diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c836c8d..d854645 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -44,7 +44,11 @@ flowview/ # Repository root (install to plugins/fl ├── flowview_upgrade.php # Background schema upgrade runner ├── flow_collector.php # NetFlow v5/v9/v10 collector daemon ├── functions.php / functions-pre13.php # Core UI + protocol decode helpers -├── import_flows.php # CLI flow file importer +├── cli/ # CLI utilities (importer, repair, materializer) +│ ├── fixprivate.php # CLI: repair domains of private-range addresses +│ ├── import_flows.php # CLI flow file importer +│ ├── materialize_queries.php # CLI: pre-build each query's partition cache (NAT on/off) +│ └── repair_tables.php # CLI: widen the raw partition table columns ├── run_schedule.php # CLI scheduled-report executor ├── INFO # Plugin metadata (name, version, compat) ├── README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fde7b01..f794228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ --- develop --- +* feature: Say which partitions a Filter is not materialized for, with NAT and without, in the tip of its Required and Running pill + +* feature: Show the number of queries of each Filter that are running now in a second pill, in the style of a disabled device and reading Running: N, beside the status pill on the Filters page, with how many are with NAT and without in its tip. The process table only has the id of a running request, so a running query is matched to a Filter by the cache key its map query is stored under, which is worked out from the Filter, and there is no pill when none are running + +* feature: Add --status to cli/materialize_queries.php to list the parallel queries that are running and the Filter each one is for, with its progress + +* feature: Have cli/materialize_queries.php get the partitions ready before it starts the Parallel Query. It waits for an ALTER TABLE that is already running on a partition it needs, such as from the schema upgrade, since a query waits for it and then ends at the Maximum Run Time, and then does the alters that are still due itself, one partition at a time with progress shown. The partition still receiving flows is left alone. It also notes the partitions that have no NAT columns, as they have no NAT data. What a partition needs altered is now decided in one place, flowview_get_partition_alter(), which the schema upgrade also uses, so the ALTER it issues is unchanged + +* feature: Materialize the partitions that have gone cold automatically. Each poller cycle flowview_process.php records the newest partition, which is the one still receiving flows, and starts cli/materialize_queries.php in the background for the partitions that have rotated since the previous cycle, for every Filter with NAT on and off. Partitions that are already materialized and accurate are skipped. When a materialization is already running, or the Parallel Query is not enabled, the record is left alone so a later cycle catches up. The first cycle only records the newest partition, so older partitions are still materialized with cli/materialize_queries.php. The script gained a --partitions option for this + +* feature: Add a Materialization column, to the right of the Filter Name and its ID on the Filters page, with a tip explaining what materialization is, showing the automatic materialization status as a pill in the form of the device status: Required (down) when rotated partitions are not materialized yet, Running (recovering) while a materialization is in progress, and Up to date (up) once every cold partition is materialized with NAT on and off. The ID column now follows the Filter Name + +* feature: Show the number of partitions each Filter has stored in the Parallel Query cache, with NAT and without, as the NAT Shards and Non-NAT Shards columns next to the ID on the Filters page, along with a Partitions column holding the total number of flow partitions so the progress of a materialization can be followed; the cache keys are worked out from the Filter without running it (run_flow_query() gained a plan-only mode, and the map query hashing is shared with the parallel query request) + +* feature: Add a 'Materialize' action to the Filters page that, after confirmation, starts cli/materialize_queries.php in the background for the selected Filters (--query-id now accepts a comma separated list), and tells you when a materialization is already running + +* feature: Recommend a forced re-materialization when a Filter is saved, from either the Filter form or the report page, with a change that alters what it materializes (report type, device, exporter, template, or any address, port, interface, AS, protocol, TCP flag or TOS restriction, plus the sort field of printed reports); display settings, the time range, cutoffs and NAT mode do not count + +* feature: Add cli/materialize_queries.php, which runs every saved FlowView query through the parallel query for the entire range of flow partitions, once with NAT on and once with NAT off (--query-id, --nat, --dry-run and --force are optional), so the partition results are already cached when a report is opened; it registers with register_process_start() for 8 hours so only one run is active at a time, reuses cached partitions that still match their partition's data bounds, rebuilds the ones that have since gained flows or been dropped, and runs each query and NAT mode in its own process so one that reaches the Maximum Run Time does not end the rest + +* dev: Move fixprivate.php, import_flows.php and repair_tables.php into the plugin's cli/ directory (listed as tombstones in manifest.json so upgrades remove the old copies); they now locate the Cacti root relative to their own location + * fix: Make bundled Net_DNS2 constructor nullability explicit for PHP 8.4 without changing accepted arguments * issue: Shorten the graph drilldown button title from 'View NetFlow Traffic In Range' to 'NetFlow' so it reads cleanly as a label in Cacti's graph action menu diff --git a/README.md b/README.md index 8de558c..561a522 100644 --- a/README.md +++ b/README.md @@ -194,11 +194,93 @@ this migration, ensure that your Cacti system, has enough space to handle all the flow data. You should check the size of your existing flows, and then verify that you have enough space to handle the data inside of a MySQL database. -Once you have done this, simply run the 'import_flows.php' script and all your +Once you have done this, simply run the 'cli/import_flows.php' script and all your legacy flow data will be imported. Remember, it's important that you define your partitioning scheme ahead of time, especially if you have large quantities of flow data streaming into the Cacti server. +## Pre-Building Query Results + +The Parallel Query caches the result of each fully covered partition, so +reports over older data open quickly once a partition has been scanned. To +build that cache ahead of time, run the following from the plugin's `cli` +directory with the Parallel Query enabled (Max Concurrent Threads of 2 or +more under the FlowView Settings): + +``` +php cli/materialize_queries.php +``` + +The script runs every saved query over every flow partition that has rotated, +once with NAT on and once with NAT off. The partition still receiving flows is +never cached, as a result cached for it would be wrong as soon as more flows +arrive. Instead, the table and charts of one report share a single scan of it, +whose result is reused for 60 seconds and then discarded. Use `--query-id=ID[,ID...]` or +`--nat=on|off` to narrow the run, and `--dry-run` to see what would be run. +Partitions that are already cached and still match their data are skipped, +while partitions that have received more flows since, or have been dropped, +are rebuilt. Each Filter being materialized is registered in the Cacti process +table as `materialize`, with the id of the Filter as the task id, for up to 8 +hours, so Filters can be materialized separately and at the same time. A Filter +that is already being materialized is skipped, unless `--force` is used. The +Filters page shows only those Filters as Running. `--status` lists the Filters +being materialized, and the parallel queries that are running now with the +Filter each one is for, with its progress. When it is not run from a terminal, +for instance from the Filters page or by the poller, its progress is written to +the Cacti log. Sending it SIGTERM or SIGINT stops its workers and removes +its registrations from the process table. + +Only the partitions a Filter does not have cached are run, a batch of them at a +time, each as its own Parallel Query, as one query for every partition can not +finish within the Maximum Run Time. A batch that does not finish is followed by +a smaller one, and what a batch cached is kept, so a run that is stopped, or +timed out, carries on from there when it is run again. + +Before it scans anything the script gets the partitions ready. A query waits +for an ALTER TABLE that is running on the partition it scans, and a long one +ends the query at the Maximum Run Time. So the script waits for an alter that +is already running on a partition it needs, for instance from the schema +upgrade, and then does the alters that are still due itself, one partition at a +time. The partition still receiving flows is left alone. The post-NAT columns +are not added, as that is a deliberate choice for large installs. The script +notes the partitions that do not have them, and they have no NAT data until +`flowview_upgrade_nat_columns.php` is run. + +You can also start it from the Filters page by selecting one or more Filters +and choosing the 'Materialize' action. It runs in the background after +you confirm, and the result is written to the Cacti log. The 'NAT Shards' and +'Non-NAT Shards' columns on that page show how many partitions of each Filter +are already cached, with NAT on and with NAT off, out of the total shown in the +'Partitions' column. + +New partitions are materialized automatically. Each poller cycle records the +newest partition, which is the one still receiving flows, and materializes the +partitions that have rotated since the previous cycle for every Filter, with +NAT on and off, in the background. The first cycle only records the newest +partition, so run the script once to materialize the existing ones. The +'Materialization' column on the Filters page shows the result for each Filter, +as Required, Running or Up to date. The partition still receiving flows is not +materialized automatically, so it does not count. + +When you save a +Filter in a way that changes the data it materializes, such as its report, +device, exporter, or any address, port, interface, AS, protocol, flag or TOS +restriction, a dialog warns you first, and you can Cancel to keep editing or +Continue to save. Once it is saved, a 'Save Succeeded' dialog reminds you to +start a materialization of the Filter with the 'Materialize' action. + +The report of a Filter is part of what is cached for it, so it can not be +changed on the report page: the Report selector is disabled once the Filter has +one. To change it, Edit the Filter and then materialize it again. Saving a +Filter does not remove anything from the cache. The results of the old settings +simply stop matching the Filter. The line under the Filters table says how many +cached shards match a Filter, which is told by their md5 value, and how many +match none, such as those of a Filter before it was changed, of a deleted +Filter, or of a report with its settings changed on the report page. The +'Prune' button on the Filters page, after Clear, removes the ones that match no +Filter, after you confirm in its dialog. It keeps everything that matches a Filter, and does nothing when +the cache keys of a Filter that has a report can not be worked out. + # Possible Bugs? If you figure out this problem, goto GitHub and create a pulls request or open diff --git a/fixprivate.php b/cli/fixprivate.php similarity index 99% rename from fixprivate.php rename to cli/fixprivate.php index 0c6bdb1..35d87a8 100644 --- a/fixprivate.php +++ b/cli/fixprivate.php @@ -23,7 +23,7 @@ +-------------------------------------------------------------------------+ */ -chdir('../../'); +chdir(__DIR__ . '/../../../'); require('./include/cli_check.php'); global $config; diff --git a/import_flows.php b/cli/import_flows.php similarity index 98% rename from import_flows.php rename to cli/import_flows.php index 6bc7e9f..84d23b6 100644 --- a/import_flows.php +++ b/cli/import_flows.php @@ -23,7 +23,7 @@ +-------------------------------------------------------------------------+ */ -chdir('../../'); +chdir(__DIR__ . '/../../../'); require('./include/cli_check.php'); global $config; diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php new file mode 100644 index 0000000..f8c4f31 --- /dev/null +++ b/cli/materialize_queries.php @@ -0,0 +1,860 @@ +#!/usr/bin/env php + $value) { + $value = is_array($value) ? end($value) : $value; + + switch ($arg) { + case 'query-id': + $query_ids = []; + + foreach (explode(',', (string) $value) as $id) { + if (!ctype_digit($id) || $id < 1) { + print 'ERROR: --query-id must be a positive integer or a comma separated list of them' . PHP_EOL; + exit(1); + } + + $query_ids[(int) $id] = (int) $id; + } + + $query_ids = array_values($query_ids); + + break; + case 'partitions': + $partitions = []; + + foreach (explode(',', (string) $value) as $table) { + if (flowview_partition_sort_key($table) === '') { + print 'ERROR: --partitions must be a comma separated list of raw partition tables' . PHP_EOL; + exit(1); + } + + $partitions[$table] = $table; + } + + $partitions = array_values($partitions); + + break; + case 'nat': + $nat = strtolower((string) $value); + + if ($nat != 'on' && $nat != 'off') { + print 'ERROR: --nat must be either on or off' . PHP_EOL; + exit(1); + } + + break; + case 'start': + $start = ctype_digit((string) $value) ? (int) $value : false; + + break; + case 'end': + $end = ctype_digit((string) $value) ? (int) $value : false; + + break; + case 'debug': + $debug = true; + + break; + case 'force': + $force = true; + + break; + case 'dry-run': + $dry_run = true; + + break; + case 'status': + $status = true; + + break; + case 'worker': + $worker = true; + + break; + case 'V': + case 'v': + case 'version': + display_version(); + exit(0); + case 'H': + case 'h': + case 'help': + display_help(); + exit(0); + default: + print 'ERROR: Invalid option ' . $arg . PHP_EOL; + display_help(); + exit(1); + } +} + +if ($status) { + materialize_show_status(); + + exit(0); +} + +/* a worker materializes exactly one query and NAT mode for the range its parent chose */ +if ($worker) { + if (cacti_sizeof($query_ids) != 1 || $nat === false || $start === false || $end === false) { + print 'ERROR: --worker requires one --query-id, --nat, --start and --end' . PHP_EOL; + exit(1); + } + + materialize_install_signal_handlers(); + + /* only the partitions of the batch are scanned, not every partition in its padded range */ + if (cacti_sizeof($partitions)) { + $GLOBALS['flowview_partition_allowlist'] = array_fill_keys($partitions, true); + } + + exit(materialize_run_worker($query_ids[0], $nat, $start, $end)); +} + +/* the filters to materialize, all of them unless some were asked for */ +if (!cacti_sizeof($query_ids)) { + $query_ids = array_map('intval', array_column(flowview_db_fetch_assoc('SELECT id + FROM plugin_flowview_queries + ORDER BY id'), 'id')); +} + +if (!cacti_sizeof($query_ids)) { + print 'NOTE: There are no FlowView queries to materialize.' . PHP_EOL; + exit(0); +} + +/** + * Each filter is registered in the process table as its own materialize + * process, with the filter id as the task id, so the filters can be + * materialized separately and what is being done can be seen there. A filter + * that already has one is left to it. + */ +if (!$force && !$dry_run) { + flowview_purge_dead_materializations(); + + $skipped = []; + + foreach ($query_ids as $query_id) { + if (register_process_start('flowview', 'materialize', $query_id, MATERIALIZE_RUNTIME_LIMIT)) { + $materialize_registered[$query_id] = $query_id; + } else { + $skipped[] = $query_id; + } + } + + if (cacti_sizeof($skipped)) { + materialize_say(sprintf('NOTE: Query(s) %s are already being materialized, so they are skipped. To override use --force', implode(', ', $skipped))); + } + + if (!cacti_sizeof($materialize_registered)) { + exit(0); + } + + register_shutdown_function('materialize_release'); + + $query_ids = array_values($materialize_registered); +} + +materialize_install_signal_handlers(); + +$threads = read_config_option('flowview_parallel_threads'); + +if (empty($threads) || $threads == 1) { + print 'ERROR: Materialization requires the Parallel Query. Set Max Concurrent Threads to 2 or more under the FlowView Settings.' . PHP_EOL; + exit(1); +} + +/* the live partition is still receiving flows, so it is never materialized */ +$tables = flowview_get_cold_partitions(flowview_get_partition_tables()); + +/* only the partitions asked for, that still exist */ +if (cacti_sizeof($partitions)) { + $tables = array_values(array_intersect($tables, $partitions)); +} + +$cold = $tables; + +$range = flowview_get_materialize_range($tables); + +if ($range === false) { + print 'NOTE: There are no flow partitions to materialize.' . PHP_EOL; + exit(0); +} + +list($start, $end) = $range; + +if (cacti_sizeof($query_ids)) { + $queries = flowview_db_fetch_assoc_prepared('SELECT id, name + FROM plugin_flowview_queries + WHERE id IN (' . implode(', ', array_fill(0, cacti_sizeof($query_ids), '?')) . ') + ORDER BY id', + $query_ids); +} else { + $queries = flowview_db_fetch_assoc('SELECT id, name + FROM plugin_flowview_queries + ORDER BY id'); +} + +if (!cacti_sizeof($queries)) { + print 'NOTE: There are no FlowView queries to materialize.' . PHP_EOL; + exit(0); +} + +$modes = $nat !== false ? [$nat] : ['on', 'off']; + +materialize_say(sprintf('FlowView Materialization: Partitions:%d From:%s To:%s Queries:%d NAT:%s', + cacti_sizeof($tables), + date('Y-m-d H:i:s', flowview_table_name_to_time(reset($tables), 'start_time')), + date('Y-m-d H:i:s', flowview_table_name_to_time(end($tables), 'end_time')), + cacti_sizeof($queries), + implode('/', $modes))); + +/** + * Any alter that is due has to be done before the parallel query starts. A + * query waits for an alter running on the partition it scans, and a long one + * ends the query at the Maximum Run Time. + */ +materialize_prepare_partitions($cold, $modes, $dry_run); + +/* partitions that are still accurate keep their cache, everything else is rebuilt below */ +if (!$dry_run) { + $live = flowview_remove_live_partition_cache(); + + if ($live > 0) { + materialize_say(sprintf('Removed %d cached result(s) of the live partition, as it is still receiving flows', $live)); + } + + $cache = flowview_invalidate_stale_shard_cache(cacti_sizeof($partitions) ? $tables : []); + + materialize_say(sprintf('Cached Partitions Still Accurate:%d, Stale Partitions Removed:%d', $cache['kept'], $cache['removed'])); +} + +$began = microtime(true); +$total = 0; +$failed = 0; + +foreach ($queries as $query) { + foreach ($modes as $mode) { + $total++; + + if ($dry_run) { + $todo = flowview_get_uncached_partitions((int) $query['id'], $mode, $cold); + + print sprintf('DRY RUN: Would materialize Query:%d [%s] NAT:%s, %d of %d partition(s) are not cached', $query['id'], $query['name'], $mode, cacti_sizeof($todo), cacti_sizeof($cold)) . PHP_EOL; + + continue; + } + + if (!materialize_query_mode($query, $mode, $cold, (int) $threads, $debug)) { + $failed++; + + materialize_say(sprintf('ERROR: Query:%d [%s] NAT:%s failed to materialize', $query['id'], $query['name'], $mode)); + } + } +} + +$time = microtime(true) - $began; + +materialize_say(sprintf('FlowView Materialization Complete: Time:%0.2f Runs:%d Failed:%d', $time, $total, $failed)); + +if ($failed > 0) { + print 'NOTE: Partitions that finished materializing are kept, so run the materializer again to continue.' . PHP_EOL; +} + +if (!$dry_run) { + cacti_log(sprintf('FLOWVIEW MATERIALIZE STATS: Time:%0.2f Runs:%d Failed:%d', $time, $total, $failed), false, 'SYSTEM'); +} + +exit($failed > 0 ? 1 : 0); + +/** + * Shows the parallel queries that are running and the Filter each one is for. + * The process table only has the id of the request, so the Filter is found + * from the cache key of the request. + * + * @return void + */ +function materialize_show_status(): void { + $materializing = flowview_get_materializing_filters(); + + if (cacti_sizeof($materializing)) { + print sprintf('Materializing Query(s): %s', implode(', ', $materializing)) . PHP_EOL; + } else { + print 'No Query is being materialized.' . PHP_EOL; + } + + $running = flowview_get_running_requests(); + + if (!cacti_sizeof($running)) { + print 'No parallel queries are running.' . PHP_EOL; + + return; + } + + $names = []; + $md5sums = []; + + foreach (flowview_db_fetch_assoc('SELECT id, name FROM plugin_flowview_queries ORDER BY id') as $filter) { + $names[(int) $filter['id']] = $filter['name']; + $md5sums[(int) $filter['id']] = flowview_get_filter_md5sums((int) $filter['id']); + } + + $requests = array_column(flowview_db_fetch_assoc_prepared('SELECT id, status, total_shards, finished_shards, cached_shards + FROM parallel_database_query + WHERE id IN (' . implode(', ', array_fill(0, cacti_sizeof($running), '?')) . ')', + array_keys($running)), null, 'id'); + + print sprintf('%d parallel queries are running:', cacti_sizeof($running)) . PHP_EOL; + + foreach ($running as $id => $md5sum) { + $for = []; + + foreach (flowview_find_request_filters($md5sums, $md5sum) as $match) { + $mode = $match['nat'] && $match['non_nat'] ? 'on/off' : ($match['nat'] ? 'on' : 'off'); + $for[] = sprintf('Query:%d [%s] NAT:%s', $match['id'], $names[$match['id']], $mode); + } + + if (isset($requests[$id])) { + $progress = sprintf('Shards:%d of %d, Already Materialized:%d, State:%s', $requests[$id]['finished_shards'], $requests[$id]['total_shards'], $requests[$id]['cached_shards'], $requests[$id]['status']); + } else { + $progress = 'The request no longer exists, its processes are still registered'; + } + + print sprintf('Request:%d %s %s', $id, cacti_sizeof($for) ? implode(', ', $for) : 'No Filter matches it, it may be an ad hoc report or a Filter that has changed', $progress) . PHP_EOL; + } +} + +/** + * Waits until no ALTER TABLE is running on any of the partitions. A query + * waits for an alter on the partition it scans, so the parallel query must + * not start while one is running. + * + * @param array $tables The partitions to wait for + * + * @return void + */ +function materialize_wait_for_alters(array $tables): void { + $last_report = 0; + + while (cacti_sizeof($running = flowview_get_running_partition_alters($tables))) { + if (time() - $last_report >= 30) { + $last_report = time(); + + arsort($running); + + materialize_say(sprintf('Waiting for an ALTER TABLE to finish on %d partition(s), the longest %s for %d seconds', cacti_sizeof($running), array_key_first($running), reset($running))); + } + + sleep(5); + } +} + +/** + * Materializes one query for one NAT mode, a batch of the partitions that are + * not cached yet at a time. One parallel query for every partition can not + * finish within the Maximum Run Time, and what it did not finish would be lost + * with it, so each batch is its own query, and the next one follows from what + * is cached after it. A batch that did not finish is followed by a smaller + * one, and the work is given up when a small one gets nowhere. + * + * @param array $query The FlowView query, with its id and name + * @param string $mode on for NAT, off for without + * @param array $cold The partitions to materialize, oldest first + * @param int $threads The number of Max Concurrent Threads + * @param bool $debug Whether to pass --debug on + * + * @return bool Whether everything that can be cached is + */ +function materialize_query_mode(array $query, string $mode, array $cold, int $threads, bool $debug): bool { + $id = (int) $query['id']; + $size = 0; + $timed_out = false; + $uncacheable = []; + + $missing = flowview_get_uncached_partitions($id, $mode, $cold); + + if (!cacti_sizeof($missing)) { + materialize_say(sprintf('Query:%d NAT:%s Already Materialized and Accurate', $id, $mode)); + + return true; + } + + materialize_say(sprintf('Query:%d NAT:%s has %d of %d partition(s) to materialize', $id, $mode, cacti_sizeof($missing), cacti_sizeof($cold))); + + while (cacti_sizeof($missing)) { + $size = flowview_materialize_batch_size($size, $timed_out, $threads); + $batch = array_slice($missing, 0, $size); + $range = flowview_get_materialize_range($batch); + + if ($range === false) { + return false; + } + + /* a separate process keeps a batch that hits the run limit from ending the rest */ + $command = [ + PHP_BINARY, + __FILE__, + '--worker', + '--query-id=' . $id, + '--nat=' . $mode, + '--start=' . $range[0], + '--end=' . $range[1], + /* the range is padded so the batch is covered, only the batch's own partitions are scanned */ + '--partitions=' . implode(',', $batch), + ]; + + if ($debug) { + $command[] = '--debug'; + } + + $status = materialize_run_command($command); + + $timed_out = $status != 0; + $remaining = flowview_get_uncached_partitions($id, $mode, array_values(array_diff($cold, $uncacheable))); + $progress = cacti_sizeof($missing) - cacti_sizeof($remaining); + + materialize_say(sprintf('Query:%d NAT:%s Batch:%d Cached:%d Remaining:%d', $id, $mode, cacti_sizeof($batch), $progress, cacti_sizeof($remaining))); + + if ($timed_out && $progress <= 0 && $size <= 4) { + return false; + } + + /* a batch that ran and still did not cache a partition has one that can not be cached */ + if (!$timed_out) { + foreach (array_intersect($batch, $remaining) as $partition) { + $uncacheable[] = $partition; + } + + $remaining = array_values(array_diff($remaining, $uncacheable)); + } + + $missing = $remaining; + } + + if (cacti_sizeof($uncacheable)) { + materialize_say(sprintf('NOTE: Query:%d NAT:%s %d partition(s) could not be cached, as their flows are not all within their range, such as %s', $id, $mode, cacti_sizeof($uncacheable), reset($uncacheable))); + } + + return true; +} + +/** + * Gets the partitions ready to be scanned. It waits for an alter that is + * already running on one of them, for instance from the schema upgrade, and + * then does the alters that are still due itself, so nothing is altered while + * the parallel query runs. The partition still receiving flows is left alone. + * + * A partition is looked at again just before it is altered, under a lock on + * it that every process altering a partition takes, so a partition that + * another materialization, the poller or the schema upgrade altered in the + * meantime is never rebuilt with a plan that is out of date. + * + * @param array $cold The partitions to prepare + * @param array $modes The NAT modes being materialized, on and off + * @param bool $dry_run Only say what would be done + * + * @return void + */ +function materialize_prepare_partitions(array $cold, array $modes, bool $dry_run): void { + if (!cacti_sizeof($cold)) { + return; + } + + $raw_engine = get_set_default_fast_engine(); + + if (!$dry_run) { + materialize_wait_for_alters($cold); + } + + $pending = flowview_get_pending_partition_alters($cold, $raw_engine); + + if (cacti_sizeof($pending)) { + materialize_say(sprintf('%s %d partition(s) that still need an ALTER TABLE', $dry_run ? 'DRY RUN: Would alter' : 'Altering', cacti_sizeof($pending))); + + $i = 0; + + foreach ($pending as $table => $plan) { + $i++; + + if ($dry_run) { + materialize_say(sprintf('DRY RUN: Would alter Table %s (%d of %d): %s', $table, $i, cacti_sizeof($pending), trim($plan['alter']))); + + continue; + } + + $began = microtime(true); + $count = cacti_sizeof($pending); + + /* the plan is worked out again under the table's lock, as another process may have altered it since */ + $result = flowview_alter_partition($table, $raw_engine, false, FLOWVIEW_ALTER_LOCK_WAIT, function ($plan) use ($table, $i, $count) { + materialize_say(sprintf('Altering Table %s (%d of %d): %s', $table, $i, $count, trim($plan['alter']))); + }); + + if ($result == 'altered') { + materialize_say(sprintf('Altered Table %s in %0.2f seconds', $table, microtime(true) - $began)); + } elseif ($result == 'current') { + materialize_say(sprintf('Table %s (%d of %d) was altered by another process', $table, $i, $count)); + } else { + materialize_say(sprintf('WARNING: Table %s (%d of %d) is still being altered by another process, it is left to it', $table, $i, $count)); + } + } + } + + $no_nat = in_array('on', $modes, true) ? flowview_get_partitions_without_nat_columns($cold) : []; + + if (cacti_sizeof($no_nat)) { + materialize_say(sprintf('NOTE: %d partition(s) have no NAT columns, so they have no NAT data. Run flowview_upgrade_nat_columns.php before materializing if they should.', cacti_sizeof($no_nat))); + } +} + +/** + * Makes SIGTERM and SIGINT end the script through materialize_sig_handler(), + * so what it registered is cleaned up, as the flow collector does. + * + * @return void + */ +function materialize_install_signal_handlers(): void { + if (function_exists('pcntl_signal')) { + pcntl_signal(SIGTERM, 'materialize_sig_handler'); + pcntl_signal(SIGINT, 'materialize_sig_handler'); + } +} + +/** + * Cleans up on SIGTERM and SIGINT and exits. The script that runs the + * materialization stops the worker it started, and removes its registration + * from the process table. A worker cancels the parallel queries it started, + * which stops their runner and shard workers and removes their registrations. + * + * @param int $signo + * + * @return void + */ +function materialize_sig_handler($signo): void { + global $worker; + + static $handling = false; + + switch ($signo) { + case SIGTERM: + case SIGINT: + if ($handling) { + return; + } + + $handling = true; + + if ($worker) { + cacti_log('WARNING: FlowView Materialization worker is shutting down by signal!', false, 'FLOWVIEW'); + + flowview_cancel_requests_in_flight(); + + /* the shutdown function must not report this as a failure to finish */ + $GLOBALS['flowview_error_reported'] = true; + } else { + cacti_log('WARNING: FlowView Materialization is shutting down by signal!', false, 'FLOWVIEW'); + + materialize_stop_child(); + materialize_release(); + } + + exit(1); + default: + /* ignore all other signals */ + } +} + +/** + * Removes the registration of each filter of this materialization from the + * process table. It does nothing the second time it is called. + * + * @return void + */ +function materialize_release(): void { + global $materialize_registered; + + foreach ($materialize_registered as $query_id) { + unregister_process('flowview', 'materialize', $query_id, (int) getmypid()); + } + + $materialize_registered = []; +} + +/** + * Says something, on the screen, and also in the Cacti log when the output is + * not a terminal. A materialization started from the Filters page, or by the + * poller, has nowhere to show it otherwise. + * + * @param string $text + * + * @return void + */ +function materialize_say(string $text): void { + print $text . PHP_EOL; + + if (!function_exists('posix_isatty') || !posix_isatty(STDOUT)) { + cacti_log($text, false, 'FLOWVIEW'); + } +} + +/** + * Runs a worker and waits for it, with its output shown as it is written. + * The worker is kept so a signal can stop it. It is started directly, not + * through a shell, so a signal sent to it reaches the worker itself. + * + * @param array $command The program and its arguments + * + * @return int The exit status, 0 on success + */ +function materialize_run_command(array $command): int { + global $materialize_child; + + $process = proc_open($command, [0 => STDIN, 1 => STDOUT, 2 => STDERR], $pipes); + + if (!is_resource($process)) { + return 1; + } + + $materialize_child = $process; + $status = 1; + + while (true) { + $info = proc_get_status($process); + + if (!$info['running']) { + /* the exit status can only be read the first time it is asked for */ + $status = $info['exitcode'] >= 0 ? $info['exitcode'] : 1; + + break; + } + + usleep(100000); + } + + $materialize_child = null; + + proc_close($process); + + return $status; +} + +/** + * Stops the worker being run, giving it time to cancel its queries first. + * + * @return void + */ +function materialize_stop_child(): void { + global $materialize_child; + + if (!is_resource($materialize_child)) { + return; + } + + proc_terminate($materialize_child, SIGTERM); + + $deadline = microtime(true) + 15; + + while (microtime(true) < $deadline) { + $info = proc_get_status($materialize_child); + + if (!$info['running']) { + return; + } + + usleep(100000); + } + + proc_terminate($materialize_child, SIGKILL); +} + +/** + * Materializes one query for one NAT mode across the given range. The query + * runs through the normal parallel query path, so each partition that is + * already cached and still accurate is reused instead of being scanned again. + * + * @param int $query_id The FlowView query to run + * @param string $nat Either on or off + * @param int $start The Unix start time of the range + * @param int $end The Unix end time of the range + * + * @return int The exit status, 0 on success + */ +function materialize_run_worker(int $query_id, string $nat, int $start, int $end): int { + $finished = false; + + /* the parallel query ends the process on a timeout, after it has said why */ + register_shutdown_function(function () use (&$finished) { + if (!$finished) { + if (empty($GLOBALS['flowview_error_reported'])) { + print 'ERROR: The materialization ended before it finished. See the Cacti log for details.' . PHP_EOL; + } + + exit(1); + } + }); + + set_request_var('usenat', $nat == 'on' ? 'true' : 'false'); + + /* run the shards, even if the same query was run before, as it is the cached shards that are wanted */ + $GLOBALS['flowview_fresh_requests'] = true; + $GLOBALS['flowview_requests_made'] = []; + + materialize_say(sprintf('Materializing Query:%d NAT:%s', $query_id, $nat)); + + $began = microtime(true); + + $data = load_data_for_filter($query_id, $start, $end); + + $finished = true; + + if ($data === false) { + materialize_say(sprintf('ERROR: Query:%d NAT:%s did not return a report. See the Cacti log for details.', $query_id, $nat)); + + return 1; + } + + /* only the requests of this worker, as the workers of other filters can run at the same time */ + $made = array_map('intval', array_values($GLOBALS['flowview_requests_made'])); + + $stats = cacti_sizeof($made) ? flowview_db_fetch_row_prepared('SELECT COUNT(*) AS requests, + COALESCE(SUM(total_shards), 0) AS shards, + COALESCE(SUM(cached_shards), 0) AS cached + FROM parallel_database_query + WHERE id IN (' . implode(', ', array_fill(0, cacti_sizeof($made), '?')) . ')', + $made) : []; + + if (cacti_sizeof($stats) && $stats['requests'] > 0) { + materialize_say(sprintf('Query:%d NAT:%s Partitions:%d AlreadyMaterialized:%d Materialized:%d Time:%0.2f', + $query_id, + $nat, + $stats['shards'], + $stats['cached'], + $stats['shards'] - $stats['cached'], + microtime(true) - $began)); + } else { + materialize_say(sprintf('Query:%d NAT:%s Already Materialized and Accurate Time:%0.2f', $query_id, $nat, microtime(true) - $began)); + } + + return 0; +} + +/** + * Prints this script's version and copyright banner. Called from + * display_help() and when invoked with '--version'. + * + * @return void + */ +function display_version(): void { + $version = get_cacti_cli_version(); + print "Cacti FlowView Query Materializer, Version $version, " . COPYRIGHT_YEARS . PHP_EOL; +} + +/** + * Prints the version banner followed by this script's command-line + * usage summary. Called when invoked with '--help' or with invalid + * arguments. + * + * @return void + */ +function display_help(): void { + display_version(); + + print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID[,ID...]] [--partitions=TABLE[,TABLE...]] [--nat=on|off] [--dry-run] [--status] [--force] [--debug]' . PHP_EOL . PHP_EOL; + + print 'Materializes each FlowView query into the parallel query cache for every' . PHP_EOL; + print 'flow partition that has rotated, once with NAT on and once with NAT off.' . PHP_EOL; + print 'The partition still receiving flows is never cached. Partitions that are' . PHP_EOL; + print 'already materialized and still accurate are skipped.' . PHP_EOL; + print 'Each Filter is registered in the process table as materialize, with its id as the task id,' . PHP_EOL; + print 'so Filters can be materialized separately. A Filter that is already being materialized' . PHP_EOL; + print 'is skipped. Registrations last up to 8 hours.' . PHP_EOL . PHP_EOL; + + print 'Options:' . PHP_EOL; + print ' --query-id=ID Only materialize this query, or a comma separated list of them.' . PHP_EOL; + print ' --partitions=TABLE Only materialize these partition tables, as the poller does for the' . PHP_EOL; + print ' partitions that have rotated.' . PHP_EOL; + print ' --nat=on|off Only materialize this NAT mode.' . PHP_EOL; + print ' --dry-run List what would be materialized and exit.' . PHP_EOL; + print ' --status List the parallel queries that are running and the Filter each is for.' . PHP_EOL; + print ' --force Run even if a Filter is already being materialized.' . PHP_EOL; + print ' --debug Provide some debug output during the run.' . PHP_EOL . PHP_EOL; +} diff --git a/repair_tables.php b/cli/repair_tables.php similarity index 98% rename from repair_tables.php rename to cli/repair_tables.php index 86b1301..1976fe1 100644 --- a/repair_tables.php +++ b/cli/repair_tables.php @@ -23,7 +23,7 @@ +-------------------------------------------------------------------------+ */ -chdir('../../'); +chdir(__DIR__ . '/../../../'); require('./include/cli_check.php'); global $config, $flowviewdb_default; diff --git a/flowview.php b/flowview.php index 0a2a903..375a88a 100644 --- a/flowview.php +++ b/flowview.php @@ -81,6 +81,10 @@ flowview_request_vars(); flowview_gettimespan(); + break; + case 'timeshift': + flowview_timeshift(); + break; case 'query': load_session_for_filter(); @@ -303,6 +307,35 @@ function load_session_for_filter() { * settings first, and then overlay others only if they are set */ if (isset_request_var('query') && get_filter_request_var('query') > 0) { + $query = flowview_db_fetch_row_prepared('SELECT * + FROM plugin_flowview_queries + WHERE id = ?', + array(get_request_var('query'))); + + /** + * The report of a filter can not be changed here, as it is part of + * what is cached for it. It is changed by editing the filter. + */ + $report = cacti_sizeof($query) ? flowview_filter_report_key($query) : ''; + + if ($report != '') { + set_request_var('report', $report); + + /* a printed report is a table, there is nothing to chart */ + if (substr($report, 0, 1) == 'p') { + set_request_var('table', 'true'); + set_request_var('bytes', 'false'); + set_request_var('packets', 'false'); + set_request_var('flows', 'false'); + + /* the saved panels are merged in below, so they have to say the same */ + $query['panel_table'] = 'on'; + $query['panel_bytes'] = ''; + $query['panel_packets'] = ''; + $query['panel_flows'] = ''; + } + } + // Handle Report Column if (isset_request_var('report')) { if (get_nfilter_request_var('report') != '0' && trim(get_nfilter_request_var('report'), 'sp') != '0') { @@ -316,11 +349,6 @@ function load_session_for_filter() { } } - $query = flowview_db_fetch_row_prepared('SELECT * - FROM plugin_flowview_queries - WHERE id = ?', - array(get_request_var('query'))); - if (cacti_sizeof($query)) { foreach($query as $column => $value) { switch($column) { @@ -489,6 +517,18 @@ function flowview_request_vars(): void { } } + /* the report is not sent for a Filter that has one, as it is part of the Filter */ + if (isset_request_var('query') && get_filter_request_var('query') > 0) { + $filter_report = flowview_filter_report_key(flowview_db_fetch_row_prepared('SELECT statistics, printed + FROM plugin_flowview_queries + WHERE id = ?', + array(get_request_var('query'))) ?: []); + + if ($filter_report != '') { + set_request_var('report', $filter_report); + } + } + /* ================= input validation and session storage ================= */ $filters = array( 'device_id' => [ diff --git a/flowview_filters.php b/flowview_filters.php index ca758a5..d74d397 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -41,7 +41,8 @@ /* 2 => __('Send Now', 'flowview'), */ 1 => __('Delete', 'flowview'), 3 => __('Disable', 'flowview'), - 4 => __('Enable', 'flowview') + 4 => __('Enable', 'flowview'), + 5 => __('Materialize', 'flowview') ); switch (get_request_var('action')) { @@ -54,6 +55,12 @@ case 'sort_filter': sort_filter(); break; + case 'check_material': + flowview_check_filter_save(); + break; + case 'prune_shards': + prune_shards(); + break; case 'edit': if (!isset_request_var('embed')) { top_header(); @@ -73,6 +80,32 @@ break; } +/** + * Removes the cached shards that match no Filter and returns to the Filters + * page. It is posted by the confirmation dialog of the Prune button. Called + * from this script's main request-dispatch switch when action=prune_shards. + * + * @return void + */ +function prune_shards(): void { + /* it removes data, so it is only done when posted from the dialog */ + if (($_SERVER['REQUEST_METHOD'] ?? 'GET') !== 'POST') { + header('Location: flowview_filters.php?header=false'); + exit; + } + + $pruned = flowview_prune_shard_cache(); + + if ($pruned === null) { + raise_message('flowview_prune', __('The cache keys of the Filters could not be worked out, so nothing was pruned.', 'flowview'), MESSAGE_LEVEL_ERROR); + } else { + raise_message('flowview_prune', __('Pruned %s cached shard(s) that did not match any Filter.', number_format_i18n($pruned, 0), 'flowview'), MESSAGE_LEVEL_INFO); + } + + header('Location: flowview_filters.php?header=false'); + exit; +} + /** * Handles the bulk-action confirmation page/submission for the saved * filters list: on first display, renders a confirmation box listing @@ -110,6 +143,22 @@ function actions_filters(): void { for ($i=0; $i + +

" . __('Click \'Continue\' to Materialize the following Filter(s) in the background, with NAT on and off, for the entire range of flow data. Partitions that are already materialized and accurate are skipped. This can take a long time on large data sets. A Filter that is already being materialized is skipped.', 'flowview') . "

+ + + "; } if (!isset($filter_array)) { @@ -293,6 +349,7 @@ function show_filters(): void { + @@ -362,6 +419,16 @@ function clearFilter() { 'align' => 'left', 'sort' => 'ASC' ), + 'id' => array( + 'display' => __('ID', 'flowview'), + 'align' => 'right', + 'sort' => 'ASC' + ), + 'nosort5' => array( + 'display' => __('Materialization', 'flowview'), + 'align' => 'center', + 'tip' => __('Materialization pre-computes the results of a Filter for each flow partition, with NAT on and off, and stores them in the Parallel Query cache, so reports over older data open quickly. New partitions are materialized automatically once they stop receiving flows. Required means partitions that have rotated are not materialized yet, Running means a materialization is in progress, and Up to date means every rotated partition is materialized. Running: N is how many queries of the Filter are working now. The partition still receiving flows is not materialized, so it does not count.', 'flowview') + ), 'device' => array( 'display' => __('Listener', 'flowview'), 'align' => 'left', @@ -372,10 +439,20 @@ function clearFilter() { 'align' => 'left', 'sort' => 'ASC' ), - 'id' => array( - 'display' => __('ID', 'flowview'), + 'nosort2' => array( + 'display' => __('NAT Shards', 'flowview'), 'align' => 'right', - 'sort' => 'ASC' + 'tip' => __('The number of partitions of this Filter that are stored in the Parallel Query cache with NAT on.', 'flowview') + ), + 'nosort3' => array( + 'display' => __('Non-NAT Shards', 'flowview'), + 'align' => 'right', + 'tip' => __('The number of partitions of this Filter that are stored in the Parallel Query cache with NAT off.', 'flowview') + ), + 'nosort4' => array( + 'display' => __('Partitions', 'flowview'), + 'align' => 'right', + 'tip' => __('The total number of flow partitions, which is the most shards a Filter can have cached with or without NAT.', 'flowview') ), 'sortfield' => array( 'display' => __('Sort Field', 'flowview'), @@ -400,9 +477,26 @@ function clearFilter() { html_header_sort_checkbox($display_array, get_request_var('sort_column'), get_request_var('sort_direction'), false); if (cacti_sizeof($filters)) { + $tables = flowview_get_partition_tables(); + $cold = flowview_get_cold_partitions($tables); + $partitions = cacti_sizeof($tables); + $materializing = flowview_get_materializing_filters(); + $requests = flowview_get_running_requests(); + foreach ($filters as $filter) { + $md5sums = flowview_get_filter_md5sums((int) $filter['id']); + $cached = flowview_get_cached_shard_tables((int) $filter['id'], $md5sums); + $queries = flowview_count_running_queries($md5sums, $requests); + + /* only the filters being materialized, or with a query running, are running */ + $running = in_array((int) $filter['id'], $materializing, true) || $queries > 0; + $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); + $missing = flowview_get_missing_partitions($cached['nat'], $cached['non_nat'], $cold); + form_alternate_row('line' . $filter['id'], true); form_selectable_cell('' . html_escape($filter['name']) . '', $filter['id']); + form_selectable_cell($filter['id'], $filter['id'], '', 'right'); + form_selectable_cell(trim(flowview_get_colored_materialization_status($status, $missing) . ' ' . flowview_get_colored_running_queries($queries, flowview_count_running_queries_by_mode($md5sums, $requests))), $filter['id'], '', 'center'); if (empty($filter['device'])) { form_selectable_cell(__('All', 'flowview'), $filter['id']); @@ -419,7 +513,10 @@ function clearFilter() { } form_selectable_cell($type, $filter['id']); - form_selectable_cell($filter['id'], $filter['id'], '', 'right'); + + form_selectable_cell(number_format_i18n(cacti_sizeof($cached['nat']), 0), $filter['id'], '', 'right'); + form_selectable_cell(number_format_i18n(cacti_sizeof($cached['non_nat']), 0), $filter['id'], '', 'right'); + form_selectable_cell(number_format_i18n($partitions, 0), $filter['id'], '', 'right'); form_selectable_cell($sort, $filter['id'], '', 'right'); form_selectable_cell($filter['resolve'], $filter['id'], '', 'right'); form_checkbox_cell($filter['name'], $filter['id']); @@ -437,5 +534,89 @@ function clearFilter() { draw_actions_dropdown($sched_actions); form_end(); + + show_shard_cache_summary(); +} + +/** + * Shows how much of the Parallel Query cache matches a Filter and how much + * does not, which is what the Prune button removes. + * + * @return void + */ +function show_shard_cache_summary(): void { + $summary = flowview_get_shard_cache_summary(); + $prune = false; + + html_start_box('', '100%', false, 3, 'center', ''); + + if ($summary === null) { + $text = __('The cached shards of the Filters could not be told apart from the others, as the cache keys of a Filter could not be worked out.', 'flowview'); + $dialog = __('The cache keys of the Filters could not be worked out, so nothing can be pruned right now. Check that each Filter has a valid Report.', 'flowview'); + } else { + $text = __('Cached Shards: %s matching a Filter. %s not matching any Filter, which Prune removes.', number_format_i18n($summary['matching']['shards'], 0), number_format_i18n($summary['other']['shards'], 0), 'flowview'); + + if ($summary['other']['shards'] > 0) { + $prune = true; + $dialog = __('This removes %s cached shard(s) that do not match any Filter, for every Filter and every user. They come from Filters that were changed or deleted, and from reports with settings changed on the report page. The %s cached shard(s) of the Filters are kept. This can not be undone.', number_format_i18n($summary['other']['shards'], 0), number_format_i18n($summary['matching']['shards'], 0), 'flowview'); + } else { + $dialog = __('Every cached shard matches a Filter. There is nothing to prune.', 'flowview'); + } + } + + print "" . html_escape($text) . ''; + + html_end_box(); + + $js = JSON_HEX_TAG | JSON_HEX_APOS | JSON_HEX_QUOT | JSON_HEX_AMP; + + print "'; + + ?> + + 'drop_multi', 'value' => '|arg1:protocols|', 'default' => '0', + /* the select2 control of Cacti 1.2.32 and later shows this when none are picked */ + 'select_all_text' => __('All Protocols', 'flowview'), 'array' => $ip_protocols_array ), 'tcpflags' => array( diff --git a/includes/functions.php b/includes/functions.php index bbe9f3f..668717b 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -23,6 +23,21 @@ +-------------------------------------------------------------------------+ */ +// The live partition is never materialized. The workers of one report page +// share a single scan of it for a short time, kept in the shard cache table +// under this marker in map_partition, behind a named lock. +if (!defined('FLOWVIEW_LIVE_SHARD_MARKER')) { + define('FLOWVIEW_LIVE_SHARD_MARKER', 'live'); + define('FLOWVIEW_LIVE_SHARD_TTL', 60); + define('FLOWVIEW_LIVE_SHARD_LOCK', 'flowview_live_'); + define('FLOWVIEW_LIVE_SHARD_LOCK_WAIT', 120); + /* a request that is not finished and has no runner after this many seconds was abandoned */ + define('FLOWVIEW_ABANDONED_REQUEST_AGE', 120); + /* every alter of a raw partition holds this lock on it, see flowview_alter_partition() */ + define('FLOWVIEW_ALTER_LOCK', 'flowview_alter_'); + define('FLOWVIEW_ALTER_LOCK_WAIT', 7200); +} + // reports_run()/reports_queue()/reports_log_and_notify() became native in // Cacti 1.3. Whether core will declare them is fixed by the Cacti version at // bootstrap, so gate the pre-1.3 shims on that rather than on function_exists(): @@ -209,6 +224,17 @@ function sort_filter(): void { } } +/** + * flowview_cacti_uses_select2 - Whether Cacti draws a multi-select as a + * select2 control itself, which it does from 1.2.32. Earlier versions leave + * the page to make it a jquery-multiselect. + * + * @return bool + */ +function flowview_cacti_uses_select2(): bool { + return defined('CACTI_VERSION') && version_compare(CACTI_VERSION, '1.2.32', '>='); +} + /** * @return void */ @@ -320,11 +346,63 @@ function edit_filter(): void { form_save_button($page, 'return'); + $js = JSON_HEX_TAG | JSON_HEX_APOS | JSON_HEX_QUOT | JSON_HEX_AMP; + + print "'; + + flowview_show_saved_dialog(); + ?> + get_request_var('device_id'), + 'statistics' => $statistical, + 'printed' => $printed, + 'sortfield' => get_nfilter_request_var('sortfield'), + ] + ); + + flowview_db_execute_prepared('UPDATE plugin_flowview_queries + SET timespan = ?, + device_id = ?, + statistics = ?, + printed = ?, + sortfield = ?, + cutofflines = ?, + cutoffoctets = ?, + graph_type = ?, + graph_height = ?, + panel_table = ?, + panel_bytes = ?, + panel_packets = ?, + panel_flows = ?, + usenat = ? + WHERE id = ?', + array( + get_request_var('timespan'), + get_request_var('device_id'), + $statistical, + $printed, + get_nfilter_request_var('sortfield'), + get_request_var('cutofflines'), + get_request_var('cutoffoctets'), + get_nfilter_request_var('graph_type'), + get_request_var('graph_height'), + get_nfilter_request_var('table') == 'true' ? 'on':'', + get_nfilter_request_var('bytes') == 'true' ? 'on':'', + get_nfilter_request_var('packets') == 'true' ? 'on':'', + get_nfilter_request_var('flows') == 'true' ? 'on':'', + get_nfilter_request_var('usenat') == 'true' ? 'on':'', + get_request_var('query') + ) + ); + + print json_encode(['rematerialize' => $material]); +} + +/** + * @return void + */ +function save_filter(): void { + global $config; + + $save = flowview_filter_form_values(); + + /* worked out before the save, as it compares with the Filter as it was */ + $material = flowview_filter_form_is_material($save); + $id = flowview_sql_save($save, 'plugin_flowview_queries', 'id', true); if (is_error_message()) { @@ -598,10 +880,15 @@ function save_filter(): void { header('Location: ' . html_escape(get_nfilter_request_var('return') . '?query=' . (empty($id) ? get_filter_request_var('id') : $id))); } - exit; + flowview_exit(); } - raise_message(1); + /* a change to what is cached is said in a dialog, with the note to materialize it again */ + if ($material) { + $_SESSION['flowview_rematerialize_saved'] = (int) $id; + } else { + raise_message(1); + } flowview_purge_query_caches($id); @@ -611,7 +898,7 @@ function save_filter(): void { header('Location: ' . html_escape(get_nfilter_request_var('return') . '?query=' . (empty($id) ? get_filter_request_var('id') : $id))); } - exit; + flowview_exit(); } /** @@ -674,6 +961,38 @@ function flowview_gettimespan(): void { print json_encode($span); } +/** + * Shifts the current date1/date2 range backward or forward by the interval + * selected in the predefined_timeshift dropdown and returns the new range as + * JSON. Uses the same interval strings as Cacti's graph time shifting. + * + * @return void + */ +function flowview_timeshift(): void { + global $graph_timeshifts_vals; + + $direction = get_nfilter_request_var('direction') == 'left' ? '-' : '+'; + $shift = get_filter_request_var('predefined_timeshift'); + $date1 = strtotime(get_nfilter_request_var('date1')); + $date2 = strtotime(get_nfilter_request_var('date2')); + $span = []; + + if (!isset($graph_timeshifts_vals[$shift])) { + $shift = GTS_1_DAY; + } + + if ($date1 !== false && $date2 !== false) { + $interval = $graph_timeshifts_vals[$shift]; + + $span['begin_now'] = strtotime($direction . $interval, $date1); + $span['end_now'] = strtotime($direction . $interval, $date2); + $span['current_value_date1'] = date('Y-m-d H:i', $span['begin_now']); + $span['current_value_date2'] = date('Y-m-d H:i', $span['end_now']); + } + + print json_encode($span); +} + /** * @param mixed $data * @@ -698,6 +1017,9 @@ function flowview_display_filter(): void { $title = __esc('Undefined Filter [ Select Filter to get Details ]', 'flowview'); + $report_fixed = false; + $report_printed = substr((string) get_nfilter_request_var('report'), 0, 1) == 'p'; + if (get_filter_request_var('query') > 0) { $row = flowview_db_fetch_row_prepared('SELECT name, statistics, printed FROM plugin_flowview_queries @@ -705,14 +1027,18 @@ function flowview_display_filter(): void { array(get_request_var('query'))); if (cacti_sizeof($row)) { + $report_fixed = flowview_filter_report_key($row) != ''; if ($row['statistics'] > 0) { $title = __esc('Statistical Report: %s [ Including overrides as specified below ]', $stat_report_array[$row['statistics']]); } elseif ($row['printed'] > 0) { - $title = __esc('Printed Report: %s [ Including overrides as specified below ]', $print_report_array[$row['statistics']]); + $title = __esc('Printed Report: %s [ Including overrides as specified below ]', $print_report_array[$row['printed']]); } } } + /* a Filter saved from its editor, that was opened from this page, comes back here */ + flowview_show_saved_dialog(); + html_start_box($title . ' ', '100%', false, 3, 'center', ''); ?> @@ -807,6 +1133,12 @@ function flowview_display_filter(): void { + 0 && !$report_fixed; + + if ($report_shown) { + ?> @@ -840,6 +1172,7 @@ function flowview_display_filter(): void { ?> + @@ -944,12 +1277,16 @@ function flowview_display_filter(): void {
'> - > - + + > + + '> - > - + + > + +
@@ -1037,20 +1374,28 @@ function flowview_display_filter(): void { - > - + + > + + - > - + + > + + - > - + + > + + - > - + + > + + @@ -1179,11 +1524,11 @@ function flowview_display_filter(): void { }); $('.timeshiftBackward').off('click').on('click', function() { - timeshiftFilterLeft(); + timeshiftFilter('left'); }); $('.timeshiftForward').off('click').on('click', function() { - timeshiftFilterRight(); + timeshiftFilter('right'); }); $('#flowview_filter').on('submit', function(event) { @@ -1471,14 +1816,16 @@ function exportData() { initTimespan(); } - if ($('#report').val().indexOf('p') >= 0) { - $('#bytes').prop('disabled', true); - $('#flows').prop('disabled', true); - $('#packets').prop('disabled', true); - } else { - $('#bytes').prop('disabled', false); - $('#flows').prop('disabled', false); - $('#packets').prop('disabled', false); + if ($('#report').length) { + if ($('#report').val().indexOf('p') >= 0) { + $('#bytes').prop('disabled', true); + $('#flows').prop('disabled', true); + $('#packets').prop('disabled', true); + } else { + $('#bytes').prop('disabled', false); + $('#flows').prop('disabled', false); + $('#packets').prop('disabled', false); + } } // Setup the charts @@ -1521,12 +1868,17 @@ function exportData() { }); }); + /* the Report is not on the page when the Filter has one, as the server knows it from the Filter */ + function reportParam() { + return $('#report').length ? '&report=' + $('#report').val() : ''; + } + function renderBarChart(type, bindto, title, label, height, width) { $.getJSON('flowview.php?action=chartdata&type=' + type + '&domains=' + $('#domains').is(':checked') + '&usenat=' + $('#usenat').is(':checked') + '&query=' + $('#query').val() + - '&report=' + $('#report').val() + + reportParam() + '&device_id=' + $('#device_id').val() + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -1591,7 +1943,7 @@ function renderTreemapChart(type, bindto, title, label, height, width) { '&domains=' + $('#domains').is(':checked') + '&usenat=' + $('#usenat').is(':checked') + '&query=' + $('#query').val() + - '&report=' + $('#report').val() + + reportParam() + '&device_id=' + $('#device_id').val() + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -1657,7 +2009,7 @@ function renderPieChart(type, bindto, title, label, height, width) { '&domains=' + $('#domains').is(':checked') + '&usenat=' + $('#usenat').is(':checked') + '&query=' + $('#query').val() + - '&report=' + $('#report').val() + + reportParam() + '&device_id=' + $('#device_id').val() + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -1747,13 +2099,13 @@ function numFormatter(num) { } function saveFilter() { - $.get(urlPath + 'plugins/flowview/flowview.php' + + $.getJSON(urlPath + 'plugins/flowview/flowview.php' + '?action=savefilter' + '&query=' + $('#query').val() + '&domains=' + $('#domains').is(':checked') + '&usenat=' + $('#usenat').is(':checked') + '×pan=' + $('#predefined_timespan').val() + - '&report=' + $('#report').val() + + reportParam() + '&device_id=' + $('#device_id').val() + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -1765,9 +2117,14 @@ function saveFilter() { '&flows=' + $('#flows').is(':checked') + '&exclude=' + $('#exclude').val() + '&graph_type=' + $('#graph_type').val() + - '&graph_height=' + $('#graph_height').val(), function() { + '&graph_height=' + $('#graph_height').val(), function(data) { Pace.stop(); - $('#text').show().text('[ ]').fadeOut(2000); + + if (data['rematerialize']) { + $('#text').show().text('[ ]').fadeOut(10000); + } else { + $('#text').show().text('[ ]').fadeOut(2000); + } }); } @@ -1808,6 +2165,30 @@ function applyTimespan() { } } + function setCustomTimespan() { + $('#predefined_timespan').val('0'); + + if ($('#predefined_timespan').selectmenu('instance') !== undefined) { + $('#predefined_timespan').selectmenu('refresh'); + } + } + + function timeshiftFilter(direction) { + $.getJSON(urlPath + 'plugins/flowview/flowview.php' + + '?action=timeshift' + + '&direction=' + direction + + '&predefined_timeshift=' + $('#predefined_timeshift').val() + + '&date1=' + encodeURIComponent($('#date1').val()) + + '&date2=' + encodeURIComponent($('#date2').val()), function(data) { + if (data['current_value_date1'] !== undefined) { + $('#date1').val(data['current_value_date1']); + $('#date2').val(data['current_value_date2']); + setCustomTimespan(); + applyFilter(); + } + }); + } + function changeQuery() { loadPageNoHeader(urlPath+'plugins/flowview/flowview.php' + '?action=query' + @@ -1823,18 +2204,21 @@ function changeQuery() { } function applyFilter(reset) { + var report = ''; + var extra = ''; + if (reset) { - var report = 0; + report = '&report=0'; var device_id = 0; } else { - var report = $('#report').val(); - var device_id = $('#device_id').val(); - } + /* only a Filter that has no Report has one on the page */ + report = reportParam(); - if (report.indexOf('p') >= 0) { - var extra='&table=true&bytes=false&packets=false&flows=false'; - } else { - var extra=''; + if ($('#report').length && $('#report').val().indexOf('p') >= 0) { + extra = '&table=true&bytes=false&packets=false&flows=false'; + } + + var device_id = $('#device_id').val(); } loadPageNoHeader(urlPath+'plugins/flowview/flowview.php' + @@ -1843,7 +2227,7 @@ function applyFilter(reset) { '&usenat=' + $('#usenat').is(':checked') + '&query=' + $('#query').val() + '&predefined_timespan=' + $('#predefined_timespan').val() + - '&report=' + report + + report + '&device_id=' + device_id + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -1862,10 +2246,7 @@ function clearFilter() { } $('#date1, #date2').on('change', function() { - $('#predefined_timespan').val('0'); - - $('#predefined_timespan').selectmenu('refresh'); - + setCustomTimespan(); }); @@ -2345,7 +2726,14 @@ function get_tables_for_query($start, $end = null) { WHERE TABLE_NAME LIKE "plugin_flowview_raw_%"'); if (cacti_sizeof($tables)) { + /* a materialization worker scans only the partitions of its batch */ + $allowed = $GLOBALS['flowview_partition_allowlist'] ?? null; + foreach($tables as $t) { + if (is_array($allowed) && !isset($allowed[$t['table']])) { + continue; + } + $parts = explode('_', $t['table']); $partition = trim($parts[3]); @@ -2515,8 +2903,20 @@ function flowview_get_chartdata(): void { $start = strtotime(get_nfilter_request_var('date1')); $end = strtotime(get_nfilter_request_var('date2')); + /* the report is not sent for a Filter that has one, as it is part of the Filter */ + if ($query_id > 0) { + $filter_report = flowview_filter_report_key(flowview_db_fetch_row_prepared('SELECT statistics, printed + FROM plugin_flowview_queries + WHERE id = ?', + [$query_id]) ?: []); + + if ($filter_report != '') { + $report = $filter_report; + } + } + if (!empty($report) && substr($report, 0, 1) == 's') { - $report = trim(get_nfilter_request_var('report'), 'sp'); + $report = trim($report, 'sp'); $output = run_flow_query(true, $query_id, $start, $end); @@ -2593,12 +2993,12 @@ function flowview_get_chartdata(): void { ]; print json_encode($chartData, JSON_NUMERIC_CHECK); - exit; + flowview_exit(); } } print json_encode([]); - exit; + flowview_exit(); } /** @@ -2715,14 +3115,24 @@ function get_category_columns($statistics, $domain) { * filter table * an error. * - * @param mixed $session - * @param mixed $query_id - * @param mixed $start - * @param mixed $end + * When $plan_only is true nothing is run. The md5sum_tables of each map + * query the parallel query would cache against, which is how the cached + * shards of the query are found, are put in $plan, and true is returned. + * They are not returned, so what this returns is only ever the report. + * $force_nat then decides the NAT mode instead of the request variables. + * Use flowview_plan_filter() rather than calling this for a plan. + * + * @param mixed $session + * @param mixed $query_id + * @param mixed $start + * @param mixed $end + * @param bool $plan_only + * @param bool|null $force_nat + * @param array|null $plan Set to the cache keys when $plan_only is true * * @return mixed */ -function run_flow_query($session, $query_id, $start, $end) { +function run_flow_query($session, $query_id, $start, $end, bool $plan_only = false, ?bool $force_nat = null, ?array &$plan = null) { global $stat_columns_array; global $config, $graph_timespans; @@ -2744,7 +3154,9 @@ function run_flow_query($session, $query_id, $start, $end) { } /* close session to allow offpage navigation */ - cacti_session_close(); + if (!$plan_only) { + cacti_session_close(); + } require($config['base_path'] . '/plugins/flowview/includes/arrays.php'); @@ -2753,6 +3165,10 @@ function run_flow_query($session, $query_id, $start, $end) { WHERE id = ?', [$query_id]); + if ($plan_only && !cacti_sizeof($data)) { + return false; + } + $title = flowview_db_fetch_cell_prepared('SELECT name FROM plugin_flowview_queries WHERE id = ?', @@ -2810,27 +3226,31 @@ function run_flow_query($session, $query_id, $start, $end) { $sql_where = get_numeric_filter($sql_where, $sql_params, $data['device_id'], 'listener_id'); } - /* ex_addr override */ - if (isset_request_var('ex_addr') && get_nfilter_request_var('ex_addr') != '0' && get_nfilter_request_var('ex_addr') != '-1') { + /* ex_addr override, an exporter address, so anything else is not one */ + $ex_addr = isset_request_var('ex_addr') ? filter_var(get_nfilter_request_var('ex_addr'), FILTER_VALIDATE_IP) : false; + + if ($ex_addr !== false) { $sql_where .= ($sql_where != '' ? ' AND ':'') . 'ex_addr = ?'; - $sql_params[] = get_request_var('ex_addr'); + $sql_params[] = $ex_addr; } elseif (isset($data['ex_addr']) && $data['ex_addr'] != '-1' && $data['ex_addr'] != '' && $data['ex_addr'] != '0') { $sql_where .= ($sql_where != '' ? ' AND ':'') . 'ex_addr = ?'; $sql_params[] = $data['ex_addr']; } - /* report override */ + /* report override, unless the filter has a report, as it is part of what is cached */ $report = ''; if (isset_request_var('report')) { - $report = get_nfilter_request_var('report'); - $nreport = trim(get_nfilter_request_var('report'), 'sp'); - if (strpos($report, 's') !== false && is_numeric($nreport)) { - $data['statistics'] = $nreport; - $data['printed'] = 0; - } elseif (strpos($report, 'p') !== false && is_numeric($nreport)) { - $data['printed'] = $nreport; - $data['statistics'] = 0; + if (flowview_filter_report_key($data) == '') { + $report = get_nfilter_request_var('report'); + $nreport = trim(get_nfilter_request_var('report'), 'sp'); + if (strpos($report, 's') !== false && is_numeric($nreport)) { + $data['statistics'] = $nreport; + $data['printed'] = 0; + } elseif (strpos($report, 'p') !== false && is_numeric($nreport)) { + $data['printed'] = $nreport; + $data['statistics'] = 0; + } } } elseif (!sizeof($data)) { return false; @@ -3348,17 +3768,22 @@ function run_flow_query($session, $query_id, $start, $end) { (preg_match('/^(bytes|packets|flows)$/i', $data['sortfield']) ? ' DESC':' ASC'); } } else { - cacti_log(sprintf('WARNING: The Flowview Query %s [%s] does not include either a Statistical or Printed Report.', $data['name'], $data['id']), false, 'FLOWVIEW'); + if (!$plan_only) { + cacti_log(sprintf('WARNING: The Flowview Query %s [%s] does not include either a Statistical or Printed Report.', $data['name'], $data['id']), false, 'FLOWVIEW'); + } + return false; } if ($sql_outer == '' || ($sql_inner == '' && $sql_inner1 == '')) { + if ($plan_only) { + return false; + } + if ($data['statistics'] > 0) { $message = __('The Statistical Report %s is not known. Please open a bug on GitHub', $data['statistics'], 'flowview'); - } elseif($data['printed'] > 0) { - $message = __('The Printed Report %s is not known. Please open a bug on GitHub', $data['printed'], 'flowview'); } else { - $message = __('No valid report was selected. Please open a bug on GitHub', 'flowview'); + $message = __('The Printed Report %s is not known. Please open a bug on GitHub', $data['printed'], 'flowview'); } raise_message('bad_report', $message, MESSAGE_LEVEL_ERROR); @@ -3369,7 +3794,9 @@ function run_flow_query($session, $query_id, $start, $end) { } /* use post-NAT data instead of the original src/dst values - issue#110 */ - if (isset_request_var('usenat')) { + if ($force_nat !== null) { + $use_nat = $force_nat; + } elseif (isset_request_var('usenat')) { $use_nat = (get_request_var('usenat') == 'true'); } else { $use_nat = (isset($data['usenat']) && $data['usenat'] == 'on'); @@ -3409,6 +3836,81 @@ function run_flow_query($session, $query_id, $start, $end) { $sql_inner_groupby2 = flowview_apply_nat_toggle($sql_inner_groupby2, $use_nat, false); } + /** + * The map (inner) structures cache their shards by an md5 of the + * query without the time range, so they can be built before the + * tables are known. + */ + if ($sql_inner1 != '') { + $stru_outer = [ + 'sql_query' => $sql_outer, + 'sql_where' => '', + 'sql_groupby' => $sql_groupby, + 'sql_having' => $sql_having, + 'sql_order' => $sql_order, + 'sql_limit' => $sql_limit, + 'sql_params' => [] + ]; + + $stru_inners = []; + + foreach ([[$sql_inner1, $sql_inner_groupby1], [$sql_inner2, $sql_inner_groupby2]] as $inner) { + $stru_inner = [ + 'sql_query' => $inner[0], + 'sql_where' => $sql_where, + 'sql_range' => $sql_range, + 'sql_groupby' => $inner[1], + 'sql_having' => '', + 'sql_order' => '', + 'sql_limit' => '', + 'sql_params' => $sql_params, + 'sql_range_params' => $sql_range_params, + 'sql_start_time' => $start, + 'sql_end_time' => $end + ]; + + /* only statistical reports have two map queries, so there is no printed limit or order here */ + $stru_inners[] = $stru_inner; + } + } else { + $stru_inner = [ + 'sql_query' => $sql_inner, + 'sql_where' => $sql_where, + 'sql_range' => $sql_range, + 'sql_having' => '', + 'sql_order' => '', + 'sql_limit' => '', + 'sql_groupby' => $sql_inner_groupby, + 'sql_params' => $sql_params, + 'sql_range_params' => $sql_range_params, + 'sql_start_time' => $start, + 'sql_end_time' => $end + ]; + + if ($request_type == 'printed') { + $stru_inner['sql_limit'] = 'LIMIT 100'; + $stru_inner['sql_order'] = $sql_order; + } + + $stru_outer = [ + 'sql_query' => $sql_outer, + 'sql_where' => '', + 'sql_having' => $sql_having, + 'sql_groupby' => $sql_groupby, + 'sql_order' => $sql_order, + 'sql_limit' => $sql_limit, + 'sql_params' => [] + ]; + + $stru_inners = [$stru_inner]; + } + + if ($plan_only) { + $plan = array_map('parallel_database_query_table_md5', $stru_inners); + + return true; + } + $tables = get_tables_for_query($start, $end); $sql = ''; $all_params = []; @@ -3476,84 +3978,7 @@ function run_flow_query($session, $query_id, $start, $end) { cacti_log(sprintf('PARALLEL STATS: Time:%0.3f Threads:%d Shards:%d', $end - $start, $threads, $shards), false, 'FLOWVIEW'); } else { - if ($sql_inner1 != '') { - $stru_inner1 = [ - 'sql_query' => $sql_inner1, - 'sql_where' => $sql_where, - 'sql_range' => $sql_range, - 'sql_groupby' => $sql_inner_groupby1, - 'sql_having' => '', - 'sql_order' => '', - 'sql_limit' => '', - 'sql_params' => $sql_params, - 'sql_range_params' => $sql_range_params, - 'sql_start_time' => $start, - 'sql_end_time' => $end - ]; - - $stru_inner2 = [ - 'sql_query' => $sql_inner2, - 'sql_where' => $sql_where, - 'sql_range' => $sql_range, - 'sql_groupby' => $sql_inner_groupby2, - 'sql_having' => '', - 'sql_order' => '', - 'sql_limit' => '', - 'sql_params' => $sql_params, - 'sql_range_params' => $sql_range_params, - 'sql_start_time' => $start, - 'sql_end_time' => $end - ]; - - $stru_outer = [ - 'sql_query' => $sql_outer, - 'sql_where' => '', - 'sql_groupby' => $sql_groupby, - 'sql_having' => $sql_having, - 'sql_order' => $sql_order, - 'sql_limit' => $sql_limit, - 'sql_params' => [] - ]; - - if ($request_type == 'printed') { - $stru_inner1['sql_limit'] = 'LIMIT 100'; - $stru_inner1['sql_order'] = $sql_order; - $stru_inner2['sql_limit'] = 'LIMIT 100'; - $stru_inner2['sql_order'] = $sql_order; - } - - $requests[] = parallel_database_query_request($tables, $stru_inner1, $stru_outer); - $requests[] = parallel_database_query_request($tables, $stru_inner2, $stru_outer); - } else { - $stru_inner = [ - 'sql_query' => $sql_inner, - 'sql_where' => $sql_where, - 'sql_range' => $sql_range, - 'sql_having' => '', - 'sql_order' => '', - 'sql_limit' => '', - 'sql_groupby' => $sql_inner_groupby, - 'sql_params' => $sql_params, - 'sql_range_params' => $sql_range_params, - 'sql_start_time' => $start, - 'sql_end_time' => $end - ]; - - if ($request_type == 'printed') { - $stru_inner['sql_limit'] = 'LIMIT 100'; - $stru_inner['sql_order'] = $sql_order; - } - - $stru_outer = [ - 'sql_query' => $sql_outer, - 'sql_where' => '', - 'sql_having' => $sql_having, - 'sql_groupby' => $sql_groupby, - 'sql_order' => $sql_order, - 'sql_limit' => $sql_limit, - 'sql_params' => [] - ]; - + foreach ($stru_inners as $stru_inner) { $requests[] = parallel_database_query_request($tables, $stru_inner, $stru_outer); } @@ -3807,132 +4232,1575 @@ function run_flow_query($session, $query_id, $start, $end) { */ $results = $results[0]; - /** @var array $sql_array */ - foreach($sql_array as $c) { - $total += $results[$c['name']]; - } + /** @var array $sql_array */ + foreach($sql_array as $c) { + $total += $results[$c['name']]; + } + + $table .= ''; + $table .= ''; + + for ($i = 0; $i < 14; $i++) { + $table .= ''; + } + + $table .= ''; + $table .= ''; + + for ($i = 0; $i < 14; $i++) { + $name = $sql_array[$i]['name']; + + if ($total > 0) { + $table .= ''; + } else { + $table .= ''; + } + } + + $table .= '
' . $sql_array[$i]['title'] . '
' . number_format_i18n(($results[$name] / $total) * 100, 2) . ' %' . __('N/A', 'flowview') . '
'; + + $table .= ''; + $table .= '
'; + $table .= ''; + + for ($i = 14; $i < 27; $i++) { + $table .= ''; + } + + $table .= ''; + $table .= ''; + + $table .= ''; + + for ($i = 14; $i < 27; $i++) { + $name = $sql_array[$i]['name']; + if ($total > 0) { + $table .= ''; + } else { + $table .= ''; + } + } + + $table .= ''; + $table .= '
' . $sql_array[$i]['title'] . '
' . number_format_i18n(($results[$name] / $total) * 100, 2) . ' %' . __('N/A', 'flowview') . '
'; + } + + $output['table'] = $table; + $output['title'] = $title; + + return $output; + } + } + + return false; +} + +/** + * @param mixed $table + * @param mixed $type + * + * @return mixed + */ +function flowview_table_name_to_time($table, $type) { + if (!preg_match('/^plugin_flowview_raw_([0-9]{7})([0-9]{2})?$/', $table, $matches)) { + return false; + } + + $suffix = $matches[1] . (isset($matches[2]) ? $matches[2] : ''); + + $year = substr($suffix, 0, 4); + $day = substr($suffix, 4, 3); + + if (strlen($suffix) == 7) { + $gran = 'days'; + $hour = 0; + } else { + $gran = 'hours'; + $hour = substr($suffix, 7, 2); + } + + $dates = flowview_convert_yeardayhour_to_date($gran, $year, $day, (int) $hour); + + return $dates[$type]; +} + +/** + * @param mixed $range + * @param mixed $year + * @param mixed $day + * @param int $hour + * + * @return array + */ +function flowview_convert_yeardayhour_to_date($range, $year, $day, int $hour = 0): array { + $datetime = new DateTime(); + $datetime->setDate((int) $year, 1, 1)->setTime(0, 0, 0); + $datetime->modify('+' . (int) $day . ' days'); + + if ($range == 'days') { + $start_date = $datetime->format('Y-m-d 00:00:00'); + $datetime->modify('+1 day'); + $end_date = $datetime->format('Y-m-d 00:00:00'); + } else { + $datetime->modify('+' . (int) $hour . ' hours'); + $start_date = $datetime->format('Y-m-d H:00:00'); + $datetime->modify('+1 hour'); + $end_date = $datetime->format('Y-m-d H:00:00'); + } + + $start_time = strtotime($start_date); + $end_time = strtotime($end_date); + + return [ + 'start_date' => $start_date, + 'start_time' => $start_time, + 'end_date' => $end_date, + 'end_time' => $end_time + ]; +} + +/** + * flowview_get_materialize_range - Given a list of raw partition table + * names, return the Unix start and end timestamps spanning every partition. + * Names that are not partition tables are ignored, and false is returned + * when none qualify. + * + * A shard is only a full scan, and therefore only cached in + * parallel_database_query_shard_cache, when the requested range lies strictly + * outside the data in the partition. The end is padded by a second, as a + * flow ends no later than it was captured. The start is padded by a day, + * because a flow can start well before the window of the partition it was + * written to, and with less the earliest partition, or one that rotated, is + * never treated as a full scan and so is never cached. A partition that does + * not exist in the extra day is simply not there to scan. + * + * @param array $tables + * + * @return array|false + */ +function flowview_get_materialize_range(array $tables): array|false { + $start = false; + $end = false; + + foreach ($tables as $table) { + $table_start = flowview_table_name_to_time($table, 'start_time'); + $table_end = flowview_table_name_to_time($table, 'end_time'); + + if ($table_start === false || $table_end === false) { + continue; + } + + if ($start === false || $table_start < $start) { + $start = $table_start; + } + + if ($end === false || $table_end > $end) { + $end = $table_end; + } + } + + if ($start === false || $end === false) { + return false; + } + + return [$start - 86400, $end + 1]; +} + +/** + * flowview_shard_cache_is_current - Decide whether a cached partition shard + * still reflects its partition. The cache stores the partition's + * MIN(start_time) and MAX(end_time) from when it was built, so it is current + * only while the partition still reports the same bounds. + * + * The raw tables keep microseconds but the cache keeps whole seconds, which + * the database either rounds or truncates, so a difference of one second is + * treated as equal. + * + * @param array $cached The min_date and max_date stored with the cached shard + * @param array $current The min_date and max_date the partition reports now + * + * @return bool + */ +function flowview_shard_cache_is_current(array $cached, array $current): bool { + foreach (['min_date', 'max_date'] as $column) { + if (empty($cached[$column]) || empty($current[$column])) { + return false; + } + + $cached_time = strtotime($cached[$column]); + $current_time = strtotime($current[$column]); + + if ($cached_time === false || $current_time === false || abs($cached_time - $current_time) > 1) { + return false; + } + } + + return true; +} + +/** + * flowview_invalidate_stale_shard_cache - Remove cached partition shards that + * no longer match their partition, either because more flows arrived or + * because the partition table is gone. Shards that are still current are + * left alone so they are never materialized a second time. + * + * When anything is removed, the finished CLI requests (user 0) are dropped + * as well so their results are rebuilt from the refreshed shards. + * + * @param array $only Only check the cache of these partition tables, or all + * of it when empty + * + * @return array The number of shards kept and removed + */ +function flowview_invalidate_stale_shard_cache(array $only = []): array { + $kept = 0; + $removed = 0; + $current = []; + + if (cacti_sizeof($only)) { + $cache = flowview_db_fetch_assoc_prepared('SELECT md5sum, map_table, map_partition, min_date, max_date + FROM parallel_database_query_shard_cache + WHERE map_table IN (' . implode(', ', array_fill(0, cacti_sizeof($only), '?')) . ') + AND map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\' + ORDER BY map_table', + array_values($only)); + } else { + $cache = flowview_db_fetch_assoc('SELECT md5sum, map_table, map_partition, min_date, max_date + FROM parallel_database_query_shard_cache + WHERE map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\' + ORDER BY map_table'); + } + + if (cacti_sizeof($cache)) { + foreach ($cache as $entry) { + $table = $entry['map_table']; + + if (!isset($current[$table])) { + if (preg_match('/^plugin_flowview_raw_[0-9]+$/', $table) && flowview_db_table_exists($table)) { + $current[$table] = flowview_db_fetch_row("SELECT MIN(start_time) AS min_date, MAX(end_time) AS max_date + FROM $table"); + } else { + $current[$table] = []; + } + } + + if (cacti_sizeof($current[$table]) && flowview_shard_cache_is_current($entry, $current[$table])) { + $kept++; + + continue; + } + + flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache + WHERE md5sum = ? + AND map_table = ? + AND map_partition = ?', + [$entry['md5sum'], $entry['map_table'], $entry['map_partition']]); + + $removed++; + } + } + + if ($removed > 0) { + $requests = flowview_db_fetch_assoc_prepared('SELECT id + FROM parallel_database_query + WHERE user_id = 0 + AND status = ?', + ['complete']); + + foreach ($requests as $request) { + parallel_database_query_cleanup($request['id'], true); + } + } + + return ['kept' => $kept, 'removed' => $removed]; +} + +/** + * flowview_launch_materialize - Start cli/materialize_queries.php in the + * background, for the given FlowView queries, or all of them when none are + * given, and for the given partitions, or all of them when none are given. + * A filter that is already being materialized is left to the run that has + * it, so the filters can be materialized separately. + * + * @param array $query_ids The FlowView query ids to materialize + * @param array $partitions The partition tables to materialize + * + * @return array The filters a materialization was started for + */ +function flowview_launch_materialize(array $query_ids = [], array $partitions = []): array { + global $config; + + if (cacti_sizeof($query_ids)) { + $targets = array_map('intval', $query_ids); + } else { + $targets = array_map('intval', array_column(flowview_db_fetch_assoc('SELECT id + FROM plugin_flowview_queries + ORDER BY id'), 'id')); + } + + $targets = array_values(array_diff(array_unique(array_filter($targets, function ($id) { + return $id > 0; + })), flowview_get_materializing_filters())); + + if (!cacti_sizeof($targets)) { + return []; + } + + $arguments = cacti_escapeshellarg($config['base_path'] . '/plugins/flowview/cli/materialize_queries.php') . + ' --query-id=' . implode(',', $targets); + + $partitions = array_filter($partitions, function ($table) { + return flowview_partition_sort_key($table) !== ''; + }); + + if (cacti_sizeof($partitions)) { + $arguments .= ' --partitions=' . implode(',', $partitions); + } + + exec_background(read_config_option('path_php_binary'), $arguments); + + return $targets; +} + +/** + * parallel_database_query_map_query - The map query of a request as the JSON + * that is stored and hashed. The time range is left out, so the same query + * over a different range shares the same cached partition shards. + * + * @param array $stru_inner The map structure of a parallel query request + * + * @return string + */ +function parallel_database_query_map_query(array $stru_inner): string { + unset($stru_inner['sql_range']); + unset($stru_inner['sql_range_params']); + unset($stru_inner['sql_start_time']); + unset($stru_inner['sql_end_time']); + + return (string) json_encode($stru_inner); +} + +/** + * parallel_database_query_table_md5 - The md5sum_tables of a map structure, + * which is the key the partition shards are cached under. + * + * @param array $stru_inner The map structure of a parallel query request + * + * @return string + */ +function parallel_database_query_table_md5(array $stru_inner): string { + return md5(parallel_database_query_map_query($stru_inner)); +} + +/** + * flowview_partition_sort_key - The partition of a raw table name as a + * fixed length key that sorts in time order, whether the partitions are + * daily or hourly. An empty key is returned for anything that is not a + * raw partition table. + * + * @param string $table + * + * @return string + */ +function flowview_partition_sort_key(string $table): string { + if (!preg_match('/^plugin_flowview_raw_([0-9]{7})([0-9]{2})?$/', $table, $matches)) { + return ''; + } + + return $matches[1] . ($matches[2] ?? '00'); +} + +/** + * flowview_get_partition_tables - The names of the raw flow partition tables + * that exist, oldest first. Anything that merely looks like a partition + * table, such as plugin_flowview_raw_abc, is left out. + * + * @return array + */ +function flowview_get_partition_tables(): array { + $tables = flowview_db_fetch_assoc('SELECT TABLE_NAME AS `table` + FROM information_schema.TABLES + WHERE TABLE_NAME LIKE "plugin_flowview_raw_%"'); + + $partitions = []; + + foreach ($tables as $table) { + if (flowview_partition_sort_key($table['table']) !== '') { + $partitions[] = $table['table']; + } + } + + usort($partitions, function ($a, $b) { + return strcmp(flowview_partition_sort_key($a), flowview_partition_sort_key($b)); + }); + + return $partitions; +} + +/** + * flowview_get_live_partition - The raw partition table that is still + * receiving flows, which is the newest one. + * + * @return string The table, or an empty string when there are no partitions + */ +function flowview_get_live_partition(): string { + $tables = flowview_get_partition_tables(); + + return cacti_sizeof($tables) ? (string) end($tables) : ''; +} + +/** + * flowview_is_full_scan - Whether a shard scans the whole of its partition, + * which is when the range starts before the earliest flow in it and ends + * after the last. Such a shard does not depend on the range, so its result + * is cached and reused by any query that covers the partition. + * + * The live partition is never a full scan. It is still receiving flows, so + * a result cached for it would be wrong as soon as more arrive. + * + * @param int $start The start of the range + * @param int $end The end of the range + * @param int $partition_start The time of the earliest flow in the partition + * @param int $partition_end The time of the latest flow in the partition + * @param bool $live Whether it is the live partition + * + * @return bool + */ +function flowview_is_full_scan(int $start, int $end, int $partition_start, int $partition_end, bool $live): bool { + return !$live && $start < $partition_start && $end > $partition_end; +} + +/** + * flowview_remove_live_partition_cache - Remove anything cached for the live + * partition, from before it was kept out of the cache. It would never be + * used, as the live partition is no longer scanned as a full scan, but it + * would be counted as materialized. + * + * @return int How many cached results were removed + */ +function flowview_remove_live_partition_cache(): int { + $live = flowview_get_live_partition(); + + if ($live == '') { + return 0; + } + + $cached = (int) flowview_db_fetch_cell_prepared('SELECT COUNT(*) + FROM parallel_database_query_shard_cache + WHERE map_table = ? + AND map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\'', + [$live]); + + if ($cached > 0) { + flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache + WHERE map_table = ? + AND map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\'', + [$live]); + } + + return $cached; +} + +/** + * flowview_live_shard_key - The key a live partition shard result is shared + * under. Requests that run the same map query, with the same range, against + * the same partition get the same key, such as the table and the charts of a + * single report page. It is derived from the map query, so it never matches + * the key of a materialized shard. + * + * @param array $query The parallel query request + * @param array $shard The shard of the live partition + * + * @return string + */ +function flowview_live_shard_key(array $query, array $shard): string { + return md5($query['md5sum_tables'] . '|' . $shard['map_table'] . '|' . $shard['map_params']); +} + +/** + * flowview_live_shard_lock - Take the lock that makes the workers of one + * live shard key run the map query one at a time, so the rest can reuse the + * result of the first. + * + * @param string $key The live shard key + * + * @return bool Whether the lock was taken within the wait time + */ +function flowview_live_shard_lock(string $key): bool { + return (int) flowview_db_fetch_cell_prepared('SELECT GET_LOCK(?, ?)', + [FLOWVIEW_LIVE_SHARD_LOCK . $key, FLOWVIEW_LIVE_SHARD_LOCK_WAIT]) === 1; +} + +/** + * flowview_live_shard_unlock - Release the lock of flowview_live_shard_lock(). + * + * @param string $key The live shard key + * + * @return void + */ +function flowview_live_shard_unlock(string $key): void { + flowview_db_fetch_cell_prepared('SELECT RELEASE_LOCK(?)', [FLOWVIEW_LIVE_SHARD_LOCK . $key]); +} + +/** + * flowview_live_shard_get - The result of a live shard that was run in the + * last FLOWVIEW_LIVE_SHARD_TTL seconds. + * + * @param string $key The live shard key + * @param string $table The live partition table + * + * @return array|null null when there is no recent result + */ +function flowview_live_shard_get(string $key, string $table): ?array { + $row = flowview_db_fetch_row_prepared('SELECT results + FROM parallel_database_query_shard_cache + WHERE md5sum = ? + AND map_table = ? + AND map_partition = ? + AND date_created >= DATE_SUB(NOW(), INTERVAL ? SECOND)', + [$key, $table, FLOWVIEW_LIVE_SHARD_MARKER, FLOWVIEW_LIVE_SHARD_TTL]); + + if (!cacti_sizeof($row)) { + return null; + } + + $results = json_decode($row['results'], true); + + return is_array($results) ? $results : null; +} + +/** + * flowview_live_shard_save - Keep the result of a live shard for + * FLOWVIEW_LIVE_SHARD_TTL seconds. + * + * @param string $key The live shard key + * @param string $table The live partition table + * @param array $results The result of the map query + * + * @return void + */ +function flowview_live_shard_save(string $key, string $table, array $results): void { + flowview_db_execute_prepared('REPLACE INTO parallel_database_query_shard_cache + (md5sum, map_table, map_partition, min_date, max_date, results, date_created) + VALUES (?, ?, ?, NOW(), NOW(), ?, NOW())', + [$key, $table, FLOWVIEW_LIVE_SHARD_MARKER, json_encode($results)]); +} + +/** + * flowview_live_shard_expire - Remove the live shard results that are no + * longer reusable. + * + * @return void + */ +function flowview_live_shard_expire(): void { + flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache + WHERE map_partition = ? + AND date_created < DATE_SUB(NOW(), INTERVAL ? SECOND)', + [FLOWVIEW_LIVE_SHARD_MARKER, FLOWVIEW_LIVE_SHARD_TTL]); +} + +/** + * flowview_get_cold_partitions - The partitions that have rotated. The + * newest partition is the one still receiving flows, every older one is cold. + * + * @param array $tables The partition tables + * + * @return array + */ +function flowview_get_cold_partitions(array $tables): array { + $keys = array_map('flowview_partition_sort_key', $tables); + + if (!cacti_sizeof($keys)) { + return []; + } + + $newest = max($keys); + + return array_values(array_filter($tables, function ($table) use ($newest) { + return flowview_partition_sort_key($table) < $newest; + })); +} + +/** + * flowview_get_uncached_partitions - The cold partitions that a filter does + * not have cached yet, with NAT or without. What a materialization still has + * to do, so it can be done a part at a time and be continued. + * + * @param int $query_id The FlowView query + * @param string $mode on for NAT, off for without + * @param array $cold The cold partitions, oldest first + * + * @return array The partitions that are not cached, oldest first + */ +function flowview_get_uncached_partitions(int $query_id, string $mode, array $cold): array { + $key = $mode == 'on' ? 'nat' : 'non_nat'; + $md5sums = flowview_get_filter_md5sums($query_id); + + /* without its cache keys all that can be done is to run it for everything */ + if (!cacti_sizeof($md5sums[$key])) { + return array_values($cold); + } + + $cached = flowview_get_cached_shard_tables($query_id, $md5sums); + + return array_values(array_diff($cold, $cached[$key])); +} + +/** + * flowview_materialize_batch_size - How many partitions a materialization + * runs through one parallel query. The query has to finish within the + * Maximum Run Time, and every year of partitions can not, so it is run for a + * part of them at a time. A batch that did not finish is followed by a + * smaller one. + * + * @param int $current The size of the last batch, or 0 for the first + * @param bool $timed_out Whether the last batch did not finish + * @param int $threads The number of Max Concurrent Threads + * + * @return int + */ +function flowview_materialize_batch_size(int $current, bool $timed_out, int $threads): int { + if ($current <= 0) { + return max(40, $threads * 25); + } + + return $timed_out ? max(4, intdiv($current, 2)) : $current; +} + +/** + * flowview_get_rotated_partitions - The partitions that went cold since the + * previous newest partition was recorded. That partition was still hot then, + * so it is the first of them, and the newest partition is still hot now. + * + * @param array $tables The partition tables + * @param string $previous The newest partition recorded by the previous cycle + * + * @return array + */ +function flowview_get_rotated_partitions(array $tables, string $previous): array { + $previous_key = flowview_partition_sort_key($previous); + $keys = array_map('flowview_partition_sort_key', $tables); + + if ($previous_key === '' || !cacti_sizeof($keys)) { + return []; + } + + $newest = max($keys); + + return array_values(array_filter($tables, function ($table) use ($previous_key, $newest) { + $key = flowview_partition_sort_key($table); + + return $key >= $previous_key && $key < $newest; + })); +} + +/** + * flowview_get_partition_alter - What a raw partition table needs altered to + * match the current layout: the character set, the keys and columns the + * queries use, the ex_addr type, and the storage engine. This is the single + * definition used by the schema upgrade and by the materializer, so a + * partition the upgrade has not reached yet is prepared the same way. + * + * Keys of $info: collation and engine of the table, has_keycol and + * has_template_id for those columns, indexes as the names of the indexes + * it has, and ex_addr_type as the type of the ex_addr column, or null. + * + * @param array $info What the partition table currently is + * @param string $raw_engine The engine a partition table that has rotated uses + * + * @return array The alter as a string, which is empty when none is needed, + * and whether ex_addr has to be rewritten afterwards + */ +function flowview_get_partition_alter(array $info, string $raw_engine): array { + $alter = ''; + $ex_change = false; + $indexes = $info['indexes'] ?? []; + + if (($info['collation'] ?? '') != 'latin1_swedish_ci') { + $alter = 'CONVERT TO CHARACTER SET latin1'; + } + + if (!empty($info['has_keycol'])) { + $alter .= ($alter != '' ? ', ':'') . 'DROP KEY keycol'; + } + + if (empty($info['has_template_id'])) { + $alter .= ($alter != '' ? ', ':'') . 'ADD COLUMN template_id int(11) unsigned NOT NULL default "0" AFTER listener_id'; + } + + foreach (['template_id', 'start_time', 'end_time'] as $index) { + if (!in_array($index, $indexes, true)) { + $alter .= ($alter != '' ? ', ':'') . "ADD INDEX $index ($index)"; + } + } + + if (isset($info['ex_addr_type']) && stripos($info['ex_addr_type'], 'VARCHAR') === false) { + $alter .= ($alter != '' ? ', ':'') . 'MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default ""'; + $ex_change = true; + } + + if (!in_array('ex_addr', $indexes, true)) { + $alter .= ($alter != '' ? ', ':'') . 'ADD INDEX ex_addr (ex_addr)'; + } + + if (($info['engine'] ?? '') != $raw_engine) { + if ($raw_engine == 'Aria') { + $alter .= ($alter != '' ? ',':'') . " ENGINE=$raw_engine ROW_FORMAT=page"; + } else { + $alter .= ($alter != '' ? ',':'') . " ENGINE=$raw_engine ROW_FORMAT=Dynamic"; + } + } + + return ['alter' => $alter, 'ex_change' => $ex_change]; +} + +/** + * flowview_apply_partition_alter - Alter a raw partition table as planned by + * flowview_get_partition_alter(). + * + * @param string $table The raw partition table + * @param array $plan The plan from flowview_get_partition_alter() + * @param bool $print Also show what is being altered, as well as logging it. + * A script that reports it itself passes false. + * + * @return void + */ +function flowview_apply_partition_alter(string $table, array $plan, bool $print = true): void { + cacti_log("Altering Table $table Using this alter " . trim($plan['alter']) . '.', $print, 'FLOWVIEW'); + + flowview_db_execute('ALTER TABLE ' . $table . ' ' . $plan['alter']); + + if ($plan['ex_change']) { + flowview_db_execute('UPDATE ' . $table . ' SET ex_addr = SUBSTRING_INDEX(ex_addr, ":", 1)'); + } +} + +/** + * flowview_get_pending_partition_alters - The partitions that still need + * altering, looked at all at once so that a long list of partitions does not + * mean a long list of queries. + * + * @param array $tables The raw partition tables to look at + * @param string $raw_engine The engine a partition table that has rotated uses + * + * @return array The plan from flowview_get_partition_alter() of each + * partition that needs one, by table + */ +function flowview_get_pending_partition_alters(array $tables, string $raw_engine): array { + if (!cacti_sizeof($tables)) { + return []; + } + + $info = []; + + /* when only a few partitions are asked about, only they are read */ + $names = array_values(array_filter($tables, function ($table) { + return preg_match('/^plugin_flowview_raw_[0-9]+$/', $table) === 1; + })); + + if (cacti_sizeof($names) && cacti_sizeof($names) <= 100) { + $which = 'AND TABLE_NAME IN ("' . implode('", "', $names) . '")'; + } else { + $which = 'AND TABLE_NAME LIKE "plugin_flowview_raw_%"'; + } + + foreach (flowview_db_fetch_assoc('SELECT TABLE_NAME, TABLE_COLLATION, ENGINE + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() + ' . $which) as $row) { + $info[$row['TABLE_NAME']] = [ + 'collation' => $row['TABLE_COLLATION'], + 'engine' => $row['ENGINE'], + 'has_keycol' => false, + 'has_template_id' => false, + 'indexes' => [], + 'ex_addr_type' => null, + ]; + } + + foreach (flowview_db_fetch_assoc('SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + ' . $which . ' + AND COLUMN_NAME IN ("keycol", "template_id", "ex_addr")') as $row) { + if (!isset($info[$row['TABLE_NAME']])) { + continue; + } + + if ($row['COLUMN_NAME'] == 'keycol') { + $info[$row['TABLE_NAME']]['has_keycol'] = true; + } elseif ($row['COLUMN_NAME'] == 'template_id') { + $info[$row['TABLE_NAME']]['has_template_id'] = true; + } else { + $info[$row['TABLE_NAME']]['ex_addr_type'] = $row['COLUMN_TYPE']; + } + } + + foreach (flowview_db_fetch_assoc('SELECT DISTINCT TABLE_NAME, INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + ' . $which . ' + AND INDEX_NAME IN ("template_id", "start_time", "end_time", "ex_addr")') as $row) { + if (isset($info[$row['TABLE_NAME']])) { + $info[$row['TABLE_NAME']]['indexes'][] = $row['INDEX_NAME']; + } + } + + $pending = []; + + foreach ($tables as $table) { + if (!isset($info[$table])) { + continue; + } + + $plan = flowview_get_partition_alter($info[$table], $raw_engine); + + if ($plan['alter'] != '') { + $pending[$table] = $plan; + } + } + + return $pending; +} + +/** + * flowview_alter_partition - Alter a raw partition table to what it should + * be, holding a lock on the table while it does. The plan is worked out + * after the lock is taken, from the table as it is then, so a table that + * another materialization, the poller or the schema upgrade altered in the + * meantime is not altered again with a plan that is out of date. Every + * place that alters a partition goes through here. + * + * @param string $table The raw partition table + * @param string $raw_engine The engine a partition table that has rotated uses + * @param bool $print Also show the alter, as well as logging it + * @param int $wait How many seconds to wait for the lock + * @param callable|null $before Called with the plan just before it is applied + * + * @return string altered, current when there was nothing to do, or busy when + * the lock was not had in time + */ +function flowview_alter_partition(string $table, string $raw_engine, bool $print = true, int $wait = FLOWVIEW_ALTER_LOCK_WAIT, ?callable $before = null): string { + $lock = FLOWVIEW_ALTER_LOCK . $table; + + if ((int) flowview_db_fetch_cell_prepared('SELECT GET_LOCK(?, ?)', [$lock, $wait]) !== 1) { + return 'busy'; + } + + try { + $pending = flowview_get_pending_partition_alters([$table], $raw_engine); + + if (!isset($pending[$table])) { + return 'current'; + } + + if ($before !== null) { + $before($pending[$table]); + } + + flowview_apply_partition_alter($table, $pending[$table], $print); + + return 'altered'; + } finally { + flowview_db_fetch_cell_prepared('SELECT RELEASE_LOCK(?)', [$lock]); + } +} + +/** + * flowview_get_running_partition_alters - The raw partition tables that an + * ALTER TABLE is running on right now, other than from this connection. A + * query against such a table waits for the alter to finish. + * + * @param array $tables The raw partition tables to look at + * + * @return array How many seconds each alter has been running, by table + */ +function flowview_get_running_partition_alters(array $tables): array { + $running = []; + + $processes = flowview_db_fetch_assoc('SELECT TIME, INFO + FROM information_schema.PROCESSLIST + WHERE ID != CONNECTION_ID() + AND INFO LIKE "ALTER TABLE%plugin_flowview_raw_%"'); + + foreach ($processes as $process) { + if (!preg_match('/^\s*ALTER\s+TABLE\s+(?:`?[A-Za-z0-9_]+`?\.)?`?(plugin_flowview_raw_[0-9]+)`?/i', (string) $process['INFO'], $matches)) { + continue; + } + + if (in_array($matches[1], $tables, true)) { + $running[$matches[1]] = max($running[$matches[1]] ?? 0, (int) $process['TIME']); + } + } + + return $running; +} + +/** + * flowview_get_partitions_without_nat_columns - The raw partition tables that + * pre-date the post-NAT columns. Queries against them still work, but have + * no NAT data to show. + * + * @param array $tables The raw partition tables to look at + * + * @return array + */ +function flowview_get_partitions_without_nat_columns(array $tables): array { + $with = array_column(flowview_db_fetch_assoc('SELECT TABLE_NAME + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME LIKE "plugin_flowview_raw_%" + AND COLUMN_NAME = "post_nat_src_addr"'), 'TABLE_NAME'); + + return array_values(array_diff($tables, $with)); +} + +/** + * flowview_get_running_requests - The parallel query requests that have a + * runner or shard worker registered in the process table, as db_query_ or + * db_shard_. They keep going after the script that started them has + * ended, for instance when it timed out. + * + * A registration counts when it is recent, or when its process is still + * alive. A recent one is trusted as the web server may not be able to see + * the process of another user, and an old one is checked, as a process that + * was killed can leave its row behind. + * + * @return array The cache key (md5sum_tables) of each request, by request id, + * which is empty for a request that no longer exists + */ +function flowview_get_running_requests(): array { + $recent = max(60, (int) read_config_option('flowview_parallel_runlimit')); + + $processes = db_fetch_assoc_prepared('SELECT pid, taskname, UNIX_TIMESTAMP() - UNIX_TIMESTAMP(started) AS age + FROM processes + WHERE tasktype = ? + AND (taskname LIKE ? OR taskname LIKE ?)', + ['flowview', 'db\_query\_%', 'db\_shard\_%']); + + $ids = []; + + foreach ($processes as $process) { + if (!preg_match('/^db_(?:query|shard)_([0-9]+)$/', $process['taskname'], $matches)) { + continue; + } + + if ((int) $process['age'] <= $recent || !function_exists('cacti_process_still_running') || cacti_process_still_running((int) $process['pid'])) { + $ids[(int) $matches[1]] = true; + } + } + + if (!cacti_sizeof($ids)) { + return []; + } + + $requests = flowview_db_fetch_assoc_prepared('SELECT id, md5sum_tables + FROM parallel_database_query + WHERE id IN (' . implode(', ', array_fill(0, cacti_sizeof($ids), '?')) . ')', + array_keys($ids)); + + $running = array_fill_keys(array_keys($ids), ''); + + foreach ($requests as $request) { + $running[(int) $request['id']] = (string) $request['md5sum_tables']; + } + + return $running; +} + +/** + * flowview_materialization_running - Whether a materialization is running. + * That is when one is registered as running, since only one may be active at + * a time, or when a parallel query still has processes registered. + * + * @return bool + */ +function flowview_materialization_running(): bool { + if (cacti_sizeof(flowview_get_materializing_filters())) { + return true; + } + + return cacti_sizeof(flowview_get_running_requests()) > 0; +} + +/** + * flowview_get_materializing_filters - The filters that are being + * materialized. Each filter a materialization works on is registered in the + * process table as materialize, with the id of the filter as the task id, so + * the filters can be materialized separately and what is being done is + * visible from the process table. + * + * @return array The filter ids + */ +function flowview_get_materializing_filters(): array { + $rows = db_fetch_assoc_prepared('SELECT taskid + FROM processes + WHERE tasktype = ? + AND taskname = ? + AND taskid > 0 + ORDER BY taskid', + ['flowview', 'materialize']); + + return array_values(array_unique(array_map('intval', array_column($rows, 'taskid')))); +} + +/** + * flowview_purge_dead_materializations - Remove the registrations of a + * materialization whose process is gone, such as one that was killed and could + * not clean up. Without this the filter would show as running, and could not + * be materialized, until the registration timed out. Meant for a script, as the + * web server may not be able to see the process. + * + * @return int The registrations removed + */ +function flowview_purge_dead_materializations(): int { + $removed = 0; + + /* without a way to tell whether a process is alive nothing is removed */ + $rows = function_exists('cacti_process_still_running') ? db_fetch_assoc_prepared('SELECT id, pid + FROM processes + WHERE tasktype = ? + AND taskname = ?', + ['flowview', 'materialize']) : []; + + foreach ($rows as $row) { + if (!cacti_process_still_running((int) $row['pid'])) { + db_execute_prepared('DELETE FROM processes + WHERE id = ?', + [$row['id']]); + + $removed++; + } + } + + return $removed; +} + +/** + * flowview_materialization_status - The automatic materialization status of + * a filter. It is current once every cold partition is cached with NAT and + * without. The hot partition is not materialized automatically, so it does + * not count. Otherwise it is running while a materialization is active, and + * required when none is. + * + * @param array $cached_nat The partitions cached for the filter with NAT + * @param array $cached_non_nat The partitions cached for the filter without NAT + * @param array $cold The cold partitions + * @param bool $running Whether a materialization is running + * + * @return string current, running or required + */ +function flowview_materialization_status(array $cached_nat, array $cached_non_nat, array $cold, bool $running): string { + if (!cacti_sizeof(array_diff($cold, $cached_nat)) && !cacti_sizeof(array_diff($cold, $cached_non_nat))) { + return 'current'; + } + + return $running ? 'running' : 'required'; +} + +/** + * flowview_get_colored_materialization_status - The pill for the automatic + * materialization status of a filter, using the same colors as the device + * status in Cacti: down when it is required, recovering while it is running + * and up once it is up to date. + * + * @param string $status current, running or required + * + * @return string + */ +function flowview_get_colored_materialization_status(string $status, ?array $missing = null): string { + $tip = ''; + + /* say which partitions are not materialized, when it is not up to date */ + if ($missing !== null && $status != 'current') { + $tip = " title='" . __esc('Not materialized yet, with NAT: %s, without NAT: %s.', flowview_describe_partitions($missing['nat'] ?? []), flowview_describe_partitions($missing['non_nat'] ?? []), 'flowview') . "'"; + } + + switch ($status) { + case 'current': + return "" . __('Up to date', 'flowview') . ''; + case 'running': + return "" . __('Running', 'flowview') . ''; + default: + return "" . __('Required', 'flowview') . ''; + } +} + +/** + * flowview_get_missing_partitions - The cold partitions that a filter is not + * materialized for, with NAT and without. + * + * @param array $cached_nat The partitions cached for the filter with NAT + * @param array $cached_non_nat The partitions cached for the filter without NAT + * @param array $cold The cold partitions + * + * @return array The partitions as nat and non_nat + */ +function flowview_get_missing_partitions(array $cached_nat, array $cached_non_nat, array $cold): array { + return [ + 'nat' => array_values(array_diff($cold, $cached_nat)), + 'non_nat' => array_values(array_diff($cold, $cached_non_nat)), + ]; +} + +/** + * flowview_describe_partitions - How many partitions there are and the first + * few of them by name, for a tip. + * + * @param array $tables The partition tables + * @param int $limit How many to name + * + * @return string + */ +function flowview_describe_partitions(array $tables, int $limit = 3): string { + if (!cacti_sizeof($tables)) { + return '0'; + } + + return cacti_sizeof($tables) . ' (' . implode(', ', array_slice($tables, 0, $limit)) . (cacti_sizeof($tables) > $limit ? ', ...' : '') . ')'; +} + +/** + * flowview_get_colored_running_queries - The pill for the number of queries + * of a filter that are running now, in the gray of a disabled device. There + * is none when nothing is running. + * + * @param int $count The number of queries running + * @param array|null $by_mode How many of them are with NAT and without, as + * nat and non_nat, to say so in the tip + * + * @return string + */ +function flowview_get_colored_running_queries(int $count, ?array $by_mode = null): string { + if ($count < 1) { + return ''; + } + + $tip = __esc('The number of queries of this Filter that are running now.', 'flowview'); + + if ($by_mode !== null) { + $tip .= ' ' . __esc('With NAT: %d, without NAT: %d.', $by_mode['nat'] ?? 0, $by_mode['non_nat'] ?? 0, 'flowview'); + } + + return "" . __('Running: %s', number_format_i18n($count, 0), 'flowview') . ''; +} + +/** + * flowview_convert_partition_engine - Change a raw partition table that has + * rotated to the storage engine they use. Nothing is done while an ALTER + * TABLE is already running on it. Until one finishes the table still shows + * its old engine, so without that check every poller cycle would start + * another that waits for the first and then rebuilds the table again. + * + * @param string $table The raw partition table + * @param string $raw_engine The engine a partition table that has rotated uses + * + * @return bool false when an alter is already running on it + */ +function flowview_convert_partition_engine(string $table, string $raw_engine): bool { + if (cacti_sizeof(flowview_get_running_partition_alters([$table]))) { + return false; + } + + /* the poller does not wait for another process altering it, and checks it again under the lock */ + $lock = FLOWVIEW_ALTER_LOCK . $table; + + if ((int) flowview_db_fetch_cell_prepared('SELECT GET_LOCK(?, ?)', [$lock, 0]) !== 1) { + return false; + } + + try { + $engine = flowview_db_fetch_cell_prepared('SELECT ENGINE + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = ?', + [$table]); + + if ($engine != '' && $engine != $raw_engine) { + flowview_db_execute("ALTER TABLE $table ENGINE=$raw_engine"); + } + } finally { + flowview_db_fetch_cell_prepared('SELECT RELEASE_LOCK(?)', [$lock]); + } + + return true; +} + +/** + * flowview_convert_cold_partitions - Change the raw partition tables that + * have rotated, and are not yet using the fast engine, to it. Every + * partition that has rotated is looked at, not just the last few, so + * partitions missed while the poller was not running, or while the plugin was + * disabled, are converted as well, newest first. + * + * Only $limit are converted per call, as each ALTER TABLE is run in turn and + * takes a while. Nothing is converted while an ALTER TABLE is running on any + * of them, so the conversions of one poller cycle never pile up on those of + * the previous one. + * + * @param string $raw_engine The engine a partition table that has rotated uses + * @param int $limit The most partitions to convert + * + * @return array The partitions that were converted + */ +function flowview_convert_cold_partitions(string $raw_engine, int $limit = 3): array { + $cold = flowview_get_cold_partitions(flowview_get_partition_tables()); + + if (!cacti_sizeof($cold) || cacti_sizeof(flowview_get_running_partition_alters($cold))) { + return []; + } + + $engines = []; + + foreach (flowview_db_fetch_assoc('SELECT TABLE_NAME, ENGINE + FROM information_schema.TABLES + WHERE TABLE_NAME LIKE "plugin_flowview_raw_%"') as $row) { + $engines[$row['TABLE_NAME']] = $row['ENGINE']; + } + + $converted = []; + + foreach (array_reverse($cold) as $table) { + if (cacti_sizeof($converted) >= $limit) { + break; + } + + if (!isset($engines[$table]) || $engines[$table] == $raw_engine) { + continue; + } + + if (flowview_convert_partition_engine($table, $raw_engine)) { + $converted[] = $table; + } + } + + return $converted; +} + +/** + * flowview_materialize_rotated_partitions - Called every poller cycle to + * materialize the partitions that went cold since the previous cycle, for + * every filter with NAT and without. Partitions that are already + * materialized and still accurate are skipped by the materializer itself. + * + * The newest partition is recorded each cycle. The record is left alone + * while nothing can be started, so a later cycle picks up what was missed. + * The first cycle only records it, as the older partitions are materialized + * with cli/materialize_queries.php. + * + * @return array The partitions a materialization was started for + */ +function flowview_materialize_rotated_partitions(): array { + $tables = flowview_get_partition_tables(); + + if (!cacti_sizeof($tables)) { + return []; + } + + $newest = end($tables); + $previous = (string) read_config_option('flowview_last_hot_partition'); + + if ($previous == '') { + set_config_option('flowview_last_hot_partition', $newest); + + return []; + } + + $rotated = flowview_get_rotated_partitions($tables, $previous); + + if (!cacti_sizeof($rotated)) { + if ($previous != $newest) { + set_config_option('flowview_last_hot_partition', $newest); + } + + return []; + } + + $threads = read_config_option('flowview_parallel_threads'); + + /* a filter is left alone while it is being materialized, so this waits for a quiet moment to catch them all up */ + if (empty($threads) || $threads == 1 || flowview_materialization_running()) { + return []; + } + + if (!cacti_sizeof(flowview_launch_materialize([], $rotated))) { + return []; + } + + set_config_option('flowview_last_hot_partition', $newest); + + return $rotated; +} + +/** + * flowview_plan_filter - The cache keys (md5sum_tables) of the map queries a + * filter runs, with NAT or without. Nothing is run. + * + * @param int $query_id The FlowView query + * @param bool $use_nat Whether it is with NAT + * + * @return array The cache keys, empty when the filter can not be planned + */ +function flowview_plan_filter(int $query_id, bool $use_nat): array { + $plan = []; + $now = time(); + + if (run_flow_query(false, $query_id, $now, $now, true, $use_nat, $plan) !== true) { + return []; + } + + return $plan; +} + +/** + * flowview_get_filter_md5sums - The cache keys (md5sum_tables) the parallel + * query stores the partitions of a filter under, with NAT and without. Some + * reports use two map queries and so have two. Nothing is run, they are + * worked out from the filter. + * + * @param int $query_id The FlowView query to work the cache keys out for + * + * @return array The cache keys as nat and non_nat + */ +function flowview_get_filter_md5sums(int $query_id): array { + $md5sums = ['nat' => [], 'non_nat' => []]; + + foreach (['nat' => true, 'non_nat' => false] as $mode => $use_nat) { + $planned = flowview_plan_filter($query_id, $use_nat); + + if (cacti_sizeof($planned)) { + $md5sums[$mode] = array_values(array_unique($planned)); + } + } + + return $md5sums; +} + +/** + * flowview_filter_report_key - The report a filter is for, in the form the + * Report selector of the report page uses, s for a statistical report and + * p for a printed one. The report is part of the cache key of the filter. + * + * @param array $filter The filter, with its statistics and printed columns + * + * @return string The report, or an empty string when the filter has none + */ +function flowview_filter_report_key(array $filter): string { + if ((int) ($filter['statistics'] ?? 0) > 0) { + return 's' . (int) $filter['statistics']; + } + + if ((int) ($filter['printed'] ?? 0) > 0) { + return 'p' . (int) $filter['printed']; + } - $table .= ''; - $table .= ''; + return ''; +} - for ($i = 0; $i < 14; $i++) { - $table .= ''; +/** + * flowview_get_filter_shard_keys - Every cache key (md5sum_tables) that a + * filter, with NAT or without, stores its partitions under. + * + * @return array|null The keys as the array keys, or null when the keys of a + * filter that has a report could not be worked out, as + * what belongs to which filter is not known then + */ +function flowview_get_filter_shard_keys(): ?array { + $keys = []; + + foreach (flowview_db_fetch_assoc('SELECT id, statistics, printed + FROM plugin_flowview_queries') as $filter) { + $md5sums = flowview_get_filter_md5sums((int) $filter['id']); + + foreach (['nat', 'non_nat'] as $mode) { + if (!cacti_sizeof($md5sums[$mode])) { + /* a filter with no report has nothing cached */ + if (flowview_filter_report_key($filter) != '') { + return null; } - $table .= ''; - $table .= ''; + continue; + } - for ($i = 0; $i < 14; $i++) { - $name = $sql_array[$i]['name']; + foreach ($md5sums[$mode] as $key) { + $keys[$key] = true; + } + } + } - if ($total > 0) { - $table .= ''; - } else { - $table .= ''; - } - } + return $keys; +} - $table .= '
' . $sql_array[$i]['title'] . '
' . number_format_i18n(($results[$name] / $total) * 100, 2) . ' %' . __('N/A', 'flowview') . '
'; +/** + * flowview_get_shard_cache_summary - What is in the shard cache, told apart + * by whether it matches a filter. A cached partition matches when its + * md5sum is the cache key of a filter, with NAT or without. The rest comes + * from queries that no filter has, such as a filter before it was changed, a + * deleted filter, or a report with settings changed on the report page. The + * short lived results of the live partition are not counted. + * + * @return array|null matching and other, each with the number of cached + * shards, and the md5sums of the other; null when the + * keys of the filters are not known + */ +function flowview_get_shard_cache_summary(): ?array { + /* the size of the results is not looked at, as that reads every cached result */ + $cached = flowview_db_fetch_assoc('SELECT md5sum, COUNT(*) AS shards + FROM parallel_database_query_shard_cache + WHERE map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\' + GROUP BY md5sum'); - $table .= ''; - $table .= '
'; - $table .= ''; + $keys = flowview_get_filter_shard_keys(); - for ($i = 14; $i < 27; $i++) { - $table .= ''; - } + if ($keys === null) { + return null; + } - $table .= ''; - $table .= ''; + $summary = [ + 'matching' => ['shards' => 0], + 'other' => ['shards' => 0, 'md5sums' => []], + ]; - $table .= ''; + foreach ($cached as $row) { + if (isset($keys[$row['md5sum']])) { + $summary['matching']['shards'] += (int) $row['shards']; + } else { + $summary['other']['shards'] += (int) $row['shards']; + $summary['other']['md5sums'][] = $row['md5sum']; + } + } - for ($i = 14; $i < 27; $i++) { - $name = $sql_array[$i]['name']; - if ($total > 0) { - $table .= ''; - } else { - $table .= ''; - } - } + return $summary; +} - $table .= ''; - $table .= '
' . $sql_array[$i]['title'] . '
' . number_format_i18n(($results[$name] / $total) * 100, 2) . ' %' . __('N/A', 'flowview') . '
'; - } +/** + * flowview_prune_shard_cache - Remove the cached shards that match no filter. + * Whatever matches a filter is kept, even when the filter is not materialized + * for every partition. + * + * @return int|null The number of shards removed, or null when nothing was + * removed because the keys of the filters are not known + */ +function flowview_prune_shard_cache(): ?int { + $summary = flowview_get_shard_cache_summary(); - $output['table'] = $table; - $output['title'] = $title; + if ($summary === null) { + return null; + } - return $output; - } + foreach ($summary['other']['md5sums'] as $md5sum) { + flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache + WHERE md5sum = ? + AND map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\'', + [$md5sum]); } - return false; + return $summary['other']['shards']; } /** - * @param mixed $table - * @param mixed $type + * flowview_count_running_queries - How many of the running requests are + * queries of a filter, with NAT or without. * - * @return mixed + * @param array $md5sums The cache keys of the filter from flowview_get_filter_md5sums() + * @param array $running The running requests from flowview_get_running_requests() + * + * @return int */ -function flowview_table_name_to_time($table, $type) { - if (!preg_match('/^plugin_flowview_raw_([0-9]{7})([0-9]{2})?$/', $table, $matches)) { - return false; - } +function flowview_count_running_queries(array $md5sums, array $running): int { + $keys = array_merge($md5sums['nat'] ?? [], $md5sums['non_nat'] ?? []); - $suffix = $matches[1] . (isset($matches[2]) ? $matches[2] : ''); + return cacti_sizeof(array_filter($running, function ($md5sum) use ($keys) { + return in_array($md5sum, $keys, true); + })); +} - $year = substr($suffix, 0, 4); - $day = substr($suffix, 4, 3); +/** + * flowview_count_running_queries_by_mode - How many of the running requests + * are queries of a filter with NAT, and how many without. A request whose + * cache key is the same in both, as for a report that has no address to + * translate, is counted in both. + * + * @param array $md5sums The cache keys of the filter from flowview_get_filter_md5sums() + * @param array $running The running requests from flowview_get_running_requests() + * + * @return array The counts as nat and non_nat + */ +function flowview_count_running_queries_by_mode(array $md5sums, array $running): array { + return [ + 'nat' => flowview_count_running_queries(['nat' => $md5sums['nat'] ?? []], $running), + 'non_nat' => flowview_count_running_queries(['non_nat' => $md5sums['non_nat'] ?? []], $running), + ]; +} - if (strlen($suffix) == 7) { - $gran = 'days'; - $hour = 0; - } else { - $gran = 'hours'; - $hour = substr($suffix, 7, 2); +/** + * flowview_find_request_filters - The filters a running request belongs to, + * and whether it is with NAT or without. The process table only knows the id + * of the request, so it is the cache key of the request that finds them. + * + * @param array $filter_md5sums The cache keys from flowview_get_filter_md5sums(), by filter id + * @param string $md5sum The cache key of the request + * + * @return array The id of each filter, with whether it matches as nat and non_nat + */ +function flowview_find_request_filters(array $filter_md5sums, string $md5sum): array { + $found = []; + + if ($md5sum === '') { + return $found; } - $dates = flowview_convert_yeardayhour_to_date($gran, $year, $day, (int) $hour); + foreach ($filter_md5sums as $id => $md5sums) { + $nat = in_array($md5sum, $md5sums['nat'] ?? [], true); + $non_nat = in_array($md5sum, $md5sums['non_nat'] ?? [], true); - return $dates[$type]; + if ($nat || $non_nat) { + $found[] = ['id' => (int) $id, 'nat' => $nat, 'non_nat' => $non_nat]; + } + } + + return $found; } /** - * @param mixed $range - * @param mixed $year - * @param mixed $day - * @param int $hour + * flowview_get_cached_shard_tables - The partitions of a filter that are + * stored in the parallel query cache, with NAT and without. Some reports + * use two map queries, and a partition only counts once every one of them is + * cached. Nothing is run, the cache keys are worked out from the filter. * - * @return array + * @param int $query_id The FlowView query to find the cached shards of + * @param array|null $md5sums The cache keys of the filter when they are + * already known, otherwise they are worked out + * + * @return array The partition tables cached as nat and non_nat */ -function flowview_convert_yeardayhour_to_date($range, $year, $day, int $hour = 0): array { - $datetime = new DateTime(); - $datetime->setDate((int) $year, 1, 1)->setTime(0, 0, 0); - $datetime->modify('+' . (int) $day . ' days'); +function flowview_get_cached_shard_tables(int $query_id, ?array $md5sums = null): array { + $cached = ['nat' => [], 'non_nat' => []]; - if ($range == 'days') { - $start_date = $datetime->format('Y-m-d 00:00:00'); - $datetime->modify('+1 day'); - $end_date = $datetime->format('Y-m-d 00:00:00'); - } else { - $datetime->modify('+' . (int) $hour . ' hours'); - $start_date = $datetime->format('Y-m-d H:00:00'); - $datetime->modify('+1 hour'); - $end_date = $datetime->format('Y-m-d H:00:00'); + if ($md5sums === null) { + $md5sums = flowview_get_filter_md5sums($query_id); } - $start_time = strtotime($start_date); - $end_time = strtotime($end_date); + foreach (['nat', 'non_nat'] as $mode) { + $keys = $md5sums[$mode] ?? []; - return [ - 'start_date' => $start_date, - 'start_time' => $start_time, - 'end_date' => $end_date, - 'end_time' => $end_time - ]; + if (!cacti_sizeof($keys)) { + continue; + } + + $tables = flowview_db_fetch_assoc_prepared('SELECT map_table + FROM parallel_database_query_shard_cache + WHERE md5sum IN (' . implode(', ', array_fill(0, cacti_sizeof($keys), '?')) . ') + GROUP BY map_table, map_partition + HAVING COUNT(DISTINCT md5sum) = ?', + array_merge($keys, [cacti_sizeof($keys)])); + + $cached[$mode] = array_column($tables, 'map_table'); + } + + return $cached; } + /** * parallel_database_query_request - Given a series of tables and * and inner (map phase) and outer (reduce phase) SQL queries, @@ -3998,19 +5866,13 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { */ $map_range = $stru_inner['sql_range']; $map_range_params = $stru_inner['sql_range_params']; - $map_query = $stru_inner; $sql_start_time = $stru_inner['sql_start_time']; $sql_end_time = $stru_inner['sql_end_time']; $md5_array = [$stru_inner, $stru_outer]; $query_md5 = md5((string) json_encode($md5_array)); - unset($map_query['sql_range']); - unset($map_query['sql_range_params']); - unset($map_query['sql_start_time']); - unset($map_query['sql_end_time']); - $reduce_query = json_encode($stru_outer); - $map_query = (string) json_encode($map_query); + $map_query = parallel_database_query_map_query($stru_inner); $table_md5 = md5($map_query); @@ -4025,6 +5887,15 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { AND user_id = ?', [$query_md5, $user_id]); + /* a request left over from a query that timed out would be waited on for nothing */ + if (!empty($request_id) && flowview_discard_abandoned_request((int) $request_id)) { + $request_id = 0; + } + + /* a materialization has to run the shards, not be given the result of an earlier request */ + if (!empty($request_id) && !empty($GLOBALS['flowview_fresh_requests']) && flowview_discard_completed_request((int) $request_id)) { + $request_id = 0; + } $time_to_live = read_config_option('flowview_parallel_time_to_live'); if (empty($time_to_live)) { @@ -4064,6 +5935,7 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { $start = $sql_start_time; $end = $sql_end_time; $index = 0; + $live = flowview_get_live_partition(); foreach($tables as $table => $details) { if (!empty($details['min_date'])) { @@ -4081,11 +5953,7 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { $fsql_where = ''; $fsql_params = []; - if ($start < $start_time && $end > $end_time) { - $full_scan = true; - } else { - $full_scan = false; - } + $full_scan = flowview_is_full_scan($start, $end, $start_time, $end_time, $table == $live); if (!$full_scan) { $fsql_where = $stru_inner['sql_where'] . ($stru_inner['sql_where'] != '' ? ' AND ':'WHERE ') . $stru_inner['sql_range']; @@ -4118,7 +5986,7 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { $index, $full_scan ? '1':'0', $details['table_name'], - $details['table_partition'], + ($table == $live && !$full_scan) ? FLOWVIEW_LIVE_SHARD_MARKER : $details['table_partition'], $map_query, json_encode($fsql_params) ) @@ -4128,9 +5996,148 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { } } + flowview_track_request((int) $request_id); + return $request_id; } +/** + * flowview_discard_abandoned_request - Remove a parallel query request that + * was started and then abandoned. When a query reaches the Maximum Run Time + * its shards and processes are cleaned up, but the request itself stays as + * scheduled. As a request is reused for the same query and range, the next + * run would find it, see it is not pending, start nothing and wait for the + * Maximum Run Time again, for as long as the request lasts. + * + * A request is abandoned when it is not finished, nothing is running it, and + * it is old enough that a runner would have registered by now. + * + * @param int $request_id + * + * @return bool Whether it was abandoned and was removed + */ +function flowview_discard_abandoned_request(int $request_id): bool { + $request = flowview_db_fetch_row_prepared('SELECT status, created + FROM parallel_database_query + WHERE id = ?', + [$request_id]); + + if (!cacti_sizeof($request) || $request['status'] == 'complete') { + return false; + } + + $created = strtotime((string) $request['created']); + + if ($created === false || time() - $created < FLOWVIEW_ABANDONED_REQUEST_AGE) { + return false; + } + + if (array_key_exists($request_id, flowview_get_running_requests())) { + return false; + } + + parallel_database_query_cleanup($request_id, true); + + return true; +} + +/** + * flowview_discard_completed_request - Remove a parallel query request that + * has finished, so the same query is run again. A request is reused for the + * same query and range until it expires, and a finished one is never run + * again, it only hands back its result. That result does not mean the cached + * shards are there: they may have been pruned, or removed because the + * partition changed. A request that is still running is left alone, as + * another process is using it. + * + * @param int $request_id + * + * @return bool Whether it was finished and was removed + */ +function flowview_discard_completed_request(int $request_id): bool { + $status = flowview_db_fetch_cell_prepared('SELECT status + FROM parallel_database_query + WHERE id = ?', + [$request_id]); + + if ($status != 'complete') { + return false; + } + + parallel_database_query_cleanup($request_id, true); + + return true; +} + +/** + * flowview_track_request - Remember a parallel query request this process + * made and has not cleaned up, so a script that is told to stop can cancel + * it. It is forgotten by parallel_database_query_cleanup(). + * + * @param int $request_id + * + * @return void + */ +function flowview_track_request(int $request_id): void { + $GLOBALS['flowview_requests_in_flight'][$request_id] = $request_id; + $GLOBALS['flowview_requests_made'][$request_id] = $request_id; +} + +/** + * flowview_cancel_requests_in_flight - Cancel the parallel query requests + * this process made that are not finished. The runner and shard workers are + * told to stop, which unregisters them from the process table, and the request + * and its tables are removed. Meant for a signal handler. + * + * @return array The requests that were cancelled + */ +function flowview_cancel_requests_in_flight(): array { + $requests = array_values($GLOBALS['flowview_requests_in_flight'] ?? []); + + foreach ($requests as $request_id) { + parallel_database_query_cleanup($request_id, true); + } + + return $requests; +} + +/** + * flowview_parallel_timeout_message - The message for a parallel query that + * did not finish within the Maximum Run Time, worded for the Cacti log and + * for a script. + * + * @param int $seconds The Maximum Run Time in seconds + * + * @return string + */ +function flowview_parallel_timeout_message(int $seconds): string { + return __('ERROR: The Parallel Query did not finish within the Maximum Run Time of %d seconds. Check the Maximum Run Time under the FlowView Settings, and that the database is not busy with another operation such as an ALTER TABLE.', $seconds, 'flowview'); +} + +/** + * flowview_report_parallel_timeout - Log that a parallel query did not finish + * within the Maximum Run Time, and say so. The web page shows the message in + * the browser, a script has no browser to run it, so it is printed. + * + * @param int $seconds The Maximum Run Time + * @param bool $web Whether this is a web request + * + * @return void + */ +function flowview_report_parallel_timeout(int $seconds, bool $web): void { + $timeout_message = flowview_parallel_timeout_message($seconds); + + cacti_log($timeout_message, false, 'FLOWVIEW'); + + if ($web) { + raise_message_javascript('parallel_query_timeout', __('The Parallel Query Timed Out. Please contact your administrator', 'flowview'), MESSAGE_LEVEL_ERROR); + } else { + print $timeout_message . PHP_EOL; + + $GLOBALS['flowview_error_reported'] = true; + } +} + /** * parallel_database_query_is_running - Checks to see if an existing parallel * database query is currently running and return a true of false. This prevents @@ -4217,13 +6224,15 @@ function parallel_database_query_run($requests) { } if ($total_time >= $max_time) { - raise_message_javascript('parallel_query_timeout', __('The Parallel Query Timed Out. Please contact your administrator', 'flowview'), MESSAGE_LEVEL_ERROR); + $web = defined('CACTI_WEB') && CACTI_WEB; + + flowview_report_parallel_timeout((int) $max_time, $web); foreach($requests as $request_id) { parallel_database_query_cleanup($request_id); } - exit; + flowview_exit($web ? 0 : 1); } if (cacti_sizeof($requests) == 1) { @@ -4422,6 +6431,8 @@ function parallel_database_query_create_reduce_table($request_id, $sql_query, $t function parallel_database_query_expire(): void { $ttl = read_config_option('flowview_parallel_time_to_live'); + flowview_live_shard_expire(); + if (empty($ttl)) { set_config_option('flowview_parallel_time_to_live', '21600'); $ttl = 21600; @@ -4448,6 +6459,8 @@ function parallel_database_query_expire(): void { * @return void */ function parallel_database_query_cleanup($request_id, bool $remove = false): void { + unset($GLOBALS['flowview_requests_in_flight'][(int) $request_id]); + $table = flowview_db_fetch_cell_prepared('SELECT map_table FROM parallel_database_query WHERE id = ?', @@ -4466,16 +6479,15 @@ function parallel_database_query_cleanup($request_id, bool $remove = false): voi WHERE query_id = ?', [$request_id]); + /* the runner and its shard workers register as db_query_ and db_shard_ */ $processes = db_fetch_assoc_prepared('SELECT * FROM processes WHERE tasktype = ? - AND taskname LIKE ?', - ['flowview', "db_query{$request_id}"]); + AND taskname IN (?, ?)', + ['flowview', "db_query_{$request_id}", "db_shard_{$request_id}"]); if (cacti_sizeof($processes)) { foreach($processes as $p) { - if (posix_kill($p['pid'], 0)) { - posix_kill($p['pid'], SIGTERM); - } + flowview_signal_process((int) $p['pid'], 'SIGTERM'); db_execute_prepared('DELETE FROM processes WHERE pid = ?', @@ -4490,6 +6502,44 @@ function parallel_database_query_cleanup($request_id, bool $remove = false): voi } } +/** + * flowview_signal_number - The number of a signal. The SIG constants only + * exist when the pcntl extension is loaded, which it is not for the web + * server, so using one there is a fatal error. The numbers are those of + * Linux. + * + * @param string $signal SIGTERM, SIGKILL or SIGINT + * + * @return int + */ +function flowview_signal_number(string $signal): int { + if (defined($signal)) { + return (int) constant($signal); + } + + $numbers = ['SIGINT' => 2, 'SIGKILL' => 9, 'SIGTERM' => 15]; + + return $numbers[$signal] ?? 15; +} + +/** + * flowview_signal_process - Send a signal to a process, from a script or from + * the web server. A process of another user can not be signalled from the web + * server, which is not an error here. + * + * @param int $pid The process + * @param string $signal SIGTERM, SIGKILL or SIGINT + * + * @return bool Whether the signal was sent + */ +function flowview_signal_process(int $pid, string $signal = 'SIGTERM'): bool { + if ($pid <= 0 || !function_exists('posix_kill')) { + return false; + } + + return posix_kill($pid, flowview_signal_number($signal)); +} + /** * @param mixed $query_id * @@ -4507,7 +6557,7 @@ function parallel_database_query_cancel($query_id): void { foreach($processes as $p) { cacti_log("WARNING: Killing FlowView Query Process with Task Name:{$p['taskname']} and PID:{$p['pid']}", false, 'FLOWVIEW'); - posix_kill($p['pid'], SIGKILL); + flowview_signal_process((int) $p['pid'], 'SIGKILL'); db_execute_prepared('DELETE FROM processes WHERE id = ?', [$p['id']]); } @@ -4744,9 +6794,26 @@ function parallel_database_child_runner($query_id, $shard_id): void { $exists = []; } - if (!cacti_sizeof($exists)) { + $live_key = ''; + $live_locked = false; + $live_reused = false; + + if (!cacti_sizeof($exists) && $shard['map_partition'] == FLOWVIEW_LIVE_SHARD_MARKER) { + $live_key = flowview_live_shard_key($query, $shard); + $live_locked = flowview_live_shard_lock($live_key); + $results = flowview_live_shard_get($live_key, $shard['map_table']); + $live_reused = $results !== null; + } + + if (!cacti_sizeof($exists) && !$live_reused) { $results = flowview_db_fetch_assoc_prepared("{$shard['map_query']}", json_decode($shard['map_params']), false, $max_cnn); - } else { + + if ($live_key != '') { + if (is_array($results)) { + flowview_live_shard_save($live_key, $shard['map_table'], $results); + } + } + } elseif (cacti_sizeof($exists)) { flowview_db_execute_prepared('UPDATE parallel_database_query SET cached_shards = cached_shards + 1 WHERE id = ?', @@ -4755,6 +6822,10 @@ function parallel_database_child_runner($query_id, $shard_id): void { $results = json_decode($exists['results'], true); } + if ($live_locked) { + flowview_live_shard_unlock($live_key); + } + if ($shard['full_scan'] == 1 && !cacti_sizeof($exists)) { $details = flowview_db_fetch_row("SELECT MIN(start_time) AS min_date, MAX(end_time) AS max_date FROM {$shard['map_table']}"); @@ -4844,13 +6915,17 @@ function db_debug($string): void { } /** + * display_domain - A DNS name for the report table. It comes from reverse + * lookups of the flow data, which anyone who controls a reverse zone can set, + * so it is escaped. + * * @param mixed $domain * - * @return mixed + * @return string */ function display_domain($domain) { if ($domain != '') { - return $domain; + return html_escape($domain); } else { return __('unresolved', 'flowview'); } diff --git a/locales/po/cacti.pot b/locales/po/cacti.pot index 23efd5d..fe3d2f1 100644 --- a/locales/po/cacti.pot +++ b/locales/po/cacti.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Cacti \n" "Report-Msgid-Bugs-To: developers@cacti.net\n" -"POT-Creation-Date: 2026-10-12 14:40-0400\n" +"POT-Creation-Date: 2026-10-22 09:18-0400\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: Cacti Developers >\n" "Language-Team: Cacti Developers \n" @@ -920,6 +920,33 @@ msgstr "" msgid "No Flowview Listeners" msgstr "" +#: flowview_filters.php +msgid "Materialize" +msgstr "" + +#: flowview_filters.php +msgid "The cache keys of the Filters could not be worked out, so nothing was pruned." +msgstr "" + +#: flowview_filters.php +#, php-format +msgid "Pruned %s cached shard(s) that did not match any Filter." +msgstr "" + +#: flowview_filters.php +#, php-format +msgid "Materialization of %d Filter(s) was started in the background. %d Filter(s) are already being materialized. Its results are reported in the Cacti log." +msgstr "" + +#: flowview_filters.php +#, php-format +msgid "Materialization of %d Filter(s) was started in the background. Its results are reported in the Cacti log." +msgstr "" + +#: flowview_filters.php +msgid "The selected Filter(s) are already being materialized. Please try again once that has finished." +msgstr "" + #: flowview_filters.php msgid "Click 'Continue' to delete the following Filter(s) and all matching Filter." msgstr "" @@ -932,6 +959,10 @@ msgstr "" msgid "Click 'Continue' to Enable the following Filters(s)." msgstr "" +#: flowview_filters.php +msgid "Click 'Continue' to Materialize the following Filter(s) in the background, with NAT on and off, for the entire range of flow data. Partitions that are already materialized and accurate are skipped. This can take a long time on large data sets. A Filter that is already being materialized is skipped." +msgstr "" + #: flowview_filters.php msgid "You must select at least one Filter." msgstr "" @@ -948,10 +979,30 @@ msgstr "" msgid "Filters" msgstr "" +#: flowview_filters.php +msgid "Prune" +msgstr "" + +#: flowview_filters.php +msgid "Remove the cached shards that do not match any Filter" +msgstr "" + #: flowview_filters.php flowview_schedules.php msgid "Filter Name" msgstr "" +#: flowview_filters.php +msgid "ID" +msgstr "" + +#: flowview_filters.php +msgid "Materialization" +msgstr "" + +#: flowview_filters.php +msgid "Materialization pre-computes the results of a Filter for each flow partition, with NAT on and off, and stores them in the Parallel Query cache, so reports over older data open quickly. New partitions are materialized automatically once they stop receiving flows. Required means partitions that have rotated are not materialized yet, Running means a materialization is in progress, and Up to date means every rotated partition is materialized. Running: N is how many queries of the Filter are working now. The partition still receiving flows is not materialized, so it does not count." +msgstr "" + #: flowview_filters.php includes/arrays.php includes/functions.php msgid "Listener" msgstr "" @@ -961,7 +1012,27 @@ msgid "Report Type" msgstr "" #: flowview_filters.php -msgid "ID" +msgid "NAT Shards" +msgstr "" + +#: flowview_filters.php +msgid "The number of partitions of this Filter that are stored in the Parallel Query cache with NAT on." +msgstr "" + +#: flowview_filters.php +msgid "Non-NAT Shards" +msgstr "" + +#: flowview_filters.php +msgid "The number of partitions of this Filter that are stored in the Parallel Query cache with NAT off." +msgstr "" + +#: flowview_filters.php +msgid "Partitions" +msgstr "" + +#: flowview_filters.php +msgid "The total number of flow partitions, which is the most shards a Filter can have cached with or without NAT." msgstr "" #: flowview_filters.php includes/arrays.php includes/functions.php @@ -972,6 +1043,36 @@ msgstr "" msgid "Resolution" msgstr "" +#: flowview_filters.php +msgid "The cached shards of the Filters could not be told apart from the others, as the cache keys of a Filter could not be worked out." +msgstr "" + +#: flowview_filters.php +msgid "The cache keys of the Filters could not be worked out, so nothing can be pruned right now. Check that each Filter has a valid Report." +msgstr "" + +#: flowview_filters.php +#, php-format +msgid "Cached Shards: %s matching a Filter. %s not matching any Filter, which Prune removes." +msgstr "" + +#: flowview_filters.php +#, php-format +msgid "This removes %s cached shard(s) that do not match any Filter, for every Filter and every user. They come from Filters that were changed or deleted, and from reports with settings changed on the report page. The %s cached shard(s) of the Filters are kept. This can not be undone." +msgstr "" + +#: flowview_filters.php +msgid "Every cached shard matches a Filter. There is nothing to prune." +msgstr "" + +#: flowview_filters.php +msgid "Prune Cached Shards" +msgstr "" + +#: flowview_filters.php +msgid "Close" +msgstr "" + #: flowview_schedules.php msgid "Send Now" msgstr "" @@ -1663,6 +1764,10 @@ msgstr "" msgid "Select the Specific Protocol for the Filter." msgstr "" +#: includes/arrays.php +msgid "All Protocols" +msgstr "" + #: includes/arrays.php msgid "TCP Flags" msgstr "" @@ -1859,6 +1964,30 @@ msgstr "" msgid "Filter: [new]" msgstr "" +#: includes/functions.php +msgid "Cached Results Affected" +msgstr "" + +#: includes/functions.php +msgid "These changes affect the cached results of this Filter. Once it is saved they no longer match it, and the Filter has to be materialized again. Press 'Continue' to save the Filter, or 'Cancel' to keep editing it." +msgstr "" + +#: includes/functions.php +msgid "The changes to this Filter affect its cached results, so it has to be materialized again. To start a materialization, select the Filter on the Filters page and choose the 'Materialize' action." +msgstr "" + +#: includes/functions.php +msgid "Save Succeeded" +msgstr "" + +#: includes/functions.php +msgid "Save Succeeded." +msgstr "" + +#: includes/functions.php +msgid "OK" +msgstr "" + #: includes/functions.php msgid "Undefined Filter [ Select Filter to get Details ]" msgstr "" @@ -2125,6 +2254,10 @@ msgstr "" msgid " " msgstr "" +#: includes/functions.php +msgid "Filter Settings Saved. A forced re-materialization of this Filter is recommended, use the Materialize action on the Filters page." +msgstr "" + #: includes/functions.php msgid "Filter Settings Saved" msgstr "" @@ -2182,10 +2315,6 @@ msgstr "" msgid "The Printed Report %s is not known. Please open a bug on GitHub" msgstr "" -#: includes/functions.php -msgid "No valid report was selected. Please open a bug on GitHub" -msgstr "" - #: includes/functions.php msgid "Source DNS" msgstr "" @@ -2230,6 +2359,42 @@ msgstr "" msgid "Bytes/Packet" msgstr "" +#: includes/functions.php +#, php-format +msgid "Not materialized yet, with NAT: %s, without NAT: %s." +msgstr "" + +#: includes/functions.php +msgid "Up to date" +msgstr "" + +#: includes/functions.php +msgid "Running" +msgstr "" + +#: includes/functions.php +msgid "Required" +msgstr "" + +#: includes/functions.php +msgid "The number of queries of this Filter that are running now." +msgstr "" + +#: includes/functions.php +#, php-format +msgid "With NAT: %d, without NAT: %d." +msgstr "" + +#: includes/functions.php +#, php-format +msgid "Running: %s" +msgstr "" + +#: includes/functions.php +#, php-format +msgid "ERROR: The Parallel Query did not finish within the Maximum Run Time of %d seconds. Check the Maximum Run Time under the FlowView Settings, and that the database is not busy with another operation such as an ALTER TABLE." +msgstr "" + #: includes/functions.php msgid "The Parallel Query Timed Out. Please contact your administrator" msgstr "" diff --git a/manifest.json b/manifest.json index 00e7668..0db1888 100644 --- a/manifest.json +++ b/manifest.json @@ -2,9 +2,12 @@ "tombstones": [ "arrays.php", "database.php", + "fixprivate.php", "flowview_security.php", "functions-pre13.php", "functions.php", + "import_flows.php", + "repair_tables.php", "themes/" ], "expected": [ @@ -13,9 +16,9 @@ "LICENSE", "Net/", "README.md", + "cli/", "config.php.dist", "css/", - "fixprivate.php", "flow_collector.php", "flowview.php", "flowview_bulkarin.php", @@ -30,7 +33,6 @@ "flowview_upgrade.php", "flowview_upgrade_nat_columns.php", "images/", - "import_flows.php", "includes/", "index.php", "irr_tables.php", @@ -38,7 +40,6 @@ "manifest.json", "maxscale/", "plugin_flowview_ports.sql", - "repair_tables.php", "run_schedule.php", "service/", "setup.php" diff --git a/phpunit.xml b/phpunit.xml index 2ec8407..ca15b0c 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -39,6 +39,7 @@ Net/DNS2/Updater.php includes/functions.php includes/database.php + includes/arrays.php diff --git a/service/flowview-query/README.md b/service/flowview-query/README.md index f56ad76..2eed6fd 100644 --- a/service/flowview-query/README.md +++ b/service/flowview-query/README.md @@ -87,8 +87,10 @@ spawning processes and polling tables can be seconds. pool; when `flowview_use_maxscale` is on, map (read) queries are routed through the MaxScale read/write-split port so shards round-robin across backends. - **Maintenance off the critical path.** A scheduler goroutine expires queries - past their `time_to_live`, ages out cached partitions, and drops cache rows for - partitions that have been rolled off — the work PHP does inline today. + past their `time_to_live`, drops cache rows for partitions that have been rolled + off, and expires the short lived live-partition results; cached results of a + cold partition are kept until the partition is dropped, the work PHP does + inline today. ### Map-reduce reduction (unchanged contract) @@ -261,9 +263,11 @@ A goroutine runs every `scheduler_interval` (and once at startup): - expires `parallel_database_query` rows past `time_to_live` and drops their intermediary map tables; -- ages out `parallel_database_query_shard_cache` rows older than - `flowview_parallel_time_to_live`; -- drops cache rows whose partition table no longer exists (rolled-off data). +- drops cache rows whose partition table no longer exists (rolled-off data); +- expires the short lived (60 second) live-partition results that the PHP + workers store with `map_partition = 'live'`. Cached results of a cold + partition are never aged out, as the materialization of a filter is built + from them. Deleting a saved query or filter should call `POST /cache/invalidate` with that signature so its cache does not linger until TTL. diff --git a/service/flowview-query/scheduler.go b/service/flowview-query/scheduler.go index e1a68ae..7c54980 100644 --- a/service/flowview-query/scheduler.go +++ b/service/flowview-query/scheduler.go @@ -103,22 +103,28 @@ func (s *Scheduler) expireQueries(ctx context.Context) error { return nil } -// expireShardCache ages out cached partition results older than the configured -// time-to-live so hot partitions stay cached while cold data is reclaimed. -func (s *Scheduler) expireShardCache(ctx context.Context) error { - ttl := int64(s.eng.settings.TimeToLive().Seconds()) - if ttl <= 0 { - return nil - } +// Live partition shard results (map_partition = 'live') are written by the PHP +// workers so the panels of one report share a single scan of the partition that +// is still receiving flows. They are only good for a short time. +const ( + liveShardMarker = "live" + liveShardTTLSeconds = 60 +) +// expireShardCache removes the short lived results of the live partition. +// Cached results of a cold partition are never aged out: they stay until the +// partition is dropped (see dropOrphanCache) or no longer matches its data, +// as the materialization of a filter is built from them. +func (s *Scheduler) expireShardCache(ctx context.Context) error { res, err := s.eng.flow.ExecContext(ctx, `DELETE FROM parallel_database_query_shard_cache - WHERE date_created < (NOW() - INTERVAL ? SECOND)`, ttl) + WHERE map_partition = ? + AND date_created < (NOW() - INTERVAL ? SECOND)`, liveShardMarker, liveShardTTLSeconds) if err != nil { return err } if n, _ := res.RowsAffected(); n > 0 { - logf("scheduler: expired %d cached shards", n) + logf("scheduler: expired %d live shard results", n) } return nil } diff --git a/service/flowview-query/scheduler_test.go b/service/flowview-query/scheduler_test.go index cef34bc..d7dc127 100644 --- a/service/flowview-query/scheduler_test.go +++ b/service/flowview-query/scheduler_test.go @@ -49,30 +49,38 @@ func TestExpireQueriesKeepsRowOnDropFailure(t *testing.T) { } } -func TestExpireShardCache(t *testing.T) { +func TestExpireShardCacheOnlyExpiresLiveResults(t *testing.T) { flow, mock, cleanup := newMock(t) defer cleanup() eng := NewEngine(flow, nil, nil, defaultSettings()) s := &Scheduler{eng: eng} - mock.ExpectExec("DELETE FROM parallel_database_query_shard_cache"). - WithArgs(int64(eng.settings.TimeToLive().Seconds())). + // Only the short lived live partition results go, whatever the time to + // live is: a cold partition stays cached until it is dropped. + mock.ExpectExec("DELETE FROM parallel_database_query_shard_cache\\s+WHERE map_partition = \\?"). + WithArgs("live", liveShardTTLSeconds). WillReturnResult(sqlmock.NewResult(0, 3)) if err := s.expireShardCache(context.Background()); err != nil { t.Fatalf("expireShardCache: %v", err) } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } } -func TestExpireShardCacheSkippedWhenTTLZero(t *testing.T) { - flow, _, cleanup := newMock(t) +func TestExpireShardCacheIgnoresTimeToLive(t *testing.T) { + flow, mock, cleanup := newMock(t) defer cleanup() settings := defaultSettings() settings.timeToLive = 0 eng := NewEngine(flow, nil, nil, settings) s := &Scheduler{eng: eng} - // No expectations: a zero TTL must skip the delete entirely. + mock.ExpectExec("DELETE FROM parallel_database_query_shard_cache"). + WithArgs("live", liveShardTTLSeconds). + WillReturnResult(sqlmock.NewResult(0, 0)) + if err := s.expireShardCache(context.Background()); err != nil { t.Fatalf("expireShardCache: %v", err) } diff --git a/setup.php b/setup.php index 8726c0f..3f6a28f 100644 --- a/setup.php +++ b/setup.php @@ -39,6 +39,21 @@ function plugin_flowview_csp_nonce(): string { return ''; } +if (!function_exists('flowview_exit')) { + /** + * Ends the request, with the exit status given. The places in + * includes/functions.php that end a request do it through here, so the + * unit tests can stand in for it and see the request end instead of ending. + * + * @param int $status + * + * @return void + */ + function flowview_exit(int $status = 0): void { + exit($status); + } +} + /** * @return bool */ diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index f8860e7..090237e 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -27,6 +27,8 @@ it('formats display helpers deterministically', function () { expect(display_domain(''))->toBe('unresolved') ->and(display_domain('example.net'))->toBe('example.net') + ->and(display_domain('.example.net'))->toBe('<img src=x onerror=alert(1)>.example.net') + ->and(display_domain("it's\".example.net"))->not->toContain("'") ->and(get_column_alignment('Bytes'))->toBe('right') ->and(get_column_alignment('Source'))->toBe('left') ->and(flowview_altrow(0))->toBe('even') @@ -146,3 +148,2177 @@ expect($sql)->toBe('(`start_time` BETWEEN ? AND ?)') ->and($params)->toBe(array('2024-01-01 00:00:00', '2024-01-02 00:00:00')); }); + +it('shifts the date range backward and forward by the selected interval', function ($direction, $shift, $date1, $date2, $expected1, $expected2) { + if (!defined('GTS_1_DAY')) { + define('GTS_1_DAY', 7); + } + + $GLOBALS['graph_timeshifts_vals'] = array(7 => '1 Day', 11 => '1 Week', 13 => '1 Month'); + + $_REQUEST['direction'] = $direction; + $_REQUEST['predefined_timeshift'] = $shift; + $_REQUEST['date1'] = $date1; + $_REQUEST['date2'] = $date2; + + ob_start(); + flowview_timeshift(); + $span = json_decode(ob_get_clean(), true); + + expect($span['current_value_date1'])->toBe($expected1) + ->and($span['current_value_date2'])->toBe($expected2); +})->with(array( + 'left one week' => array('left', 11, '2026-10-13 12:02', '2026-10-20 12:02', '2026-10-06 12:02', '2026-10-13 12:02'), + 'right one week' => array('right', 11, '2026-10-13 12:02', '2026-10-20 12:02', '2026-10-20 12:02', '2026-10-27 12:02'), + 'right one month' => array('right', 13, '2026-09-01 00:00', '2026-09-30 23:59', '2026-10-01 00:00', '2026-10-30 23:59'), + 'unknown interval is 1day' => array('left', 99, '2026-10-13 12:02', '2026-10-20 12:02', '2026-10-12 12:02', '2026-10-19 12:02'), +)); + +it('returns an empty range when the time shift dates cannot be parsed', function () { + $_REQUEST['direction'] = 'left'; + $_REQUEST['predefined_timeshift'] = 7; + $_REQUEST['date1'] = 'not a date'; + $_REQUEST['date2'] = ''; + + ob_start(); + flowview_timeshift(); + + expect(json_decode(ob_get_clean(), true))->toBe(array()); +}); + +it('spans every partition when materializing and pads the bounds so shards are full scans', function () { + $range = flowview_get_materialize_range(array( + 'plugin_flowview_raw_2024003', + 'not_a_partition_table', + 'plugin_flowview_raw_2024001', + 'plugin_flowview_raw_202400105', + )); + + // A flow can start before the window of its partition, so the start has a day of room. + expect($range)->toBe(array( + strtotime('2024-01-02 00:00:00 UTC') - 86400, + strtotime('2024-01-05 00:00:00 UTC') + 1, + )); +}); + +it('has no materialization range without a valid partition table', function ($tables) { + expect(flowview_get_materialize_range($tables))->toBeFalse(); +})->with(array( + 'no tables' => array(array()), + 'not partitions' => array(array('plugin_flowview_queries', 'plugin_flowview_raw_abc')), +)); + +it('treats a cached partition shard as current only while the partition bounds match', function ($cached, $current, $expected) { + expect(flowview_shard_cache_is_current($cached, $current))->toBe($expected); +})->with(array( + 'identical bounds' => array( + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:59:59'), + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:59:59'), + true, + ), + 'microsecond precision' => array( + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:59:59'), + array('min_date' => '2026-10-20 12:00:00.400000', 'max_date' => '2026-10-20 12:59:59.900000'), + true, + ), + 'partition has more data' => array( + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:30:00'), + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:45:00'), + false, + ), + 'partition starts earlier' => array( + array('min_date' => '2026-10-20 12:05:00', 'max_date' => '2026-10-20 12:30:00'), + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:30:00'), + false, + ), + 'empty partition' => array( + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:30:00'), + array('min_date' => null, 'max_date' => null), + false, + ), + 'unset cached bounds' => array( + array('min_date' => '0000-00-00', 'max_date' => '0000-00-00'), + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:30:00'), + false, + ), +)); + +it('keeps accurate cached partitions and removes ones that changed or were dropped', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024001', 'map_partition' => '', 'min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00'), + array('md5sum' => 'bbb', 'map_table' => 'plugin_flowview_raw_2024001', 'map_partition' => '', 'min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00'), + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024002', 'map_partition' => '', 'min_date' => '2024-01-03 00:00:00', 'max_date' => '2024-01-03 12:00:00'), + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024009', 'map_partition' => '', 'min_date' => '2024-01-10 00:00:00', 'max_date' => '2024-01-10 23:00:00'), + )); + + // Each distinct partition table is checked once: it exists, then its current bounds. + plugin_test_queue_db_result('db_fetch_cell', 1); + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-02 00:00:00.120000', 'max_date' => '2024-01-02 23:00:00.480000')); + plugin_test_queue_db_result('db_fetch_cell', 1); + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-03 00:00:00', 'max_date' => '2024-01-03 20:00:00')); + plugin_test_queue_db_result('db_fetch_cell', 0); + + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 9))); + + expect(flowview_invalidate_stale_shard_cache())->toBe(array('kept' => 2, 'removed' => 2)); + + $deleted = array(); + foreach ($GLOBALS['__test_db_calls'] as $call) { + if ($call['fn'] == 'db_execute_prepared' && strpos($call['sql'], 'DELETE FROM parallel_database_query_shard_cache') === 0) { + $deleted[] = $call['params']; + } + } + + expect($deleted)->toBe(array( + array('aaa', 'plugin_flowview_raw_2024002', ''), + array('aaa', 'plugin_flowview_raw_2024009', ''), + )); + + $requests = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && preg_match('/^DELETE FROM parallel_database_query\s+WHERE id = \?/', $call['sql']) === 1; + })); + + expect($requests)->toHaveCount(1) + ->and($requests[0]['params'])->toBe(array(9)); +}); + +it('leaves finished requests alone when every cached partition is still accurate', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024001', 'map_partition' => '', 'min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00'), + )); + plugin_test_queue_db_result('db_fetch_cell', 1); + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00')); + + expect(flowview_invalidate_stale_shard_cache())->toBe(array('kept' => 1, 'removed' => 0)); + + $writes = array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }); + + expect($writes)->toBeEmpty(); +}); + +it('flags only changes that alter what a filter materializes', function ($old, $new, $expected) { + expect(flowview_filter_change_is_material($old, $new))->toBe($expected); +})->with(array( + 'new filter' => array( + array(), + array('sourceip' => '10.0.0.1', 'statistics' => 10), + false, + ), + 'nothing changed' => array( + array('device_id' => '3', 'sourceip' => '10.0.0.1', 'statistics' => '10', 'printed' => '0'), + array('device_id' => '3', 'sourceip' => '10.0.0.1', 'statistics' => '10', 'printed' => '0'), + false, + ), + 'display and time settings' => array( + array('name' => 'Top Talkers', 'timespan' => '3', 'graph_type' => 'bar', 'cutofflines' => '20', 'usenat' => '', 'statistics' => '10'), + array('name' => 'Renamed', 'timespan' => '5', 'graph_type' => 'pie', 'cutofflines' => '50', 'usenat' => 'on', 'statistics' => '10'), + false, + ), + 'source address changed' => array( + array('sourceip' => '10.0.0.1'), + array('sourceip' => '10.0.0.2'), + true, + ), + 'protocol added' => array( + array('protocols' => ''), + array('protocols' => '6, 17'), + true, + ), + 'report type changed' => array( + array('statistics' => '10', 'printed' => '0'), + array('statistics' => '11', 'printed' => '0'), + true, + ), + 'device restricted' => array( + array('device_id' => '0'), + array('device_id' => '3'), + true, + ), + 'unrestricted saved differently' => array( + array('device_id' => '0', 'ex_addr' => '-1', 'protocols' => '0', 'template_id' => ''), + array('device_id' => '', 'ex_addr' => '', 'protocols' => '', 'template_id' => '0'), + false, + ), + 'sort field on printed report' => array( + array('printed' => '2', 'statistics' => '0', 'sortfield' => 'bytes'), + array('printed' => '2', 'statistics' => '0', 'sortfield' => 'packets'), + true, + ), + 'sort field on statistics report' => array( + array('printed' => '0', 'statistics' => '10', 'sortfield' => 'bytes'), + array('printed' => '0', 'statistics' => '10', 'sortfield' => 'packets'), + false, + ), + 'partial save only compares what it carries' => array( + array('device_id' => '3', 'sourceip' => '10.0.0.1', 'statistics' => '10'), + array('device_id' => '3'), + false, + ), +)); + +it('tells the user to start a materialization with the Materialize action', function () { + expect(flowview_rematerialize_recommendation())->toContain('materialized again') + ->and(flowview_rematerialize_recommendation())->toContain("'Materialize' action"); +}); + +describe('warning before a Filter is saved with a change to what is cached', function () { + beforeEach(function () { + plugin_test_reset(); + + /* the dialog uses the CSP nonce of setup.php */ + require_once __DIR__ . '/../../setup.php'; + }); + + it('does not warn for a new Filter, as it has nothing cached', function () { + expect(flowview_filter_form_is_material(array('id' => '', 'sourceip' => '10.0.0.1')))->toBeFalse() + ->and($GLOBALS['__test_db_calls'])->toBe(array()); + }); + + it('warns when the change alters what is cached', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array('id' => 4, 'sourceip' => '10.0.0.1', 'statistics' => '10', 'printed' => '0')); + + expect(flowview_filter_form_is_material(array('id' => '4', 'sourceip' => '10.0.0.2', 'statistics' => '10', 'printed' => '0')))->toBeTrue(); + }); + + it('does not warn for a change of how it is shown', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array('id' => 4, 'name' => 'Old', 'sourceip' => '10.0.0.1', 'statistics' => '10', 'printed' => '0', 'graph_type' => 'bar')); + + expect(flowview_filter_form_is_material(array('id' => '4', 'name' => 'New', 'sourceip' => '10.0.0.1', 'statistics' => '10', 'printed' => '0', 'graph_type' => 'pie')))->toBeFalse(); + }); + + it('answers the form with JSON, without saving anything', function () { + $_REQUEST = array('id' => '4', 'name' => 'Top', 'sourceip' => '10.0.0.2', 'statistics' => '10', 'printed' => '0'); + + plugin_test_queue_db_result('db_fetch_row_prepared', array('id' => 4, 'sourceip' => '10.0.0.1', 'statistics' => '10', 'printed' => '0')); + + ob_start(); + flowview_check_filter_save(); + $output = ob_get_clean(); + + expect(json_decode($output, true))->toBe(array('material' => true)) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + }); + + it('says the save succeeded and to materialize again, once, in a dialog', function () { + $_SESSION['flowview_rematerialize_saved'] = 4; + + ob_start(); + flowview_show_saved_dialog(); + $first = ob_get_clean(); + + ob_start(); + flowview_show_saved_dialog(); + $second = ob_get_clean(); + + expect($first)->toContain("id='flowview_saved_dialog'") + ->and($first)->toContain('Save Succeeded') + ->and($first)->toContain('Materialize') + ->and($first)->toContain("\$('#flowview_saved_dialog').dialog(") + ->and($first)->toContain('modal: false') + ->and($second)->toBe('') + ->and(isset($_SESSION['flowview_rematerialize_saved']))->toBeFalse(); + }); + + it('shows nothing when nothing was saved with a change to what is cached', function () { + ob_start(); + flowview_show_saved_dialog(); + + expect(ob_get_clean())->toBe(''); + }); + + it('warns from the Save button with Cancel and Continue, and no longer raises a warning message', function () { + $source = plugin_test_read_source('includes/functions.php'); + + expect($source)->toContain("id='flowview_save_warning'") + ->and($source)->toContain("\$('#chk #submit').off('click.flowview').on('click.flowview'") + ->and($source)->toContain("{ name: 'action', value: 'check_material' }") + ->and($source)->toContain("\$('#flowview_save_warning').dialog(") + ->and($source)->toContain('sessionMessageCancel') + ->and($source)->toContain('sessionMessageContinue') + ->and($source)->not->toContain("raise_message('flowview_rematerialize'") + ->and(plugin_test_read_source('flowview_filters.php'))->toContain("case 'check_material':"); + }); +}); + +it('starts the materialization in the background for the given queries', function () { + if (!function_exists('exec_background')) { + function exec_background($filename, $args = '', $redirect_args = '') { + $GLOBALS['__test_exec'][] = array($filename, $args); + } + } + + if (!function_exists('cacti_escapeshellarg')) { + function cacti_escapeshellarg($string) { + return escapeshellarg($string); + } + } + + $GLOBALS['__test_exec'] = array(); + $original_base_path = $GLOBALS['config']['base_path']; + $GLOBALS['config']['base_path'] = '/var/www/cacti'; + $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; + + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 4))); + + try { + $started = flowview_launch_materialize(array('3', 5, '7; rm -rf /', 4, 0, 5)); + } finally { + $GLOBALS['config']['base_path'] = $original_base_path; + } + + expect($started)->toBe(array(3, 5, 7)) + ->and($GLOBALS['__test_exec'])->toBe(array( + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=3,5,7"), + )) + ->and($GLOBALS['__test_db_calls'][0]['params'])->toBe(array('flowview', 'materialize')); +}); + +it('does not start a materialization for filters that are already being materialized', function () { + $GLOBALS['__test_exec'] = array(); + + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 3))); + + expect(flowview_launch_materialize(array(3)))->toBe(array()) + ->and($GLOBALS['__test_exec'])->toBe(array()); +}); + +it('materializes every filter that is not already being materialized when none are given', function () { + $GLOBALS['__test_exec'] = array(); + $original_base_path = $GLOBALS['config']['base_path']; + $GLOBALS['config']['base_path'] = '/var/www/cacti'; + $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; + + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 1), array('id' => 4), array('id' => 9))); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 4))); + + try { + $started = flowview_launch_materialize(); + } finally { + $GLOBALS['config']['base_path'] = $original_base_path; + } + + expect($started)->toBe(array(1, 9)) + ->and($GLOBALS['__test_exec'])->toBe(array( + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=1,9"), + )); +}); + +it('keys the cached shards on the map query without the time range', function () { + $inner = array( + 'sql_query' => 'SELECT src_addr, SUM(bytes) AS bytes', + 'sql_where' => 'WHERE protocol IN (?)', + 'sql_range' => '(`start_time` BETWEEN ? AND ?)', + 'sql_groupby' => 'GROUP BY src_addr', + 'sql_params' => array(6), + 'sql_range_params' => array('2024-01-01 00:00:00', '2024-01-02 00:00:00'), + 'sql_start_time' => 1704067200, + 'sql_end_time' => 1704153600, + ); + + $later = $inner; + $later['sql_range_params'] = array('2024-02-01 00:00:00', '2024-02-02 00:00:00'); + $later['sql_start_time'] = 1706745600; + $later['sql_end_time'] = 1706832000; + + $other = $inner; + $other['sql_where'] = 'WHERE protocol IN (?, ?)'; + + expect(parallel_database_query_table_md5($inner))->toBe(md5(parallel_database_query_map_query($inner))) + ->and(parallel_database_query_table_md5($later))->toBe(parallel_database_query_table_md5($inner)) + ->and(parallel_database_query_table_md5($other))->not->toBe(parallel_database_query_table_md5($inner)) + ->and(parallel_database_query_map_query($inner))->not->toContain('sql_range'); +}); + +it('plans the cache keys of a filter for each NAT mode without running it', function () { + if (!function_exists('array_rekey')) { + function array_rekey($array, $key, $value) { + return array(); + } + } + + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + $plan = function ($nat) use ($filter) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + return flowview_plan_filter(4, $nat); + }; + + $nat = $plan(true); + $non_nat = $plan(false); + + expect($nat)->toBeArray()->toHaveCount(1) + ->and($nat[0])->toMatch('/^[0-9a-f]{32}$/') + ->and($non_nat[0])->toMatch('/^[0-9a-f]{32}$/') + ->and($nat[0])->not->toBe($non_nat[0]) + ->and($plan(true))->toBe($nat); + + $writes = array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }); + + expect($writes)->toBeEmpty(); +}); + +it('gives the plan of a filter only through its out parameter, never as what the report returns', function () { + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + $plan = array(); + $result = run_flow_query(false, 4, 1704067200, 1704153600, true, true, $plan); + + expect($result)->toBeTrue() + ->and($plan)->toHaveCount(1) + ->and($plan[0])->toMatch('/^[0-9a-f]{32}$/'); +}); + +it('has no plan for a filter that can not be planned', function () { + expect(flowview_plan_filter(99, true))->toBe(array()); +}); + +it('only takes an exporter address as the stream override', function ($value, $used) { + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + plugin_test_reset(); + + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + $none = flowview_plan_filter(4, true); + + $_REQUEST['ex_addr'] = $value; + + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + $with = flowview_plan_filter(4, true); + + plugin_test_reset(); + + expect($none)->toHaveCount(1) + ->and($with != $none)->toBe($used); +})->with(array( + 'an IPv4 address' => array('10.1.2.3', true), + 'an IPv6 address' => array('2001:db8::1', true), + 'markup' => array('', false), + 'All' => array('0', false), + 'a host and a port' => array('10.1.2.3:2055', false), +)); + +it('finds the partitions of a filter that are cached with and without NAT', function () { + if (!function_exists('array_rekey')) { + function array_rekey($array, $key, $value) { + return array(); + } + } + + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + // Each mode plans the filter (a row and the name) and then looks up its cache rows. + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_cell_prepared', 'Top Talkers'); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('map_table' => 'plugin_flowview_raw_2024001'), + array('map_table' => 'plugin_flowview_raw_2024002'), + )); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_cell_prepared', 'Top Talkers'); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('map_table' => 'plugin_flowview_raw_2024001'), + )); + + expect(flowview_get_cached_shard_tables(4))->toBe(array( + 'nat' => array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), + 'non_nat' => array('plugin_flowview_raw_2024001'), + )); + + $lookups = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'] ?? '', 'FROM parallel_database_query_shard_cache') !== false; + })); + + expect($lookups)->toHaveCount(2) + ->and($lookups[0]['params'])->toHaveCount(2) + ->and($lookups[0]['params'][1])->toBe(1) + ->and($lookups[0]['params'][0])->not->toBe($lookups[1]['params'][0]); +}); + +it('finds nothing cached for a filter that cannot be planned', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + + expect(flowview_get_cached_shard_tables(99))->toBe(array('nat' => array(), 'non_nat' => array())); +}); + +it('lists only genuine raw partition tables, oldest first', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('table' => 'plugin_flowview_raw_202400105'), + array('table' => 'plugin_flowview_raw_2024001'), + array('table' => 'plugin_flowview_raw_abc'), + array('table' => 'plugin_flowview_rawX2024001'), + array('table' => 'plugin_flowview_raw_2024001_bak'), + array('table' => 'plugin_flowview_raw_2023364'), + )); + + expect(flowview_get_partition_tables())->toBe(array( + 'plugin_flowview_raw_2023364', + 'plugin_flowview_raw_2024001', + 'plugin_flowview_raw_202400105', + )); +}); + +it('lists no partition tables when none exist', function () { + expect(flowview_get_partition_tables())->toBe(array()); +}); + +it('keys daily and hourly partitions so they sort in time order', function () { + expect(flowview_partition_sort_key('plugin_flowview_raw_2024001'))->toBe('202400100') + ->and(flowview_partition_sort_key('plugin_flowview_raw_202400105'))->toBe('202400105') + ->and(flowview_partition_sort_key('plugin_flowview_raw_abc'))->toBe('') + ->and(flowview_partition_sort_key('plugin_flowview_queries'))->toBe(''); +}); + +it('treats every partition but the newest as cold', function ($tables, $expected) { + expect(flowview_get_cold_partitions($tables))->toBe($expected); +})->with(array( + 'none' => array(array(), array()), + 'only the hot one' => array(array('plugin_flowview_raw_2024003'), array()), + 'several' => array( + array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003'), + array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), + ), + 'hourly' => array( + array('plugin_flowview_raw_202400105', 'plugin_flowview_raw_202400106'), + array('plugin_flowview_raw_202400105'), + ), +)); + +it('finds the partitions that rotated since the previous newest one', function ($previous, $expected) { + $tables = array( + 'plugin_flowview_raw_2024001', + 'plugin_flowview_raw_2024002', + 'plugin_flowview_raw_2024003', + 'plugin_flowview_raw_2024004', + ); + + expect(flowview_get_rotated_partitions($tables, $previous))->toBe($expected); +})->with(array( + 'nothing recorded' => array('', array()), + 'not a partition' => array('whatever', array()), + 'newest is unchanged' => array('plugin_flowview_raw_2024004', array()), + 'one rotated' => array('plugin_flowview_raw_2024003', array('plugin_flowview_raw_2024003')), + 'several rotated' => array('plugin_flowview_raw_2024002', array('plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003')), + 'previous is gone' => array('plugin_flowview_raw_2024002_old', array()), + 'recorded is ahead' => array('plugin_flowview_raw_2024009', array()), +)); + +it('reports the automatic materialization status of a filter', function ($nat, $non_nat, $running, $expected) { + $cold = array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'); + + expect(flowview_materialization_status($nat, $non_nat, $cold, $running))->toBe($expected); +})->with(array( + 'both complete' => array(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), false, 'current'), + 'complete while running' => array(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), true, 'current'), + 'hot one cached too' => array(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003'), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), false, 'current'), + 'nat missing one' => array(array('plugin_flowview_raw_2024001'), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), false, 'required'), + 'non nat missing' => array(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), array(), false, 'required'), + 'incomplete and running' => array(array(), array(), true, 'running'), + 'nothing cached' => array(array(), array(), false, 'required'), +)); + +it('is up to date when there are no cold partitions to materialize', function () { + expect(flowview_materialization_status(array(), array(), array(), false))->toBe('current'); +}); + +it('is required when the live partition is cached but an older one is not, and names it', function () { + $tables = array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003', 'plugin_flowview_raw_2024004'); + $cold = flowview_get_cold_partitions($tables); + + // every partition but the oldest is cached, and that includes the live one + $cached = array('plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003', 'plugin_flowview_raw_2024004'); + + expect(flowview_materialization_status($cached, $cached, $cold, false))->toBe('required') + ->and(flowview_get_missing_partitions($cached, $cached, $cold))->toBe(array( + 'nat' => array('plugin_flowview_raw_2024001'), + 'non_nat' => array('plugin_flowview_raw_2024001'), + )); +}); + +it('is up to date when every older partition is cached and only the live one is not', function () { + $tables = array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003'); + $cold = flowview_get_cold_partitions($tables); + $cached = array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'); + + expect(flowview_materialization_status($cached, $cached, $cold, false))->toBe('current') + ->and(flowview_get_missing_partitions($cached, $cached, $cold))->toBe(array('nat' => array(), 'non_nat' => array())); +}); + +it('describes the partitions that are missing in a tip', function ($tables, $expected) { + expect(flowview_describe_partitions($tables))->toBe($expected); +})->with(array( + 'none' => array(array(), '0'), + 'one' => array(array('plugin_flowview_raw_2024001'), '1 (plugin_flowview_raw_2024001)'), + 'three' => array(array('a', 'b', 'c'), '3 (a, b, c)'), + 'too many' => array(array('a', 'b', 'c', 'd', 'e'), '5 (a, b, c, ...)'), +)); + +it('says which partitions are missing in the tip of a pill that is not up to date', function () { + $missing = array('nat' => array('plugin_flowview_raw_2024001'), 'non_nat' => array()); + + expect(flowview_get_colored_materialization_status('required', $missing)) + ->toContain("deviceDown' title='Not materialized yet, with NAT: 1 (plugin_flowview_raw_2024001), without NAT: 0.'") + ->and(flowview_get_colored_materialization_status('running', $missing))->toContain('deviceRecovering\' title=') + ->and(flowview_get_colored_materialization_status('current', $missing))->not->toContain('title='); +}); + +it('shows the status pills in the form of the device status', function ($status, $class, $label) { + expect(flowview_get_colored_materialization_status($status))->toBe("" . $label . ''); +})->with(array( + 'current' => array('current', 'deviceUp', 'Up to date'), + 'running' => array('running', 'deviceRecovering', 'Running'), + 'required' => array('required', 'deviceDown', 'Required'), + 'unknown' => array('anything', 'deviceDown', 'Required'), +)); + +it('words the parallel query timeout for the log and a script, not the browser', function () { + $message = flowview_parallel_timeout_message(300); + + expect($message)->toStartWith('ERROR: ') + ->and($message)->toContain('300 seconds') + ->and($message)->toContain('Maximum Run Time') + ->and($message)->not->toContain('and($message)->not->toContain('DOMPurify'); +}); + +it('starts a materialization for only the partitions it is given', function () { + if (!function_exists('exec_background')) { + function exec_background($filename, $args = '', $redirect_args = '') { + $GLOBALS['__test_exec'][] = array($filename, $args); + } + } + + if (!function_exists('cacti_escapeshellarg')) { + function cacti_escapeshellarg($string) { + return escapeshellarg($string); + } + } + + $GLOBALS['__test_exec'] = array(); + $original_base_path = $GLOBALS['config']['base_path']; + $GLOBALS['config']['base_path'] = '/var/www/cacti'; + $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; + + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 1), array('id' => 2))); + + try { + $started = flowview_launch_materialize(array(), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024001; rm', 'plugin_flowview_raw_2024002')); + } finally { + $GLOBALS['config']['base_path'] = $original_base_path; + } + + expect($started)->toBe(array(1, 2)) + ->and($GLOBALS['__test_exec'])->toBe(array( + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=1,2 --partitions=plugin_flowview_raw_2024001,plugin_flowview_raw_2024002"), + )); +}); + +it('only checks the cache of the partitions it is limited to', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024001', 'map_partition' => '', 'min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00'), + )); + plugin_test_queue_db_result('db_fetch_cell', 1); + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00')); + + expect(flowview_invalidate_stale_shard_cache(array('plugin_flowview_raw_2024001')))->toBe(array('kept' => 1, 'removed' => 0)); + + $lookups = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_fetch_assoc_prepared' && strpos($call['sql'], 'map_table IN (?)') !== false; + })); + + expect($lookups)->toHaveCount(1) + ->and($lookups[0]['params'])->toBe(array('plugin_flowview_raw_2024001')); +}); + +describe('materializing the partitions that rotated', function () { + beforeEach(function () { + if (!function_exists('exec_background')) { + function exec_background($filename, $args = '', $redirect_args = '') { + $GLOBALS['__test_exec'][] = array($filename, $args); + } + } + + if (!function_exists('cacti_escapeshellarg')) { + function cacti_escapeshellarg($string) { + return escapeshellarg($string); + } + } + + $GLOBALS['__test_exec'] = array(); + $GLOBALS['__test_config']['flowview_parallel_threads'] = 4; + $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; + + $this->tables = array( + array('table' => 'plugin_flowview_raw_2024001'), + array('table' => 'plugin_flowview_raw_2024002'), + array('table' => 'plugin_flowview_raw_2024003'), + ); + }); + + it('only records the newest partition on the first cycle', function () { + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024003') + ->and($GLOBALS['__test_exec'])->toBe(array()); + }); + + it('does nothing when no partition has rotated', function () { + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024003'; + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024003') + ->and($GLOBALS['__test_exec'])->toBe(array()); + }); + + it('starts a materialization for the partitions that rotated and records the newest', function () { + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024002'; + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 1), array('id' => 2))); + + $original_base_path = $GLOBALS['config']['base_path']; + $GLOBALS['config']['base_path'] = '/var/www/cacti'; + + try { + $started = flowview_materialize_rotated_partitions(); + } finally { + $GLOBALS['config']['base_path'] = $original_base_path; + } + + expect($started)->toBe(array('plugin_flowview_raw_2024002')) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024003') + ->and($GLOBALS['__test_exec'])->toBe(array( + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=1,2 --partitions=plugin_flowview_raw_2024002"), + )); + }); + + it('leaves the record alone while a materialization is running so it is caught up later', function () { + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024001'; + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 3))); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024001') + ->and($GLOBALS['__test_exec'])->toBe(array()); + }); + + it('leaves the record alone when the parallel query is not enabled', function () { + $GLOBALS['__test_config']['flowview_parallel_threads'] = 1; + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024001'; + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024001') + ->and($GLOBALS['__test_exec'])->toBe(array()); + }); + + it('does nothing without any partitions', function () { + plugin_test_queue_db_result('db_fetch_assoc', array()); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config'])->not->toHaveKey('flowview_last_hot_partition'); + }); +}); + +describe('preparing the partitions before they are scanned', function () { + $current = array( + 'collation' => 'latin1_swedish_ci', + 'engine' => 'Aria', + 'has_keycol' => false, + 'has_template_id' => true, + 'indexes' => array('template_id', 'start_time', 'end_time', 'ex_addr'), + 'ex_addr_type' => 'varchar(46)', + ); + + it('needs no alter for a partition that is already current', function () use ($current) { + expect(flowview_get_partition_alter($current, 'Aria'))->toBe(array('alter' => '', 'ex_change' => false)); + }); + + it('alters only the engine of a partition that has rotated but kept the live engine', function ($raw_engine, $expected) use ($current) { + $info = $current; + $info['engine'] = 'InnoDB'; + + expect(flowview_get_partition_alter($info, $raw_engine))->toBe(array('alter' => $expected, 'ex_change' => false)); + })->with(array( + 'aria' => array('Aria', ' ENGINE=Aria ROW_FORMAT=page'), + 'myisam' => array('MyISAM', ' ENGINE=MyISAM ROW_FORMAT=Dynamic'), + )); + + it('plans every alter a partition from an old version needs, as the schema upgrade always did', function () { + $info = array( + 'collation' => 'utf8mb4_general_ci', + 'engine' => 'InnoDB', + 'has_keycol' => true, + 'has_template_id' => false, + 'indexes' => array(), + 'ex_addr_type' => 'varbinary(16)', + ); + + expect(flowview_get_partition_alter($info, 'Aria'))->toBe(array( + 'alter' => 'CONVERT TO CHARACTER SET latin1, DROP KEY keycol, ADD COLUMN template_id int(11) unsigned NOT NULL default "0" AFTER listener_id, ' . + 'ADD INDEX template_id (template_id), ADD INDEX start_time (start_time), ADD INDEX end_time (end_time), ' . + 'MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default "", ADD INDEX ex_addr (ex_addr), ENGINE=Aria ROW_FORMAT=page', + 'ex_change' => true, + )); + }); + + it('leaves ex_addr alone when it is a varchar in any case or is missing', function ($type) use ($current) { + $info = $current; + $info['ex_addr_type'] = $type; + + expect(flowview_get_partition_alter($info, 'Aria'))->toBe(array('alter' => '', 'ex_change' => false)); + })->with(array('VARCHAR(46)', 'varchar(20)', null)); + + it('adds only the index a partition is missing', function () use ($current) { + $info = $current; + $info['indexes'] = array('template_id', 'start_time', 'ex_addr'); + + expect(flowview_get_partition_alter($info, 'Aria')['alter'])->toBe('ADD INDEX end_time (end_time)'); + }); + + it('alters a partition and rewrites ex_addr afterwards when it was converted', function () { + flowview_apply_partition_alter('plugin_flowview_raw_2024001', array('alter' => 'MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default ""', 'ex_change' => true)); + + $sql = array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql'); + + expect(array_values($sql))->toBe(array( + 'ALTER TABLE plugin_flowview_raw_2024001 MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default ""', + 'UPDATE plugin_flowview_raw_2024001 SET ex_addr = SUBSTRING_INDEX(ex_addr, ":", 1)', + )); + }); + + it('does not rewrite ex_addr when it was not converted', function () { + flowview_apply_partition_alter('plugin_flowview_raw_2024001', array('alter' => 'ADD INDEX end_time (end_time)', 'ex_change' => false)); + + $sql = array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql'); + + expect(array_values($sql))->toBe(array('ALTER TABLE plugin_flowview_raw_2024001 ADD INDEX end_time (end_time)')); + }); + + it('logs an alter and shows it too, as the schema upgrade always did', function () { + $GLOBALS['__test_cacti_log'] = array(); + $GLOBALS['__test_cacti_log_print'] = array(); + + flowview_apply_partition_alter('plugin_flowview_raw_2024001', array('alter' => 'ADD INDEX end_time (end_time)', 'ex_change' => false)); + + expect($GLOBALS['__test_cacti_log'])->toBe(array('Altering Table plugin_flowview_raw_2024001 Using this alter ADD INDEX end_time (end_time).')) + ->and($GLOBALS['__test_cacti_log_print'])->toBe(array(true)); + }); + + it('only logs an alter when the caller reports it itself, so it is not said twice', function () { + $GLOBALS['__test_cacti_log'] = array(); + $GLOBALS['__test_cacti_log_print'] = array(); + + flowview_apply_partition_alter('plugin_flowview_raw_2024001', array('alter' => ' ENGINE=Aria ROW_FORMAT=page', 'ex_change' => false), false); + + expect($GLOBALS['__test_cacti_log'])->toBe(array('Altering Table plugin_flowview_raw_2024001 Using this alter ENGINE=Aria ROW_FORMAT=page.')) + ->and($GLOBALS['__test_cacti_log_print'])->toBe(array(false)); + }); + + it('finds the partitions that still need an alter in a single pass', function () { + $table = function ($name, $engine) { + return array('TABLE_NAME' => $name, 'TABLE_COLLATION' => 'latin1_swedish_ci', 'ENGINE' => $engine); + }; + + $columns = function ($name) { + return array( + array('TABLE_NAME' => $name, 'COLUMN_NAME' => 'template_id', 'COLUMN_TYPE' => 'int(11) unsigned'), + array('TABLE_NAME' => $name, 'COLUMN_NAME' => 'ex_addr', 'COLUMN_TYPE' => 'varchar(46)'), + ); + }; + + $indexes = function ($name, $names) { + return array_map(function ($index) use ($name) { + return array('TABLE_NAME' => $name, 'INDEX_NAME' => $index); + }, $names); + }; + + $all = array('template_id', 'start_time', 'end_time', 'ex_addr'); + + plugin_test_queue_db_result('db_fetch_assoc', array( + $table('plugin_flowview_raw_2024001', 'Aria'), + $table('plugin_flowview_raw_2024002', 'InnoDB'), + $table('plugin_flowview_raw_2024003', 'Aria'), + )); + plugin_test_queue_db_result('db_fetch_assoc', array_merge( + $columns('plugin_flowview_raw_2024001'), + $columns('plugin_flowview_raw_2024002'), + $columns('plugin_flowview_raw_2024003') + )); + plugin_test_queue_db_result('db_fetch_assoc', array_merge( + $indexes('plugin_flowview_raw_2024001', $all), + $indexes('plugin_flowview_raw_2024002', $all), + $indexes('plugin_flowview_raw_2024003', array('template_id', 'start_time', 'ex_addr')) + )); + + $pending = flowview_get_pending_partition_alters( + array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003', 'plugin_flowview_raw_2024009'), + 'Aria' + ); + + expect(array_keys($pending))->toBe(array('plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003')) + ->and($pending['plugin_flowview_raw_2024002']['alter'])->toBe(' ENGINE=Aria ROW_FORMAT=page') + ->and($pending['plugin_flowview_raw_2024003']['alter'])->toBe('ADD INDEX end_time (end_time)') + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_fetch_assoc'; + }))->toHaveCount(3); + }); + + it('looks at nothing when there are no partitions', function () { + expect(flowview_get_pending_partition_alters(array(), 'Aria'))->toBe(array()) + ->and($GLOBALS['__test_db_calls'])->toBe(array()); + }); + + it('finds the alters running on the partitions it needs', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TIME' => 120, 'INFO' => 'ALTER TABLE plugin_flowview_raw_2024001 ADD INDEX end_time (end_time)'), + array('TIME' => 5, 'INFO' => 'alter table `plugin_flowview_raw_2024002` ENGINE=Aria'), + array('TIME' => 300, 'INFO' => 'ALTER TABLE `flowview`.`plugin_flowview_raw_2024001` ENGINE=Aria'), + array('TIME' => 9, 'INFO' => 'ALTER TABLE plugin_flowview_raw_2024099 ENGINE=Aria'), + array('TIME' => 3, 'INFO' => 'ALTER TABLE plugin_flowview_arin_information MODIFY COLUMN comments VARCHAR(512)'), + array('TIME' => 1, 'INFO' => null), + )); + + expect(flowview_get_running_partition_alters(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002')))->toBe(array( + 'plugin_flowview_raw_2024001' => 300, + 'plugin_flowview_raw_2024002' => 5, + )); + }); + + it('finds no running alters when there are none', function () { + expect(flowview_get_running_partition_alters(array('plugin_flowview_raw_2024001')))->toBe(array()); + }); + + it('converts the engine of a partition when no alter is running on it', function () { + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + plugin_test_queue_db_result('db_fetch_cell_prepared', 'InnoDB'); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + + expect(flowview_convert_partition_engine('plugin_flowview_raw_202629214', 'Aria'))->toBeTrue(); + + $sql = array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql'); + + $locks = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], '_LOCK(') !== false; + })); + + expect(array_values($sql))->toBe(array('ALTER TABLE plugin_flowview_raw_202629214 ENGINE=Aria')) + ->and($locks[0]['sql'])->toContain('GET_LOCK') + ->and($locks[0]['params'])->toBe(array('flowview_alter_plugin_flowview_raw_202629214', 0)) + ->and($locks[1]['sql'])->toContain('RELEASE_LOCK'); + }); + + it('does not wait for, or repeat, an alter another process holds the lock for', function () { + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_cell_prepared', '0'); + + expect(flowview_convert_partition_engine('plugin_flowview_raw_202629214', 'Aria'))->toBeFalse() + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }))->toBeEmpty(); + }); + + it('does not convert a partition that was converted while it waited for the lock', function () { + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + plugin_test_queue_db_result('db_fetch_cell_prepared', 'Aria'); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + + expect(flowview_convert_partition_engine('plugin_flowview_raw_202629214', 'Aria'))->toBeTrue() + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }))->toBeEmpty(); + }); + + it('does not start the same engine conversion again while one is running', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TIME' => 1605, 'INFO' => 'ALTER TABLE plugin_flowview_raw_202629214 ENGINE=Aria'), + )); + + expect(flowview_convert_partition_engine('plugin_flowview_raw_202629214', 'Aria'))->toBeFalse(); + + $writes = array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }); + + expect($writes)->toBeEmpty(); + }); + + it('still converts a partition when the alter running is on another one', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TIME' => 12, 'INFO' => 'ALTER TABLE plugin_flowview_raw_202629213 ENGINE=Aria'), + )); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + plugin_test_queue_db_result('db_fetch_cell_prepared', 'InnoDB'); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + + expect(flowview_convert_partition_engine('plugin_flowview_raw_202629214', 'Aria'))->toBeTrue(); + }); + + it('finds the partitions that have no NAT columns', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TABLE_NAME' => 'plugin_flowview_raw_2024001'), + array('TABLE_NAME' => 'plugin_flowview_raw_2024003'), + )); + + expect(flowview_get_partitions_without_nat_columns(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003'))) + ->toBe(array('plugin_flowview_raw_2024002')); + }); +}); + +describe('finding a materialization that is still running', function () { + beforeEach(function () { + if (!function_exists('cacti_process_still_running')) { + function cacti_process_still_running($pid) { + return in_array($pid, $GLOBALS['__test_live_pids'], true); + } + } + + $GLOBALS['__test_live_pids'] = array(); + }); + + it('finds the runner and shard workers of a request by the names they register', function () { + if (!function_exists('posix_kill')) { + $this->markTestSkipped('The posix extension is not available.'); + } + + // A pid that cannot be a live process, so nothing is signalled. + plugin_test_queue_db_result('db_fetch_assoc_prepared', array()); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('pid' => 4999999, 'taskname' => 'db_query_991'), + array('pid' => 4999998, 'taskname' => 'db_shard_991'), + )); + + parallel_database_query_cleanup(991); + + $lookup = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_fetch_assoc_prepared' && strpos($call['sql'], 'FROM processes') !== false; + })); + + $removed = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && strpos($call['sql'], 'DELETE FROM processes') === 0; + })); + + expect($lookup)->toHaveCount(1) + ->and($lookup[0]['params'])->toBe(array('flowview', 'db_query_991', 'db_shard_991')) + ->and(array_column($removed, 'params'))->toBe(array(array(4999999), array(4999998))); + }); + + it('finds the running requests from the process table', function () { + $GLOBALS['__test_live_pids'] = array(222); + + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + // old, and the process is gone + array('pid' => 111, 'taskname' => 'db_query_990', 'age' => 5000), + // old, but the process is alive + array('pid' => 222, 'taskname' => 'db_shard_991', 'age' => 5000), + array('pid' => 223, 'taskname' => 'db_shard_991', 'age' => 5000), + // recent, so it is trusted even though the process cannot be seen + array('pid' => 333, 'taskname' => 'db_shard_992', 'age' => 10), + array('pid' => 444, 'taskname' => 'db_shard_abc', 'age' => 10), + )); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('id' => 991, 'md5sum_tables' => 'aaaa'), + array('id' => 992, 'md5sum_tables' => 'bbbb'), + )); + + expect(flowview_get_running_requests())->toBe(array(991 => 'aaaa', 992 => 'bbbb')); + + $lookup = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM parallel_database_query') !== false; + })); + + // it is not limited to the requests of one user + expect($lookup[0]['sql'])->not->toContain('user_id') + ->and($lookup[0]['params'])->toBe(array(991, 992)); + }); + + it('keeps a running request whose row is gone, without a cache key', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('pid' => 111, 'taskname' => 'db_query_991', 'age' => 5))); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array()); + + expect(flowview_get_running_requests())->toBe(array(991 => '')); + }); + + it('ignores the rows left behind by processes that are gone', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('pid' => 111, 'taskname' => 'db_query_991', 'age' => 9000), + array('pid' => 222, 'taskname' => 'db_shard_991', 'age' => 9000), + )); + + expect(flowview_get_running_requests())->toBe(array()) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM parallel_database_query') !== false; + }))->toBeEmpty(); + }); + + it('finds nothing without any runner processes', function () { + expect(flowview_get_running_requests())->toBe(array()) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM parallel_database_query') !== false; + }))->toBeEmpty(); + }); + + it('looks for the runner and shard worker names in the process table', function () { + flowview_get_running_requests(); + + $lookup = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM processes') !== false; + })); + + expect($lookup)->toHaveCount(1) + ->and($lookup[0]['params'])->toBe(array('flowview', 'db\_query\_%', 'db\_shard\_%')); + }); + + it('is running while a filter is registered as being materialized', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 3))); + + expect(flowview_materialization_running())->toBeTrue() + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM processes') !== false && ($call['params'][1] ?? '') === 'db\_query\_%'; + }))->toBeEmpty(); + }); + + it('is still running after the script that started it has ended', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array()); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('pid' => 222, 'taskname' => 'db_shard_991', 'age' => 12))); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 991, 'md5sum_tables' => 'aaaa'))); + + expect(flowview_materialization_running())->toBeTrue(); + }); + + it('is not running when nothing is registered and no runner is registered', function () { + expect(flowview_materialization_running())->toBeFalse(); + }); +}); + +describe('showing the queries of a filter that are running', function () { + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + beforeEach(function () { + if (!function_exists('array_rekey')) { + function array_rekey($array, $key, $value) { + return array(); + } + } + + if (!function_exists('number_format_i18n')) { + function number_format_i18n($number, $decimals = 0) { + return number_format($number, $decimals); + } + } + }); + + it('works out the cache keys of a filter for each NAT mode', function () use ($filter) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + $md5sums = flowview_get_filter_md5sums(4); + + expect($md5sums['nat'])->toHaveCount(1) + ->and($md5sums['non_nat'])->toHaveCount(1) + ->and($md5sums['nat'][0])->toMatch('/^[0-9a-f]{32}$/') + ->and($md5sums['nat'][0])->not->toBe($md5sums['non_nat'][0]); + }); + + it('has no cache keys for a filter that cannot be planned', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + + expect(flowview_get_filter_md5sums(99))->toBe(array('nat' => array(), 'non_nat' => array())); + }); + + it('uses the cache keys it is given instead of planning the filter again', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('map_table' => 'plugin_flowview_raw_2024001'))); + + $cached = flowview_get_cached_shard_tables(4, array('nat' => array('aaaa'), 'non_nat' => array())); + + expect($cached)->toBe(array('nat' => array('plugin_flowview_raw_2024001'), 'non_nat' => array())) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_fetch_row_prepared'; + }))->toBeEmpty(); + }); + + it('counts the running requests that are queries of a filter', function ($running, $expected) { + $md5sums = array('nat' => array('aaaa'), 'non_nat' => array('bbbb', 'cccc')); + + expect(flowview_count_running_queries($md5sums, $running))->toBe($expected); + })->with(array( + 'none running' => array(array(), 0), + 'one with nat' => array(array(991 => 'aaaa'), 1), + 'both modes' => array(array(991 => 'aaaa', 992 => 'bbbb'), 2), + 'two of the report' => array(array(991 => 'bbbb', 992 => 'cccc'), 2), + 'someone elses' => array(array(991 => 'zzzz'), 0), + 'mixed' => array(array(991 => 'aaaa', 992 => 'zzzz', 993 => 'cccc'), 2), + )); + + it('counts nothing for a filter that has no cache keys', function () { + expect(flowview_count_running_queries(array('nat' => array(), 'non_nat' => array()), array(991 => 'aaaa')))->toBe(0); + }); + + it('counts the running queries of a filter with NAT and without', function () { + $md5sums = array('nat' => array('aaaa'), 'non_nat' => array('bbbb')); + + expect(flowview_count_running_queries_by_mode($md5sums, array(991 => 'aaaa', 992 => 'bbbb', 993 => 'bbbb', 994 => 'zzzz'))) + ->toBe(array('nat' => 1, 'non_nat' => 2)); + }); + + it('finds the filter a running request belongs to, by its cache key', function () { + $filters = array( + 1 => array('nat' => array('aaaa'), 'non_nat' => array('bbbb')), + 15 => array('nat' => array('cccc'), 'non_nat' => array('cccc', 'dddd')), + 16 => array('nat' => array(), 'non_nat' => array()), + ); + + expect(flowview_find_request_filters($filters, 'aaaa'))->toBe(array(array('id' => 1, 'nat' => true, 'non_nat' => false))) + ->and(flowview_find_request_filters($filters, 'bbbb'))->toBe(array(array('id' => 1, 'nat' => false, 'non_nat' => true))) + ->and(flowview_find_request_filters($filters, 'cccc'))->toBe(array(array('id' => 15, 'nat' => true, 'non_nat' => true))) + ->and(flowview_find_request_filters($filters, 'zzzz'))->toBe(array()) + ->and(flowview_find_request_filters($filters, ''))->toBe(array()); + }); + + it('says how many running queries are with NAT and without in the tip', function () { + $pill = flowview_get_colored_running_queries(3, array('nat' => 1, 'non_nat' => 2)); + + expect($pill)->toContain('Running: 3') + ->and($pill)->toContain('With NAT: 1, without NAT: 2.'); + }); + + it('shows the number of running queries in a pill like a disabled device', function () { + $pill = flowview_get_colored_running_queries(1234); + + expect($pill)->toStartWith("and($pill)->toContain('Running: 1,234') + ->and($pill)->toContain('title='); + }); + + it('shows no pill when nothing is running', function ($count) { + expect(flowview_get_colored_running_queries($count))->toBe(''); + })->with(array(0, -1)); +}); + +describe('keeping the live partition out of the cache', function () { + it('scans the whole of a partition only when the range covers all of it', function ($start, $end, $live, $expected) { + // the partition holds flows from 1000 to 2000 + expect(flowview_is_full_scan($start, $end, 1000, 2000, $live))->toBe($expected); + })->with(array( + 'covers it' => array(500, 3000, false, true), + 'starts after its first flow' => array(1500, 3000, false, false), + 'ends before its last flow' => array(500, 1500, false, false), + 'starts at its first flow' => array(1000, 3000, false, false), + 'ends at its last flow' => array(500, 2000, false, false), + 'covers it but is live' => array(500, 3000, true, false), + )); + + it('finds the live partition as the newest one', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('table' => 'plugin_flowview_raw_202400105'), + array('table' => 'plugin_flowview_raw_202400107'), + array('table' => 'plugin_flowview_raw_202400106'), + array('table' => 'plugin_flowview_raw_abc'), + )); + + expect(flowview_get_live_partition())->toBe('plugin_flowview_raw_202400107'); + }); + + it('has no live partition when there are no partitions', function () { + expect(flowview_get_live_partition())->toBe(''); + }); + + it('removes what was cached for the live partition', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('table' => 'plugin_flowview_raw_2024001'), + array('table' => 'plugin_flowview_raw_2024002'), + )); + plugin_test_queue_db_result('db_fetch_cell_prepared', 3); + + expect(flowview_remove_live_partition_cache())->toBe(3); + + $deleted = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && strpos($call['sql'], 'DELETE FROM parallel_database_query_shard_cache') === 0; + })); + + expect($deleted)->toHaveCount(1) + ->and($deleted[0]['params'])->toBe(array('plugin_flowview_raw_2024002')); + }); + + it('removes nothing when the live partition has nothing cached', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('table' => 'plugin_flowview_raw_2024001'))); + plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + + expect(flowview_remove_live_partition_cache())->toBe(0) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + }); + + it('does nothing without any partitions', function () { + expect(flowview_remove_live_partition_cache())->toBe(0) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'] ?? '', 'parallel_database_query_shard_cache') !== false; + }))->toBeEmpty(); + }); + + it('keeps the short lived live shard results out of the removal and the staleness checks', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('table' => 'plugin_flowview_raw_2024001'))); + plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + + flowview_remove_live_partition_cache(); + flowview_invalidate_stale_shard_cache(); + flowview_invalidate_stale_shard_cache(array('plugin_flowview_raw_2024001')); + + $selects = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'] ?? '', 'FROM parallel_database_query_shard_cache') !== false; + })); + + expect($selects)->not->toBeEmpty(); + + foreach ($selects as $call) { + expect($call['sql'])->toContain("map_partition != 'live'"); + } + }); +}); + +describe('sharing one scan of the live partition', function () { + beforeEach(function () { + plugin_test_reset(); + }); + + $query = array('md5sum_tables' => 'abc'); + $shard = array('map_table' => 'plugin_flowview_raw_2024001', 'map_params' => '["x"]'); + + it('derives one key for the same map query, table and range', function () use ($query, $shard) { + $key = flowview_live_shard_key($query, $shard); + + expect($key)->toHaveLength(32) + ->and(flowview_live_shard_key($query, $shard))->toBe($key) + ->and(flowview_live_shard_key(array('md5sum_tables' => 'def'), $shard))->not->toBe($key) + ->and(flowview_live_shard_key($query, array_merge($shard, array('map_params' => '["y"]'))))->not->toBe($key) + ->and(flowview_live_shard_key($query, array_merge($shard, array('map_table' => 'plugin_flowview_raw_2024002'))))->not->toBe($key) + ->and($key)->not->toBe('abc'); + }); + + it('takes and releases a named lock on that key', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + + expect(flowview_live_shard_lock('k1'))->toBeTrue(); + + flowview_live_shard_unlock('k1'); + + $calls = array_values($GLOBALS['__test_db_calls']); + + expect($calls[0]['sql'])->toContain('GET_LOCK') + ->and($calls[0]['params'])->toBe(array('flowview_live_k1', FLOWVIEW_LIVE_SHARD_LOCK_WAIT)) + ->and($calls[1]['sql'])->toContain('RELEASE_LOCK') + ->and($calls[1]['params'])->toBe(array('flowview_live_k1')); + }); + + it('reports a lock that was not obtained in time', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', '0'); + + expect(flowview_live_shard_lock('k1'))->toBeFalse(); + }); + + it('reuses a recent live result', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array('results' => json_encode(array(array('bytes' => 5))))); + + expect(flowview_live_shard_get('k1', 'plugin_flowview_raw_2024001'))->toBe(array(array('bytes' => 5))); + + $call = $GLOBALS['__test_db_calls'][0]; + + expect($call['params'])->toBe(array('k1', 'plugin_flowview_raw_2024001', 'live', FLOWVIEW_LIVE_SHARD_TTL)) + ->and($call['sql'])->toContain('date_created >= DATE_SUB(NOW(), INTERVAL ? SECOND)'); + }); + + it('reuses an empty live result', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array('results' => '[]')); + + expect(flowview_live_shard_get('k1', 'plugin_flowview_raw_2024001'))->toBe(array()); + }); + + it('has nothing to reuse without a recent live result', function () { + expect(flowview_live_shard_get('k1', 'plugin_flowview_raw_2024001'))->toBeNull(); + }); + + it('has nothing to reuse when the stored result is unreadable', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array('results' => 'not json')); + + expect(flowview_live_shard_get('k1', 'plugin_flowview_raw_2024001'))->toBeNull(); + }); + + it('keeps a live result under the live marker', function () { + flowview_live_shard_save('k1', 'plugin_flowview_raw_2024001', array(array('bytes' => 5))); + + $call = $GLOBALS['__test_db_calls'][0]; + + expect($call['sql'])->toStartWith('REPLACE INTO parallel_database_query_shard_cache') + ->and($call['params'])->toBe(array('k1', 'plugin_flowview_raw_2024001', 'live', json_encode(array(array('bytes' => 5))))); + }); + + it('removes only expired live results', function () { + flowview_live_shard_expire(); + + $call = $GLOBALS['__test_db_calls'][0]; + + expect($call['sql'])->toContain('map_partition = ?') + ->and($call['sql'])->toContain('date_created < DATE_SUB(NOW(), INTERVAL ? SECOND)') + ->and($call['params'])->toBe(array('live', FLOWVIEW_LIVE_SHARD_TTL)); + }); + + it('expires them with the poller clean up of parallel queries', function () { + parallel_database_query_expire(); + + $sql = array_column($GLOBALS['__test_db_calls'], 'sql'); + + expect(array_filter($sql, fn ($s) => strpos($s, "DELETE FROM parallel_database_query_shard_cache") === 0))->toHaveCount(1); + }); +}); + +describe('converting the partitions that have rotated', function () { + beforeEach(function () { + plugin_test_reset(); + + /* each conversion takes the table's alter lock, checks its engine and releases the lock */ + $this->granted = function (int $conversions) { + for ($i = 0; $i < $conversions; $i++) { + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + plugin_test_queue_db_result('db_fetch_cell_prepared', 'InnoDB'); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + } + }; + + $this->partitions = function (array $engines) { + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table) { + return array('table' => $table); + }, array_keys($engines))); + }; + + $this->alters = function () { + return array_values(array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql')); + }; + }); + + it('converts every cold partition that is not on the fast engine, newest first, and never the live one', function () { + $engines = array( + 'plugin_flowview_raw_202629214' => 'InnoDB', + 'plugin_flowview_raw_202629215' => 'Aria', + 'plugin_flowview_raw_202629216' => 'InnoDB', + 'plugin_flowview_raw_202629217' => 'InnoDB', + ); + + ($this->partitions)($engines); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table, $engine) { + return array('TABLE_NAME' => $table, 'ENGINE' => $engine); + }, array_keys($engines), $engines)); + ($this->granted)(2); + + expect(flowview_convert_cold_partitions('Aria'))->toBe(array('plugin_flowview_raw_202629216', 'plugin_flowview_raw_202629214')) + ->and(($this->alters)())->toBe(array( + 'ALTER TABLE plugin_flowview_raw_202629216 ENGINE=Aria', + 'ALTER TABLE plugin_flowview_raw_202629214 ENGINE=Aria', + )); + }); + + it('catches up on partitions that were missed for more than a few hours', function () { + $engines = array(); + + foreach (range(10, 20) as $hour) { + $engines['plugin_flowview_raw_2026292' . $hour] = 'InnoDB'; + } + + ($this->partitions)($engines); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table, $engine) { + return array('TABLE_NAME' => $table, 'ENGINE' => $engine); + }, array_keys($engines), $engines)); + ($this->granted)(10); + + expect(flowview_convert_cold_partitions('Aria', 100))->toHaveCount(10); + }); + + it('converts only as many as it is allowed to in one go', function () { + $engines = array( + 'plugin_flowview_raw_202629214' => 'InnoDB', + 'plugin_flowview_raw_202629215' => 'InnoDB', + 'plugin_flowview_raw_202629216' => 'InnoDB', + 'plugin_flowview_raw_202629217' => 'InnoDB', + 'plugin_flowview_raw_202629218' => 'InnoDB', + ); + + ($this->partitions)($engines); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table, $engine) { + return array('TABLE_NAME' => $table, 'ENGINE' => $engine); + }, array_keys($engines), $engines)); + ($this->granted)(2); + + expect(flowview_convert_cold_partitions('Aria', 2))->toBe(array('plugin_flowview_raw_202629217', 'plugin_flowview_raw_202629216')); + }); + + it('does nothing while an alter is running on any cold partition', function () { + ($this->partitions)(array( + 'plugin_flowview_raw_202629214' => 'InnoDB', + 'plugin_flowview_raw_202629215' => 'InnoDB', + 'plugin_flowview_raw_202629216' => 'InnoDB', + )); + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TIME' => 14, 'INFO' => 'ALTER TABLE plugin_flowview_raw_202629214 ENGINE=Aria ROW_FORMAT=page'), + )); + + expect(flowview_convert_cold_partitions('Aria'))->toBe(array()) + ->and(($this->alters)())->toBe(array()); + }); + + it('does nothing when the cold partitions are all on the fast engine', function () { + $engines = array( + 'plugin_flowview_raw_202629214' => 'Aria', + 'plugin_flowview_raw_202629215' => 'Aria', + 'plugin_flowview_raw_202629216' => 'InnoDB', + ); + + ($this->partitions)($engines); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table, $engine) { + return array('TABLE_NAME' => $table, 'ENGINE' => $engine); + }, array_keys($engines), $engines)); + + expect(flowview_convert_cold_partitions('Aria'))->toBe(array()) + ->and(($this->alters)())->toBe(array()); + }); + + it('does nothing when there is no cold partition', function () { + ($this->partitions)(array('plugin_flowview_raw_202629216' => 'InnoDB')); + + expect(flowview_convert_cold_partitions('Aria'))->toBe(array()); + }); +}); + +describe('altering a partition under its lock, from the table as it is then', function () { + beforeEach(function () { + plugin_test_reset(); + + /* the schema of one partition, read after the lock is taken */ + $this->schema = function (string $table, string $engine) { + plugin_test_queue_db_result('db_fetch_assoc', array(array('TABLE_NAME' => $table, 'TABLE_COLLATION' => 'latin1_swedish_ci', 'ENGINE' => $engine))); + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TABLE_NAME' => $table, 'COLUMN_NAME' => 'template_id', 'COLUMN_TYPE' => 'int(11) unsigned'), + array('TABLE_NAME' => $table, 'COLUMN_NAME' => 'ex_addr', 'COLUMN_TYPE' => 'varchar(46)'), + )); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($index) use ($table) { + return array('TABLE_NAME' => $table, 'INDEX_NAME' => $index); + }, array('template_id', 'start_time', 'end_time', 'ex_addr'))); + }; + + $this->alters = function () { + return array_values(array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql')); + }; + }); + + it('applies the plan it works out after it has the lock, and releases the lock', function () { + $table = 'plugin_flowview_raw_202629214'; + $seen = array(); + + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + ($this->schema)($table, 'InnoDB'); + + $result = flowview_alter_partition($table, 'Aria', false, 60, function ($plan) use (&$seen) { + $seen[] = trim($plan['alter']); + }); + + $lock = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], '_LOCK(') !== false; + })); + + expect($result)->toBe('altered') + ->and($seen)->toBe(array('ENGINE=Aria ROW_FORMAT=page')) + ->and(($this->alters)())->toBe(array("ALTER TABLE $table ENGINE=Aria ROW_FORMAT=page")) + ->and($lock[0]['params'])->toBe(array("flowview_alter_$table", 60)) + ->and($lock[1]['sql'])->toContain('RELEASE_LOCK'); + }); + + it('does not alter a table another process already altered', function () { + $table = 'plugin_flowview_raw_202629214'; + + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + ($this->schema)($table, 'Aria'); + + expect(flowview_alter_partition($table, 'Aria', false, 60))->toBe('current') + ->and(($this->alters)())->toBe(array()) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'RELEASE_LOCK') !== false; + }))->toHaveCount(1); + }); + + it('leaves a table alone when the lock is not had in time', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', '0'); + + expect(flowview_alter_partition('plugin_flowview_raw_202629214', 'Aria', false, 1))->toBe('busy') + ->and(($this->alters)())->toBe(array()) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'information_schema') !== false; + }))->toBeEmpty(); + }); + + it('reads only the partitions it is asked about, when there are few', function () { + flowview_get_pending_partition_alters(array('plugin_flowview_raw_202629214', 'plugin_flowview_raw_202629215'), 'Aria'); + + $reads = array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'information_schema') !== false; + }), 'sql'); + + expect($reads)->toHaveCount(3); + + foreach ($reads as $sql) { + expect($sql)->toContain('AND TABLE_NAME IN ("plugin_flowview_raw_202629214", "plugin_flowview_raw_202629215")') + ->and($sql)->not->toContain('LIKE'); + } + }); + + it('is how the materializer, the poller and the schema upgrade alter a partition', function () { + expect(plugin_test_read_source('cli/materialize_queries.php'))->toContain('flowview_alter_partition($table, $raw_engine, false, FLOWVIEW_ALTER_LOCK_WAIT') + ->and(plugin_test_read_source('cli/materialize_queries.php'))->not->toContain('flowview_apply_partition_alter(') + ->and(plugin_test_read_source('flowview_upgrade.php'))->toContain("flowview_alter_partition(\$t['TABLE_NAME'], \$raw_engine)") + ->and(plugin_test_read_source('flowview_upgrade.php'))->not->toContain('flowview_apply_partition_alter(') + ->and(plugin_test_read_source('includes/functions.php'))->toMatch('/function flowview_convert_partition_engine.*?GET_LOCK\(\?, \?\)\', \[\$lock, 0\]/s'); + }); +}); + +describe('materializing only the partitions of a batch', function () { + beforeEach(function () { + plugin_test_reset(); + + if (!function_exists('array_rekey')) { + function array_rekey($array, $key, $value) { + return array(); + } + } + + unset($GLOBALS['flowview_partition_allowlist']); + }); + + afterEach(function () { + unset($GLOBALS['flowview_partition_allowlist']); + }); + + $tables = function () { + /* includes/arrays.php reads three lists first */ + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array( + array('table' => 'plugin_flowview_raw_2024001'), + array('table' => 'plugin_flowview_raw_2024002'), + array('table' => 'plugin_flowview_raw_2024003'), + )); + + foreach (range(1, 3) as $unused) { + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00', 'table_partition' => '')); + } + + return get_tables_for_query(strtotime('2024-01-01 00:00:00'), strtotime('2024-01-10 00:00:00')); + }; + + it('scans every partition in the range for a report', function () use ($tables) { + expect(array_keys($tables()))->toBe(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003')); + }); + + it('scans only the partitions of the batch for a materialization worker', function () use ($tables) { + $GLOBALS['flowview_partition_allowlist'] = array('plugin_flowview_raw_2024002' => true); + + expect(array_keys($tables()))->toBe(array('plugin_flowview_raw_2024002')); + }); + + it('passes the batch to the worker, which starts without a shell', function () { + $source = plugin_test_read_source('cli/materialize_queries.php'); + + expect($source)->toContain("'--partitions=' . implode(',', \$batch),") + ->and($source)->toContain("\$GLOBALS['flowview_partition_allowlist'] = array_fill_keys(\$partitions, true);") + ->and($source)->toContain('function materialize_run_command(array $command): int {') + ->and($source)->not->toContain('escapeshellarg(PHP_BINARY)'); + }); +}); + +describe('showing a printed report as a table', function () { + it('makes the saved panels agree before they are merged into the request', function () { + expect(plugin_test_read_source('flowview.php'))->toMatch("/set_request_var\('flows', 'false'\);\s*\n(\s*\/\*[^\n]*\*\/\n)?\s*\\\$query\['panel_table'\]\s*= 'on';\s*\n\s*\\\$query\['panel_bytes'\]\s*= '';\s*\n\s*\\\$query\['panel_packets'\] = '';\s*\n\s*\\\$query\['panel_flows'\]\s*= '';/"); + }); +}); + +describe('showing only the filters being materialized as running', function () { + beforeEach(function () { + plugin_test_reset(); + + if (!function_exists('cacti_process_still_running')) { + function cacti_process_still_running($pid) { + return in_array($pid, $GLOBALS['__test_live_pids'], true); + } + } + + $GLOBALS['__test_live_pids'] = array(); + }); + + it('finds the filters being materialized from their task ids', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('taskid' => 3), + array('taskid' => '7'), + array('taskid' => 3), + )); + + expect(flowview_get_materializing_filters())->toBe(array(3, 7)); + + $lookup = $GLOBALS['__test_db_calls'][0]; + + expect($lookup['params'])->toBe(array('flowview', 'materialize')) + ->and($lookup['sql'])->toContain('taskid > 0'); + }); + + it('finds no filter being materialized when none is registered', function () { + expect(flowview_get_materializing_filters())->toBe(array()); + }); + + it('removes the registrations of a materialization whose process is gone', function () { + $GLOBALS['__test_live_pids'] = array(222); + + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('id' => 10, 'pid' => 111), + array('id' => 11, 'pid' => 222), + array('id' => 12, 'pid' => '333'), + )); + + expect(flowview_purge_dead_materializations())->toBe(2); + + $deleted = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && strpos($call['sql'], 'DELETE FROM processes') === 0; + })); + + expect(array_column($deleted, 'params'))->toBe(array(array(10), array(12))); + }); + + it('is not running for a filter that is not registered, while another is', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 3))); + + expect(in_array(1, flowview_get_materializing_filters(), true))->toBeFalse(); + }); +}); + +describe('keeping the report of a filter fixed and pruning what no filter caches', function () { + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + beforeEach(function () { + plugin_test_reset(); + }); + + /* queue the lookups of one filter for each NAT mode, and return its cache keys */ + $keys_of = function (array $filter) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + $keys = flowview_get_filter_md5sums((int) $filter['id']); + + plugin_test_reset(); + + return $keys; + }; + + it('names the report of a filter the way the report selector does', function ($filter, $expected) { + expect(flowview_filter_report_key($filter))->toBe($expected); + })->with(array( + 'statistical' => array(array('statistics' => 10, 'printed' => 0), 's10'), + 'printed' => array(array('statistics' => '0', 'printed' => '2'), 'p2'), + 'none' => array(array('statistics' => 0, 'printed' => 0), ''), + 'missing' => array(array(), ''), + )); + + it('does not let the request change the report of a filter', function () use ($filter) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + $plain = flowview_plan_filter(4, true); + + $_REQUEST['report'] = 's11'; + + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + expect(flowview_plan_filter(4, true))->toBe($plain); + }); + + it('still lets the request pick the report of a filter that has none', function () use ($filter) { + $none = array_merge($filter, array('statistics' => 0, 'printed' => 0)); + + plugin_test_queue_db_result('db_fetch_row_prepared', $none); + + expect(flowview_plan_filter(4, true))->toBe(array()); + + $_REQUEST['report'] = 's10'; + + plugin_test_queue_db_result('db_fetch_row_prepared', $none); + + expect(flowview_plan_filter(4, true))->toBeArray()->toHaveCount(1); + }); + + it('knows the cache keys of every filter', function () use ($filter, $keys_of) { + $keys = $keys_of($filter); + + plugin_test_queue_db_result('db_fetch_assoc', array( + array('id' => 4, 'statistics' => 10, 'printed' => 0), + array('id' => 5, 'statistics' => 10, 'printed' => 0), + )); + + foreach (range(1, 4) as $unused) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + } + + expect(flowview_get_filter_shard_keys())->toBe(array( + $keys['nat'][0] => true, + $keys['non_nat'][0] => true, + )); + }); + + it('does not know the cache keys when a filter that has a report cannot be planned', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + + expect(flowview_get_filter_shard_keys())->toBeNull(); + }); + + it('knows the cache keys when only a filter that has no report cannot be planned', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 0, 'printed' => 0))); + + expect(flowview_get_filter_shard_keys())->toBe(array()); + }); + + it('tells the cached shards of the filters from the others', function () use ($filter, $keys_of) { + $keys = $keys_of($filter); + + plugin_test_queue_db_result('db_fetch_assoc', array( + array('md5sum' => $keys['nat'][0], 'shards' => 40), + array('md5sum' => $keys['non_nat'][0], 'shards' => 38), + array('md5sum' => 'old1', 'shards' => 12), + array('md5sum' => 'old2', 'shards' => 3), + )); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + expect(flowview_get_shard_cache_summary())->toBe(array( + 'matching' => array('shards' => 78), + 'other' => array('shards' => 15, 'md5sums' => array('old1', 'old2')), + )); + + $lookup = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM parallel_database_query_shard_cache') !== false; + })); + + expect($lookup[0]['sql'])->toContain("map_partition != 'live'"); + }); + + it('removes only the cached shards that match no filter', function () use ($filter, $keys_of) { + $keys = $keys_of($filter); + + plugin_test_queue_db_result('db_fetch_assoc', array( + array('md5sum' => $keys['nat'][0], 'shards' => 40), + array('md5sum' => 'old1', 'shards' => 12), + array('md5sum' => 'old2', 'shards' => 3), + )); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + expect(flowview_prune_shard_cache())->toBe(15); + + $deleted = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared'; + })); + + expect($deleted)->toHaveCount(2) + ->and($deleted[0]['params'])->toBe(array('old1')) + ->and($deleted[1]['params'])->toBe(array('old2')) + ->and($deleted[0]['sql'])->toContain('DELETE FROM parallel_database_query_shard_cache') + ->and($deleted[0]['sql'])->toContain("map_partition != 'live'"); + }); + + it('removes nothing when there is nothing to prune', function () use ($filter, $keys_of) { + $keys = $keys_of($filter); + + plugin_test_queue_db_result('db_fetch_assoc', array(array('md5sum' => $keys['nat'][0], 'shards' => 40))); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + expect(flowview_prune_shard_cache())->toBe(0) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + }); + + it('removes nothing when the cache keys of the filters are not known', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('md5sum' => 'old1', 'shards' => 12))); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + + expect(flowview_prune_shard_cache())->toBeNull() + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + }); +}); + +describe('stopping the processes of a parallel query', function () { + beforeEach(function () { + plugin_test_reset(); + $GLOBALS['flowview_requests_in_flight'] = array(); + }); + + it('names signals without needing the pcntl constants', function () { + expect(flowview_signal_number('SIGTERM'))->toBe(15) + ->and(flowview_signal_number('SIGKILL'))->toBe(9) + ->and(flowview_signal_number('SIGINT'))->toBe(2) + ->and(flowview_signal_number('SIGNOTREAL'))->toBe(15); + }); + + it('does not signal something that is not a process', function () { + if (!function_exists('posix_kill')) { + $this->markTestSkipped('The posix extension is not available.'); + } + + expect(flowview_signal_process(0))->toBeFalse() + ->and(flowview_signal_process(-5))->toBeFalse() + ->and(flowview_signal_process(4999999))->toBeFalse(); + }); + + it('does not use a signal constant that the web server does not have in the clean up', function () { + $source = plugin_test_read_source('includes/functions.php'); + + expect($source)->not->toMatch('/posix_kill\([^)]*\bSIG[A-Z]+\b/'); + }); + + it('remembers a request until it is cleaned up', function () { + flowview_track_request(991); + flowview_track_request(992); + + parallel_database_query_cleanup(991); + + expect($GLOBALS['flowview_requests_in_flight'])->toBe(array(992 => 992)); + }); + + it('cancels the requests that are not finished, removing their rows', function () { + flowview_track_request(991); + flowview_track_request(992); + + expect(flowview_cancel_requests_in_flight())->toBe(array(991, 992)) + ->and($GLOBALS['flowview_requests_in_flight'])->toBe(array()); + + $removed = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && preg_match('/^DELETE FROM parallel_database_query\s+WHERE id = \?/', $call['sql']) === 1; + })); + + expect(array_column($removed, 'params'))->toBe(array(array(991), array(992))); + }); + + it('cancels nothing when no request is in flight', function () { + expect(flowview_cancel_requests_in_flight())->toBe(array()); + }); + + it('removes a finished request so the same query is run again', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', 'complete'); + + expect(flowview_discard_completed_request(991))->toBeTrue(); + + $removed = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && preg_match('/^DELETE FROM parallel_database_query\s+WHERE id = \?/', $call['sql']) === 1; + })); + + expect(array_column($removed, 'params'))->toBe(array(array(991))); + }); + + it('leaves a request that is still being run, as another process is using it', function ($status) { + plugin_test_queue_db_result('db_fetch_cell_prepared', $status); + + expect(flowview_discard_completed_request(991))->toBeFalse() + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + })->with(array('pending', 'scheduled', 'running', '')); + + it('prunes from a button with a confirmation dialog and a post, not from the actions', function () { + $source = plugin_test_read_source('flowview_filters.php'); + + expect($source)->not->toContain("6 => __('Prune Cached Shards'") + ->and($source)->not->toContain("drp_action') == '6'") + ->and($source)->not->toContain('function confirm_prune_shards') + ->and($source)->toContain("id='prune_dialog'") + ->and($source)->toContain("\$('#prune_dialog').dialog(") + ->and($source)->toMatch("/\\\$\('#prune_dialog'\)\.dialog\(\{\s*modal: false,/") + ->and($source)->toContain("\$.post('flowview_filters.php', data)") + ->and($source)->toMatch("/function prune_shards\(\): void \{.*?REQUEST_METHOD'\] \?\? 'GET'\) !== 'POST'/s") + ->and($source)->toMatch("/id='clear'[^\n]*\n\s*