diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c836c8d..5b9a53e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -41,7 +41,7 @@ flowview/ # Repository root (install to plugins/fl ├── flowview_schedules.php # Scheduled report administration ├── flowview_process.php # CLI: parallel query execution / report runner ├── flowview_cleanup.php # CLI: data retention cleanup -├── flowview_upgrade.php # Background schema upgrade runner +├── cli/flowview_upgrade.php # Background schema upgrade runner ├── flow_collector.php # NetFlow v5/v9/v10 collector daemon ├── functions.php / functions-pre13.php # Core UI + protocol decode helpers ├── import_flows.php # CLI flow file importer @@ -104,7 +104,7 @@ context (e.g. arithmetic, strict `===` type comparisons). Use `flowview_setup_table()`/`flowview_drop_table()` helpers in `setup.php` rather than ad hoc `CREATE TABLE`/`DROP TABLE` calls scattered across pages. ### Upgrade Handling -Schema upgrades run in the **background** via `flowview_upgrade.php` (spawned with `exec_background()` from `plugin_flowview_check_upgrade()`), since flow tables can be very large: +Schema upgrades run in the **background** via `cli/flowview_upgrade.php` (spawned with `exec_background()` from `plugin_flowview_check_upgrade()`), since flow tables can be very large: ```php function plugin_flowview_check_upgrade($force = false) { @@ -116,7 +116,7 @@ function plugin_flowview_check_upgrade($force = false) { if ($current != $old || $force) { $php_binary = read_config_option('path_php_binary'); - exec_background($php_binary, '-q ' . $config['base_path'] . '/plugins/flowview/flowview_upgrade.php'); + exec_background($php_binary, '-q ' . $config['base_path'] . '/plugins/flowview/cli/flowview_upgrade.php'); raise_message('flowview_upgrade', __('Please be advised the Flowview plugins Tables are being upgraded in the background...', 'flowview'), MESSAGE_LEVEL_INFO); } } diff --git a/CHANGELOG.md b/CHANGELOG.md index fde7b01..6824563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ --- develop --- +* feature: Add Cisco Apps (NBAR) reports for application-bearing NetFlow v9 exports, with IPv4/IPv6 application totals, conversations, paged connections, per-account capture/network options, recent-record status, first/last observation dates and a router setup guide +* feature: Maintain reusable per-capture NBAR hourly/daily summaries through the existing Cacti poller, with raw partial-boundary fallback and a separate retention setting capped at 30 days +* feature: Store raw NBAR records in time-sliced tables (plugin_flowview_nbar_data_[]) that follow the Database Partitioning Scheme setting, and expire them by dropping whole slices * fix: Make bundled Net_DNS2 constructor nullability explicit for PHP 8.4 without changing accepted arguments * issue: Shorten the graph drilldown button title from 'View NetFlow Traffic In Range' to 'NetFlow' so it reads cleanly as a label in Cacti's graph action menu diff --git a/README.md b/README.md index 8de558c..7c06b45 100644 --- a/README.md +++ b/README.md @@ -209,5 +209,91 @@ an issue. Got any ideas or complaints, please see the Cacti forums or GitHub for a resolution. + +## Cisco Apps (NBAR) + +Cisco Apps is a separate application report within FlowView. The current NBAR +integration reads application IDs and names from NetFlow v9 data and options +templates, including IPv4 and IPv6 records. IPFIX application decoding is not +implemented by this integration. Existing FlowView reports retain their normal +collector and report paths. + +Configure a FlowView listener for each export stream and configure the router to +export application-bearing records to it. The report's Setup guide contains a +Cisco 1100 Series example with placeholders for the collector, UDP port and router +interfaces. Check Cisco documentation for the exact platform and IOS XE release. +Creating a listener alone does not enable NBAR on the router. + +The report offers Top applications, Conversations and Connections. Filter by IP +matches either endpoint. Conversations requires the account's Internal Networks +settings to identify hosts and upload/download direction. Combine Options accepts +two or three captures; leave them all unselected to save networks without combining. +Choose one observation point per traffic path: overlapping LAN/WAN captures add +both copies and can overcount traffic. + +STATUS describes recent retained application records for each selected capture, +independently of historical report filters. A quiet capture can show no recent +NBAR despite a working exporter. It is not a connectivity test. + +First seen and Last seen are the earliest/latest retained observation minutes in +the selected period. They do not prove continuous activity, or give exact connection +start/end times. Missing dates are shown as Unavailable. Average bandwidth divides +data volume by the full selected period. Column sorting applies to the displayed +page; resizing is saved in that browser. + +NBAR History Retention is separate from the existing FlowView Data Retention Policy. +It offers 1, 3, 7, 14 or 30 days, with a seven-day default. Timespan choices and custom +ranges respect this setting. The existing Cacti poller maintains summaries and +performs bounded cleanup; no extra scheduled task is required. A shorter retention +setting cannot be undone to recover deleted history. If polling stops, maintenance +and cleanup stop too. Shared maintenance locks are scoped to the FlowView database. + +## Cisco Apps (NBAR) historical Connections + +Connections reads the automatically maintained daily history for each selected capture. +Opening a report or changing its IP filter does not queue a new complete connection build. +The normal FlowView poller maintains this history; no additional cron job is required. + +Pages contain up to 100 connections, ordered by total data volume across the selected +captures and period. Indexed candidates are checked against every selected daily segment +so byte, packet and record totals remain exact. Partial days use hourly or raw data. +Previous, Next and a numeric page field provide navigation without a large dropdown or +an expensive full-result count. Very deep pages or broadly tied results can still require +more work; a shorter period or IP filter reduces that work. + +A small account-scoped session cache holds nearby pages for 120 seconds for past periods +and 30 seconds for periods including current data. This is separate from retained history. +The existing authenticated FlowView realm controls access; capture selections and filters are included in cache keys. + +## Reusable NBAR daily summaries + +Top Applications, Conversations and Connections share daily summaries built automatically +per capture, independently of report filters and Cacti accounts. Calendar days run from +local midnight to the following midnight in the Cacti server timezone, including daylight +saving changes. Completed hours are added to today's summary in bounded transactions. +Completed days are reused; late exports invalidate affected summaries and trigger a rebuild. +Partial days and unpublished or changed intervals fall back to hourly summaries or raw records. +IP, application and IP-version filters are applied to the preserved summary fields. + +Daily history follows Cisco Apps (NBAR) History Retention, currently selectable up to +30 days. Retention is a rolling data window, not a new retention period beginning each time +a report is opened. Daily summaries are shared source data and are reused for different +periods, filters and users, subject to listener permissions. Connections reads pages from +this history rather than preparing another full copy for every selection. + ----------------------------------------------- Copyright (c) 2004-2026 - The Cacti Group, Inc. + +NBAR daily summary materialization and observation-date backfill require MariaDB statement deadlines. On MySQL 8, these workers safely defer their DML rather than running without a bound; reports continue through hourly/raw sources. Exact counts use the MySQL SELECT execution-time hint. + +New NBAR records retain the actual exporter address through hourly and daily summaries. Legacy rows keep their counters with an unknown exporter; an exact single-address listener may still resolve their names, but earlier multi-exporter rows cannot be attributed retrospectively. + +NBAR maintenance is launched independently of the poller. A registered supervisor runs the PHP maintenance child with a 90-second process deadline. MariaDB hourly mutations also set and restore a verified 20-second statement deadline. MySQL hourly builds remain available only inside the supervised child; raw records remain the fallback until a checkpoint is published. + +NBAR timespan requests keep their session state separate from Flow Reports. Conversation totals exclude IPv4 and IPv6 multicast endpoints, and feed lookup failures are shown separately from empty captures. Retention maintenance drains additional batches across raw, hourly and daily tables within a shared ten-second cleanup budget, under the supervised worker deadline. + +Template decoding rejects zero-length fields and invalidates the affected cached template until a valid refresh arrives. Chart fallback labels are grouped by their complete expression, so applications without learned names keep separate slices. + +The NBAR template cache retains at most 256 templates / 8,192 field definitions in total and 64 templates / 2,048 fields per exporter, across all source IDs. Least recently used entries are evicted and inactive templates expire after 30 minutes (removed lazily on packet activity). Evicted templates are learned again from the next exporter refresh. Numeric data fields must fit unsigned 64-bit values; options scope fields are decoded separately and are not treated as counters. + +NBAR maintenance supervision uses PHP's direct argv-array process API rather than Cacti's version-dependent timeout command interface, so startup and hard child termination work on the minimum supported Cacti 1.2.29 and newer releases. Both output pipes are drained in bounded chunks. diff --git a/flowview_upgrade.php b/cli/flowview_upgrade.php similarity index 94% rename from flowview_upgrade.php rename to cli/flowview_upgrade.php index a0e8224..0c452e8 100644 --- a/flowview_upgrade.php +++ b/cli/flowview_upgrade.php @@ -23,11 +23,13 @@ +-------------------------------------------------------------------------+ */ -chdir('../../'); +chdir(__DIR__ . '/../../../'); require('./include/cli_check.php'); +require_once('./lib/poller.php'); require_once('./plugins/flowview/includes/functions.php'); require_once('./plugins/flowview/setup.php'); require_once('./plugins/flowview/includes/database.php'); +require_once('./plugins/flowview/includes/nbar_common.php'); flowview_connect(); @@ -38,6 +40,7 @@ array_shift($parms); $forcever = ''; +$nbar_worker = ''; if (cacti_sizeof($parms)) { foreach($parms as $parameter) { @@ -49,6 +52,13 @@ } switch ($arg) { + case '--nbar-worker': + if (!in_array($value, ['supervise', 'build'], true)) { + exit(1); + } + $nbar_worker = $value; + + break; case '--forcever': $forcever = $value; break; @@ -71,6 +81,17 @@ } +// Internal maintenance modes never enter the schema-upgrade path. +if ($nbar_worker !== '') { + if ($nbar_worker === 'supervise') { + fv_nbar_supervise_maintenance(); + } elseif ((int) read_config_option('flowview_nbar_schema_version') >= 2) { + fv_nbar_cleanup(); + fv_nbar_summary_refresh(20, true); + } + exit(0); +} + /* we need to rerun the upgrade, force the current version */ if ($forcever == '') { $old_version = db_fetch_cell('SELECT version FROM plugin_config WHERE directory = "flowview"'); @@ -82,24 +103,37 @@ $current_version = $info['version']; /* do a version check */ -if ($forcever == '' && $old_version == $current_version) { +if ($forcever == '' && $old_version == $current_version && (int) read_config_option('flowview_nbar_schema_version') >= 2) { cacti_log('Your Flowview is already up to date (v' . $current_version . ' vs v' . $old_version . ') not upgrading. Use --forcever to override', true, 'FLOWVIEW'); + + unregister_process('flowview', 'upgrade_launch', 0); + exit(0); } -if (!register_process_start('flowview', 'upgrade', 1, 86400)) { +if (!register_process_start('flowview', 'upgrade', 1, 14400)) { if (empty($forcever)) { print 'WARNING: Running process detected. To override use --forcever' . PHP_EOL; exit(1); } } +// The launcher row holds the web worker's PID. Hand it to this process so a timeout recovery can only +// ever signal this upgrade, never a PHP-FPM worker that has since been reused for another request. +if (db_table_exists('processes')) { + db_execute_prepared('UPDATE processes SET pid = ? WHERE tasktype = ? AND taskname = ? AND taskid = ?', [getmypid(), 'flowview', 'upgrade_launch', 0]); +} + +// release this run and the web launcher's lock however the script ends, including a fatal error +register_shutdown_function(static function (): void { + unregister_process('flowview', 'upgrade', 1); + unregister_process('flowview', 'upgrade_launch', 0); +}); + cacti_log('Upgrading from v' . $old_version . ' to ' . $current_version, true, 'FLOWVIEW'); flowview_upgrade($current_version, $old_version); -unregister_process('flowview', 'upgrade', 1); - exit(0); /** @@ -119,6 +153,11 @@ * @return void */ function flowview_upgrade($current, $old): void { + if (!flowview_db_create_nbar_tables()) { + throw new RuntimeException('Unable to upgrade NBAR history schema'); + } + set_config_option('flowview_nbar_schema_version', '2'); + global $config, $info; if ($current != $old) { diff --git a/css/nbar.css b/css/nbar.css new file mode 100644 index 0000000..16a12df --- /dev/null +++ b/css/nbar.css @@ -0,0 +1,288 @@ +/* Copyright (C) 2004-2026 The Cacti Group. GPL-2.0-or-later. */ +#nbarTable th .ui-resizable-handle, +#nbarConversationTable th .ui-resizable-handle { + display: none !important; + pointer-events: none !important; +} +.flowview-nbar-table-wrap { + position: relative; + width: 100%; + max-width: 100%; + min-width: 0; + /* Saved column widths must scroll here instead of widening the report bar. */ + contain: inline-size; + overflow-x: auto; + display: block; +} +.flowview-nbar-resize { + display: block !important; + z-index: 10; +} +.flowview-nbar-resize .tablesorter-resizable-handle { + width: 12px !important; + pointer-events: auto; +} +.flowview-nbar-resize .tablesorter-resizable-handle:before { + content: ''; + position: absolute; + top: 6px; + left: 4px; + height: 14px; + border-left: 1px solid currentColor; + border-right: 1px solid currentColor; + width: 2px; + opacity: 0.35; +} +.flowview-nbar-resize .tablesorter-resizable-handle:hover { + background: rgba(127, 127, 127, 0.25); +} +.nbarSettingsRow { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + gap: 8px 12px; + padding: 4px; +} +.nbarSetting { + display: flex; + align-items: center; + gap: 6px; +} +.nbarSettingsRow > details { + max-width: 100%; + min-width: 0; +} +.nbarSettingsRow > details > div { + width: 400px; + max-width: 100%; + box-sizing: border-box; + white-space: normal; + overflow-wrap: anywhere; +} +.nbarSettingsRow textarea { + width: 100%; + max-width: 100%; + box-sizing: border-box; +} +.nbarSettingsRow summary { + cursor: pointer; +} +.nbarChart { + max-width: 100%; +} +.nbarReportBar { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + gap: 12px; +} +.nbarReportBar > form { + flex: 1 1 660px; + min-width: 0; + margin: 0; +} +.nbarChart { + flex: 0 1 340px; + padding: 4px 12px; + box-sizing: border-box; + margin-right: 40px; +} +.nbarChartBody { + display: flex; + align-items: center; + gap: 14px; + margin-top: 6px; +} +.nbarChartPie { + width: 112px; + height: 112px; + border-radius: 50%; + flex-shrink: 0; +} +.nbarChart ul { + list-style: none; + margin: 0; + padding: 0; + flex: 1; + min-width: 0; +} +.nbarChart li { + display: flex; + align-items: center; + gap: 6px; + padding: 0; + line-height: 1.35; + white-space: nowrap; +} +.nbarChartKey { + width: 9px; + height: 9px; + flex-shrink: 0; +} +.nbarChartName { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; +} +@media (max-width: 1100px) { + .nbarChart { + flex-basis: 100%; + padding-left: 4px; + margin-right: 0; + } + .nbarChartBody { + max-width: 340px; + } +} +#nbarTable, +#nbarConversationTable { + table-layout: auto; + width: 100%; +} +#nbarTable th, +#nbarTable td, +#nbarConversationTable th, +#nbarConversationTable td { + white-space: normal !important; + overflow-wrap: anywhere; +} +#nbarTable th .textSubHeaderDark, +#nbarConversationTable th .textSubHeaderDark { + white-space: normal !important; +} +/* Observation dates must never wrap one character per line in a narrow column. */ +#nbarTable th:nth-last-child(-n + 2), +#nbarTable td:nth-last-child(-n + 2), +#nbarConversationTable th:nth-last-child(-n + 2), +#nbarConversationTable td:nth-last-child(-n + 2) { + min-width: 136px; + white-space: nowrap !important; + overflow-wrap: normal; +} +#nbarTable th:nth-last-child(-n + 2) .textSubHeaderDark, +#nbarConversationTable th:nth-last-child(-n + 2) .textSubHeaderDark { + white-space: nowrap !important; +} +#nbarCombineDialog { + width: min(640px, 90vw); + max-height: 85vh; + padding: 0; + position: fixed; + margin: auto; + box-sizing: border-box; + overflow: hidden; + font-weight: normal; +} +#nbarCombineDialog::backdrop { + background: rgba(0, 0, 0, 0.4); +} +#nbarCombineDialog .nbarCombineHeader { + display: flex; + justify-content: space-between; + align-items: center; + gap: 16px; + padding: 12px 16px; +} +#nbarCombineDialog .nbarCombineBody { + padding: 16px; + max-height: calc(85vh - 75px); + overflow: auto; + box-sizing: border-box; + font-weight: normal; + line-height: 1.45; +} +#nbarCombineDialog textarea { + width: 100%; + box-sizing: border-box; + max-width: 100%; +} +#nbarCombineDialog h3 { + margin: 0 0 8px; +} +#nbarCombineDialog p, +#nbarCombineDialog small { + font-weight: normal; + font-size: inherit; +} +#nbarSetupGuide { + width: min(900px, 90vw); + max-width: 90vw; + max-height: 85vh; + padding: 0; + position: fixed; + margin: auto; + box-sizing: border-box; + overflow: hidden; + font-weight: normal; +} +#nbarSetupGuide::backdrop { + background: rgba(0, 0, 0, 0.4); +} +#nbarSetupGuide .nbarGuideHeader { + display: flex; + justify-content: space-between; + align-items: center; + gap: 16px; + padding: 12px 16px; + box-sizing: border-box; +} +#nbarSetupGuide .nbarGuideBody { + padding: 16px; + max-height: calc(85vh - 75px); + overflow-y: auto; + overflow-x: hidden; + box-sizing: border-box; + min-width: 0; + font-weight: normal; + line-height: 1.45; +} +#nbarSetupGuide .nbarGuideBody p, +#nbarSetupGuide .nbarGuideBody small { + font-weight: normal; +} +#nbarSetupGuide h3 { + font-size: 1.15em; + margin: 0 0 0.8em; + font-weight: bold; +} +#nbarSetupGuide h3:not(:first-child) { + margin-top: 1.3em; +} +#nbarSetupGuide p { + margin: 0 0 1em; + overflow-wrap: anywhere; +} +#nbarSetupGuide pre { + padding: 12px; + width: 100%; + max-width: 100%; + box-sizing: border-box; + overflow: auto; + white-space: pre; + font-weight: normal; + font-family: monospace; + line-height: 1.3; +} +#nbarSetupGuide .nbarGuideHeader button { + flex-shrink: 0; +} +#fv_section_tabs { + padding-bottom: 0; +} +#fv_section_tabs + table { + margin-top: 0; +} +#fv_section_tabs a[aria-current='page'] { + font-weight: bold; + text-decoration: none; +} +#fv_section_tabs a:focus-visible { + outline: 2px solid currentColor; + outline-offset: 2px; +} + +/* Use the theme foreground while retaining the surrounding report text size. */ +.nbarReportText { + font-size: inherit; + font-weight: normal; +} diff --git a/flow_collector.php b/flow_collector.php index 0759e8a..13c0b9f 100644 --- a/flow_collector.php +++ b/flow_collector.php @@ -43,6 +43,7 @@ require_once($config['base_path'] . '/lib/poller.php'); require_once($config['base_path'] . '/plugins/flowview/setup.php'); require_once($config['base_path'] . '/plugins/flowview/includes/functions.php'); +require_once($config['base_path'] . '/plugins/flowview/includes/nbar_collector.php'); chdir(__DIR__); ini_set('max_execution_time', '-1'); @@ -1275,6 +1276,7 @@ function process_fv9($p, $ex_addr): void { global $templates, $tlengths, $allfields, $pacmap, $listener_id; flowview_connect(); + fv_nbar_packet($p, $ex_addr); if (!isset($templates[$ex_addr])) { $templates[$ex_addr] = []; @@ -1422,6 +1424,10 @@ function process_fv9($p, $ex_addr): void { $field = substr($p, $h, $t['length']); + // Decode Cisco reduced-length three-byte unsigned integers. + if ($t['unpack'] === 'N' && $t['length'] == 3 && strlen($field) == 3) { + $field = chr(0) . $field; + } $field = unpack($t['unpack'], $field) ?: []; if ($t['pack'] == 'ipv4Address') { @@ -1729,6 +1735,11 @@ function process_fv10($p, $ex_addr): void { $id = $t['field_id']; $field = substr($p, $h, $t['length']); + + // Decode Cisco reduced-length three-byte unsigned integers. + if ($t['unpack'] === 'N' && $t['length'] == 3 && strlen($field) == 3) { + $field = chr(0) . $field; + } $field = unpack($t['unpack'], $field) ?: []; if ($t['pack'] == 'ipv4Address') { diff --git a/flowview.php b/flowview.php index 0a2a903..5320876 100644 --- a/flowview.php +++ b/flowview.php @@ -31,10 +31,25 @@ global $config; require_once($config['base_path'] . '/plugins/flowview/setup.php'); require_once($config['base_path'] . '/plugins/flowview/includes/functions.php'); +require_once($config['base_path'] . '/plugins/flowview/includes/nbar_common.php'); +require_once($config['base_path'] . '/plugins/flowview/includes/nbar_count.php'); +require_once($config['base_path'] . '/plugins/flowview/includes/nbar_timespan.php'); +require_once($config['base_path'] . '/plugins/flowview/flowview_sections.php'); require_once($config['base_path'] . '/lib/time.php'); flowview_connect(); +if (get_nfilter_request_var('action') === 'nbar_count') { + fv_nbar_count_request(); +} + +if (get_nfilter_request_var('action') === 'applications' || + (get_nfilter_request_var('action') === 'view' && get_nfilter_request_var('report') === 'nbar')) { + general_header(); + include($config['base_path'] . '/plugins/flowview/nbar_report.php'); + exit; +} + set_default_action(); ini_set('max_execution_time', 240); @@ -78,6 +93,10 @@ break; case 'gettimespan': + if (fv_nbar_timespan_request()) { + break; + } + flowview_request_vars(); flowview_gettimespan(); @@ -88,6 +107,8 @@ default: general_header(); + fv_section_save('flows', fv_section_request_params()); + fv_section_navigation('flows'); flowview_request_vars(); $title = load_session_for_filter(); @@ -153,7 +174,7 @@ function save_filter_as(): void { WHERE id = ?', array(get_request_var('query'))); - if (cacti_sizeof($save)) { + if (is_array($save) && cacti_sizeof($save)) { $save['name'] = get_request_var('sname'); if (isset_request_var('report')) { diff --git a/flowview_devices.php b/flowview_devices.php index bf52069..6e2a0c9 100644 --- a/flowview_devices.php +++ b/flowview_devices.php @@ -184,7 +184,7 @@ function export_template(): void { exit; } else { - raise_message_javascript('notemplate', __('No Flow Template was found for Device ID:%s and Template ID:%s', $device_id, $template_id, 'flowview'), MESSAGE_LEVEL_ERROR); + raise_message_javascript('notemplate', __('No Flow Template was found for Device ID:%s and Template ID:%s', $device_id, $template_id, 'flowview'), '', MESSAGE_LEVEL_ERROR); exit; } diff --git a/flowview_schedules.php b/flowview_schedules.php index 77b3ea7..996657f 100644 --- a/flowview_schedules.php +++ b/flowview_schedules.php @@ -364,7 +364,7 @@ function view_log_data(): void { $log_data = db_fetch_row_prepared('SELECT * FROM reports_log WHERE id = ?', [$id]); - if (cacti_sizeof($log_data)) { + if (is_array($log_data) && cacti_sizeof($log_data)) { $data = json_decode($log_data['report_raw_data'], true); $columns = array_keys($data[0]); @@ -390,7 +390,7 @@ function download_log_data(): void { $log_data = db_fetch_row_prepared('SELECT * FROM reports_log WHERE id = ?', [$id]); - if (cacti_sizeof($log_data)) { + if (is_array($log_data) && cacti_sizeof($log_data)) { $data = json_decode($log_data['report_raw_data'], true); $columns = array_keys($data[0]); diff --git a/flowview_sections.php b/flowview_sections.php new file mode 100644 index 0000000..7f05718 --- /dev/null +++ b/flowview_sections.php @@ -0,0 +1,155 @@ + $v) { + if (!is_scalar($v)) { + unset($p[$k]); + } + } + $p['predefined_timespan'] = 1; // Re-enter reports with a fresh half-hour window. + + unset($p['date1'],$p['date2']); + $c = json_decode($p['context'] ?? '', true); + + if (is_array($c)) { + unset($c['date1'],$c['date2']); + $p['context'] = json_encode($c); + } + $p['action'] = 'view'; + $p['section'] = $section; + + if ($section === 'nbar') { + $p['report'] = 'nbar'; + } elseif (($p['report'] ?? '') === 'nbar') { + unset($p['report']); + } + + return $p; +} +/** + * Build a local URL restoring one report section. + * + * @param string $section Report section identifier. + * + * @return string + */ +function fv_section_url(string $section): string { + return 'flowview.php?' . http_build_query(fv_section_params($section)); +} +/** + * Persist account-specific section navigation without drilldown addresses. + * + * @param string $section Report section identifier. + * @param array $params Scalar navigation fields. + * + * @return void + */ +function fv_section_save(string $section, array $params): void { + unset($params['app'],$params['internal'],$params['external'],$params['section']); + $value = json_encode($params); + + if (read_user_setting('flowview_last_section', '', true) !== $section) { + set_user_setting('flowview_last_section', $section); + } + + if (read_user_setting('flowview_section_' . $section, '', true) !== $value) { + set_user_setting('flowview_section_' . $section, $value); + } +} +/** + * Render theme-aware navigation between Flow Reports and Cisco Apps. + * + * @param string $selected Currently selected report section. + * + * @return void + */ +function fv_section_navigation(string $selected): void { + global $config; + print ''; + print ''; +} +$section_entry = array_diff_key(fv_section_request_params(), array_flip(['headercontent'])); + +if (($_SERVER['REQUEST_METHOD'] ?? '') === 'GET' && (!$section_entry || (cacti_sizeof($section_entry) === 1 && ($section_entry['action'] ?? '') === 'view'))) { + $last = read_user_setting('flowview_last_section', '', true); + + if (in_array($last, ['flows', 'nbar'], true)) { + $restore = fv_section_url($last); + + if (get_nfilter_request_var('header') === 'false') { + $restore .= '&header=false'; + } + + if (isset_request_var('headercontent')) { + $restore .= '&headercontent=true'; + } + header('Location: ' . $restore); + exit; + } +} + +if (get_nfilter_request_var('section') === 'nbar') { + set_request_var('action', 'view'); + set_request_var('report', 'nbar'); +} diff --git a/flowview_upgrade_nat_columns.php b/flowview_upgrade_nat_columns.php index 8838a61..13a0e03 100644 --- a/flowview_upgrade_nat_columns.php +++ b/flowview_upgrade_nat_columns.php @@ -29,7 +29,7 @@ * 'plugin_flowview_raw_*' partition tables. * * This is deliberately a standalone CLI rather than part of the automatic - * background schema upgrade (flowview_upgrade.php): a busy install can have + * background schema upgrade (cli/flowview_upgrade.php): a busy install can have * hundreds of large raw partition tables, and altering all of them can take * a long time. Run this manually (or on a schedule) after upgrading the * plugin to backfill existing partitions; new partitions created going diff --git a/includes/database.php b/includes/database.php index b457a30..100a802 100644 --- a/includes/database.php +++ b/includes/database.php @@ -506,7 +506,7 @@ function flowview_db_get_table_column_types($table, mixed $cnn_id = false) { $columns = db_fetch_assoc("SHOW COLUMNS FROM $table", false, $flowview_cnn); $cols = []; - if (cacti_sizeof($columns)) { + if (is_array($columns) && cacti_sizeof($columns)) { foreach($columns as $col) { $cols[$col['Field']] = ['type' => $col['Type'], 'null' => $col['Null'], 'default' => $col['Default'], 'extra' => $col['Extra']]; } @@ -514,3 +514,218 @@ function flowview_db_get_table_column_types($table, mixed $cnn_id = false) { return $cols; } + +/** + * flowview_db_create_nbar_tables - creates the Cisco Apps (NBAR) tables if they do not already exist + * + * The application names, the hourly and daily summaries, the daily coverage + * checkpoints and the observation date backfill checkpoints. The daily summary + * tables store the start of the day in their hour column. The raw per-minute + * records are not created here: they live in time slices that are created as + * records arrive, see flowview_db_create_nbar_data_partition(). Every table is + * checked after it is created, so a failure is reported instead of surfacing + * later as a missing table. + * + * @return bool true when every table exists + */ +function flowview_db_create_nbar_tables(): bool { + $tables = [ + 'plugin_flowview_nbar_names' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_names` ( + `router` varchar(45) NOT NULL, + `app_id` varchar(32) NOT NULL, + `name` varchar(255) NOT NULL, + `updated` int(10) unsigned NOT NULL, + PRIMARY KEY (`router`,`app_id`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR application names learned from exporter options templates'", + + 'plugin_flowview_nbar_hours' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_hours` ( + `listener_id` int(10) unsigned NOT NULL, + `hour` int(10) unsigned NOT NULL, + `revision` bigint(20) unsigned NOT NULL DEFAULT 0, + `built` bigint(20) DEFAULT -1, + `connection_built` bigint(20) DEFAULT -1, + PRIMARY KEY (`listener_id`,`hour`), + KEY `hour` (`hour`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR hourly summary build state per capture'", + + 'plugin_flowview_nbar_apps' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_apps` ( + `listener_id` int(10) unsigned NOT NULL, + `hour` int(10) unsigned NOT NULL, + `app_id` varchar(32) NOT NULL, + `family` tinyint(3) unsigned NOT NULL, + `bytes` bigint(20) unsigned NOT NULL, + `packets` bigint(20) unsigned NOT NULL, + `flows` bigint(20) unsigned NOT NULL, + `first_seen` int(10) unsigned DEFAULT NULL, + `last_seen` int(10) unsigned DEFAULT NULL, + `router` varchar(45) NOT NULL DEFAULT '', + PRIMARY KEY (`listener_id`,`hour`,`app_id`,`family`,`router`), + KEY `hour` (`hour`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR hourly traffic per application'", + + 'plugin_flowview_nbar_pairs' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_pairs` ( + `listener_id` int(10) unsigned NOT NULL, + `hour` int(10) unsigned NOT NULL, + `app_id` varchar(32) NOT NULL, + `src` varbinary(16) NOT NULL, + `dst` varbinary(16) NOT NULL, + `bytes` bigint(20) unsigned NOT NULL, + `packets` bigint(20) unsigned NOT NULL, + `flows` bigint(20) unsigned NOT NULL, + `first_seen` int(10) unsigned DEFAULT NULL, + `last_seen` int(10) unsigned DEFAULT NULL, + `router` varchar(45) NOT NULL DEFAULT '', + PRIMARY KEY (`listener_id`,`hour`,`app_id`,`src`,`dst`,`router`), + KEY `hour` (`hour`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR hourly traffic per application and endpoint pair'", + + 'plugin_flowview_nbar_connections' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_connections` ( + `listener_id` int(10) unsigned NOT NULL, + `hour` int(10) unsigned NOT NULL, + `app_id` varchar(32) NOT NULL, + `src` varbinary(16) NOT NULL, + `dst` varbinary(16) NOT NULL, + `sport` smallint(5) unsigned NOT NULL, + `dport` smallint(5) unsigned NOT NULL, + `protocol` tinyint(3) unsigned NOT NULL, + `bytes` bigint(20) unsigned NOT NULL, + `packets` bigint(20) unsigned NOT NULL, + `flows` bigint(20) unsigned NOT NULL, + `first_seen` int(10) unsigned DEFAULT NULL, + `last_seen` int(10) unsigned DEFAULT NULL, + `router` varchar(45) NOT NULL DEFAULT '', + PRIMARY KEY (`listener_id`,`hour`,`app_id`,`src`,`dst`,`sport`,`dport`,`protocol`,`router`), + KEY `hour` (`hour`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR hourly traffic per application connection'", + + 'plugin_flowview_nbar_daily_apps' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_daily_apps` ( + `listener_id` int(10) unsigned NOT NULL, + `hour` int(10) unsigned NOT NULL, + `app_id` varchar(32) NOT NULL, + `family` tinyint(3) unsigned NOT NULL, + `bytes` bigint(20) unsigned NOT NULL, + `packets` bigint(20) unsigned NOT NULL, + `flows` bigint(20) unsigned NOT NULL, + `first_seen` int(10) unsigned DEFAULT NULL, + `last_seen` int(10) unsigned DEFAULT NULL, + `router` varchar(45) NOT NULL DEFAULT '', + PRIMARY KEY (`listener_id`,`hour`,`app_id`,`family`,`router`), + KEY `hour` (`hour`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR daily traffic per application'", + + 'plugin_flowview_nbar_daily_pairs' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_daily_pairs` ( + `listener_id` int(10) unsigned NOT NULL, + `hour` int(10) unsigned NOT NULL, + `app_id` varchar(32) NOT NULL, + `src` varbinary(16) NOT NULL, + `dst` varbinary(16) NOT NULL, + `bytes` bigint(20) unsigned NOT NULL, + `packets` bigint(20) unsigned NOT NULL, + `flows` bigint(20) unsigned NOT NULL, + `first_seen` int(10) unsigned DEFAULT NULL, + `last_seen` int(10) unsigned DEFAULT NULL, + `router` varchar(45) NOT NULL DEFAULT '', + PRIMARY KEY (`listener_id`,`hour`,`app_id`,`src`,`dst`,`router`), + KEY `hour` (`hour`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR daily traffic per application and endpoint pair'", + + 'plugin_flowview_nbar_daily_connections' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_daily_connections` ( + `listener_id` int(10) unsigned NOT NULL, + `hour` int(10) unsigned NOT NULL, + `app_id` varchar(32) NOT NULL, + `src` varbinary(16) NOT NULL, + `dst` varbinary(16) NOT NULL, + `sport` smallint(5) unsigned NOT NULL, + `dport` smallint(5) unsigned NOT NULL, + `protocol` tinyint(3) unsigned NOT NULL, + `bytes` bigint(20) unsigned NOT NULL, + `packets` bigint(20) unsigned NOT NULL, + `flows` bigint(20) unsigned NOT NULL, + `first_seen` int(10) unsigned DEFAULT NULL, + `last_seen` int(10) unsigned DEFAULT NULL, + `router` varchar(45) NOT NULL DEFAULT '', + PRIMARY KEY (`listener_id`,`hour`,`app_id`,`src`,`dst`,`sport`,`dport`,`protocol`,`router`), + KEY `hour` (`hour`), + KEY `capture_day_bytes` (`listener_id`,`hour`,`bytes`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR daily traffic per application connection'", + + 'plugin_flowview_nbar_days' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_days` ( + `listener_id` int(10) unsigned NOT NULL, + `day` int(10) unsigned NOT NULL, + `covered_to` int(10) unsigned NOT NULL, + `revision` decimal(30,0) NOT NULL, + `timezone` varchar(64) CHARACTER SET ascii NOT NULL, + `updated` int(10) unsigned NOT NULL, + PRIMARY KEY (`listener_id`,`day`), + KEY `day` (`day`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR daily summary coverage checkpoints'", + + 'plugin_flowview_nbar_seen_upgrade' => "CREATE TABLE IF NOT EXISTS `plugin_flowview_nbar_seen_upgrade` ( + `kind` tinyint(3) unsigned NOT NULL, + `listener_id` int(10) unsigned NOT NULL, + `period` int(10) unsigned NOT NULL, + `done` tinyint(3) unsigned NOT NULL DEFAULT 0, + `phase` tinyint(3) unsigned NOT NULL DEFAULT 0, + `app_cursor` varchar(32) DEFAULT NULL, + PRIMARY KEY (`kind`,`listener_id`,`period`), + KEY `pending` (`done`,`kind`,`period`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR observation date backfill checkpoints'", + ]; + + foreach ($tables as $table => $sql) { + flowview_db_execute($sql); + + if (flowview_db_fetch_cell("SHOW TABLES LIKE '$table'") !== $table) { + return false; + } + } + + return true; +} + +/** + * flowview_db_create_nbar_data_partition - creates one time slice of the raw NBAR records if it does not already exist + * + * The raw records are partitioned the way the raw flow tables are, one table per hour or per + * day, named plugin_flowview_nbar_data_[]. The table name is checked + * before it is used in the statement. + * + * @param string $table The slice table name. + * + * @return bool true when the slice exists + */ +function flowview_db_create_nbar_data_partition(string $table): bool { + if (!preg_match('/^plugin_flowview_nbar_data_[0-9]{7}([0-9]{2})?$/D', $table)) { + return false; + } + + flowview_db_execute("CREATE TABLE IF NOT EXISTS `$table` ( + `listener_id` int(10) unsigned NOT NULL, + `minute` int(10) unsigned NOT NULL, + `app_id` varchar(32) NOT NULL, + `src` varbinary(16) NOT NULL, + `dst` varbinary(16) NOT NULL, + `sport` smallint(5) unsigned NOT NULL, + `dport` smallint(5) unsigned NOT NULL, + `protocol` tinyint(3) unsigned NOT NULL, + `bytes` bigint(20) unsigned NOT NULL, + `packets` bigint(20) unsigned NOT NULL, + `flows` bigint(20) unsigned NOT NULL, + `router` varchar(45) NOT NULL DEFAULT '', + PRIMARY KEY (`listener_id`,`minute`,`app_id`,`src`,`dst`,`sport`,`dport`,`protocol`,`router`), + KEY `minute` (`minute`)) + ENGINE=InnoDB + COMMENT='Plugin Flowview - NBAR raw per-minute application flow records'"); + + return flowview_db_fetch_cell("SHOW TABLES LIKE '$table'") === $table; +} diff --git a/includes/functions-pre13.php b/includes/functions-pre13.php index 1a5f1c5..316fc50 100644 --- a/includes/functions-pre13.php +++ b/includes/functions-pre13.php @@ -40,7 +40,7 @@ * @param array $raw_data Reference, the report's raw result data. * @param mixed $oput_raw Reference, the raw rendered output. * @param string $oput_html Reference, the HTML-rendered output body. - * @param string|null $oput_text Reference, the plain-text output body (defaulted to '' if null). + * @param string $oput_text Reference, the plain-text output body. * @param array $attachments Optional list of file attachments to include. * @param array $headers Optional extra email headers to include. * @@ -69,7 +69,7 @@ function reports_log_and_notify($id, $start_time, $report_type, $source, $source $from[0] = $fromemail; $from[1] = $fromname; - if (cacti_sizeof($report)) { + if (is_array($report) && cacti_sizeof($report)) { if ($report['notification'] != '') { $notifications = json_decode($report['notification'], true); @@ -114,9 +114,9 @@ function reports_log_and_notify($id, $start_time, $report_type, $source, $source WHERE id = ?', [$data['id']]); - if (cacti_sizeof($list)) { + if (is_array($list) && cacti_sizeof($list)) { /* process the format file */ - $report_tag = ''; + $report_tag = false; $theme = 'modern'; $output_html = ''; $format = $list['format_file'] != '' ? $list['format_file']:'default'; @@ -269,7 +269,7 @@ function reports_run($id): bool { WHERE id = ?', [$id]); - if (cacti_sizeof($report)) { + if (is_array($report) && cacti_sizeof($report)) { db_execute_prepared('UPDATE reports_queued SET status = ?, start_time = ? WHERE id = ?', diff --git a/includes/functions.php b/includes/functions.php index bbe9f3f..f6702cc 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -944,12 +944,16 @@ function flowview_display_filter(): void { '> - > - + + > + + '> - > - + + > + + @@ -1037,20 +1041,28 @@ function flowview_display_filter(): void { - > - + + > + + - > - + + > + + - > - + + > + + - > - + + > + + @@ -1968,7 +1980,7 @@ function plugin_flowview_run_schedule($id, $report_id): void { } /* process the format file */ - $report_tag = ''; + $report_tag = false; $theme = 'modern'; $body_html = ''; $format = $schedule['format_file'] != '' ? $schedule['format_file']:'default'; @@ -2832,7 +2844,7 @@ function run_flow_query($session, $query_id, $start, $end) { $data['printed'] = $nreport; $data['statistics'] = 0; } - } elseif (!sizeof($data)) { + } elseif (!cacti_sizeof($data)) { return false; } @@ -4217,7 +4229,7 @@ function parallel_database_query_run($requests) { } if ($total_time >= $max_time) { - raise_message_javascript('parallel_query_timeout', __('The Parallel Query Timed Out. Please contact your administrator', 'flowview'), MESSAGE_LEVEL_ERROR); + raise_message_javascript('parallel_query_timeout', __('The Parallel Query Timed Out. Please contact your administrator', 'flowview'), '', MESSAGE_LEVEL_ERROR); foreach($requests as $request_id) { parallel_database_query_cleanup($request_id); diff --git a/includes/nbar_chart.php b/includes/nbar_chart.php new file mode 100644 index 0000000..7df2337 --- /dev/null +++ b/includes/nbar_chart.php @@ -0,0 +1,99 @@ +=? AND d.minutegetMessage(), false, 'FLOWVIEW'); + print ''; + + return; + } + + if ($host !== '' && filter_var($host, FILTER_VALIDATE_IP)) { + $where .= ' AND (d.src=INET6_ATON(?) OR d.dst=INET6_ATON(?))'; + array_push($args, $host, $host); + } + $rows = fv_nbar_query_cached( + "SELECT COALESCE(NULLIF(n.name,''),CONCAT('Application ',d.app_id)) name,SUM(d.bytes) bytes FROM (SELECT d.listener_id,d.app_id,d.router,SUM(d.bytes) bytes FROM " . $source . ' d WHERE ' . $where . ' GROUP BY d.listener_id,d.app_id,d.router) d JOIN plugin_flowview_devices l ON l.id=d.listener_id LEFT JOIN plugin_flowview_nbar_names n ON n.router=COALESCE(NULLIF(d.router,SPACE(0)),l.allowfrom) AND n.app_id=d.app_id GROUP BY COALESCE(NULLIF(n.name,\'\'),CONCAT(\'Application \',d.app_id)) ORDER BY bytes DESC', + $args + ); + + if (!is_array($rows)) { + print ''; + + return; + } + $total = array_sum(array_column($rows, 'bytes')); + $items = array_slice($rows, 0, 5); + $other = $total - array_sum(array_column($items, 'bytes')); + + if ($other > 0) { + $items[] = ['name' => __('Other', 'flowview'), 'bytes' => $other]; + } + $colors = ['#4c9bd6', '#73ae53', '#e2a545', '#a780c5', '#de7970', '#91979d']; + $stops = []; + $legend = ''; + $position = 0; + + foreach ($items as $i => $item) { + $bytes = (float) $item['bytes']; + + if ($bytes <= 0) { + continue; + } + $next = min(100, $position + $bytes / $total * 100); + $stops[] = $colors[$i] . ' ' . round($position, 5) . '% ' . round($next, 5) . '%'; + $position = $next; + $value = $bytes >= 1000000000 ? number_format($bytes / 1000000000, 2) . ' GB' : number_format($bytes / 1000000, 2) . ' MB'; + $legend .= '
  • ' . fv_nbar_h($item['name']) . '' . $value . '
  • '; + } + print ''; +} diff --git a/includes/nbar_collector.php b/includes/nbar_collector.php new file mode 100644 index 0000000..3b26303 --- /dev/null +++ b/includes/nbar_collector.php @@ -0,0 +1,320 @@ +pruned < 60) { + return; + } + + foreach ($this->entries as $key => $entry) { + if ($entry['used'] <= $now - 1800) { + unset($this->entries[$key]); + } + } + $this->pruned = $now; + } + + public function remove(string $router, string $source, int $id): void { + unset($this->entries[$router . ':' . $source . ':' . $id]); + } + + public function store(string $router, string $source, int $id, array $template, int $now): bool { + $this->prune($now); + $this->remove($router, $source, $id); + $fields = cacti_count($template[0]); + + if ($fields > 2048) { + return false; + } + + while (true) { + $usage = $this->usage($router); + $local = $usage['exporter_templates'] >= 64 || $usage['exporter_fields'] + $fields > 2048; + + if (!$local && $usage['templates'] < 256 && $usage['fields'] + $fields <= 8192) { + break; + } + + foreach ($this->entries as $key => $entry) { + if (!$local || $entry['router'] === $router) { + unset($this->entries[$key]); + + break; + } + } + } + $this->entries[$router . ':' . $source . ':' . $id] = ['router' => $router, 'used' => $now, 'template' => $template]; + + return true; + } + + public function get(string $router, string $source, int $id, int $now): ?array { + $key = $router . ':' . $source . ':' . $id; + + if (!isset($this->entries[$key])) { + return null; + } + $entry = $this->entries[$key]; + unset($this->entries[$key]); + + if ($entry['used'] <= $now - 1800) { + return null; + } + $entry['used'] = $now; + $this->entries[$key] = $entry; + + return $entry['template']; + } + + public function usage(string $router): array { + $usage = ['templates' => cacti_count($this->entries), 'fields' => 0, 'exporter_templates' => 0, 'exporter_fields' => 0]; + + foreach ($this->entries as $entry) { + $fields = cacti_count($entry['template'][0]); + $usage['fields'] += $fields; + + if ($entry['router'] === $router) { + $usage['exporter_templates']++; + $usage['exporter_fields'] += $fields; + } + } + + return $usage; + } +} + +/** + * Decode network-order counters without rounding unsigned 64-bit values. + * + * @param string $s Input text or network-order bytes. + * + * @return int|string + */ +function fv_nbar_uint(string $s): int|string { + if (strlen($s) > 8) { + throw new InvalidArgumentException('NBAR numeric fields must fit an unsigned 64-bit value'); + } + $n = 0; + + foreach (unpack('C*', $s) ?: [] as $v) { + if (is_int($n) && $n <= intdiv(PHP_INT_MAX - $v, 256)) { + $n = (int) ($n * 256 + $v); + + continue; + } + $digits = (string) $n; + $carry = $v; + $out = ''; + + for ($i = strlen($digits) - 1; $i >= 0; $i--) { + $carry += ((int) $digits[$i]) * 256; + $out = ($carry % 10) . $out; + $carry = intdiv($carry, 10); + } + $n = ($carry > 0 ? (string) $carry : '') . $out; + } + + return $n; +} + +/** + * Decode a network-order integer of at most four bytes, which always fits a native int. + * + * @param string $s Network-order bytes. + * + * @return int + */ +function fv_nbar_int(string $s): int { + return (int) fv_nbar_uint($s); +} + +/** + * Decode template-defined NetFlow v9 application records and names. + * + * @param string $p NetFlow v9 packet bytes. + * @param string $router Exporter source IP address. + * + * @return void + */ +function fv_nbar_packet(string $p, string $router): void { + global $listener_id; + static $templates = null; + + if (strlen($p) < 20 || fv_nbar_int(substr($p, 0, 2)) !== 9) { + return; + } + $templates ??= new FlowviewNbarTemplateCache(); + $now = time(); + $templates->prune($now); + $source = bin2hex(substr($p, 16, 4)); + $unix = fv_nbar_int(substr($p, 8, 4)); + $uptime = fv_nbar_int(substr($p, 4, 4)); + $batches = []; + $hours = []; + + for ($o = 20; $o + 4 <= strlen($p);) { + $id = fv_nbar_int(substr($p, $o, 2)); + $len = fv_nbar_int(substr($p, $o + 2, 2)); + + if ($len < 4 || $o + $len > strlen($p)) { + break; + } + $end = $o + $len; + $q = $o + 4; + + if ($id === 0 || $id === 1) { + while ($q + ($id === 0 ? 4 : 6) <= $end) { + $tid = fv_nbar_int(substr($p, $q, 2)); + + if ($tid < 256) { + break; + } + + if ($id === 0) { + $count = fv_nbar_int(substr($p, $q + 2, 2)); + $size = $count * 4; + $q += 4; + } else { + $size = fv_nbar_int(substr($p, $q + 2, 2)) + fv_nbar_int(substr($p, $q + 4, 2)); + $q += 6; + } + + if ($size < 4 || $size % 4 || $q + $size > $end) { + $templates->remove($router, $source, $tid); + + break; + } + $fields = []; + $total = 0; + $valid = true; + + for ($z = 0; $z < $size; $z += 4) { + $f = fv_nbar_int(substr($p, $q + $z, 2)); + $l = fv_nbar_int(substr($p, $q + $z + 2, 2)); + $width = [1 => 8, 2 => 8, 4 => 1, 7 => 2, 11 => 2, 21 => 4][$f] ?? null; + + if ($l === 0 || ($id === 0 && $width !== null && $l > $width)) { + $valid = false; + + break; + } + $fields[] = [$f, $l]; + $total += $l; + } + + if ($valid && $total > 0 && $total <= 65535) { + $templates->store($router, $source, $tid, [$fields, $total, $id === 1], $now); + } else { + $templates->remove($router, $source, $tid); + } + $q += $size; + } + } elseif ($id >= 256 && ($template = $templates->get($router, $source, $id, $now)) !== null) { + [$fields, $total, $options] = $template; + + while ($q + $total <= $end) { + $data = []; + + foreach ($fields as $f) { + $data[$f[0]] = substr($p, $q, $f[1]); + $q += $f[1]; + } + + if (!isset($data[95])) { + continue; + } + $aid = bin2hex($data[95]); + + if ($options) { + $name = trim($data[96] ?? '', chr(0) . ' '); + + if ($name !== '' && strlen($name) <= 255 && preg_match('/^[a-zA-Z0-9_.: -]+$/D', $name)) { + flowview_db_execute_prepared('INSERT INTO plugin_flowview_nbar_names VALUES (?,?,?,?) ON DUPLICATE KEY UPDATE name=VALUES(name),updated=VALUES(updated)', [$router, $aid, $name, time()]); + } + + continue; + } + $src = $data[27] ?? $data[8] ?? ''; + $dst = $data[28] ?? $data[12] ?? ''; + + if (!in_array(strlen($src), [4, 16], true) || !in_array(strlen($dst), [4, 16], true) || !isset($data[1])) { + continue; + } + $ts = $unix; + + if (isset($data[21])) { + $age = ($uptime - fv_nbar_int($data[21]) + 4294967296) % 4294967296; + $ts = $unix - (int) floor($age / 1000); + } + + if ($ts < time() - fv_nbar_retention_days() * 86400 || $ts > time() + 300) { + continue; + } + $hours[(int) (floor($ts / 3600) * 3600)] = true; + $minute = (int) (floor($ts / 60) * 60); + $table = fv_nbar_data_table($minute); + $batches[$table] ??= ['rows' => [], 'args' => []]; + $batches[$table]['rows'][] = '(?,?,?,UNHEX(?),UNHEX(?),?,?,?,?,?,1,?)'; + array_push( + $batches[$table]['args'], + (int) $listener_id, + $minute, + $aid, + bin2hex($src), + bin2hex($dst), + fv_nbar_uint($data[7] ?? chr(0)), + fv_nbar_uint($data[11] ?? chr(0)), + fv_nbar_uint($data[4] ?? chr(0)), + fv_nbar_uint($data[1]), + fv_nbar_uint($data[2] ?? chr(0)), + $router + ); + } + } + $o = $end; + } + $statements = []; + + foreach ($batches as $table => $batch) { + if (!fv_nbar_data_ensure($table)) { + cacti_log('WARNING: NBAR raw table ' . $table . ' could not be created', false, 'FLOWVIEW'); + + return; + } + $statements[] = ['INSERT INTO `' . $table . '` VALUES ' . implode(',', $batch['rows']) . ' ON DUPLICATE KEY UPDATE bytes=bytes+VALUES(bytes),packets=packets+VALUES(packets),flows=flows+1', $batch['args']]; + } + + if ($statements && !fv_nbar_summary_store($statements, (int) $listener_id, $hours)) { + cacti_log('WARNING: NBAR record transaction failed', false, 'FLOWVIEW'); + } +} diff --git a/includes/nbar_common.php b/includes/nbar_common.php new file mode 100644 index 0000000..eed088b --- /dev/null +++ b/includes/nbar_common.php @@ -0,0 +1,208 @@ += 60) { + $value = (int) read_config_option('flowview_nbar_retention_days', true); + $days = in_array($value, [1, 3, 7, 14, 30], true) ? $value : 7; + $checked = time(); + } + + return $days; +} +/** + * Expire raw records and summaries: whole raw slices are dropped, summaries are deleted in bounded batches. + * + * @return void + */ +function fv_nbar_cleanup(int|float $budget = 10): void { + // One deadline for the raw slice drops and every summary deletion batch, so dropping a large + // backlog of slices cannot use up the supervisor's time before the summaries are maintained. + $deadline = microtime(true) + max(0, (float) $budget); + $cutoff = (int) (floor((time() - fv_nbar_retention_days() * 86400) / 3600) * 3600); + $queries = []; + + fv_nbar_data_drop_expired(time() - fv_nbar_retention_days() * 86400, $deadline); + + foreach (['apps', 'pairs', 'connections', 'hours'] as $table) { + $queries[] = ["DELETE FROM plugin_flowview_nbar_$table WHERE hour<$cutoff LIMIT 50000", 50000]; + } + $queries = array_merge($queries, fv_nbar_daily_cleanup_queries()); + fv_nbar_cleanup_batches($queries, $deadline); +} + +/** Drain every retention level fairly until empty or the shared deadline expires. */ +function fv_nbar_cleanup_batches(array $queries, float $deadline): void { + while ($queries && microtime(true) < $deadline) { + foreach ($queries as $key => [$sql, $limit]) { + if (microtime(true) >= $deadline) { + return; + } + + if (flowview_db_execute($sql) === false) { + unset($queries[$key]); + + continue; + } + + if ((int) flowview_db_affected_rows() < $limit) { + unset($queries[$key]); + } + } + } +} + +/** Establish a verified DML deadline; MySQL SELECT timeouts do not bound INSERT SELECT. */ +function fv_nbar_worker_timeout(int $seconds): ?float { + $old = flowview_db_fetch_cell('SELECT @@session.max_statement_time', '', false); + + if (!is_numeric($old) || (float) $old < 0) { + return null; + } + + if (!flowview_db_execute('SET SESSION max_statement_time=' . $seconds, false)) { + return null; + } + + return (float) $old; +} + +/** Launch maintenance without holding the Cacti poller on aggregate queries. */ +function fv_nbar_launch_maintenance(): bool { + global $config; + $php = read_config_option('path_php_binary'); + + if (!is_string($php) || $php === '') { + cacti_log('NBAR maintenance requires a configured PHP binary.', false, 'FLOWVIEW'); + + return false; + } + exec_background($php, '-q ' . escapeshellarg($config['base_path'] . '/plugins/flowview/cli/flowview_upgrade.php') . ' --nbar-worker=supervise'); + + return true; +} + +/** Run an isolated maintenance child with a hard process deadline. */ +function fv_nbar_supervise_maintenance(): void { + global $config; + + if (!register_process_start('flowview', 'nbar_maintenance', 0, 120)) { + return; + } + + try { + // Invoke PHP directly; older Cacti timeout helpers accept strings only. + $command = [(string) read_config_option('path_php_binary'), '-q', $config['base_path'] . '/plugins/flowview/cli/flowview_upgrade.php', '--nbar-worker=build']; + $status = fv_nbar_run_child($command, 90); + + if ($status !== 0) { + cacti_log('NBAR maintenance child failed or exceeded its 90-second deadline.', false, 'FLOWVIEW'); + } + } finally { + unregister_process('flowview', 'nbar_maintenance', 0); + } +} + +/** + * Supervise a direct PHP child independently of Cacti's versioned command API. + * + * @param list $command Executable followed by its arguments. + * @param int $seconds Hard process deadline in seconds. + * + * @return int + */ +function fv_nbar_run_child(array $command, int $seconds): int { + $pipes = []; + $process = false; + $executable = $command[0] ?? ''; + + if (is_string($executable) && $executable !== '' && (strpbrk($executable, '/\\') === false || is_executable($executable))) { + try { + $process = @proc_open($command, [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes, null, null, ['bypass_shell' => true]); + } catch (Throwable $error) { + $process = false; + } + } + + if (!is_resource($process)) { + return 125; + } + + try { + fclose($pipes[0]); + stream_set_blocking($pipes[1], false); + stream_set_blocking($pipes[2], false); + $deadline = hrtime(true) + max(1, $seconds) * 1000000000; + + while (true) { + // Drain both pipes in bounded chunks; child output cannot fill memory. + stream_get_contents($pipes[1], 8192); + stream_get_contents($pipes[2], 8192); + $state = proc_get_status($process); + + if (!$state['running']) { + return (int) $state['exitcode']; + } + + if (hrtime(true) >= $deadline) { + proc_terminate($process, 9); + + return 124; + } + usleep(10000); + } + } finally { + foreach ($pipes as $pipe) { + if (is_resource($pipe)) { + fclose($pipe); + } + } + proc_close($process); + } +} diff --git a/includes/nbar_connections.php b/includes/nbar_connections.php new file mode 100644 index 0000000..6ddb4b0 --- /dev/null +++ b/includes/nbar_connections.php @@ -0,0 +1,422 @@ + strlen($b) ?: strcmp($a, $b); +} +/** + * Identify an exported tuple using the resolved application-name rank. + * + * @param array $row Exported tuple or observation fields. + * @param array $names Application-name ranks by capture. + * @param int $listener Capture identifier. + * + * @return string + */ +function fv_nbar_connection_key(array $row, array $names, int $listener): string { + return json_encode([(string) $row['app_id'], (int) ($names[$listener][($row['router'] ?? '') . ':' . $row['app_id']]['name_id'] ?? 0), $row['src_hex'], $row['dst_hex'], (int) $row['sport'], (int) $row['dport'], (int) $row['protocol'], (string) ($row['router'] ?? '')], JSON_THROW_ON_ERROR); +} +/** + * Order connection totals by exact byte count and deterministic tuple fields. + * + * @param array $a First connection total with its normalized tuple key. + * @param array $b Second connection total with its normalized tuple key. + * + * @return int + */ +function fv_nbar_connection_compare(array $a, array $b): int { + $cmp = fv_nbar_decimal_compare($b['bytes'], $a['bytes']); + + if ($cmp) { + return $cmp; + } + + foreach ($a['_key'] as $i => $v) { + $cmp = ($i === 0 || $i === 2 || $i === 3 || $i === 7) ? strcmp((string) $v, (string) $b['_key'][$i]) : ($v <=> $b['_key'][$i]); + + if ($cmp) { + return $cmp; + } + } + + return 0; +} +// Read completed hours independently instead of regrouping a whole partial day. +/** + * Plan independently reusable complete hours and raw partial boundaries. + * + * @param int $listener Capture identifier. + * @param int $from Inclusive boundary in Unix seconds. + * @param int $to Exclusive boundary in Unix seconds. + * @param int $now Current Unix time. + * + * @return array + */ +function fv_nbar_connection_edges(int $listener, int $from, int $to, int $now): array { + $segments = []; + $ready = flowview_db_fetch_assoc_prepared('SELECT hour FROM plugin_flowview_nbar_hours WHERE listener_id=? AND hour>=? AND hour=0 AND connection_built=revision', [$listener, (int) (ceil($from / 3600) * 3600), (int) (floor(min($to, $now - 300) / 3600) * 3600)]); + + if (!is_array($ready)) { + throw new RuntimeException('Cannot read hourly summary state'); + } + $hours = array_fill_keys(array_column($ready, 'hour'), true); + $tables = fv_nbar_data_tables(); + + for ($cursor = $from; $cursor < $to; $cursor = $end) { + $hour = (int) (floor($cursor / 3600) * 3600); + $end = min($to, $hour + 3600); + + if ($cursor == $hour && $end == $hour + 3600 && isset($hours[$hour])) { + $source = '(SELECT listener_id,hour minute,app_id,src,dst,sport,dport,protocol,bytes,packets,flows,first_seen,last_seen,router FROM plugin_flowview_nbar_connections WHERE listener_id=' . (int) $listener . ' AND hour=' . $hour . ')'; + $segments[] = ['listener' => $listener, 'source' => $source, 'daily' => true]; + } else { + $source = fv_nbar_data_source(fv_nbar_data_columns(), 'listener_id=' . (int) $listener . ' AND minute>=' . (int) $cursor . ' AND minute<' . (int) $end, (int) $cursor, (int) $end, $tables); + $segments[] = ['listener' => $listener, 'source' => $source, 'daily' => false]; + } + } + + return $segments; +} +/** + * Plan a capture range using current daily, hourly and raw revisions. + * + * @param array $listeners Selected capture identifiers. + * @param int $start Inclusive range start in Unix seconds. + * @param int $end Exclusive range end in Unix seconds. + * @param int $now Current Unix time. + * + * @return array + */ +function fv_nbar_connection_plan(array $listeners, int $start, int $end, int $now): array { + $segments = []; + + foreach ($listeners as $listener) { + $listener = (int) $listener; + + if ($listener <= 0) { + throw new InvalidArgumentException('Invalid capture'); + } + $days = flowview_db_fetch_assoc_prepared('SELECT * FROM plugin_flowview_nbar_days WHERE listener_id=? AND day>=? AND covered_to<=? AND covered_to>day ORDER BY day', [$listener, $start, $end]); + + if (!is_array($days)) { + throw new RuntimeException('Cannot read daily history'); + } + $cursor = $start; + + foreach ($days as $day) { + $from = (int) $day['day']; + $to = (int) $day['covered_to']; + + if ($day['timezone'] !== date_default_timezone_get() || $from < $cursor || $to > (int) (floor(($now - 300) / 3600) * 3600)) { + continue; + } + + if (fv_nbar_daily_signature($listener, $from, $to) !== (string) $day['revision']) { + continue; + } + + if ($from > $cursor) { + $segments = array_merge($segments, fv_nbar_connection_edges($listener, $cursor, $from, $now)); + } + $segments[] = ['listener' => $listener, 'source' => '(SELECT listener_id,hour minute,app_id,src,dst,sport,dport,protocol,bytes,packets,flows,first_seen,last_seen,router FROM plugin_flowview_nbar_daily_connections WHERE listener_id=' . $listener . ' AND hour=' . $from . ')', 'daily' => true]; + $cursor = $to; + } + + if ($cursor < $end) { + $segments = array_merge($segments, fv_nbar_connection_edges($listener, $cursor, $end, $now)); + } + } + + return $segments; +} +/** + * Read an exact connection page from a consistent history snapshot. + * + * @param array $listeners Selected capture identifiers. + * @param int $start Inclusive range start in Unix seconds. + * @param int $end Exclusive range end in Unix seconds. + * @param string $where Validated filter predicate. + * @param array $args Prepared statement values. + * @param int $page One-based connection page. + * @param int $size Bounded page size including the next-page sentinel. + * + * @return array + */ +function fv_nbar_connections_direct(array $listeners, int $start, int $end, string $where, array $args, int $page, int $size = 101): array { + $listeners = array_values(array_unique(array_map('intval', $listeners))); + + if (!$listeners || cacti_sizeof($listeners) > 3) { + throw new InvalidArgumentException('Choose one to three captures'); + } + $page = max(1, (int) $page); + $size = max(101, min(501, (int) $size)); + $offset = ($page - 1) * 100; + $need = $offset + $size; + // Keep page retrieval bounded. Deep jumps use the exact SQL fallback below. + $limit = max(256, min(4096, $need * 2)); + + if (!flowview_db_execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ') || !flowview_db_execute('START TRANSACTION WITH CONSISTENT SNAPSHOT')) { + throw new RuntimeException('Cannot open history snapshot'); + } + + try { + $segments = fv_nbar_connection_plan($listeners, (int) $start, (int) $end, time()); + $nameRows = flowview_db_fetch_assoc_prepared('SELECT l.id listener_id,l.allowfrom,n.router,n.app_id,n.name,DENSE_RANK() OVER (ORDER BY n.name) name_id FROM plugin_flowview_devices l CROSS JOIN plugin_flowview_nbar_names n WHERE l.id IN (' . implode(',', $listeners) . ')', []); + + if (!is_array($nameRows)) { + throw new RuntimeException('Cannot read application names'); + } + $names = []; + + foreach ($nameRows as $row) { + if ($row['app_id'] !== null) { + $names[(int) $row['listener_id']][($row['router'] ?? '') . ':' . $row['app_id']] = $row; + + if (($row['router'] ?? '') === ($row['allowfrom'] ?? '')) { + $names[(int) $row['listener_id']][':' . $row['app_id']] = $row; + } + } + } + $fields = 'd.router,d.app_id,HEX(d.src) src_hex,HEX(d.dst) dst_hex,d.sport,d.dport,d.protocol'; + $group = 'd.router,d.app_id,d.src,d.dst,d.sport,d.dport,d.protocol'; + $counters = 'd.bytes bytes,d.packets packets,d.flows records,d.first_seen,d.last_seen'; + $sum = 'SUM(d.bytes) bytes,SUM(d.packets) packets,SUM(d.flows) records,IF(COUNT(d.first_seen)=COUNT(*),MIN(d.first_seen),NULL) first_seen,IF(COUNT(d.last_seen)=COUNT(*),MAX(d.last_seen),NULL) last_seen'; + $order = 'bytes DESC'; + + for ($round = 0; $round < 7 && $need <= 32768; $round++,$limit *= 2) { + $candidates = []; + $bound = '0'; + $exhausted = true; + + foreach ($segments as $segment) { + $sql = 'SELECT ' . $fields . ',' . ($segment['daily'] ? $counters : $sum) . ' FROM ' . $segment['source'] . ' d WHERE ' . $where . ($segment['daily'] ? '' : ' GROUP BY ' . $group) . ' ORDER BY ' . $order . ' LIMIT ' . ($limit + 1); + $rows = flowview_db_fetch_assoc_prepared($sql, $args); + + if (!is_array($rows)) { + throw new RuntimeException('Cannot read connection page'); + } + + if (cacti_sizeof($rows) > $limit) { + $exhausted = false; + $last = array_pop($rows); + $bound = fv_nbar_decimal_add($bound, $last['bytes']); + } + + foreach ($rows as $row) { + $candidates[fv_nbar_connection_key($row, $names, $segment['listener'])] = $row; + } + } + + if (!$candidates) { + flowview_db_execute('COMMIT'); + + return []; + } + + // Sparse histories produce different candidates for each day. Bound + // cross-day probes before they cost more than one grouped SQL query. + if (cacti_sizeof($candidates) >= 1024) { + break; + } + // Look up every candidate in every day, including small contributions + // outside each day's top list. The final totals are exact, not sampled. + $totals = []; + + foreach ($candidates as $key => $row) { + $tuple = json_decode($key, true); + $row['name'] = null; + + foreach ($nameRows as $name) { + if ((int) $name['name_id'] === $tuple[1]) { + $row['name'] = $name['name']; + + break; + } + } + $row['first_seen'] = null; + $row['last_seen'] = null; + $row['_seen_missing'] = false; + $row['bytes'] = '0'; + $row['packets'] = '0'; + $row['records'] = '0'; + $totals[$key] = $row; + } + $tuples = []; + + foreach ($candidates as $row) { + $tuples[json_encode([$row['router'] ?? '', $row['app_id'], $row['src_hex'], $row['dst_hex'], $row['sport'], $row['dport'], $row['protocol']])] = $row; + } + + foreach ($segments as $segment) { + foreach (array_chunk(array_values($tuples), 400) as $chunk) { + $lookup = []; + $values = []; + + foreach ($chunk as $row) { + $lookup[] = '(?,?,UNHEX(?),UNHEX(?),?,?,?)'; + array_push($values, $row['router'] ?? '', $row['app_id'], $row['src_hex'], $row['dst_hex'], $row['sport'], $row['dport'], $row['protocol']); + } + $sql = 'SELECT ' . $fields . ',' . ($segment['daily'] ? $counters : $sum) . ' FROM ' . $segment['source'] . ' d WHERE ' . $where . ' AND (' . $group . ') IN (' . implode(',', $lookup) . ')' . ($segment['daily'] ? '' : ' GROUP BY ' . $group); + $rows = flowview_db_fetch_assoc_prepared($sql, array_merge($args, $values)); + + if (!is_array($rows)) { + throw new RuntimeException('Cannot total connection page'); + } + + foreach ($rows as $row) { + $key = fv_nbar_connection_key($row, $names, $segment['listener']); + + if (!isset($totals[$key])) { + continue; + } + fv_nbar_seen_merge($totals[$key], $row); + + foreach (['bytes', 'packets', 'records'] as $counter) { + $totals[$key][$counter] = fv_nbar_decimal_add($totals[$key][$counter], $row[$counter]); + } + } + } + } + $ranked = []; + + foreach ($totals as $key => $row) { + if ($row['_seen_missing']) { + $row['first_seen'] = null; + $row['last_seen'] = null; + } unset($row['_seen_missing']); + $row['_key'] = json_decode($key, true); + $ranked[] = $row; + } + usort($ranked, 'fv_nbar_connection_compare'); + + // Any unseen connection has at most $bound bytes across all segments. + // Strict inequality also protects deterministic ties at page boundaries. + if ($exhausted || (cacti_sizeof($ranked) >= $need && fv_nbar_decimal_compare($ranked[$need - 1]['bytes'], $bound) > 0)) { + $result = array_slice($ranked, $offset, $size); + + foreach ($result as &$row) { + $row['src'] = inet_ntop((string) hex2bin($row['src_hex'])); + $row['dst'] = inet_ntop((string) hex2bin($row['dst_hex'])); + unset($row['_key'],$row['src_hex'],$row['dst_hex']); + } unset($row); + flowview_db_execute('COMMIT'); + + return $result; + } + } + // Exact fallback for exceptionally deep pages or a flat traffic distribution. + $source = '(' . implode(' UNION ALL ', array_map(static function (array $segment): string {return 'SELECT * FROM ' . $segment['source'] . ' s'; }, $segments)) . ')'; + $result = flowview_db_fetch_assoc_prepared(str_replace(' LIMIT 101 OFFSET ', ' LIMIT ' . $size . ' OFFSET ', fv_nbar_connections_sql($source, $where, $page)), $args); + + if (!is_array($result)) { + throw new RuntimeException('Cannot read connection page'); + } + flowview_db_execute('COMMIT'); + + return $result; + } catch (Throwable $e) { + flowview_db_execute('ROLLBACK'); + + throw $e; + } +} + +// Cache requested pages, never an entire report. Authorization is in the key. +/** + * Cache only requested connection pages within the current account. + * + * @param array $listeners Selected capture identifiers. + * @param int $start Inclusive range start in Unix seconds. + * @param int $end Exclusive range end in Unix seconds. + * @param string $where Validated filter predicate. + * @param array $args Prepared statement values. + * @param int $page One-based connection page. + * + * @return array + */ +function fv_nbar_connections_cached(array $listeners, int $start, int $end, string $where, array $args, int $page): array { + if (session_status() !== PHP_SESSION_ACTIVE) { + return fv_nbar_connections_direct($listeners, $start, $end, $where, $args, $page); + } + $block = max(1, (int) $page); + $slice = 0; + $key = hash('sha256', serialize(['requested-page-seen-v3', $_SESSION['sess_user_id'] ?? 0, $listeners, $start, $end, $where, $args, $block])); + $cache = $_SESSION['fv_nbar_connection_pages'] ?? []; + + foreach ($cache as $id => $entry) { + if ($entry['until'] <= time()) { + unset($cache[$id]); + } + } + + if (isset($cache[$key])) { + $_SESSION['fv_nbar_connection_pages'] = $cache; + + return array_slice($cache[$key]['rows'], $slice, 101); + } + $rows = fv_nbar_connections_direct($listeners, $start, $end, $where, $args, $block, 101); + + if (strlen(serialize($rows)) <= 524288) { + while (cacti_sizeof($cache) >= 3) { + array_shift($cache); + } $cache[$key] = ['until' => time() + ($end < time() - 300 ? 120 : 30), 'rows' => $rows]; + } + $_SESSION['fv_nbar_connection_pages'] = $cache; + + return array_slice($rows, $slice, 101); +} diff --git a/includes/nbar_count.php b/includes/nbar_count.php new file mode 100644 index 0000000..d9664cb --- /dev/null +++ b/includes/nbar_count.php @@ -0,0 +1,191 @@ + $where, 'listeners' => array_map('intval', $listeners), 'start' => (int) $start, 'end' => (int) $end]; + $source = fv_nbar_summary_source('connections', $listeners, (int) $start, (int) $end); + } + $sql = fv_nbar_count_sql($source, $where); + $key = hash('sha256', serialize([$_SESSION['sess_user_id'] ?? 0, $scope ?? $sql, $args])); + $cache = $_SESSION['fv_nbar_counts'] ?? []; + + foreach ($cache as $k => $entry) { + if ($entry['expires'] < time()) { + unset($cache[$k]); + } + } + + if (!isset($cache[$key])) { + while (cacti_sizeof($cache) >= 12) { + array_shift($cache); + } + $cache[$key] = ['sql' => $sql, 'args' => $args, 'expires' => time() + 3600, 'ttl' => $end >= time() - 120 ? 60 : 600] + ($scope === null ? [] : ['scope' => $scope]); + } + $_SESSION['fv_nbar_counts'] = $cache; + + return $key; +} +/** + * Send a JSON count response and terminate the HTTP request. + * + * @param array $payload JSON response fields. + * + * @return never + */ +function fv_nbar_count_response(array $payload): never { + header('Content-Type: application/json'); + header('Cache-Control: no-store'); + print json_encode($payload); + exit; +} +/** + * Handle an authenticated count request without blocking account navigation. + * + * @return never + */ +function fv_nbar_count_request(): never { + $key = get_nfilter_request_var('key'); + + if (!is_string($key) || !preg_match('/^[a-f0-9]{64}$/D', $key)) { + fv_nbar_count_response(['status' => 'expired']); + } + $entry = $_SESSION['fv_nbar_counts'][$key] ?? null; + + if (!$entry || $entry['expires'] < time()) { + fv_nbar_count_response(['status' => 'expired']); + } + + if (($entry['failed_until'] ?? 0) > time()) { + fv_nbar_count_response(['status' => 'unavailable']); + } + + if (isset($entry['total']) && $entry['until'] > time()) { + fv_nbar_count_response(['status' => 'ready', 'pages' => (int) ceil($entry['total'] / 100), 'total' => $entry['total']]); + } + // Release the account session lock so navigation remains available during counting. + session_write_close(); + $locked = flowview_db_fetch_cell("SELECT GET_LOCK(CONCAT('flowview_nbar_page_count:',LEFT(SHA2(DATABASE(),256),16)),0)"); + + if ((int) $locked !== 1) { + fv_nbar_count_response(['status' => 'busy']); + } + // Return promptly; subsequent polls read the completed session cache. + // PHP-FPM can finish this bounded calculation after closing the response. + ini_set('session.use_cookies', '0'); + session_cache_limiter(''); + $background = function_exists('fastcgi_finish_request'); + + if ($background) { + header('Content-Type: application/json'); + header('Cache-Control: no-store'); + print json_encode(['status' => 'busy']); + fastcgi_finish_request(); + } + $row = null; + + try { + set_time_limit(100); + $sql = isset($entry['scope']) ? fv_nbar_count_sql(fv_nbar_summary_source('connections', $entry['scope']['listeners'], $entry['scope']['start'], $entry['scope']['end']), $entry['scope']['where']) : $entry['sql']; + $row = fv_nbar_count_query($sql, $entry['args']); + } catch (RuntimeException $e) { + // The raw slices could not be listed, so the count is reported as unavailable. + cacti_log('NBAR count: ' . $e->getMessage(), false, 'FLOWVIEW'); + } finally { + flowview_db_fetch_cell("SELECT RELEASE_LOCK(CONCAT('flowview_nbar_page_count:',LEFT(SHA2(DATABASE(),256),16)))"); + } + + if (!is_array($row) || !isset($row['total'])) { + session_start(); + + if (isset($_SESSION['fv_nbar_counts'][$key])) { + $_SESSION['fv_nbar_counts'][$key]['failed_until'] = time() + 60; + } + session_write_close(); + + if ($background) { + exit; + } + fv_nbar_count_response(['status' => 'unavailable']); + } + $total = (int) $row['total']; + session_start(); + + if (isset($_SESSION['fv_nbar_counts'][$key])) { + $_SESSION['fv_nbar_counts'][$key]['total'] = $total; + $_SESSION['fv_nbar_counts'][$key]['until'] = time() + $entry['ttl']; + } + session_write_close(); + + if ($background) { + exit; + } + fv_nbar_count_response(['status' => 'ready', 'pages' => (int) ceil($total / 100), 'total' => $total]); +} + +/** Bound a SELECT on the actual FlowView server rather than Cacti's server. */ +function fv_nbar_count_query(string $sql, array $args): array|false|null { + $version = flowview_db_fetch_cell('SELECT VERSION()'); + + if (is_string($version) && stripos($version, 'MariaDB') !== false) { + return flowview_db_fetch_row_prepared('SET STATEMENT max_statement_time=90 FOR ' . $sql, $args); + } + + if (is_string($version) && preg_match('/^8[.]/', $version)) { + return flowview_db_fetch_row_prepared(preg_replace('/^SELECT /', 'SELECT /*+ MAX_EXECUTION_TIME(90000) */ ', $sql), $args); + } + + return false; +} diff --git a/includes/nbar_daily.php b/includes/nbar_daily.php new file mode 100644 index 0000000..cd720aa --- /dev/null +++ b/includes/nbar_daily.php @@ -0,0 +1,311 @@ +getTimestamp(), $day->modify('+1 day')->getTimestamp()]; +} +/** + * Require every overlapping hour to match its completed raw revision. + * + * @param int $listener Capture identifier. + * @param int $from Inclusive boundary in Unix seconds. + * @param int $to Exclusive boundary in Unix seconds. + * + * @return string|null + */ +function fv_nbar_daily_signature(int $listener, int $from, int $to): ?string { + $lo = (int) (floor($from / 3600) * 3600); + $hi = (int) (ceil($to / 3600) * 3600); + $r = flowview_db_fetch_row_prepared('SELECT COUNT(*) n, COALESCE(SUM(revision),0) rev, + COALESCE(SUM(built=revision AND connection_built=revision AND built>=0),0) ready + FROM plugin_flowview_nbar_hours WHERE listener_id=? AND hour>=? AND hour=0),0)=$count + FROM plugin_flowview_nbar_hours WHERE listener_id=$listener AND hour>=$lo AND hour<$hi)"; +} +/** + * Add one hour atomically or restart a day invalidated by late records. + * + * @param int $listener Capture identifier. + * @param int $day Local calendar-day start in Unix seconds. + * @param int $target Requested exclusive summary boundary. + * + * @return int + */ +function fv_nbar_daily_build(int $listener, int $day, int $target): int { + $listener = (int) $listener; + $day = (int) $day; + $target = (int) $target; + + if (!flowview_db_execute('START TRANSACTION WITH CONSISTENT SNAPSHOT')) { + return 0; + } + $old = flowview_db_fetch_row_prepared('SELECT * FROM plugin_flowview_nbar_days WHERE listener_id=? AND day=?', [$listener, $day]); + $valid = $old && $old['timezone'] === date_default_timezone_get() && (int) $old['covered_to'] <= $target && + fv_nbar_daily_signature($listener, $day, (int) $old['covered_to']) === (string) $old['revision']; + $from = $valid ? (int) $old['covered_to'] : $day; + + if ($from >= $target) { + flowview_db_execute('ROLLBACK'); + + return 0; + } + $to = min($target, (int) (floor($from / 3600) * 3600 + 3600)); + $revision = fv_nbar_daily_signature($listener, $day, $to); + + if ($revision === null) { + flowview_db_execute('ROLLBACK'); + + return 0; + } + $statement_timeout = fv_nbar_worker_timeout(20); + + if ($statement_timeout === null) { + flowview_db_execute('ROLLBACK'); + + return 0; + } + + try { + foreach (['apps' => 'app_id,LENGTH(src)', 'pairs' => 'app_id,src,dst', 'connections' => 'app_id,src,dst,sport,dport,protocol'] as $kind => $fields) { + if (!$valid && !flowview_db_execute_prepared("DELETE FROM plugin_flowview_nbar_daily_$kind WHERE listener_id=? AND hour=?", [$listener, $day])) { + throw new RuntimeException('Daily reset failed'); + } + $source = fv_nbar_hourly_source($kind, [$listener], $from, $to); + $sql = "INSERT INTO plugin_flowview_nbar_daily_$kind SELECT $listener,$day,$fields,SUM(bytes),SUM(packets),SUM(flows),IF(COUNT(first_seen)=COUNT(*),MIN(first_seen),NULL),IF(COUNT(last_seen)=COUNT(*),MAX(last_seen),NULL),router + FROM $source d WHERE d.listener_id=$listener AND d.minute>=$from AND d.minute<$to GROUP BY $fields,router + ON DUPLICATE KEY UPDATE bytes=bytes+VALUES(bytes),packets=packets+VALUES(packets),flows=flows+VALUES(flows),first_seen=LEAST(first_seen,VALUES(first_seen)),last_seen=GREATEST(last_seen,VALUES(last_seen))"; + + if (!flowview_db_execute($sql)) { + throw new RuntimeException('Daily aggregate failed'); + } + } + + if (!flowview_db_execute_prepared( + 'INSERT INTO plugin_flowview_nbar_days + (listener_id,day,covered_to,revision,timezone,updated) VALUES (?,?,?,?,?,?) + ON DUPLICATE KEY UPDATE covered_to=VALUES(covered_to),revision=VALUES(revision),timezone=VALUES(timezone),updated=VALUES(updated)', + [$listener, $day, $to, $revision, date_default_timezone_get(), time()] + )) { + throw new RuntimeException('Daily checkpoint failed'); + } + + if (!flowview_db_execute('COMMIT')) { + throw new RuntimeException('Daily commit failed'); + } + + return 1; + } catch (Throwable $e) { + flowview_db_execute('ROLLBACK'); + cacti_log('NBAR daily summary: ' . $e->getMessage(), false, 'FLOWVIEW'); + + return 0; + } finally { + flowview_db_execute('SET SESSION max_statement_time=' . $statement_timeout); + } +} +/** + * Maintain reusable calendar-day summaries within a worker budget. + * + * @param int $budget Worker time budget in seconds. + * + * @return int + */ +function fv_nbar_daily_refresh(int $budget = 20): int { + if ((int) flowview_db_fetch_cell("SELECT GET_LOCK(CONCAT('flowview_nbar_daily:',LEFT(SHA2(DATABASE(),256),16)),0)") !== 1) { + return 0; + } + $deadline = microtime(true) + max(1, (int) $budget); + $done = 0; + + try { + $cutoff = time() - fv_nbar_retention_days() * 86400; + $closed = (int) (floor((time() - 300) / 3600) * 3600); + $listeners = flowview_db_fetch_assoc('SELECT listener_id,MIN(hour) first_hour FROM plugin_flowview_nbar_hours GROUP BY listener_id'); + $candidates = []; + + foreach ($listeners as $row) { + $first_record = fv_nbar_data_edge((int) $row['listener_id'])['minute'] ?? 0; + + if (!$first_record) { + continue; + } + $row['first_hour'] = (int) (floor((int) $first_record / 3600) * 3600); + [$first] = fv_nbar_daily_bounds(max($cutoff, (int) $first_record)); + + if ($first >= $cutoff) { + for ($empty = (int) (floor($first / 3600) * 3600); $empty < (int) $row['first_hour']; $empty += 3600) { + flowview_db_execute_prepared('INSERT IGNORE INTO plugin_flowview_nbar_hours (listener_id,hour,revision,built,connection_built) VALUES (?,?,0,0,0)', [(int) $row['listener_id'], $empty]); + } + } + [$today] = fv_nbar_daily_bounds($closed); + + for ($day = $first; $day <= $today; $day = $next) { + [, $next] = fv_nbar_daily_bounds($day); + + if ($day < $cutoff) { + continue; + } + $candidates[] = [(int) $row['listener_id'], $day, min($next, $closed)]; + } + } + // Finish older days first; each checkpoint appends only the next hour. + usort($candidates, static function (array $a, array $b): int {return $a[1] <=> $b[1] ?: $a[0] <=> $b[0]; }); + + do { + $progress = 0; + + foreach ($candidates as [$listener,$day,$target]) { + if (microtime(true) >= $deadline) { + break 2; + } + $progress += fv_nbar_daily_build($listener, $day, $target); + } + $done += $progress; + } while ($progress && microtime(true) < $deadline); + } finally { + flowview_db_fetch_cell("SELECT RELEASE_LOCK(CONCAT('flowview_nbar_daily:',LEFT(SHA2(DATABASE(),256),16)))"); + } + + return $done; +} +/** + * Expire daily summary rows and checkpoints in bounded batches. + * + * @return void + */ +function fv_nbar_daily_cleanup(): void { + fv_nbar_cleanup_batches(fv_nbar_daily_cleanup_queries(), microtime(true) + 10); +} +/** Return allowlisted daily retention batches for the shared cleanup worker. */ +function fv_nbar_daily_cleanup_queries(): array { + $cutoff = time() - fv_nbar_retention_days() * 86400; + $queries = []; + + foreach (['apps', 'pairs', 'connections'] as $kind) { + $queries[] = ["DELETE FROM plugin_flowview_nbar_daily_$kind WHERE hour<$cutoff LIMIT 5000", 5000]; + } + $queries[] = ["DELETE FROM plugin_flowview_nbar_days WHERE day<$cutoff LIMIT 100", 100]; + + return $queries; +} +/** + * Use valid full-day summaries and hourly or raw boundary records. + * + * @param string $kind Allowlisted summary type. + * @param array $listeners Selected capture identifiers. + * @param int $start Inclusive range start in Unix seconds. + * @param int $end Exclusive range end in Unix seconds. + * @param int|null $now Current Unix time, or null to read the clock. + * + * @return string + */ +function fv_nbar_summary_source(string $kind, array $listeners, int $start, int $end, ?int $now = null): string { + if (!in_array($kind, ['apps', 'pairs', 'connections'], true)) { + throw new InvalidArgumentException('Unknown summary kind'); + } + $listeners = array_values(array_unique(array_filter(array_map('intval', $listeners), static function (int $id): bool {return $id > 0; }))); + + $start = (int) $start; + $end = (int) $end; + + if (!$listeners) { + return fv_nbar_data_source(fv_nbar_data_columns(), "minute>=$start AND minute<$end", $start, $end); + } + $now = $now ?? time(); + + if ($end - $start < 3600) { + return fv_nbar_hourly_source($kind, $listeners, $start, $end, $now); + } + $parts = []; + + foreach ($listeners as $listener) { + $days = flowview_db_fetch_assoc_prepared('SELECT * FROM plugin_flowview_nbar_days WHERE listener_id=? AND day>=? AND covered_to<=? AND covered_to>day ORDER BY day', [$listener, $start, $end]); + $cursor = $start; + + foreach ($days as $row) { + $day = (int) $row['day']; + $to = (int) $row['covered_to']; + + if ($row['timezone'] !== date_default_timezone_get() || $day < $cursor || $to > (int) (floor(($now - 300) / 3600) * 3600)) { + continue; + } + + if ($day > $cursor) { + $s = fv_nbar_hourly_source($kind, [$listener], $cursor, $day, $now); + $parts[] = "SELECT * FROM $s fallback_before WHERE listener_id=$listener AND minute>=$cursor AND minute<$day"; + } + $ready = fv_nbar_daily_guard($listener, $day, $to, $row['revision']); + + if ($kind === 'apps') { + $fields = 'listener_id,hour minute,app_id,REPEAT(CHAR(0),family) src,REPEAT(CHAR(0),family) dst,0 sport,0 dport,0 protocol,bytes,packets,flows,first_seen,last_seen,router'; + } elseif ($kind === 'pairs') { + $fields = 'listener_id,hour minute,app_id,src,dst,0 sport,0 dport,0 protocol,bytes,packets,flows,first_seen,last_seen,router'; + } else { + $fields = 'listener_id,hour minute,app_id,src,dst,sport,dport,protocol,bytes,packets,flows,first_seen,last_seen,router'; + } + $parts[] = "SELECT $fields FROM plugin_flowview_nbar_daily_$kind WHERE listener_id=$listener AND hour=$day AND ($ready)"; + $s = fv_nbar_hourly_source($kind, [$listener], $day, $to, $now); + $parts[] = "SELECT * FROM $s fallback_day WHERE listener_id=$listener AND minute>=$day AND minute<$to AND NOT ($ready)"; + $cursor = $to; + } + + if ($cursor < $end) { + $s = fv_nbar_hourly_source($kind, [$listener], $cursor, $end, $now); + $parts[] = "SELECT * FROM $s fallback_after WHERE listener_id=$listener AND minute>=$cursor AND minute<$end"; + } + } + + return '(' . implode(' UNION ALL ', $parts) . ')'; +} diff --git a/includes/nbar_dns.php b/includes/nbar_dns.php new file mode 100644 index 0000000..8ee6f35 --- /dev/null +++ b/includes/nbar_dns.php @@ -0,0 +1,115 @@ + time()) { + return $names[$ip] = $negative[$ip]['name']; + } + + if ($started === null) { + $started = microtime(true); + } + + if (microtime(true) - $started > 1) { + return $names[$ip] = __('Not resolved', 'flowview'); + } + + if ($resolver === null) { + set_include_path($config['base_path'] . '/plugins/flowview' . PATH_SEPARATOR . get_include_path()); + require_once($config['base_path'] . '/plugins/flowview/Net/DNS2.php'); + $servers = array_filter([read_config_option('settings_dns_primary'), read_config_option('settings_dns_secondary')]); + $resolver = new Net_DNS2_Resolver($servers ? ['nameservers' => array_values($servers)] : []); + $resolver->timeout = 1; + } + + try { + $response = $resolver->query($ip, 'PTR'); + + foreach ($response->answer as $answer) { + if (!empty($answer->ptrdname)) { + $name = rtrim($answer->ptrdname, '.'); + flowview_db_execute_prepared('INSERT INTO plugin_flowview_dnscache (ip,host,source,arin_verified,arin_id,time) VALUES (?,?,?,0,0,?) ON DUPLICATE KEY UPDATE host=VALUES(host),source=VALUES(source),time=VALUES(time)', [$ip, $name, 'DNS', time()]); + + return $names[$ip] = $name; + } + } + + return $names[$ip] = fv_nbar_dns_negative($ip, __('No reverse DNS', 'flowview'), 300); + } catch (Exception $e) { + return $names[$ip] = fv_nbar_dns_negative($ip, $e->getCode() === 3 ? __('No reverse DNS', 'flowview') : __('Not resolved', 'flowview'), $e->getCode() === 3 ? 300 : 30); + } +} + +/** + * Cache a negative DNS result with bounded account-session storage. + * + * @param string $ip Endpoint IP address. + * @param string $name Cached DNS result label. + * @param int $ttl Cache lifetime in seconds. + * + * @return string + */ +function fv_nbar_dns_negative(string $ip, string $name, int $ttl): string { + $cache = $_SESSION['fv_nbar_dns_negative'] ?? []; + + foreach ($cache as $key => $entry) { + if ($entry['until'] <= time()) { + unset($cache[$key]); + } + } + $cache[$ip] = ['name' => $name, 'until' => time() + $ttl]; + + if (cacti_sizeof($cache) > 512) { + $cache = array_slice($cache, -512, null, true); + } + $_SESSION['fv_nbar_dns_negative'] = $cache; + + return $name; +} diff --git a/includes/nbar_feed_status.php b/includes/nbar_feed_status.php new file mode 100644 index 0000000..1ec72c7 --- /dev/null +++ b/includes/nbar_feed_status.php @@ -0,0 +1,67 @@ +' . fv_nbar_h(__('STATUS:', 'flowview')) . ' '; + $items = []; + + foreach ($devices as $device) { + $id = (int) $device['id']; + + if (!in_array($id, $capture_ids, true)) { + continue; + } + // Indexed lookup independent of historical report filters. + $latest = fv_nbar_data_edge($id, true); + + if (!is_array($latest)) { + $items[] = '' . fv_nbar_h($device['name']) . ': ' . fv_nbar_h(__('Unable to load NBAR status. Please try again.', 'flowview')) . ''; + + continue; + } + $last = isset($latest['minute']) ? (int) $latest['minute'] : 0; + $receiving = $last > 0 && $last >= $now - 300 && $last <= $now + 60; + $label = $receiving ? __('Receiving', 'flowview') : __('No recent NBAR', 'flowview'); + $seen = $last > 0 ? __('Last NBAR record: %s', date('Y-m-d H:i', $last), 'flowview') : __('No retained NBAR records', 'flowview'); + $icon = $receiving ? 'fa-check-circle' : 'fa-times-circle'; + $color = $receiving ? '#27852b' : '#d32f2f'; + $items[] = ' ' . fv_nbar_h($device['name']) . ': ' . fv_nbar_h($label) . ' (' . fv_nbar_h($seen) . ')'; + } + print implode('   |   ', $items); + + if (!$items) { + print fv_nbar_h(__('No captures selected', 'flowview')); + } + print ''; +} diff --git a/includes/nbar_partition.php b/includes/nbar_partition.php new file mode 100644 index 0000000..6299938 --- /dev/null +++ b/includes/nbar_partition.php @@ -0,0 +1,273 @@ +[]. The Database Partitioning Scheme + * setting picks hourly (the default) or daily slices, slices are created when the first record + * for them arrives, and expired slices are dropped instead of deleted row by row. + * + * Slices are named by UTC time. NBAR's hour boundaries are UTC aligned, so a UTC hour is always + * inside exactly one hourly slice, and the repeated hour when daylight saving time ends cannot + * make two different hours share a table. Readers never recompute names: they list the slices + * that exist and test each slice's time range, so slices from before a scheme change keep working. + */ + +/** + * The raw record columns shared by every slice, with the observation dates the summaries expect. + * + * @return string + */ +function fv_nbar_data_columns(): string { + return 'listener_id,minute,app_id,src,dst,sport,dport,protocol,bytes,packets,flows,minute first_seen,minute last_seen,router'; +} + +/** + * Name the slice that holds the records of a given minute. + * + * @param int $time Unix time of the record. + * + * @return string + */ +function fv_nbar_data_table(int $time): string { + $suffix = gmdate('Y', $time) . substr('000' . gmdate('z', $time), -3); + + if ((string) read_config_option('flowview_partition') !== '0') { + $suffix .= gmdate('H', $time); + } + + return 'plugin_flowview_nbar_data_' . $suffix; +} + +/** + * The time range [start, end) covered by a slice, taken from its name. + * + * @param string $table Slice table name. + * + * @return array{0: int, 1: int}|null null when the name is not a slice. + */ +function fv_nbar_data_bounds(string $table): ?array { + if (!preg_match('/^plugin_flowview_nbar_data_([0-9]{4})([0-9]{3})([0-9]{2})?$/D', $table, $match)) { + return null; + } + $hourly = isset($match[3]); + + if ((int) $match[2] > 365 || ($hourly && (int) $match[3] > 23)) { + return null; + } + $start = (int) gmmktime($hourly ? (int) $match[3] : 0, 0, 0, 1, 1 + (int) $match[2], (int) $match[1]); + + return [$start, $start + ($hourly ? 3600 : 86400)]; +} + +/** + * List the slices that exist, oldest first. + * + * @throws RuntimeException When the database cannot list its tables, so a failure is never mistaken for "no slices". + * + * @return list + */ +function fv_nbar_data_tables(): array { + $rows = flowview_db_fetch_assoc("SHOW TABLES LIKE 'plugin\\_flowview\\_nbar\\_data\\_%'"); + + if (!is_array($rows)) { + throw new RuntimeException('Cannot list the NBAR raw tables'); + } + $tables = []; + + foreach ($rows as $row) { + $name = (string) reset($row); + $bounds = fv_nbar_data_bounds($name); + + if ($bounds !== null) { + $tables[] = ['table' => $name, 'start' => $bounds[0], 'end' => $bounds[1]]; + } + } + usort($tables, static fn (array $a, array $b): int => [$a['start'], $a['table']] <=> [$b['start'], $b['table']]); + + return $tables; +} + +/** + * The existing slices that overlap a time range. + * + * @param int $from Inclusive range start in Unix seconds. + * @param int $to Exclusive range end in Unix seconds. + * @param list|null $tables Slices from fv_nbar_data_tables(), or null to list them. + * + * @return list + */ +function fv_nbar_data_slices(int $from, int $to, ?array $tables = null): array { + $slices = []; + + foreach ($tables ?? fv_nbar_data_tables() as $slice) { + if ($slice['start'] < $to && $slice['end'] > $from) { + $slices[] = $slice['table']; + } + } + + return $slices; +} + +/** + * One SELECT per slice that overlaps the range, each carrying the same filter so no slice is read whole. + * + * @param string $columns Select list. + * @param string $where Trusted filter. + * @param int $from Inclusive range start in Unix seconds. + * @param int $to Exclusive range end in Unix seconds. + * @param list|null $tables Slices from fv_nbar_data_tables(), or null to list them. + * + * @return list + */ +function fv_nbar_data_parts(string $columns, string $where, int $from, int $to, ?array $tables = null): array { + return array_map( + static fn (string $table): string => "SELECT $columns FROM `$table` WHERE $where", + fv_nbar_data_slices($from, $to, $tables) + ); +} + +/** + * A relation over the slices of a range. An empty relation with the right columns is returned + * when no slice exists yet, so reports on a fresh installation still run. + * + * @param string $columns Select list. + * @param string $where Trusted filter. + * @param int $from Inclusive range start in Unix seconds. + * @param int $to Exclusive range end in Unix seconds. + * @param list|null $tables Slices from fv_nbar_data_tables(), or null to list them. + * + * @return string + */ +function fv_nbar_data_source(string $columns, string $where, int $from, int $to, ?array $tables = null): string { + $parts = fv_nbar_data_parts($columns, $where, $from, $to, $tables); + + if (!$parts) { + $parts[] = "SELECT $columns FROM (SELECT CAST(0 AS UNSIGNED) listener_id,CAST(0 AS UNSIGNED) minute,CAST('' AS CHAR(32)) app_id,CAST('' AS BINARY(16)) src,CAST('' AS BINARY(16)) dst,CAST(0 AS UNSIGNED) sport,CAST(0 AS UNSIGNED) dport,CAST(0 AS UNSIGNED) protocol,CAST(0 AS UNSIGNED) bytes,CAST(0 AS UNSIGNED) packets,CAST(0 AS UNSIGNED) flows,CAST('' AS CHAR(45)) router) fv_nbar_empty WHERE 1=0"; + } + + return '(' . implode(' UNION ALL ', $parts) . ')'; +} + +/** + * Create a slice the first time a record for it arrives. The answer is remembered for a minute so + * the collector does not check for the table on every packet. + * + * @param string $table Slice table name. + * + * @return bool + */ +function fv_nbar_data_ensure(string $table): bool { + if (($GLOBALS['fv_nbar_data_known'][$table] ?? 0) > time() - 60) { + return true; + } + + if (!flowview_db_create_nbar_data_partition($table)) { + return false; + } + $GLOBALS['fv_nbar_data_known'][$table] = time(); + + return true; +} + +/** + * Drop the slices that ended before the retention cutoff. + * + * @param int $cutoff Unix time before which records are no longer retained. + * @param float|null $deadline Unix time with microseconds after which no further slice is dropped, or null for no limit. + * + * @return int The number of slices dropped. + */ +function fv_nbar_data_drop_expired(int $cutoff, ?float $deadline = null): int { + $dropped = 0; + + try { + $tables = fv_nbar_data_tables(); + } catch (RuntimeException $e) { + // Nothing is dropped; retention simply runs again on the next maintenance pass. + return 0; + } + + // Oldest slices first, so an interrupted pass leaves only the newest expired slices for the next one. + foreach ($tables as $slice) { + if ($deadline !== null && microtime(true) >= $deadline) { + break; + } + + if ($slice['end'] <= $cutoff && flowview_db_execute('DROP TABLE IF EXISTS `' . $slice['table'] . '`') !== false) { + unset($GLOBALS['fv_nbar_data_known'][$slice['table']]); + $dropped++; + } + } + + return $dropped; +} + +/** + * The oldest or newest retained record of a capture. + * + * The slices are probed in batches of 32, one query per batch, so a capture with no records does + * not cost a query per retained slice. + * + * @param int $listener Capture identifier. + * @param bool $latest True for the newest record instead of the oldest. + * + * @return array|false A row with the minute, an empty array when the capture has no records, or false when the database cannot be read. + */ +function fv_nbar_data_edge(int $listener, bool $latest = false): array|false { + try { + $tables = fv_nbar_data_tables(); + } catch (RuntimeException $e) { + return false; + } + + // Slices from different schemes overlap, so the first slice with a record is not necessarily the + // answer. Keep going until no remaining slice can hold an earlier (or later) minute. + if ($latest) { + usort($tables, static fn (array $a, array $b): int => [$b['end'], $b['table']] <=> [$a['end'], $a['table']]); + } + $found = []; + + foreach (array_chunk($tables, 32) as $batch) { + if ($found && ($latest ? $batch[0]['end'] <= (int) $found['minute'] : $batch[0]['start'] >= (int) $found['minute'])) { + break; + } + $members = array_map( + static fn (array $slice): string => '(SELECT minute FROM `' . $slice['table'] . '` WHERE listener_id=? ORDER BY minute' . ($latest ? ' DESC' : '') . ' LIMIT 1)', + $batch + ); + $row = flowview_db_fetch_row_prepared('SELECT ' . ($latest ? 'MAX' : 'MIN') . '(minute) minute FROM (' . implode(' UNION ALL ', $members) . ') edge', array_fill(0, cacti_sizeof($batch), $listener)); + + if (!is_array($row)) { + return false; + } + + if (isset($row['minute']) && (!$found || ($latest ? (int) $row['minute'] > (int) $found['minute'] : (int) $row['minute'] < (int) $found['minute']))) { + $found = ['minute' => $row['minute']]; + } + } + + return $found; +} diff --git a/includes/nbar_preferences.php b/includes/nbar_preferences.php new file mode 100644 index 0000000..e9e459c --- /dev/null +++ b/includes/nbar_preferences.php @@ -0,0 +1,201 @@ + strlen($packed) * 8) { + return false; + } + $out[] = inet_ntop($packed) . '/' . $bits; + + if (cacti_sizeof($out) > 32) { + return false; + } + } + + return array_values(array_unique($out)); +} +/** + * Load account preferences and validate a protected settings submission. + * + * @param array $devices Available capture definitions. + * + * @return array + */ +function fv_nbar_preferences_init(array $devices): array { + $defaults = ['prefixes' => [], 'listeners' => [], 'data_unit' => 'GB', 'rate_unit' => 'Mbps', 'dns' => '1']; + $saved = json_decode((string) read_user_setting('flowview_nbar_preferences', ''), true); + $p = array_replace($defaults, is_array($saved) ? $saved : ($_SESSION['fv_nbar_preferences'] ?? [])); + + if (!is_array($p['prefixes']) || !fv_nbar_cidrs(implode("\n", $p['prefixes']))) { + $p['prefixes'] = $defaults['prefixes']; + } + $p['listeners'] = is_array($p['listeners']) ? array_values(array_intersect(array_map('intval', $p['listeners']), array_map('intval', array_column($devices, 'id')))) : $defaults['listeners']; + + if (!$p['listeners']) { + $p['listeners'] = $defaults['listeners']; + } + $message = ''; + $GLOBALS['fv_nbar_preferences_saved'] = false; + + if (($_SERVER['REQUEST_METHOD'] ?? '') === 'POST' && get_nfilter_request_var('prefs_apply') === '1') { + if (!function_exists('csrf_check') || !csrf_check(false)) { + http_response_code(403); + exit; + } + $candidate = $p; + $prefixes = is_string(get_nfilter_request_var('local_prefixes', null)) ? fv_nbar_cidrs(get_nfilter_request_var('local_prefixes')) : false; + $ids = is_array(get_nfilter_request_var('site_listeners', null)) ? get_nfilter_request_var('site_listeners') : []; + $ids = array_filter($ids, 'is_scalar'); + $ids = array_values(array_intersect(array_map('intval', $ids), array_map('intval', array_column($devices, 'id')))); + + if ($prefixes === false) { + $message = __('Settings were not saved. Enter valid IPv4/IPv6 networks, or leave the list blank.', 'flowview'); + } elseif (cacti_sizeof(array_unique($ids)) > 3 || (isset_request_var('combine_apply') && cacti_sizeof(array_unique($ids)) === 1)) { + $message = __('Settings were not saved. Choose two or three captures to combine, or clear all selections to save Internal Networks only.', 'flowview'); + } else { + $candidate['prefixes'] = $prefixes; + $candidate['listeners'] = array_values(array_unique($ids)); + + foreach (['data_unit' => ['Auto', 'MB', 'GB'], 'rate_unit' => ['Auto', 'Kbps', 'Mbps'], 'dns' => ['0', '1']] as $key => $allowed) { + if (is_string(get_nfilter_request_var($key, null)) && in_array(get_nfilter_request_var($key), $allowed, true)) { + $candidate[$key] = get_nfilter_request_var($key); + } + } + $_SESSION['fv_nbar_preferences'] = $p = $candidate; + $GLOBALS['fv_nbar_preferences_saved'] = true; + set_user_setting('flowview_nbar_preferences', json_encode($p)); + $message = ''; + } + } + $GLOBALS['fv_nbar_preferences'] = $p; + $GLOBALS['fv_nbar_preferences_message'] = $message; + + return $p; +} +/** + * Build the internal-network predicate for a permitted address column. + * + * @param string $f Permitted address-column name. + * + * @return string + */ +function fv_nbar_local_sql(string $f): string { + if (!in_array($f, ['d.src', 'd.dst'], true)) { + throw new InvalidArgumentException('Invalid address field'); + } + $clauses = []; + + foreach ($GLOBALS['fv_nbar_preferences']['prefixes'] as $cidr) { + [$ip,$bits] = explode('/', $cidr); + $packed = (string) inet_pton($ip); + $bits = (int) $bits; + $whole = intdiv($bits, 8); + $rest = $bits % 8; + $checks = ['LENGTH(' . $f . ')=' . strlen($packed)]; + + if ($whole) { + $checks[] = "LEFT(HEX($f)," . ($whole * 2) . ")='" . strtoupper(bin2hex(substr($packed, 0, $whole))) . "'"; + } + + if ($rest) { + $mask = (255 << (8 - $rest)) & 255; + $value = ord($packed[$whole]) & $mask; + $checks[] = "(ORD(SUBSTRING($f," . ($whole + 1) . ",1)) & $mask)=$value"; + } + $clauses[] = '(' . implode(' AND ', $checks) . ')'; + } + + return $clauses ? '(' . implode(' OR ', $clauses) . ')' : '(0)'; +} +/** + * Render protected capture options, networks and display preferences. + * + * @param array $devices Available capture definitions. + * @param string $base Report base URL. + * @param string $mode Current report mode. + * @param string $aid Current application identifier. + * @param string $parent_app Parent application drilldown identifier. + * + * @return void + */ +function fv_nbar_preferences_render(array $devices, string $base, string $mode, string $aid, string $parent_app): void { + global $config; + $p = $GLOBALS['fv_nbar_preferences']; + print '
    '; + + if ($GLOBALS['fv_nbar_preferences_message'] !== '') { + print '

    ' . fv_nbar_h($GLOBALS['fv_nbar_preferences_message']) . '

    '; + } + print '
    '; + + foreach (['data_unit' => [__('Data units', 'flowview'), ['Auto', 'MB', 'GB']], 'rate_unit' => [__('Bandwidth units', 'flowview'), ['Auto', 'Kbps', 'Mbps']], 'dns' => [__('DNS names', 'flowview'), ['1' => __('Show', 'flowview'), '0' => __('Hide', 'flowview')]]] as $key => $definition) { + print '
    '; + } + print '
    ' . html_escape(__('Options', 'flowview')) . '

    ' . html_escape(__('Combine Options', 'flowview')) . '

    '; + + foreach ($devices as $d) { + print ''; + } + print '' . html_escape(__('Choose two or three captures to combine, normally one LAN capture per router. Leave all captures unticked to save Internal Networks only. Overlapping LAN and WAN captures are added together and can count traffic twice.', 'flowview')) . '
    '; + print '

    ' . html_escape(__('One internal IPv4 or IPv6 network per line. Used to identify internal hosts and upload/download direction in Conversations. Update delegated IPv6 prefixes when they change, and keep old prefixes for retained history. Exported flows do not automatically identify your site networks.', 'flowview')) . '

    '; + print ''; + print ''; +} + +/** Exclude multicast endpoints from internal/external conversations. */ +function fv_nbar_unicast_sql(string $field): string { + if (!in_array($field, ['d.src', 'd.dst'], true)) { + throw new InvalidArgumentException('Invalid address field'); + } + + return "NOT ((LENGTH($field)=4 AND (ORD(SUBSTRING($field,1,1)) & 240)=224) OR (LENGTH($field)=16 AND ORD(SUBSTRING($field,1,1))=255))"; +} diff --git a/includes/nbar_query.php b/includes/nbar_query.php new file mode 100644 index 0000000..2018904 --- /dev/null +++ b/includes/nbar_query.php @@ -0,0 +1,92 @@ + $entry) { + if ($entry['until'] <= $now) { + unset($cache[$id]); + } + } + + if (isset($cache[$key])) { + $_SESSION['fv_nbar_query_cache'] = $cache; + + return $slice === null ? $cache[$key]['rows'] : array_slice($cache[$key]['rows'], $slice, 101); + } + $rows = flowview_db_fetch_assoc_prepared($sql, $args); + + // Cache successful, compact result sets only; never retain query failures. + if (is_array($rows) && strlen(serialize($rows)) <= 524288) { + while (cacti_sizeof($cache) >= 12) { + array_shift($cache); + } + $cache[$key] = ['until' => time() + ($slice === null ? 30 : 120), 'rows' => $rows]; + } + $_SESSION['fv_nbar_query_cache'] = $cache; + + return $slice === null || !is_array($rows) ? $rows : array_slice($rows, $slice, 101); +} + +/** + * Build an exact grouped connection-page query over a trusted source. + * + * @param string $source Trusted summary or raw SQL source. + * @param string $where Validated filter predicate. + * @param int $page One-based connection page. + * + * @return string + */ +function fv_nbar_connections_sql(string $source, string $where, int $page): string { + // Use compact name ranks while grouping; resolve display names after paging. + $map = '(SELECT router,app_id,DENSE_RANK() OVER (ORDER BY name) name_id FROM plugin_flowview_nbar_names)'; + $group = 'd.app_id,COALESCE(n.name_id,0),d.src,d.dst,d.sport,d.dport,d.protocol,d.router'; + $inner = 'SELECT d.router,d.app_id,COALESCE(n.name_id,0) name_id,SUM(d.bytes) bytes,SUM(d.packets) packets,SUM(d.flows) records,IF(COUNT(d.first_seen)=COUNT(*),MIN(d.first_seen),NULL) first_seen,IF(COUNT(d.last_seen)=COUNT(*),MAX(d.last_seen),NULL) last_seen,d.src,d.dst,d.sport,d.dport,d.protocol FROM ' . $source . ' d JOIN plugin_flowview_devices l ON l.id=d.listener_id LEFT JOIN ' . $map . ' n ON n.router=COALESCE(NULLIF(d.router,SPACE(0)),l.allowfrom) AND n.app_id=d.app_id WHERE ' . $where . ' GROUP BY ' . $group . ' ORDER BY bytes DESC,' . $group . ' LIMIT 101 OFFSET ' . (($page - 1) * 100); + + return 'SELECT d.router,d.app_id,n.name,d.bytes,d.packets,d.records,d.first_seen,d.last_seen,INET6_NTOA(d.src) src,INET6_NTOA(d.dst) dst,d.sport,d.dport,d.protocol FROM (' . $inner . ') d LEFT JOIN (SELECT name,DENSE_RANK() OVER (ORDER BY name) name_id FROM plugin_flowview_nbar_names GROUP BY name) n ON n.name_id=d.name_id ORDER BY d.bytes DESC,d.app_id,d.name_id,d.src,d.dst,d.sport,d.dport,d.protocol,d.router'; +} diff --git a/includes/nbar_seen.php b/includes/nbar_seen.php new file mode 100644 index 0000000..cc4a5ac --- /dev/null +++ b/includes/nbar_seen.php @@ -0,0 +1,205 @@ + 0 ? date('Y-m-d H:i', $stamp) : __('Unavailable', 'flowview'); + $hint = __('FlowView observation within the selected period; minute precision. Time zone: %s', date_default_timezone_get(), 'flowview'); + $html .= '' . fv_nbar_h($text) . ''; + } + + return $html; +} +/** + * Merge observation dates while preserving missing-date information. + * + * @param array $total Accumulated dates and missing-date marker. + * @param array $row Exported tuple or observation fields. + * + * @return void + */ +function fv_nbar_seen_merge(array &$total, array $row): void { + if (!isset($row['first_seen'],$row['last_seen'])) { + $total['_seen_missing'] = true; + + return; + } + $first = (int) $row['first_seen']; + $last = (int) $row['last_seen']; + $total['first_seen'] = $total['first_seen'] === null ? $first : min($total['first_seen'], $first); + $total['last_seen'] = $total['last_seen'] === null ? $last : max($total['last_seen'], $last); +} +/** + * Backfill dates only where retained counters exactly match source totals. + * + * @param int $budget Worker time budget in seconds. + * + * @return int + */ +function fv_nbar_seen_backfill(int $budget = 10): int { + if ((int) flowview_db_fetch_cell("SELECT GET_LOCK(CONCAT('flowview_nbar_summary:',LEFT(SHA2(DATABASE(),256),16)),0)") !== 1) { + return 0; + } + $daily = false; + $done = 0; + $began = microtime(true); + $old = null; + $isolation = flowview_db_fetch_cell('SELECT @@session.transaction_isolation', '', false); + + if (!in_array($isolation, ['READ-UNCOMMITTED', 'READ-COMMITTED', 'REPEATABLE-READ', 'SERIALIZABLE'], true)) { + $isolation = flowview_db_fetch_cell('SELECT @@session.tx_isolation', '', false); + } + + try { + $daily = (int) flowview_db_fetch_cell("SELECT GET_LOCK(CONCAT('flowview_nbar_daily:',LEFT(SHA2(DATABASE(),256),16)),0)") === 1; + + if (!$daily) { + return 0; + } + + if (!in_array($isolation, ['READ-UNCOMMITTED', 'READ-COMMITTED', 'REPEATABLE-READ', 'SERIALIZABLE'], true)) { + return 0; + } + $old = fv_nbar_worker_timeout(20); + + if ($old === null || !flowview_db_execute('SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED')) { + return 0; + } + $jobs = flowview_db_fetch_assoc('SELECT kind,listener_id,period,phase,app_cursor FROM plugin_flowview_nbar_seen_upgrade WHERE done=0 ORDER BY kind,period DESC,listener_id LIMIT 100'); + + foreach ($jobs as $job) { + if (microtime(true) - $began >= $budget) { + break; + } + $listener = (int) $job['listener_id']; + $period = (int) $job['period']; + $isDay = (int) $job['kind'] === 1; + $end = $period + 3600; + $prefix = ''; + + if ($isDay) { + $prefix = 'daily_'; + $end = (int) flowview_db_fetch_cell_prepared('SELECT covered_to FROM plugin_flowview_nbar_days WHERE listener_id=? AND day=?', [$listener, $period]); + } + $ok = true; + $phase = 0; + + foreach (['apps' => 'app_id,LENGTH(src) family', 'pairs' => 'app_id,src,dst', 'connections' => 'app_id,src,dst,sport,dport,protocol'] as $kind => $fields) { + if ($isDay && $phase < (int) $job['phase']) { + $phase++; + + continue; + } + $group = ($kind === 'apps' ? 'app_id,LENGTH(src)' : $fields) . ',router'; + $keys = array_merge($kind === 'apps' ? ['app_id', 'family'] : explode(',', $fields), ['router']); + $column = flowview_db_fetch_row("SHOW FULL COLUMNS FROM plugin_flowview_nbar_$prefix$kind LIKE 'app_id'"); + $collation = $column['Collation']; + + if (!preg_match('/^[a-zA-Z0-9_]+$/D', $collation)) { + throw new RuntimeException('Invalid application collation'); + } + $join = implode(' AND ', array_map(static function (string $k) use ($collation): string {return $k === 'app_id' ? "s.$k=r.$k COLLATE $collation" : "s.$k=r.$k"; }, $keys)); + $source = $isDay ? fv_nbar_hourly_source($kind, [$listener], $period, $end) : fv_nbar_data_source(fv_nbar_data_columns(), "listener_id=$listener AND minute>=$period AND minute<$end", $period, $end); + $dates = 'IF(COUNT(first_seen)=COUNT(*),MIN(first_seen),NULL) first_seen,IF(COUNT(last_seen)=COUNT(*),MAX(last_seen),NULL) last_seen'; + $sql = "UPDATE plugin_flowview_nbar_$prefix$kind s JOIN (SELECT $fields,router,SUM(bytes) bytes,SUM(packets) packets,SUM(flows) flows,$dates FROM $source d WHERE listener_id=$listener AND minute>=$period AND minute<$end GROUP BY $group) r ON $join SET s.first_seen=r.first_seen,s.last_seen=r.last_seen WHERE s.listener_id=$listener AND s.hour=$period AND s.bytes=r.bytes AND s.packets=r.packets AND s.flows=r.flows AND (s.first_seen IS NULL OR s.last_seen IS NULL)"; + $applications = [null]; + + if ($isDay && $kind === 'connections') { + $applications = array_column(flowview_db_fetch_assoc_prepared('SELECT DISTINCT app_id FROM plugin_flowview_nbar_daily_apps WHERE listener_id=? AND hour=? AND app_id>? ORDER BY app_id', [$listener, $period, ($job['app_cursor'] ?? '')]), 'app_id'); + } + + foreach ($applications as $application) { + if (microtime(true) - $began >= $budget) { + $ok = false; + + break; + } + $statement = $sql; + + if ($application !== null) { + $value = 'CONVERT(0x' . bin2hex($application) . ' USING utf8mb4) COLLATE ' . $collation; + $statement = str_replace(' GROUP BY ', " AND app_id=$value GROUP BY ", $statement); + $statement .= " AND s.app_id=$value"; + } + + if (flowview_db_execute($statement) === false) { + $ok = false; + + break; + } + + if ($application !== null) { + flowview_db_execute_prepared('UPDATE plugin_flowview_nbar_seen_upgrade SET app_cursor=? WHERE kind=? AND listener_id=? AND period=?', [$application, $job['kind'], $listener, $period]); + } + } + + if (!$ok) { + break; + } + $phase++; + + if ($isDay) { + flowview_db_execute_prepared('UPDATE plugin_flowview_nbar_seen_upgrade SET phase=? WHERE kind=? AND listener_id=? AND period=?', [$phase, $job['kind'], $listener, $period]); + } + } + + if (!$ok) { + break; + } + + if (flowview_db_execute_prepared('UPDATE plugin_flowview_nbar_seen_upgrade SET done=1 WHERE kind=? AND listener_id=? AND period=?', [$job['kind'], $listener, $period]) === false) { + break; + } + $done++; + } + } catch (Throwable $e) { + cacti_log('NBAR observation upgrade: ' . $e->getMessage(), false, 'FLOWVIEW'); + } finally { + if ($old !== null) { + flowview_db_execute('SET SESSION max_statement_time=' . $old); + } + + if (in_array($isolation, ['READ-UNCOMMITTED', 'READ-COMMITTED', 'REPEATABLE-READ', 'SERIALIZABLE'], true)) { + flowview_db_execute('SET SESSION TRANSACTION ISOLATION LEVEL ' . str_replace('-', ' ', $isolation)); + } + + if ($daily) { + flowview_db_fetch_cell("SELECT RELEASE_LOCK(CONCAT('flowview_nbar_daily:',LEFT(SHA2(DATABASE(),256),16)))"); + } + flowview_db_fetch_cell("SELECT RELEASE_LOCK(CONCAT('flowview_nbar_summary:',LEFT(SHA2(DATABASE(),256),16)))"); + } + + return $done; +} diff --git a/includes/nbar_summary.php b/includes/nbar_summary.php new file mode 100644 index 0000000..096318e --- /dev/null +++ b/includes/nbar_summary.php @@ -0,0 +1,255 @@ + $statements One prepared INSERT per raw slice, as [sql, values]. + * @param int $listener Capture identifier. + * @param array $hours Affected UTC hour boundaries. + * + * @return bool + */ +function fv_nbar_summary_store(array $statements, int $listener, array $hours): bool { + if (flowview_db_execute('START TRANSACTION') === false) { + return false; + } + $ok = true; + + foreach ($statements as [$sql, $args]) { + if (flowview_db_execute_prepared($sql, $args) === false) { + $ok = false; + + break; + } + } + + if ($ok) { + $values = []; + + foreach (array_keys($hours) as $hour) { + $values[] = '(' . (int) $listener . ',' . (int) $hour . ',1,-1)'; + } + $ok = flowview_db_execute('INSERT INTO plugin_flowview_nbar_hours (listener_id,hour,revision,built) VALUES ' . implode(',', $values) . ' ON DUPLICATE KEY UPDATE revision=revision+1') !== false; + } + + if (!$ok) { + flowview_db_execute('ROLLBACK'); + + return false; + } + + return flowview_db_execute('COMMIT') !== false; +} + +/** + * Maintain closed-hour summaries under a database-scoped worker lock. + * + * @param int $budget Worker time budget in seconds. + * + * @return int + */ +function fv_nbar_summary_refresh(int $budget = 20, bool $supervised = false): int { + $lock = flowview_db_fetch_row("SELECT GET_LOCK(CONCAT('flowview_nbar_summary:',LEFT(SHA2(DATABASE(),256),16)),0) acquired"); + + if ((int) ($lock['acquired'] ?? 0) !== 1) { + return 0; + } + $done = 0; + $started = microtime(true); + + try { + $cutoff = time() - fv_nbar_retention_days() * 86400; + $closed = (int) (floor((time() - 300) / 3600) * 3600); + $listeners = flowview_db_fetch_assoc('SELECT id FROM plugin_flowview_devices'); + + foreach ($listeners as $listener) { + $id = (int) $listener['id']; + $first = fv_nbar_data_edge($id); + + if (!isset($first['minute'])) { + continue; + } + $values = []; + + for ($h = (int) (floor(max($cutoff, (int) $first['minute']) / 3600) * 3600); $h < $closed; $h += 3600) { + $values[] = "($id,$h,0,-1)"; + } + + if ($values && flowview_db_execute('INSERT IGNORE INTO plugin_flowview_nbar_hours (listener_id,hour,revision,built) VALUES ' . implode(',', $values)) === false) { + return $done; + } + } + $pending = flowview_db_fetch_assoc('SELECT listener_id,hour,revision FROM plugin_flowview_nbar_hours WHERE hour>=' . (int) (ceil($cutoff / 3600) * 3600) . " AND hour<$closed AND (built<0 OR built<>revision OR connection_built<0 OR connection_built<>revision) ORDER BY hour DESC,listener_id LIMIT 100"); + + foreach ($pending as $item) { + if (microtime(true) - $started >= max(1, (int) $budget)) { + break; + } + + if (fv_nbar_summary_build((int) $item['listener_id'], (int) $item['hour'], $supervised)) { + $done++; + } + } + } finally { + flowview_db_fetch_row("SELECT RELEASE_LOCK(CONCAT('flowview_nbar_summary:',LEFT(SHA2(DATABASE(),256),16))) released"); + } + + return $done + fv_nbar_daily_refresh() + fv_nbar_seen_backfill(5); +} + +/** + * Publish an hourly summary only if its raw revision remains unchanged. + * + * @param int $listener Capture identifier. + * @param int $hour UTC hour boundary in Unix seconds. + * + * @return bool + */ +function fv_nbar_summary_build(int $listener, int $hour, bool $supervised = false): bool { + // A MySQL build without DML timeouts is allowed only in the supervised child. + $old_timeout = fv_nbar_worker_timeout(20); + + if ($old_timeout === null && !$supervised) { + return false; + } + + try { + // READ COMMITTED avoids holding raw-record locks during INSERT ... SELECT. + if (flowview_db_execute('SET TRANSACTION ISOLATION LEVEL READ COMMITTED') === false || flowview_db_execute('START TRANSACTION') === false) { + return false; + } + $state = flowview_db_fetch_row_prepared('SELECT revision FROM plugin_flowview_nbar_hours WHERE listener_id=? AND hour=?', [$listener, $hour]); + + if (!isset($state['revision'])) { + throw new RuntimeException('Missing summary state'); + } + $raw = fv_nbar_data_source('listener_id,minute,app_id,src,dst,sport,dport,protocol,bytes,packets,flows,router', 'listener_id=' . $listener . ' AND minute>=' . $hour . ' AND minute<' . ($hour + 3600), $hour, $hour + 3600); + $where = ' FROM ' . $raw . ' fv_nbar_hour'; + $specs = [ + 'apps' => 'app_id,LENGTH(src)', + 'pairs' => 'app_id,src,dst', + 'connections' => 'app_id,src,dst,sport,dport,protocol' + ]; + + foreach ($specs as $table => $fields) { + if (flowview_db_execute_prepared("DELETE FROM plugin_flowview_nbar_$table WHERE listener_id=? AND hour=?", [$listener, $hour]) === false) { + throw new RuntimeException('Summary delete failed'); + } + $sql = "INSERT INTO plugin_flowview_nbar_$table SELECT $listener,$hour,$fields,SUM(bytes),SUM(packets),SUM(flows),MIN(minute),MAX(minute),router$where GROUP BY $fields,router"; + + if (flowview_db_execute($sql) === false) { + throw new RuntimeException('Summary aggregate failed'); + } + } + $latest = flowview_db_fetch_row_prepared('SELECT revision FROM plugin_flowview_nbar_hours WHERE listener_id=? AND hour=?', [$listener, $hour]); + + if (!isset($latest['revision']) || (string) $latest['revision'] !== (string) $state['revision']) { + throw new RuntimeException('Hour changed during summary build'); + } + + if (flowview_db_execute_prepared('UPDATE plugin_flowview_nbar_hours SET built=?,connection_built=? WHERE listener_id=? AND hour=?', [$state['revision'], $state['revision'], $listener, $hour]) === false) { + throw new RuntimeException('Summary publish failed'); + } + + if (flowview_db_execute('COMMIT') === false) { + throw new RuntimeException('Summary commit failed'); + } + + return true; + } catch (Throwable $e) { + flowview_db_execute('ROLLBACK'); + + return false; + } finally { + if ($old_timeout !== null) { + flowview_db_execute('SET SESSION max_statement_time=' . $old_timeout); + } + } +} + +/** + * Choose mutually exclusive ready-summary or raw branches for each block. + * + * @param string $kind Allowlisted summary type. + * @param array $listeners Selected capture identifiers. + * @param int $start Inclusive range start in Unix seconds. + * @param int $end Exclusive range end in Unix seconds. + * @param int|null $now Current Unix time, or null to read the clock. + * + * @return string + */ +function fv_nbar_hourly_source(string $kind, array $listeners, int $start, int $end, ?int $now = null): string { + if (!in_array($kind, ['apps', 'pairs', 'connections'], true)) { + throw new InvalidArgumentException('Unknown summary kind'); + } + $listeners = array_values(array_unique(array_filter(array_map('intval', $listeners), static function (int $id): bool {return $id > 0; }))); + $start = (int) $start; + $end = (int) $end; + $raw = fv_nbar_data_columns(); + $tables = fv_nbar_data_tables(); + + if (!$listeners) { + return fv_nbar_data_source($raw, "minute>=$start AND minute<$end", $start, $end, $tables); + } + $full_start = (int) (ceil($start / 3600) * 3600); + $full_end = min((int) (floor($end / 3600) * 3600), (int) (floor((($now ?? time()) - 300) / 3600) * 3600)); + + if ($full_end <= $full_start) { + return fv_nbar_data_source($raw, 'listener_id IN (' . implode(',', $listeners) . ") AND minute>=$start AND minute<$end", $start, $end, $tables); + } + $built_column = $kind === 'connections' ? 'connection_built' : 'built'; + $parts = []; + + foreach ($listeners as $listener) { + if ($start < $full_start) { + array_push($parts, ...fv_nbar_data_parts($raw, "listener_id=$listener AND minute>=$start AND minute<$full_start", $start, $full_start, $tables)); + } + + if ($full_end < $end) { + array_push($parts, ...fv_nbar_data_parts($raw, "listener_id=$listener AND minute>=$full_end AND minute<$end", $full_end, $end, $tables)); + } + + for ($h = $full_start; $h < $full_end; $h = $to) { + $to = min($h + 21600, $full_end); + $count = (int) (($to - $h) / 3600); + $ready = "(SELECT COUNT(*) FROM plugin_flowview_nbar_hours WHERE listener_id=$listener AND hour>=$h AND hour<$to AND $built_column>=0 AND $built_column=revision)=$count"; + + if ($kind === 'apps') { + $fields = 'listener_id,hour minute,app_id,REPEAT(CHAR(0),family) src,REPEAT(CHAR(0),family) dst,0 sport,0 dport,0 protocol,bytes,packets,flows,first_seen,last_seen,router'; + } elseif ($kind === 'pairs') { + $fields = 'listener_id,hour minute,app_id,src,dst,0 sport,0 dport,0 protocol,bytes,packets,flows,first_seen,last_seen,router'; + } else { + $fields = 'listener_id,hour minute,app_id,src,dst,sport,dport,protocol,bytes,packets,flows,first_seen,last_seen,router'; + } + $parts[] = "SELECT $fields FROM plugin_flowview_nbar_$kind WHERE listener_id=$listener AND hour>=$h AND hour<$to AND ($ready)"; + array_push($parts, ...fv_nbar_data_parts($raw, "listener_id=$listener AND minute>=$h AND minute<$to AND NOT ($ready)", $h, $to, $tables)); + } + } + + return '(' . implode(' UNION ALL ', $parts) . ')'; +} diff --git a/includes/nbar_timespan.php b/includes/nbar_timespan.php new file mode 100644 index 0000000..fc44a45 --- /dev/null +++ b/includes/nbar_timespan.php @@ -0,0 +1,96 @@ + $label) { + $span = []; + get_timespan($span, $now, $id, (int) $weekday); + $from = strtotime($span['current_value_date1']); + $to = strtotime($span['current_value_date2']); + + if ($from >= $cutoff && $to > $from && $to <= $now) { + // The exact retention option below replaces duplicate rolling periods. + if ($from === $cutoff && $to === $now) { + continue; + } + $options[$id] = $label; + } + } + $options[1000] = $days === 1 ? __('Last Day', 'flowview') : __('Last %s Days', $days, 'flowview'); + + return $options; +} +/** + * Clamp a requested range to retained history and the current time. + * + * @param int $start Inclusive range start in Unix seconds. + * @param int $end Exclusive range end in Unix seconds. + * @param int $now Current Unix time. + * @param int $days Configured retention in days. + * + * @return array + */ +function fv_nbar_bound_dates(int $start, int $end, int $now, int $days): array { + $cutoff = $now - $days * 86400; + $from = max($cutoff, $start); + $to = min($now, $end); + + if ($to <= $from) { + return [$now - 1800, $now, true]; + } + + return [$from, $to, $from !== $start || $to !== $end]; +} + +/** Handle NBAR timespans without mutating legacy Flow Reports session state. */ +function fv_nbar_timespan_request(): bool { + if (get_nfilter_request_var('report') !== 'nbar') { + return false; + } + header('Content-Type: application/json'); + + if ((int) get_filter_request_var('predefined_timespan') === 1000) { + $end = (int) (floor(time() / 60) * 60); + $start = $end - fv_nbar_retention_days() * 86400; + print json_encode(['current_value_date1' => date('Y-m-d H:i', $start), 'current_value_date2' => date('Y-m-d H:i', $end), 'begin_now' => $start, 'end_now' => $end]); + } else { + flowview_gettimespan(); + } + + return true; +} diff --git a/js/nbar-preferences.js b/js/nbar-preferences.js new file mode 100644 index 0000000..ee5ac7b --- /dev/null +++ b/js/nbar-preferences.js @@ -0,0 +1,96 @@ +/* Copyright (C) 2004-2026 The Cacti Group. GPL-2.0-or-later. */ + +$(function () { + var prefs = window.fvNbarDisplay; + var combineDialog = document.getElementById('nbarCombineDialog'), + combineSnapshot; + $('#nbarCombineOptions').on('click', function () { + combineSnapshot = { + prefixes: $('#nbarPrefixes').val(), + picks: $('#nbarCombineDialog input[type=checkbox]') + .map(function () { + return this.checked; + }) + .get(), + }; + if (!combineDialog.open) combineDialog.showModal(); + }); + $('#nbarCombineClose,#nbarCombineCancel').on('click', function () { + combineDialog.close(); + }); + $(combineDialog).on('close', function () { + if (!combineSnapshot) return; + $('#nbarPrefixes').val(combineSnapshot.prefixes); + $('#nbarCombineDialog input[type=checkbox]').each(function (i) { + this.checked = combineSnapshot.picks[i]; + }); + updateRouterSelection(); + }); + + var routerPicks = $('#nbarPreferences input[name="site_listeners[]"]'); + function updateRouterSelection() { + var count = routerPicks.filter(':checked').length; + routerPicks.not(':checked').prop('disabled', count >= 3); + $('#nbarRouterSelectionCount').text( + prefs.labels.selection.replace('{count}', count) + + (count === 1 ? prefs.labels.single : count === 0 ? prefs.labels.off : ''), + ); + $('#nbarCombineSave').prop('disabled', count === 1 || count > 3); + } + $('#nbarCombineSave').on('click', function () { + var count = routerPicks.filter(':checked').length; + if (count === 1 || count > 3) return false; + if (count >= 2) $('#nbar_filter [name=listener]').val('0'); + $('', { type: 'hidden', name: 'combine_apply', value: '1' }).appendTo('#nbar_filter'); + }); + routerPicks.on('change', updateRouterSelection); + updateRouterSelection(); + $('#nbarTable,#nbarConversationTable').each(function () { + var table = $(this); + table.find('thead th').each(function (index) { + var th = $(this), + label = th.text(), + data = th.attr('data-nbar-unit') === 'data', + rate = th.attr('data-nbar-unit') === 'rate', + dns = th.attr('data-nbar-unit') === 'dns'; + if (dns && prefs.dns === '0') { + th.hide(); + table.find('tbody tr').each(function () { + $(this).children().eq(index).hide(); + }); + } + if (!data && !rate) return; + var choice = data ? prefs.data : prefs.rate; + th.text( + label.replace(data ? 'GB' : 'Mbps', choice === 'Auto' ? (data ? 'MB / GB' : 'Kbps / Mbps') : choice), + ); + th.attr('title', rate ? prefs.labels.rate : prefs.labels.data); + table.find('tbody tr').each(function () { + var cell = $(this).children().eq(index), + value = Number(cell.attr('data-text')); + if (!cell.is('[data-text]') || !Number.isFinite(value) || value < 0) return; + var unit = choice, + result = value; + if (data) { + if (choice === 'Auto') unit = value >= 1 ? 'GB' : 'MB'; + if (unit === 'MB') result = value * 1000; + } else { + if (choice === 'Auto') unit = value >= 1 ? 'Mbps' : 'Kbps'; + if (unit === 'Kbps') result = value * 1000; + } + var precision = choice === 'Auto' ? 2 : data ? 4 : 3, + threshold = Math.pow(10, -precision); + var text = + result > 0 && result < threshold + ? '<' + threshold.toFixed(precision) + : result.toLocaleString(undefined, { + minimumFractionDigits: precision, + maximumFractionDigits: precision, + }); + cell.text(text + (choice === 'Auto' ? ' ' + unit : '')); + cell.attr('title', data ? (value * 1e9).toLocaleString() + ' bytes' : value.toLocaleString() + ' Mbps'); + }); + }); + if (prefs.dns === '0') table.find('tbody td small').hide(); + }); +}); diff --git a/js/nbar-report.js b/js/nbar-report.js new file mode 100644 index 0000000..16b5a33 --- /dev/null +++ b/js/nbar-report.js @@ -0,0 +1,45 @@ +/* Copyright (C) 2004-2026 The Cacti Group. GPL-2.0-or-later. */ +$(function () { + $('#nbarSetupLink').on('click', function (e) { + e.preventDefault(); + var guide = document.getElementById('nbarSetupGuide'); + if (!guide.open) guide.showModal(); + }); +}); +$(function () { + $('#nbar_filter [name=mode]').on('change', function () { + $('#nbar_filter [name=app],#nbar_filter [name=internal],#nbar_filter [name=external]').remove(); + }); +}); + +$(function () { + function refreshTimespan() { + var v = $('#nbar_period').val(); + if (v !== '0') + $.getJSON('flowview.php', { action: 'gettimespan', report: 'nbar', predefined_timespan: v }, function (d) { + $('#nbar_date1').val(d.current_value_date1); + $('#nbar_date2').val(d.current_value_date2); + }); + } + function flowReturn(report) { + var u = new URL($('#nbar_return').attr('href'), location.href); + if (report) u.searchParams.set('report', report); + u.searchParams.set('device_id', $('#nbar_filter [name=listener]').val()); + u.searchParams.set('predefined_timespan', $('#nbar_period').val() === '1000' ? '0' : $('#nbar_period').val()); + u.searchParams.set('date1', $('#nbar_date1').val()); + u.searchParams.set('date2', $('#nbar_date2').val()); + location.href = u.href; + } + $('#nbar_report_select').on('change', function () { + if (this.value !== 'nbar') flowReturn(this.value); + }); + $('#nbar_return').on('click', function (e) { + e.preventDefault(); + flowReturn(); + }); + $('#nbar_period').on('change', refreshTimespan); + $('#nbar_date1,#nbar_date2').on('change', function () { + $('#nbar_period').val('0'); + if (theme !== 'classic') $('#nbar_period').selectmenu('refresh'); + }); +}); diff --git a/js/nbar-setup-guide.js b/js/nbar-setup-guide.js new file mode 100644 index 0000000..2eb47d2 --- /dev/null +++ b/js/nbar-setup-guide.js @@ -0,0 +1,6 @@ +/* Copyright (C) 2004-2026 The Cacti Group. GPL-2.0-or-later. */ +$(function () { + $('#nbarSetupClose').on('click', function () { + document.getElementById('nbarSetupGuide').close(); + }); +}); diff --git a/js/nbar-tables.js b/js/nbar-tables.js new file mode 100644 index 0000000..bf864c7 --- /dev/null +++ b/js/nbar-tables.js @@ -0,0 +1,142 @@ +/* Copyright (C) 2004-2026 The Cacti Group. GPL-2.0-or-later. */ + +window.flowviewNbarInitTable = function (spec) { + $(function () { + var table = $(spec.table); + if (!table.length || !$.fn.tablesorter) return; + + var columns = table.find('th').length; + var firstNumber = spec.conversations ? 3 : columns - 6; + var headers = {}; + var savedSort = [[firstNumber, 1]]; + var storage; + try { + storage = spec.conversations ? localStorage : sessionStorage; + var saved = JSON.parse(storage.getItem(spec.sortKey)); + if ( + Array.isArray(saved) && + saved.length && + saved.every(function (item) { + return ( + Array.isArray(item) && + Number.isInteger(item[0]) && + item[0] >= 0 && + item[0] < columns && + (item[1] === 0 || item[1] === 1) + ); + }) + ) + savedSort = saved; + } catch (error) {} + + function saveSort() { + try { + if (storage && table[0].config) { + storage.setItem(spec.sortKey, JSON.stringify(table[0].config.sortList)); + } + } catch (error) {} + } + + for (var index = firstNumber; index < columns; index++) { + headers[index] = { sorter: 'digit', sortInitialOrder: 'desc' }; + } + + var widths = spec.conversations + ? [20, 20, 12, 8, 8, 8, 8, 8, 8] + : columns === 14 + ? [9, 11, 10, 5, 11, 10, 5, 5, 7, 7, 5, 5, 5, 5] + : [28, 14, 18, 10, 10, 10, 10]; + table + .tablesorter({ + headers: headers, + sortList: savedSort, + initialized: function (element) { + // Cacti also initializes jQuery UI handles. Use the table widget's + // handles consistently so saved widths and Safari dragging agree. + var config = element.config; + var widget = $.tablesorter.getWidgetById('resizable'); + if (!widget || !$.tablesorter.resizable) return; + config.widgetOptions = $.extend({}, widget.options, config.widgetOptions); + var options = config.widgetOptions; + // Old saved widths can exceed the saved table width, leaving the + // last column with no space. Repair them before the widget loads. + if ($.tablesorter.storage) { + try { + var stored = $.tablesorter.storage(element, 'tablesorter-resizable'); + if (Array.isArray(stored) && stored.length) { + var valid = + stored.length === columns && + stored.every(function (width) { + return typeof width === 'number' && Number.isFinite(width) && width >= 0; + }); + var safe = valid + ? stored.map(function (width, index) { + if (!config.$headerIndexed[index].is(':visible')) return 0; + return Math.max(index >= columns - 2 ? 136 : 48, width); + }) + : []; + $.tablesorter.storage(element, 'tablesorter-resizable', safe); + $.tablesorter.storage( + element, + 'tablesorter-table-resized-width', + safe.length + ? Math.max( + $(element).parent().width(), + safe.reduce(function (sum, width) { + return sum + width + 8; + }, 0), + ) + : 'auto', + ); + } + } catch (error) {} + } + $(element).removeClass('hasResizable'); + $.tablesorter.resizable.init(config, options); + config.widgets.push('resizable'); + config.widgetInit.resizable = true; + options.$resizable_container + .addClass('flowview-nbar-resize') + .children() + .attr('title', spec.resizeLabel) + .each(function (index) { + var header = $(config.$headerIndexed[index]); + $(this).css('color', header.find('.textSubHeaderDark').css('color') || header.css('color')); + }); + $(element).trigger('resizableUpdate'); + }, + widgets: ['zebra'], + widgetOptions: { + resizable: true, + storage_tableId: spec.conversations + ? 'flowview-nbar-conversation-columns-v2' + : 'flowview-nbar-columns-v2-' + columns, + resizable_widths: widths.map(function (width) { + return width + '%'; + }), + }, + widgetZebra: { css: ['even', 'odd'] }, + headerTemplate: '
    {content} {icon}
    ', + cssIconAsc: 'fa-sort-up', + cssIconDesc: 'fa-sort-down', + cssIconNone: 'fa-sort', + cssIcon: 'fa', + }) + .on('sortEnd', saveSort); + + $('.nbarPageLink').on('click', saveSort); + $('.nbarPageJump').on('click', function () { + var input = $(this).closest('.nbarPagination').find('.nbarPageNumber')[0]; + if (input && input.checkValidity()) { + saveSort(); + location.href = this.dataset.url + '&page=' + Number.parseInt(input.value, 10); + } + }); + $('.nbarPageNumber').on('keydown', function (event) { + if (event.key === 'Enter') { + event.preventDefault(); + $(this).closest('.nbarPagination').find('.nbarPageJump').trigger('click'); + } + }); + }); +}; diff --git a/locales/po/cacti.pot b/locales/po/cacti.pot index 23efd5d..86e6f5a 100644 --- a/locales/po/cacti.pot +++ b/locales/po/cacti.pot @@ -6,11 +6,12 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: Cacti \n" +"Project-Id-Version: Cacti 5.1\n" "Report-Msgid-Bugs-To: developers@cacti.net\n" -"POT-Creation-Date: 2026-10-12 14:40-0400\n" +"POT-Creation-Date: 2026-10-19 12:10-0400 +" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: Cacti Developers >\n" +"Last-Translator: Cacti Developers \n" "Language-Team: Cacti Developers \n" "Language: \n" "MIME-Version: 1.0\n" @@ -480,7 +481,7 @@ msgid "Default" msgstr "" #: flowview_databases.php flowview_devices.php flowview_filters.php -#: flowview_schedules.php includes/functions.php +#: flowview_schedules.php includes/functions.php nbar_report.php msgid "Go" msgstr "" @@ -490,7 +491,7 @@ msgid "Set/Refresh Filters" msgstr "" #: flowview_databases.php flowview_devices.php flowview_filters.php -#: flowview_schedules.php includes/functions.php +#: flowview_schedules.php includes/functions.php nbar_report.php msgid "Clear" msgstr "" @@ -518,7 +519,7 @@ msgstr[0] "" msgstr[1] "" #: flowview_databases.php flowview_devices.php flowview_filters.php -#: flowview_schedules.php includes/functions.php +#: flowview_schedules.php includes/functions.php includes/nbar_preferences.php msgid "Cancel" msgstr "" @@ -714,6 +715,7 @@ msgid "Port this collector will listen on." msgstr "" #: flowview_devices.php includes/arrays.php includes/functions.php +#: nbar_report.php msgid "Protocol" msgstr "" @@ -1240,6 +1242,18 @@ msgstr "" msgid "Yes" msgstr "" +#: flowview_sections.php +msgid "FlowView sections" +msgstr "" + +#: flowview_sections.php +msgid "Cisco Apps (NBAR)" +msgstr "" + +#: flowview_sections.php +msgid "Flow Reports" +msgstr "" + #: includes/arrays.php includes/functions.php msgid "Select a Report" msgstr "" @@ -1272,11 +1286,11 @@ msgstr "" msgid "UDP/TCP Port" msgstr "" -#: includes/arrays.php includes/functions.php +#: includes/arrays.php includes/functions.php nbar_report.php msgid "Source IP" msgstr "" -#: includes/arrays.php +#: includes/arrays.php nbar_report.php msgid "Destination IP" msgstr "" @@ -1398,7 +1412,7 @@ msgstr "" msgid "Bytes" msgstr "" -#: includes/arrays.php includes/functions.php +#: includes/arrays.php includes/functions.php nbar_report.php msgid "Packets" msgstr "" @@ -1913,7 +1927,7 @@ msgstr "" msgid "Edit the Flow Filter" msgstr "" -#: includes/functions.php +#: includes/functions.php includes/nbar_preferences.php msgid "Save" msgstr "" @@ -1992,19 +2006,19 @@ msgstr "" msgid "Show the post-NAT (translated) IP/DNS name instead of the original values" msgstr "" -#: includes/functions.php +#: includes/functions.php nbar_report.php msgid "Timespan" msgstr "" -#: includes/functions.php +#: includes/functions.php nbar_report.php msgid "Custom" msgstr "" -#: includes/functions.php +#: includes/functions.php nbar_report.php msgid "From" msgstr "" -#: includes/functions.php +#: includes/functions.php nbar_report.php msgid "To" msgstr "" @@ -2186,7 +2200,7 @@ msgstr "" msgid "No valid report was selected. Please open a bug on GitHub" msgstr "" -#: includes/functions.php +#: includes/functions.php nbar_report.php msgid "Source DNS" msgstr "" @@ -2404,6 +2418,474 @@ msgstr "" msgid "InnoDB (Slow, High Concurrency)" msgstr "" +#: includes/nbar_chart.php +msgid "Unable to load application chart. Please try again." +msgstr "" + +#: includes/nbar_chart.php +msgid "Other" +msgstr "" + +#: includes/nbar_chart.php +msgid "Top 5 applications" +msgstr "" + +#: includes/nbar_chart.php +msgid "Top 5 applications by data" +msgstr "" + +#: includes/nbar_chart.php +msgid "Application data share for the selected period" +msgstr "" + +#: includes/nbar_chart.php +msgid "No application data in this period" +msgstr "" + +#: includes/nbar_dns.php +msgid "No reverse DNS" +msgstr "" + +#: includes/nbar_dns.php +msgid "Not resolved" +msgstr "" + +#: includes/nbar_feed_status.php +msgid "STATUS:" +msgstr "" + +#: includes/nbar_feed_status.php +msgid "Unable to load NBAR status. Please try again." +msgstr "" + +#: includes/nbar_feed_status.php +msgid "No recent NBAR" +msgstr "" + +#: includes/nbar_feed_status.php +msgid "Receiving" +msgstr "" + +#: includes/nbar_feed_status.php +#, php-format +msgid "Last NBAR record: %s" +msgstr "" + +#: includes/nbar_feed_status.php +msgid "No retained NBAR records" +msgstr "" + +#: includes/nbar_feed_status.php +msgid "No captures selected" +msgstr "" + +#: includes/nbar_preferences.php +msgid "Settings were not saved. Enter valid IPv4/IPv6 networks, or leave the list blank." +msgstr "" + +#: includes/nbar_preferences.php +msgid "Settings were not saved. Choose two or three captures to combine, or clear all selections to save Internal Networks only." +msgstr "" + +#: includes/nbar_preferences.php +msgid "Bandwidth units" +msgstr "" + +#: includes/nbar_preferences.php +msgid "DNS names" +msgstr "" + +#: includes/nbar_preferences.php +msgid "Data units" +msgstr "" + +#: includes/nbar_preferences.php +msgid "Hide" +msgstr "" + +#: includes/nbar_preferences.php nbar_report.php +msgid "Show" +msgstr "" + +#: includes/nbar_preferences.php +msgid "Auto" +msgstr "" + +#: includes/nbar_preferences.php nbar_setup_guide.php +msgid "Close" +msgstr "" + +#: includes/nbar_preferences.php +msgid "Combine Options" +msgstr "" + +#: includes/nbar_preferences.php nbar_report.php +msgid "Options" +msgstr "" + +#: includes/nbar_preferences.php +msgid "Choose two or three captures to combine, normally one LAN capture per router. Leave all captures unticked to save Internal Networks only. Overlapping LAN and WAN captures are added together and can count traffic twice." +msgstr "" + +#: includes/nbar_preferences.php +msgid "Internal Networks" +msgstr "" + +#: includes/nbar_preferences.php +msgid "One internal IPv4 or IPv6 network per line. Used to identify internal hosts and upload/download direction in Conversations. Update delegated IPv6 prefixes when they change, and keep old prefixes for retained history. Exported flows do not automatically identify your site networks." +msgstr "" + +#: includes/nbar_preferences.php +msgid " — combining off" +msgstr "" + +#: includes/nbar_preferences.php +msgid " — select another capture or untick all to save" +msgstr "" + +#: includes/nbar_preferences.php +msgid "Average rate across the selected timespan." +msgstr "" + +#: includes/nbar_preferences.php +msgid "Decimal units: 1 GB = 1,000 MB." +msgstr "" + +#: includes/nbar_preferences.php +msgid "{count} of 3 selected" +msgstr "" + +#: includes/nbar_seen.php +msgid "Unavailable" +msgstr "" + +#: includes/nbar_seen.php +#, php-format +msgid "FlowView observation within the selected period; minute precision. Time zone: %s" +msgstr "" + +#: includes/nbar_timespan.php +#, php-format +msgid "Last %s Days" +msgstr "" + +#: includes/nbar_timespan.php +msgid "Last Day" +msgstr "" + +#: nbar_conversations.php +msgid "Configure your site networks in Application settings to label internal and external devices. Top applications and Connections work directly with exported IPv4 and IPv6 addresses." +msgstr "" + +#: nbar_conversations.php +msgid "Unable to load conversations. Please try again." +msgstr "" + +#: nbar_conversations.php nbar_report.php +msgid "← Back to top applications" +msgstr "" + +#: nbar_conversations.php +msgid "Internal Host ↔ External Host. Click a device to see its individual connections. Bandwidth is averaged over the selected timespan." +msgstr "" + +#: nbar_conversations.php nbar_report.php +msgid "Application" +msgstr "" + +#: nbar_conversations.php +msgid "Download (Avg Mbps)" +msgstr "" + +#: nbar_conversations.php +msgid "Download (GB)" +msgstr "" + +#: nbar_conversations.php +msgid "External Host" +msgstr "" + +#: nbar_conversations.php nbar_report.php +msgid "First seen" +msgstr "" + +#: nbar_conversations.php +msgid "Internal Host" +msgstr "" + +#: nbar_conversations.php nbar_report.php +msgid "Last seen" +msgstr "" + +#: nbar_conversations.php +msgid "Upload (Avg Mbps)" +msgstr "" + +#: nbar_conversations.php +msgid "Upload (GB)" +msgstr "" + +#: nbar_conversations.php nbar_report.php +#, php-format +msgid "Awaiting name (%s)" +msgstr "" + +#: nbar_conversations.php +msgid "No conversations for this period. Allow 1–2 minutes for exports, or choose a longer timespan." +msgstr "" + +#: nbar_conversations.php nbar_report.php +msgid "First seen and Last seen show the earliest and latest FlowView observations within the selected period, at minute precision. They do not identify continuous activity or exact flow start/end times. Unavailable means the retained summary has no complete observation dates." +msgstr "" + +#: nbar_conversations.php nbar_report.php +msgid "Report information" +msgstr "" + +#: nbar_conversations.php +#, php-format +msgid "Top 100 conversations by combined data volume. Each row groups an Internal Host, External Host and application across both directions. Uses only the selected LAN captures to avoid counting WAN copies. Local and multicast traffic is excluded. DNS names identify servers, not the exact website requested. History retention: %d days." +msgstr "" + +#: nbar_conversations.php nbar_report.php +msgid "Drag to resize column" +msgstr "" + +#: nbar_report.php +msgid "NBAR history is being upgraded. Refresh this report shortly." +msgstr "" + +#: nbar_report.php +#, php-format +msgid "That period is outside the %s-day NBAR retention setting. Showing the last half hour." +msgstr "" + +#: nbar_report.php +#, php-format +msgid "Dates adjusted to the %s-day NBAR retention window." +msgstr "" + +#: nbar_report.php +msgid "Application Report: Cisco Apps Recognition (NBAR)" +msgstr "" + +#: nbar_report.php +msgid "Router / capture" +msgstr "" + +#: nbar_report.php +msgid "Combined routers" +msgstr "" + +#: nbar_report.php +msgid "Setup guide" +msgstr "" + +#: nbar_report.php +msgid "IP version" +msgstr "" + +#: nbar_report.php +msgid "IPv4 and IPv6" +msgstr "" + +#: nbar_report.php +msgid "Connections" +msgstr "" + +#: nbar_report.php +msgid "Conversations" +msgstr "" + +#: nbar_report.php +msgid "Top applications" +msgstr "" + +#: nbar_report.php +msgid "Any IPv4 or IPv6 address" +msgstr "" + +#: nbar_report.php +msgid "Filter by IP" +msgstr "" + +#: nbar_report.php +msgid "Show traffic where this IP is the source or destination. Leave blank to show all." +msgstr "" + +#: nbar_report.php +msgid "Unable to read the NBAR records. Please try again." +msgstr "" + +#: nbar_report.php +msgid "← Back to conversations" +msgstr "" + +#: nbar_report.php +msgid "Unable to load connections. Try a shorter timespan or filter by IP." +msgstr "" + +#: nbar_report.php +msgid "Unable to load applications. Please try again." +msgstr "" + +#: nbar_report.php +msgid "Counting total pages…" +msgstr "" + +#: nbar_report.php +msgid "Page" +msgstr "" + +#: nbar_report.php +msgid "Up to 100 connections per page, ordered by data volume." +msgstr "" + +#: nbar_report.php +msgid "Previous" +msgstr "" + +#: nbar_report.php +msgid "Next" +msgstr "" + +#: nbar_report.php +#, php-format +msgid "Application %s" +msgstr "" + +#: nbar_report.php +msgid "Destination DNS" +msgstr "" + +#: nbar_report.php +msgid "Destination port" +msgstr "" + +#: nbar_report.php +msgid "Reverse DNS hostname" +msgstr "" + +#: nbar_report.php +msgid "Source port" +msgstr "" + +#: nbar_report.php +msgid "Bandwidth (Avg Mbps)" +msgstr "" + +#: nbar_report.php +msgid "Data (GB)" +msgstr "" + +#: nbar_report.php +msgid "First FlowView observation in the selected period" +msgstr "" + +#: nbar_report.php +msgid "Last FlowView observation in the selected period" +msgstr "" + +#: nbar_report.php +msgid "Records" +msgstr "" + +#: nbar_report.php +msgid "No application records in this view yet. Try a longer period or another traffic view." +msgstr "" + +#: nbar_report.php +msgid "Total pages unavailable" +msgstr "" + +#: nbar_report.php +msgid "of {pages} pages ({connections} connections)" +msgstr "" + +#: nbar_report.php +msgid "Results are ordered by data volume; Connections shows up to 100 per page. Column sorting applies to the displayed page. NBAR history uses the configured retention setting. Daily summaries are reused across report selections; partial days use hourly summaries, and partial or changed hours use raw records. Connections pages are read from shared daily history, with nearby pages briefly cached for navigation. Average Mbps covers the selected period; allow 1–2 minutes for exports. Capture visibility depends on the device exporter and observation point. NAT can hide internal IPv4 addresses on a WAN capture. Select one observation point per traffic path to avoid duplicate totals. Cisco NBAR may classify encrypted traffic as SSL, QUIC or Unknown. History starts when export is enabled." +msgstr "" + +#: nbar_setup_guide.php +msgid "1. FlowView listener" +msgstr "" + +#: nbar_setup_guide.php +msgid "2. Router configuration example" +msgstr "" + +#: nbar_setup_guide.php +msgid "Create and enable a listener with the correct UDP port and allowed router export-source address. Start or restart the capture service after adding a listener. Allow the UDP port through the collector firewall. The export source must reach the collector. An IPv4 exporter can carry both IPv4 and IPv6 records." +msgstr "" + +#: nbar_setup_guide.php +msgid "Replace the red placeholders with your own values:" +msgstr "" + +#: nbar_setup_guide.php +msgid "Requirements" +msgstr "" + +#: nbar_setup_guide.php +msgid "Setup guide — Cisco Apps (NBAR)" +msgstr "" + +#: nbar_setup_guide.php +msgid "This configuration example is based on a Cisco 1100 Series ISR installation using NBAR2, Flexible NetFlow and application export. For other router types, refer to Cisco documentation for your exact router model and IOS XE release for supported commands, features and configuration requirements. Input/output monitoring and licence requirements vary by platform. Protocol packs must match their supported IOS XE release and NBAR engine. This guide does not apply router changes." +msgstr "" + +#: nbar_setup_guide.php +msgid "is a router interface whose address can reach the collector;" +msgstr "" + +#: nbar_setup_guide.php +msgid "is the Cacti collector address;" +msgstr "" + +#: nbar_setup_guide.php +msgid "is the listener UDP port;" +msgstr "" + +#: nbar_setup_guide.php +msgid "is the routed LAN interface to monitor. The export source identifies the router sending flow records; it does not choose which traffic is captured. It may differ from the monitored LAN VLAN. Set the FlowView listener to allow the chosen export-source IP. This example uses NetFlow v9 application records from a Cisco 1100 Series ISR. It omits optional routing metadata. Check Cisco documentation for defaults and supported record fields on your platform. The NBAR decoder currently supports application records exported using NetFlow v9; IPFIX application records are not decoded by this integration." +msgstr "" + +#: nbar_setup_guide.php +msgid "3. Capture and report settings" +msgstr "" + +#: nbar_setup_guide.php +msgid "4. Verify collection" +msgstr "" + +#: nbar_setup_guide.php +msgid "For example, replace" +msgstr "" + +#: nbar_setup_guide.php +msgid "Top applications and Connections use exported addresses directly. Conversations needs internal IPv4 and IPv6 networks in Options to distinguish Internal Hosts from External Hosts. Update delegated IPv6 prefixes when they change; retain old prefixes while viewing their history." +msgstr "" + +#: nbar_setup_guide.php +msgid "Use your monitor and exporter names. Generate traffic and check both caches and export counters. Allow 1-2 minutes for exports to appear. Check QFP load during a speed test; overhead depends on platform, traffic and capture count." +msgstr "" + +#: nbar_setup_guide.php +msgid "if that is the routed gateway interface for your LAN. Apply both the IPv4 and IPv6 monitors in input and output directions on that interface, as shown above. Input captures traffic arriving from that LAN towards the WAN; output captures traffic returning from the WAN to that LAN. This covers LAN-to-WAN traffic routed through that interface without also attaching the same monitors to the WAN. Use your actual routed LAN interface or VLAN interface; Vlan1 is an example, not a requirement. Traffic routed between VLANs can also cross this interface, so the capture is not limited to Internet traffic. Locally switched traffic that does not cross the router is outside this capture. Avoid combining LAN and WAN copies of the same traffic. For combined routers, select up to three captures, normally one LAN capture per router." +msgstr "" + +#: nbar_setup_guide.php +msgid "with" +msgstr "" + +#: nbar_setup_guide.php +msgid "Average bandwidth covers the selected timespan, not an instantaneous or peak rate. Conversations combines both directions for each Internal Host, External Host and application. Classification comes from the router; encrypted traffic can remain SSL or QUIC. DNS names identify hosts, not the exact HTTPS page." +msgstr "" + +#: nbar_setup_guide.php +msgid "Reading reports" +msgstr "" + #: setup.php msgid "NetFlow User" msgstr "" @@ -2647,6 +3129,34 @@ msgstr "" msgid "%d Year" msgstr "" +#: setup.php +msgid "Cisco Apps (NBAR) History Retention" +msgstr "" + +#: setup.php +msgid "Days of application history to retain. Reducing this removes older history during scheduled cleanup; increasing it cannot restore expired data." +msgstr "" + +#: setup.php +msgid "1 Day" +msgstr "" + +#: setup.php +msgid "14 Days" +msgstr "" + +#: setup.php +msgid "3 Days" +msgstr "" + +#: setup.php +msgid "30 Days" +msgstr "" + +#: setup.php +msgid "7 Days" +msgstr "" + #: setup.php msgid "Database Partitioning Scheme" msgstr "" diff --git a/manifest.json b/manifest.json index 00e7668..df291f7 100644 --- a/manifest.json +++ b/manifest.json @@ -3,6 +3,7 @@ "arrays.php", "database.php", "flowview_security.php", + "flowview_upgrade.php", "functions-pre13.php", "functions.php", "themes/" @@ -13,6 +14,7 @@ "LICENSE", "Net/", "README.md", + "cli/", "config.php.dist", "css/", "fixprivate.php", @@ -27,16 +29,20 @@ "flowview_process.php", "flowview_runner.php", "flowview_schedules.php", - "flowview_upgrade.php", + "flowview_sections.php", "flowview_upgrade_nat_columns.php", "images/", "import_flows.php", "includes/", "index.php", "irr_tables.php", + "js/", "locales/", "manifest.json", "maxscale/", + "nbar_conversations.php", + "nbar_report.php", + "nbar_setup_guide.php", "plugin_flowview_ports.sql", "repair_tables.php", "run_schedule.php", diff --git a/nbar_conversations.php b/nbar_conversations.php new file mode 100644 index 0000000..6f52a5c --- /dev/null +++ b/nbar_conversations.php @@ -0,0 +1,129 @@ + $nbar_preferences + * @var string $listener_scope + * @var int $lid + * @var int $start + * @var int $end + * @var int $seconds + * @var string $family + * @var string $aid + * @var string $host + * @var string $base + * @var array $combined_lids + */ +if (empty($nbar_preferences['prefixes'])) { + print '

    ' . html_escape(__('Configure your site networks in Application settings to label internal and external devices. Top applications and Connections work directly with exported IPv4 and IPv6 addresses.', 'flowview')) . '

    '; + + return; +} +$where = $listener_scope . ' AND d.minute>=? AND d.minutegetMessage(), false, 'FLOWVIEW'); + print '

    ' . fv_nbar_h(__('Unable to load conversations. Please try again.', 'flowview')) . '

    '; + + return; +} +$sql = "SELECT $fields FROM $nbar_source d JOIN plugin_flowview_devices l ON l.id=d.listener_id LEFT JOIN plugin_flowview_nbar_names n ON n.router=COALESCE(NULLIF(d.router,SPACE(0)),l.allowfrom) AND n.app_id=d.app_id WHERE $where GROUP BY d.app_id,n.name,internal,external ORDER BY total DESC LIMIT 100"; +$rows = fv_nbar_query_cached($sql, $args); + +if (!is_array($rows)) { + print '

    ' . fv_nbar_h(__('Unable to load conversations. Please try again.', 'flowview')) . '

    '; + + return; +} +$convbase = $base . '&mode=conversations' . ($aid !== '' ? '&app=' . $aid : ''); + +if ($aid !== '') { + print '

    ' . html_escape(__('← Back to top applications', 'flowview')) . '

    '; +} +print '

    ' . html_escape(__('Internal Host ↔ External Host. Click a device to see its individual connections. Bandwidth is averaged over the selected timespan.', 'flowview')) . '

    '; +print '
    '; + +foreach ([__('Internal Host', 'flowview'), __('External Host', 'flowview'), __('Application', 'flowview'), __('Download (GB)', 'flowview'), __('Upload (GB)', 'flowview'), __('Download (Avg Mbps)', 'flowview'), __('Upload (Avg Mbps)', 'flowview'), __('First seen', 'flowview'), __('Last seen', 'flowview')] as $index => $h) { + $unit = [3 => 'data', 4 => 'data', 5 => 'rate', 6 => 'rate'][$index] ?? ''; + print ''; +} +print ''; +$i = 0; + +foreach ($rows as $r) { + $link = $base . '&parent_app=' . rawurlencode($aid) . '&mode=connections&app=' . $r['app_id'] . '&internal=' . rawurlencode($r['internal']) . '&external=' . rawurlencode($r['external']); + print ''; + + foreach (['internal', 'external'] as $f) { + $dns = fv_nbar_dns($r[$f]); + print ''; + } + print ''; + + foreach ([$r['download'] / 1e9, $r['upload'] / 1e9, $r['download'] * 8 / $seconds / 1e6, $r['upload'] * 8 / $seconds / 1e6] as $j => $v) { + $precision = $j < 2 ? 4 : 3; + $threshold = pow(10, -$precision); + print ''; + } + print fv_nbar_seen_cells($r) . ''; +} + +if (!$rows) { + print ''; +} +print '
    ' . fv_nbar_h($h) . '
    ' . fv_nbar_h($r[$f]) . '
    ' . fv_nbar_h($dns) . '
    ' . fv_nbar_h($r['name'] ?? __('Awaiting name (%s)', $r['app_id'], 'flowview')) . '' . ($v > 0 && $v < $threshold ? '<' . number_format($threshold, $precision) : number_format($v, $precision)) . '
    ' . html_escape(__('No conversations for this period. Allow 1–2 minutes for exports, or choose a longer timespan.', 'flowview')) . '
    ' . html_escape(__('Report information', 'flowview')) . '

    ' . html_escape(__('First seen and Last seen show the earliest and latest FlowView observations within the selected period, at minute precision. They do not identify continuous activity or exact flow start/end times. Unavailable means the retained summary has no complete observation dates.', 'flowview')) . '

    ' . html_escape(__('Top 100 conversations by combined data volume. Each row groups an Internal Host, External Host and application across both directions. Uses only the selected LAN captures to avoid counting WAN copies. Local and multicast traffic is excluded. DNS names identify servers, not the exact website requested. History retention: %d days.', fv_nbar_retention_days(), 'flowview')) . '

    '; + +print ''; diff --git a/nbar_report.php b/nbar_report.php new file mode 100644 index 0000000..824f007 --- /dev/null +++ b/nbar_report.php @@ -0,0 +1,469 @@ + $graph_timespans + * @var array> $stat_report_array + * @var array> $print_report_array + */ +if ((int) read_config_option('flowview_nbar_schema_version') < 2) { + plugin_flowview_check_upgrade(); + print '

    ' . html_escape(__('NBAR history is being upgraded. Refresh this report shortly.', 'flowview')) . '

    '; + bottom_footer(); + exit; +} + +// Report controls use GET links or a protected POST form. +$nbar_request = []; + +foreach (['listener', 'period', 'family', 'mode', 'app', 'host', 'page', 'parent_app', 'context', 'predefined_timespan', 'date1', 'date2', 'lock_range', 'internal', 'external'] as $key) { + $value = get_nfilter_request_var($key, null); + + if (is_scalar($value)) { + $nbar_request[$key] = (string) $value; + } +} + +$devices = flowview_db_fetch_assoc("SELECT id,name FROM plugin_flowview_devices WHERE enabled='on' ORDER BY id"); +$nbar_preferences = fv_nbar_preferences_init($devices); +$lid = (int) ($nbar_request['listener'] ?? ($devices[0]['id'] ?? 0)); +$valid = array_column($devices, 'id'); + +if ($lid !== 0 && !in_array($lid, $valid, true)) { + $lid = (int) ($valid[0] ?? 0); +} +$seconds = (int) ($nbar_request['period'] ?? 300); + +if (!in_array($seconds, [300, 900, 3600, 86400, 604800], true)) { + $seconds = 300; +} +$family = (string) ($nbar_request['family'] ?? 'all'); + +if (!in_array($family, ['all', '4', '6'], true)) { + $family = 'all'; +} +$mode = (string) ($nbar_request['mode'] ?? 'conversations'); + +if (!in_array($mode, ['apps', 'conversations', 'connections'], true)) { + $mode = 'conversations'; +} +$combined_lids = array_values(array_intersect($nbar_preferences['listeners'], array_map('intval', $valid))); + +if (!empty($GLOBALS['fv_nbar_preferences_saved']) && isset_request_var('combine_apply') && cacti_sizeof($combined_lids) >= 2) { + $lid = 0; +} + +if ($lid === 0 && !$combined_lids) { + $lid = (int) ($valid[0] ?? 0); +} +// Capture ids start at 1, so 0 matches nothing when no capture device exists and IN () is never built. +$listener_scope = $lid === 0 ? 'd.listener_id IN (' . implode(',', $combined_lids ?: [0]) . ')' : 'd.listener_id=?'; +$aid = (string) ($nbar_request['app'] ?? ''); + +if (!preg_match('/^[a-f0-9]{2,32}$/D', $aid)) { + $aid = ''; +} +$host = (string) ($nbar_request['host'] ?? ''); + +if (!filter_var($host, FILTER_VALIDATE_IP)) { + $host = ''; +} +$page = max(1, min(1000000, (int) ($nbar_request['page'] ?? 1))); +$parent_app = (string) ($nbar_request['parent_app'] ?? ''); + +if (!preg_match('/^[a-f0-9]{2,32}$/D', $parent_app)) { + $parent_app = ''; +} +$end = (int) (floor(time() / 60) * 60) - 60; +$start = $end - $seconds; +$timespan_context = json_decode((string) ($nbar_request['context'] ?? ''), true); +$timespan = (int) ($nbar_request['predefined_timespan'] ?? ($timespan_context['predefined_timespan'] ?? 0)); +$custom_start = strtotime((string) ($nbar_request['date1'] ?? '')); +$custom_end = strtotime((string) ($nbar_request['date2'] ?? '')); +$has_dates = $custom_start && $custom_end && $custom_end > $custom_start; +$custom = $has_dates; + +if (!isset($nbar_request['predefined_timespan']) && !isset($timespan_context['predefined_timespan']) && !$custom) { + $legacy = [3600 => GT_LAST_HOUR, 86400 => GT_LAST_DAY, 604800 => GT_LAST_WEEK]; + + if (isset($nbar_request['period'])) { + $timespan = $legacy[$seconds] ?? 0; + } else { + $timespan = GT_LAST_HALF_HOUR; + } +} + +if ($timespan > 0 && $timespan !== 1000 && !isset($graph_timespans[$timespan])) { + $timespan = GT_LAST_HOUR; +} +$nbar_now = (int) (floor(time() / 60) * 60); +$nbar_retention_days = fv_nbar_retention_days(); +$nbar_timespans = fv_nbar_timespan_options($graph_timespans, $nbar_now, $nbar_retention_days, read_user_setting('first_weekdayid')); +$nbar_range_notice = ''; + +if ($timespan > 0 && !isset($nbar_timespans[$timespan])) { + $timespan = GT_LAST_HALF_HOUR; + $nbar_range_notice = __('That period is outside the %s-day NBAR retention setting. Showing the last half hour.', $nbar_retention_days, 'flowview'); + unset($nbar_request['lock_range']); +} + +if ($timespan === 1000) { + $end = $nbar_now; + $start = $end - $nbar_retention_days * 86400; + $custom = false; +} elseif ($timespan > 0) { + $nbar_span = []; + get_timespan($nbar_span, $nbar_now, $timespan, read_user_setting('first_weekdayid')); + $start = strtotime($nbar_span['current_value_date1']); + $end = strtotime($nbar_span['current_value_date2']); + $custom = false; +} else { + if ($custom) { + $start = $custom_start; + $end = $custom_end; + } else { + $end = (int) (floor(time() / 60) * 60); + $start = $end - $seconds; + } + $timespan = 0; + $custom = true; +} + +if ($timespan === 0 && $has_dates) { + $start = $custom_start; + $end = $custom_end; +} + +if (!empty($nbar_request['lock_range']) && $has_dates) { + $start = $custom_start; + $end = $custom_end; +} +[$start, $end, $nbar_dates_adjusted] = fv_nbar_bound_dates($start, $end, $nbar_now, $nbar_retention_days); + +if ($nbar_dates_adjusted) { + $timespan = 0; + $custom = true; + $nbar_range_notice = __('Dates adjusted to the %s-day NBAR retention window.', $nbar_retention_days, 'flowview'); +} +// The primary key starts with capture/minute, so each lookup reads its first record. +$nbar_earliest = null; +$nbar_data_error = false; + +foreach ($lid === 0 ? $combined_lids : [$lid] as $nbar_capture_id) { + $nbar_first = fv_nbar_data_edge($nbar_capture_id); + + if ($nbar_first === false) { + $nbar_data_error = true; + } elseif (isset($nbar_first['minute'])) { + $nbar_first_minute = max($nbar_now - $nbar_retention_days * 86400, (int) $nbar_first['minute']); + $nbar_earliest = $nbar_earliest === null ? $nbar_first_minute : min($nbar_earliest, $nbar_first_minute); + } +} +$seconds = max(1, $end - $start); +$context = json_decode((string) ($nbar_request['context'] ?? ''), true); + +if (!is_array($context)) { + $context = []; +} +$allowed = ['query', 'report', 'predefined_timespan', 'ex_addr', 'sortfield', 'cutofflines', 'cutoffoctets', 'exclude', 'graph_type', 'graph_height', 'date1', 'date2', 'domains']; +$context = array_intersect_key($context, array_flip($allowed)); + +foreach ($context as $key => $value) { + if (!is_scalar($value)) { + unset($context[$key]); + } +} + +if (!preg_match('/^[sp][0-9]+$/', (string) ($context['report'] ?? ''))) { + $context['report'] = 's10'; +} +$report_key = (int) substr($context['report'], 1); + +if (!isset(($context['report'][0] === 's' ? $stat_report_array : $print_report_array)[$report_key])) { + $context['report'] = 's10'; +} +$context_json = json_encode($context, JSON_THROW_ON_ERROR); +$base = 'flowview.php?' . http_build_query(['action' => 'view', 'report' => 'nbar', 'listener' => $lid, 'predefined_timespan' => $timespan, 'lock_range' => 1, 'family' => $family, 'date1' => date('Y-m-d H:i', $start), 'date2' => date('Y-m-d H:i', $end), 'context' => $context_json, 'host' => $host]); +$return = $context; +$return['action'] = 'view'; +$return['device_id'] = $lid; +$return['date1'] = date('Y-m-d H:i', $start); +$return['date2'] = date('Y-m-d H:i', $end); +$return['predefined_timespan'] = $timespan; +$return['section'] = 'flows'; +$return_url = 'flowview.php?' . http_build_query($return); +parse_str((string) parse_url($base, PHP_URL_QUERY), $section_params); +$section_params['mode'] = $mode; +fv_section_save('nbar', $section_params); +fv_section_navigation('nbar'); +html_start_box(__('Application Report: Cisco Apps Recognition (NBAR)', 'flowview'), '100%', false, 3, 'center', ''); +print ''; +fv_nbar_feed_status($devices, array_map('intval', $lid === 0 ? $combined_lids : [$lid])); +print ''; +print '
    '; +print ''; + +foreach (['action' => 'view', 'report' => 'nbar', 'context' => $context_json, 'app' => $aid, 'parent_app' => $parent_app] as $key => $value) { + print ''; +} + +foreach (['internal', 'external'] as $f) { + if ($mode === 'connections' && filter_var($nbar_request[$f] ?? '', FILTER_VALIDATE_IP)) { + print ''; + } +} +print '
    ' . html_escape(__('Router / capture', 'flowview')) . '' . html_escape(__('Clear', 'flowview')) . '
    '; +print '
    ' . html_escape(__('IP version', 'flowview')) . '' . html_escape(__('Show', 'flowview')) . '
    '; +print '
    ' . html_escape(__('Filter by IP', 'flowview')) . '
    '; +print '
    ' . html_escape(__('Timespan', 'flowview')) . '' . html_escape(__('From', 'flowview')) . '' . html_escape(__('To', 'flowview')) . '
    '; + +if ($nbar_range_notice !== '') { + print '

    ' . fv_nbar_h($nbar_range_notice) . '

    '; +} +fv_nbar_preferences_render($devices, $base, $mode, $aid, $parent_app); +print '
    '; + +if ($nbar_data_error) { + print '

    ' . fv_nbar_h(__('Unable to read the NBAR records. Please try again.', 'flowview')) . '

    '; +} else { + fv_nbar_chart($listener_scope, $lid === 0 ? [$start, $end] : [$lid, $start, $end], $family, $host); +} +print '
    '; +html_end_box(); +include(__DIR__ . '/nbar_setup_guide.php'); + +print ''; +print ''; +print ''; + +if ($nbar_data_error) { + bottom_footer(); + + return; +} + +if ($mode === 'conversations') { + include(__DIR__ . '/nbar_conversations.php'); + bottom_footer(); + exit; +} + +if ($mode === 'connections' && $aid !== '' && !filter_var($nbar_request['internal'] ?? '', FILTER_VALIDATE_IP)) { + print '

    ' . html_escape(__('← Back to top applications', 'flowview')) . '

    '; +} +$where = $listener_scope . ' AND d.minute>=? AND d.minute' . html_escape(__('← Back to conversations', 'flowview')) . '   ' . fv_nbar_h($internal) . ' ↔ ' . fv_nbar_h($external) . '

    '; +} +$fields = 'd.app_id,n.name,SUM(d.bytes) bytes,SUM(d.packets) packets,SUM(d.flows) records,IF(COUNT(d.first_seen)=COUNT(*),MIN(d.first_seen),NULL) first_seen,IF(COUNT(d.last_seen)=COUNT(*),MAX(d.last_seen),NULL) last_seen'; +$group = 'd.app_id,n.name'; + +if ($mode === 'connections') { + $fields .= ',INET6_NTOA(d.src) src,INET6_NTOA(d.dst) dst,d.sport,d.dport,d.protocol'; + $group .= ',d.src,d.dst,d.sport,d.dport,d.protocol'; +} +$nbar_source = null; + +if ($mode === 'apps') { + try { + $nbar_source = fv_nbar_summary_source($host === '' ? 'apps' : 'pairs', $lid === 0 ? $combined_lids : [$lid], $start, $end); + } catch (RuntimeException $e) { + cacti_log('NBAR applications: ' . $e->getMessage(), false, 'FLOWVIEW'); + print '

    ' . fv_nbar_h(__('Unable to load applications. Please try again.', 'flowview')) . '

    '; + bottom_footer(); + + return; + } +} + +if ($mode === 'apps') { + // Resolve names after reducing raw records to one row per capture/application. + $sql = 'SELECT d.app_id,n.name,SUM(d.bytes) bytes,SUM(d.packets) packets,SUM(d.flows) records,IF(COUNT(d.first_seen)=COUNT(*),MIN(d.first_seen),NULL) first_seen,IF(COUNT(d.last_seen)=COUNT(*),MAX(d.last_seen),NULL) last_seen FROM (SELECT d.listener_id,d.app_id,d.router,SUM(d.bytes) bytes,SUM(d.packets) packets,SUM(d.flows) flows,IF(COUNT(d.first_seen)=COUNT(*),MIN(d.first_seen),NULL) first_seen,IF(COUNT(d.last_seen)=COUNT(*),MAX(d.last_seen),NULL) last_seen FROM ' . $nbar_source . ' d WHERE ' . $where . ' GROUP BY d.listener_id,d.app_id,d.router) d JOIN plugin_flowview_devices l ON l.id=d.listener_id LEFT JOIN plugin_flowview_nbar_names n ON n.router=COALESCE(NULLIF(d.router,SPACE(0)),l.allowfrom) AND n.app_id=d.app_id GROUP BY d.app_id,n.name ORDER BY bytes DESC LIMIT 100'; +} + +if ($mode === 'connections') { + try { + $nbar_count_key = fv_nbar_count_context('', $where, $args, $end, $lid === 0 ? $combined_lids : [$lid], $start); + $rows = fv_nbar_connections_cached($lid === 0 ? $combined_lids : [$lid], $start, $end, $where, $args, $page); + } catch (Throwable $e) { + cacti_log('NBAR connection page: ' . $e->getMessage(), false, 'FLOWVIEW'); + print '

    ' . fv_nbar_h(__('Unable to load connections. Try a shorter timespan or filter by IP.', 'flowview')) . '

    '; + bottom_footer(); + + return; + } +} else { + $rows = fv_nbar_query_cached($sql, $args); + + if (!is_array($rows)) { + print '

    ' . fv_nbar_h(__('Unable to load applications. Please try again.', 'flowview')) . '

    '; + bottom_footer(); + + return; + } +} +$has_next = $mode === 'connections' && cacti_sizeof($rows) > 100; + +if ($has_next) { + $rows = array_slice($rows, 0, 100); +} +$nbar_pagination = ''; + +if ($mode === 'connections') { + $page_params = ['mode' => 'connections', 'app' => $aid, 'parent_app' => $parent_app, 'host' => $host, 'internal' => $internal, 'external' => $external]; + $nbar_page_key = hash('sha256', $base . '&' . http_build_query($page_params)); + $nbar_pagination = '
    '; + $nbar_page_total = '' . fv_nbar_h(__('Counting total pages…', 'flowview')) . ''; + $nbar_pagination .= '' . $nbar_page_total . '' . fv_nbar_h(__('Up to 100 connections per page, ordered by data volume.', 'flowview')) . ''; + + if ($page > 1) { + $nbar_pagination .= '' . __('Previous', 'flowview') . ''; + } + + if ($has_next && $page < 1000000) { + $nbar_pagination .= '' . __('Next', 'flowview') . ''; + } + $nbar_pagination .= '
    '; + print $nbar_pagination; +} + +if ($mode === 'connections' && $aid !== '') { + $selected_name = $rows[0]['name'] ?? __('Application %s', $aid, 'flowview'); + print '

    ' . fv_nbar_h(strtoupper($selected_name)) . ' — Connections

    '; +} +print '
    '; + +if ($mode === 'connections') { + print ''; +} +print ''; +$nbar_row = 0; + +foreach ($rows as $r) { + $name = $r['name'] ?? __('Awaiting name (%s)', $r['app_id'], 'flowview'); + print ''; + + if ($mode === 'connections') { + foreach (['src', 'sport', 'dst', 'dport'] as $f) { + print ''; + + if ($f === 'src' || $f === 'dst') { + print ''; + } + } + print ''; + } + + print '' . fv_nbar_seen_cells($r) . ''; +} + +if (!$rows) { + print ''; +} +print '
    ' . html_escape(__('Application', 'flowview')) . '' . html_escape(__('Source IP', 'flowview')) . '' . html_escape(__('Source DNS', 'flowview')) . '' . html_escape(__('Source port', 'flowview')) . '' . html_escape(__('Destination IP', 'flowview')) . '' . html_escape(__('Destination DNS', 'flowview')) . '' . html_escape(__('Destination port', 'flowview')) . '' . html_escape(__('Protocol', 'flowview')) . '' . html_escape(__('Data (GB)', 'flowview')) . '' . html_escape(__('Bandwidth (Avg Mbps)', 'flowview')) . '' . html_escape(__('Packets', 'flowview')) . '' . html_escape(__('Records', 'flowview')) . '' . html_escape(__('First seen', 'flowview')) . '' . html_escape(__('Last seen', 'flowview')) . '
    '; + + if ($mode === 'connections') { + print fv_nbar_h($name); + } else { + print '' . fv_nbar_h($name) . ''; + } + print '' . fv_nbar_h($r[$f]) . '' . fv_nbar_h(fv_nbar_dns($r[$f])) . '' . fv_nbar_h([6 => 'TCP', 17 => 'UDP', 1 => 'ICMP', 58 => 'ICMPv6'][$r['protocol']] ?? $r['protocol']) . '' . ($r['bytes'] > 0 && $r['bytes'] < 100000 ? '<0.0001' : number_format($r['bytes'] / 1e9, 4)) . '' . ($r['bytes'] > 0 && $r['bytes'] * 8 / $seconds / 1e6 < 0.001 ? '<0.001' : number_format($r['bytes'] * 8 / $seconds / 1e6, 3)) . '' . number_format($r['packets']) . '' . number_format($r['records']) . '
    ' . html_escape(__('No application records in this view yet. Try a longer period or another traffic view.', 'flowview')) . '
    '; +print $nbar_pagination; + +if ($mode === 'connections') { + print ''; +} + +print '
    ' . html_escape(__('Report information', 'flowview')) . '

    ' . html_escape(__('First seen and Last seen show the earliest and latest FlowView observations within the selected period, at minute precision. They do not identify continuous activity or exact flow start/end times. Unavailable means the retained summary has no complete observation dates.', 'flowview')) . '

    ' . html_escape(__('Results are ordered by data volume; Connections shows up to 100 per page. Column sorting applies to the displayed page. NBAR history uses the configured retention setting. Daily summaries are reused across report selections; partial days use hourly summaries, and partial or changed hours use raw records. Connections pages are read from shared daily history, with nearby pages briefly cached for navigation. Average Mbps covers the selected period; allow 1–2 minutes for exports. Capture visibility depends on the device exporter and observation point. NAT can hide internal IPv4 addresses on a WAN capture. Select one observation point per traffic path to avoid duplicate totals. Cisco NBAR may classify encrypted traffic as SSL, QUIC or Unknown. History starts when export is enabled.', 'flowview')) . '

    '; +print ''; +bottom_footer(); diff --git a/nbar_setup_guide.php b/nbar_setup_guide.php new file mode 100644 index 0000000..8ab6bf3 --- /dev/null +++ b/nbar_setup_guide.php @@ -0,0 +1,99 @@ + $config Cacti's global configuration, available in nbar_report.php's scope. */ +if (!function_exists('fv_nbar_h')) { + http_response_code(403); + exit; +} +?> +

    COLLECTOR_IP LISTENER_PORT EXPORT_SOURCE VlanXX

    flow record CACTI_NBAR4
    + match ipv4 tos
    + match ipv4 protocol
    + match ipv4 source address
    + match ipv4 destination address
    + match transport source-port
    + match transport destination-port
    + match interface input
    + match application name
    + collect transport tcp flags
    + collect interface output
    + collect counter bytes long
    + collect counter packets long
    + collect timestamp sys-uptime first
    + collect timestamp sys-uptime last
    +!
    +flow record CACTI_NBAR6
    + match ipv6 traffic-class
    + match ipv6 flow-label
    + match ipv6 protocol
    + match ipv6 extension map
    + match ipv6 source address
    + match ipv6 destination address
    + match transport source-port
    + match transport destination-port
    + match interface input
    + match flow direction
    + match application name
    + collect transport tcp flags
    + collect interface output
    + collect counter bytes long
    + collect counter packets long
    + collect timestamp sys-uptime first
    + collect timestamp sys-uptime last
    +!
    +flow exporter CACTI_LAN
    + destination COLLECTOR_IP
    + source EXPORT_SOURCE
    + transport udp LISTENER_PORT
    + template data timeout 60
    + option application-table timeout 60
    +!
    +flow monitor CACTI_APP4_LAN
    + exporter CACTI_LAN
    + cache timeout active 60
    + record CACTI_NBAR4
    +!
    +flow monitor CACTI_APP6_LAN
    + exporter CACTI_LAN
    + cache timeout active 60
    + record CACTI_NBAR6
    +!
    +interface VlanXX
    + ip flow monitor CACTI_APP4_LAN input
    + ip flow monitor CACTI_APP4_LAN output
    + ipv6 flow monitor CACTI_APP6_LAN input
    + ipv6 flow monitor CACTI_APP6_LAN output

    VlanXX Vlan1

    show ip nbar version
    +show ip nbar protocol-pack active
    +show flow interface
    +show flow exporter CACTI_LAN statistics
    +show flow monitor CACTI_APP4_LAN statistics
    +show flow monitor CACTI_APP6_LAN statistics
    +show flow monitor CACTI_APP6_LAN cache format table
    +show platform hardware qfp active datapath utilization
    +


    Copyright (c) 2004-2026 The Cacti Group.
    +'; diff --git a/phpunit.xml b/phpunit.xml index 2ec8407..ea1e6e4 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -39,6 +39,24 @@ Net/DNS2/Updater.php includes/functions.php includes/database.php + includes/nbar_chart.php + includes/nbar_collector.php + includes/nbar_common.php + includes/nbar_connections.php + nbar_conversations.php + includes/nbar_count.php + includes/nbar_daily.php + includes/nbar_dns.php + includes/nbar_feed_status.php + includes/nbar_partition.php + includes/nbar_preferences.php + includes/nbar_query.php + nbar_report.php + includes/nbar_seen.php + nbar_setup_guide.php + includes/nbar_summary.php + includes/nbar_timespan.php + flowview_sections.php diff --git a/run_schedule.php b/run_schedule.php index 17c052c..2799c6b 100644 --- a/run_schedule.php +++ b/run_schedule.php @@ -124,7 +124,7 @@ LIMIT 1', ['flowview', 'pending']); - if (cacti_sizeof($report)) { + if (is_array($report) && cacti_sizeof($report)) { reports_run($report['id']); $run++; @@ -155,7 +155,7 @@ AND rq.id = ?', ['flowview', $report_id]); - if (cacti_sizeof($report)) { + if (is_array($report) && cacti_sizeof($report)) { $id = $report['source_id']; /* silently end if the registered process is still running, or process table missing */ diff --git a/setup.php b/setup.php index 8726c0f..0548c8a 100644 --- a/setup.php +++ b/setup.php @@ -23,6 +23,8 @@ +-------------------------------------------------------------------------+ */ +require_once(__DIR__ . '/includes/nbar_common.php'); + /** * Return the CSP nonce attribute for inline ', 'bytes' => 2000000000], + ['name' => 'Two', 'bytes' => 4000000], ['name' => 'Three', 'bytes' => 3000000], + ['name' => 'Four', 'bytes' => 2000000], ['name' => 'Five', 'bytes' => 1000000], + ['name' => 'Six', 'bytes' => 500000], ['name' => 'Zero', 'bytes' => 0], + ]); + ob_start(); + fv_nbar_chart('d.listener_id=?', [1, 1800000000, 1800001800], 'all'); + $html = ob_get_clean(); + expect($html)->toContain('<script>', '2.00 GB', 'Other', '0.50 MB', 'conic-gradient'); + expect($html)->not->toContain('