From 426b69aeaaf65e2faa147b8b5305122c91e066ae Mon Sep 17 00:00:00 2001 From: Lakshmi Pradeep Date: Fri, 16 Oct 2026 10:21:07 -0700 Subject: [PATCH 1/2] sysfs: parse NVMe controller and namespace diag counters Linux 7.2 added a diag/ attribute group to NVMe controllers and namespace block devices: /sys/class/nvme//diag/command_error_count /sys/class/nvme//diag/reset_count /sys/class/nvme//diag/reconnect_count (fabrics only) /sys/class/nvme///diag/command_retries_count /sys/class/nvme///diag/command_error_count /sys/class/nvme///diag/multipath_failover_count Add NVMeDevice.Diag and NVMeDevice.NamespaceDiag to read them on demand, so NVMeClass does not read the extra files for callers that do not need them. NVMeDevice gains a Path field with its sysfs directory for the methods to read from. Every counter is a pointer and stays nil when the running kernel does not provide the attribute, so older kernels and PCIe controllers (which have no reconnect_count) keep working. Signed-off-by: Lakshmi Pradeep --- sysfs/class_nvme.go | 88 +++++++++++++++++++++++++++++++++- sysfs/class_nvme_test.go | 100 +++++++++++++++++++++++++++++++++++++++ testdata/fixtures.ttar | 62 ++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 1 deletion(-) diff --git a/sysfs/class_nvme.go b/sysfs/class_nvme.go index aa91d7b2..e528d46d 100644 --- a/sysfs/class_nvme.go +++ b/sysfs/class_nvme.go @@ -16,6 +16,7 @@ package sysfs import ( + "errors" "fmt" "os" "path/filepath" @@ -51,9 +52,30 @@ type NVMeNamespace struct { CapacityBytes uint64 // calculated: SizeBlocks * 512 } +// NVMeNamespaceDiag holds the diagnostic counters of a namespace block device +// from /sys/class/nvme///diag, added in Linux 7.2. A nil +// field means the kernel does not expose that counter. Root can reset the +// counters by writing to them. Use NVMeDevice.NamespaceDiag to read them. +type NVMeNamespaceDiag struct { + CommandRetries *uint64 // diag/command_retries_count: I/O commands retried + CommandErrors *uint64 // diag/command_error_count: I/O commands completed with an error + MultipathFailovers *uint64 // diag/multipath_failover_count: I/O failed over to another path +} + +// NVMeControllerDiag holds the diagnostic counters of a controller from +// /sys/class/nvme//diag, added in Linux 7.2. A nil field means the +// kernel does not expose that counter. Root can reset the counters by writing +// to them. Use NVMeDevice.Diag to read them. +type NVMeControllerDiag struct { + CommandErrors *uint64 // diag/command_error_count: admin commands completed with an error + Resets *uint64 // diag/reset_count: controller resets + Reconnects *uint64 // diag/reconnect_count: reconnect attempts, fabrics transports only +} + // NVMeDevice contains info from files in /sys/class/nvme for a single NVMe device. type NVMeDevice struct { Name string + Path string // sysfs directory of the device, /sys/class/nvme/ Serial string // /sys/class/nvme//serial Model string // /sys/class/nvme//model State string // /sys/class/nvme//state @@ -92,7 +114,7 @@ func (fs FS) NVMeClass() (NVMeClass, error) { // Parse one NVMe device. func (fs FS) parseNVMeDevice(name string) (*NVMeDevice, error) { path := fs.sys.Path(nvmeClassPath, name) - device := NVMeDevice{Name: name} + device := NVMeDevice{Name: name, Path: path} // Parse device-level attributes for _, f := range [...]string{"firmware_rev", "model", "serial", "state", "cntlid"} { @@ -191,3 +213,67 @@ func (fs FS) parseNVMeDevice(name string) (*NVMeDevice, error) { return &device, nil } + +// Diag reads the controller's diagnostic counters from +// /sys/class/nvme//diag. They are read on demand rather than by +// NVMeClass, so callers that do not need them avoid the extra file reads. +func (d NVMeDevice) Diag() (NVMeControllerDiag, error) { + var ( + diag NVMeControllerDiag + err error + ) + dir := filepath.Join(d.Path, "diag") + for _, c := range [...]struct { + file string + dest **uint64 + }{ + {"command_error_count", &diag.CommandErrors}, + {"reset_count", &diag.Resets}, + {"reconnect_count", &diag.Reconnects}, + } { + if *c.dest, err = readOptionalUint(filepath.Join(dir, c.file)); err != nil { + return diag, err + } + } + return diag, nil +} + +// NamespaceDiag reads the diagnostic counters of one of the device's namespace +// block devices, given by name as in NVMeNamespace.Name (e.g. nvme0n1 or +// nvme0c1n1), from /sys/class/nvme///diag. +func (d NVMeDevice) NamespaceDiag(namespace string) (NVMeNamespaceDiag, error) { + var ( + diag NVMeNamespaceDiag + err error + ) + if !nvmeNamespacePattern.MatchString(namespace) { + return diag, fmt.Errorf("invalid NVMe namespace name %q", namespace) + } + dir := filepath.Join(d.Path, namespace, "diag") + for _, c := range [...]struct { + file string + dest **uint64 + }{ + {"command_retries_count", &diag.CommandRetries}, + {"command_error_count", &diag.CommandErrors}, + {"multipath_failover_count", &diag.MultipathFailovers}, + } { + if *c.dest, err = readOptionalUint(filepath.Join(dir, c.file)); err != nil { + return diag, err + } + } + return diag, nil +} + +// readOptionalUint reads an unsigned integer sysfs attribute, returning nil +// when the attribute does not exist on the running kernel. +func readOptionalUint(path string) (*uint64, error) { + v, err := parsers.SysReadUintFromFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to read file %q: %w", path, err) + } + return &v, nil +} diff --git a/sysfs/class_nvme_test.go b/sysfs/class_nvme_test.go index 06994ecf..14d97484 100644 --- a/sysfs/class_nvme_test.go +++ b/sysfs/class_nvme_test.go @@ -43,6 +43,7 @@ func TestNVMeClass(t *testing.T) { want := NVMeClass{ "nvme0": NVMeDevice{ Name: "nvme0", + Path: filepath.Join(sysTestFixtures, "class/nvme/nvme0"), FirmwareRevision: "1B2QEXP7", Model: "Samsung SSD 970 PRO 512GB", Serial: "S680HF8N190894I", @@ -66,6 +67,7 @@ func TestNVMeClass(t *testing.T) { // namespace is the hidden per-path device nvmecn. "nvme1": NVMeDevice{ Name: "nvme1", + Path: filepath.Join(sysTestFixtures, "class/nvme/nvme1"), FirmwareRevision: "1B2QEXP7", Model: "Samsung SSD 970 PRO 512GB", Serial: "S680HF8N190894I", @@ -92,6 +94,104 @@ func TestNVMeClass(t *testing.T) { } } +func TestNVMeDeviceDiag(t *testing.T) { + if _, err := os.Stat(sysTestFixtures); os.IsNotExist(err) { + t.Skip("Test fixtures not available, skipping NVMe diag tests") + return + } + + fs, err := NewFS(sysTestFixtures) + if err != nil { + t.Fatal(err) + } + + nc, err := fs.NVMeClass() + if err != nil { + t.Fatal(err) + } + + for _, tt := range []struct { + device string + namespace string + ctrl NVMeControllerDiag + ns NVMeNamespaceDiag + }{ + { + // PCIe controller: no reconnect_count, no multipath_failover_count. + device: "nvme0", + namespace: "nvme0n0", + ctrl: NVMeControllerDiag{ + CommandErrors: new(uint64(3)), + Resets: new(uint64(1)), + }, + ns: NVMeNamespaceDiag{ + CommandRetries: new(uint64(12)), + CommandErrors: new(uint64(2)), + }, + }, + { + // Fabrics controller with a multipath per-path namespace. + device: "nvme1", + namespace: "nvme0c1n1", + ctrl: NVMeControllerDiag{ + CommandErrors: new(uint64(0)), + Resets: new(uint64(2)), + Reconnects: new(uint64(5)), + }, + ns: NVMeNamespaceDiag{ + CommandRetries: new(uint64(4)), + CommandErrors: new(uint64(1)), + MultipathFailovers: new(uint64(7)), + }, + }, + } { + t.Run(tt.device, func(t *testing.T) { + d := nc[tt.device] + + ctrl, err := d.Diag() + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tt.ctrl, ctrl); diff != "" { + t.Errorf("unexpected controller diag (-want +got):\n%s", diff) + } + + ns, err := d.NamespaceDiag(tt.namespace) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tt.ns, ns); diff != "" { + t.Errorf("unexpected namespace diag (-want +got):\n%s", diff) + } + }) + } + + if _, err := nc["nvme0"].NamespaceDiag("../nvme1"); err == nil { + t.Error("expected an error for an invalid namespace name") + } +} + +func TestNVMeDeviceDiagMissing(t *testing.T) { + // A kernel older than 7.2 has no diag directory: every counter is nil. + d := NVMeDevice{Name: "nvme0", Path: t.TempDir()} + + ctrl, err := d.Diag() + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(NVMeControllerDiag{}, ctrl); diff != "" { + t.Errorf("unexpected controller diag (-want +got):\n%s", diff) + } + + ns, err := d.NamespaceDiag("nvme0n1") + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(NVMeNamespaceDiag{}, ns); diff != "" { + t.Errorf("unexpected namespace diag (-want +got):\n%s", diff) + } +} + func TestNVMeNamespaceParsingWithMockData(t *testing.T) { // Create a temporary directory structure for testing namespace parsing tempDir := t.TempDir() diff --git a/testdata/fixtures.ttar b/testdata/fixtures.ttar index 409658b8..802d1b13 100644 --- a/testdata/fixtures.ttar +++ b/testdata/fixtures.ttar @@ -7197,6 +7197,19 @@ Lines: 1 1997 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/nvme/nvme0/diag +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme0/diag/command_error_count +Lines: 1 +3 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme0/diag/reset_count +Lines: 1 +1 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/sys/class/nvme/nvme0/firmware_rev Lines: 1 1B2QEXP7 @@ -7215,6 +7228,19 @@ Lines: 1 optimized Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/nvme/nvme0/nvme0n0/diag +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme0/nvme0n0/diag/command_error_count +Lines: 1 +2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme0/nvme0n0/diag/command_retries_count +Lines: 1 +12 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/sys/class/nvme/nvme0/nvme0n0/nuse Lines: 1 488281250 @@ -7251,6 +7277,24 @@ Lines: 1 2 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/nvme/nvme1/diag +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme1/diag/command_error_count +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme1/diag/reconnect_count +Lines: 1 +5 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme1/diag/reset_count +Lines: 1 +2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/sys/class/nvme/nvme1/firmware_rev Lines: 1 1B2QEXP7 @@ -7269,6 +7313,24 @@ Lines: 1 non-optimized Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/nvme/nvme1/nvme0c1n1/diag +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme1/nvme0c1n1/diag/command_error_count +Lines: 1 +1 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme1/nvme0c1n1/diag/command_retries_count +Lines: 1 +4 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme/nvme1/nvme0c1n1/diag/multipath_failover_count +Lines: 1 +7 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/sys/class/nvme/nvme1/nvme0c1n1/nsid Lines: 1 1 From e3b2072a8d31972a629b5230cb39adea8fb17e54 Mon Sep 17 00:00:00 2001 From: Lakshmi Pradeep Date: Fri, 16 Oct 2026 10:21:07 -0700 Subject: [PATCH 2/2] sysfs: parse NVMe multipath namespace head diag counters Linux 7.2 exposes two counters on each native-multipath namespace head: /sys/class/nvme-subsystem///diag/io_requeue_no_usable_path_count /sys/class/nvme-subsystem///diag/io_fail_no_available_path_count They show I/O that was queued or failed because every path to the namespace was down, which per-path counters cannot show. Add NVMeSubsystem.NamespaceDiag to read them on demand for one of the subsystem's namespaces, so NVMeSubsystemClass does not read the extra files. NVMeSubsystem gains a Path field with its sysfs directory for the method to read from. Counters the kernel does not provide stay nil. Signed-off-by: Lakshmi Pradeep --- sysfs/class_nvme_subsystem.go | 42 +++++++++++++++++++++++++++++- sysfs/class_nvme_subsystem_test.go | 36 +++++++++++++++++++++++++ testdata/fixtures.ttar | 13 +++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/sysfs/class_nvme_subsystem.go b/sysfs/class_nvme_subsystem.go index 99e00f84..9f35aa6a 100644 --- a/sysfs/class_nvme_subsystem.go +++ b/sysfs/class_nvme_subsystem.go @@ -18,6 +18,7 @@ package sysfs import ( "fmt" "os" + "path/filepath" "regexp" "strings" @@ -35,6 +36,9 @@ var ( type NVMeSubsystem struct { // Name is the subsystem directory name, e.g. "nvme-subsys0". Name string + // Path is the subsystem's sysfs directory, + // /sys/class/nvme-subsystem/. + Path string // NQN is the NVMe Qualified Name from subsysnqn. NQN string // Model is the subsystem model string. @@ -50,6 +54,19 @@ type NVMeSubsystem struct { Namespaces []string } +// NVMeSubsystemNamespaceDiag holds the diagnostic counters of a multipath +// namespace head from /sys/class/nvme-subsystem///diag. +// A nil field means the kernel does not expose that counter. Use +// NVMeSubsystem.NamespaceDiag to read them. +type NVMeSubsystemNamespaceDiag struct { + // IORequeueNoUsablePath counts I/O requeued because no path was usable + // (diag/io_requeue_no_usable_path_count). + IORequeueNoUsablePath *uint64 + // IOFailNoAvailablePath counts I/O failed because no path was available + // (diag/io_fail_no_available_path_count). + IOFailNoAvailablePath *uint64 +} + // NVMeSubsystemController contains info about a single NVMe controller // within an NVMe subsystem. type NVMeSubsystemController struct { @@ -94,7 +111,7 @@ func (fs FS) NVMeSubsystemClass() (NVMeSubsystemClass, error) { func (fs FS) parseNVMeSubsystem(name string) (*NVMeSubsystem, error) { path := fs.sys.Path(nvmeSubsystemClassPath, name) - subsys := &NVMeSubsystem{Name: name} + subsys := &NVMeSubsystem{Name: name, Path: path} for _, attr := range [...]struct { file string @@ -156,3 +173,26 @@ func (fs FS) parseNVMeSubsystemController(subsysName, ctrlName string) (*NVMeSub return ctrl, nil } + +// NamespaceDiag reads the diagnostic counters of one of the subsystem's +// multipath namespace head devices, given by name as in Namespaces (e.g. +// nvme0n1), from /sys/class/nvme-subsystem///diag. They +// are read on demand rather than by NVMeSubsystemClass, so callers that do +// not need them avoid the extra file reads. +func (s NVMeSubsystem) NamespaceDiag(namespace string) (NVMeSubsystemNamespaceDiag, error) { + var ( + diag NVMeSubsystemNamespaceDiag + err error + ) + if !nvmeNamespaceRE.MatchString(namespace) { + return diag, fmt.Errorf("invalid NVMe namespace name %q", namespace) + } + dir := filepath.Join(s.Path, namespace, "diag") + if diag.IORequeueNoUsablePath, err = readOptionalUint(filepath.Join(dir, "io_requeue_no_usable_path_count")); err != nil { + return diag, err + } + if diag.IOFailNoAvailablePath, err = readOptionalUint(filepath.Join(dir, "io_fail_no_available_path_count")); err != nil { + return diag, err + } + return diag, nil +} diff --git a/sysfs/class_nvme_subsystem_test.go b/sysfs/class_nvme_subsystem_test.go index 79957121..637d6939 100644 --- a/sysfs/class_nvme_subsystem_test.go +++ b/sysfs/class_nvme_subsystem_test.go @@ -37,6 +37,7 @@ func TestNVMeSubsystemClass(t *testing.T) { want := NVMeSubsystemClass{ { Name: "nvme-subsys0", + Path: filepath.Join(sysTestFixtures, "class/nvme-subsystem/nvme-subsys0"), NQN: "nqn.2014-08.org.nvmexpress:uuid:a34c4f3a-0d6f-5cec-dead-beefcafebabe", Model: "Dell PowerStore", Serial: "SN12345678", @@ -50,6 +51,7 @@ func TestNVMeSubsystemClass(t *testing.T) { }, { Name: "nvme-subsys1", + Path: filepath.Join(sysTestFixtures, "class/nvme-subsystem/nvme-subsys1"), NQN: "nqn.2014-08.org.nvmexpress:uuid:b45d5e4b-1e7f-6ded-beef-deadcafe1234", Model: "NetApp ONTAP", Serial: "NTAP98765", @@ -65,6 +67,40 @@ func TestNVMeSubsystemClass(t *testing.T) { if diff := cmp.Diff(want, got); diff != "" { t.Fatalf("unexpected NVMeSubsystemClass (-want +got):\n%s", diff) } + + // Multipath head diag counters, read on demand. + for _, tt := range []struct { + subsys int + namespace string + want NVMeSubsystemNamespaceDiag + }{ + { + subsys: 0, + namespace: "nvme0n1", + want: NVMeSubsystemNamespaceDiag{ + IORequeueNoUsablePath: new(uint64(9)), + IOFailNoAvailablePath: new(uint64(1)), + }, + }, + { + // No diag directory, as on kernels older than 7.2. + subsys: 1, + namespace: "nvme3n1", + want: NVMeSubsystemNamespaceDiag{}, + }, + } { + diag, err := got[tt.subsys].NamespaceDiag(tt.namespace) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tt.want, diag); diff != "" { + t.Errorf("unexpected %s diag (-want +got):\n%s", tt.namespace, diff) + } + } + + if _, err := got[0].NamespaceDiag("../nvme-subsys1/nvme3n1"); err == nil { + t.Error("expected an error for an invalid namespace name") + } } func TestNVMeSubsystemClassNotPresent(t *testing.T) { diff --git a/testdata/fixtures.ttar b/testdata/fixtures.ttar index 802d1b13..b6f18ae7 100644 --- a/testdata/fixtures.ttar +++ b/testdata/fixtures.ttar @@ -7401,6 +7401,19 @@ Mode: 644 Directory: fixtures/sys/class/nvme-subsystem/nvme-subsys0/nvme0n1 Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/nvme-subsystem/nvme-subsys0/nvme0n1/diag +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme-subsystem/nvme-subsys0/nvme0n1/diag/io_fail_no_available_path_count +Lines: 1 +1 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/nvme-subsystem/nvme-subsys0/nvme0n1/diag/io_requeue_no_usable_path_count +Lines: 1 +9 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Directory: fixtures/sys/class/nvme-subsystem/nvme-subsys0/nvme1 Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -