sysfs: parse NVMe diag counters (controller, namespace, multipath head)

sysfs/class_nvme.go

@@ -16,6 +16,7 @@
 package sysfs
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -51,9 +52,30 @@ type NVMeNamespace struct {
 	CapacityBytes    uint64 // calculated: SizeBlocks * 512
 }
 
+// NVMeNamespaceDiag holds the diagnostic counters of a namespace block device
+// from /sys/class/nvme/<device>/<namespace>/diag, added in Linux 7.2. A nil
+// field means the kernel does not expose that counter. Root can reset the
+// counters by writing to them. Use NVMeDevice.NamespaceDiag to read them.
+type NVMeNamespaceDiag struct {
+	CommandRetries     *uint64 // diag/command_retries_count: I/O commands retried
+	CommandErrors      *uint64 // diag/command_error_count: I/O commands completed with an error
+	MultipathFailovers *uint64 // diag/multipath_failover_count: I/O failed over to another path
+}
+
+// NVMeControllerDiag holds the diagnostic counters of a controller from
+// /sys/class/nvme/<device>/diag, added in Linux 7.2. A nil field means the
+// kernel does not expose that counter. Root can reset the counters by writing
+// to them. Use NVMeDevice.Diag to read them.
+type NVMeControllerDiag struct {
+	CommandErrors *uint64 // diag/command_error_count: admin commands completed with an error
+	Resets        *uint64 // diag/reset_count: controller resets
+	Reconnects    *uint64 // diag/reconnect_count: reconnect attempts, fabrics transports only
+}
+
 // NVMeDevice contains info from files in /sys/class/nvme for a single NVMe device.
 type NVMeDevice struct {
 	Name             string
+	Path             string          // sysfs directory of the device, /sys/class/nvme/<Name>
 	Serial           string          // /sys/class/nvme/<Name>/serial
 	Model            string          // /sys/class/nvme/<Name>/model
 	State            string          // /sys/class/nvme/<Name>/state
@@ -92,7 +114,7 @@ func (fs FS) NVMeClass() (NVMeClass, error) {
 // Parse one NVMe device.
 func (fs FS) parseNVMeDevice(name string) (*NVMeDevice, error) {
 	path := fs.sys.Path(nvmeClassPath, name)
-	device := NVMeDevice{Name: name}
+	device := NVMeDevice{Name: name, Path: path}
 
 	// Parse device-level attributes
 	for _, f := range [...]string{"firmware_rev", "model", "serial", "state", "cntlid"} {
@@ -191,3 +213,67 @@ func (fs FS) parseNVMeDevice(name string) (*NVMeDevice, error) {
 
 	return &device, nil
 }
+
+// Diag reads the controller's diagnostic counters from
+// /sys/class/nvme/<device>/diag. They are read on demand rather than by
+// NVMeClass, so callers that do not need them avoid the extra file reads.
+func (d NVMeDevice) Diag() (NVMeControllerDiag, error) {
+	var (
+		diag NVMeControllerDiag
+		err  error
+	)
+	dir := filepath.Join(d.Path, "diag")
+	for _, c := range [...]struct {
+		file string
+		dest **uint64
+	}{
+		{"command_error_count", &diag.CommandErrors},
+		{"reset_count", &diag.Resets},
+		{"reconnect_count", &diag.Reconnects},
+	} {
+		if *c.dest, err = readOptionalUint(filepath.Join(dir, c.file)); err != nil {
+			return diag, err
+		}
+	}
+	return diag, nil
+}
+
+// NamespaceDiag reads the diagnostic counters of one of the device's namespace
+// block devices, given by name as in NVMeNamespace.Name (e.g. nvme0n1 or
+// nvme0c1n1), from /sys/class/nvme/<device>/<namespace>/diag.
+func (d NVMeDevice) NamespaceDiag(namespace string) (NVMeNamespaceDiag, error) {
+	var (
+		diag NVMeNamespaceDiag
+		err  error
+	)
+	if !nvmeNamespacePattern.MatchString(namespace) {
+		return diag, fmt.Errorf("invalid NVMe namespace name %q", namespace)
+	}
+	dir := filepath.Join(d.Path, namespace, "diag")
+	for _, c := range [...]struct {
+		file string
+		dest **uint64
+	}{
+		{"command_retries_count", &diag.CommandRetries},
+		{"command_error_count", &diag.CommandErrors},
+		{"multipath_failover_count", &diag.MultipathFailovers},
+	} {
+		if *c.dest, err = readOptionalUint(filepath.Join(dir, c.file)); err != nil {
+			return diag, err
+		}
+	}
+	return diag, nil
+}
+
+// readOptionalUint reads an unsigned integer sysfs attribute, returning nil
+// when the attribute does not exist on the running kernel.
+func readOptionalUint(path string) (*uint64, error) {
+	v, err := parsers.SysReadUintFromFile(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to read file %q: %w", path, err)
+	}
+	return &v, nil
+}

sysfs/class_nvme_subsystem.go

@@ -18,6 +18,7 @@ package sysfs
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -35,6 +36,9 @@ var (
 type NVMeSubsystem struct {
 	// Name is the subsystem directory name, e.g. "nvme-subsys0".
 	Name string
+	// Path is the subsystem's sysfs directory,
+	// /sys/class/nvme-subsystem/<Name>.
+	Path string
 	// NQN is the NVMe Qualified Name from subsysnqn.
 	NQN string
 	// Model is the subsystem model string.
@@ -50,6 +54,19 @@ type NVMeSubsystem struct {
 	Namespaces []string
 }
 
+// NVMeSubsystemNamespaceDiag holds the diagnostic counters of a multipath
+// namespace head from /sys/class/nvme-subsystem/<subsys>/<namespace>/diag.
+// A nil field means the kernel does not expose that counter. Use
+// NVMeSubsystem.NamespaceDiag to read them.
+type NVMeSubsystemNamespaceDiag struct {
+	// IORequeueNoUsablePath counts I/O requeued because no path was usable
+	// (diag/io_requeue_no_usable_path_count).
+	IORequeueNoUsablePath *uint64
+	// IOFailNoAvailablePath counts I/O failed because no path was available
+	// (diag/io_fail_no_available_path_count).
+	IOFailNoAvailablePath *uint64
+}
+
 // NVMeSubsystemController contains info about a single NVMe controller
 // within an NVMe subsystem.
 type NVMeSubsystemController struct {
@@ -94,7 +111,7 @@ func (fs FS) NVMeSubsystemClass() (NVMeSubsystemClass, error) {
 
 func (fs FS) parseNVMeSubsystem(name string) (*NVMeSubsystem, error) {
 	path := fs.sys.Path(nvmeSubsystemClassPath, name)
-	subsys := &NVMeSubsystem{Name: name}
+	subsys := &NVMeSubsystem{Name: name, Path: path}
 
 	for _, attr := range [...]struct {
 		file string
@@ -156,3 +173,26 @@ func (fs FS) parseNVMeSubsystemController(subsysName, ctrlName string) (*NVMeSub
 
 	return ctrl, nil
 }
+
+// NamespaceDiag reads the diagnostic counters of one of the subsystem's
+// multipath namespace head devices, given by name as in Namespaces (e.g.
+// nvme0n1), from /sys/class/nvme-subsystem/<subsys>/<namespace>/diag. They
+// are read on demand rather than by NVMeSubsystemClass, so callers that do
+// not need them avoid the extra file reads.
+func (s NVMeSubsystem) NamespaceDiag(namespace string) (NVMeSubsystemNamespaceDiag, error) {
+	var (
+		diag NVMeSubsystemNamespaceDiag
+		err  error
+	)
+	if !nvmeNamespaceRE.MatchString(namespace) {
+		return diag, fmt.Errorf("invalid NVMe namespace name %q", namespace)
+	}
+	dir := filepath.Join(s.Path, namespace, "diag")
+	if diag.IORequeueNoUsablePath, err = readOptionalUint(filepath.Join(dir, "io_requeue_no_usable_path_count")); err != nil {
+		return diag, err
+	}
+	if diag.IOFailNoAvailablePath, err = readOptionalUint(filepath.Join(dir, "io_fail_no_available_path_count")); err != nil {
+		return diag, err
+	}
+	return diag, nil
+}

sysfs/class_nvme_subsystem_test.go

@@ -37,6 +37,7 @@ func TestNVMeSubsystemClass(t *testing.T) {
 	want := NVMeSubsystemClass{
 		{
 			Name:     "nvme-subsys0",
+			Path:     filepath.Join(sysTestFixtures, "class/nvme-subsystem/nvme-subsys0"),
 			NQN:      "nqn.2014-08.org.nvmexpress:uuid:a34c4f3a-0d6f-5cec-dead-beefcafebabe",
 			Model:    "Dell PowerStore",
 			Serial:   "SN12345678",
@@ -50,6 +51,7 @@ func TestNVMeSubsystemClass(t *testing.T) {
 		},
 		{
 			Name:     "nvme-subsys1",
+			Path:     filepath.Join(sysTestFixtures, "class/nvme-subsystem/nvme-subsys1"),
 			NQN:      "nqn.2014-08.org.nvmexpress:uuid:b45d5e4b-1e7f-6ded-beef-deadcafe1234",
 			Model:    "NetApp ONTAP",
 			Serial:   "NTAP98765",
@@ -65,6 +67,40 @@ func TestNVMeSubsystemClass(t *testing.T) {
 	if diff := cmp.Diff(want, got); diff != "" {
 		t.Fatalf("unexpected NVMeSubsystemClass (-want +got):\n%s", diff)
 	}
+
+	// Multipath head diag counters, read on demand.
+	for _, tt := range []struct {
+		subsys    int
+		namespace string
+		want      NVMeSubsystemNamespaceDiag
+	}{
+		{
+			subsys:    0,
+			namespace: "nvme0n1",
+			want: NVMeSubsystemNamespaceDiag{
+				IORequeueNoUsablePath: new(uint64(9)),
+				IOFailNoAvailablePath: new(uint64(1)),
+			},
+		},
+		{
+			// No diag directory, as on kernels older than 7.2.
+			subsys:    1,
+			namespace: "nvme3n1",
+			want:      NVMeSubsystemNamespaceDiag{},
+		},
+	} {
+		diag, err := got[tt.subsys].NamespaceDiag(tt.namespace)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if diff := cmp.Diff(tt.want, diag); diff != "" {
+			t.Errorf("unexpected %s diag (-want +got):\n%s", tt.namespace, diff)
+		}
+	}
+
+	if _, err := got[0].NamespaceDiag("../nvme-subsys1/nvme3n1"); err == nil {
+		t.Error("expected an error for an invalid namespace name")
+	}
 }
 
 func TestNVMeSubsystemClassNotPresent(t *testing.T) {

sysfs/class_nvme_test.go

@@ -43,6 +43,7 @@ func TestNVMeClass(t *testing.T) {
 	want := NVMeClass{
 		"nvme0": NVMeDevice{
 			Name:             "nvme0",
+			Path:             filepath.Join(sysTestFixtures, "class/nvme/nvme0"),
 			FirmwareRevision: "1B2QEXP7",
 			Model:            "Samsung SSD 970 PRO 512GB",
 			Serial:           "S680HF8N190894I",
@@ -66,6 +67,7 @@ func TestNVMeClass(t *testing.T) {
 		// namespace is the hidden per-path device nvme<subsys>c<ctrl>n<inst>.
 		"nvme1": NVMeDevice{
 			Name:             "nvme1",
+			Path:             filepath.Join(sysTestFixtures, "class/nvme/nvme1"),
 			FirmwareRevision: "1B2QEXP7",
 			Model:            "Samsung SSD 970 PRO 512GB",
 			Serial:           "S680HF8N190894I",
@@ -92,6 +94,104 @@ func TestNVMeClass(t *testing.T) {
 	}
 }
 
+func TestNVMeDeviceDiag(t *testing.T) {
+	if _, err := os.Stat(sysTestFixtures); os.IsNotExist(err) {
+		t.Skip("Test fixtures not available, skipping NVMe diag tests")
+		return
+	}
+
+	fs, err := NewFS(sysTestFixtures)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	nc, err := fs.NVMeClass()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tt := range []struct {
+		device    string
+		namespace string
+		ctrl      NVMeControllerDiag
+		ns        NVMeNamespaceDiag
+	}{
+		{
+			// PCIe controller: no reconnect_count, no multipath_failover_count.
+			device:    "nvme0",
+			namespace: "nvme0n0",
+			ctrl: NVMeControllerDiag{
+				CommandErrors: new(uint64(3)),
+				Resets:        new(uint64(1)),
+			},
+			ns: NVMeNamespaceDiag{
+				CommandRetries: new(uint64(12)),
+				CommandErrors:  new(uint64(2)),
+			},
+		},
+		{
+			// Fabrics controller with a multipath per-path namespace.
+			device:    "nvme1",
+			namespace: "nvme0c1n1",
+			ctrl: NVMeControllerDiag{
+				CommandErrors: new(uint64(0)),
+				Resets:        new(uint64(2)),
+				Reconnects:    new(uint64(5)),
+			},
+			ns: NVMeNamespaceDiag{
+				CommandRetries:     new(uint64(4)),
+				CommandErrors:      new(uint64(1)),
+				MultipathFailovers: new(uint64(7)),
+			},
+		},
+	} {
+		t.Run(tt.device, func(t *testing.T) {
+			d := nc[tt.device]
+
+			ctrl, err := d.Diag()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(tt.ctrl, ctrl); diff != "" {
+				t.Errorf("unexpected controller diag (-want +got):\n%s", diff)
+			}
+
+			ns, err := d.NamespaceDiag(tt.namespace)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(tt.ns, ns); diff != "" {
+				t.Errorf("unexpected namespace diag (-want +got):\n%s", diff)
+			}
+		})
+	}
+
+	if _, err := nc["nvme0"].NamespaceDiag("../nvme1"); err == nil {
+		t.Error("expected an error for an invalid namespace name")
+	}
+}
+
+func TestNVMeDeviceDiagMissing(t *testing.T) {
+	// A kernel older than 7.2 has no diag directory: every counter is nil.
+	d := NVMeDevice{Name: "nvme0", Path: t.TempDir()}
+
+	ctrl, err := d.Diag()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(NVMeControllerDiag{}, ctrl); diff != "" {
+		t.Errorf("unexpected controller diag (-want +got):\n%s", diff)
+	}
+
+	ns, err := d.NamespaceDiag("nvme0n1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(NVMeNamespaceDiag{}, ns); diff != "" {
+		t.Errorf("unexpected namespace diag (-want +got):\n%s", diff)
+	}
+}
+
 func TestNVMeNamespaceParsingWithMockData(t *testing.T) {
 	// Create a temporary directory structure for testing namespace parsing
 	tempDir := t.TempDir()