Merge pull request #140 from pridhiviraj/sysdump_fixes

Fix system dump test case by enabling the dump policy.

Author: stewartsmith

common/OpTestFSP.py

@@ -299,6 +299,20 @@ def wait_for_ipling(self, timeout=10):
             time.sleep(5)
         return BMC_CONST.FW_SUCCESS
 
+    def wait_for_dump_to_start(self):
+        count = 0
+        # Dump maximum can start in one minute(So lets wait for 3 mins)
+        while count < 3:
+            if self.get_sys_status() == "dumping":
+                return True
+            count += 1
+            time.sleep(60)
+        else:
+            print "Current system status: %s" % self.get_sys_status()
+            print "Current progress code: %s" % self.get_progress_code()
+            raise OpTestError("System dump not started even after 3 minutes")
+
+
     ##
     # @brief Wait for system to reach runtime
     # @returns 0 on success or throws exception
@@ -318,6 +332,15 @@ def wait_for_runtime(self, timeout=10):
             time.sleep(5)
         return BMC_CONST.FW_SUCCESS
 
+    def enable_system_dump(self):
+        print "Enabling the system dump policy"
+        self.fspc.run_command("sysdump -sp enableSys")
+        res = self.fspc.run_command("sysdump -vp")
+        if "System dumps             Enabled       Enabled" in res:
+            print "System dump policy enabled successfully"
+            return True
+        raise OpTestError("Failed to enable system dump policy")
+
     ##
     # @brief Trigger system dump from fsp
     # @returns True on success or raises exception
@@ -340,6 +363,8 @@ def trigger_system_dump(self):
     # @returns True on success or throws exception
     #
     def wait_for_systemdump_to_finish(self):
+        self.wait_for_dump_to_start()
+        # If dump starts then wait for finish it
         count = 0
         while count < 30:
             res = self.fspc.run_command("sysdump -qall")

op-test

@@ -60,6 +60,7 @@ from testcases import OpTestMCColdResetEffects
 from testcases import OpTestDumps
 from testcases import HostLogin
 from testcases import OpalMsglog
+from testcases import KernelLog
 from testcases import OpTestFlash
 from testcases import OpalErrorLog
 from testcases import LightPathDiagnostics
@@ -101,6 +102,7 @@ class SkirootSuite():
         self.s.addTest(OpTestHeartbeat.HeartbeatSkiroot())
         self.s.addTest(OpTestNVRAM.SkirootNVRAM())
         self.s.addTest(OpalMsglog.Skiroot())
+        self.s.addTest(KernelLog.Skiroot())
         self.s.addTest(DPO.DPOSkiroot())
 #        self.s.addTest(OpTestEnergyScale.runtime_suite())
     def suite(self):
@@ -126,6 +128,7 @@ class HostSuite():
         self.s.addTest(OpTestSensors.OpTestSensors())
         self.s.addTest(OpalErrorLog.BasicTest())
         self.s.addTest(OpalMsglog.Host())
+        self.s.addTest(KernelLog.Host())
         self.s.addTest(OpTestPrdDaemon.OpTestPrdDaemon())
     def suite(self):
         return self.s
@@ -308,6 +311,9 @@ if OpTestConfiguration.conf.args.list_suites:
         print '{0:34}{1}'.format(key, suites[key].__doc__)
     exit(0)
 
+reload(sys)
+sys.setdefaultencoding("utf8")
+
 t = unittest.TestSuite()
 
 if OpTestConfiguration.conf.args.run_suite:

testcases/KernelLog.py

@@ -0,0 +1,58 @@
+#!/usr/bin/python
+# OpenPOWER Automated Test Project
+#
+# Contributors Listed Below - COPYRIGHT 2017
+# [+] International Business Machines Corp.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+import unittest
+
+import OpTestConfiguration
+from common.OpTestSystem import OpSystemState
+from common.OpTestConstants import OpTestConstants as BMC_CONST
+
+class KernelLog():
+    def setUp(self):
+        conf = OpTestConfiguration.conf
+        self.cv_HOST = conf.host()
+        self.cv_IPMI = conf.ipmi()
+        self.cv_SYSTEM = conf.system()
+
+    def runTest(self):
+        self.setup_test()
+        if "skiroot" in self.test:
+            cmd = "dmesg -r|grep '<[4321]>'"
+        elif "host" in self.test:
+            cmd = "dmesg -T --level=alert,crit,err,warn"
+        else:
+            raise Exception("Unknow test type")
+
+        log_entries = self.c.run_command(cmd)
+        msg = '\n'.join(filter(None, log_entries))
+        self.assertTrue( len(log_entries) == 0, "Warnings/Errors in Kernel log:\n%s" % msg)
+
+class Skiroot(KernelLog, unittest.TestCase):
+    def setup_test(self):
+        self.test = "skiroot"
+        self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
+        self.c = self.cv_SYSTEM.sys_get_ipmi_console()
+        self.cv_SYSTEM.host_console_unique_prompt()
+
+class Host(KernelLog, unittest.TestCase):
+    def setup_test(self):
+        self.test = "host"
+        self.cv_SYSTEM.goto_state(OpSystemState.OS)
+        self.c = self.cv_SYSTEM.host().get_ssh_connection()

testcases/OpTestDumps.py

@@ -25,6 +25,10 @@
 #  Different dumps for fsp platforms
 #  fipsdump
 #  system dump
+#      host_dump_boottime --> Trigger dump when kernel boots(either Petitboot or OS kernel)
+#      host_dump_runtime  --> Trigger dump when system is in OS or already booted to OS
+#      nmi_dump           --> Trigger system dump by sending NMI interrupts to processors
+#
 
 import time
 import subprocess
@@ -57,7 +61,7 @@ def tearDown(self):
     def trigger_dump(self):
         if self.test == "nmi_dump":
             self.cv_IPMI.ipmi_power_diag()
-        elif self.test == "host_dump":
+        elif "host_dump" in self.test:
             self.cv_FSP.trigger_system_dump()
         else:
             raise Exception("Unknown test type")
@@ -138,15 +142,20 @@ def runTest(self):
             self.skipTest("FSP Platform OPAL specific dump tests")
         self.cv_HOST.host_check_command("opal-dump-parse")
         self.cv_HOST.host_run_command("rm -rf /var/log/dump/SYSDUMP*")
+        self.cv_HOST.host_run_command("rm -rf /var/log/dump/Opal-log*")
+        self.cv_HOST.host_run_command("rm -rf /var/log/dump/HostBoot-Runtime-log*")
+        self.cv_HOST.host_run_command("rm -rf /var/log/dump/printk*")
         self.cv_FSP.fsp_get_console()
         if not self.cv_FSP.mount_exists():
             raise OpTestError("Please mount NFS and retry the test")
 
         self.set_up()
         print self.cv_FSP.fsp_run_command("smgr mfgState")
+        self.cv_FSP.enable_system_dump()
         self.cv_FSP.clear_fsp_errors()
-        self.cv_FSP.power_off_sys()
-        self.cv_FSP.power_on_sys()
+        if "host_dump_boottime" in self.test:
+            self.cv_FSP.power_off_sys()
+            self.cv_FSP.power_on_sys()
         self.util.PingFunc(self.cv_HOST.ip, BMC_CONST.PING_RETRY_POWERCYCLE)
         self.trigger_dump()
         self.cv_FSP.wait_for_systemdump_to_finish()
@@ -164,6 +173,14 @@ def runTest(self):
         self.assertIn("Opal", '\n'.join(res), "sysdump test failed in dumping Opal-log section")
         self.assertIn("HostBoot-Runtime-log", '\n'.join(res), "sysdump test failed in dumping HBRT section")
         self.assertIn("printk", '\n'.join(res), "sysdump test failed in dumping printk section")
+        self.cv_HOST.host_run_command("cd /var/log/dump/")
+        self.cv_HOST.host_run_command("opal-dump-parse -s 1 /var/log/dump/SYSDUMP*")
+        self.cv_HOST.host_run_command("cat Opal-log*")
+        self.cv_HOST.host_run_command("opal-dump-parse -s 2  /var/log/dump/SYSDUMP*")
+        self.cv_HOST.host_run_command("cat HostBoot-Runtime-log*")
+        self.cv_HOST.host_run_command("opal-dump-parse -s 128 /var/log/dump/SYSDUMP*")
+        self.cv_HOST.host_run_command("cat printk*")
+
 
 class FIPS_DUMP(OpTestDumps, unittest.TestCase):
 
@@ -201,10 +218,16 @@ def runTest(self):
             dumpname, size = self.fipsdump_initiate_from_host()
             self.verify_fipsdump(dumpname, size)
 
-class HOST_DUMP(SYSTEM_DUMP):
+class HOST_DUMP_BOOTTIME(SYSTEM_DUMP):
+
+    def set_up(self):
+        self.test = "host_dump_boottime"
+
+class HOST_DUMP_RUNTIME(SYSTEM_DUMP):
 
     def set_up(self):
-        self.test = "host_dump"
+        self.test = "host_dump_runtime"
+
 
 class NMI_DIAG_DUMP(SYSTEM_DUMP):
 
@@ -214,6 +237,7 @@ def set_up(self):
 def suite():
     s = unittest.TestSuite()
     s.addTest(FIPS_DUMP())
-    s.addTest(HOST_DUMP())
+    s.addTest(HOST_DUMP_RUNTIME())
     s.addTest(NMI_DIAG_DUMP())
+    s.addTest(HOST_DUMP_BOOTTIME())
     return s

testcases/fspresetReload.py

@@ -174,7 +174,10 @@ def check_for_fsp_host_interfaces(self):
         self.assertTrue(self.check_for_inbandipmi(), "inband ipmi interface failed after fsp rr")
         self.assertTrue(self.check_for_sensors(), "inband sensors failed after fsp rr")
         self.assertTrue(self.check_for_nvram(), "nvram interface failed after fsp rr")
-        self.check_for_sol_console()
+        try:
+            self.check_for_sol_console()
+        except:
+            pass
 
 
     def look_for_in_opal_log(self, pattern):
@@ -245,6 +248,10 @@ class HIRTorture(HIR):
     def number_of_resets(self):
         return 20
 
+class SIRTorture(SIR):
+    def number_of_resets(self):
+        return 20
+
 def suite():
     s = unittest.TestSuite()
     s.addTest(FIR())
@@ -255,4 +262,5 @@ def torture_suite():
     s = unittest.TestSuite()
     s.addTest(FIRTorture())
     s.addTest(HIRTorture())
+    s.addTest(SIRTorture())
     return s