Merge remote-tracking branch 'pridhiviraj/openbmc_fixes'

Author: stewartsmith

common/Exceptions.py

@@ -37,6 +37,24 @@ def __init__(self, command, output, exitcode):
     def __str__(self):
         return "Command '%s' exited with %d.\nOutput:\n%s" % (self.command, self.exitcode, self.output)
 
+class SSHSessionDisconnected(Exception):
+    '''
+    SSH session/console was disconnected unexpectedly. e.g. it may have crashed
+    '''
+    def __init__(self, notice):
+        self.notice = notice
+    def __str__(self):
+        return "SSH session/console disconnected due to '%s'" % self.notice
+
+class LoginFailure(Exception):
+    '''
+    SSH/IPMI Login failure - username/password may be wrong
+    '''
+    def __init__(self, notice):
+        self.notice = notice
+    def __str__(self):
+        return "Login failure due to '%s'" % self.notice
+
 class BMCDisconnected(Exception):
     '''
     BMC Cosnole was disconnected unexpectedly. e.g. it may have crashed

common/OpTestBMC.py

@@ -37,10 +37,11 @@
 import subprocess
 from OpTestIPMI import OpTestIPMI
 from OpTestSSH import OpTestSSH
+from OpTestUtil import OpTestUtil
 from OpTestConstants import OpTestConstants as BMC_CONST
 from OpTestError import OpTestError
 from OpTestWeb import OpTestWeb
-from Exceptions import CommandFailed
+from Exceptions import CommandFailed, SSHSessionDisconnected
 
 class OpTestBMC():
     def __init__(self, ip=None, username=None, password=None,
@@ -57,6 +58,7 @@ def __init__(self, ip=None, username=None, password=None,
         self.known_hosts_file = known_hosts_file
         self.ssh = OpTestSSH(ip, username, password, logfile, prompt='\[PEXPECT\]#',
                 check_ssh_keys=check_ssh_keys, known_hosts_file=known_hosts_file)
+        self.util = OpTestUtil()
 
     def bmc_host(self):
         return self.cv_bmcIP
@@ -87,10 +89,13 @@ def reboot(self):
         retries = 0
         try:
             self.ssh.run_command('reboot')
-        except pexpect.EOF:
+        except SSHSessionDisconnected:
             pass
         print 'Sent reboot command now waiting for reboot to complete...'
-        time.sleep(10)
+        # Wait for BMC to go down.
+        self.util.ping_fail_check(self.cv_bmcIP)
+        # Wait for BMC to ping back.
+        self.util.PingFunc(self.cv_bmcIP, BMC_CONST.PING_RETRY_FOR_STABILITY)
         '''  Ping the system until it reboots  '''
         while True:
             try:

common/OpTestOpenBMC.py

@@ -28,7 +28,7 @@
 from OpTestIPMI import OpTestIPMI
 from OpTestUtil import OpTestUtil
 from OpTestBMC import OpTestBMC
-from Exceptions import CommandFailed
+from Exceptions import CommandFailed, LoginFailure
 from common.OpTestError import OpTestError
 from OpTestConstants import OpTestConstants as BMC_CONST
 from common import OPexpect
@@ -51,6 +51,8 @@ def __init__(self, binary="curl",
         self.password = password
         self.binary = binary
         self.logresult = True
+        self.cmd_bkup = ""
+        self.login_retry = 0
 
     def feed_data(self, dbus_object=None, action=None,
                   operation=None, command=None,
@@ -137,8 +139,13 @@ def arguments(self):
         args += self.dbus_interface()
         return args
 
+    def bkup_cmd(self, cmd):
+        self.cmd_bkup = cmd
+
     def run(self, background=False, cmdprefix=None):
-        if cmdprefix:
+        if self.cmd_bkup:
+            cmd = self.cmd_bkup
+        elif cmdprefix:
             cmd = cmdprefix + self.binary + self.arguments() + cmd
         else:
             cmd = self.binary + self.arguments()
@@ -155,16 +162,26 @@ def run(self, background=False, cmdprefix=None):
             # TODO - need python 2.7
             # output = check_output(cmd, stderr=subprocess.STDOUT, shell=True)
             try:
-                cmd = subprocess.Popen(cmd, stderr=subprocess.STDOUT,
+                obj = subprocess.Popen(cmd, stderr=subprocess.STDOUT,
                                        stdout=subprocess.PIPE, shell=True)
             except Exception as e:
                 l_msg = "Curl Command Failed"
                 print l_msg
                 print str(e)
                 raise OpTestError(l_msg)
-            output = cmd.communicate()[0]
+            output = obj.communicate()[0]
             if self.logresult:
                 print output
+            if '"description": "Login required"' in output:
+                if self.login_retry > 5:
+                    raise LoginFailure("Rest Login retry exceeded")
+                output = ""
+                cmd_bkup =  cmd
+                self.login()
+                self.login_retry += 1
+                self.bkup_cmd(cmd_bkup)
+                output = self.run()
+                self.cmd_bkup = ""
             if '"status": "error"' in output:
                 print output
                 raise FailedCurlInvocation(cmd, output)
@@ -173,6 +190,16 @@ def run(self, background=False, cmdprefix=None):
     def log_result(self):
         self.logresult = True
 
+    def login(self):
+        data = '\'{"data": [ "%s", "%s" ] }\'' % (self.username, self.password)
+        self.feed_data(dbus_object="/login", operation='w', command="POST", data=data)
+        try:
+            output = self.run()
+        except FailedCurlInvocation as fci:
+            output = fci.output
+        if '"description": "Invalid username or password"' in output:
+            raise LoginFailure("Rest login invalid username or password")
+
 class HostManagement():
     def __init__(self, ip=None, username=None, password=None):
         self.hostname = ip
@@ -182,7 +209,9 @@ def __init__(self, ip=None, username=None, password=None):
                              username=username,
                              password=password)
         self.util = OpTestUtil()
+        self.util.PingFunc(self.hostname, BMC_CONST.PING_RETRY_FOR_STABILITY)
         self.login()
+        self.wait_for_bmc_runtime()
 
     '''
     curl -c cjar -k -X POST -H "Content-Type: application/json" \
@@ -191,7 +220,12 @@ def __init__(self, ip=None, username=None, password=None):
     def login(self):
         data = '\'{"data": [ "%s", "%s" ] }\'' % (self.username, self.password)
         self.curl.feed_data(dbus_object="/login", operation='w', command="POST", data=data)
-        self.curl.run()
+        try:
+            output = self.curl.run()
+        except FailedCurlInvocation as fci:
+            output = fci.output
+        if '"description": "Invalid username or password"' in output:
+            raise LoginFailure("Rest login invalid username or password")
 
     '''
     Logout:

common/OpTestSSH.py

@@ -22,7 +22,7 @@
 import time
 import pexpect
 
-from Exceptions import CommandFailed
+from Exceptions import CommandFailed, SSHSessionDisconnected
 import OpTestSystem
 try:
     from common import OPexpect
@@ -131,9 +131,9 @@ def set_unique_prompt(self, console):
         except pexpect.EOF as cf:
             print cf
             if self.check_ssh_keys:
-                raise Exception("SSH session exited early - bad host key?")
+                raise SSHSessionDisconnected("SSH session exited early - bad host key?")
             else:
-                raise Exception("SSH session exited early!")
+                raise SSHSessionDisconnected("SSH session exited early!")
 
     def build_prompt(self):
         if self.prompt:
@@ -212,7 +212,7 @@ def retry_password(self, console, command):
             raise CommandFailed(command, 'Retry Password TIMEOUT ' + ''.join(retry_list_output), -1)
           elif (rc == 3):
             self.state = ConsoleState.DISCONNECTED
-            raise Exception("SSH session exited early!")
+            raise SSHSessionDisconnected("SSH session exited early!")
 
         return retry_list_output, echo_rc
 
@@ -249,7 +249,7 @@ def handle_password(self, console, command):
           failure_list_output += handle_list_output
           if (rc == 3):
             self.state = ConsoleState.DISCONNECTED
-            raise Exception("SSH session exited early!")
+            raise SSHSessionDisconnected("SSH session exited early!")
           else:
             raise CommandFailed(command, ''.join(failure_list_output), -1)
         return list_output, echo_rc
@@ -292,7 +292,7 @@ def run_command(self, command, timeout=60):
           output_list, echo_rc = self.try_sendcontrol(console, command)
         else:
           self.state = ConsoleState.DISCONNECTED
-          raise Exception("SSH session exited early!")
+          raise SSHSessionDisconnected("SSH session exited early!")
         res = output_list
         if echo_rc != 0:
           raise CommandFailed(command, res, echo_rc)