Merge pull request #365 from cdburkard/devel/test_output

print useful output for tests upon failure

mininet/test/test_hifi.py

@@ -40,24 +40,47 @@ class testOptionsTopoCommon( object ):
 
     switchClass = None # overridden in subclasses
 
-    def runOptionsTopoTest( self, n, hopts=None, lopts=None ):
+    def runOptionsTopoTest( self, n, msg, hopts=None, lopts=None ):
         "Generic topology-with-options test runner."
         mn = Mininet( topo=SingleSwitchOptionsTopo( n=n, hopts=hopts,
                                                     lopts=lopts ),
                       host=CPULimitedHost, link=TCLink,
                       switch=self.switchClass, waitConnected=True )
         dropped = mn.run( mn.ping )
-        self.assertEqual( dropped, 0 )
-
-    def assertWithinTolerance(self, measured, expected, tolerance_frac):
+        hoptsStr = ', '.join( '%s: %s' % ( opt, value )
+                              for opt, value in hopts.items() )
+        loptsStr = ', '.join( '%s: %s' % ( opt, value )
+                              for opt, value in lopts.items() )
+        msg += ( '%s%% of pings were dropped during mininet.ping().\n'
+                 'Topo = SingleSwitchTopo, %s hosts\n'
+                 'hopts = %s\n'
+                 'lopts = %s\n'
+                 'host = CPULimitedHost\n'
+                 'link = TCLink\n'
+                 'Switch = %s\n'
+                 % ( dropped, n, hoptsStr, loptsStr, self.switchClass ) )
+
+        self.assertEqual( dropped, 0, msg=msg )
+
+    def assertWithinTolerance( self, measured, expected, tolerance_frac, msg ):
         """Check that a given value is within a tolerance of expected
         tolerance_frac: less-than-1.0 value; 0.8 would yield 20% tolerance.
         """
-        self.assertGreaterEqual( float(measured),
-                                 float(expected) * tolerance_frac )
-        self.assertLessEqual( float( measured ),
-                                 float(expected) + (1-tolerance_frac)
-                                 * float( expected ) )
+        upperBound = ( float( expected ) + ( 1 - tolerance_frac ) *
+                       float( expected ) )
+        lowerBound = float( expected ) * tolerance_frac
+        info = ( 'measured value is out of bounds\n'
+                 'expected value: %s\n'
+                 'measured value: %s\n'
+                 'failure tolerance: %s\n'
+                 'upper bound: %s\n'
+                 'lower bound: %s\n'
+                 % ( expected, measured, tolerance_frac,
+                     upperBound, lowerBound ) )
+        msg += info
+
+        self.assertGreaterEqual( float( measured ),lowerBound, msg=msg )
+        self.assertLessEqual( float( measured ), upperBound, msg=msg )
 
     def testCPULimits( self ):
         "Verify topology creation with CPU limits set for both schedulers."
@@ -72,9 +95,22 @@ def testCPULimits( self ):
         mn.start()
         results = mn.runCpuLimitTest( cpu=CPU_FRACTION )
         mn.stop()
+        hostUsage = '\n'.join( 'h%s: %s' %
+                               ( n + 1, results[ ( n - 1 ) * 5: ( n * 5 ) - 1 ] )
+                               for n in range( N ) )
+        hoptsStr = ', '.join( '%s: %s' % ( opt, value )
+                              for opt, value in hopts.items() )
+        msg = ( '\nTesting cpu limited to %d%% of cpu per host\n'
+                'cpu usage percent per host:\n%s\n'
+                'Topo = SingleSwitchTopo, %s hosts\n'
+                'hopts = %s\n'
+                'host = CPULimitedHost\n'
+                'Switch = %s\n'
+                % ( CPU_FRACTION * 100, hostUsage, N, hoptsStr, self.switchClass ) )
         for pct in results:
             #divide cpu by 100 to convert from percentage to fraction
-            self.assertWithinTolerance( pct/100, CPU_FRACTION, CPU_TOLERANCE )
+            self.assertWithinTolerance( pct/100, CPU_FRACTION,
+                                        CPU_TOLERANCE, msg )
 
     def testLinkBandwidth( self ):
         "Verify that link bandwidths are accurate within a bound."
@@ -89,9 +125,20 @@ def testLinkBandwidth( self ):
                       link=TCLink, switch=self.switchClass,
                       waitConnected=True )
         bw_strs = mn.run( mn.iperf, format='m' )
+        loptsStr = ', '.join( '%s: %s' % ( opt, value )
+                              for opt, value in lopts.items() )
+        msg = ( '\nTesting link bandwidth limited to %d Mbps per link\n'
+                'iperf results[ client, server ]: %s\n'
+                'Topo = SingleSwitchTopo, %s hosts\n'
+                'Link = TCLink\n'
+                'lopts = %s\n'
+                'host = default\n'
+                'switch = %s\n'
+                % ( BW, bw_strs, N, loptsStr, self.switchClass ) )
+
         for bw_str in bw_strs:
             bw = float( bw_str.split(' ')[0] )
-            self.assertWithinTolerance( bw, BW, BW_TOLERANCE )
+            self.assertWithinTolerance( bw, BW, BW_TOLERANCE, msg )
 
     def testLinkDelay( self ):
         "Verify that link delays are accurate within a bound."
@@ -111,12 +158,26 @@ def testLinkDelay( self ):
         # pylint: disable-msg=W0612
         node, dest, ping_outputs = test_outputs
         sent, received, rttmin, rttavg, rttmax, rttdev = ping_outputs
-        self.assertEqual( sent, received )
+        pingFailMsg = 'sent %s pings, only received %s' % ( sent, received )
+        self.assertEqual( sent, received, msg=pingFailMsg )
         # pylint: enable-msg=W0612
+        loptsStr = ', '.join( '%s: %s' % ( opt, value )
+                              for opt, value in lopts.items() )
+        msg = ( '\nTesting Link Delay of %s ms\n'
+                'ping results across 4 links:\n'
+                '(Sent, Received, rttmin, rttavg, rttmax, rttdev)\n'
+                '%s\n'
+                'Topo = SingleSwitchTopo, %s hosts\n'
+                'Link = TCLink\n'
+                'lopts = %s\n'
+                'host = default'
+                'switch = %s\n'
+                % ( DELAY_MS, ping_outputs, N, loptsStr, self.switchClass ) )
+
         for rttval in [rttmin, rttavg, rttmax]:
             # Multiply delay by 4 to cover there & back on two links
             self.assertWithinTolerance( rttval, DELAY_MS * 4.0, 
-                                        DELAY_TOLERANCE)
+                                        DELAY_TOLERANCE, msg )
 
 
     def testLinkLoss( self ):
@@ -135,36 +196,54 @@ def testLinkLoss( self ):
         for _ in range(REPS):
             dropped_total += mn.ping(timeout='1')
         mn.stop()
-        self.assertGreater( dropped_total, 0 )
+
+        loptsStr = ', '.join( '%s: %s' % ( opt, value )
+                              for opt, value in lopts.items() )
+        msg = ( '\nTesting packet loss with %d%% loss rate\n'
+                'number of dropped pings during mininet.ping(): %s\n'
+                'expected number of dropped packets: 1\n'
+                'Topo = SingleSwitchTopo, %s hosts\n'
+                'Link = TCLink\n'
+                'lopts = %s\n'
+                'host = default\n'
+                'switch = %s\n'
+                % ( LOSS_PERCENT, dropped_total, N, loptsStr, self.switchClass ) )
+
+        self.assertGreater( dropped_total, 0, msg )
 
     def testMostOptions( self ):
         "Verify topology creation with most link options and CPU limits."
         lopts = { 'bw': 10, 'delay': '5ms', 'use_htb': True }
         hopts = { 'cpu': 0.5 / N }
-        self.runOptionsTopoTest( N, hopts=hopts, lopts=lopts )
+        msg = '\nTesting many cpu and link options\n'
+        self.runOptionsTopoTest( N, msg, hopts=hopts, lopts=lopts )
 
 # pylint: enable=E1101
 
 class testOptionsTopoOVSKernel( testOptionsTopoCommon, unittest.TestCase ):
     """Verify ability to create networks with host and link options
        (OVS kernel switch)."""
+    longMessage = True
     switchClass = OVSSwitch
 
 @unittest.skip( 'Skipping OVS user switch test for now' )
 class testOptionsTopoOVSUser( testOptionsTopoCommon, unittest.TestCase ):
     """Verify ability to create networks with host and link options
        (OVS user switch)."""
+    longMessage = True
     switchClass = partial( OVSSwitch, datapath='user' )
 
 @unittest.skipUnless( quietRun( 'which ivs-ctl' ), 'IVS is not installed' )
 class testOptionsTopoIVS( testOptionsTopoCommon, unittest.TestCase ):
     "Verify ability to create networks with host and link options (IVS)."
+    longMessage = True
     switchClass = IVSSwitch
 
 @unittest.skipUnless( quietRun( 'which ofprotocol' ),
                      'Reference user switch is not installed' )
 class testOptionsTopoUserspace( testOptionsTopoCommon, unittest.TestCase ):
     "Verify ability to create networks with host and link options (UserSwitch)."
+    longMessage = True
     switchClass = UserSwitch
 
 if __name__ == '__main__':