Version 10 (modified by, 9 years ago) (diff)


OpenFlow Load Balancer Tutorial

Appendix: Hints and Explanations

A. LabWiki Script for the LoadBalancer

defProperty('theSender', "outside-loadbaltest", "ID of sender node")
defProperty('theReceiver', "inside-loadbaltest", "ID of receiver node")
defProperty('theSwitch',"switch-loadbaltest", "ID of switch node")
defProperty('left', "eth1", "Left Interface name")
defProperty('right', "eth3", "Right Interface name")
defProperty('setinterval','1.0',"Time between iperf")
defProperty('serverip', "","Server interface IP")
defProperty('clientip', "","Client interface IP")
defProperty('setbandwidth', "100m", "Throughput of Sender")
defApplication('trace') do |app|
  app.description = 'Packet capture'
  app.binary_path = "/usr/bin/trace-oml2"
  app.defProperty('filter', 'Filter expression BPFEXP', '-f', {:type => :string, :mnemonic => 'f'})
  app.defProperty('interface', 'Interface to trace', '-i',{:type => :string, :mnemonic => 'i', :default => '"eth0"'})

app.defMeasurement("ip") do |m|
    m.defMetric('pktid',    :uint64, ' internal packet ID to link MPs')
    m.defMetric('ip_tos',   :uint32, ' Type of Service')
    m.defMetric('ip_len',   :uint32, ' Total Length')
    m.defMetric('ip_id',    :int32,  ' Identification')
    m.defMetric('ip_off',   :uint32, ' IP Fragment offset (and flags)')
    m.defMetric('ip_ttl',   :uint32, ' Time to Live')
    m.defMetric('ip_proto', :uint32, ' Protocol')
    m.defMetric('ip_sum',   :uint32, ' Checksum')
    m.defMetric('ip_src',   :string, ' Source Address')
    m.defMetric('ip_dst',   :string, ' Destination Address')
    m.defMetric('ip_sizeofpacket', :uint32, ' Size of the Packet')
    m.defMetric('ip_ts',    :double, ' timestamp of the measurement')

defApplication('iperf') do |app| 
  app.description = "Iperf is a traffic generator and bandwidth measurement
tool. It provides generators producing various forms of packet streams and port
for sending these packets via various transports, such as TCP and UDP."
  app.binary_path = "/usr/bin/iperf_oml2"

  #app.defProperty('interval', 'pause n seconds between periodic bandwidth reports', '-i',
   # :type => :double, :unit => "seconds", :default => '1.')
  app.defProperty('len', 'set length read/write buffer to n (default 8 KB)', '-l',
		  :type => :integer, :unit => "KiBytes")
  app.defProperty('print_mss', 'print TCP maximum segment size (MTU - TCP/IP header)', '-m',
		  :type => :boolean)
  app.defProperty('output', 'output the report or error message to this specified file', '-o',
		  :type => :string)
  app.defProperty('port', 'set server port to listen on/connect to to n (default 5001)', '-p',
		  :type => :integer)
  app.defProperty('udp', 'use UDP rather than TCP', '-u',
		  :type => :boolean,
		  :order => 2)
  app.defProperty('window', 'TCP window size (socket buffer size)', '-w',
		  :type => :integer, :unit => "Bytes")
  app.defProperty('bind', 'bind to <host>, an interface or multicast address', '-B',
		  :type => :string)
  app.defProperty('compatibility', 'for use with older versions does not sent extra msgs', '-C',
		  :type => :boolean)
  app.defProperty('mss', 'set TCP maximum segment size (MTU - 40 bytes)', '-M',
		  :type => :integer, :unit => "Bytes")
  app.defProperty('nodelay', 'set TCP no delay, disabling Nagle\'s Algorithm', '-N',
		  :type => :boolean)
  app.defProperty('IPv6Version', 'set the domain to IPv6', '-V',
		  :type => :boolean)
  app.defProperty('reportexclude', 'exclude C(connection) D(data) M(multicast) S(settings) V(server) reports', '-x',
		  :type => :string, :unit => "[CDMSV]")
  app.defProperty('reportstyle', 'C or c for CSV report, O or o for OML', '-y',
		  :type => :string, :unit => "[CcOo]", :default => "o") # Use OML reporting by default

  app.defProperty('server', 'run in server mode', '-s',
		  :type => :boolean)

  app.defProperty('bandwidth', 'set target bandwidth to n bits/sec (default 1 Mbit/sec)', '-b',
		  :type => :string, :unit => "Mbps")
  app.defProperty('client', 'run in client mode, connecting to <host>', '-c',
		  :type => :string,
		  :order => 1)
  app.defProperty('dualtest', 'do a bidirectional test simultaneously', '-d',
		  :type => :boolean)
  app.defProperty('num', 'number of bytes to transmit (instead of -t)', '-n',
		  :type => :integer, :unit => "Bytes")
  app.defProperty('tradeoff', 'do a bidirectional test individually', '-r',
		  :type => :boolean)
  app.defProperty('time', 'time in seconds to transmit for (default 10 secs)', '-t',
		  :type => :integer, :unit => "seconds")
  app.defProperty('fileinput', 'input the data to be transmitted from a file', '-F',
		  :type => :string)
  app.defProperty('stdin', 'input the data to be transmitted from stdin', '-I',
		  :type => :boolean)
  app.defProperty('listenport', 'port to recieve bidirectional tests back on', '-L',
		  :type => :integer)
  app.defProperty('parallel', 'number of parallel client threads to run', '-P',
		  :type => :integer)
  app.defProperty('ttl', 'time-to-live, for multicast (default 1)', '-T',
		  :type => :integer,
		  :default => 1)
  app.defProperty('linux_congestion', 'set TCP congestion control algorithm (Linux only)', '-Z',
		  :type => :boolean)

  app.defMeasurement("application"){ |m|
    m.defMetric('pid', :integer)
    m.defMetric('version', :string)
    m.defMetric('cmdline', :string)
    m.defMetric('starttime_s', :integer)
    m.defMetric('starttime_us', :integer)

  app.defMeasurement("settings"){ |m|
    m.defMetric('pid', :integer)
    m.defMetric('server_mode', :integer)
    m.defMetric('bind_address', :string)
    m.defMetric('multicast', :integer)
    m.defMetric('multicast_ttl', :integer)
    m.defMetric('transport_protocol', :integer)
    m.defMetric('window_size', :integer)
    m.defMetric('buffer_size', :integer)

  app.defMeasurement("connection"){ |m|
    m.defMetric('pid', :integer)
    m.defMetric('connection_id', :integer)
    m.defMetric('local_address', :string)
    m.defMetric('local_port', :integer)
    m.defMetric('remote_address', :string)
    m.defMetric('remote_port', :integer)

  app.defMeasurement("transfer"){ |m|
    m.defMetric('pid', :integer)
    m.defMetric('connection_id', :integer)
    m.defMetric('begin_interval', :double)
    m.defMetric('end_interval', :double)
    m.defMetric('size', :uint64)

  app.defMeasurement("losses"){ |m|
    m.defMetric('pid', :integer)
    m.defMetric('connection_id', :integer)
    m.defMetric('begin_interval', :double)
    m.defMetric('end_interval', :double)
    m.defMetric('total_datagrams', :integer)
    m.defMetric('lost_datagrams', :integer)

  app.defMeasurement("jitter"){ |m|
    m.defMetric('pid', :integer)
    m.defMetric('connection_id', :integer)
    m.defMetric('begin_interval', :double)
    m.defMetric('end_interval', :double)
    m.defMetric('jitter', :double)

  app.defMeasurement("packets"){ |m|
    m.defMetric('pid', :integer)
    m.defMetric('connection_id', :integer)
    m.defMetric('packet_id', :integer)
    m.defMetric('packet_size', :integer)
    m.defMetric('packet_time_s', :integer)
    m.defMetric('packet_time_us', :integer)
    m.defMetric('packet_sent_time_s', :integer)
    m.defMetric('packet_sent_time_us', :integer)


defGroup('Servers', property.theReceiver) do |node|
  node.addApplication("iperf") do |app|
        #app.setProperty('interval', property.setinterval)
        app.measure('transfer', :samples => 1)

defGroup('Sender',property.theSender) do |node|
    node.addApplication("iperf") do |app|
        app.setProperty('parallel', 5)
        app.measure('transfer', :samples => 1)
defGroup('Monitor1', property.theSwitch) do |node|
  node.addApplication("trace") do |app|
    app.setProperty("interface", property.left)
    app.setProperty("filter", 'tcp')
    app.measure("tcp", :samples => 1)
defGroup('Monitor', property.theSwitch) do |node|
  node.addApplication("trace") do |app|
    app.setProperty("interface", property.right)
    app.setProperty("filter", 'tcp')
    app.measure("tcp", :samples => 1)

onEvent(:ALL_UP_AND_INSTALLED) do |event|
    info "starting"
     after 2 do
    after 100 do
      group ('Sender').stopApplications
    group ('Monitor1').stopApplications
     info "All applications stopped." 
##define the graphs that we want to display##
#defGraph 'Cumulated number of Bytes' do |g|
#'network').select(:oml_ts_server, :tx_bytes, :oml_sender_id)
#  g.caption "Total Bytes"
#  g.type 'line_chart3'
#  g.mapping :x_axis => :oml_ts_server, :y_axis => :tx_bytes, :group_by => :oml_sender_id
#  g.xaxis :legend => 'time', :ticks => {:format => 's'}
#  g.yaxis :legend => '', :ticks => {:format => 'Byte'}

defGraph 'TCP Throughput Bytes-per-Second' do |g|'tcp').select(:oml_ts_server, :tcp_packet_size, :oml_sender_id)
  g.caption "TCP throughput"
  g.type 'line_chart3'
  g.mapping :x_axis => :oml_ts_server, :y_axis => :tcp_packet_size, :group_by => :oml_sender_id
  g.xaxis :legend => 'time', :ticks => {:format => 's'}
  g.yaxis :legend => '', :ticks => {:format => 'Bytes/s'}

B. About the OpenFlow controller load-balancer.rb

  • Trema web site:
  • Treme ruby API document:
  • Functions used in our tutorial:
    • start(): is the function that will be called when the OpenFlow Controller is started. Here in our case, we read the file /tmp/portmap and figures out which OpenFlow port points to which path
    • switch_ready(): is the function that will be called each time a switch connects to the OpenFlow Controller. Here in our case, we allow all non-TCP flows to pass (including ARP and ICMP packets) and ask new inbound TCP flow to go to the controller. We also starts a "timer" function that calls "query_stats()" once every 2 seconds.
    • query_stats(): is the function that sends out a flow_stats_request to get the current statistics about each flow.
    • packet_in(): is the function that will be called each time a packet arrives at the controller. Here in our case, we call "decide_path()" to get path decisions, then send flow entry back to the OpenFlow Switch to instruct the switch which path to take for this new TCP flow.
    • stats_reply(): is the function that will be called when the OpenFlow Controller receives a flow_stats_reply message from the OpenFlow Switch. Here in our case, we update the flow statistics so that "decide_path()" can make the right decision.
    • send_flow_mod_add(): is the function that you should use to add a flow entry into an OpenFlow Switch.
    • decide_path(): is the function that makes path decisions. It returns the path choices based on flow statistics.
  • The Whole Process:
    • When the OpenFlow switch is ready, our controller starts a function that asks for flow stats once every 2 seconds.
    • The OpenFlow switch will reply with statistics information about all flows in its flow table.
    • This flow statistics message will be fetched by the "stats_reply" function in the OpenFlow controller implemented by the user on node "Switch".
    • As a result, our controller updates its knowledge about both left and right path once every 2 seconds.
    • Upon the arrival of a new TCP flow, the OpenFlow controller decides which path to send the new flow to, based on the updated flow statistics.

The FlowStatsReply message is in the following format:
  :length => 96,
  :table_id => 0,
  :match =>
  :duration_sec => 10,
  :duration_nsec => 106000000,
  :priority => 0,
  :idle_timeout => 0,
  :hard_timeout => 0,
  :cookie => 0xabcd,
  :packet_count => 1,
  :byte_count => 1,
  :actions => [ ]

C. About The Rspec file OpenFlowLBExo.rspec

  • The Rspec file describes a topology we showed earlier--each node is assigned with certain number of interfaces with pre-defined IP addresses
  • Some of the nodes are loaded with softwares and post-scripts. We will take node "Switch" as an example since it is the most complicated one.
    • The following section in the Rspec file for node "Switch":
        <install url="" install_path="/tmp"/>
      means it is going to download that tar ball from the specified URL and extract to directory "/tmp"
    • The following section in the Rspec file for node "Switch":
        <execute shell="bash" command=" sudo cp /tmp/of-topo-setup/gimibot/ / ; sh / $self.Name() $sliceName; sh /tmp/of-topo-setup/lb-setup $switch.IP("outside-switch") $switch.IP("switch-left") $switch.IP("switch-right")"/>
      names the post-boot script that ExoGENI is going to run for you after the nodes are booted.
  • More information about "/tmp/": It is a "hook" to the LabWiki interface. Experimenter run this so that LabWiki knows the name of the slice and the hostname of the particular node that OML/OMF toolkits are running on.
  • Take a look at the script for setting up the switch node, it is located at the node at "/tmp/of-topo-setup/lb-setup".

D. Tips: Debugging an OpenFlow Controller

You will find it helpful to know what is going on inside your OpenFlow controller and its associated switch when implementing these exercises.
This section contains a few tips that may help you out if you are using the Open vSwitch implementation provided with this tutorial. If you are using a hardware OpenFlow switch, your instructor can help you find equivalent commands.

  • ovs-vsctl
    Open vSwitch switches are primarily configured using the ovs-vsctl command. For exploring, you may find the ovs-vsctl show command useful, as it dumps the status of all virtual switches on the local Open vSwitch instance. Once you have some information on the local switch configurations, ovs-vsctl provides a broad range of capabilities that you will likely find useful for expanding your network setup to more complex configurations for testing and verification. In particular, the subcommands add-br, add-port, and set-controller may be of interest.
  • ovs-ofctl
    The switch host configured by the given rspec listens for incoming OpenFlow connections on localhost port 6634. You can use this to query the switch state using the ovs-ofctl command. In particular, you may find the dump-tables and dump-flows subcommands useful. For example, sudo ovs-ofctl dump-flows tcp: will output lines that look like this:
    cookie=0x4, duration=6112.717s, table=0, n packets=1, n bytes=74, idle age=78,priority=5,tcp,
    nw src= actions=CONTROLLER:65535
    This indicates that any TCP segment with source IP in the subnet should be sent to the OpenFlow controller for processing, that it has been 78 seconds since such a segment was last seen, that one such segment has been seen so far, and the total number of bytes in packets matching this rule is 74. The other fields are perhaps interesting, but you will probably not need them for debugging. (Unless, of course, you choose to use multiple tables — an exercise in OpenFlow 1.1 functionality left to the reader.)
  • Unix utilities
    You will want to use a variety of Unix utilities, in addition to the tools listed in ExerciseLayout, to test your controllers. The standard ping and /usr/sbin/arping tools are useful for debugging connectivity (but make sure your controller passes ICMP ECHO REQUEST and REPLY packets and ARP traffic, respectively!), and the command netstat -an will show all active network connections on a Unix host; the TCP connections of interest in this exercise will be at the top of the listing. The format of netstat output is out of the scope of this tutorial, but information is available online and in the manual pages.
  • Linux netem
    Use the tc command to enable and configure delay and lossrate constraints on the outgoing interfaces for traffic traveling from the OpenFlow switch to the Aggregator node. To configure a path with a 20 ms delay and 10% lossrate on eth2, you would issue the command:
    sudo tc qdisc add dev eth2 root handle 1:0 netem delay 20ms loss 2%
    Use the "tc qdisc change" command to reconfigure existing links,instead of "tc qdisc add".