#!/usr/bin/env python3
#===============================================================================
# Copyright 2012 NetApp, Inc. All Rights Reserved,
# contribution by Jorge Mora <mora@netapp.com>
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#===============================================================================
import os
import time
import posix
import ctypes
import traceback
import nfstest_config as c
from formatstr import crc32
from packet.nfs.nfs3_const import *
from packet.nfs.nfs4_const import *
from nfstest.test_util import TestUtil

# Module constants
__author__    = "Jorge Mora (%s)" % c.NFSTEST_AUTHOR_EMAIL
__copyright__ = "Copyright (C) 2012 NetApp, Inc."
__license__   = "GPL v2"
__version__   = "1.5"

USAGE = """%prog --server <server> [options]

Direct I/O tests
================
Functional direct I/O tests verify that every READ/WRITE is sent to the server
instead of the client caching the requests. Client bypasses read ahead by
sending the READ with only the requested bytes. Verify the client correctly
handles eof marker when reading the whole file. Verify client ignores delegation
while writing a file.

Direct I/O on pNFS tests verify the client sends the READ/WRITE to the correct
DS or the MDS if using a PAGESIZE aligned buffer or not, respectively.

Direct I/O data correctness tests verify that a file written with buffered I/O
is read correctly with direct I/O. Verify that a file written with direct I/O
is read correctly with buffered I/O.

Vectored I/O tests verify coalescence of multiple vectors into one READ/WRITE
packet when all vectors are PAGESIZE aligned. Vectors with different alignments
are sent on separate packets.

Valid for NFSv4.0 and NFSv4.1 including pNFS.

Examples:
    The only required option is --server
    $ %prog --server 192.168.0.11

Notes:
    The user id in the local host must have access to run commands as root
    using the 'sudo' command without the need for a password."""

# Test script ID
SCRIPT_ID = "DIO"

TESTNAMES = [
    'eof',
    'correctness',
    'fstat',
    'read',
    'read_ahead',
    'basic',
    'rsize',
    'wsize',
    'aligned',
    'nonaligned',
    'diffalign',
    'stripesize',
    'vectored_io',
]

# Constants
MDS = 0
DS  = 1
SERVER = 2
mds_map = {
       MDS: 'MDS',
        DS: 'DS',
    SERVER: 'server',
}

class iovec(ctypes.Structure):
    """
       struct iovec {
           void  *iov_base;    /* Starting address */
           size_t iov_len;     /* Number of bytes to transfer */
       };
    """
    _fields_ = [
        ("iov_base", ctypes.c_void_p),
        ("iov_len",  ctypes.c_ulong),
    ]

class DioTest(TestUtil):
    """DioTest object

       DioTest() -> New test object

       Usage:
           x = DioTest()

           # Verify the client correctly handles eof marker when reading
           # the end of the file
           x.verify_eof()

           # Verify client sends a READ request after writing when the file
           # is open for both read and write
           x.verify_read()

           # Verify basic direct I/O functionality
           x.verify_basic_dio(write=True)

           # Vectored I/O test
           tinfo = [
               {'size':4096, 'aligned':True,  'server':MDS},
               {'size':4096, 'aligned':True},
               {'size':4096, 'aligned':True},
           ]
           x.vectored_io(tinfo)

           x.exit()
    """
    def __init__(self, **kwargs):
        """Constructor

           Initialize object's private data.
        """
        TestUtil.__init__(self, **kwargs)
        self.opts.version = "%prog " + __version__

        # Set default script options
        self.opts.set_defaults(filesize=262144)
        self.opts.set_defaults(mtopts="hard,intr")

        # Options specific for this test script
        hmsg = "List of I/O types to test [default: '%default']"
        self.test_opgroup.add_option("--iotype", default='read,write', help=hmsg)
        hmsg = "List of buffered I/O types to test [default: '%default']"
        self.test_opgroup.add_option("--biotype", default='none,read,write', help=hmsg)
        hmsg = "Use delegation on tests [default: both without and with delegation]"
        self.test_opgroup.add_option("--withdeleg", default='false,true', help=hmsg)
        self.scan_options()

        self.r_bsize = 4*self.rsize
        self.r_mtbsize = self.rsize
        self.w_bsize = 4*self.wsize
        self.w_mtbsize = self.wsize

        # Disable createtraces option
        self.createtraces = False

        # Flag which defines if client is using new style direct I/O where
        # the use of non-aligned buffers result in sending the I/O to the
        # DS instead of the MDS
        self.newstyle = False

        # Flag is True when pNFS is available
        self.ispnfs = False

        # Flags are True if delegations are available
        self.read_deleg  = False
        self.write_deleg = False

        if self.nfs_version < 3:
            self.config("Option nfsversion must be 3 or above")

        # Process --iotype option
        self.io_list = self.get_list(self.iotype, {'read':False, 'write':True})
        if self.io_list is None:
            self.opts.error("invalid type given in --iotype [%s]" % self.iotype)

        # Process --biotype option
        self.bio_list = self.get_list(self.biotype, {'none':None, 'read':False, 'write':True})
        if self.bio_list is None:
            self.opts.error("invalid type given in --biotype [%s]" % self.biotype)

        # Process --withdeleg option
        self.deleg_list = self.get_list(self.withdeleg, {'false':False, 'true':True})
        if self.deleg_list is None:
            self.opts.error("invalid type given in --withdeleg [%s]" % self.withdeleg)

        self.testidx = 1
        self.fbuffers = []

        # Prototypes for libc functions
        self.libc.malloc.argtypes = [ctypes.c_long]
        self.libc.malloc.restype = ctypes.c_void_p
        self.libc.posix_memalign.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_long, ctypes.c_long]
        self.libc.posix_memalign.restype = ctypes.c_int
        self.libc.read.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_long]
        self.libc.read.restype = ctypes.c_int
        self.libc.write.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_long]
        self.libc.write.restype = ctypes.c_int
        self.libc.lseek.argtypes = [ctypes.c_int, ctypes.c_long, ctypes.c_int]
        self.libc.lseek.restype = ctypes.c_long
        self.libc.memcpy.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long]
        self.libc.memcpy.restype = ctypes.c_void_p
        self.libc.readv.argtypes = [ctypes.c_int, ctypes.POINTER(iovec), ctypes.c_int]
        self.libc.readv.restype = ctypes.c_ulong
        self.libc.writev.argtypes = [ctypes.c_int, ctypes.POINTER(iovec), ctypes.c_int]
        self.libc.writev.restype = ctypes.c_ulong

        self.bsize = self.rsize if self.rsize > self.wsize else self.wsize

    def _check_delegations(self):
        """Check if delegations are granted"""
        self.umount()
        self.trace_start()
        self.mount()
        try:
            fd = None
            ofd = None
            oofile = self.abspath(self.files[0])
            self.dprint('DBG4', "Open file %s so open owner sticks around" % oofile)
            ofd = open(oofile, 'r')
            self.create_file()
            rfile = self.abspath(self.files[1])
            self.dprint('DBG2', "Open file %s for reading" % rfile)
            fd = open(rfile, 'r')
            fd.read(self.rsize)
        finally:
            if ofd:
                ofd.close()
            if fd:
                fd.close()
            self.umount()
            self.trace_stop()

        self.trace_open()
        self.set_pktlist()
        (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=self.files[1])
        self.read_deleg = False if deleg_stateid is None else True
        self.pktt.rewind()
        (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=self.files[2])
        self.write_deleg = False if deleg_stateid is None else True
        if not self.read_deleg:
            self.dprint('INFO', "READ  delegations are not available -- skipping tests expecting read  delegations")
        if not self.write_deleg:
            self.dprint('INFO', "WRITE delegations are not available -- skipping tests expecting write delegations")
        self.pktt.close()

    def setup(self, **kwargs):
        """Setup test environment"""
        if self.rsize != self.wsize:
            raise Exception("CONFIG error: options rsize and wsize must have the same value")
        elif self.rsize % self.PAGESIZE > 0:
            raise Exception("CONFIG error: option rsize must be a multiple of %d (PAGESIZE)" % self.PAGESIZE)

        self.umount()
        self.trace_start()
        self.mount()
        statfs = os.statvfs(self.mtpoint)

        if len(self.basename) == 0 and statfs.f_bsize < 3*self.rsize:
            raise Exception("CONFIG error: mount options rsize and wsize must be greater than or equal to 3 times the value of option rsize")

        super(DioTest, self).setup(**kwargs)

        self.umount()
        self.trace_stop()

        if self.nfs_version > 3:
            self.trace_open()
            self.set_pktlist()
            (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=self.files[0])
            (layoutget, layoutget_res) = self.find_layoutget(filehandle)
            (pktcall, pktreply, dslist) = self.find_getdeviceinfo()
            if layoutget and len(self.dslist):
                self.ispnfs = True

            if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
                if len(pktreply.NFSop.device_addr.versions) > 1:
                    raise Exception("Support for only one device info in NFS flex files layout type")
                self.r_mtbsize = pktreply.NFSop.device_addr.versions[0].rsize
                self.w_mtbsize = pktreply.NFSop.device_addr.versions[0].wsize
                self.r_bsize = 4*self.r_mtbsize
                self.w_bsize = 4*self.w_mtbsize

            if self.ispnfs:
                if self.layout is None:
                    raise Exception("Could not find layout")
                self.stripe_size = self.layout['stripe_size']
                if self.stripe_size > 0 and self.stripe_size < 3*self.rsize:
                    raise Exception("CONFIG error: option rsize must be less or equal to %d (stripe size / 3)" % int(self.stripe_size/3))
            self.pktt.close()

            # Check if delegations are granted
            self._check_delegations()

        # Non-aligned contiguous vectors on aligned offset
        self.file_handles = {False:None, True:None}
        tinfo = [
            {'size':self.rsize, 'aligned':False, 'server':MDS},
            {'size':self.rsize, 'aligned':False, 'server':MDS},
            {'size':self.rsize, 'aligned':False, 'server':MDS},
        ]
        self.mount()
        out = self.vectored_io(tinfo, write=False, contiguous=True, check=True)
        self.umount()
        self.newstyle = len(out) == 1 and out[0] == 3*self.rsize
        if self.newstyle:
            self.dprint('INFO', "Direct I/O data is sent to the DS regardless of buffer alignment")

    def mem_alloc(self, size, aligned=True, fill=False, offset=0):
        """Allocate buffer.

           size:
               Number of bytes to allocate
           aligned:
               Aligned buffer on a PAGESIZE boundary if true [default: True]
           fill:
               Fill buffer with a predetermined pattern
           offset:
               Offset used in creating fill data

           Return allocated buffer.

           See also free_buffers()
        """
        align_str = ""
        buffer = None
        if aligned:
            # Allocate aligned buffer
            buffer = ctypes.c_void_p()
            self.libc.posix_memalign(ctypes.byref(buffer), self.PAGESIZE, size)
        else:
            # Make sure the buffer is not aligned
            align_str = "non-"
            buffers = []
            for i in range(30):
                buffers.append(ctypes.c_void_p())
                buffers[-1].value = self.libc.malloc(size)
                if buffers[-1].value & (self.PAGESIZE-1) != 0:
                    # Found non-aligned buffer
                    buffer = buffers.pop()
                    break

            for buf in buffers:
                # Free all unused buffers
                self.libc.free(buf)

            if buffer is None:
                raise Exception("Could not allocate %saligned buffer" % align_str)

        self.dprint('DBG3', "Allocated %saligned buffer of %d bytes @ 0x%x" % (align_str, size, buffer.value))
        # Save allocated buffer so it can be freed by free_buffers()
        self.fbuffers.append(buffer)
        if fill:
            # Fill buffer
            data = self.data_pattern(offset, size)
            pdata = ctypes.create_string_buffer(data)
            self.libc.memcpy(buffer, pdata, size);
        return buffer

    def alloc_buffers(self, **kwargs):
        """Allocate buffers used by do_read and do_write."""
        size = kwargs.pop('size', 2*self.bsize)
        # Make sure the buffer is not aligned
        self.nonaligned_buffer = self.mem_alloc(size, aligned=False)
        # Allocate aligned buffer
        self.aligned_buffer = self.mem_alloc(size, aligned=True)

    def free_buffers(self):
        """Free all allocated buffers created by mem_alloc()."""
        try:
            if len(self.fbuffers):
                while len(self.fbuffers):
                    if self.fbuffers[0] != None:
                        self.dprint('DBG4', "Freeing allocated buffer 0x%x" % self.fbuffers[0].value)
                        self.libc.free(self.fbuffers[0])
                    # Remove buffer from list so this function can be called
                    # multiple times
                    self.fbuffers.pop(0)
        except Exception:
            pass

    def do_read(self, fd, offset, size, aligned=True, delay=None):
        """Wrapper for system call read().

           fd:
               File descriptor returned from open() system call
           offset:
               Start reading at this file position
           size:
               Number of bytes to read
           aligned:
               Use aligned buffer if true [default: True]
           delay:
               Delay read in seconds [default: --iodelay]

           Return data read.
        """
        buffer = self.aligned_buffer if aligned else self.nonaligned_buffer
        self.dprint('DBG3', "Read file %d@%d" % (size, offset))
        self.libc.lseek(fd, offset, 0)
        count = self.libc.read(fd, buffer, size)
        self.dprint('DBG4', "Read returned %d bytes" % count)
        data = ctypes.string_at(buffer, count)
        # Slow down traffic for tcpdump to capture all packets
        self.delay_io(delay)
        return data

    def do_write(self, fd, offset, size, aligned=True, delay=None):
        """Wrapper for system call write().

           fd:
               File descriptor returned from open() system call
           offset:
               Start writing at this file position
           size:
               Number of bytes to write
           aligned:
               Use aligned buffer if true [default: True]
           delay:
               Delay write in seconds [default: --iodelay]

           Return number of bytes written.
        """
        buffer = self.aligned_buffer if aligned else self.nonaligned_buffer
        data = self.data_pattern(offset, size)
        pdata = ctypes.create_string_buffer(data)
        self.libc.memcpy(buffer, pdata, size);
        self.dprint('DBG3', "Write file %d@%d" % (size, offset))
        self.libc.lseek(fd, offset, 0)
        count = self.libc.write(fd, buffer, size)
        self.dprint('DBG4', "Write returned %d bytes" % count)
        # Slow down traffic for tcpdump to capture all packets
        self.delay_io(delay)
        return count

    def _get_info(self, direct=True):
        """Return tuple (O_DIRECT, ' (O_DIRECT)') if direct option is true,
           (0, '') otherwise.
        """
        if direct:
            info = " (O_DIRECT)"
            open_args = posix.O_DIRECT
        else:
            info = ""
            open_args = 0
        return (open_args, info)

    def read_file(self, absfile, rsize=None, direct=True, aligned=True, delay=None):
        """Read file and compare read data with known data pattern.

           absfile:
               File to read
           rsize:
               Number of bytes to write per call [default: --rsize]
           direct:
               Open file using O_DIRECT if true [default: True]
           aligned:
               Use aligned buffer if true [default: True]
           delay:
               Delay each read in seconds [default: --iodelay]

           Return total number of bytes read.
        """
        (open_args, info) = self._get_info(direct)
        self.dprint('DBG2', "Open file %s for reading%s" % (absfile, info))
        fd = posix.open(absfile, posix.O_RDONLY|open_args)
        try:
            offset = 0
            rsize = self.rsize if rsize is None else rsize
            while True:
                data = self.do_read(fd, offset, rsize, aligned=aligned, delay=delay)
                count = len(data)
                cdata = self.data_pattern(offset, count)
                if count == 0 or data != cdata:
                    break
                offset += count
        finally:
            posix.close(fd)
        return offset

    def write_file(self, absfile, wsize=None, direct=True, aligned=True, delay=None):
        """Write file with known data pattern.

           absfile:
               File to write
           wsize:
               Number of bytes to write per call [default: --wsize]
           direct:
               Open file using O_DIRECT if true [default: True]
           aligned:
               Use aligned buffer if true [default: True]
           delay:
               Delay each write in seconds [default: --iodelay]

           Return total number of bytes written.
        """
        (open_args, info) = self._get_info(direct)
        self.dprint('DBG2', "Open file %s for writing%s" % (absfile, info))
        fd = posix.open(absfile, posix.O_WRONLY|posix.O_CREAT|open_args, 0o644)
        offset = 0
        wsize = self.wsize if wsize is None else wsize
        while offset < self.filesize:
            count = self.filesize - offset
            if count > wsize:
                count = wsize
            count = self.do_write(fd, offset, wsize, delay=delay)
            offset += count
        posix.close(fd)
        return offset

    def verify_eof(self, aligned=True):
        """Verify eof marker is handled correctly when reading the end
           of the file.

           aligned:
               Use aligned buffer on read() if true [default: True]
        """
        try:
            fd = None
            align_str = "" if aligned else "non-"
            self.test_group("Verify eof marker is handled correctly when reading eof using %saligned buffer" % align_str)

            self.umount()
            self.mount()
            buffer = self.mem_alloc(2*self.rsize, aligned=aligned)
            absfile = self.abspath(self.files[0])
            self.dprint('DBG2', "Open file %s for reading" % absfile)
            fd = posix.open(absfile, posix.O_RDONLY|posix.O_DIRECT)
            offset = self.filesize - self.rsize
            self.dprint('DBG3', "Read file %d@%d" % (self.rsize, offset))
            self.libc.lseek(fd, offset, 0)
            count = self.libc.read(fd, buffer, self.rsize)
            offset += count
            self.test(count == self.rsize, "READ right before end of file should return correct read count (%d)" % self.rsize, failmsg=", returned read count = %d" % count)
            self.dprint('DBG3', "Read file %d@%d" % (self.rsize, offset))
            count = self.libc.read(fd, buffer, self.rsize)
            self.test(count == 0, "READ at end of file should return read count = 0", failmsg=", returned read count = %d" % count)
        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            if fd:
                posix.close(fd)
            self.free_buffers()
            self.umount()

    def eof_test(self):
        """Verify eof marker is handled correctly when reading the end
           of the file.
        """
        self.verify_eof()
        #self.verify_eof(aligned=False)

    def verify_read(self, read_ahead=False, aligned=True):
        """Verify READ is sent with only the requested bytes bypassing
           read ahead when read_ahead is True.
           Verify READ is sent after writing when the file is open for
           both read and write when read_ahead is False.
        """
        try:
            fd = None
            if read_ahead:
                self.test_group("Verify READ is sent with only the requested bytes bypassing read ahead")
            else:
                self.test_group("Verify READ is sent after writing when the file is open for both read and write")

            self.alloc_buffers()
            self.umount()
            self.trace_start()
            self.mount()

            if read_ahead:
                filename = self.files[0]
                absfile = self.abspath(filename)
                self.dprint('DBG2', "Open file %s for reading" % absfile)
                fd = posix.open(absfile, posix.O_RDONLY|posix.O_DIRECT)
            else:
                self.get_filename()
                filename = self.filename
                self.dprint('DBG2', "Open file %s for writing" % self.absfile)
                fd = posix.open(self.absfile, posix.O_RDWR|posix.O_CREAT|posix.O_DIRECT)

                offset = 0
                for i in range(3):
                    count = self.do_write(fd, offset, self.wsize, aligned=aligned)
                    offset += count

            data = self.do_read(fd, 0, self.rsize, aligned=aligned)
            self.test(data == self.data_pattern(0, self.rsize), "READ data should be correct")
        except Exception:
            self.test(False, traceback.format_exc())
            return
        finally:
            if fd:
                posix.close(fd)
            self.umount()
            self.free_buffers()
            self.trace_stop()

        try:
            self.trace_open()
            self.set_pktlist()
            (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=filename)
            stateid = deleg_stateid if deleg_stateid else open_stateid
            self.find_layoutget(filehandle)
            (pktcall, pktreply, dslist) = self.find_getdeviceinfo()

            nfs_version = self.nfs_version
            if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
                item = pktreply.NFSop.device_addr.versions[0]
                nfs_version = float("%d.%d" % (item.version, item.minorversion))
                filehandle = self.layout['filehandles'][0]

            if nfs_version < 4:
                match_str = "NFS.argop == %d and NFS.fh == b'%s'" % (NFSPROC3_READ, self.pktt.escape(filehandle))
            else:
                match_str = "NFS.argop == %d and NFS.stateid.other == b'%s'" % (OP_READ, self.pktt.escape(stateid))

            pkt = self.pktt.match(match_str)
            self.test(pkt, "READ should be sent to the server")
            if pkt:
                roffset = pkt.NFSop.offset
                rcount = pkt.NFSop.count
                self.test(roffset == 0 and rcount == self.rsize, "READ should be sent with correct offset (%d) and count (%d)" % (0, self.rsize))
            if read_ahead:
                pkt = self.pktt.match(match_str)
                self.test(not pkt, "Extra READs should not be sent to the server")
        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            self.pktt.close()

    def read_test(self):
        """Verify READ is sent after writing when the file is open for
           both read and write.
        """
        self.verify_read()

    def read_ahead_test(self):
        """Verify READ is sent with only the requested bytes bypassing
           read ahead.
        """
        self.verify_read(read_ahead=True)

    def correctness_test(self):
        """Verify data correctness when reading/writing using direct I/O.
           File created with buffered I/O is read correctly with direct I/O.
           File created with direct I/O is read correctly with buffered I/O.
        """
        try:
            self.test_group("Verify data correctness when reading/writing using direct I/O")
            iosize = max(self.rsize, self.wsize, int(self.filesize/8))
            self.alloc_buffers(size=iosize)
            self.umount()
            self.mount()

            # Read file using direct I/O on a file created with buffered I/O
            # and verify data read with known data on file
            count = self.read_file(self.abspath(self.files[0]), rsize=iosize, delay=0)
            self.test(count == self.filesize, "File created with buffered I/O is read correctly with direct I/O")

            # Create file using direct I/O
            self.get_filename()
            self.write_file(self.absfile, wsize=iosize, delay=0)

            self.umount()
            self.mount()

            # Verify written data by reading the file using buffered I/O
            count = self.read_file(self.absfile, rsize=iosize, direct=False, delay=0)
            self.test(count == self.filesize, "File created with direct I/O is read correctly with buffered I/O")
            self.umount()
        except Exception:
            self.test(False, traceback.format_exc())
            self.free_buffers()

    def fstat_test(self):
        """Verify fstat() gets correct file size after writing."""
        try:
            fd = None
            self.test_group("Verify fstat() gets correct file size after writing")
            wsize = max(self.wsize, int(self.filesize/8))
            self.alloc_buffers(size=wsize)
            self.umount()
            self.mount()

            self.get_filename()
            self.dprint('DBG2', "Open file %s for writing" % self.absfile)
            fd = posix.open(self.absfile, posix.O_WRONLY|posix.O_CREAT|posix.O_DIRECT, 0o644)

            idx = 0
            ngood = 0
            offset = 0
            while offset < self.filesize:
                count = self.do_write(fd, offset, wsize, delay=0)
                offset += count
                idx += 1
                fs = posix.fstat(fd)
                if fs.st_size == offset:
                    ngood += 1
            self.test(ngood == idx, "The fstat() should get correct file size after every write")

            # Write at a large offset of 10G
            offset = 10 * 1024 * 1024 * 1024
            count = self.do_write(fd, offset, wsize, delay=0)
            fs = posix.fstat(fd)
            size = offset + count
            self.test(fs.st_size==size, "The fstat() should get correct file size after writing at offset = 10G", failmsg="\nexpecting %d, got %d" % (size, fs.st_size))

        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            if fd:
                posix.close(fd)
            self.umount()
            self.free_buffers()

    def verify_basic_dio(self, write, bsize=None, mtbsize=None, nio=None, align_hash=[], buffered_write=None, deleg=False):
        """Verify basic direct I/O functionality.

           write:
               Test writing if true, otherwise test reading
           bsize:
               Block size used for I/O [default: --rsize/2]
           mtbsize:
               Block size to used on mount [default: not specified on mount]
           nio:
               Number of READ/WRITE packets each read()/write() request will
               generate on the wire [default: calculated using bsize and mtbsize]
           align_hash:
               List of expected 'alignments' on pNFS, if item is True the I/O
               is expected to go to the DS, otherwise to the MDS [default: []]
           buffered_write:
               Open another file for buffered I/O. If true use buffered write,
               if false use buffered read [default: None(no buffered I/O)]
           deleg:
               Expect to get a delegation if true [default: False]
        """
        try:
            fd = None
            bfd = None
            ofd = None
            io_str  = "WRITE" if write else "READ"
            io_mode = posix.O_WRONLY|posix.O_CREAT if write else posix.O_RDONLY
            bio_str  = "WRITE" if buffered_write else "READ"
            bio_mode = os.O_WRONLY|os.O_CREAT if buffered_write else os.O_RDONLY

            if not bsize:
                bsize = int(self.rsize/2)
            self.alloc_buffers(size=bsize)

            self.umount()
            self.trace_start()
            if mtbsize:
                self.mount(mtopts="hard,intr,rsize=%d,wsize=%d" % (mtbsize, mtbsize))
            else:
                self.mount()

            if nio is None:
                if mtbsize:
                    nio = int(bsize / mtbsize) + (1 if bsize > mtbsize and bsize % mtbsize else 0)
                else:
                    nio = 1
                b_size = mtbsize if nio > 1 else bsize
            else:
                b_size = int(bsize / nio)

            if write:
                while len(self.files) < 4:
                    self.get_filename()
                filename = self.files[3]
            else:
                filename = self.files[0]
            absfile = self.abspath(filename)

            if deleg:
                oofile = self.abspath(self.files[2])
                self.dprint('DBG4', "Open file %s so open owner sticks around" % oofile)
                ofd = open(oofile, 'r')

            self.dprint('DBG2', "Open file %s for %s" % (absfile, io_str))
            fd = posix.open(absfile, io_mode|posix.O_DIRECT)

            if buffered_write != None:
                # Open file for buffered I/O
                if buffered_write:
                    while len(self.files) < 5:
                        self.get_filename()
                    bfile = self.files[4]
                else:
                    bfile = self.files[1]
                babsfile = self.abspath(bfile)
                self.dprint('DBG2', "Open file %s for %s (buffered)" % (babsfile, bio_str))
                bfd = os.open(babsfile, bio_mode)

            off = 0
            boffset = 0
            test_hash = []
            N = len(align_hash) if len(align_hash) else 3
            for i in range(N):
                b_off = off
                if len(align_hash):
                    aligned = align_hash[i]
                else:
                    aligned = True
                for j in range(nio):
                    test_hash.append({'offset':b_off, 'size':b_size, 'aligned':aligned, 'grpidx':i})
                    b_off += b_size
                if write:
                    count = self.do_write(fd, off, bsize, aligned=aligned)
                else:
                    data = self.do_read(fd, off, bsize, aligned=aligned)
                if buffered_write != None:
                    # Buffered I/O
                    if buffered_write:
                        self.dprint('DBG3', "Write file %d@%d (buffered)" % (bsize, boffset))
                        count = os.write(bfd, self.data_pattern(boffset, bsize))
                        self.dprint('DBG4', "Write returned %d bytes" % count)
                    else:
                        self.dprint('DBG3', "Read file %d@%d (buffered)" % (bsize, boffset))
                        data = os.read(bfd, bsize)
                        self.dprint('DBG4', "Read returned %d bytes" % len(data))
                    boffset += bsize
                    # Slow down traffic for tcpdump to capture all packets
                    self.delay_io()
                off += 2*bsize
        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            if fd:
                posix.close(fd)
            if bfd:
                os.close(bfd)
            if deleg:
                ofd.close()
            self.umount()
            self.free_buffers()
            self.trace_stop()

        try:
            self.trace_open()
            self.set_pktlist()
            (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=filename)
            save_index = self.pktt.get_index()
            if deleg_stateid != None:
                self.dprint('DBG3', "Delegation is granted")
            stateid = deleg_stateid if deleg_stateid else open_stateid
            self.find_layoutget(filehandle)
            (pktcall, pktreply, dslist) = self.find_getdeviceinfo()
            self.pktt.rewind(save_index)

            nfs_version = self.nfs_version
            if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
                item = pktreply.NFSop.device_addr.versions[0]
                nfs_version = float("%d.%d" % (item.version, item.minorversion))
                filehandle = self.layout['filehandles'][0]

            if nfs_version < 4:
                io_op  = NFSPROC3_WRITE if write else NFSPROC3_READ
            else:
                io_op  = OP_WRITE if write else OP_READ

            match_str = "NFS.argop == %d" % io_op
            if nfs_version < 4:
                match_str += " and NFS.fh == b'%s'" % self.pktt.escape(filehandle)
            else:
                match_str += " and NFS.stateid.other == b'%s'" % self.pktt.escape(stateid)

            idx = 0
            xids = {}
            io_h = {}
            err_h = {}
            while self.pktt.match(match_str):
                save_index = self.pktt.get_index()
                pkt     = self.pktt.pkt
                roffset = pkt.NFSop.offset
                ipaddr  = pkt.ip.dst
                pobj    = pkt.udp if pkt.tcp is None else pkt.tcp
                port    = pobj.dst_port
                xid     = pkt.rpc.xid
                pktr    = self.pktt.match("RPC.type == 1 and RPC.xid == %d" % xid)
                self.pktt.rewind(save_index)
                if xids.get(xid, None) is None:
                    # Save xid to keep track of re-transmitted packets
                    xids[xid] = 1
                else:
                    # Skip re-transmitted packets
                    continue
                if pktr == "nfs" and pktr.nfs.status != NFS4_OK:
                    # Server returned error for this I/O operation
                    errstr = nfsstat4.get(pktr.nfs.status)
                    if err_h.get(errstr) is None:
                        err_h[errstr] = 1
                    else:
                        err_h[errstr] += 1

                # Get size of I/O data
                rsize = pkt.NFSop.count

                if io_h.get(roffset) == rsize:
                    # This (offset, size) has already been processed
                    continue
                else:
                    io_h[roffset] = rsize

                if len(test_hash) <= idx:
                    # Got unexpected READ/WRITE
                    self.test(0, "%s (%d@%d) should not be sent" % (io_str, rsize, roffset))
                else:
                    # Check READ/WRITE for expected offset and size
                    toffset  = test_hash[idx].get('offset')
                    tsize    = test_hash[idx].get('size')
                    taligned = test_hash[idx].get('aligned')
                    tgrpidx  = test_hash[idx].get('grpidx')
                    expr = (toffset == roffset and tsize == rsize)
                    if not expr and ((mtbsize and bsize > mtbsize) or nio > 1):
                        # The I/O size is greater than the mount block size
                        # check if this write belongs to the same I/O group
                        for item in test_hash:
                            if item.get('grpidx') == tgrpidx:
                                expr = (item.get('offset') == roffset and item.get('size') == rsize)
                                if expr:
                                    # I/O belongs to the expected I/O group
                                    toffset  = item.get('offset')
                                    tsize    = item.get('size')
                                    taligned = item.get('aligned')
                                    self.dprint('DBG4', "%s (%d@%d) is sent out of order but within the same I/O call" % (io_str, tsize, toffset))
                                    break
                    if len(align_hash) and (self.newstyle or taligned) and self.ispnfs:
                        dsidx = 0
                        ds_index = None
                        for ds in self.dslist:
                            for item in ds:
                                if ipaddr == item['ipaddr'] and port == item['port']:
                                    ds_index = dsidx
                                    break
                            dsidx += 1
                        if ds_index is None:
                            # XXX this could be a mirror -- mirrors are not supported yet
                            continue

                    msg  = "%s (%d@%d) should be sent with correct offset and count" % (io_str, tsize, toffset)
                    fmsg = ", got packet (%d@%d)" %(rsize, roffset)
                    self.test(expr, msg, failmsg=fmsg)

                    if len(align_hash):
                        if (self.newstyle or taligned) and self.ispnfs:
                            out = self.verify_stripe(toffset, tsize, ds_index)
                            self.test(out, "%s should be sent to the correct DS%s" % (io_str, "" if ds_index is None else "(%d)"%ds_index))
                        else:
                            rserver = MDS if self.ispnfs else SERVER
                            expr = ipaddr == self.server_ipaddr
                            if self.proto in ("tcp", "udp"):
                                expr = expr and port == self.port
                            self.test(expr, "%s should be sent to the %s" % (io_str, mds_map[rserver]))
                idx += 1
            for err in err_h:
                self.test(False, "%s fails with %s, number of failures found: %d" % (io_str, err, err_h[err]))

            for item in test_hash[idx:]:
                # Check for expected READ/WRITE packets which were not found
                toffset = item.get('offset')
                tsize   = item.get('size')
                self.test(0, "%s (%d@%d) should be sent" % (io_str, tsize, toffset))

            if buffered_write != None:
                self.pktt.rewind()
                (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=bfile)
                stateid = deleg_stateid if deleg_stateid else open_stateid
                self.find_layoutget(filehandle)
                (pktcall, pktreply, dslist) = self.find_getdeviceinfo()
                total_size = 0
                rmatch = True
                xids = {}

                nfs_version = self.nfs_version
                if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
                    item = pktreply.NFSop.device_addr.versions[0]
                    nfs_version = float("%d.%d" % (item.version, item.minorversion))
                    filehandle = self.layout['filehandles'][0]

                if nfs_version < 4:
                    bio_op = NFSPROC3_WRITE if buffered_write else NFSPROC3_READ
                else:
                    bio_op = OP_WRITE if buffered_write else OP_READ

                if nfs_version < 4:
                    match_str = "NFS.argop == %d and NFS.fh == b'%s'" % (bio_op, self.pktt.escape(filehandle))
                else:
                    match_str = "NFS.argop == %d and NFS.stateid.other == b'%s'" % (bio_op, self.pktt.escape(stateid))

                while self.pktt.match(match_str):
                    pkt = self.pktt.pkt
                    roffset = pkt.NFSop.offset
                    xid     = pkt.rpc.xid
                    if xids.get(xid, None) is None:
                        # Save xid to keep track of re-transmitted packets
                        xids[xid] = 1
                    else:
                        # Skip re-transmitted packets
                        continue
                    rsize = pkt.NFSop.count
                    if rsize != bsize:
                        rmatch = False
                    total_size += rsize
                expr = (buffered_write and total_size == boffset) or (total_size >= boffset)
                msg  = "%ss should be sent with correct size for buffered I/O" % bio_str
                fmsg = "; expecting %d, got %d" % (boffset, total_size)
                #XXX
                #self.test(expr, msg, failmsg=fmsg)
                self.test(not rmatch, "%ss should be cached for buffered I/O" % bio_str)
        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            self.pktt.close()

    def basic_dio(self, testname):
        """Verify basic direct I/O functionality for given testname."""
        try:
            no_striping = self.layout is not None and self.layout.get('stripe_size') == 0

            for write in self.io_list:
                for deleg in self.deleg_list:
                    if deleg and (write and not self.write_deleg or not write and not self.read_deleg):
                        # Skip delegation testing for I/O since delegation
                        # was not granted for this particular I/O
                        continue
                    for bwrite in self.bio_list:
                        io_str   = "WRITE" if write  else "READ"
                        bio_str  = "WRITE" if bwrite else "READ"
                        size_str = "wsize" if write  else "rsize"
                        wr_str   = "writing from" if write  else "reading into"
                        dstr = " with deleg" if deleg else ""
                        bstr = "" if bwrite is None else " and having %s buffered I/O to another file" % bio_str
                        if self.ispnfs:
                            ds_str   = "correct %s" % mds_map[DS]
                            if self.newstyle:
                                mds_str  = ds_str
                            else:
                                mds_str  = mds_map[MDS]
                            both_str = "both %s and correct %s" % (mds_map[MDS], mds_map[DS])
                        else:
                            ds_str   = mds_map[SERVER]
                            mds_str  = mds_map[SERVER]
                            both_str = mds_map[SERVER]

                        if testname == "basic":
                            self.test_group("Verify %s packet is sent for each %s%s%s" % (io_str, io_str.lower(), dstr, bstr))
                            self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg)
                        elif testname == "rsize" and not write:
                            self.test_group("Verify multiple %s packets are sent for each %s having request size > %s%s%s" % (io_str, io_str.lower(), size_str, dstr, bstr))
                            self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.r_bsize, mtbsize=self.r_mtbsize)
                        elif testname == "wsize" and write:
                            self.test_group("Verify multiple %s packets are sent for each %s having request size > %s%s%s" % (io_str, io_str.lower(), size_str, dstr, bstr))
                            self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.w_bsize, mtbsize=self.w_mtbsize)
                        elif testname == "aligned":
                            self.test_group("Verify %s is sent to %s when PAGESIZE aligned%s%s" % (io_str, ds_str, dstr, bstr))
                            self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.rsize, align_hash=[True, True, True, True])
                        elif testname == "nonaligned":
                            self.test_group("Verify %s is sent to %s when not PAGESIZE aligned%s%s" % (io_str, mds_str, dstr, bstr))
                            ahash = [True, True, True, True] if no_striping else [False, False, False, False]
                            self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.rsize, align_hash=ahash)
                        elif testname == "diffalign":
                            self.test_group("Verify %ss are sent to %s on same open using buffers with different alignments%s%s" % (io_str, both_str, dstr, bstr))
                            ahash = [True, True, True, True] if no_striping else [True, False, True, False]
                            self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.rsize, align_hash=ahash)
                        elif testname == "stripesize" and self.ispnfs and not no_striping:
                            self.test_group("Verify multiple %s packets are sent for each %s having request size > stripe size%s%s" % (io_str, io_str.lower(), dstr, bstr))
                            self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=2*self.stripe_size, mtbsize=4*self.stripe_size, nio=2, align_hash=[True, True])
        except Exception:
            self.test(False, traceback.format_exc())

    def basic_test(self):
        """Verify a packet is sent for each I/O request."""
        self.basic_dio('basic')

    def rsize_test(self):
        """Verify multiple READ packets are sent for each read request
           having request size > rsize.
        """
        self.basic_dio('rsize')

    def wsize_test(self):
        """Verify multiple WRITE packets are sent for each write request
           having request size > wsize
        """
        self.basic_dio('wsize')

    def aligned_test(self):
        """Verify packet is sent to correct DS server when using a memory
           which is PAGESIZE aligned.
        """
        self.basic_dio('aligned')

    def nonaligned_test(self):
        """Verify packet is sent to the MDS when using a memory which
           is not PAGESIZE aligned.
        """
        self.basic_dio('nonaligned')

    def diffalign_test(self):
        """Verify packets are sent to both the MDS and correct DS on same open
           using buffers with different alignments.
        """
        self.basic_dio('diffalign')

    def stripesize_test(self):
        """Verify multiple packets are sent for each request having the
           request size greater than stripe size.
        """
        self.basic_dio('stripesize')

    def vectored_io(self, tinfo, offset=0, contiguous=False, write=False, check=False):
        """Verify vectored I/O functionality.

           tinfo:
               List of vector definitions and expected results. Each vector
               definition is a dictionary having the following keys:
                   size:    size of vector
                   aligned: buffer alignment of vector
                   server:  expected server where the vector data is going to
                            if not specified, this vector data is part of the
                            previous packet
           offset:
               Read/write at given offset [default: 0]
           contiguous:
               Vectors are contiguous if true [default: False]
           write:
               Test writing if true, otherwise test reading [default: False]
           check:
               Run the test but don't PASS/FAIL it, just return a list of
               all I/O sizes sent to the server [default: False]
        """
        try:
            # Generate test hash
            fd = None
            off = offset
            head_vec = []
            test_hash = []
            total_size = 0

            if self.newstyle and contiguous:
                tlist = [item for item in tinfo if item["aligned"]]
                if len(tlist) == 0 or len(tlist) == len(tinfo):
                    # All vectors have the same alignment
                    tinfo[0]["server"] = DS
                    for item in tinfo[1:]:
                        item.pop("server", None)

            for item in tinfo:
                server = item.get('server')
                if server != None:
                    if self.newstyle and server == MDS:
                        server = DS
                    test_hash.append({'server':server, 'size':item['size'], 'offset':off})
                else:
                    test_hash[-1]['size'] += item['size']
                align_str = "" if item['aligned'] else "non-"
                if not contiguous or len(head_vec) == 0:
                    head_vec.append("%saligned(%d)" % (align_str, item['size']))
                else:
                    head_vec.append("(%d)" % item['size'])
                total_size += item['size']
                off += item['size']
            if not self.ispnfs:
                for item in test_hash:
                    item['server'] = 2

            con_str = "" if contiguous else "non-"
            off_str = "@%d " % offset if offset else ""
            io_str  = "WRITE" if write else "READ"
            io_mode = posix.O_WRONLY|posix.O_CREAT if write else posix.O_RDONLY

            if not check:
                self.test_group("Verify %s %s%scontiguous vector [%s]" % (io_str, off_str, con_str, ", ".join(head_vec)))
                sub_testname = "%s_%03d" % (self.testname, self.testidx)
                self.dprint("INFO", "Running %s" % sub_testname)
                self.testidx += 1

            idx = 0
            off = offset
            buffers = []
            nvecs = len(tinfo)
            vec_str = "IOvecs("
            for item in tinfo:
                if contiguous:
                    if idx == 0:
                        buffers.append(self.mem_alloc(total_size, aligned=item['aligned'], fill=write, offset=offset))
                    else:
                        buffer = ctypes.c_void_p()
                        buffer.value = buffers[idx-1].value + tinfo[idx-1]['size']
                        buffers.append(buffer)
                else:
                    buffers.append(self.mem_alloc(item['size'], aligned=item['aligned'], fill=write, offset=off))
                    off += item['size']
                vec_str += "iovec(buffers[%d], %d)," % (idx, item['size'])
                idx += 1
            vec_str += ")"

            # Create array of iovec structures
            IOvecs = iovec * nvecs
            vectors = eval(vec_str)

            self.trace_start()
            if write:
                while len(self.files) < 4:
                    self.get_filename()
                filename = self.files[3]
            else:
                filename = self.files[0]
            absfile = self.abspath(filename)
            self.dprint('DBG2', "Open file %s for %s" % (absfile, io_str))
            fd = posix.open(absfile, io_mode|posix.O_DIRECT)
            self.libc.lseek(fd, offset, 0)
            wr_str = "Writing" if write  else "Reading"
            fi_str = "to" if write else "from"
            self.dprint('DBG3', "%s %d vectors %s file %s " % (wr_str, nvecs, fi_str, absfile))
            if write:
                count = self.libc.writev(fd, vectors, nvecs)
            else:
                count = self.libc.readv(fd, vectors, nvecs)
            # Slow down traffic for tcpdump to capture all packets
            self.delay_io()
        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            if fd:
                self.dprint('DBG3', "Close file %s " % absfile)
                posix.close(fd)
            self.trace_stop()

        try:
            fd = None
            self.trace_open()
            self.set_pktlist()
            nfs_version = self.nfs_version
            (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=filename, claimfh=self.file_handles[write], anyclaim=True)
            if filehandle:
                self.file_handles[write] = filehandle
                save_index = self.pktt.get_index()
                self.stateid = deleg_stateid if deleg_stateid else open_stateid
                self.find_layoutget(filehandle)
                (pktcall, pktreply, dslist) = self.find_getdeviceinfo()

                if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
                    item = pktreply.NFSop.device_addr.versions[0]
                    nfs_version = float("%d.%d" % (item.version, item.minorversion))
                    filehandle = self.layout['filehandles'][0]

                self.pktt.rewind(save_index)
            elif not check:
                if self.pktcall and not self.pktreply:
                    copstr = "OPEN"
                    if nfs_version < 4:
                        copstr = str(self.pktcall.nfs.op)[9:]
                    self.warning("Could not find %s reply for file" % copstr)
                elif nfs_version >= 4 and not self.stateid and not self.pktcall:
                    self.warning("Could not find OPEN call for file")

            dsismds = False
            for ds in self.dslist:
                for item in ds:
                    if self.server_ipaddr == item['ipaddr'] and self.port == item['port']:
                        dsismds = True
                        break

            if nfs_version < 4:
                io_op = NFSPROC3_WRITE if write else NFSPROC3_READ
            else:
                io_op = OP_WRITE if write else OP_READ

            match_str = "NFS.argop == %d" % io_op
            if nfs_version < 4 and filehandle:
                match_str += " and crc32(NFS.fh) == 0x%08x" % crc32(filehandle)
            elif filehandle:
                # There is a valid state id
                match_str += " and crc32(NFS.stateid.other) == 0x%08x" % crc32(self.stateid)

            xids = {}
            ret = []
            received = []
            pkt_hash = {}
            while self.pktt.match(match_str):
                pkt = self.pktt.pkt
                ipaddr = pkt.ip.dst
                pobj   = pkt.udp if pkt.tcp is None else pkt.tcp
                port   = pobj.dst_port
                xid    = pkt.rpc.xid
                if xids.get(xid, None) is None:
                    # Save xid to keep track of re-transmitted packets
                    xids[xid] = 1
                else:
                    # Skip re-transmitted packets
                    continue
                rsize = pkt.NFSop.count
                if self.ispnfs:
                    rserver = MDS if (ipaddr == self.server_ipaddr and port == self.port) else DS
                else:
                    rserver = SERVER
                off = pkt.NFSop.offset
                pkt_hash[off] = {'server':rserver, 'size':rsize}
                received.append("%s(%d)" % (mds_map[rserver], rsize))
                ret.append(rsize)

            if check:
                return ret

            dsmsg = " where DS == MDS" if dsismds else ""
            self.dprint('INFO', "Client sent [%s]%s" % (", ".join(received), dsmsg))

            for item in test_hash:
                server = item.get('server')
                size   = item.get('size')
                pktinfo = pkt_hash.pop(item['offset'], None)
                if pktinfo is None:
                    self.test(False, "%s should be sent to the %s with size %d" % (io_str, mds_map[server], size))
                else:
                    rserver = pktinfo.get('server')
                    rsize   = pktinfo.get('size')
                    srvtest = dsismds or server == rserver
                    srvrmsg = " not the %s" % mds_map[rserver] if not srvtest else ""
                    sizemsg = " not %d" % rsize if size != rsize else ""
                    self.test(srvtest and size == rsize, "%s should be sent to the %s%s with offset %d and size %d%s" % \
                              (io_str, mds_map[server], srvrmsg, item['offset'], size, sizemsg))
            for off in pkt_hash:
                rserver = pkt_hash[off].get('server')
                rsize   = pkt_hash[off].get('size')
                self.test(False, "%s should not be sent to the %s with offset %d and size %d" % \
                          (io_str, mds_map[rserver], off, rsize))

            # Check data on all vectors
            idx = 0
            for item in tinfo:
                size = item['size']
                data = ctypes.string_at(buffers[idx], size)
                if write:
                    if fd is None:
                       fd = open(absfile, 'rb')
                       fd.seek(offset)
                    rdata = fd.read(size)
                    self.test(data == rdata, "WRITE vector(%d) data should be correct" % size)
                else:
                    self.test(data == self.data_pattern(offset, size), "READ vector(%d) data should be correct" % size)
                offset += size
                idx += 1
        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            if fd:
                fd.close()
            self.free_buffers()
            self.pktt.close()

    def _vectored_io_test(self):
        """Verify vectored I/O functionality."""
        self.testidx = 1
        hsize = int(self.PAGESIZE/2)
        for write in self.io_list:
            #=======================================================================
            # Non-contiguous vectors on aligned offset
            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write)

            tinfo = [
                {'size':self.rsize, 'aligned':True,  'server':DS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write)

            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':True,  'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write)

            tinfo = [
                {'size':self.rsize, 'aligned':True,  'server':DS},
                {'size':self.rsize, 'aligned':True},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write)

            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':True,  'server':MDS},
            ]
            self.vectored_io(tinfo, write=write)

            tinfo = [
                {'size':self.rsize, 'aligned':True,  'server':DS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':True,  'server':MDS},
            ]
            self.vectored_io(tinfo, write=write)

            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':True,  'server':MDS},
                {'size':self.rsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write)

            tinfo = [
                {'size':self.rsize, 'aligned':True, 'server':DS},
                {'size':self.rsize, 'aligned':True},
                {'size':self.rsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write)

            #=======================================================================
            # Non-aligned contiguous vectors on aligned offset
            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            #=======================================================================
            # Non-aligned contiguous vectors on aligned offset with various sizes
            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            #=======================================================================
            # Aligned contiguous vectors on aligned offset
            tinfo = [
                {'size':self.rsize, 'aligned':True, 'server':DS},
                {'size':self.rsize, 'aligned':True},
                {'size':self.rsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            #=======================================================================
            # Aligned contiguous vectors on aligned offset with various sizes
            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
                {'size':self.rsize,       'aligned':True, 'server':DS},
                {'size':self.rsize,       'aligned':True, 'server':DS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':True, 'server':DS},
                {'size':self.rsize-hsize, 'aligned':True},
                {'size':self.rsize,       'aligned':True, 'server':DS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
                {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
                {'size':self.rsize,       'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':True, 'server':DS},
                {'size':self.rsize,       'aligned':True},
                {'size':self.rsize-hsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
                {'size':self.rsize,       'aligned':True, 'server':DS},
                {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':True, 'server':DS},
                {'size':self.rsize-hsize, 'aligned':True},
                {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
                {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
                {'size':self.rsize-hsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, contiguous=True)

            #=======================================================================
            # Non-contiguous vectors on non-aligned offset
            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize, 'aligned':True,  'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':True,  'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize, 'aligned':True,  'server':MDS},
                {'size':self.rsize, 'aligned':True},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':True,  'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize, 'aligned':True,  'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':True,  'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':True,  'server':MDS},
                {'size':self.rsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize, 'aligned':True},
                {'size':self.rsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            #=======================================================================
            # Non-contiguous vectors on non-aligned offset with various sizes
            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize,       'aligned':True, 'server':MDS},
                {'size':self.rsize,       'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':True,  'server':MDS},
                {'size':self.rsize,       'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':True,  'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize,       'aligned':True, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize)

            #=======================================================================
            # Non-aligned contiguous vectors on non-aligned offset
            tinfo = [
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            #=======================================================================
            # Non-aligned contiguous vectors on non-aligned offset with various sizes
            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            #=======================================================================
            # Aligned contiguous vectors on non-aligned offset
            tinfo = [
                {'size':self.rsize, 'aligned':True,  'server':MDS},
                {'size':self.rsize, 'aligned':True},
                {'size':self.rsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            #=======================================================================
            # Aligned contiguous vectors on non-aligned offset with various sizes
            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize,       'aligned':True, 'server':MDS},
                {'size':self.rsize,       'aligned':True, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':True, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':True},
                {'size':self.rsize,       'aligned':True, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize,       'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':True, 'server':MDS},
                {'size':self.rsize,       'aligned':True},
                {'size':self.rsize-hsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize,       'aligned':True, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize,       'aligned':True, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':True},
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

            tinfo = [
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
                {'size':self.rsize-hsize, 'aligned':True},
            ]
            self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)

    def vectored_io_test(self):
        """Verify vectored I/O functionality."""
        self.file_handles = {False:None, True:None}
        self.umount()
        try:
            self.mount()
            self._vectored_io_test()
        finally:
            self.umount()

################################################################################
#  Entry point
x = DioTest(usage=USAGE, testnames=TESTNAMES, sid=SCRIPT_ID)

try:
    x.setup(nfiles = 2)

    # Run all the tests
    x.run_tests()
except Exception:
    x.test(False, traceback.format_exc())
finally:
    x.cleanup()
    x.exit()
