Adding upstream version 1:2.6.4.upstream/1%2.6.4upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1145 insertions, 0 deletions

diff --git a/tools/mountstats/mountstats.py b/tools/mountstats/mountstats.py
new file mode 100755
index 0000000..8e129c8
--- /dev/null
+++ b/tools/mountstats/mountstats.py
@@ -0,0 +1,1145 @@
+#!/usr/bin/python3
+# -*- python-mode -*-
+"""Parse /proc/self/mountstats and display it in human readable form
+"""
+
+from __future__ import print_function
+import datetime as datetime
+
+__copyright__ = """
+Copyright (C) 2005, Chuck Lever <cel@netapp.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+MA 02110-1301 USA
+"""
+
+import sys, os, time
+from operator import itemgetter, add
+try:
+    import argparse
+except ImportError:
+    print('%s:  Failed to import argparse - make sure argparse is installed!'
+        % sys.argv[0])
+    sys.exit(1)
+
+Mountstats_version = '0.3'
+
+def difference(x, y):
+    """Used for a map() function
+    """
+    return x - y
+
+NfsEventCounters = [
+    'inoderevalidates',
+    'dentryrevalidates',
+    'datainvalidates',
+    'attrinvalidates',
+    'vfsopen',
+    'vfslookup',
+    'vfspermission',
+    'vfsupdatepage',
+    'vfsreadpage',
+    'vfsreadpages',
+    'vfswritepage',
+    'vfswritepages',
+    'vfsreaddir',
+    'vfssetattr',
+    'vfsflush',
+    'vfsfsync',
+    'vfslock',
+    'vfsrelease',
+    'congestionwait',
+    'setattrtrunc',
+    'extendwrite',
+    'sillyrenames',
+    'shortreads',
+    'shortwrites',
+    'delay',
+    'pnfsreads',
+    'pnfswrites'
+]
+
+NfsByteCounters = [
+    'normalreadbytes',
+    'normalwritebytes',
+    'directreadbytes',
+    'directwritebytes',
+    'serverreadbytes',
+    'serverwritebytes',
+    'readpages',
+    'writepages'
+]
+
+XprtUdpCounters = [
+    'port',
+    'bind_count',
+    'rpcsends',
+    'rpcreceives',
+    'badxids',
+    'inflightsends',
+    'backlogutil',
+    'maxslots',
+    'sendutil',
+    'pendutil'
+]
+
+XprtTcpCounters = [
+    'port',
+    'bind_count',
+    'connect_count',
+    'connect_time',
+    'idle_time',
+    'rpcsends',
+    'rpcreceives',
+    'badxids',
+    'inflightsends',
+    'backlogutil',
+    'maxslots',
+    'sendutil',
+    'pendutil'
+]
+
+XprtRdmaCounters = [
+    'port',
+    'bind_count',
+    'connect_count',
+    'connect_time',
+    'idle_time',
+    'rpcsends',
+    'rpcreceives',
+    'badxids',
+    'inflightsends',
+    'backlogutil',
+    'read_segments',
+    'write_segments',
+    'reply_segments',
+    'total_rdma_req',
+    'total_rdma_rep',
+    'pullup',
+    'fixup',
+    'hardway',
+    'failed_marshal',
+    'bad_reply',
+    'nomsg_calls',
+    'recovered_mrs',
+    'orphaned_mrs',
+    'allocated_mrs',
+    'local_invalidates',
+    'empty_sendctx_q',
+    'reply_waits_for_send',
+]
+
+Nfsv3ops = [
+    'NULL',
+    'GETATTR',
+    'SETATTR',
+    'LOOKUP',
+    'ACCESS',
+    'READLINK',
+    'READ',
+    'WRITE',
+    'CREATE',
+    'MKDIR',
+    'SYMLINK',
+    'MKNOD',
+    'REMOVE',
+    'RMDIR',
+    'RENAME',
+    'LINK',
+    'READDIR',
+    'READDIRPLUS',
+    'FSSTAT',
+    'FSINFO',
+    'PATHCONF',
+    'COMMIT'
+]
+
+# This list should be kept in-sync with the NFSPROC4_CLNT_* enum in
+# include/linux/nfs4.h in the kernel.
+Nfsv4ops = [
+    'NULL',
+    'READ',
+    'WRITE',
+    'COMMIT',
+    'OPEN',
+    'OPEN_CONFIRM',
+    'OPEN_NOATTR',
+    'OPEN_DOWNGRADE',
+    'CLOSE',
+    'SETATTR',
+    'FSINFO',
+    'RENEW',
+    'SETCLIENTID',
+    'SETCLIENTID_CONFIRM',
+    'LOCK',
+    'LOCKT',
+    'LOCKU',
+    'ACCESS',
+    'GETATTR',
+    'LOOKUP',
+    'LOOKUP_ROOT',
+    'REMOVE',
+    'RENAME',
+    'LINK',
+    'SYMLINK',
+    'CREATE',
+    'PATHCONF',
+    'STATFS',
+    'READLINK',
+    'READDIR',
+    'SERVER_CAPS',
+    'DELEGRETURN',
+    'GETACL',
+    'SETACL',
+    'FS_LOCATIONS',
+    'RELEASE_LOCKOWNER',
+    'SECINFO',
+    'FSID_PRESENT',
+    'EXCHANGE_ID',
+    'CREATE_SESSION',
+    'DESTROY_SESSION',
+    'SEQUENCE',
+    'GET_LEASE_TIME',
+    'RECLAIM_COMPLETE',
+    'LAYOUTGET',
+    'GETDEVICEINFO',
+    'LAYOUTCOMMIT',
+    'LAYOUTRETURN',
+    'SECINFO_NO_NAME',
+    'TEST_STATEID',
+    'FREE_STATEID',
+    'GETDEVICELIST',
+    'BIND_CONN_TO_SESSION',
+    'DESTROY_CLIENTID',
+    'SEEK',
+    'ALLOCATE',
+    'DEALLOCATE',
+    'LAYOUTSTATS',
+    'CLONE',
+    'COPY',
+    'OFFLOAD_CANCEL',
+    'LOOKUPP',
+    'LAYOUTERROR',
+    'COPY_NOTIFY'
+]
+
+class DeviceData:
+    """DeviceData objects provide methods for parsing and displaying
+    data for a single mount grabbed from /proc/self/mountstats
+    """
+    def __init__(self):
+        self.__nfs_data = dict()
+        self.__rpc_data = dict()
+        self.__rpc_data['ops'] = []
+
+    def __parse_nfs_line(self, words):
+        if words[0] == 'device':
+            self.__nfs_data['export'] = words[1]
+            self.__nfs_data['mountpoint'] = words[4]
+            self.__nfs_data['fstype'] = words[7]
+            if words[7].find('nfs') != -1 and words[7] != 'nfsd':
+                self.__nfs_data['statvers'] = words[8]
+        elif 'nfs' in words or 'nfs4' in words:
+            self.__nfs_data['export'] = words[0]
+            self.__nfs_data['mountpoint'] = words[3]
+            self.__nfs_data['fstype'] = words[6]
+            if words[6].find('nfs') != -1 and words[6] != 'nfsd':
+                self.__nfs_data['statvers'] = words[7]
+        elif words[0] == 'age:':
+            self.__nfs_data['age'] = int(words[1])
+        elif words[0] == 'opts:':
+            self.__nfs_data['mountoptions'] = ''.join(words[1:]).split(',')
+        elif words[0] == 'caps:':
+            self.__nfs_data['servercapabilities'] = ''.join(words[1:]).split(',')
+        elif words[0] == 'nfsv4:':
+            self.__nfs_data['nfsv4flags'] = ''.join(words[1:]).split(',')
+        elif words[0] == 'sec:':
+            keys = ''.join(words[1:]).split(',')
+            self.__nfs_data['flavor'] = int(keys[0].split('=')[1])
+            self.__nfs_data['pseudoflavor'] = 0
+            if self.__nfs_data['flavor'] == 6:
+                self.__nfs_data['pseudoflavor'] = int(keys[1].split('=')[1])
+        elif words[0] == 'events:':
+            i = 1
+            for key in NfsEventCounters:
+                try:
+                    self.__nfs_data[key] = int(words[i])
+                except IndexError as err:
+                    self.__nfs_data[key] = 0
+                i += 1
+        elif words[0] == 'bytes:':
+            i = 1
+            for key in NfsByteCounters:
+                self.__nfs_data[key] = int(words[i])
+                i += 1
+
+    def __parse_rpc_line(self, words):
+        if words[0] == 'RPC':
+            self.__rpc_data['statsvers'] = float(words[3])
+            self.__rpc_data['programversion'] = words[5]
+        elif words[0] == 'xprt:':
+            self.__rpc_data['protocol'] = words[1]
+            if words[1] == 'udp':
+                i = 2
+                for key in XprtUdpCounters:
+                    if i < len(words):
+                        self.__rpc_data[key] = int(words[i])
+                    i += 1
+            elif words[1] == 'tcp':
+                i = 2
+                for key in XprtTcpCounters:
+                    if i < len(words):
+                        self.__rpc_data[key] = int(words[i])
+                    i += 1
+            elif words[1] == 'rdma':
+                i = 2
+                for key in XprtRdmaCounters:
+                    if i < len(words):
+                        self.__rpc_data[key] = int(words[i])
+                    i += 1
+        elif words[0] == 'per-op':
+            self.__rpc_data['per-op'] = words
+        else:
+            op = words[0][:-1]
+            self.__rpc_data['ops'] += [op]
+            self.__rpc_data[op] = [int(word) for word in words[1:]]
+            if len(self.__rpc_data[op]) < 9:
+                self.__rpc_data[op] += [0]
+
+    def parse_stats(self, lines):
+        """Turn a list of lines from a mount stat file into a 
+        dictionary full of stats, keyed by name
+        """
+        found = False
+        for line in lines:
+            words = line.split()
+            if len(words) == 0:
+                continue
+            if (not found and words[0] != 'RPC'):
+                self.__parse_nfs_line(words)
+                continue
+
+            found = True
+            self.__parse_rpc_line(words)
+
+    def is_nfs_mountpoint(self):
+        """Return True if this is an NFS or NFSv4 mountpoint,
+        otherwise return False
+        """
+        if self.__nfs_data['fstype'] == 'nfs':
+            return True
+        elif self.__nfs_data['fstype'] == 'nfs4':
+            return True
+        return False
+
+    def nfs_version(self):
+        if self.is_nfs_mountpoint():
+            prog, vers = self.__rpc_data['programversion'].split('/')
+            return int(vers)
+
+    def display_raw_stats(self):
+        """Prints out stats in the same format as /proc/self/mountstats
+        """
+        print('device %s mounted on %s with fstype %s %s' % \
+            (self.__nfs_data['export'], self.__nfs_data['mountpoint'], \
+            self.__nfs_data['fstype'], self.__nfs_data['statvers']))
+        print('\topts:\t%s' % ','.join(self.__nfs_data['mountoptions']))
+        print('\tage:\t%d' % self.__nfs_data['age'])
+        print('\tcaps:\t%s' % ','.join(self.__nfs_data['servercapabilities']))
+        print('\tsec:\tflavor=%d,pseudoflavor=%d' % (self.__nfs_data['flavor'], \
+            self.__nfs_data['pseudoflavor']))
+        print('\tevents:\t%s' % " ".join([str(self.__nfs_data[key]) for key in NfsEventCounters]))
+        print('\tbytes:\t%s' % " ".join([str(self.__nfs_data[key]) for key in NfsByteCounters]))
+        print('\tRPC iostats version: %1.1f p/v: %s (nfs)' % (self.__rpc_data['statsvers'], \
+            self.__rpc_data['programversion']))
+        if self.__rpc_data['protocol'] == 'udp':
+            print('\txprt:\tudp %s' % " ".join([str(self.__rpc_data[key]) for key in XprtUdpCounters]))
+        elif self.__rpc_data['protocol'] == 'tcp':
+            print('\txprt:\ttcp %s' % " ".join([str(self.__rpc_data[key]) for key in XprtTcpCounters]))
+        elif self.__rpc_data['protocol'] == 'rdma':
+            print('\txprt:\trdma %s' % " ".join([str(self.__rpc_data[key]) for key in XprtRdmaCounters]))
+        else:
+            raise Exception('Unknown RPC transport protocol %s' % self.__rpc_data['protocol'])
+        print('\tper-op statistics')
+        prog, vers = self.__rpc_data['programversion'].split('/')
+        if vers == '3':
+            for op in Nfsv3ops:
+                print('\t%12s: %s' % (op, " ".join(str(x) for x in self.__rpc_data[op])))
+        elif vers == '4':
+            for op in Nfsv4ops:
+                try:
+                    print('\t%12s: %s' % (op, " ".join(str(x) for x in self.__rpc_data[op])))
+                except KeyError:
+                    continue
+        else:
+            print('\tnot implemented for version %d' % vers)
+        print()
+
+    def display_stats_header(self):
+        print('Stats for %s mounted on %s:' % \
+            (self.__nfs_data['export'], self.__nfs_data['mountpoint']))
+        print()
+
+    def display_nfs_options(self):
+        """Pretty-print the NFS options
+        """
+        print('  NFS mount options: %s' % ','.join(self.__nfs_data['mountoptions']))
+        print('  NFS mount age: %s' % datetime.timedelta(seconds = self.__nfs_data['age']))
+        print('  NFS server capabilities: %s' % ','.join(self.__nfs_data['servercapabilities']))
+        if 'nfsv4flags' in self.__nfs_data:
+            print('  NFSv4 capability flags: %s' % ','.join(self.__nfs_data['nfsv4flags']))
+        if 'pseudoflavor' in self.__nfs_data:
+            print('  NFS security flavor: %d  pseudoflavor: %d' % \
+                (self.__nfs_data['flavor'], self.__nfs_data['pseudoflavor']))
+        else:
+            print('  NFS security flavor: %d' % self.__nfs_data['flavor'])
+
+    def display_nfs_events(self):
+        """Pretty-print the NFS event counters
+        """
+        print()
+        print('Cache events:')
+        print('  data cache invalidated %d times' % self.__nfs_data['datainvalidates'])
+        print('  attribute cache invalidated %d times' % self.__nfs_data['attrinvalidates'])
+        print()
+        print('VFS calls:')
+        print('  VFS requested %d inode revalidations' % self.__nfs_data['inoderevalidates'])
+        print('  VFS requested %d dentry revalidations' % self.__nfs_data['dentryrevalidates'])
+        print()
+        print('  VFS called nfs_readdir() %d times' % self.__nfs_data['vfsreaddir'])
+        print('  VFS called nfs_lookup() %d times' % self.__nfs_data['vfslookup'])
+        print('  VFS called nfs_permission() %d times' % self.__nfs_data['vfspermission'])
+        print('  VFS called nfs_file_open() %d times' % self.__nfs_data['vfsopen'])
+        print('  VFS called nfs_file_flush() %d times' % self.__nfs_data['vfsflush'])
+        print('  VFS called nfs_lock() %d times' % self.__nfs_data['vfslock'])
+        print('  VFS called nfs_fsync() %d times' % self.__nfs_data['vfsfsync'])
+        print('  VFS called nfs_file_release() %d times' % self.__nfs_data['vfsrelease'])
+        print()
+        print('VM calls:')
+        print('  VFS called nfs_readpage() %d times' % self.__nfs_data['vfsreadpage'])
+        print('  VFS called nfs_readpages() %d times' % self.__nfs_data['vfsreadpages'])
+        print('  VFS called nfs_writepage() %d times' % self.__nfs_data['vfswritepage'])
+        print('  VFS called nfs_writepages() %d times' % self.__nfs_data['vfswritepages'])
+        print()
+        print('Generic NFS counters:')
+        print('  File size changing operations:')
+        print('    truncating SETATTRs: %d  extending WRITEs: %d' % \
+            (self.__nfs_data['setattrtrunc'], self.__nfs_data['extendwrite']))
+        print('  %d silly renames' % self.__nfs_data['sillyrenames'])
+        print('  short reads: %d  short writes: %d' % \
+            (self.__nfs_data['shortreads'], self.__nfs_data['shortwrites']))
+        print('  NFSERR_DELAYs from server: %d' % self.__nfs_data['delay'])
+        print('  pNFS READs: %d' % self.__nfs_data['pnfsreads'])
+        print('  pNFS WRITEs: %d' % self.__nfs_data['pnfswrites'])
+
+    def display_nfs_bytes(self):
+        """Pretty-print the NFS event counters
+        """
+        print()
+        print('NFS byte counts:')
+        print('  applications read %d bytes via read(2)' % self.__nfs_data['normalreadbytes'])
+        print('  applications wrote %d bytes via write(2)' % self.__nfs_data['normalwritebytes'])
+        print('  applications read %d bytes via O_DIRECT read(2)' % self.__nfs_data['directreadbytes'])
+        print('  applications wrote %d bytes via O_DIRECT write(2)' % self.__nfs_data['directwritebytes'])
+        print('  client read %d bytes via NFS READ' % self.__nfs_data['serverreadbytes'])
+        print('  client wrote %d bytes via NFS WRITE' % self.__nfs_data['serverwritebytes'])
+
+    def display_rpc_generic_stats(self):
+        """Pretty-print the generic RPC stats
+        """
+        sends = self.__rpc_data['rpcsends']
+
+        print('RPC statistics:')
+
+        print('  %d RPC requests sent, %d RPC replies received (%d XIDs not found)' % \
+            (sends, self.__rpc_data['rpcreceives'], self.__rpc_data['badxids']))
+        if sends != 0:
+            print('  average backlog queue length: %d' % \
+                (float(self.__rpc_data['backlogutil']) / sends))
+
+    def display_rpc_op_stats(self):
+        """Pretty-print the per-op stats
+        """
+        sends = self.__rpc_data['rpcsends']
+
+        allstats = []
+        for op in self.__rpc_data['ops']:
+            allstats.append([op] + self.__rpc_data[op])
+
+        print()
+        for stats in sorted(allstats, key=itemgetter(1), reverse=True):
+            count = stats[1]
+            if count != 0:
+                print('%s:' % stats[0])
+                ops_pcnt = 0
+                if sends != 0:
+                    ops_pcnt = (count * 100) / sends
+                print('\t%d ops (%d%%)' % \
+                    (count, ops_pcnt), end=' ')
+                retrans = stats[2] - count
+                if retrans != 0:
+                    print('\t%d retrans (%d%%)' % (retrans, ((retrans * 100) / count)), end=' ')
+                    print('\t%d major timeouts' % stats[3], end='')
+                if len(stats) >= 10 and stats[9] != 0:
+                    print('\t%d errors (%d%%)' % (stats[9], ((stats[9] * 100) / count)))
+                else:
+                    print('')
+                print('\tavg bytes sent per op: %d\tavg bytes received per op: %d' % \
+                    (stats[4] / count, stats[5] / count))
+                print('\tbacklog wait: %f' % (float(stats[6]) / count), end=' ')
+                print('\tRTT: %f' % (float(stats[7]) / count), end=' ')
+                print('\ttotal execute time: %f (milliseconds)' % \
+                    (float(stats[8]) / count))
+
+    def client_rpc_stats(self):
+        """Tally high-level rpc stats for the nfsstat command
+        """
+        sends = 0
+        trans = 0
+        authrefrsh = 0
+        for op in self.__rpc_data['ops']:
+            sends += self.__rpc_data[op][0]
+            trans += self.__rpc_data[op][1]
+        retrans = trans - sends
+        # authrefresh stats don't actually get captured in
+        # /proc/self/mountstats, so we fudge it here
+        authrefrsh = sends
+        return (sends, retrans, authrefrsh)
+
+    def display_nfsstat_stats(self):
+        """Pretty-print nfsstat-style stats
+        """
+        sends = 0
+        for op in self.__rpc_data['ops']:
+            sends += self.__rpc_data[op][0]
+        if sends == 0:
+            return
+        print()
+        vers = self.nfs_version()
+        print('Client nfs v%d' % vers)
+        info = []
+        for op in self.__rpc_data['ops']:
+            print('%-13s' % str.lower(op)[:12], end='')
+            count = self.__rpc_data[op][0]
+            pct = (count * 100) / sends
+            info.append((count, pct))
+            if (self.__rpc_data['ops'].index(op) + 1) % 6 == 0:
+                print()
+                for (count, pct) in info:
+                    print('%-8u%3u%% ' % (count, pct), end='')
+                print()
+                info = []
+        print()
+        if len(info) > 0:
+            for (count, pct) in info:
+                print('%-8u%3u%% ' % (count, pct), end='')
+            print()
+
+    def compare_iostats(self, old_stats):
+        """Return the difference between two sets of stats
+        """
+        if old_stats.__nfs_data['age'] > self.__nfs_data['age']:
+            return self
+
+        result = DeviceData()
+        protocol = self.__rpc_data['protocol']
+
+        # copy self into result
+        for key, value in self.__nfs_data.items():
+            result.__nfs_data[key] = value
+        for key, value in self.__rpc_data.items():
+            result.__rpc_data[key] = value
+
+        # compute the difference of each item in the list
+        # note the copy loop above does not copy the lists, just
+        # the reference to them.  so we build new lists here
+        # for the result object.
+        for op in result.__rpc_data['ops']:
+            try:
+                result.__rpc_data[op] = list(map(difference, self.__rpc_data[op], old_stats.__rpc_data[op]))
+            except KeyError:
+                continue
+
+        # update the remaining keys
+        if protocol == 'udp':
+            for key in XprtUdpCounters:
+                result.__rpc_data[key] -= old_stats.__rpc_data[key]
+        elif protocol == 'tcp':
+            for key in XprtTcpCounters:
+                result.__rpc_data[key] -= old_stats.__rpc_data[key]
+        elif protocol == 'rdma':
+            for key in XprtRdmaCounters:
+                result.__rpc_data[key] -= old_stats.__rpc_data[key]
+        result.__nfs_data['age'] -= old_stats.__nfs_data['age']
+        for key in NfsEventCounters:
+            result.__nfs_data[key] -= old_stats.__nfs_data[key]
+        for key in NfsByteCounters:
+            result.__nfs_data[key] -= old_stats.__nfs_data[key]
+        return result
+
+    def setup_accumulator(self, ops):
+        """Initialize a DeviceData instance to tally stats for all mountpoints
+        with the same major version. This is for the nfsstat command.
+        """
+        if ops == Nfsv3ops:
+            self.__rpc_data['programversion'] = '100003/3'
+            self.__nfs_data['fstype'] = 'nfs'
+        elif ops == Nfsv4ops:
+            self.__rpc_data['programversion'] = '100003/4'
+            self.__nfs_data['fstype'] = 'nfs4'
+        self.__rpc_data['ops'] = ops
+        for op in ops:
+            self.__rpc_data[op] = [0 for i in range(9)]
+
+    def accumulate_iostats(self, new_stats):
+        """Accumulate counters from all RPC op buckets in new_stats.  This is
+        for the nfsstat command.
+        """
+        for op in new_stats.__rpc_data['ops']:
+            try:
+                self.__rpc_data[op] = list(map(add, self.__rpc_data[op], new_stats.__rpc_data[op]))
+            except KeyError:
+                continue
+
+    def __print_rpc_op_stats(self, op, sample_time):
+        """Print generic stats for one RPC op
+        """
+        if op not in self.__rpc_data:
+            return
+
+        rpc_stats = self.__rpc_data[op]
+        ops = float(rpc_stats[0])
+        retrans = float(rpc_stats[1] - rpc_stats[0])
+        kilobytes = float(rpc_stats[3] + rpc_stats[4]) / 1024
+        queued_for = float(rpc_stats[5])
+        rtt = float(rpc_stats[6])
+        exe = float(rpc_stats[7])
+        if len(rpc_stats) >= 9:
+            errs = int(rpc_stats[8])
+
+        # prevent floating point exceptions
+        if ops != 0:
+            kb_per_op = kilobytes / ops
+            retrans_percent = (retrans * 100) / ops
+            rtt_per_op = rtt / ops
+            exe_per_op = exe / ops
+            queued_for_per_op = queued_for / ops
+            if len(rpc_stats) >= 9:
+                errs_percent = (errs * 100) / ops
+        else:
+            kb_per_op = 0.0
+            retrans_percent = 0.0
+            rtt_per_op = 0.0
+            exe_per_op = 0.0
+            queued_for_per_op = 0.0
+            errs_percent = 0.0
+
+        op += ':'
+        print(format(op.lower(), '<16s'), end='')
+        print(format('ops/s', '>8s'), end='')
+        print(format('kB/s', '>16s'), end='')
+        print(format('kB/op', '>16s'), end='')
+        print(format('retrans', '>16s'), end='')
+        print(format('avg RTT (ms)', '>16s'), end='')
+        print(format('avg exe (ms)', '>16s'), end='')
+        print(format('avg queue (ms)', '>16s'), end='')
+        if len(rpc_stats) >= 9:
+            print(format('errors', '>16s'), end='')
+        print()
+
+        print(format((ops / sample_time), '>24.3f'), end='')
+        print(format((kilobytes / sample_time), '>16.3f'), end='')
+        print(format(kb_per_op, '>16.3f'), end='')
+        retransmits = '{0:>10.0f} ({1:>3.1f}%)'.format(retrans, retrans_percent).strip()
+        print(format(retransmits, '>16'), end='')
+        print(format(rtt_per_op, '>16.3f'), end='')
+        print(format(exe_per_op, '>16.3f'), end='')
+        print(format(queued_for_per_op, '>16.3f'), end='')
+        if len(rpc_stats) >= 9:
+            errors = '{0:>10.0f} ({1:>3.1f}%)'.format(errs, errs_percent).strip()
+            print(format(errors, '>16'), end='')
+        print()
+
+    def display_iostats(self, sample_time):
+        """Display NFS and RPC stats in an iostat-like way
+        """
+        sends = float(self.__rpc_data['rpcsends'])
+        if sample_time == 0:
+            sample_time = float(self.__nfs_data['age'])
+        #  sample_time could still be zero if the export was just mounted.
+        #  Set it to 1 to avoid divide by zero errors in this case since we'll
+        #  likely still have relevant mount statistics to show.
+        #
+        if sample_time == 0:
+            sample_time = 1;
+        if sends != 0:
+            backlog = (float(self.__rpc_data['backlogutil']) / sends) / sample_time
+        else:
+            backlog = 0.0
+
+        print()
+        print('%s mounted on %s:' % \
+            (self.__nfs_data['export'], self.__nfs_data['mountpoint']))
+        print()
+
+        print(format('ops/s', '>16') + format('rpc bklog', '>16'))
+        print(format((sends / sample_time), '>16.3f'), end='')
+        print(format(backlog, '>16.3f'))
+        print()
+
+        self.__print_rpc_op_stats('READ', sample_time)
+        self.__print_rpc_op_stats('WRITE', sample_time)
+        sys.stdout.flush()
+
+    def display_xprt_stats(self):
+        """Pretty-print the xprt statistics
+        """
+        if self.__rpc_data['protocol'] == 'udp':
+            print('\tTransport protocol: udp')
+            print('\tSource port: %d' % self.__rpc_data['port'])
+            print('\tBind count: %d' % self.__rpc_data['bind_count'])
+            print('\tRPC requests: %d' % self.__rpc_data['rpcsends'])
+            print('\tRPC replies: %d' % self.__rpc_data['rpcreceives'])
+            print('\tXIDs not found: %d' % self.__rpc_data['badxids'])
+            print('\tMax slots: %d' % self.__rpc_data['maxslots'])
+            if self.__rpc_data['rpcsends'] != 0:
+                print('\tAvg backlog length: %d' % \
+                    (float(self.__rpc_data['backlogutil']) / self.__rpc_data['rpcsends']))
+                print('\tAvg send queue length: %d' % \
+                    (float(self.__rpc_data['sendutil']) / self.__rpc_data['rpcsends']))
+                print('\tAvg pending queue length: %d' % \
+                    (float(self.__rpc_data['pendutil']) / self.__rpc_data['rpcsends']))
+        elif self.__rpc_data['protocol'] == 'tcp':
+            print('\tTransport protocol: tcp')
+            print('\tSource port: %d' % self.__rpc_data['port'])
+            print('\tBind count: %d' % self.__rpc_data['bind_count'])
+            print('\tConnect count: %d' % self.__rpc_data['connect_count'])
+            print('\tConnect time: %d seconds' % self.__rpc_data['connect_time'])
+            print('\tIdle time: %d seconds' % self.__rpc_data['idle_time'])
+            print('\tRPC requests: %d' % self.__rpc_data['rpcsends'])
+            print('\tRPC replies: %d' % self.__rpc_data['rpcreceives'])
+            print('\tXIDs not found: %d' % self.__rpc_data['badxids'])
+            print('\tMax slots: %d' % self.__rpc_data['maxslots'])
+            if self.__rpc_data['rpcsends'] != 0:
+                print('\tAvg backlog length: %d' % \
+                    (float(self.__rpc_data['backlogutil']) / self.__rpc_data['rpcsends']))
+                print('\tAvg send queue length: %d' % \
+                    (float(self.__rpc_data['sendutil']) / self.__rpc_data['rpcsends']))
+                print('\tAvg pending queue length: %d' % \
+                    (float(self.__rpc_data['pendutil']) / self.__rpc_data['rpcsends']))
+        elif self.__rpc_data['protocol'] == 'rdma':
+            print('\tTransport protocol: rdma')
+            print('\tConnect count: %d' % self.__rpc_data['connect_count'])
+            print('\tConnect time: %d seconds' % self.__rpc_data['connect_time'])
+            print('\tIdle time: %d seconds' % self.__rpc_data['idle_time'])
+            sends = self.__rpc_data['rpcsends']
+            print('\tRPC requests: %d' % self.__rpc_data['rpcsends'])
+            print('\tRPC replies: %d' % self.__rpc_data['rpcreceives'])
+            print('\tXIDs not found: %d' % self.__rpc_data['badxids'])
+            if self.__rpc_data['rpcsends'] != 0:
+                print('\tAvg backlog length: %d' % \
+                    (float(self.__rpc_data['backlogutil']) / self.__rpc_data['rpcsends']))
+            print('\tRead segments: %d' % self.__rpc_data['read_segments'])
+            print('\tWrite segments: %d' % self.__rpc_data['write_segments'])
+            print('\tReply segments: %d' % self.__rpc_data['reply_segments'])
+            print('\tRegistered: %d bytes' % self.__rpc_data['total_rdma_req'])
+            print('\tRDMA received: %d bytes' % self.__rpc_data['total_rdma_rep'])
+            print('\tTotal pull-up: %d bytes' % self.__rpc_data['pullup'])
+            print('\tTotal fix-up: %d bytes' % self.__rpc_data['fixup'])
+            print('\tHardway allocations: %d bytes' % self.__rpc_data['hardway'])
+            print('\tFailed marshals: %d' % self.__rpc_data['failed_marshal'])
+            print('\tBad replies: %d' % self.__rpc_data['bad_reply'])
+
+            """ Counters not present in all kernels """
+            if 'nomsg_calls' in self.__rpc_data:
+                print('\tRDMA_NOMSG calls: %d' % self.__rpc_data['nomsg_calls'])
+            if 'allocated_mrs' in self.__rpc_data:
+                print('\tAllocated MRs: %d' % self.__rpc_data['allocated_mrs'])
+            if 'recovered_mrs' in self.__rpc_data:
+                print('\tRecovered MRs: %d' % self.__rpc_data['recovered_mrs'])
+            if 'orphaned_mrs' in self.__rpc_data:
+                print('\tOrphaned MRs: %d' % self.__rpc_data['orphaned_mrs'])
+            if 'local_invalidates' in self.__rpc_data:
+                print('\tLocal Invalidates needed: %d' % self.__rpc_data['local_invalidates'])
+            if 'empty_sendctx_q' in self.__rpc_data:
+                print('\tEmpty sendctx queue count: %d' % self.__rpc_data['empty_sendctx_q'])
+            if 'reply_waits_for_send' in self.__rpc_data:
+                print('\tReplies that waited for Send completion: %d' % self.__rpc_data['reply_waits_for_send'])
+        else:
+            raise Exception('Unknown RPC transport protocol %s' % self.__rpc_data['protocol'])
+
+def parse_stats_file(f):
+    """pop the contents of a mountstats file into a dictionary,
+    keyed by mount point.  each value object is a list of the
+    lines in the mountstats file corresponding to the mount
+    point named in the key.
+    """
+    ms_dict = dict()
+    key = ''
+
+    f.seek(0)
+    for line in f.readlines():
+        words = line.split()
+        if len(words) == 0:
+            continue
+        if words[0] == 'device':
+            key = words[4]
+            new = [ line.strip() ]
+        elif 'nfs' in words or 'nfs4' in words:
+            key = words[3]
+            new = [ line.strip() ]
+        else:
+            new += [ line.strip() ]
+        ms_dict[key] = new
+
+    return ms_dict
+
+def print_mountstats(stats, nfs_only, rpc_only, raw, xprt_only):
+    if nfs_only:
+       stats.display_stats_header()
+       stats.display_nfs_options()
+       stats.display_nfs_events()
+       stats.display_nfs_bytes()
+    elif rpc_only:
+       stats.display_stats_header()
+       stats.display_rpc_generic_stats()
+       stats.display_rpc_op_stats()
+    elif raw:
+       stats.display_raw_stats()
+    elif xprt_only:
+       stats.display_stats_header()
+       stats.display_xprt_stats()
+    else:
+       stats.display_stats_header()
+       stats.display_nfs_options()
+       stats.display_nfs_bytes()
+       stats.display_rpc_generic_stats()
+       stats.display_rpc_op_stats()
+    print()
+
+def mountstats_command(args):
+    """Mountstats command
+    """
+    mountstats = parse_stats_file(args.infile)
+    mountpoints = [os.path.normpath(mp) for mp in args.mountpoints]
+
+    # make certain devices contains only NFS mount points
+    if len(mountpoints) > 0:
+        check = []
+        for device in mountpoints:
+            stats = DeviceData()
+            try:
+                stats.parse_stats(mountstats[device])
+                if stats.is_nfs_mountpoint():
+                    check += [device]
+            except KeyError:
+                continue
+        mountpoints = check
+    else:
+        for device, descr in mountstats.items():
+            stats = DeviceData()
+            stats.parse_stats(descr)
+            if stats.is_nfs_mountpoint():
+                mountpoints += [device]
+    if len(mountpoints) == 0:
+        print('No NFS mount points were found')
+        return 1
+
+    if args.since:
+        old_mountstats = parse_stats_file(args.since)
+
+    for mp in mountpoints:
+        stats = DeviceData()
+        stats.parse_stats(mountstats[mp])
+        if not args.since:
+            print_mountstats(stats, args.nfs_only, args.rpc_only, args.raw, args.xprt_only)
+        elif args.since and mp not in old_mountstats:
+            print_mountstats(stats, args.nfs_only, args.rpc_only, args.raw, args.xprt_only)
+        else:
+            old_stats = DeviceData()
+            old_stats.parse_stats(old_mountstats[mp])
+            diff_stats = stats.compare_iostats(old_stats)
+            print_mountstats(diff_stats, args.nfs_only, args.rpc_only, args.raw, args.xprt_only)
+
+    args.infile.close()
+    if args.since:
+        args.since.close()
+    return 0
+
+def nfsstat_command(args):
+    """nfsstat-like command for NFS mount points
+    """
+    mountstats = parse_stats_file(args.infile)
+    mountpoints = [os.path.normpath(mp) for mp in args.mountpoints]
+    v3stats = DeviceData()
+    v3stats.setup_accumulator(Nfsv3ops)
+    v4stats = DeviceData()
+    v4stats.setup_accumulator(Nfsv4ops)
+
+    # ensure stats get printed if neither v3 nor v4 was specified
+    if args.show_v3 or args.show_v4:
+        show_both = False
+    else:
+        show_both = True
+
+    # make certain devices contains only NFS mount points
+    if len(mountpoints) > 0:
+        check = []
+        for device in mountpoints:
+            stats = DeviceData()
+            try:
+                stats.parse_stats(mountstats[device])
+                if stats.is_nfs_mountpoint():
+                    check += [device]
+            except KeyError:
+                continue
+        mountpoints = check
+    else:
+        for device, descr in mountstats.items():
+            stats = DeviceData()
+            stats.parse_stats(descr)
+            if stats.is_nfs_mountpoint():
+                mountpoints += [device]
+    if len(mountpoints) == 0:
+        print('No NFS mount points were found')
+        return 1
+
+    if args.since:
+        old_mountstats = parse_stats_file(args.since)
+
+    for mp in mountpoints:
+        stats = DeviceData()
+        stats.parse_stats(mountstats[mp])
+        vers = stats.nfs_version()
+
+        if not args.since:
+            acc_stats = stats
+        elif args.since and mp not in old_mountstats:
+            acc_stats = stats
+        else:
+            old_stats = DeviceData()
+            old_stats.parse_stats(old_mountstats[mp])
+            acc_stats = stats.compare_iostats(old_stats)
+
+        if vers == 3 and (show_both or args.show_v3):
+           v3stats.accumulate_iostats(acc_stats)
+        elif vers == 4 and (show_both or args.show_v4):
+           v4stats.accumulate_iostats(acc_stats)
+
+    sends, retrans, authrefrsh = map(add, v3stats.client_rpc_stats(), v4stats.client_rpc_stats())
+    print('Client rpc stats:')
+    print('calls      retrans    authrefrsh')
+    print('%-11u%-11u%-11u' % (sends, retrans, authrefrsh))
+
+    if show_both or args.show_v3:
+        v3stats.display_nfsstat_stats()
+    if show_both or args.show_v4:
+        v4stats.display_nfsstat_stats()
+
+    args.infile.close()
+    if args.since:
+        args.since.close()
+    return 0
+
+def print_iostat_summary(old, new, devices, time):
+    for device in devices:
+        stats = DeviceData()
+        stats.parse_stats(new[device])
+        if not old or device not in old:
+            stats.display_iostats(time)
+        else:
+            if ("fstype autofs" not in str(old[device])) and ("fstype autofs" not in str(new[device])):
+                old_stats = DeviceData()
+                old_stats.parse_stats(old[device])
+                diff_stats = stats.compare_iostats(old_stats)
+                diff_stats.display_iostats(time)
+
+def iostat_command(args):
+    """iostat-like command for NFS mount points
+    """
+    mountstats = parse_stats_file(args.infile)
+    devices = [os.path.normpath(mp) for mp in args.mountpoints]
+
+    if args.since:
+        old_mountstats = parse_stats_file(args.since)
+    else:
+        old_mountstats = None
+
+    # make certain devices contains only NFS mount points
+    if len(devices) > 0:
+        check = []
+        for device in devices:
+            stats = DeviceData()
+            try:
+                stats.parse_stats(mountstats[device])
+                if stats.is_nfs_mountpoint():
+                    check += [device]
+            except KeyError:
+                continue
+        devices = check
+    else:
+        for device, descr in mountstats.items():
+            stats = DeviceData()
+            stats.parse_stats(descr)
+            if stats.is_nfs_mountpoint():
+                devices += [device]
+    if len(devices) == 0:
+        print('No NFS mount points were found')
+        return 1
+
+    sample_time = 0
+
+    if args.interval is None:
+        print_iostat_summary(old_mountstats, mountstats, devices, sample_time)
+        return
+
+    if args.count is not None:
+        count = args.count
+        while count != 0:
+            print_iostat_summary(old_mountstats, mountstats, devices, sample_time)
+            old_mountstats = mountstats
+            time.sleep(args.interval)
+            sample_time = args.interval
+            mountstats = parse_stats_file(args.infile)
+            count -= 1
+    else: 
+        while True:
+            print_iostat_summary(old_mountstats, mountstats, devices, sample_time)
+            old_mountstats = mountstats
+            time.sleep(args.interval)
+            sample_time = args.interval
+            mountstats = parse_stats_file(args.infile)
+
+    args.infile.close()
+    if args.since:
+        args.since.close()
+    return 0
+
+class ICMAction(argparse.Action):
+    """Custom action to deal with interval, count, and mountpoints.
+    """
+    def __call__(self, parser, namespace, values, option_string=None):
+        if namespace.mountpoints is None:
+            namespace.mountpoints = []
+        if values is None:
+            return
+        elif (type(values) == type([])):
+            for value in values:
+                self._handle_one(namespace, value)
+        else:
+            self._handle_one(namespace, values)
+
+    def _handle_one(self, namespace, value):
+        try:
+            intval = int(value)
+            if namespace.infile.name != '/proc/self/mountstats':
+                raise argparse.ArgumentError(self, "not allowed with argument -f/--file or -S/--since")
+            self._handle_int(namespace, intval)
+        except ValueError:
+            namespace.mountpoints.append(value)
+
+    def _handle_int(self, namespace, value):
+        if namespace.interval is None:
+            namespace.interval = value
+        elif namespace.count is None:
+            namespace.count = value
+        else:
+            raise argparse.ArgumentError(self, "too many integer arguments")
+
+def main():
+    parser = argparse.ArgumentParser(epilog='For specific sub-command help, '
+        'run \'mountstats SUB-COMMAND -h|--help\'')
+    subparsers = parser.add_subparsers(help='sub-command help')
+
+    common_parser = argparse.ArgumentParser(add_help=False)
+    common_parser.add_argument('-v', '--version', action='version',
+        version='mountstats ' + Mountstats_version)
+    common_parser.add_argument('-f', '--file', default=open('/proc/self/mountstats', 'r'),
+        type=argparse.FileType('r'), dest='infile',
+        help='Read stats from %(dest)s instead of /proc/self/mountstats')
+    common_parser.add_argument('-S', '--since', type=argparse.FileType('r'),
+        metavar='SINCEFILE',
+        help='Show difference between current stats and those in SINCEFILE')
+
+    mountstats_parser = subparsers.add_parser('mountstats',
+        parents=[common_parser],
+        help='Display a combination of per-op RPC statistics, NFS event counts, and NFS byte counts. '
+            'This is the default sub-command if no sub-command is given.')
+    group = mountstats_parser.add_mutually_exclusive_group()
+    group.add_argument('-n', '--nfs', action='store_true', dest='nfs_only',
+        help='Display only the NFS statistics')
+    group.add_argument('-r', '--rpc', action='store_true', dest='rpc_only',
+        help='Display only the RPC statistics')
+    group.add_argument('-R', '--raw', action='store_true',
+        help='Display only the raw statistics')
+    group.add_argument('-x', '--xprt', action='store_true', dest='xprt_only',
+        help='Display only the xprt statistics')
+    # The mountpoints argument cannot be moved into the common_parser because
+    # it will screw up the parsing of the iostat arguments (interval and count)
+    mountstats_parser.add_argument('mountpoints', nargs='*', metavar='mountpoint',
+        help='Display statistics for this mountpoint. More than one may be specified. '
+            'If absent, statistics for all NFS mountpoints will be generated.')
+    mountstats_parser.set_defaults(func=mountstats_command)
+
+    nfsstat_parser = subparsers.add_parser('nfsstat',
+        parents=[common_parser],
+        help='Display nfsstat-like statistics.')
+    nfsstat_parser.add_argument('-3', action='store_true', dest='show_v3',
+        help='Show NFS version 3 statistics')
+    nfsstat_parser.add_argument('-4', action='store_true', dest='show_v4',
+        help='Show NFS version 4 statistics')
+    # The mountpoints argument cannot be moved into the common_parser because
+    # it will screw up the parsing of the iostat arguments (interval and count)
+    nfsstat_parser.add_argument('mountpoints', nargs='*', metavar='mountpoint',
+        help='Display statistics for this mountpoint. More than one may be specified. '
+            'If absent, statistics for all NFS mountpoints will be generated.')
+    nfsstat_parser.set_defaults(func=nfsstat_command)
+
+    iostat_parser = subparsers.add_parser('iostat',
+        parents=[common_parser],
+        help='Display iostat-like statistics.')
+    iostat_parser.add_argument('interval', nargs='?', action=ICMAction,
+        help='Number of seconds between reports. If absent, only one report will '
+            'be generated.')
+    iostat_parser.add_argument('count', nargs='?', action=ICMAction,
+        help='Number of reports generated at <interval> seconds apart. If absent, '
+            'reports will be generated continuously.')
+    # The mountpoints argument cannot be moved into the common_parser because
+    # it will screw up the parsing of the iostat arguments (interval and count)
+    iostat_parser.add_argument('mountpoints', nargs='*', action=ICMAction, metavar='mountpoint',
+        help='Display statsistics for this mountpoint. More than one may be specified. '
+            'If absent, statistics for all NFS mountpoints will be generated.')
+    iostat_parser.set_defaults(func=iostat_command)
+ 
+    args = parser.parse_args()
+    return args.func(args)
+
+try:
+    if __name__ == '__main__':
+        # Run the mounstats sub-command if no sub-command (or the help flag)
+        # is given.  If the argparse module ever gets support for optional
+        # (default) sub-commands, then this can be changed.
+        if len(sys.argv) == 1:
+            sys.argv.insert(1, 'mountstats')
+        elif sys.argv[1] not in ['-h', '--help', 'mountstats', 'iostat', 'nfsstat']:
+            sys.argv.insert(1, 'mountstats')
+        res = main()
+        sys.stdout.close()
+        sys.stderr.close()
+        sys.exit(res)
+except (KeyboardInterrupt, RuntimeError):
+    sys.exit(1)
+except IOError:
+    pass
+