path: root/tools/check_typed_item_calls.py

#!/usr/bin/env python3
# Wireshark - Network traffic analyzer
# By Gerald Combs <gerald@wireshark.org>
# Copyright 1998 Gerald Combs
#
# SPDX-License-Identifier: GPL-2.0-or-later

import os
import re
import argparse
import signal
import subprocess

# This utility scans the dissector code for various issues.
# TODO:
# - Create maps from type -> display types for hf items (see display (FIELDDISPLAY)) in docs/README.dissector


# Try to exit soon after Ctrl-C is pressed.
should_exit = False

def signal_handler(sig, frame):
    global should_exit
    should_exit = True
    print('You pressed Ctrl+C - exiting')

signal.signal(signal.SIGINT, signal_handler)


warnings_found = 0
errors_found = 0

def name_has_one_of(name, substring_list):
    for word in substring_list:
        if name.lower().find(word) != -1:
            return True
    return False

# An individual call to an API we are interested in.
# Used by APICheck below.
class Call:
    def __init__(self, hf_name, macros, line_number=None, length=None, fields=None):
        self.hf_name = hf_name
        self.line_number = line_number
        self.fields = fields
        self.length = None
        if length:
            try:
                self.length = int(length)
            except:
                if length.isupper():
                    if length in macros:
                        try:
                            self.length = int(macros[length])
                        except:
                            pass
                pass


# These are variable names that have been seen to be used in calls..
common_hf_var_names = { 'hf_index', 'hf_item', 'hf_idx', 'hf_x', 'hf_id', 'hf_cookie', 'hf_flag',
                        'hf_dos_time', 'hf_dos_date', 'hf_value', 'hf_num',
                        'hf_cause_value', 'hf_uuid',
                        'hf_endian', 'hf_ip', 'hf_port', 'hf_suff', 'hf_string', 'hf_uint',
                        'hf_tag', 'hf_type', 'hf_hdr', 'hf_field', 'hf_opcode', 'hf_size',
                        'hf_entry', 'field' }

item_lengths = {}
item_lengths['FT_CHAR']  = 1
item_lengths['FT_UINT8']  = 1
item_lengths['FT_INT8']   = 1
item_lengths['FT_UINT16'] = 2
item_lengths['FT_INT16']  = 2
item_lengths['FT_UINT24'] = 3
item_lengths['FT_INT24']  = 3
item_lengths['FT_UINT32'] = 4
item_lengths['FT_INT32']  = 4
item_lengths['FT_UINT40'] = 5
item_lengths['FT_INT40']  = 5
item_lengths['FT_UINT48'] = 6
item_lengths['FT_INT48']  = 6
item_lengths['FT_UINT56'] = 7
item_lengths['FT_INT56']  = 7
item_lengths['FT_UINT64'] = 8
item_lengths['FT_INT64']  = 8
item_lengths['FT_ETHER']  = 6
# TODO: other types...


# A check for a particular API function.
class APICheck:
    def __init__(self, fun_name, allowed_types, positive_length=False):
        self.fun_name = fun_name
        self.allowed_types = allowed_types
        self.positive_length = positive_length
        self.calls = []

        if fun_name.startswith('ptvcursor'):
            # RE captures function name + 1st 2 args (always ptvc + hfindex)
            self.p = re.compile('[^\n]*' +  self.fun_name + '\s*\(([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+)')
        elif fun_name.find('add_bitmask') == -1:
            # Normal case.
            # RE captures function name + 1st 2 args (always tree + hfindex + length)
            self.p = re.compile('[^\n]*' +  self.fun_name + '\s*\(([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+),\s*[a-zA-Z0-9_]+,\s*[a-zA-Z0-9_]+,\s*([a-zA-Z0-9_]+)')
        else:
            # _add_bitmask functions.
            # RE captures function name + 1st + 4th args (always tree + hfindex)
            # 6th arg is 'fields'
            self.p = re.compile('[^\n]*' +  self.fun_name + '\s*\(([a-zA-Z0-9_]+),\s*[a-zA-Z0-9_]+,\s*[a-zA-Z0-9_]+,\s*([a-zA-Z0-9_]+)\s*,\s*[a-zA-Z0-9_]+\s*,\s*([a-zA-Z0-9_]+)\s*,')

        self.file = None
        self.mask_allowed = True
        if fun_name.find('proto_tree_add_bits_') != -1:
            self.mask_allowed = False


    def find_calls(self, file, macros):
        self.file = file
        self.calls = []

        with open(file, 'r', encoding="utf8") as f:
            contents = f.read()
            lines = contents.splitlines()
            total_lines = len(lines)
            for line_number,line in enumerate(lines):
                # Want to check this, and next few lines
                to_check = lines[line_number-1] + '\n'
                # Nothing to check if function name isn't in it
                if to_check.find(self.fun_name) != -1:
                    # Ok, add the next file lines before trying RE
                    for i in range(1, 4):
                        if to_check.find(';') != -1:
                            break
                        elif line_number+i < total_lines:
                            to_check += (lines[line_number-1+i] + '\n')
                    m = self.p.search(to_check)
                    if m:
                        fields = None
                        length = None

                        if self.fun_name.find('add_bitmask') != -1:
                            fields = m.group(3)
                        else:
                            if self.p.groups == 3:
                                length = m.group(3)

                        # Add call. We have length if re had 3 groups.
                        num_groups = self.p.groups
                        self.calls.append(Call(m.group(2),
                                               macros,
                                               line_number=line_number,
                                               length=length,
                                               fields=fields))

    # Return true if bit position n is set in value.
    def check_bit(self, value, n):
        return (value & (0x1 << n)) != 0

    def does_mask_cover_value(self, mask, value):
        # Walk past any l.s. 0 bits in value
        n = 0

        mask_start = n
        # Walk through any bits that are set and check they are in mask
        while self.check_bit(value, n) and n <= 63:
            if not self.check_bit(mask, n):
                return False
            n += 1

        return True

    def check_against_items(self, items_defined, items_declared, items_declared_extern, check_missing_items=False,
                            field_arrays=None):
        global errors_found
        global warnings_found

        for call in self.calls:

            # Check lengths, but for now only for APIs that have length in bytes.
            if self.fun_name.find('add_bits') == -1 and call.hf_name in items_defined:
                if call.length and items_defined[call.hf_name].item_type in item_lengths:
                    if item_lengths[items_defined[call.hf_name].item_type] < call.length:
                        print('Warning:', self.file + ':' + str(call.line_number),
                              self.fun_name + ' called for', call.hf_name, ' - ',
                              'item type is', items_defined[call.hf_name].item_type, 'but call has len', call.length)
                        warnings_found += 1

            # Needs a +ve length
            if self.positive_length and call.length != None:
                if call.length != -1 and call.length <= 0:
                    print('Error: ' +  self.fun_name + '(.., ' + call.hf_name + ', ...) called at ' +
                          self.file + ':' + str(call.line_number) +
                          ' with length ' + str(call.length) + ' - must be > 0 or -1')
                    errors_found += 1

            if call.hf_name in items_defined:
                # Is type allowed?
                if not items_defined[call.hf_name].item_type in self.allowed_types:
                    print('Error: ' +  self.fun_name + '(.., ' + call.hf_name + ', ...) called at ' +
                          self.file + ':' + str(call.line_number) +
                          ' with type ' + items_defined[call.hf_name].item_type)
                    print('    (allowed types are', self.allowed_types, ')\n')
                    errors_found += 1
                # No mask allowed
                if not self.mask_allowed and items_defined[call.hf_name].mask_value != 0:
                    print('Error: ' +  self.fun_name + '(.., ' + call.hf_name + ', ...) called at ' +
                          self.file + ':' + str(call.line_number) +
                          ' with mask ' + items_defined[call.hf_name].mask + '    (must be zero!)\n')
                    errors_found += 1

            if self.fun_name.find('add_bitmask') != -1 and call.hf_name in items_defined and field_arrays:
                if call.fields in field_arrays:
                    if (items_defined[call.hf_name].mask_value and
                        field_arrays[call.fields][1] != 0 and items_defined[call.hf_name].mask_value != field_arrays[call.fields][1]):
                        # TODO: only really a problem if bit is set in array but not in top-level item?
                        if not self.does_mask_cover_value(items_defined[call.hf_name].mask_value,
                                                          field_arrays[call.fields][1]):
                            print('Warning:', self.file, call.hf_name, call.fields, "masks don't match. root=",
                                items_defined[call.hf_name].mask,
                                "array has", hex(field_arrays[call.fields][1]))
                            warnings_found += 1

            if check_missing_items:
                if call.hf_name in items_declared and not call.hf_name in items_declared_extern:
                #not in common_hf_var_names:
                    print('Warning:', self.file + ':' + str(call.line_number),
                          self.fun_name + ' called for "' + call.hf_name + '"', ' - but no item found')
                    warnings_found += 1


# Specialization of APICheck for add_item() calls
class ProtoTreeAddItemCheck(APICheck):
    def __init__(self, ptv=None):

        # RE will capture whole call.

        if not ptv:
            # proto_item *
            # proto_tree_add_item(proto_tree *tree, int hfindex, tvbuff_t *tvb,
            #                     const gint start, gint length, const guint encoding)
            self.fun_name = 'proto_tree_add_item'
            self.p = re.compile('[^\n]*' + self.fun_name + '\s*\(\s*[a-zA-Z0-9_]+?,\s*([a-zA-Z0-9_]+?),\s*[a-zA-Z0-9_\+\s]+?,\s*[^,.]+?,\s*(.+),\s*([^,.]+?)\);')
        else:
            # proto_item *
            # ptvcursor_add(ptvcursor_t *ptvc, int hfindex, gint length,
            #               const guint encoding)
            self.fun_name = 'ptvcursor_add'
            self.p = re.compile('[^\n]*' + self.fun_name + '\s*\([^,.]+?,\s*([^,.]+?),\s*([^,.]+?),\s*([a-zA-Z0-9_\-\>]+)')


    def find_calls(self, file, macros):
        self.file = file
        self.calls = []
        with open(file, 'r', encoding="utf8") as f:

            contents = f.read()
            lines = contents.splitlines()
            total_lines = len(lines)
            for line_number,line in enumerate(lines):
                # Want to check this, and next few lines
                to_check = lines[line_number-1] + '\n'
                # Nothing to check if function name isn't in it
                fun_idx = to_check.find(self.fun_name)
                if fun_idx != -1:
                    # Ok, add the next file lines before trying RE
                    for i in range(1, 5):
                        if to_check.find(';') != -1:
                            break
                        elif line_number+i < total_lines:
                            to_check += (lines[line_number-1+i] + '\n')
                    # Lose anything before function call itself.
                    to_check = to_check[fun_idx:]
                    m = self.p.search(to_check)
                    if m:
                        # Throw out if parens not matched
                        if m.group(0).count('(') != m.group(0).count(')'):
                            continue

                        enc = m.group(3)
                        hf_name = m.group(1)
                        if not enc.startswith('ENC_'):
                            if not enc in { 'encoding', 'enc', 'client_is_le', 'cigi_byte_order', 'endian', 'endianess', 'machine_encoding', 'byte_order', 'bLittleEndian',
                                            'p_mq_parm->mq_str_enc', 'p_mq_parm->mq_int_enc',
                                            'iEnc', 'strid_enc', 'iCod', 'nl_data->encoding',
                                            'argp->info->encoding', 'gquic_info->encoding', 'writer_encoding',
                                            'tds_get_int2_encoding(tds_info)',
                                            'tds_get_int4_encoding(tds_info)',
                                            'tds_get_char_encoding(tds_info)',
                                            'info->encoding',
                                            'item->encoding',
                                            'DREP_ENC_INTEGER(drep)', 'string_encoding', 'item', 'type',
                                            'dvb_enc_to_item_enc(encoding)',
                                            'packet->enc',
                                            'IS_EBCDIC(uCCS) ? ENC_EBCDIC : ENC_ASCII',
                                            'DREP_ENC_INTEGER(hdr->drep)',
                                            'dhcp_uuid_endian',
                                            'payload_le',
                                            'local_encoding',
                                            'big_endian',
                                            'hf_data_encoding',
                                            'IS_EBCDIC(eStr) ? ENC_EBCDIC : ENC_ASCII',
                                            'big_endian ? ENC_BIG_ENDIAN : ENC_LITTLE_ENDIAN',
                                            '(skip == 1) ? ENC_BIG_ENDIAN : ENC_LITTLE_ENDIAN',
                                            'pdu_info->sbc', 'pdu_info->mbc',
                                            'seq_info->txt_enc | ENC_NA',
                                            'BASE_SHOW_UTF_8_PRINTABLE',
                                            'dhcp_secs_endian',
                                            'is_mdns ? ENC_UTF_8|ENC_NA : ENC_ASCII|ENC_NA',
                                            'xl_encoding'
                                          }:
                                global warnings_found

                                print('Warning:', self.file + ':' + str(line_number),
                                      self.fun_name + ' called for "' + hf_name + '"',  'check last/enc param:', enc, '?')
                                warnings_found += 1
                        self.calls.append(Call(hf_name, macros, line_number=line_number, length=m.group(2)))

    def check_against_items(self, items_defined, items_declared, items_declared_extern,
                            check_missing_items=False, field_arrays=None):
        # For now, only complaining if length if call is longer than the item type implies.
        #
        # Could also be bugs where the length is always less than the type allows.
        # Would involve keeping track (in the item) of whether any call had used the full length.

        global warnings_found

        for call in self.calls:
            if call.hf_name in items_defined:
                if call.length and items_defined[call.hf_name].item_type in item_lengths:
                    if item_lengths[items_defined[call.hf_name].item_type] < call.length:
                        print('Warning:', self.file + ':' + str(call.line_number),
                              self.fun_name + ' called for', call.hf_name, ' - ',
                              'item type is', items_defined[call.hf_name].item_type, 'but call has len', call.length)
                        warnings_found += 1
            elif check_missing_items:
                if call.hf_name in items_declared and not call.hf_name in items_declared_extern:
                #not in common_hf_var_names:
                    print('Warning:', self.file + ':' + str(call.line_number),
                          self.fun_name + ' called for "' + call.hf_name + '"', ' - but no item found')
                    warnings_found += 1



##################################################################################################
# This is a set of items (by filter name) where we know that the bitmask is non-contiguous,
# but is still believed to be correct.
known_non_contiguous_fields = { 'wlan.fixed.capabilities.cfpoll.sta',
                                'wlan.wfa.ie.wme.qos_info.sta.reserved',
                                'btrfcomm.frame_type',   # https://os.itec.kit.edu/downloads/sa_2006_roehricht-martin_flow-control-in-bluez.pdf
                                'capwap.control.message_element.ac_descriptor.dtls_policy.r', # RFC 5415
                                'couchbase.extras.subdoc.flags.reserved',
                                'wlan.fixed.capabilities.cfpoll.ap',   # These are 3 separate bits...
                                'wlan.wfa.ie.wme.tspec.ts_info.reserved', # matches other fields in same sequence
                                'zbee_zcl_se.pp.attr.payment_control_configuration.reserved', # matches other fields in same sequence
                                'zbee_zcl_se.pp.snapshot_payload_cause.reserved',  # matches other fields in same sequence
                                'ebhscr.eth.rsv',  # matches other fields in same sequence
                                'v120.lli',  # non-contiguous field (http://www.acacia-net.com/wwwcla/protocol/v120_l2.htm)
                                'stun.type.class',
                                'bssgp.csg_id', 'tiff.t6.unused', 'artnet.ip_prog_reply.unused',
                                'telnet.auth.mod.enc', 'osc.message.midi.bender', 'btle.data_header.rfu',
                                'stun.type.method', # figure 3 in rfc 5389
                                'tds.done.status', # covers all bits in bitset
                                'hf_iax2_video_csub',  # RFC 5456, table 8.7
                                'iax2.video.subclass',
                                'dnp3.al.ana.int',
                                'pwcesopsn.cw.lm',
                                'gsm_a.rr.format_id', # EN 301 503
                                'siii.mst.phase', # comment in code seems convinced
                                'xmcp.type.class',
                                'xmcp.type.method',
                                'hf_hiqnet_flags',
                                'hf_hiqnet_flagmask',
                                'hf_h223_mux_mpl',
                                'rdp.flags.pkt'
                              }
##################################################################################################


field_widths = {
    'FT_BOOLEAN' : 64,   # TODO: Width depends upon 'display' field
    'FT_CHAR'    : 8,
    'FT_UINT8'   : 8,
    'FT_INT8'    : 8,
    'FT_UINT16'  : 16,
    'FT_INT16'   : 16,
    'FT_UINT24'  : 24,
    'FT_INT24'   : 24,
    'FT_UINT32'  : 32,
    'FT_INT32'   : 32,
    'FT_UINT40'  : 40,
    'FT_INT40'   : 40,
    'FT_UINT48'  : 48,
    'FT_INT48'   : 48,
    'FT_UINT56'  : 56,
    'FT_INT56'   : 56,
    'FT_UINT64'  : 64,
    'FT_INT64'   : 64
}

# TODO: most of these might as well be strings...
def is_ignored_consecutive_filter(filter):
    ignore_patterns = [
        re.compile(r'^elf.sh_type'),
        re.compile(r'^elf.p_type'),
        re.compile(r'^btavrcp.pdu_id'),
        re.compile(r'^nstrace.trcdbg.val(\d+)'),
        re.compile(r'^netlogon.dummy_string'),
        re.compile(r'^opa.reserved'),
        re.compile(r'^mpls_pm.timestamp\d\..*'),
        re.compile(r'^wassp.data.mu_mac'),
        re.compile(r'^thrift.type'),
        re.compile(r'^quake2.game.client.command.move.angles'),
        re.compile(r'^ipp.enum_value'),
        re.compile(r'^idrp.error.subcode'),
        re.compile(r'^ftdi-ft.lValue'),
        re.compile(r'^6lowpan.src'),
        re.compile(r'^couchbase.flex_frame.frame.id'),
        re.compile(r'^rtps.param.id'),
        re.compile(r'^rtps.locator.port'),
        re.compile(r'^sigcomp.udvm.value'),
        re.compile(r'^opa.mad.attributemodifier.n'),
        re.compile(r'^smb.cmd'),
        re.compile(r'^sctp.checksum'),
        re.compile(r'^dhcp.option.end'),
        re.compile(r'^nfapi.num.bf.vector.bf.value'),
        re.compile(r'^dnp3.al.range.abs'),
        re.compile(r'^dnp3.al.range.quantity'),
        re.compile(r'^dnp3.al.index'),
        re.compile(r'^dnp3.al.size'),
        re.compile(r'^ftdi-ft.hValue'),
        re.compile(r'^homeplug_av.op_attr_cnf.data.sw_sub'),
        re.compile(r'^radiotap.he_mu.preamble_puncturing'),
        re.compile(r'^ndmp.file'),
        re.compile(r'^ocfs2.dlm.lvb'),
        re.compile(r'^oran_fh_cus.reserved'),
        re.compile(r'^qnet6.kif.msgsend.msg.read.xtypes0-7'),
        re.compile(r'^qnet6.kif.msgsend.msg.write.xtypes0-7'),
        re.compile(r'^mih.sig_strength'),
        re.compile(r'^couchbase.flex_frame.frame.len'),
        re.compile(r'^nvme-rdma.read_to_host_req'),
        re.compile(r'^rpcap.dummy'),
        re.compile(r'^sflow.flow_sample.output_interface'),
        re.compile(r'^socks.results'),
        re.compile(r'^opa.mad.attributemodifier.p'),
        re.compile(r'^v5ua.efa'),
        re.compile(r'^zbncp.data.tx_power'),
        re.compile(r'^zbncp.data.nwk_addr'),
        re.compile(r'^zbee_zcl_hvac.pump_config_control.attr.ctrl_mode'),
        re.compile(r'^nat-pmp.external_port'),
        re.compile(r'^zbee_zcl.attr.float'),
        re.compile(r'^wpan-tap.phr.fsk_ms.mode'),
        re.compile(r'^mysql.exec_flags'),
        re.compile(r'^pim.metric_pref'),
        re.compile(r'^modbus.regval_float'),
        re.compile(r'^alcap.cau.value'),
        re.compile(r'^bpv7.crc_field'),
        re.compile(r'^at.chld.mode'),
        re.compile(r'^btl2cap.psm'),
        re.compile(r'^srvloc.srvtypereq.nameauthlistlen'),
        re.compile(r'^a11.ext.code'),
        re.compile(r'^adwin_config.port'),
        re.compile(r'^afp.unknown'),
        re.compile(r'^ansi_a_bsmap.mid.digit_1'),
        re.compile(r'^ber.unknown.OCTETSTRING'),
        re.compile(r'^btatt.handle'),
        re.compile(r'^btl2cap.option_flushto'),
        re.compile(r'^cip.network_segment.prod_inhibit'),
        re.compile(r'^cql.result.rows.table_name'),
        re.compile(r'^dcom.sa.vartype'),
        re.compile(r'^f5ethtrailer.slot'),
        re.compile(r'^ipdr.cm_ipv6_addr'),
        re.compile(r'^mojito.kuid'),
        re.compile(r'^mtp3.priority'),
        re.compile(r'^pw.cw.length'),
        re.compile(r'^rlc.ciphered_data'),
        re.compile(r'^vp8.pld.pictureid'),
        re.compile(r'^gryphon.sched.channel'),
        re.compile(r'^pn_io.ioxs'),
        re.compile(r'^pn_dcp.block_qualifier_reset'),
        re.compile(r'^pn_dcp.suboption_device_instance'),
        re.compile(r'^nfs.attr'),
        re.compile(r'^nfs.create_session_flags'),
        re.compile(r'^rmt-lct.toi64'),
        re.compile(r'^gryphon.data.header_length'),
        re.compile(r'^quake2.game.client.command.move.movement'),
        re.compile(r'^isup.parameter_type'),
        re.compile(r'^cip.port'),
        re.compile(r'^adwin.fifo_no'),
        re.compile(r'^bthci_evt.hci_vers_nr'),
        re.compile(r'^gryphon.usdt.stmin_active'),
        re.compile(r'^dnp3.al.anaout.int'),
        re.compile(r'^dnp3.al.ana.int'),
        re.compile(r'^dnp3.al.cnt'),
        re.compile(r'^bthfp.chld.mode'),
        re.compile(r'^nat-pmp.pml'),
        re.compile(r'^isystemactivator.actproperties.ts.hdr'),
        re.compile(r'^rtpdump.txt_addr'),
        re.compile(r'^unistim.vocoder.id'),
        re.compile(r'^mac.ueid'),
        re.compile(r'cip.symbol.size'),
        re.compile(r'dnp3.al.range.start'),
        re.compile(r'dnp3.al.range.stop'),
        re.compile(r'gtpv2.mp'),
        re.compile(r'gvcp.cmd.resend.firstpacketid'),
        re.compile(r'gvcp.cmd.resend.lastpacketid'),
        re.compile(r'wlan.bf.reserved'),
        re.compile(r'opa.sa.reserved'),
        re.compile(r'rmt-lct.ext_tol_transfer_len'),
        re.compile(r'pn_io.error_code2'),
        re.compile(r'gryphon.ldf.schedsize'),
        re.compile(r'wimaxmacphy.burst_opt_mimo_matrix_indicator'),
        re.compile(r'alcap.*bwt.*.[b|f]w'),
        re.compile(r'ccsds.packet_type'),
        re.compile(r'iso15765.flow_control.stmin'),
        re.compile(r'msdo.PieceSize'),
        re.compile(r'opa.clasportinfo.redirect.reserved'),
        re.compile(r'p_mul.unused'),
        re.compile(r'btle.control.phys.le_[1|2]m_phy'),
        re.compile(r'opa.pm.dataportcounters.reserved'),
        re.compile(r'opa.switchinfo.switchcapabilitymask.reserved'),
        re.compile(r'nvme-rdma.read_from_host_resp'),
        re.compile(r'nvme-rdma.write_to_host_req'),
        re.compile(r'netlink-route.ifla_linkstats.rx_errors.fifo_errs'),
        re.compile(r'mtp3mg.japan_spare'),
        re.compile(r'ixveriwave.errors.ip_checksum_error'),
        re.compile(r'ansi_a_bsmap.cm2.scm.bc_entry.opmode[0|1]')
    ]

    for patt in ignore_patterns:
        if patt.match(filter):
            return True
    return False


class ValueString:
    def __init__(self, file, name, vals, macros, do_extra_checks=False):
        self.file = file
        self.name = name
        self.raw_vals = vals
        self.parsed_vals = {}
        self.seen_labels = set()
        self.valid = True
        self.min_value =  99999
        self.max_value = -99999

        # Now parse out each entry in the value_string
        matches = re.finditer(r'\{\s*([0-9_A-Za-z]*)\s*,\s*(".*?")\s*}\s*,', self.raw_vals)
        for m in matches:
            value,label = m.group(1), m.group(2)
            if value in macros:
                value = macros[value]
            elif any(not c in '0123456789abcdefABCDEFxX' for c in value):
                self.valid = False
                return

            try:
                # Read according to the appropriate base.
                if value.lower().startswith('0x'):
                    value = int(value, 16)
                elif value.startswith('0b'):
                    value = int(value[2:], 2)
                elif value.startswith('0'):
                    value = int(value, 8)
                else:
                    value = int(value, 10)
            except:
                return

            global warnings_found

            # Check for value conflict before inserting
            if value in self.parsed_vals and label != self.parsed_vals[value]:
                print('Warning:', self.file, ': value_string', self.name, '- value ', value, 'repeated with different values - was',
                    self.parsed_vals[value], 'now', label)
                warnings_found += 1
            else:
                # Add into table, while checking for repeated label
                self.parsed_vals[value] = label
                if do_extra_checks and label in self.seen_labels:
                    # These are commonly repeated..
                    exceptions = [ 'reserved', 'invalid', 'unused', 'not used', 'unknown', 'undefined', 'spare',
                                   'unallocated', 'not assigned', 'implementation specific', 'unspecified',
                                   'other', 'for further study', 'future', 'vendor specific', 'obsolete', 'none',
                                   'shall not be used', 'national use', 'unassigned', 'oem', 'user defined',
                                   'manufacturer specific', 'not specified', 'proprietary', 'operator-defined',
                                   'dynamically allocated', 'user specified', 'xxx', 'default', 'planned', 'not req' ]
                    excepted = False
                    for ex in exceptions:
                        if label.lower().find(ex) != -1:
                            excepted = True
                            break

                    if not excepted:
                        print('Warning:', self.file, ': value_string', self.name, '- label ', label, 'repeated')
                        warnings_found += 1
                else:
                    self.seen_labels.add(label)

                if value > self.max_value:
                    self.max_value = value
                if value < self.min_value:
                    self.min_value = value

    def extraChecks(self):
        global warnings_found

        # Look for one value missing in range (quite common...)
        num_items = len(self.parsed_vals)
        span = self.max_value - self.min_value + 1
        if num_items > 4 and span > num_items and (span-num_items <=1):
            for val in range(self.min_value, self.max_value):
                if not val in self.parsed_vals:
                    print('Warning:', self.file, ': value_string', self.name, '- value', val, 'missing?', '(', num_items, 'entries)')
                    global warnings_found
                    warnings_found += 1

        # Do most of the labels match the number?
        matching_label_entries = set()
        for val in self.parsed_vals:
            if self.parsed_vals[val].find(str(val)) != -1:
                # TODO: pick out multiple values rather than concat into wrong number
                parsed_value = int(''.join(d for d in self.parsed_vals[val] if d.isdecimal()))
                if val == parsed_value:
                    matching_label_entries.add(val)

        if len(matching_label_entries) >= 4 and len(matching_label_entries) > 0 and len(matching_label_entries) < num_items and len(matching_label_entries) >= num_items-1:
            # Be forgiving about first or last entry
            first_val = list(self.parsed_vals)[0]
            last_val =  list(self.parsed_vals)[-1]
            if not first_val in matching_label_entries or not last_val in matching_label_entries:
                return
            print('Warning:', self.file, ': value_string', self.name, 'Labels match value except for 1!', matching_label_entries, num_items, self)

        # Do all labels start with lower-or-upper char?
        startLower,startUpper = 0,0
        for val in self.parsed_vals:
            first_letter = self.parsed_vals[val][1]
            if first_letter.isalpha():
                if first_letter.isupper():
                    startUpper += 1
                else:
                    startLower += 1
        if startLower > 0 and startUpper > 0:
            if startLower+startUpper > 10 and (startLower <=3 or startUpper <=3):
                standouts = []
                if startLower < startUpper:
                    standouts += [self.parsed_vals[val] for val in self.parsed_vals if self.parsed_vals[val][1].islower()]
                if startLower > startUpper:
                    standouts += [self.parsed_vals[val] for val in self.parsed_vals if self.parsed_vals[val][1].isupper()]

                print('Note:', self.file, ': value_string', self.name, 'mix of upper', startUpper, 'and lower', startLower, standouts)


    def __str__(self):
        return  self.name + '= { ' + self.raw_vals + ' }'


class RangeStringEntry:
    def __init__(self, min, max, label):
        self.min = min
        self.max = max
        self.label = label

    def hides(self, min, max):
        return min >= self.min and max <= self.max

    def __str__(self):
        return '(' + str(self.min) + ', ' + str(self.max) + ') -> ' + self.label


class RangeString:
    def __init__(self, file, name, vals, macros, do_extra_checks=False):
        self.file = file
        self.name = name
        self.raw_vals = vals
        self.parsed_vals = []
        self.seen_labels = set()
        self.valid = True
        self.min_value =  99999
        self.max_value = -99999

        # Now parse out each entry in the value_string
        matches = re.finditer(r'\{\s*([0-9_A-Za-z]*)\s*,\s*([0-9_A-Za-z]*)\s*,\s*(".*?")\s*}\s*,', self.raw_vals)
        for m in matches:
            min,max,label = m.group(1), m.group(2), m.group(3)
            if min in macros:
                min = macros[min]
            elif any(not c in '0123456789abcdefABCDEFxX' for c in min):
                self.valid = False
                return
            if max in macros:
                max = macros[max]
            elif any(not c in '0123456789abcdefABCDEFxX' for c in max):
                self.valid = False
                return


            try:
                # Read according to the appropriate base.
                if min.lower().startswith('0x'):
                    min = int(min, 16)
                elif min.startswith('0b'):
                    min = int(min[2:], 2)
                elif min.startswith('0'):
                    min = int(min, 8)
                else:
                    min = int(min, 10)

                if max.lower().startswith('0x'):
                    max = int(max, 16)
                elif max.startswith('0b'):
                    max = int(max[2:], 2)
                elif max.startswith('0'):
                    max = int(max, 8)
                else:
                    max = int(max, 10)
            except:
                return

            # Now check what we've found.
            global warnings_found

            if min < self.min_value:
                self.min_value = min
            # For overall max value, still use min of each entry.
            # It is common for entries to extend to e.g. 0xff, but at least we can check for items
            # that can never match if we only chec the min.
            if min > self.max_value:
                self.max_value = min

            # This value should not be entirely hidden by earlier entries
            for prev in self.parsed_vals:
                if prev.hides(min, max):
                    print('Warning:', self.file, ': range_string label', label, 'hidden by', prev)
                    warnings_found += 1

            # Max should not be > min
            if min > max:
                print('Warning:', self.file, ': range_string', self.name, 'entry', label, 'min', min, '>', max)
                warnings_found += 1

            # Check label.
            if label[1:-1].startswith(' ') or label[1:-1].endswith(' '):
                print('Warning:', self.file, ': range_string', self.name, 'entry', label, 'starts or ends with space')
                warnings_found += 1

            # OK, add this entry
            self.parsed_vals.append(RangeStringEntry(min, max, label))

    def extraChecks(self):
        pass
        # TODO: some checks over all entries.  e.g.,
        # - can multiple values be coalesced into 1?
        # - if in all cases min==max, suggest value_string instead?




# Look for value_string entries in a dissector file.  Return a dict name -> ValueString
def findValueStrings(filename, macros, do_extra_checks=False):
    vals_found = {}

    #static const value_string radio_type_vals[] =
    #{
    #    { 0,      "FDD"},
    #    { 1,      "TDD"},
    #    { 0, NULL }
    #};

    with open(filename, 'r', encoding="utf8") as f:
        contents = f.read()

        # Remove comments so as not to trip up RE.
        contents = removeComments(contents)

        matches =   re.finditer(r'.*const value_string\s*([a-zA-Z0-9_]*)\s*\[\s*\]\s*\=\s*\{([\{\}\d\,a-zA-Z0-9_\-\*\#\.:\/\(\)\'\s\"]*)\};', contents)
        for m in matches:
            name = m.group(1)
            vals = m.group(2)
            vals_found[name] = ValueString(filename, name, vals, macros, do_extra_checks)

    return vals_found

# Look for value_string entries in a dissector file.  Return a dict name -> ValueString
def findRangeStrings(filename, macros, do_extra_checks=False):
    vals_found = {}

    #static const range_string symbol_table_shndx_rvals[] = {
    #    { 0x0000, 0x0000,  "Undefined" },
    #    { 0x0001, 0xfeff,  "Normal Section" },
    #    { 0, 0, NULL }
    #};

    with open(filename, 'r', encoding="utf8") as f:
        contents = f.read()

        # Remove comments so as not to trip up RE.
        contents = removeComments(contents)

        matches =   re.finditer(r'.*const range_string\s*([a-zA-Z0-9_]*)\s*\[\s*\]\s*\=\s*\{([\{\}\d\,a-zA-Z0-9_\-\*\#\.:\/\(\)\'\s\"]*)\};', contents)
        for m in matches:
            name = m.group(1)
            vals = m.group(2)
            vals_found[name] = RangeString(filename, name, vals, macros, do_extra_checks)

    return vals_found



# The relevant parts of an hf item.  Used as value in dict where hf variable name is key.
class Item:

    # Keep the previous few items
    previousItems = []

    def __init__(self, filename, hf, filter, label, item_type, display, strings, macros,
                 value_strings, range_strings,
                 mask=None, check_mask=False, mask_exact_width=False, check_label=False,
                 check_consecutive=False, blurb=''):
        self.filename = filename
        self.hf = hf
        self.filter = filter
        self.label = label
        self.mask = mask
        self.strings = strings
        self.mask_exact_width = mask_exact_width

        global warnings_found

        self.set_mask_value(macros)

        if check_consecutive:
            for previous_index,previous_item in enumerate(Item.previousItems):
                if previous_item.filter == filter:
                    if label != previous_item.label:
                        if not is_ignored_consecutive_filter(self.filter):
                            print('Warning:', filename, hf, ': - filter "' + filter +
                                '" appears ' + str(previous_index+1) + ' items before - labels are "' + previous_item.label + '" and "' + label + '"')
                            warnings_found += 1

            # Add this one to front of (short) previous list
            Item.previousItems = [self] + Item.previousItems
            if len(Item.previousItems) > 5:
                # Get rid of oldest one now
                #Item.previousItems = Item.previousItems[:-1]
                Item.previousItems.pop()

        self.item_type = item_type
        self.display = display

        # Optionally check label (short and long).
        if check_label:
            self.check_label(label, 'label')
            #self.check_label(blurb, 'blurb')

        # Optionally check that mask bits are contiguous
        if check_mask:
            if self.mask_read and not mask in { 'NULL', '0x0', '0', '0x00' }:
                self.check_contiguous_bits(mask)
                self.check_num_digits(self.mask)
                # N.B., if last entry in set is removed, see around 18,000 warnings
                self.check_digits_all_zeros(self.mask)

        # N.B. these checks are already done by checkApis.pl
        if strings.find('RVALS') != -1 and display.find('BASE_RANGE_STRING') == -1:
            print('Warning: ' + filename, hf, 'filter "' + filter + ' strings has RVALS but display lacks BASE_RANGE_STRING')
            warnings_found += 1

        # For RVALS, is BASE_RANGE_STRING also set (checked by checkApis.pl)?
        if strings.find('VALS_EXT_PTR') != -1 and display.find('BASE_EXT_STRING') == -1:
            print('Warning: ' + filename, hf, 'filter "' + filter + ' strings has VALS_EXT_PTR but display lacks BASE_EXT_STRING')
            warnings_found += 1

        # For VALS, lookup the corresponding ValueString and try to check range.
        vs_re = re.compile(r'VALS\(([a-zA-Z0-9_]*)\)')
        m = vs_re.search(strings)
        if m:
            self.vs_name = m.group(1)
            if self.vs_name in value_strings:
                vs = value_strings[self.vs_name]
                self.check_value_string_range(vs.min_value, vs.max_value)

        # For RVALS, lookup the corresponding RangeString and try to check range.
        rs_re = re.compile(r'RVALS\(([a-zA-Z0-9_]*)\)')
        m = rs_re.search(strings)
        if m:
            self.rs_name = m.group(1)
            if self.rs_name in range_strings:
                rs = range_strings[self.rs_name]
                self.check_range_string_range(rs.min_value, rs.max_value)


    def __str__(self):
        return 'Item ({0} "{1}" {2} type={3}:{4} {5} mask={6})'.format(self.filename, self.label, self.filter, self.item_type, self.display, self.strings, self.mask)

    def check_label(self, label, label_name):
        global warnings_found

        # TODO: this is masking a bug where the re for the item can't cope with macro for containing ',' for mask arg..
        if label.count('"') == 1:
            return

        if label.startswith(' ') or label.endswith(' '):
            print('Warning: ' + self.filename, self.hf, 'filter "' + self.filter, label_name,  '"' + label + '" begins or ends with a space')
            warnings_found += 1

        if (label.count('(') != label.count(')') or
            label.count('[') != label.count(']') or
            label.count('{') != label.count('}')):
            # Ignore if includes quotes, as may be unbalanced.
            if label.find("'") == -1:
                print('Warning: ' + self.filename, self.hf, 'filter "' + self.filter + '"', label_name, '"' + label + '"', 'has unbalanced parens/braces/brackets')
                warnings_found += 1
        if self.item_type != 'FT_NONE' and label.endswith(':'):
            print('Warning: ' + self.filename, self.hf, 'filter "' + self.filter + '"', label_name, '"' + label + '"', 'ends with an unnecessary colon')
            warnings_found += 1


    def set_mask_value(self, macros):
        try:
            self.mask_read = True

            # Substitute mask if found as a macro..
            if self.mask in macros:
                self.mask = macros[self.mask]
            elif any(not c in '0123456789abcdefABCDEFxX' for c in self.mask):
                self.mask_read = False
                self.mask_value = 0
                return


            # Read according to the appropriate base.
            if self.mask.startswith('0x'):
                self.mask_value = int(self.mask, 16)
            elif self.mask.startswith('0'):
                self.mask_value = int(self.mask, 8)
            else:
                self.mask_value = int(self.mask, 10)
        except:
            self.mask_read = False
            self.mask_value = 0

    def check_value_string_range(self, vs_min, vs_max):
        item_width = self.get_field_width_in_bits()

        if item_width is None:
            # Type field defined by macro?
            return

        if self.mask_value > 0:
            # Distance between first and last '1'
            bitBools = bin(self.mask_value)[2:]
            mask_width = bitBools.rfind('1') - bitBools.find('1') + 1
        else:
            # No mask is effectively a full mask..
            mask_width = item_width

        item_max = (2 ** mask_width)
        if vs_max > item_max:
            global warnings_found
            print('Warning:', self.filename, self.hf, 'filter=', self.filter,
                  self.strings, "has max value", vs_max, '(' + hex(vs_max) + ')', "which doesn't fit into", mask_width, 'bits',
                  '( mask is', hex(self.mask_value), ')')
            warnings_found += 1

    def check_range_string_range(self, rs_min, rs_max):
        item_width = self.get_field_width_in_bits()

        if item_width is None:
            # Type field defined by macro?
            return

        if self.mask_value > 0:
            # Distance between first and last '1'
            bitBools = bin(self.mask_value)[2:]
            mask_width = bitBools.rfind('1') - bitBools.find('1') + 1
        else:
            # No mask is effectively a full mask..
            mask_width = item_width

        item_max = (2 ** mask_width)
        if rs_max > item_max:
            global warnings_found
            print('Warning:', self.filename, self.hf, 'filter=', self.filter,
                  self.strings, "has values", rs_min, rs_max, '(' + hex(rs_max) + ')', "which doesn't fit into", mask_width, 'bits',
                  '( mask is', hex(self.mask_value), ')')
            warnings_found += 1




    # Return true if bit position n is set in value.
    def check_bit(self, value, n):
        return (value & (0x1 << n)) != 0

    # Output a warning if non-contigous bits are found in the mask (guint64).
    # Note that this legimately happens in several dissectors where multiple reserved/unassigned
    # bits are conflated into one field.
    # - there is probably a cool/efficient way to check this (+1 => 1-bit set?)
    def check_contiguous_bits(self, mask):
        if not self.mask_value:
            return

        # Do see legitimate non-contiguous bits often for these..
        if name_has_one_of(self.hf, ['reserved', 'unknown', 'unused', 'spare']):
            return
        if name_has_one_of(self.label, ['reserved', 'unknown', 'unused', 'spare']):
            return


        # Walk past any l.s. 0 bits
        n = 0
        while not self.check_bit(self.mask_value, n) and n <= 63:
            n += 1
        if n==63:
            return

        mask_start = n
        # Walk through any bits that are set
        while self.check_bit(self.mask_value, n) and n <= 63:
            n += 1
        n += 1

        if n >= 63:
            return

        # Look up the field width
        field_width = 0
        if not self.item_type in field_widths:
            print('unexpected item_type is ', self.item_type)
            field_width = 64
        else:
            field_width = self.get_field_width_in_bits()


        # Its a problem is the mask_width is > field_width - some of the bits won't get looked at!?
        mask_width = n-1-mask_start
        if field_width is not None and (mask_width > field_width):
            # N.B. No call, so no line number.
            print(self.filename + ':', self.hf, 'filter=', self.filter, self.item_type, 'so field_width=', field_width,
                  'but mask is', mask, 'which is', mask_width, 'bits wide!')
            global warnings_found
            warnings_found += 1
        # Now, any more zero set bits are an error!
        if self.filter in known_non_contiguous_fields or self.filter.startswith('rtpmidi'):
            # Don't report if we know this one is Ok.
            # TODO: also exclude items that are used as root in add_bitmask() calls?
            return
        while n <= 63:
            if self.check_bit(self.mask_value, n):
                print('Warning:', self.filename, self.hf, 'filter=', self.filter, ' - mask with non-contiguous bits',
                      mask, '(', hex(self.mask_value), ')')
                warnings_found += 1
                return
            n += 1

    def get_field_width_in_bits(self):
        if self.item_type == 'FT_BOOLEAN':
            if self.display == 'NULL':
                return 8  # i.e. 1 byte
            elif self.display == 'BASE_NONE':
                return 8
            elif self.display == 'SEP_DOT':   # from proto.h, only meant for FT_BYTES
                return 64
            else:
                try:
                    # For FT_BOOLEAN, modifier is just numerical number of bits. Round up to next nibble.
                    return int((int(self.display) + 3)/4)*4
                except:
                    return None
        else:
            if self.item_type in field_widths:
                # Lookup fixed width for this type
                return field_widths[self.item_type]
            else:
                return None

    def check_num_digits(self, mask):
        if mask.startswith('0x') and len(mask) > 3:
            global warnings_found
            global errors_found

            width_in_bits = self.get_field_width_in_bits()
            # Warn if odd number of digits.  TODO: only if >= 5?
            if len(mask) % 2  and self.item_type != 'FT_BOOLEAN':
                print('Warning:', self.filename, self.hf, 'filter=', self.filter, ' - mask has odd number of digits', mask,
                      'expected max for', self.item_type, 'is', int(width_in_bits/4))
                warnings_found += 1

            if self.item_type in field_widths:
                # Longer than it should be?
                if width_in_bits is None:
                    return
                if len(mask)-2 > width_in_bits/4:
                    extra_digits = mask[2:2+(len(mask)-2 - int(width_in_bits/4))]
                    # Its definitely an error if any of these are non-zero, as they won't have any effect!
                    if extra_digits != '0'*len(extra_digits):
                        print('Error:', self.filename, self.hf, 'filter=', self.filter, 'mask', self.mask, "with len is", len(mask)-2,
                              "but type", self.item_type, " indicates max of", int(width_in_bits/4),
                              "and extra digits are non-zero (" + extra_digits + ")")
                        errors_found += 1
                    else:
                        # Has extra leading zeros, still confusing, so warn.
                        print('Warning:', self.filename, self.hf, 'filter=', self.filter, 'mask', self.mask, "with len", len(mask)-2,
                              "but type", self.item_type, " indicates max of", int(width_in_bits/4))
                        warnings_found += 1

                # Strict/fussy check - expecting mask length to match field width exactly!
                # Currently only doing for FT_BOOLEAN, and don't expect to be in full for 64-bit fields!
                if self.mask_exact_width:
                    ideal_mask_width = int(width_in_bits/4)
                    if self.item_type == 'FT_BOOLEAN' and ideal_mask_width < 16 and len(mask)-2 != ideal_mask_width:
                        print('Warning:', self.filename, self.hf, 'filter=', self.filter, 'mask', self.mask, "with len", len(mask)-2,
                                "but type", self.item_type, "|", self.display,  " indicates should be", int(width_in_bits/4))
                        warnings_found += 1

            else:
                # This type shouldn't have a mask set at all.
                print('Warning:', self.filename, self.hf, 'filter=', self.filter, ' - item has type', self.item_type, 'but mask set:', mask)
                warnings_found += 1

    def check_digits_all_zeros(self, mask):
        if mask.startswith('0x') and len(mask) > 3:
            if mask[2:] == '0'*(len(mask)-2):
                print('Warning:', self.filename, self.hf, 'filter=', self.filter, ' - item mask has all zeros - this is confusing! :', '"' + mask + '"')
                global warnings_found
                warnings_found += 1

    # A mask where all bits are set should instead be 0.
    # Exceptions might be where:
    # - in add_bitmask()
    # - represents flags, but dissector is not yet decoding them
    def check_full_mask(self, mask, field_arrays):
        if self.item_type == "FT_BOOLEAN":
            return
        if self.label.lower().find('mask') != -1 or self.label.lower().find('flag') != -1 or self.label.lower().find('bitmap') != -1:
            return
        if mask.startswith('0x') and len(mask) > 3:
            width_in_bits = self.get_field_width_in_bits()
            if not width_in_bits:
                return
            num_digits = int(width_in_bits / 4)
            if num_digits is None:
                return
            if mask[2:] == 'f'*num_digits   or   mask[2:] == 'F'*num_digits:
                # Don't report if appears in a 'fields' array
                for arr in field_arrays:
                    list = field_arrays[arr][0]
                    if self.hf in list:
                        # These need to have a mask - don't judge for being 0
                        return

                print('Note:', self.filename, self.hf, 'filter=', self.filter, " - mask is all set - if only want value (rather than bits), set 0 instead? :", '"' + mask + '"')

    # An item that appears in a bitmask set, needs to have a non-zero mask.
    def check_mask_if_in_field_array(self, mask, field_arrays):
        # Work out if this item appears in a field array
        found = False
        array_name = None
        for arr in field_arrays:
            list = field_arrays[arr][0]
            if self.hf in list:
                # These need to have a mask - don't judge for being 0
                found = True
                array_name = arr
                break

        if found:
            # It needs to have a non-zero mask.
            if self.mask_read and self.mask_value == 0:
                print('Error:', self.filename, self.hf, 'is in fields array', arr, 'but has a zero mask - this is not allowed')
                global errors_found
                errors_found += 1



    # Return True if appears to be a match
    def check_label_vs_filter(self, reportError=True, reportNumericalMismatch=True):
        global warnings_found

        last_filter = self.filter.split('.')[-1]
        last_filter_orig = last_filter
        last_filter = last_filter.replace('-', '')
        last_filter = last_filter.replace('_', '')
        last_filter = last_filter.replace(' ', '')
        label = self.label
        label_orig = label
        label = label.replace(' ', '')
        label = label.replace('-', '')
        label = label.replace('_', '')
        label = label.replace('(', '')
        label = label.replace(')', '')
        label = label.replace('/', '')
        label = label.replace("'", '')


        # OK if filter is abbrev of label.
        label_words = self.label.split(' ')
        label_words = [w for w in label_words if len(w)]
        if len(label_words) == len(last_filter):
            #print(label_words)
            abbrev_letters = [w[0] for w in label_words]
            abbrev = ''.join(abbrev_letters)
            if abbrev.lower() == last_filter.lower():
                return True

        # If both have numbers, they should probably match!
        label_numbers =  [int(n) for n in re.findall(r'\d+', label_orig)]
        filter_numbers = [int(n) for n in re.findall(r'\d+', last_filter_orig)]
        if len(label_numbers) == len(filter_numbers) and label_numbers != filter_numbers:
            if reportNumericalMismatch:
                print('Note:', self.filename, self.hf, 'label="' + self.label + '" has different **numbers** from  filter="' + self.filter + '"')
                print(label_numbers, filter_numbers)
            return False

        # If they match after trimming number from filter, they should match.
        if label.lower() == last_filter.lower().rstrip("0123456789"):
            return True

        # Are they just different?
        if label.lower().find(last_filter.lower()) == -1:
            if reportError:
                print('Warning:', self.filename, self.hf, 'label="' + self.label + '" does not seem to match filter="' + self.filter + '"')
                warnings_found += 1
            return False

        return True


class CombinedCallsCheck:
    def __init__(self, file, apiChecks):
        self.file = file
        self.apiChecks = apiChecks
        self.get_all_calls()

    def get_all_calls(self):
        self.all_calls = []
        # Combine calls into one list.
        for check in self.apiChecks:
            self.all_calls += check.calls

        # Sort by line number.
        self.all_calls.sort(key=lambda x:x.line_number)

    def check_consecutive_item_calls(self):
        lines = open(self.file, 'r', encoding="utf8").read().splitlines()

        prev = None
        for call in self.all_calls:

            # These names commonly do appear together..
            if name_has_one_of(call.hf_name, [ 'unused', 'unknown', 'spare', 'reserved', 'default']):
                return

            if prev and call.hf_name == prev.hf_name:
                # More compelling if close together..
                if call.line_number>prev.line_number and call.line_number-prev.line_number <= 4:
                    scope_different = False
                    for l in range(prev.line_number, call.line_number-1):
                        if lines[l].find('{') != -1 or lines[l].find('}') != -1 or lines[l].find('else') != -1 or lines[l].find('break;') != -1 or lines[l].find('if ') != -1:
                            scope_different = True
                            break
                    # Also more compelling if check for and scope changes { } in lines in-between?
                    if not scope_different:
                        print('Warning:', f + ':' + str(call.line_number),
                              call.hf_name + ' called consecutively at line', call.line_number, '- previous at', prev.line_number)
                        global warnings_found
                        warnings_found += 1
            prev = call




# These are APIs in proto.c that check a set of types at runtime and can print '.. is not of type ..' to the console
# if the type is not suitable.
apiChecks = []
apiChecks.append(APICheck('proto_tree_add_item_ret_uint', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32'}, positive_length=True))
apiChecks.append(APICheck('proto_tree_add_item_ret_int', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'}))
apiChecks.append(APICheck('ptvcursor_add_ret_uint', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32'}, positive_length=True))
apiChecks.append(APICheck('ptvcursor_add_ret_int', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'}, positive_length=True))
apiChecks.append(APICheck('ptvcursor_add_ret_string', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'}))
apiChecks.append(APICheck('ptvcursor_add_ret_boolean', { 'FT_BOOLEAN'}, positive_length=True))
apiChecks.append(APICheck('proto_tree_add_item_ret_uint64', { 'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64'}, positive_length=True))
apiChecks.append(APICheck('proto_tree_add_item_ret_int64', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'}, positive_length=True))
apiChecks.append(APICheck('proto_tree_add_item_ret_boolean', { 'FT_BOOLEAN'}, positive_length=True))
apiChecks.append(APICheck('proto_tree_add_item_ret_string_and_length', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'}))
apiChecks.append(APICheck('proto_tree_add_item_ret_display_string_and_length', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING',
                                                                                 'FT_STRINGZPAD', 'FT_STRINGZTRUNC', 'FT_BYTES', 'FT_UINT_BYTES'}))
apiChecks.append(APICheck('proto_tree_add_item_ret_time_string', { 'FT_ABSOLUTE_TIME', 'FT_RELATIVE_TIME'}))
apiChecks.append(APICheck('proto_tree_add_uint', {  'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_FRAMENUM'}))
apiChecks.append(APICheck('proto_tree_add_uint_format_value', {  'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_FRAMENUM'}))
apiChecks.append(APICheck('proto_tree_add_uint_format', {  'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_FRAMENUM'}))
apiChecks.append(APICheck('proto_tree_add_uint64', { 'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64', 'FT_FRAMENUM'}))
apiChecks.append(APICheck('proto_tree_add_int64', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'}))
apiChecks.append(APICheck('proto_tree_add_int64_format_value', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'}))
apiChecks.append(APICheck('proto_tree_add_int64_format', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'}))
apiChecks.append(APICheck('proto_tree_add_int', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'}))
apiChecks.append(APICheck('proto_tree_add_int_format_value', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'}))
apiChecks.append(APICheck('proto_tree_add_int_format', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'}))
apiChecks.append(APICheck('proto_tree_add_boolean', { 'FT_BOOLEAN'}))
apiChecks.append(APICheck('proto_tree_add_boolean64', { 'FT_BOOLEAN'}))
apiChecks.append(APICheck('proto_tree_add_float', { 'FT_FLOAT'}))
apiChecks.append(APICheck('proto_tree_add_float_format', { 'FT_FLOAT'}))
apiChecks.append(APICheck('proto_tree_add_float_format_value', { 'FT_FLOAT'}))
apiChecks.append(APICheck('proto_tree_add_double', { 'FT_DOUBLE'}))
apiChecks.append(APICheck('proto_tree_add_double_format', { 'FT_DOUBLE'}))
apiChecks.append(APICheck('proto_tree_add_double_format_value', { 'FT_DOUBLE'}))
apiChecks.append(APICheck('proto_tree_add_string', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'}))
apiChecks.append(APICheck('proto_tree_add_string_format', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'}))
apiChecks.append(APICheck('proto_tree_add_string_format_value', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'}))
apiChecks.append(APICheck('proto_tree_add_guid', { 'FT_GUID'}))
apiChecks.append(APICheck('proto_tree_add_oid', { 'FT_OID'}))
apiChecks.append(APICheck('proto_tree_add_none_format', { 'FT_NONE'}))
apiChecks.append(APICheck('proto_tree_add_item_ret_varint', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32', 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64',
                                                              'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_FRAMENUM',
                                                              'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64',}))
apiChecks.append(APICheck('proto_tree_add_boolean_bits_format_value', { 'FT_BOOLEAN'}))
apiChecks.append(APICheck('proto_tree_add_boolean_bits_format_value64', { 'FT_BOOLEAN'}))
apiChecks.append(APICheck('proto_tree_add_ascii_7bits_item', { 'FT_STRING'}))
# TODO: positions are different, and takes 2 hf_fields..
#apiChecks.append(APICheck('proto_tree_add_checksum', { 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32'}))
apiChecks.append(APICheck('proto_tree_add_int64_bits_format_value', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'}))

# TODO: add proto_tree_add_bytes_item, proto_tree_add_time_item ?

bitmask_types = { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32',
                  'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32',
                  'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64',
                  'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64',
                   'FT_BOOLEAN'}
apiChecks.append(APICheck('proto_tree_add_bitmask', bitmask_types))
apiChecks.append(APICheck('proto_tree_add_bitmask_tree', bitmask_types))
apiChecks.append(APICheck('proto_tree_add_bitmask_ret_uint64', bitmask_types))
apiChecks.append(APICheck('proto_tree_add_bitmask_with_flags', bitmask_types))
apiChecks.append(APICheck('proto_tree_add_bitmask_with_flags_ret_uint64', bitmask_types))
apiChecks.append(APICheck('proto_tree_add_bitmask_value', bitmask_types))
apiChecks.append(APICheck('proto_tree_add_bitmask_value_with_flags', bitmask_types))
apiChecks.append(APICheck('proto_tree_add_bitmask_len', bitmask_types))
# N.B., proto_tree_add_bitmask_list does not have a root item, just a subtree...

add_bits_types = { 'FT_CHAR', 'FT_BOOLEAN',
                   'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64',
                   'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32', 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64',
                    'FT_BYTES'}
apiChecks.append(APICheck('proto_tree_add_bits_item',    add_bits_types))
apiChecks.append(APICheck('proto_tree_add_bits_ret_val', add_bits_types))

# TODO: doesn't even have an hf_item !
#apiChecks.append(APICheck('proto_tree_add_bitmask_text', bitmask_types))

# Check some ptvcuror calls too.
apiChecks.append(APICheck('ptvcursor_add_ret_uint', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32'}))
apiChecks.append(APICheck('ptvcursor_add_ret_int', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'}))
apiChecks.append(APICheck('ptvcursor_add_ret_boolean', { 'FT_BOOLEAN'}))


# Also try to check proto_tree_add_item() calls (for length)
apiChecks.append(ProtoTreeAddItemCheck())
apiChecks.append(ProtoTreeAddItemCheck(True)) # for ptvcursor_add()



def removeComments(code_string):
    code_string = re.sub(re.compile(r"/\*.*?\*/",re.DOTALL ) ,"" , code_string) # C-style comment
    code_string = re.sub(re.compile(r"//.*?\n" ) ,"" , code_string)             # C++-style comment
    code_string = re.sub(re.compile(r"#if 0.*?#endif",re.DOTALL ) ,"" , code_string) # Ignored region

    return code_string

# Test for whether the given file was automatically generated.
def isGeneratedFile(filename):
    # Check file exists - e.g. may have been deleted in a recent commit.
    if not os.path.exists(filename):
        return False

    # Open file
    f_read = open(os.path.join(filename), 'r', encoding="utf8")
    lines_tested = 0
    for line in f_read:
        # The comment to say that its generated is near the top, so give up once
        # get a few lines down.
        if lines_tested > 10:
            f_read.close()
            return False
        if (line.find('Generated automatically') != -1 or
            line.find('Generated Automatically') != -1 or
            line.find('Autogenerated from') != -1 or
            line.find('is autogenerated') != -1 or
            line.find('automatically generated by Pidl') != -1 or
            line.find('Created by: The Qt Meta Object Compiler') != -1 or
            line.find('This file was generated') != -1 or
            line.find('This filter was automatically generated') != -1 or
            line.find('This file is auto generated, do not edit!') != -1):

            f_read.close()
            return True
        lines_tested = lines_tested + 1

    # OK, looks like a hand-written file!
    f_read.close()
    return False


def find_macros(filename):
    macros = {}
    with open(filename, 'r', encoding="utf8") as f:
        contents = f.read()
        # Remove comments so as not to trip up RE.
        contents = removeComments(contents)

        matches = re.finditer( r'#define\s*([A-Z0-9_]*)\s*([0-9xa-fA-F]*)\n', contents)
        for m in matches:
            # Store this mapping.
            macros[m.group(1)] = m.group(2)
    return macros


# Look for hf items (i.e. full item to be registered) in a dissector file.
def find_items(filename, macros, value_strings, range_strings,
               check_mask=False, mask_exact_width=False, check_label=False, check_consecutive=False):
    is_generated = isGeneratedFile(filename)
    items = {}
    with open(filename, 'r', encoding="utf8") as f:
        contents = f.read()
        # Remove comments so as not to trip up RE.
        contents = removeComments(contents)

        # N.B. re extends all the way to HFILL to avoid greedy matching
        # TODO: fix a problem where re can't cope with mask that involve a macro with commas in it...
        matches = re.finditer( r'.*\{\s*\&(hf_[a-z_A-Z0-9]*)\s*,\s*{\s*\"(.*?)\"\s*,\s*\"(.*?)\"\s*,\s*(.*?)\s*,\s*([0-9A-Z_\|\s]*?)\s*,\s*(.*?)\s*,\s*(.*?)\s*,\s*([a-zA-Z0-9\W\s_\u00f6\u00e4]*?)\s*,\s*HFILL', contents)
        for m in matches:
            # Store this item.
            hf = m.group(1)

            blurb = m.group(8)
            if blurb.startswith('"'):
                blurb = blurb[1:-1]

            items[hf] = Item(filename, hf, filter=m.group(3), label=m.group(2), item_type=m.group(4),
                             display=m.group(5),
                             strings=m.group(6),
                             macros=macros,
                             value_strings=value_strings,
                             range_strings=range_strings,
                             mask=m.group(7),
                             blurb=blurb,
                             check_mask=check_mask,
                             mask_exact_width=mask_exact_width,
                             check_label=check_label,
                             check_consecutive=(not is_generated and check_consecutive))
    return items


# Looking for args to ..add_bitmask_..() calls that are not NULL-terminated or  have repeated items.
# TODO: some dissectors have similar-looking hf arrays for other reasons, so need to cross-reference with
# the 6th arg of ..add_bitmask_..() calls...
# TODO: return items (rather than local checks) from here so can be checked against list of calls for given filename
def find_field_arrays(filename, all_fields, all_hf):
    field_entries = {}
    global warnings_found
    with open(filename, 'r', encoding="utf8") as f:
        contents = f.read()
        # Remove comments so as not to trip up RE.
        contents = removeComments(contents)

        # Find definition of hf array
        matches = re.finditer(r'static\s*g?int\s*\*\s*const\s+([a-zA-Z0-9_]*)\s*\[\]\s*\=\s*\{([a-zA-Z0-9,_\&\s]*)\}', contents)
        for m in matches:
            name = m.group(1)
            # Ignore if not used in a call to an _add_bitmask_ API
            if not name in all_fields:
                continue

            fields_text = m.group(2)
            fields_text = fields_text.replace('&', '')
            fields_text = fields_text.replace(',', '')

            # Get list of each hf field in the array
            fields = fields_text.split()

            if fields[0].startswith('ett_'):
                continue
            if fields[-1].find('NULL') == -1 and fields[-1] != '0':
                print('Warning:', filename, name, 'is not NULL-terminated - {', ', '.join(fields), '}')
                warnings_found += 1
                continue

            # Do any hf items reappear?
            seen_fields = set()
            for f in fields:
                if f in seen_fields:
                    print(filename, name, f, 'already added!')
                    warnings_found += 1
                seen_fields.add(f)

            # Check for duplicated flags among entries..
            combined_mask = 0x0
            for f in fields[0:-1]:
                if f in all_hf:
                    new_mask = all_hf[f].mask_value
                    if new_mask & combined_mask:
                        print('Warning:', filename, name, 'has overlapping mask - {', ', '.join(fields), '} combined currently', hex(combined_mask), f, 'adds', hex(new_mask))
                        warnings_found += 1
                    combined_mask |= new_mask

            # Make sure all entries have the same width
            set_field_width = None
            for f in fields[0:-1]:
                if f in all_hf:
                    new_field_width = all_hf[f].get_field_width_in_bits()
                    if set_field_width is not None and new_field_width != set_field_width:
                        # Its not uncommon for fields to be used in multiple sets, some of which can be different widths..
                        print('Note:', filename, name, 'set items not all same width - {', ', '.join(fields), '} seen', set_field_width, 'now', new_field_width)
                    set_field_width = new_field_width

            # Add entry to table
            field_entries[name] = (fields[0:-1], combined_mask)

    return field_entries

def find_item_declarations(filename):
    items = set()

    with open(filename, 'r', encoding="utf8") as f:
        lines = f.read().splitlines()
        p = re.compile(r'^static int (hf_[a-zA-Z0-9_]*)\s*\=\s*-1;')
        for line in lines:
            m = p.search(line)
            if m:
                items.add(m.group(1))
    return items

def find_item_extern_declarations(filename):
    items = set()
    with open(filename, 'r', encoding="utf8") as f:
        lines = f.read().splitlines()
        p = re.compile(r'^\s*(hf_[a-zA-Z0-9_]*)\s*\=\s*proto_registrar_get_id_byname\s*\(')
        for line in lines:
            m = p.search(line)
            if m:
                items.add(m.group(1))
    return items


def is_dissector_file(filename):
    p = re.compile(r'.*(packet|file)-.*\.c$')
    return p.match(filename)


def findDissectorFilesInFolder(folder, recursive=False):
    dissector_files = []

    if recursive:
        for root, subfolders, files in os.walk(folder):
            for f in files:
                if should_exit:
                    return
                f = os.path.join(root, f)
                dissector_files.append(f)
    else:
        for f in sorted(os.listdir(folder)):
            if should_exit:
                return
            filename = os.path.join(folder, f)
            dissector_files.append(filename)

    return [x for x in filter(is_dissector_file, dissector_files)]



# Run checks on the given dissector file.
def checkFile(filename, check_mask=False, mask_exact_width=False, check_label=False, check_consecutive=False,
              check_missing_items=False, check_bitmask_fields=False, label_vs_filter=False, extra_value_string_checks=False):
    # Check file exists - e.g. may have been deleted in a recent commit.
    if not os.path.exists(filename):
        print(filename, 'does not exist!')
        return

    # Find simple macros so can substitute into items and calls.
    macros = find_macros(filename)

    # Find (and sanity-check) value_strings
    value_strings = findValueStrings(filename, macros, do_extra_checks=extra_value_string_checks)
    if extra_value_string_checks:
        for name in value_strings:
            value_strings[name].extraChecks()

    # Find (and sanity-check) range_strings
    range_strings = findRangeStrings(filename, macros, do_extra_checks=extra_value_string_checks)
    if extra_value_string_checks:
        for name in range_strings:
            range_strings[name].extraChecks()



    # Find important parts of items.
    items_defined = find_items(filename, macros, value_strings, range_strings,
                               check_mask, mask_exact_width, check_label, check_consecutive)
    items_extern_declared = {}

    items_declared = {}
    if check_missing_items:
        items_declared = find_item_declarations(filename)
        items_extern_declared = find_item_extern_declarations(filename)

    fields = set()

    # Get 'fields' out of calls
    for c in apiChecks:
        c.find_calls(filename, macros)
        for call in c.calls:
            # From _add_bitmask() calls
            if call.fields:
                fields.add(call.fields)

    # Checking for lists of fields for add_bitmask calls
    field_arrays = {}
    if check_bitmask_fields:
        field_arrays = find_field_arrays(filename, fields, items_defined)

    if check_mask and check_bitmask_fields:
        for i in items_defined:
            item = items_defined[i]
            item.check_full_mask(item.mask, field_arrays)
            item.check_mask_if_in_field_array(item.mask, field_arrays)

    # Now actually check the calls
    for c in apiChecks:
        c.check_against_items(items_defined, items_declared, items_extern_declared, check_missing_items, field_arrays)


    if label_vs_filter:
        matches = 0
        for hf in items_defined:
            if items_defined[hf].check_label_vs_filter(reportError=False, reportNumericalMismatch=True):
                matches += 1

        # Only checking if almost every field does match.
        checking = len(items_defined) and matches<len(items_defined) and ((matches / len(items_defined)) > 0.93)
        if checking:
            print(filename, ':', matches, 'label-vs-filter matches of out of', len(items_defined), 'so reporting mismatches')
            for hf in items_defined:
                items_defined[hf].check_label_vs_filter(reportError=True, reportNumericalMismatch=False)



#################################################################
# Main logic.

# command-line args.  Controls which dissector files should be checked.
# If no args given, will just scan epan/dissectors folder.
parser = argparse.ArgumentParser(description='Check calls in dissectors')
parser.add_argument('--file', action='append',
                    help='specify individual dissector file to test')
parser.add_argument('--folder', action='store', default='',
                    help='specify folder to test')
parser.add_argument('--commits', action='store',
                    help='last N commits to check')
parser.add_argument('--open', action='store_true',
                    help='check open files')
parser.add_argument('--mask', action='store_true',
                   help='when set, check mask field too')
parser.add_argument('--mask-exact-width', action='store_true',
                   help='when set, check width of mask against field width')
parser.add_argument('--label', action='store_true',
                   help='when set, check label field too')
parser.add_argument('--consecutive', action='store_true',
                    help='when set, copy copy/paste errors between consecutive items')
parser.add_argument('--missing-items', action='store_true',
                    help='when set, look for used items that were never registered')
parser.add_argument('--check-bitmask-fields', action='store_true',
                    help='when set, attempt to check arrays of hf items passed to add_bitmask() calls')
parser.add_argument('--label-vs-filter', action='store_true',
                    help='when set, check whether label matches last part of filter')
parser.add_argument('--extra-value-string-checks', action='store_true',
                    help='when set, do extra checks on parsed value_strings')
parser.add_argument('--all-checks', action='store_true',
                    help='when set, apply all checks to selected files')


args = parser.parse_args()

# Turn all checks on.
if args.all_checks:
    args.mask = True
    args.mask_exact_width = True
    args.consecutive = True
    args.check_bitmask_fields = True
    #args.label = True
    args.label_vs_filter = True
    args.extra_value_string_checks

if args.check_bitmask_fields:
    args.mask = True


# Get files from wherever command-line args indicate.
files = []
if args.file:
    # Add specified file(s)
    for f in args.file:
        if not os.path.isfile(f):
            print('Chosen file', f, 'does not exist.')
            exit(1)
        else:
            files.append(f)
elif args.folder:
    # Add all files from a given folder.
    folder = args.folder
    if not os.path.isdir(folder):
        print('Folder', folder, 'not found!')
        exit(1)
    # Find files from folder.
    print('Looking for files in', folder)
    files = findDissectorFilesInFolder(folder, recursive=True)
elif args.commits:
    # Get files affected by specified number of commits.
    command = ['git', 'diff', '--name-only', '--diff-filter=d', 'HEAD~' + args.commits]
    files = [f.decode('utf-8')
             for f in subprocess.check_output(command).splitlines()]
    # Will examine dissector files only
    files = list(filter(lambda f : is_dissector_file(f), files))
elif args.open:
    # Unstaged changes.
    command = ['git', 'diff', '--name-only', '--diff-filter=d']
    files = [f.decode('utf-8')
             for f in subprocess.check_output(command).splitlines()]
    # Only interested in dissector files.
    files = list(filter(lambda f : is_dissector_file(f), files))
    # Staged changes.
    command = ['git', 'diff', '--staged', '--name-only', '--diff-filter=d']
    files_staged = [f.decode('utf-8')
                    for f in subprocess.check_output(command).splitlines()]
    # Only interested in dissector files.
    files_staged = list(filter(lambda f : is_dissector_file(f), files_staged))
    for f in files_staged:
        if not f in files:
            files.append(f)
else:
    # Find all dissector files.
    files  = findDissectorFilesInFolder(os.path.join('epan', 'dissectors'))
    files += findDissectorFilesInFolder(os.path.join('plugins', 'epan'), recursive=True)


# If scanning a subset of files, list them here.
print('Examining:')
if args.file or args.commits or args.open:
    if files:
        print(' '.join(files), '\n')
    else:
        print('No files to check.\n')
else:
    print('All dissector modules\n')


# Now check the files.
for f in files:
    if should_exit:
        exit(1)
    checkFile(f, check_mask=args.mask, mask_exact_width=args.mask_exact_width, check_label=args.label,
              check_consecutive=args.consecutive, check_missing_items=args.missing_items,
              check_bitmask_fields=args.check_bitmask_fields, label_vs_filter=args.label_vs_filter,
              extra_value_string_checks=args.extra_value_string_checks)

    # Do checks against all calls.
    if args.consecutive:
        combined_calls = CombinedCallsCheck(f, apiChecks)
        # This hasn't really found any issues, but shows lots of false positives (and are difficult to investigate)
        #combined_calls.check_consecutive_item_calls()


# Show summary.
print(warnings_found, 'warnings')
if errors_found:
    print(errors_found, 'errors')
    exit(1)