#!/usr/bin/env python3 # Wireshark - Network traffic analyzer # By Gerald Combs # Copyright 1998 Gerald Combs # # SPDX-License-Identifier: GPL-2.0-or-later import os import re import argparse import signal import subprocess # This utility scans the dissector code for various issues. # TODO: # - Create maps from type -> display types for hf items (see display (FIELDDISPLAY)) in docs/README.dissector # Try to exit soon after Ctrl-C is pressed. should_exit = False def signal_handler(sig, frame): global should_exit should_exit = True print('You pressed Ctrl+C - exiting') signal.signal(signal.SIGINT, signal_handler) warnings_found = 0 errors_found = 0 def name_has_one_of(name, substring_list): for word in substring_list: if name.lower().find(word) != -1: return True return False # An individual call to an API we are interested in. # Used by APICheck below. class Call: def __init__(self, hf_name, macros, line_number=None, length=None, fields=None): self.hf_name = hf_name self.line_number = line_number self.fields = fields self.length = None if length: try: self.length = int(length) except: if length.isupper(): if length in macros: try: self.length = int(macros[length]) except: pass pass # These are variable names that have been seen to be used in calls.. common_hf_var_names = { 'hf_index', 'hf_item', 'hf_idx', 'hf_x', 'hf_id', 'hf_cookie', 'hf_flag', 'hf_dos_time', 'hf_dos_date', 'hf_value', 'hf_num', 'hf_cause_value', 'hf_uuid', 'hf_endian', 'hf_ip', 'hf_port', 'hf_suff', 'hf_string', 'hf_uint', 'hf_tag', 'hf_type', 'hf_hdr', 'hf_field', 'hf_opcode', 'hf_size', 'hf_entry', 'field' } item_lengths = {} item_lengths['FT_CHAR'] = 1 item_lengths['FT_UINT8'] = 1 item_lengths['FT_INT8'] = 1 item_lengths['FT_UINT16'] = 2 item_lengths['FT_INT16'] = 2 item_lengths['FT_UINT24'] = 3 item_lengths['FT_INT24'] = 3 item_lengths['FT_UINT32'] = 4 item_lengths['FT_INT32'] = 4 item_lengths['FT_UINT40'] = 5 item_lengths['FT_INT40'] = 5 item_lengths['FT_UINT48'] = 6 item_lengths['FT_INT48'] = 6 item_lengths['FT_UINT56'] = 7 item_lengths['FT_INT56'] = 7 item_lengths['FT_UINT64'] = 8 item_lengths['FT_INT64'] = 8 item_lengths['FT_ETHER'] = 6 # TODO: other types... # A check for a particular API function. class APICheck: def __init__(self, fun_name, allowed_types, positive_length=False): self.fun_name = fun_name self.allowed_types = allowed_types self.positive_length = positive_length self.calls = [] if fun_name.startswith('ptvcursor'): # RE captures function name + 1st 2 args (always ptvc + hfindex) self.p = re.compile('[^\n]*' + self.fun_name + '\s*\(([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+)') elif fun_name.find('add_bitmask') == -1: # Normal case. # RE captures function name + 1st 2 args (always tree + hfindex + length) self.p = re.compile('[^\n]*' + self.fun_name + '\s*\(([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+),\s*[a-zA-Z0-9_]+,\s*[a-zA-Z0-9_]+,\s*([a-zA-Z0-9_]+)') else: # _add_bitmask functions. # RE captures function name + 1st + 4th args (always tree + hfindex) # 6th arg is 'fields' self.p = re.compile('[^\n]*' + self.fun_name + '\s*\(([a-zA-Z0-9_]+),\s*[a-zA-Z0-9_]+,\s*[a-zA-Z0-9_]+,\s*([a-zA-Z0-9_]+)\s*,\s*[a-zA-Z0-9_]+\s*,\s*([a-zA-Z0-9_]+)\s*,') self.file = None self.mask_allowed = True if fun_name.find('proto_tree_add_bits_') != -1: self.mask_allowed = False def find_calls(self, file, macros): self.file = file self.calls = [] with open(file, 'r', encoding="utf8") as f: contents = f.read() lines = contents.splitlines() total_lines = len(lines) for line_number,line in enumerate(lines): # Want to check this, and next few lines to_check = lines[line_number-1] + '\n' # Nothing to check if function name isn't in it if to_check.find(self.fun_name) != -1: # Ok, add the next file lines before trying RE for i in range(1, 4): if to_check.find(';') != -1: break elif line_number+i < total_lines: to_check += (lines[line_number-1+i] + '\n') m = self.p.search(to_check) if m: fields = None length = None if self.fun_name.find('add_bitmask') != -1: fields = m.group(3) else: if self.p.groups == 3: length = m.group(3) # Add call. We have length if re had 3 groups. num_groups = self.p.groups self.calls.append(Call(m.group(2), macros, line_number=line_number, length=length, fields=fields)) # Return true if bit position n is set in value. def check_bit(self, value, n): return (value & (0x1 << n)) != 0 def does_mask_cover_value(self, mask, value): # Walk past any l.s. 0 bits in value n = 0 mask_start = n # Walk through any bits that are set and check they are in mask while self.check_bit(value, n) and n <= 63: if not self.check_bit(mask, n): return False n += 1 return True def check_against_items(self, items_defined, items_declared, items_declared_extern, check_missing_items=False, field_arrays=None): global errors_found global warnings_found for call in self.calls: # Check lengths, but for now only for APIs that have length in bytes. if self.fun_name.find('add_bits') == -1 and call.hf_name in items_defined: if call.length and items_defined[call.hf_name].item_type in item_lengths: if item_lengths[items_defined[call.hf_name].item_type] < call.length: print('Warning:', self.file + ':' + str(call.line_number), self.fun_name + ' called for', call.hf_name, ' - ', 'item type is', items_defined[call.hf_name].item_type, 'but call has len', call.length) warnings_found += 1 # Needs a +ve length if self.positive_length and call.length != None: if call.length != -1 and call.length <= 0: print('Error: ' + self.fun_name + '(.., ' + call.hf_name + ', ...) called at ' + self.file + ':' + str(call.line_number) + ' with length ' + str(call.length) + ' - must be > 0 or -1') errors_found += 1 if call.hf_name in items_defined: # Is type allowed? if not items_defined[call.hf_name].item_type in self.allowed_types: print('Error: ' + self.fun_name + '(.., ' + call.hf_name + ', ...) called at ' + self.file + ':' + str(call.line_number) + ' with type ' + items_defined[call.hf_name].item_type) print(' (allowed types are', self.allowed_types, ')\n') errors_found += 1 # No mask allowed if not self.mask_allowed and items_defined[call.hf_name].mask_value != 0: print('Error: ' + self.fun_name + '(.., ' + call.hf_name + ', ...) called at ' + self.file + ':' + str(call.line_number) + ' with mask ' + items_defined[call.hf_name].mask + ' (must be zero!)\n') errors_found += 1 if self.fun_name.find('add_bitmask') != -1 and call.hf_name in items_defined and field_arrays: if call.fields in field_arrays: if (items_defined[call.hf_name].mask_value and field_arrays[call.fields][1] != 0 and items_defined[call.hf_name].mask_value != field_arrays[call.fields][1]): # TODO: only really a problem if bit is set in array but not in top-level item? if not self.does_mask_cover_value(items_defined[call.hf_name].mask_value, field_arrays[call.fields][1]): print('Warning:', self.file, call.hf_name, call.fields, "masks don't match. root=", items_defined[call.hf_name].mask, "array has", hex(field_arrays[call.fields][1])) warnings_found += 1 if check_missing_items: if call.hf_name in items_declared and not call.hf_name in items_declared_extern: #not in common_hf_var_names: print('Warning:', self.file + ':' + str(call.line_number), self.fun_name + ' called for "' + call.hf_name + '"', ' - but no item found') warnings_found += 1 # Specialization of APICheck for add_item() calls class ProtoTreeAddItemCheck(APICheck): def __init__(self, ptv=None): # RE will capture whole call. if not ptv: # proto_item * # proto_tree_add_item(proto_tree *tree, int hfindex, tvbuff_t *tvb, # const gint start, gint length, const guint encoding) self.fun_name = 'proto_tree_add_item' self.p = re.compile('[^\n]*' + self.fun_name + '\s*\(\s*[a-zA-Z0-9_]+?,\s*([a-zA-Z0-9_]+?),\s*[a-zA-Z0-9_\+\s]+?,\s*[^,.]+?,\s*(.+),\s*([^,.]+?)\);') else: # proto_item * # ptvcursor_add(ptvcursor_t *ptvc, int hfindex, gint length, # const guint encoding) self.fun_name = 'ptvcursor_add' self.p = re.compile('[^\n]*' + self.fun_name + '\s*\([^,.]+?,\s*([^,.]+?),\s*([^,.]+?),\s*([a-zA-Z0-9_\-\>]+)') def find_calls(self, file, macros): self.file = file self.calls = [] with open(file, 'r', encoding="utf8") as f: contents = f.read() lines = contents.splitlines() total_lines = len(lines) for line_number,line in enumerate(lines): # Want to check this, and next few lines to_check = lines[line_number-1] + '\n' # Nothing to check if function name isn't in it fun_idx = to_check.find(self.fun_name) if fun_idx != -1: # Ok, add the next file lines before trying RE for i in range(1, 5): if to_check.find(';') != -1: break elif line_number+i < total_lines: to_check += (lines[line_number-1+i] + '\n') # Lose anything before function call itself. to_check = to_check[fun_idx:] m = self.p.search(to_check) if m: # Throw out if parens not matched if m.group(0).count('(') != m.group(0).count(')'): continue enc = m.group(3) hf_name = m.group(1) if not enc.startswith('ENC_'): if not enc in { 'encoding', 'enc', 'client_is_le', 'cigi_byte_order', 'endian', 'endianess', 'machine_encoding', 'byte_order', 'bLittleEndian', 'p_mq_parm->mq_str_enc', 'p_mq_parm->mq_int_enc', 'iEnc', 'strid_enc', 'iCod', 'nl_data->encoding', 'argp->info->encoding', 'gquic_info->encoding', 'writer_encoding', 'tds_get_int2_encoding(tds_info)', 'tds_get_int4_encoding(tds_info)', 'tds_get_char_encoding(tds_info)', 'info->encoding', 'item->encoding', 'DREP_ENC_INTEGER(drep)', 'string_encoding', 'item', 'type', 'dvb_enc_to_item_enc(encoding)', 'packet->enc', 'IS_EBCDIC(uCCS) ? ENC_EBCDIC : ENC_ASCII', 'DREP_ENC_INTEGER(hdr->drep)', 'dhcp_uuid_endian', 'payload_le', 'local_encoding', 'big_endian', 'hf_data_encoding', 'IS_EBCDIC(eStr) ? ENC_EBCDIC : ENC_ASCII', 'big_endian ? ENC_BIG_ENDIAN : ENC_LITTLE_ENDIAN', '(skip == 1) ? ENC_BIG_ENDIAN : ENC_LITTLE_ENDIAN', 'pdu_info->sbc', 'pdu_info->mbc', 'seq_info->txt_enc | ENC_NA', 'BASE_SHOW_UTF_8_PRINTABLE', 'dhcp_secs_endian', 'is_mdns ? ENC_UTF_8|ENC_NA : ENC_ASCII|ENC_NA', 'xl_encoding' }: global warnings_found print('Warning:', self.file + ':' + str(line_number), self.fun_name + ' called for "' + hf_name + '"', 'check last/enc param:', enc, '?') warnings_found += 1 self.calls.append(Call(hf_name, macros, line_number=line_number, length=m.group(2))) def check_against_items(self, items_defined, items_declared, items_declared_extern, check_missing_items=False, field_arrays=None): # For now, only complaining if length if call is longer than the item type implies. # # Could also be bugs where the length is always less than the type allows. # Would involve keeping track (in the item) of whether any call had used the full length. global warnings_found for call in self.calls: if call.hf_name in items_defined: if call.length and items_defined[call.hf_name].item_type in item_lengths: if item_lengths[items_defined[call.hf_name].item_type] < call.length: print('Warning:', self.file + ':' + str(call.line_number), self.fun_name + ' called for', call.hf_name, ' - ', 'item type is', items_defined[call.hf_name].item_type, 'but call has len', call.length) warnings_found += 1 elif check_missing_items: if call.hf_name in items_declared and not call.hf_name in items_declared_extern: #not in common_hf_var_names: print('Warning:', self.file + ':' + str(call.line_number), self.fun_name + ' called for "' + call.hf_name + '"', ' - but no item found') warnings_found += 1 ################################################################################################## # This is a set of items (by filter name) where we know that the bitmask is non-contiguous, # but is still believed to be correct. known_non_contiguous_fields = { 'wlan.fixed.capabilities.cfpoll.sta', 'wlan.wfa.ie.wme.qos_info.sta.reserved', 'btrfcomm.frame_type', # https://os.itec.kit.edu/downloads/sa_2006_roehricht-martin_flow-control-in-bluez.pdf 'capwap.control.message_element.ac_descriptor.dtls_policy.r', # RFC 5415 'couchbase.extras.subdoc.flags.reserved', 'wlan.fixed.capabilities.cfpoll.ap', # These are 3 separate bits... 'wlan.wfa.ie.wme.tspec.ts_info.reserved', # matches other fields in same sequence 'zbee_zcl_se.pp.attr.payment_control_configuration.reserved', # matches other fields in same sequence 'zbee_zcl_se.pp.snapshot_payload_cause.reserved', # matches other fields in same sequence 'ebhscr.eth.rsv', # matches other fields in same sequence 'v120.lli', # non-contiguous field (http://www.acacia-net.com/wwwcla/protocol/v120_l2.htm) 'stun.type.class', 'bssgp.csg_id', 'tiff.t6.unused', 'artnet.ip_prog_reply.unused', 'telnet.auth.mod.enc', 'osc.message.midi.bender', 'btle.data_header.rfu', 'stun.type.method', # figure 3 in rfc 5389 'tds.done.status', # covers all bits in bitset 'hf_iax2_video_csub', # RFC 5456, table 8.7 'iax2.video.subclass', 'dnp3.al.ana.int', 'pwcesopsn.cw.lm', 'gsm_a.rr.format_id', # EN 301 503 'siii.mst.phase', # comment in code seems convinced 'xmcp.type.class', 'xmcp.type.method', 'hf_hiqnet_flags', 'hf_hiqnet_flagmask', 'hf_h223_mux_mpl', 'rdp.flags.pkt' } ################################################################################################## field_widths = { 'FT_BOOLEAN' : 64, # TODO: Width depends upon 'display' field 'FT_CHAR' : 8, 'FT_UINT8' : 8, 'FT_INT8' : 8, 'FT_UINT16' : 16, 'FT_INT16' : 16, 'FT_UINT24' : 24, 'FT_INT24' : 24, 'FT_UINT32' : 32, 'FT_INT32' : 32, 'FT_UINT40' : 40, 'FT_INT40' : 40, 'FT_UINT48' : 48, 'FT_INT48' : 48, 'FT_UINT56' : 56, 'FT_INT56' : 56, 'FT_UINT64' : 64, 'FT_INT64' : 64 } # TODO: most of these might as well be strings... def is_ignored_consecutive_filter(filter): ignore_patterns = [ re.compile(r'^elf.sh_type'), re.compile(r'^elf.p_type'), re.compile(r'^btavrcp.pdu_id'), re.compile(r'^nstrace.trcdbg.val(\d+)'), re.compile(r'^netlogon.dummy_string'), re.compile(r'^opa.reserved'), re.compile(r'^mpls_pm.timestamp\d\..*'), re.compile(r'^wassp.data.mu_mac'), re.compile(r'^thrift.type'), re.compile(r'^quake2.game.client.command.move.angles'), re.compile(r'^ipp.enum_value'), re.compile(r'^idrp.error.subcode'), re.compile(r'^ftdi-ft.lValue'), re.compile(r'^6lowpan.src'), re.compile(r'^couchbase.flex_frame.frame.id'), re.compile(r'^rtps.param.id'), re.compile(r'^rtps.locator.port'), re.compile(r'^sigcomp.udvm.value'), re.compile(r'^opa.mad.attributemodifier.n'), re.compile(r'^smb.cmd'), re.compile(r'^sctp.checksum'), re.compile(r'^dhcp.option.end'), re.compile(r'^nfapi.num.bf.vector.bf.value'), re.compile(r'^dnp3.al.range.abs'), re.compile(r'^dnp3.al.range.quantity'), re.compile(r'^dnp3.al.index'), re.compile(r'^dnp3.al.size'), re.compile(r'^ftdi-ft.hValue'), re.compile(r'^homeplug_av.op_attr_cnf.data.sw_sub'), re.compile(r'^radiotap.he_mu.preamble_puncturing'), re.compile(r'^ndmp.file'), re.compile(r'^ocfs2.dlm.lvb'), re.compile(r'^oran_fh_cus.reserved'), re.compile(r'^qnet6.kif.msgsend.msg.read.xtypes0-7'), re.compile(r'^qnet6.kif.msgsend.msg.write.xtypes0-7'), re.compile(r'^mih.sig_strength'), re.compile(r'^couchbase.flex_frame.frame.len'), re.compile(r'^nvme-rdma.read_to_host_req'), re.compile(r'^rpcap.dummy'), re.compile(r'^sflow.flow_sample.output_interface'), re.compile(r'^socks.results'), re.compile(r'^opa.mad.attributemodifier.p'), re.compile(r'^v5ua.efa'), re.compile(r'^zbncp.data.tx_power'), re.compile(r'^zbncp.data.nwk_addr'), re.compile(r'^zbee_zcl_hvac.pump_config_control.attr.ctrl_mode'), re.compile(r'^nat-pmp.external_port'), re.compile(r'^zbee_zcl.attr.float'), re.compile(r'^wpan-tap.phr.fsk_ms.mode'), re.compile(r'^mysql.exec_flags'), re.compile(r'^pim.metric_pref'), re.compile(r'^modbus.regval_float'), re.compile(r'^alcap.cau.value'), re.compile(r'^bpv7.crc_field'), re.compile(r'^at.chld.mode'), re.compile(r'^btl2cap.psm'), re.compile(r'^srvloc.srvtypereq.nameauthlistlen'), re.compile(r'^a11.ext.code'), re.compile(r'^adwin_config.port'), re.compile(r'^afp.unknown'), re.compile(r'^ansi_a_bsmap.mid.digit_1'), re.compile(r'^ber.unknown.OCTETSTRING'), re.compile(r'^btatt.handle'), re.compile(r'^btl2cap.option_flushto'), re.compile(r'^cip.network_segment.prod_inhibit'), re.compile(r'^cql.result.rows.table_name'), re.compile(r'^dcom.sa.vartype'), re.compile(r'^f5ethtrailer.slot'), re.compile(r'^ipdr.cm_ipv6_addr'), re.compile(r'^mojito.kuid'), re.compile(r'^mtp3.priority'), re.compile(r'^pw.cw.length'), re.compile(r'^rlc.ciphered_data'), re.compile(r'^vp8.pld.pictureid'), re.compile(r'^gryphon.sched.channel'), re.compile(r'^pn_io.ioxs'), re.compile(r'^pn_dcp.block_qualifier_reset'), re.compile(r'^pn_dcp.suboption_device_instance'), re.compile(r'^nfs.attr'), re.compile(r'^nfs.create_session_flags'), re.compile(r'^rmt-lct.toi64'), re.compile(r'^gryphon.data.header_length'), re.compile(r'^quake2.game.client.command.move.movement'), re.compile(r'^isup.parameter_type'), re.compile(r'^cip.port'), re.compile(r'^adwin.fifo_no'), re.compile(r'^bthci_evt.hci_vers_nr'), re.compile(r'^gryphon.usdt.stmin_active'), re.compile(r'^dnp3.al.anaout.int'), re.compile(r'^dnp3.al.ana.int'), re.compile(r'^dnp3.al.cnt'), re.compile(r'^bthfp.chld.mode'), re.compile(r'^nat-pmp.pml'), re.compile(r'^isystemactivator.actproperties.ts.hdr'), re.compile(r'^rtpdump.txt_addr'), re.compile(r'^unistim.vocoder.id'), re.compile(r'^mac.ueid'), re.compile(r'cip.symbol.size'), re.compile(r'dnp3.al.range.start'), re.compile(r'dnp3.al.range.stop'), re.compile(r'gtpv2.mp'), re.compile(r'gvcp.cmd.resend.firstpacketid'), re.compile(r'gvcp.cmd.resend.lastpacketid'), re.compile(r'wlan.bf.reserved'), re.compile(r'opa.sa.reserved'), re.compile(r'rmt-lct.ext_tol_transfer_len'), re.compile(r'pn_io.error_code2'), re.compile(r'gryphon.ldf.schedsize'), re.compile(r'wimaxmacphy.burst_opt_mimo_matrix_indicator'), re.compile(r'alcap.*bwt.*.[b|f]w'), re.compile(r'ccsds.packet_type'), re.compile(r'iso15765.flow_control.stmin'), re.compile(r'msdo.PieceSize'), re.compile(r'opa.clasportinfo.redirect.reserved'), re.compile(r'p_mul.unused'), re.compile(r'btle.control.phys.le_[1|2]m_phy'), re.compile(r'opa.pm.dataportcounters.reserved'), re.compile(r'opa.switchinfo.switchcapabilitymask.reserved'), re.compile(r'nvme-rdma.read_from_host_resp'), re.compile(r'nvme-rdma.write_to_host_req'), re.compile(r'netlink-route.ifla_linkstats.rx_errors.fifo_errs'), re.compile(r'mtp3mg.japan_spare'), re.compile(r'ixveriwave.errors.ip_checksum_error'), re.compile(r'ansi_a_bsmap.cm2.scm.bc_entry.opmode[0|1]') ] for patt in ignore_patterns: if patt.match(filter): return True return False class ValueString: def __init__(self, file, name, vals, macros, do_extra_checks=False): self.file = file self.name = name self.raw_vals = vals self.parsed_vals = {} self.seen_labels = set() self.valid = True self.min_value = 99999 self.max_value = -99999 # Now parse out each entry in the value_string matches = re.finditer(r'\{\s*([0-9_A-Za-z]*)\s*,\s*(".*?")\s*}\s*,', self.raw_vals) for m in matches: value,label = m.group(1), m.group(2) if value in macros: value = macros[value] elif any(not c in '0123456789abcdefABCDEFxX' for c in value): self.valid = False return try: # Read according to the appropriate base. if value.lower().startswith('0x'): value = int(value, 16) elif value.startswith('0b'): value = int(value[2:], 2) elif value.startswith('0'): value = int(value, 8) else: value = int(value, 10) except: return global warnings_found # Check for value conflict before inserting if value in self.parsed_vals and label != self.parsed_vals[value]: print('Warning:', self.file, ': value_string', self.name, '- value ', value, 'repeated with different values - was', self.parsed_vals[value], 'now', label) warnings_found += 1 else: # Add into table, while checking for repeated label self.parsed_vals[value] = label if do_extra_checks and label in self.seen_labels: # These are commonly repeated.. exceptions = [ 'reserved', 'invalid', 'unused', 'not used', 'unknown', 'undefined', 'spare', 'unallocated', 'not assigned', 'implementation specific', 'unspecified', 'other', 'for further study', 'future', 'vendor specific', 'obsolete', 'none', 'shall not be used', 'national use', 'unassigned', 'oem', 'user defined', 'manufacturer specific', 'not specified', 'proprietary', 'operator-defined', 'dynamically allocated', 'user specified', 'xxx', 'default', 'planned', 'not req' ] excepted = False for ex in exceptions: if label.lower().find(ex) != -1: excepted = True break if not excepted: print('Warning:', self.file, ': value_string', self.name, '- label ', label, 'repeated') warnings_found += 1 else: self.seen_labels.add(label) if value > self.max_value: self.max_value = value if value < self.min_value: self.min_value = value def extraChecks(self): global warnings_found # Look for one value missing in range (quite common...) num_items = len(self.parsed_vals) span = self.max_value - self.min_value + 1 if num_items > 4 and span > num_items and (span-num_items <=1): for val in range(self.min_value, self.max_value): if not val in self.parsed_vals: print('Warning:', self.file, ': value_string', self.name, '- value', val, 'missing?', '(', num_items, 'entries)') global warnings_found warnings_found += 1 # Do most of the labels match the number? matching_label_entries = set() for val in self.parsed_vals: if self.parsed_vals[val].find(str(val)) != -1: # TODO: pick out multiple values rather than concat into wrong number parsed_value = int(''.join(d for d in self.parsed_vals[val] if d.isdecimal())) if val == parsed_value: matching_label_entries.add(val) if len(matching_label_entries) >= 4 and len(matching_label_entries) > 0 and len(matching_label_entries) < num_items and len(matching_label_entries) >= num_items-1: # Be forgiving about first or last entry first_val = list(self.parsed_vals)[0] last_val = list(self.parsed_vals)[-1] if not first_val in matching_label_entries or not last_val in matching_label_entries: return print('Warning:', self.file, ': value_string', self.name, 'Labels match value except for 1!', matching_label_entries, num_items, self) # Do all labels start with lower-or-upper char? startLower,startUpper = 0,0 for val in self.parsed_vals: first_letter = self.parsed_vals[val][1] if first_letter.isalpha(): if first_letter.isupper(): startUpper += 1 else: startLower += 1 if startLower > 0 and startUpper > 0: if startLower+startUpper > 10 and (startLower <=3 or startUpper <=3): standouts = [] if startLower < startUpper: standouts += [self.parsed_vals[val] for val in self.parsed_vals if self.parsed_vals[val][1].islower()] if startLower > startUpper: standouts += [self.parsed_vals[val] for val in self.parsed_vals if self.parsed_vals[val][1].isupper()] print('Note:', self.file, ': value_string', self.name, 'mix of upper', startUpper, 'and lower', startLower, standouts) def __str__(self): return self.name + '= { ' + self.raw_vals + ' }' class RangeStringEntry: def __init__(self, min, max, label): self.min = min self.max = max self.label = label def hides(self, min, max): return min >= self.min and max <= self.max def __str__(self): return '(' + str(self.min) + ', ' + str(self.max) + ') -> ' + self.label class RangeString: def __init__(self, file, name, vals, macros, do_extra_checks=False): self.file = file self.name = name self.raw_vals = vals self.parsed_vals = [] self.seen_labels = set() self.valid = True self.min_value = 99999 self.max_value = -99999 # Now parse out each entry in the value_string matches = re.finditer(r'\{\s*([0-9_A-Za-z]*)\s*,\s*([0-9_A-Za-z]*)\s*,\s*(".*?")\s*}\s*,', self.raw_vals) for m in matches: min,max,label = m.group(1), m.group(2), m.group(3) if min in macros: min = macros[min] elif any(not c in '0123456789abcdefABCDEFxX' for c in min): self.valid = False return if max in macros: max = macros[max] elif any(not c in '0123456789abcdefABCDEFxX' for c in max): self.valid = False return try: # Read according to the appropriate base. if min.lower().startswith('0x'): min = int(min, 16) elif min.startswith('0b'): min = int(min[2:], 2) elif min.startswith('0'): min = int(min, 8) else: min = int(min, 10) if max.lower().startswith('0x'): max = int(max, 16) elif max.startswith('0b'): max = int(max[2:], 2) elif max.startswith('0'): max = int(max, 8) else: max = int(max, 10) except: return # Now check what we've found. global warnings_found if min < self.min_value: self.min_value = min # For overall max value, still use min of each entry. # It is common for entries to extend to e.g. 0xff, but at least we can check for items # that can never match if we only chec the min. if min > self.max_value: self.max_value = min # This value should not be entirely hidden by earlier entries for prev in self.parsed_vals: if prev.hides(min, max): print('Warning:', self.file, ': range_string label', label, 'hidden by', prev) warnings_found += 1 # Max should not be > min if min > max: print('Warning:', self.file, ': range_string', self.name, 'entry', label, 'min', min, '>', max) warnings_found += 1 # Check label. if label[1:-1].startswith(' ') or label[1:-1].endswith(' '): print('Warning:', self.file, ': range_string', self.name, 'entry', label, 'starts or ends with space') warnings_found += 1 # OK, add this entry self.parsed_vals.append(RangeStringEntry(min, max, label)) def extraChecks(self): pass # TODO: some checks over all entries. e.g., # - can multiple values be coalesced into 1? # - if in all cases min==max, suggest value_string instead? # Look for value_string entries in a dissector file. Return a dict name -> ValueString def findValueStrings(filename, macros, do_extra_checks=False): vals_found = {} #static const value_string radio_type_vals[] = #{ # { 0, "FDD"}, # { 1, "TDD"}, # { 0, NULL } #}; with open(filename, 'r', encoding="utf8") as f: contents = f.read() # Remove comments so as not to trip up RE. contents = removeComments(contents) matches = re.finditer(r'.*const value_string\s*([a-zA-Z0-9_]*)\s*\[\s*\]\s*\=\s*\{([\{\}\d\,a-zA-Z0-9_\-\*\#\.:\/\(\)\'\s\"]*)\};', contents) for m in matches: name = m.group(1) vals = m.group(2) vals_found[name] = ValueString(filename, name, vals, macros, do_extra_checks) return vals_found # Look for value_string entries in a dissector file. Return a dict name -> ValueString def findRangeStrings(filename, macros, do_extra_checks=False): vals_found = {} #static const range_string symbol_table_shndx_rvals[] = { # { 0x0000, 0x0000, "Undefined" }, # { 0x0001, 0xfeff, "Normal Section" }, # { 0, 0, NULL } #}; with open(filename, 'r', encoding="utf8") as f: contents = f.read() # Remove comments so as not to trip up RE. contents = removeComments(contents) matches = re.finditer(r'.*const range_string\s*([a-zA-Z0-9_]*)\s*\[\s*\]\s*\=\s*\{([\{\}\d\,a-zA-Z0-9_\-\*\#\.:\/\(\)\'\s\"]*)\};', contents) for m in matches: name = m.group(1) vals = m.group(2) vals_found[name] = RangeString(filename, name, vals, macros, do_extra_checks) return vals_found # The relevant parts of an hf item. Used as value in dict where hf variable name is key. class Item: # Keep the previous few items previousItems = [] def __init__(self, filename, hf, filter, label, item_type, display, strings, macros, value_strings, range_strings, mask=None, check_mask=False, mask_exact_width=False, check_label=False, check_consecutive=False, blurb=''): self.filename = filename self.hf = hf self.filter = filter self.label = label self.mask = mask self.strings = strings self.mask_exact_width = mask_exact_width global warnings_found self.set_mask_value(macros) if check_consecutive: for previous_index,previous_item in enumerate(Item.previousItems): if previous_item.filter == filter: if label != previous_item.label: if not is_ignored_consecutive_filter(self.filter): print('Warning:', filename, hf, ': - filter "' + filter + '" appears ' + str(previous_index+1) + ' items before - labels are "' + previous_item.label + '" and "' + label + '"') warnings_found += 1 # Add this one to front of (short) previous list Item.previousItems = [self] + Item.previousItems if len(Item.previousItems) > 5: # Get rid of oldest one now #Item.previousItems = Item.previousItems[:-1] Item.previousItems.pop() self.item_type = item_type self.display = display # Optionally check label (short and long). if check_label: self.check_label(label, 'label') #self.check_label(blurb, 'blurb') # Optionally check that mask bits are contiguous if check_mask: if self.mask_read and not mask in { 'NULL', '0x0', '0', '0x00' }: self.check_contiguous_bits(mask) self.check_num_digits(self.mask) # N.B., if last entry in set is removed, see around 18,000 warnings self.check_digits_all_zeros(self.mask) # N.B. these checks are already done by checkApis.pl if strings.find('RVALS') != -1 and display.find('BASE_RANGE_STRING') == -1: print('Warning: ' + filename, hf, 'filter "' + filter + ' strings has RVALS but display lacks BASE_RANGE_STRING') warnings_found += 1 # For RVALS, is BASE_RANGE_STRING also set (checked by checkApis.pl)? if strings.find('VALS_EXT_PTR') != -1 and display.find('BASE_EXT_STRING') == -1: print('Warning: ' + filename, hf, 'filter "' + filter + ' strings has VALS_EXT_PTR but display lacks BASE_EXT_STRING') warnings_found += 1 # For VALS, lookup the corresponding ValueString and try to check range. vs_re = re.compile(r'VALS\(([a-zA-Z0-9_]*)\)') m = vs_re.search(strings) if m: self.vs_name = m.group(1) if self.vs_name in value_strings: vs = value_strings[self.vs_name] self.check_value_string_range(vs.min_value, vs.max_value) # For RVALS, lookup the corresponding RangeString and try to check range. rs_re = re.compile(r'RVALS\(([a-zA-Z0-9_]*)\)') m = rs_re.search(strings) if m: self.rs_name = m.group(1) if self.rs_name in range_strings: rs = range_strings[self.rs_name] self.check_range_string_range(rs.min_value, rs.max_value) def __str__(self): return 'Item ({0} "{1}" {2} type={3}:{4} {5} mask={6})'.format(self.filename, self.label, self.filter, self.item_type, self.display, self.strings, self.mask) def check_label(self, label, label_name): global warnings_found # TODO: this is masking a bug where the re for the item can't cope with macro for containing ',' for mask arg.. if label.count('"') == 1: return if label.startswith(' ') or label.endswith(' '): print('Warning: ' + self.filename, self.hf, 'filter "' + self.filter, label_name, '"' + label + '" begins or ends with a space') warnings_found += 1 if (label.count('(') != label.count(')') or label.count('[') != label.count(']') or label.count('{') != label.count('}')): # Ignore if includes quotes, as may be unbalanced. if label.find("'") == -1: print('Warning: ' + self.filename, self.hf, 'filter "' + self.filter + '"', label_name, '"' + label + '"', 'has unbalanced parens/braces/brackets') warnings_found += 1 if self.item_type != 'FT_NONE' and label.endswith(':'): print('Warning: ' + self.filename, self.hf, 'filter "' + self.filter + '"', label_name, '"' + label + '"', 'ends with an unnecessary colon') warnings_found += 1 def set_mask_value(self, macros): try: self.mask_read = True # Substitute mask if found as a macro.. if self.mask in macros: self.mask = macros[self.mask] elif any(not c in '0123456789abcdefABCDEFxX' for c in self.mask): self.mask_read = False self.mask_value = 0 return # Read according to the appropriate base. if self.mask.startswith('0x'): self.mask_value = int(self.mask, 16) elif self.mask.startswith('0'): self.mask_value = int(self.mask, 8) else: self.mask_value = int(self.mask, 10) except: self.mask_read = False self.mask_value = 0 def check_value_string_range(self, vs_min, vs_max): item_width = self.get_field_width_in_bits() if item_width is None: # Type field defined by macro? return if self.mask_value > 0: # Distance between first and last '1' bitBools = bin(self.mask_value)[2:] mask_width = bitBools.rfind('1') - bitBools.find('1') + 1 else: # No mask is effectively a full mask.. mask_width = item_width item_max = (2 ** mask_width) if vs_max > item_max: global warnings_found print('Warning:', self.filename, self.hf, 'filter=', self.filter, self.strings, "has max value", vs_max, '(' + hex(vs_max) + ')', "which doesn't fit into", mask_width, 'bits', '( mask is', hex(self.mask_value), ')') warnings_found += 1 def check_range_string_range(self, rs_min, rs_max): item_width = self.get_field_width_in_bits() if item_width is None: # Type field defined by macro? return if self.mask_value > 0: # Distance between first and last '1' bitBools = bin(self.mask_value)[2:] mask_width = bitBools.rfind('1') - bitBools.find('1') + 1 else: # No mask is effectively a full mask.. mask_width = item_width item_max = (2 ** mask_width) if rs_max > item_max: global warnings_found print('Warning:', self.filename, self.hf, 'filter=', self.filter, self.strings, "has values", rs_min, rs_max, '(' + hex(rs_max) + ')', "which doesn't fit into", mask_width, 'bits', '( mask is', hex(self.mask_value), ')') warnings_found += 1 # Return true if bit position n is set in value. def check_bit(self, value, n): return (value & (0x1 << n)) != 0 # Output a warning if non-contigous bits are found in the mask (guint64). # Note that this legimately happens in several dissectors where multiple reserved/unassigned # bits are conflated into one field. # - there is probably a cool/efficient way to check this (+1 => 1-bit set?) def check_contiguous_bits(self, mask): if not self.mask_value: return # Do see legitimate non-contiguous bits often for these.. if name_has_one_of(self.hf, ['reserved', 'unknown', 'unused', 'spare']): return if name_has_one_of(self.label, ['reserved', 'unknown', 'unused', 'spare']): return # Walk past any l.s. 0 bits n = 0 while not self.check_bit(self.mask_value, n) and n <= 63: n += 1 if n==63: return mask_start = n # Walk through any bits that are set while self.check_bit(self.mask_value, n) and n <= 63: n += 1 n += 1 if n >= 63: return # Look up the field width field_width = 0 if not self.item_type in field_widths: print('unexpected item_type is ', self.item_type) field_width = 64 else: field_width = self.get_field_width_in_bits() # Its a problem is the mask_width is > field_width - some of the bits won't get looked at!? mask_width = n-1-mask_start if field_width is not None and (mask_width > field_width): # N.B. No call, so no line number. print(self.filename + ':', self.hf, 'filter=', self.filter, self.item_type, 'so field_width=', field_width, 'but mask is', mask, 'which is', mask_width, 'bits wide!') global warnings_found warnings_found += 1 # Now, any more zero set bits are an error! if self.filter in known_non_contiguous_fields or self.filter.startswith('rtpmidi'): # Don't report if we know this one is Ok. # TODO: also exclude items that are used as root in add_bitmask() calls? return while n <= 63: if self.check_bit(self.mask_value, n): print('Warning:', self.filename, self.hf, 'filter=', self.filter, ' - mask with non-contiguous bits', mask, '(', hex(self.mask_value), ')') warnings_found += 1 return n += 1 def get_field_width_in_bits(self): if self.item_type == 'FT_BOOLEAN': if self.display == 'NULL': return 8 # i.e. 1 byte elif self.display == 'BASE_NONE': return 8 elif self.display == 'SEP_DOT': # from proto.h, only meant for FT_BYTES return 64 else: try: # For FT_BOOLEAN, modifier is just numerical number of bits. Round up to next nibble. return int((int(self.display) + 3)/4)*4 except: return None else: if self.item_type in field_widths: # Lookup fixed width for this type return field_widths[self.item_type] else: return None def check_num_digits(self, mask): if mask.startswith('0x') and len(mask) > 3: global warnings_found global errors_found width_in_bits = self.get_field_width_in_bits() # Warn if odd number of digits. TODO: only if >= 5? if len(mask) % 2 and self.item_type != 'FT_BOOLEAN': print('Warning:', self.filename, self.hf, 'filter=', self.filter, ' - mask has odd number of digits', mask, 'expected max for', self.item_type, 'is', int(width_in_bits/4)) warnings_found += 1 if self.item_type in field_widths: # Longer than it should be? if width_in_bits is None: return if len(mask)-2 > width_in_bits/4: extra_digits = mask[2:2+(len(mask)-2 - int(width_in_bits/4))] # Its definitely an error if any of these are non-zero, as they won't have any effect! if extra_digits != '0'*len(extra_digits): print('Error:', self.filename, self.hf, 'filter=', self.filter, 'mask', self.mask, "with len is", len(mask)-2, "but type", self.item_type, " indicates max of", int(width_in_bits/4), "and extra digits are non-zero (" + extra_digits + ")") errors_found += 1 else: # Has extra leading zeros, still confusing, so warn. print('Warning:', self.filename, self.hf, 'filter=', self.filter, 'mask', self.mask, "with len", len(mask)-2, "but type", self.item_type, " indicates max of", int(width_in_bits/4)) warnings_found += 1 # Strict/fussy check - expecting mask length to match field width exactly! # Currently only doing for FT_BOOLEAN, and don't expect to be in full for 64-bit fields! if self.mask_exact_width: ideal_mask_width = int(width_in_bits/4) if self.item_type == 'FT_BOOLEAN' and ideal_mask_width < 16 and len(mask)-2 != ideal_mask_width: print('Warning:', self.filename, self.hf, 'filter=', self.filter, 'mask', self.mask, "with len", len(mask)-2, "but type", self.item_type, "|", self.display, " indicates should be", int(width_in_bits/4)) warnings_found += 1 else: # This type shouldn't have a mask set at all. print('Warning:', self.filename, self.hf, 'filter=', self.filter, ' - item has type', self.item_type, 'but mask set:', mask) warnings_found += 1 def check_digits_all_zeros(self, mask): if mask.startswith('0x') and len(mask) > 3: if mask[2:] == '0'*(len(mask)-2): print('Warning:', self.filename, self.hf, 'filter=', self.filter, ' - item mask has all zeros - this is confusing! :', '"' + mask + '"') global warnings_found warnings_found += 1 # A mask where all bits are set should instead be 0. # Exceptions might be where: # - in add_bitmask() # - represents flags, but dissector is not yet decoding them def check_full_mask(self, mask, field_arrays): if self.item_type == "FT_BOOLEAN": return if self.label.lower().find('mask') != -1 or self.label.lower().find('flag') != -1 or self.label.lower().find('bitmap') != -1: return if mask.startswith('0x') and len(mask) > 3: width_in_bits = self.get_field_width_in_bits() if not width_in_bits: return num_digits = int(width_in_bits / 4) if num_digits is None: return if mask[2:] == 'f'*num_digits or mask[2:] == 'F'*num_digits: # Don't report if appears in a 'fields' array for arr in field_arrays: list = field_arrays[arr][0] if self.hf in list: # These need to have a mask - don't judge for being 0 return print('Note:', self.filename, self.hf, 'filter=', self.filter, " - mask is all set - if only want value (rather than bits), set 0 instead? :", '"' + mask + '"') # An item that appears in a bitmask set, needs to have a non-zero mask. def check_mask_if_in_field_array(self, mask, field_arrays): # Work out if this item appears in a field array found = False array_name = None for arr in field_arrays: list = field_arrays[arr][0] if self.hf in list: # These need to have a mask - don't judge for being 0 found = True array_name = arr break if found: # It needs to have a non-zero mask. if self.mask_read and self.mask_value == 0: print('Error:', self.filename, self.hf, 'is in fields array', arr, 'but has a zero mask - this is not allowed') global errors_found errors_found += 1 # Return True if appears to be a match def check_label_vs_filter(self, reportError=True, reportNumericalMismatch=True): global warnings_found last_filter = self.filter.split('.')[-1] last_filter_orig = last_filter last_filter = last_filter.replace('-', '') last_filter = last_filter.replace('_', '') last_filter = last_filter.replace(' ', '') label = self.label label_orig = label label = label.replace(' ', '') label = label.replace('-', '') label = label.replace('_', '') label = label.replace('(', '') label = label.replace(')', '') label = label.replace('/', '') label = label.replace("'", '') # OK if filter is abbrev of label. label_words = self.label.split(' ') label_words = [w for w in label_words if len(w)] if len(label_words) == len(last_filter): #print(label_words) abbrev_letters = [w[0] for w in label_words] abbrev = ''.join(abbrev_letters) if abbrev.lower() == last_filter.lower(): return True # If both have numbers, they should probably match! label_numbers = [int(n) for n in re.findall(r'\d+', label_orig)] filter_numbers = [int(n) for n in re.findall(r'\d+', last_filter_orig)] if len(label_numbers) == len(filter_numbers) and label_numbers != filter_numbers: if reportNumericalMismatch: print('Note:', self.filename, self.hf, 'label="' + self.label + '" has different **numbers** from filter="' + self.filter + '"') print(label_numbers, filter_numbers) return False # If they match after trimming number from filter, they should match. if label.lower() == last_filter.lower().rstrip("0123456789"): return True # Are they just different? if label.lower().find(last_filter.lower()) == -1: if reportError: print('Warning:', self.filename, self.hf, 'label="' + self.label + '" does not seem to match filter="' + self.filter + '"') warnings_found += 1 return False return True class CombinedCallsCheck: def __init__(self, file, apiChecks): self.file = file self.apiChecks = apiChecks self.get_all_calls() def get_all_calls(self): self.all_calls = [] # Combine calls into one list. for check in self.apiChecks: self.all_calls += check.calls # Sort by line number. self.all_calls.sort(key=lambda x:x.line_number) def check_consecutive_item_calls(self): lines = open(self.file, 'r', encoding="utf8").read().splitlines() prev = None for call in self.all_calls: # These names commonly do appear together.. if name_has_one_of(call.hf_name, [ 'unused', 'unknown', 'spare', 'reserved', 'default']): return if prev and call.hf_name == prev.hf_name: # More compelling if close together.. if call.line_number>prev.line_number and call.line_number-prev.line_number <= 4: scope_different = False for l in range(prev.line_number, call.line_number-1): if lines[l].find('{') != -1 or lines[l].find('}') != -1 or lines[l].find('else') != -1 or lines[l].find('break;') != -1 or lines[l].find('if ') != -1: scope_different = True break # Also more compelling if check for and scope changes { } in lines in-between? if not scope_different: print('Warning:', f + ':' + str(call.line_number), call.hf_name + ' called consecutively at line', call.line_number, '- previous at', prev.line_number) global warnings_found warnings_found += 1 prev = call # These are APIs in proto.c that check a set of types at runtime and can print '.. is not of type ..' to the console # if the type is not suitable. apiChecks = [] apiChecks.append(APICheck('proto_tree_add_item_ret_uint', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32'}, positive_length=True)) apiChecks.append(APICheck('proto_tree_add_item_ret_int', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'})) apiChecks.append(APICheck('ptvcursor_add_ret_uint', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32'}, positive_length=True)) apiChecks.append(APICheck('ptvcursor_add_ret_int', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'}, positive_length=True)) apiChecks.append(APICheck('ptvcursor_add_ret_string', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'})) apiChecks.append(APICheck('ptvcursor_add_ret_boolean', { 'FT_BOOLEAN'}, positive_length=True)) apiChecks.append(APICheck('proto_tree_add_item_ret_uint64', { 'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64'}, positive_length=True)) apiChecks.append(APICheck('proto_tree_add_item_ret_int64', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'}, positive_length=True)) apiChecks.append(APICheck('proto_tree_add_item_ret_boolean', { 'FT_BOOLEAN'}, positive_length=True)) apiChecks.append(APICheck('proto_tree_add_item_ret_string_and_length', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'})) apiChecks.append(APICheck('proto_tree_add_item_ret_display_string_and_length', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC', 'FT_BYTES', 'FT_UINT_BYTES'})) apiChecks.append(APICheck('proto_tree_add_item_ret_time_string', { 'FT_ABSOLUTE_TIME', 'FT_RELATIVE_TIME'})) apiChecks.append(APICheck('proto_tree_add_uint', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_FRAMENUM'})) apiChecks.append(APICheck('proto_tree_add_uint_format_value', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_FRAMENUM'})) apiChecks.append(APICheck('proto_tree_add_uint_format', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_FRAMENUM'})) apiChecks.append(APICheck('proto_tree_add_uint64', { 'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64', 'FT_FRAMENUM'})) apiChecks.append(APICheck('proto_tree_add_int64', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'})) apiChecks.append(APICheck('proto_tree_add_int64_format_value', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'})) apiChecks.append(APICheck('proto_tree_add_int64_format', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'})) apiChecks.append(APICheck('proto_tree_add_int', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'})) apiChecks.append(APICheck('proto_tree_add_int_format_value', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'})) apiChecks.append(APICheck('proto_tree_add_int_format', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'})) apiChecks.append(APICheck('proto_tree_add_boolean', { 'FT_BOOLEAN'})) apiChecks.append(APICheck('proto_tree_add_boolean64', { 'FT_BOOLEAN'})) apiChecks.append(APICheck('proto_tree_add_float', { 'FT_FLOAT'})) apiChecks.append(APICheck('proto_tree_add_float_format', { 'FT_FLOAT'})) apiChecks.append(APICheck('proto_tree_add_float_format_value', { 'FT_FLOAT'})) apiChecks.append(APICheck('proto_tree_add_double', { 'FT_DOUBLE'})) apiChecks.append(APICheck('proto_tree_add_double_format', { 'FT_DOUBLE'})) apiChecks.append(APICheck('proto_tree_add_double_format_value', { 'FT_DOUBLE'})) apiChecks.append(APICheck('proto_tree_add_string', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'})) apiChecks.append(APICheck('proto_tree_add_string_format', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'})) apiChecks.append(APICheck('proto_tree_add_string_format_value', { 'FT_STRING', 'FT_STRINGZ', 'FT_UINT_STRING', 'FT_STRINGZPAD', 'FT_STRINGZTRUNC'})) apiChecks.append(APICheck('proto_tree_add_guid', { 'FT_GUID'})) apiChecks.append(APICheck('proto_tree_add_oid', { 'FT_OID'})) apiChecks.append(APICheck('proto_tree_add_none_format', { 'FT_NONE'})) apiChecks.append(APICheck('proto_tree_add_item_ret_varint', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32', 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64', 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_FRAMENUM', 'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64',})) apiChecks.append(APICheck('proto_tree_add_boolean_bits_format_value', { 'FT_BOOLEAN'})) apiChecks.append(APICheck('proto_tree_add_boolean_bits_format_value64', { 'FT_BOOLEAN'})) apiChecks.append(APICheck('proto_tree_add_ascii_7bits_item', { 'FT_STRING'})) # TODO: positions are different, and takes 2 hf_fields.. #apiChecks.append(APICheck('proto_tree_add_checksum', { 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32'})) apiChecks.append(APICheck('proto_tree_add_int64_bits_format_value', { 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64'})) # TODO: add proto_tree_add_bytes_item, proto_tree_add_time_item ? bitmask_types = { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32', 'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64', 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64', 'FT_BOOLEAN'} apiChecks.append(APICheck('proto_tree_add_bitmask', bitmask_types)) apiChecks.append(APICheck('proto_tree_add_bitmask_tree', bitmask_types)) apiChecks.append(APICheck('proto_tree_add_bitmask_ret_uint64', bitmask_types)) apiChecks.append(APICheck('proto_tree_add_bitmask_with_flags', bitmask_types)) apiChecks.append(APICheck('proto_tree_add_bitmask_with_flags_ret_uint64', bitmask_types)) apiChecks.append(APICheck('proto_tree_add_bitmask_value', bitmask_types)) apiChecks.append(APICheck('proto_tree_add_bitmask_value_with_flags', bitmask_types)) apiChecks.append(APICheck('proto_tree_add_bitmask_len', bitmask_types)) # N.B., proto_tree_add_bitmask_list does not have a root item, just a subtree... add_bits_types = { 'FT_CHAR', 'FT_BOOLEAN', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32', 'FT_UINT40', 'FT_UINT48', 'FT_UINT56', 'FT_UINT64', 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32', 'FT_INT40', 'FT_INT48', 'FT_INT56', 'FT_INT64', 'FT_BYTES'} apiChecks.append(APICheck('proto_tree_add_bits_item', add_bits_types)) apiChecks.append(APICheck('proto_tree_add_bits_ret_val', add_bits_types)) # TODO: doesn't even have an hf_item ! #apiChecks.append(APICheck('proto_tree_add_bitmask_text', bitmask_types)) # Check some ptvcuror calls too. apiChecks.append(APICheck('ptvcursor_add_ret_uint', { 'FT_CHAR', 'FT_UINT8', 'FT_UINT16', 'FT_UINT24', 'FT_UINT32'})) apiChecks.append(APICheck('ptvcursor_add_ret_int', { 'FT_INT8', 'FT_INT16', 'FT_INT24', 'FT_INT32'})) apiChecks.append(APICheck('ptvcursor_add_ret_boolean', { 'FT_BOOLEAN'})) # Also try to check proto_tree_add_item() calls (for length) apiChecks.append(ProtoTreeAddItemCheck()) apiChecks.append(ProtoTreeAddItemCheck(True)) # for ptvcursor_add() def removeComments(code_string): code_string = re.sub(re.compile(r"/\*.*?\*/",re.DOTALL ) ,"" , code_string) # C-style comment code_string = re.sub(re.compile(r"//.*?\n" ) ,"" , code_string) # C++-style comment code_string = re.sub(re.compile(r"#if 0.*?#endif",re.DOTALL ) ,"" , code_string) # Ignored region return code_string # Test for whether the given file was automatically generated. def isGeneratedFile(filename): # Check file exists - e.g. may have been deleted in a recent commit. if not os.path.exists(filename): return False # Open file f_read = open(os.path.join(filename), 'r', encoding="utf8") lines_tested = 0 for line in f_read: # The comment to say that its generated is near the top, so give up once # get a few lines down. if lines_tested > 10: f_read.close() return False if (line.find('Generated automatically') != -1 or line.find('Generated Automatically') != -1 or line.find('Autogenerated from') != -1 or line.find('is autogenerated') != -1 or line.find('automatically generated by Pidl') != -1 or line.find('Created by: The Qt Meta Object Compiler') != -1 or line.find('This file was generated') != -1 or line.find('This filter was automatically generated') != -1 or line.find('This file is auto generated, do not edit!') != -1): f_read.close() return True lines_tested = lines_tested + 1 # OK, looks like a hand-written file! f_read.close() return False def find_macros(filename): macros = {} with open(filename, 'r', encoding="utf8") as f: contents = f.read() # Remove comments so as not to trip up RE. contents = removeComments(contents) matches = re.finditer( r'#define\s*([A-Z0-9_]*)\s*([0-9xa-fA-F]*)\n', contents) for m in matches: # Store this mapping. macros[m.group(1)] = m.group(2) return macros # Look for hf items (i.e. full item to be registered) in a dissector file. def find_items(filename, macros, value_strings, range_strings, check_mask=False, mask_exact_width=False, check_label=False, check_consecutive=False): is_generated = isGeneratedFile(filename) items = {} with open(filename, 'r', encoding="utf8") as f: contents = f.read() # Remove comments so as not to trip up RE. contents = removeComments(contents) # N.B. re extends all the way to HFILL to avoid greedy matching # TODO: fix a problem where re can't cope with mask that involve a macro with commas in it... matches = re.finditer( r'.*\{\s*\&(hf_[a-z_A-Z0-9]*)\s*,\s*{\s*\"(.*?)\"\s*,\s*\"(.*?)\"\s*,\s*(.*?)\s*,\s*([0-9A-Z_\|\s]*?)\s*,\s*(.*?)\s*,\s*(.*?)\s*,\s*([a-zA-Z0-9\W\s_\u00f6\u00e4]*?)\s*,\s*HFILL', contents) for m in matches: # Store this item. hf = m.group(1) blurb = m.group(8) if blurb.startswith('"'): blurb = blurb[1:-1] items[hf] = Item(filename, hf, filter=m.group(3), label=m.group(2), item_type=m.group(4), display=m.group(5), strings=m.group(6), macros=macros, value_strings=value_strings, range_strings=range_strings, mask=m.group(7), blurb=blurb, check_mask=check_mask, mask_exact_width=mask_exact_width, check_label=check_label, check_consecutive=(not is_generated and check_consecutive)) return items # Looking for args to ..add_bitmask_..() calls that are not NULL-terminated or have repeated items. # TODO: some dissectors have similar-looking hf arrays for other reasons, so need to cross-reference with # the 6th arg of ..add_bitmask_..() calls... # TODO: return items (rather than local checks) from here so can be checked against list of calls for given filename def find_field_arrays(filename, all_fields, all_hf): field_entries = {} global warnings_found with open(filename, 'r', encoding="utf8") as f: contents = f.read() # Remove comments so as not to trip up RE. contents = removeComments(contents) # Find definition of hf array matches = re.finditer(r'static\s*g?int\s*\*\s*const\s+([a-zA-Z0-9_]*)\s*\[\]\s*\=\s*\{([a-zA-Z0-9,_\&\s]*)\}', contents) for m in matches: name = m.group(1) # Ignore if not used in a call to an _add_bitmask_ API if not name in all_fields: continue fields_text = m.group(2) fields_text = fields_text.replace('&', '') fields_text = fields_text.replace(',', '') # Get list of each hf field in the array fields = fields_text.split() if fields[0].startswith('ett_'): continue if fields[-1].find('NULL') == -1 and fields[-1] != '0': print('Warning:', filename, name, 'is not NULL-terminated - {', ', '.join(fields), '}') warnings_found += 1 continue # Do any hf items reappear? seen_fields = set() for f in fields: if f in seen_fields: print(filename, name, f, 'already added!') warnings_found += 1 seen_fields.add(f) # Check for duplicated flags among entries.. combined_mask = 0x0 for f in fields[0:-1]: if f in all_hf: new_mask = all_hf[f].mask_value if new_mask & combined_mask: print('Warning:', filename, name, 'has overlapping mask - {', ', '.join(fields), '} combined currently', hex(combined_mask), f, 'adds', hex(new_mask)) warnings_found += 1 combined_mask |= new_mask # Make sure all entries have the same width set_field_width = None for f in fields[0:-1]: if f in all_hf: new_field_width = all_hf[f].get_field_width_in_bits() if set_field_width is not None and new_field_width != set_field_width: # Its not uncommon for fields to be used in multiple sets, some of which can be different widths.. print('Note:', filename, name, 'set items not all same width - {', ', '.join(fields), '} seen', set_field_width, 'now', new_field_width) set_field_width = new_field_width # Add entry to table field_entries[name] = (fields[0:-1], combined_mask) return field_entries def find_item_declarations(filename): items = set() with open(filename, 'r', encoding="utf8") as f: lines = f.read().splitlines() p = re.compile(r'^static int (hf_[a-zA-Z0-9_]*)\s*\=\s*-1;') for line in lines: m = p.search(line) if m: items.add(m.group(1)) return items def find_item_extern_declarations(filename): items = set() with open(filename, 'r', encoding="utf8") as f: lines = f.read().splitlines() p = re.compile(r'^\s*(hf_[a-zA-Z0-9_]*)\s*\=\s*proto_registrar_get_id_byname\s*\(') for line in lines: m = p.search(line) if m: items.add(m.group(1)) return items def is_dissector_file(filename): p = re.compile(r'.*(packet|file)-.*\.c$') return p.match(filename) def findDissectorFilesInFolder(folder, recursive=False): dissector_files = [] if recursive: for root, subfolders, files in os.walk(folder): for f in files: if should_exit: return f = os.path.join(root, f) dissector_files.append(f) else: for f in sorted(os.listdir(folder)): if should_exit: return filename = os.path.join(folder, f) dissector_files.append(filename) return [x for x in filter(is_dissector_file, dissector_files)] # Run checks on the given dissector file. def checkFile(filename, check_mask=False, mask_exact_width=False, check_label=False, check_consecutive=False, check_missing_items=False, check_bitmask_fields=False, label_vs_filter=False, extra_value_string_checks=False): # Check file exists - e.g. may have been deleted in a recent commit. if not os.path.exists(filename): print(filename, 'does not exist!') return # Find simple macros so can substitute into items and calls. macros = find_macros(filename) # Find (and sanity-check) value_strings value_strings = findValueStrings(filename, macros, do_extra_checks=extra_value_string_checks) if extra_value_string_checks: for name in value_strings: value_strings[name].extraChecks() # Find (and sanity-check) range_strings range_strings = findRangeStrings(filename, macros, do_extra_checks=extra_value_string_checks) if extra_value_string_checks: for name in range_strings: range_strings[name].extraChecks() # Find important parts of items. items_defined = find_items(filename, macros, value_strings, range_strings, check_mask, mask_exact_width, check_label, check_consecutive) items_extern_declared = {} items_declared = {} if check_missing_items: items_declared = find_item_declarations(filename) items_extern_declared = find_item_extern_declarations(filename) fields = set() # Get 'fields' out of calls for c in apiChecks: c.find_calls(filename, macros) for call in c.calls: # From _add_bitmask() calls if call.fields: fields.add(call.fields) # Checking for lists of fields for add_bitmask calls field_arrays = {} if check_bitmask_fields: field_arrays = find_field_arrays(filename, fields, items_defined) if check_mask and check_bitmask_fields: for i in items_defined: item = items_defined[i] item.check_full_mask(item.mask, field_arrays) item.check_mask_if_in_field_array(item.mask, field_arrays) # Now actually check the calls for c in apiChecks: c.check_against_items(items_defined, items_declared, items_extern_declared, check_missing_items, field_arrays) if label_vs_filter: matches = 0 for hf in items_defined: if items_defined[hf].check_label_vs_filter(reportError=False, reportNumericalMismatch=True): matches += 1 # Only checking if almost every field does match. checking = len(items_defined) and matches 0.93) if checking: print(filename, ':', matches, 'label-vs-filter matches of out of', len(items_defined), 'so reporting mismatches') for hf in items_defined: items_defined[hf].check_label_vs_filter(reportError=True, reportNumericalMismatch=False) ################################################################# # Main logic. # command-line args. Controls which dissector files should be checked. # If no args given, will just scan epan/dissectors folder. parser = argparse.ArgumentParser(description='Check calls in dissectors') parser.add_argument('--file', action='append', help='specify individual dissector file to test') parser.add_argument('--folder', action='store', default='', help='specify folder to test') parser.add_argument('--commits', action='store', help='last N commits to check') parser.add_argument('--open', action='store_true', help='check open files') parser.add_argument('--mask', action='store_true', help='when set, check mask field too') parser.add_argument('--mask-exact-width', action='store_true', help='when set, check width of mask against field width') parser.add_argument('--label', action='store_true', help='when set, check label field too') parser.add_argument('--consecutive', action='store_true', help='when set, copy copy/paste errors between consecutive items') parser.add_argument('--missing-items', action='store_true', help='when set, look for used items that were never registered') parser.add_argument('--check-bitmask-fields', action='store_true', help='when set, attempt to check arrays of hf items passed to add_bitmask() calls') parser.add_argument('--label-vs-filter', action='store_true', help='when set, check whether label matches last part of filter') parser.add_argument('--extra-value-string-checks', action='store_true', help='when set, do extra checks on parsed value_strings') parser.add_argument('--all-checks', action='store_true', help='when set, apply all checks to selected files') args = parser.parse_args() # Turn all checks on. if args.all_checks: args.mask = True args.mask_exact_width = True args.consecutive = True args.check_bitmask_fields = True #args.label = True args.label_vs_filter = True args.extra_value_string_checks if args.check_bitmask_fields: args.mask = True # Get files from wherever command-line args indicate. files = [] if args.file: # Add specified file(s) for f in args.file: if not os.path.isfile(f): print('Chosen file', f, 'does not exist.') exit(1) else: files.append(f) elif args.folder: # Add all files from a given folder. folder = args.folder if not os.path.isdir(folder): print('Folder', folder, 'not found!') exit(1) # Find files from folder. print('Looking for files in', folder) files = findDissectorFilesInFolder(folder, recursive=True) elif args.commits: # Get files affected by specified number of commits. command = ['git', 'diff', '--name-only', '--diff-filter=d', 'HEAD~' + args.commits] files = [f.decode('utf-8') for f in subprocess.check_output(command).splitlines()] # Will examine dissector files only files = list(filter(lambda f : is_dissector_file(f), files)) elif args.open: # Unstaged changes. command = ['git', 'diff', '--name-only', '--diff-filter=d'] files = [f.decode('utf-8') for f in subprocess.check_output(command).splitlines()] # Only interested in dissector files. files = list(filter(lambda f : is_dissector_file(f), files)) # Staged changes. command = ['git', 'diff', '--staged', '--name-only', '--diff-filter=d'] files_staged = [f.decode('utf-8') for f in subprocess.check_output(command).splitlines()] # Only interested in dissector files. files_staged = list(filter(lambda f : is_dissector_file(f), files_staged)) for f in files_staged: if not f in files: files.append(f) else: # Find all dissector files. files = findDissectorFilesInFolder(os.path.join('epan', 'dissectors')) files += findDissectorFilesInFolder(os.path.join('plugins', 'epan'), recursive=True) # If scanning a subset of files, list them here. print('Examining:') if args.file or args.commits or args.open: if files: print(' '.join(files), '\n') else: print('No files to check.\n') else: print('All dissector modules\n') # Now check the files. for f in files: if should_exit: exit(1) checkFile(f, check_mask=args.mask, mask_exact_width=args.mask_exact_width, check_label=args.label, check_consecutive=args.consecutive, check_missing_items=args.missing_items, check_bitmask_fields=args.check_bitmask_fields, label_vs_filter=args.label_vs_filter, extra_value_string_checks=args.extra_value_string_checks) # Do checks against all calls. if args.consecutive: combined_calls = CombinedCallsCheck(f, apiChecks) # This hasn't really found any issues, but shows lots of false positives (and are difficult to investigate) #combined_calls.check_consecutive_item_calls() # Show summary. print(warnings_found, 'warnings') if errors_found: print(errors_found, 'errors') exit(1)