summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/build/check_gn_headers.py
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/libwebrtc/build/check_gn_headers.py')
-rwxr-xr-xthird_party/libwebrtc/build/check_gn_headers.py310
1 files changed, 310 insertions, 0 deletions
diff --git a/third_party/libwebrtc/build/check_gn_headers.py b/third_party/libwebrtc/build/check_gn_headers.py
new file mode 100755
index 0000000000..9bdbba895f
--- /dev/null
+++ b/third_party/libwebrtc/build/check_gn_headers.py
@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Find header files missing in GN.
+
+This script gets all the header files from ninja_deps, which is from the true
+dependency generated by the compiler, and report if they don't exist in GN.
+"""
+
+from __future__ import print_function
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from multiprocessing import Process, Queue
+
+SRC_DIR = os.path.abspath(
+ os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
+DEPOT_TOOLS_DIR = os.path.join(SRC_DIR, 'third_party', 'depot_tools')
+
+
+def GetHeadersFromNinja(out_dir, skip_obj, q):
+ """Return all the header files from ninja_deps"""
+
+ def NinjaSource():
+ cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-t', 'deps']
+ # A negative bufsize means to use the system default, which usually
+ # means fully buffered.
+ popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
+ for line in iter(popen.stdout.readline, ''):
+ yield line.rstrip()
+
+ popen.stdout.close()
+ return_code = popen.wait()
+ if return_code:
+ raise subprocess.CalledProcessError(return_code, cmd)
+
+ ans, err = set(), None
+ try:
+ ans = ParseNinjaDepsOutput(NinjaSource(), out_dir, skip_obj)
+ except Exception as e:
+ err = str(e)
+ q.put((ans, err))
+
+
+def ParseNinjaDepsOutput(ninja_out, out_dir, skip_obj):
+ """Parse ninja output and get the header files"""
+ all_headers = {}
+
+ # Ninja always uses "/", even on Windows.
+ prefix = '../../'
+
+ is_valid = False
+ obj_file = ''
+ for line in ninja_out:
+ if line.startswith(' '):
+ if not is_valid:
+ continue
+ if line.endswith('.h') or line.endswith('.hh'):
+ f = line.strip()
+ if f.startswith(prefix):
+ f = f[6:] # Remove the '../../' prefix
+ # build/ only contains build-specific files like build_config.h
+ # and buildflag.h, and system header files, so they should be
+ # skipped.
+ if f.startswith(out_dir) or f.startswith('out'):
+ continue
+ if not f.startswith('build'):
+ all_headers.setdefault(f, [])
+ if not skip_obj:
+ all_headers[f].append(obj_file)
+ else:
+ is_valid = line.endswith('(VALID)')
+ obj_file = line.split(':')[0]
+
+ return all_headers
+
+
+def GetHeadersFromGN(out_dir, q):
+ """Return all the header files from GN"""
+
+ tmp = None
+ ans, err = set(), None
+ try:
+ # Argument |dir| is needed to make sure it's on the same drive on Windows.
+ # dir='' means dir='.', but doesn't introduce an unneeded prefix.
+ tmp = tempfile.mkdtemp(dir='')
+ shutil.copy2(os.path.join(out_dir, 'args.gn'),
+ os.path.join(tmp, 'args.gn'))
+ # Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
+ gn_exe = 'gn.bat' if sys.platform == 'win32' else 'gn'
+ subprocess.check_call([
+ os.path.join(DEPOT_TOOLS_DIR, gn_exe), 'gen', tmp, '--ide=json', '-q'])
+ gn_json = json.load(open(os.path.join(tmp, 'project.json')))
+ ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
+ except Exception as e:
+ err = str(e)
+ finally:
+ if tmp:
+ shutil.rmtree(tmp)
+ q.put((ans, err))
+
+
+def ParseGNProjectJSON(gn, out_dir, tmp_out):
+ """Parse GN output and get the header files"""
+ all_headers = set()
+
+ for _target, properties in gn['targets'].iteritems():
+ sources = properties.get('sources', [])
+ public = properties.get('public', [])
+ # Exclude '"public": "*"'.
+ if type(public) is list:
+ sources += public
+ for f in sources:
+ if f.endswith('.h') or f.endswith('.hh'):
+ if f.startswith('//'):
+ f = f[2:] # Strip the '//' prefix.
+ if f.startswith(tmp_out):
+ f = out_dir + f[len(tmp_out):]
+ all_headers.add(f)
+
+ return all_headers
+
+
+def GetDepsPrefixes(q):
+ """Return all the folders controlled by DEPS file"""
+ prefixes, err = set(), None
+ try:
+ gclient_exe = 'gclient.bat' if sys.platform == 'win32' else 'gclient'
+ gclient_out = subprocess.check_output([
+ os.path.join(DEPOT_TOOLS_DIR, gclient_exe),
+ 'recurse', '--no-progress', '-j1',
+ 'python', '-c', 'import os;print os.environ["GCLIENT_DEP_PATH"]'],
+ universal_newlines=True)
+ for i in gclient_out.split('\n'):
+ if i.startswith('src/'):
+ i = i[4:]
+ prefixes.add(i)
+ except Exception as e:
+ err = str(e)
+ q.put((prefixes, err))
+
+
+def IsBuildClean(out_dir):
+ cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-n']
+ try:
+ out = subprocess.check_output(cmd)
+ return 'no work to do.' in out
+ except Exception as e:
+ print(e)
+ return False
+
+def ParseWhiteList(whitelist):
+ out = set()
+ for line in whitelist.split('\n'):
+ line = re.sub(r'#.*', '', line).strip()
+ if line:
+ out.add(line)
+ return out
+
+
+def FilterOutDepsedRepo(files, deps):
+ return {f for f in files if not any(f.startswith(d) for d in deps)}
+
+
+def GetNonExistingFiles(lst):
+ out = set()
+ for f in lst:
+ if not os.path.isfile(f):
+ out.add(f)
+ return out
+
+
+def main():
+
+ def DumpJson(data):
+ if args.json:
+ with open(args.json, 'w') as f:
+ json.dump(data, f)
+
+ def PrintError(msg):
+ DumpJson([])
+ parser.error(msg)
+
+ parser = argparse.ArgumentParser(description='''
+ NOTE: Use ninja to build all targets in OUT_DIR before running
+ this script.''')
+ parser.add_argument('--out-dir', metavar='OUT_DIR', default='out/Release',
+ help='output directory of the build')
+ parser.add_argument('--json',
+ help='JSON output filename for missing headers')
+ parser.add_argument('--whitelist', help='file containing whitelist')
+ parser.add_argument('--skip-dirty-check', action='store_true',
+ help='skip checking whether the build is dirty')
+ parser.add_argument('--verbose', action='store_true',
+ help='print more diagnostic info')
+
+ args, _extras = parser.parse_known_args()
+
+ if not os.path.isdir(args.out_dir):
+ parser.error('OUT_DIR "%s" does not exist.' % args.out_dir)
+
+ if not args.skip_dirty_check and not IsBuildClean(args.out_dir):
+ dirty_msg = 'OUT_DIR looks dirty. You need to build all there.'
+ if args.json:
+ # Assume running on the bots. Silently skip this step.
+ # This is possible because "analyze" step can be wrong due to
+ # underspecified header files. See crbug.com/725877
+ print(dirty_msg)
+ DumpJson([])
+ return 0
+ else:
+ # Assume running interactively.
+ parser.error(dirty_msg)
+
+ d_q = Queue()
+ d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, True, d_q,))
+ d_p.start()
+
+ gn_q = Queue()
+ gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
+ gn_p.start()
+
+ deps_q = Queue()
+ deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
+ deps_p.start()
+
+ d, d_err = d_q.get()
+ gn, gn_err = gn_q.get()
+ missing = set(d.keys()) - gn
+ nonexisting = GetNonExistingFiles(gn)
+
+ deps, deps_err = deps_q.get()
+ missing = FilterOutDepsedRepo(missing, deps)
+ nonexisting = FilterOutDepsedRepo(nonexisting, deps)
+
+ d_p.join()
+ gn_p.join()
+ deps_p.join()
+
+ if d_err:
+ PrintError(d_err)
+ if gn_err:
+ PrintError(gn_err)
+ if deps_err:
+ PrintError(deps_err)
+ if len(GetNonExistingFiles(d)) > 0:
+ print('Non-existing files in ninja deps:', GetNonExistingFiles(d))
+ PrintError('Found non-existing files in ninja deps. You should ' +
+ 'build all in OUT_DIR.')
+ if len(d) == 0:
+ PrintError('OUT_DIR looks empty. You should build all there.')
+ if any((('/gen/' in i) for i in nonexisting)):
+ PrintError('OUT_DIR looks wrong. You should build all there.')
+
+ if args.whitelist:
+ whitelist = ParseWhiteList(open(args.whitelist).read())
+ missing -= whitelist
+ nonexisting -= whitelist
+
+ missing = sorted(missing)
+ nonexisting = sorted(nonexisting)
+
+ DumpJson(sorted(missing + nonexisting))
+
+ if len(missing) == 0 and len(nonexisting) == 0:
+ return 0
+
+ if len(missing) > 0:
+ print('\nThe following files should be included in gn files:')
+ for i in missing:
+ print(i)
+
+ if len(nonexisting) > 0:
+ print('\nThe following non-existing files should be removed from gn files:')
+ for i in nonexisting:
+ print(i)
+
+ if args.verbose:
+ # Only get detailed obj dependency here since it is slower.
+ GetHeadersFromNinja(args.out_dir, False, d_q)
+ d, d_err = d_q.get()
+ print('\nDetailed dependency info:')
+ for f in missing:
+ print(f)
+ for cc in d[f]:
+ print(' ', cc)
+
+ print('\nMissing headers sorted by number of affected object files:')
+ count = {k: len(v) for (k, v) in d.iteritems()}
+ for f in sorted(count, key=count.get, reverse=True):
+ if f in missing:
+ print(count[f], f)
+
+ if args.json:
+ # Assume running on the bots. Temporarily return 0 before
+ # https://crbug.com/937847 is fixed.
+ return 0
+ return 1
+
+
+if __name__ == '__main__':
+ sys.exit(main())