summaryrefslogtreecommitdiffstats
path: root/scripts/spdxcheck.py
blob: 8b8fb115fc819507c6ac3e12970d9ae8cabf6b69 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
# Copyright Thomas Gleixner <tglx@linutronix.de>

from argparse import ArgumentParser
from ply import lex, yacc
import locale
import traceback
import fnmatch
import sys
import git
import re
import os

class ParserException(Exception):
    def __init__(self, tok, txt):
        self.tok = tok
        self.txt = txt

class SPDXException(Exception):
    def __init__(self, el, txt):
        self.el = el
        self.txt = txt

class SPDXdata(object):
    def __init__(self):
        self.license_files = 0
        self.exception_files = 0
        self.licenses = [ ]
        self.exceptions = { }

class dirinfo(object):
    def __init__(self):
        self.missing = 0
        self.total = 0
        self.files = []

    def update(self, fname, basedir, miss):
        self.total += 1
        self.missing += miss
        if miss:
            fname = './' + fname
            bdir = os.path.dirname(fname)
            if bdir == basedir.rstrip('/'):
                self.files.append(fname)

# Read the spdx data from the LICENSES directory
def read_spdxdata(repo):

    # The subdirectories of LICENSES in the kernel source
    # Note: exceptions needs to be parsed as last directory.
    license_dirs = [ "preferred", "dual", "deprecated", "exceptions" ]
    lictree = repo.head.commit.tree['LICENSES']

    spdx = SPDXdata()

    for d in license_dirs:
        for el in lictree[d].traverse():
            if not os.path.isfile(el.path):
                continue

            exception = None
            for l in open(el.path, encoding="utf-8").readlines():
                if l.startswith('Valid-License-Identifier:'):
                    lid = l.split(':')[1].strip().upper()
                    if lid in spdx.licenses:
                        raise SPDXException(el, 'Duplicate License Identifier: %s' %lid)
                    else:
                        spdx.licenses.append(lid)

                elif l.startswith('SPDX-Exception-Identifier:'):
                    exception = l.split(':')[1].strip().upper()
                    spdx.exceptions[exception] = []

                elif l.startswith('SPDX-Licenses:'):
                    for lic in l.split(':')[1].upper().strip().replace(' ', '').replace('\t', '').split(','):
                        if not lic in spdx.licenses:
                            raise SPDXException(None, 'Exception %s missing license %s' %(exception, lic))
                        spdx.exceptions[exception].append(lic)

                elif l.startswith("License-Text:"):
                    if exception:
                        if not len(spdx.exceptions[exception]):
                            raise SPDXException(el, 'Exception %s is missing SPDX-Licenses' %exception)
                        spdx.exception_files += 1
                    else:
                        spdx.license_files += 1
                    break
    return spdx

class id_parser(object):

    reserved = [ 'AND', 'OR', 'WITH' ]
    tokens = [ 'LPAR', 'RPAR', 'ID', 'EXC' ] + reserved

    precedence = ( ('nonassoc', 'AND', 'OR'), )

    t_ignore = ' \t'

    def __init__(self, spdx):
        self.spdx = spdx
        self.lasttok = None
        self.lastid = None
        self.lexer = lex.lex(module = self, reflags = re.UNICODE)
        # Initialize the parser. No debug file and no parser rules stored on disk
        # The rules are small enough to be generated on the fly
        self.parser = yacc.yacc(module = self, write_tables = False, debug = False)
        self.lines_checked = 0
        self.checked = 0
        self.excluded = 0
        self.spdx_valid = 0
        self.spdx_errors = 0
        self.spdx_dirs = {}
        self.dirdepth = -1
        self.basedir = '.'
        self.curline = 0
        self.deepest = 0

    def set_dirinfo(self, basedir, dirdepth):
        if dirdepth >= 0:
            self.basedir = basedir
            bdir = basedir.lstrip('./').rstrip('/')
            if bdir != '':
                parts = bdir.split('/')
            else:
                parts = []
            self.dirdepth = dirdepth + len(parts)

    # Validate License and Exception IDs
    def validate(self, tok):
        id = tok.value.upper()
        if tok.type == 'ID':
            if not id in self.spdx.licenses:
                raise ParserException(tok, 'Invalid License ID')
            self.lastid = id
        elif tok.type == 'EXC':
            if id not in self.spdx.exceptions:
                raise ParserException(tok, 'Invalid Exception ID')
            if self.lastid not in self.spdx.exceptions[id]:
                raise ParserException(tok, 'Exception not valid for license %s' %self.lastid)
            self.lastid = None
        elif tok.type != 'WITH':
            self.lastid = None

    # Lexer functions
    def t_RPAR(self, tok):
        r'\)'
        self.lasttok = tok.type
        return tok

    def t_LPAR(self, tok):
        r'\('
        self.lasttok = tok.type
        return tok

    def t_ID(self, tok):
        r'[A-Za-z.0-9\-+]+'

        if self.lasttok == 'EXC':
            print(tok)
            raise ParserException(tok, 'Missing parentheses')

        tok.value = tok.value.strip()
        val = tok.value.upper()

        if val in self.reserved:
            tok.type = val
        elif self.lasttok == 'WITH':
            tok.type = 'EXC'

        self.lasttok = tok.type
        self.validate(tok)
        return tok

    def t_error(self, tok):
        raise ParserException(tok, 'Invalid token')

    def p_expr(self, p):
        '''expr : ID
                | ID WITH EXC
                | expr AND expr
                | expr OR expr
                | LPAR expr RPAR'''
        pass

    def p_error(self, p):
        if not p:
            raise ParserException(None, 'Unfinished license expression')
        else:
            raise ParserException(p, 'Syntax error')

    def parse(self, expr):
        self.lasttok = None
        self.lastid = None
        self.parser.parse(expr, lexer = self.lexer)

    def parse_lines(self, fd, maxlines, fname):
        self.checked += 1
        self.curline = 0
        fail = 1
        try:
            for line in fd:
                line = line.decode(locale.getpreferredencoding(False), errors='ignore')
                self.curline += 1
                if self.curline > maxlines:
                    break
                self.lines_checked += 1
                if line.find("SPDX-License-Identifier:") < 0:
                    continue
                expr = line.split(':')[1].strip()
                # Remove trailing comment closure
                if line.strip().endswith('*/'):
                    expr = expr.rstrip('*/').strip()
                # Remove trailing xml comment closure
                if line.strip().endswith('-->'):
                    expr = expr.rstrip('-->').strip()
                # Special case for SH magic boot code files
                if line.startswith('LIST \"'):
                    expr = expr.rstrip('\"').strip()
                self.parse(expr)
                self.spdx_valid += 1
                #
                # Should we check for more SPDX ids in the same file and
                # complain if there are any?
                #
                fail = 0
                break

        except ParserException as pe:
            if pe.tok:
                col = line.find(expr) + pe.tok.lexpos
                tok = pe.tok.value
                sys.stdout.write('%s: %d:%d %s: %s\n' %(fname, self.curline, col, pe.txt, tok))
            else:
                sys.stdout.write('%s: %d:0 %s\n' %(fname, self.curline, pe.txt))
            self.spdx_errors += 1

        if fname == '-':
            return

        base = os.path.dirname(fname)
        if self.dirdepth > 0:
            parts = base.split('/')
            i = 0
            base = '.'
            while i < self.dirdepth and i < len(parts) and len(parts[i]):
                base += '/' + parts[i]
                i += 1
        elif self.dirdepth == 0:
            base = self.basedir
        else:
            base = './' + base.rstrip('/')
        base += '/'

        di = self.spdx_dirs.get(base, dirinfo())
        di.update(fname, base, fail)
        self.spdx_dirs[base] = di

class pattern(object):
    def __init__(self, line):
        self.pattern = line
        self.match = self.match_file
        if line == '.*':
            self.match = self.match_dot
        elif line.endswith('/'):
            self.pattern = line[:-1]
            self.match = self.match_dir
        elif line.startswith('/'):
            self.pattern = line[1:]
            self.match = self.match_fn

    def match_dot(self, fpath):
        return os.path.basename(fpath).startswith('.')

    def match_file(self, fpath):
        return os.path.basename(fpath) == self.pattern

    def match_fn(self, fpath):
        return fnmatch.fnmatchcase(fpath, self.pattern)

    def match_dir(self, fpath):
        if self.match_fn(os.path.dirname(fpath)):
            return True
        return fpath.startswith(self.pattern)

def exclude_file(fpath):
    for rule in exclude_rules:
        if rule.match(fpath):
            return True
    return False

def scan_git_tree(tree, basedir, dirdepth):
    parser.set_dirinfo(basedir, dirdepth)
    for el in tree.traverse():
        if not os.path.isfile(el.path):
            continue
        if exclude_file(el.path):
            parser.excluded += 1
            continue
        with open(el.path, 'rb') as fd:
            parser.parse_lines(fd, args.maxlines, el.path)

def scan_git_subtree(tree, path, dirdepth):
    for p in path.strip('/').split('/'):
        tree = tree[p]
    scan_git_tree(tree, path.strip('/'), dirdepth)

def read_exclude_file(fname):
    rules = []
    if not fname:
        return rules
    with open(fname) as fd:
        for line in fd:
            line = line.strip()
            if line.startswith('#'):
                continue
            if not len(line):
                continue
            rules.append(pattern(line))
    return rules

if __name__ == '__main__':

    ap = ArgumentParser(description='SPDX expression checker')
    ap.add_argument('path', nargs='*', help='Check path or file. If not given full git tree scan. For stdin use "-"')
    ap.add_argument('-d', '--dirs', action='store_true',
                    help='Show [sub]directory statistics.')
    ap.add_argument('-D', '--depth', type=int, default=-1,
                    help='Directory depth for -d statistics. Default: unlimited')
    ap.add_argument('-e', '--exclude',
                    help='File containing file patterns to exclude. Default: scripts/spdxexclude')
    ap.add_argument('-f', '--files', action='store_true',
                    help='Show files without SPDX.')
    ap.add_argument('-m', '--maxlines', type=int, default=15,
                    help='Maximum number of lines to scan in a file. Default 15')
    ap.add_argument('-v', '--verbose', action='store_true', help='Verbose statistics output')
    args = ap.parse_args()

    # Sanity check path arguments
    if '-' in args.path and len(args.path) > 1:
        sys.stderr.write('stdin input "-" must be the only path argument\n')
        sys.exit(1)

    try:
        # Use git to get the valid license expressions
        repo = git.Repo(os.getcwd())
        assert not repo.bare

        # Initialize SPDX data
        spdx = read_spdxdata(repo)

        # Initialize the parser
        parser = id_parser(spdx)

    except SPDXException as se:
        if se.el:
            sys.stderr.write('%s: %s\n' %(se.el.path, se.txt))
        else:
            sys.stderr.write('%s\n' %se.txt)
        sys.exit(1)

    except Exception as ex:
        sys.stderr.write('FAIL: %s\n' %ex)
        sys.stderr.write('%s\n' %traceback.format_exc())
        sys.exit(1)

    try:
        fname = args.exclude
        if not fname:
            fname = os.path.join(os.path.dirname(__file__), 'spdxexclude')
        exclude_rules = read_exclude_file(fname)
    except Exception as ex:
        sys.stderr.write('FAIL: Reading exclude file %s: %s\n' %(fname, ex))
        sys.exit(1)

    try:
        if len(args.path) and args.path[0] == '-':
            stdin = os.fdopen(sys.stdin.fileno(), 'rb')
            parser.parse_lines(stdin, args.maxlines, '-')
        else:
            if args.path:
                for p in args.path:
                    if os.path.isfile(p):
                        parser.parse_lines(open(p, 'rb'), args.maxlines, p)
                    elif os.path.isdir(p):
                        scan_git_subtree(repo.head.reference.commit.tree, p,
                                         args.depth)
                    else:
                        sys.stderr.write('path %s does not exist\n' %p)
                        sys.exit(1)
            else:
                # Full git tree scan
                scan_git_tree(repo.head.commit.tree, '.', args.depth)

            ndirs = len(parser.spdx_dirs)
            dirsok = 0
            if ndirs:
                for di in parser.spdx_dirs.values():
                    if not di.missing:
                        dirsok += 1

            if args.verbose:
                sys.stderr.write('\n')
                sys.stderr.write('License files:     %12d\n' %spdx.license_files)
                sys.stderr.write('Exception files:   %12d\n' %spdx.exception_files)
                sys.stderr.write('License IDs        %12d\n' %len(spdx.licenses))
                sys.stderr.write('Exception IDs      %12d\n' %len(spdx.exceptions))
                sys.stderr.write('\n')
                sys.stderr.write('Files excluded:    %12d\n' %parser.excluded)
                sys.stderr.write('Files checked:     %12d\n' %parser.checked)
                sys.stderr.write('Lines checked:     %12d\n' %parser.lines_checked)
                if parser.checked:
                    pc = int(100 * parser.spdx_valid / parser.checked)
                    sys.stderr.write('Files with SPDX:   %12d %3d%%\n' %(parser.spdx_valid, pc))
                    missing = parser.checked - parser.spdx_valid
                    mpc = int(100 * missing / parser.checked)
                    sys.stderr.write('Files without SPDX:%12d %3d%%\n' %(missing, mpc))
                sys.stderr.write('Files with errors: %12d\n' %parser.spdx_errors)
                if ndirs:
                    sys.stderr.write('\n')
                    sys.stderr.write('Directories accounted: %8d\n' %ndirs)
                    pc = int(100 * dirsok / ndirs)
                    sys.stderr.write('Directories complete:  %8d %3d%%\n' %(dirsok, pc))

            if ndirs and ndirs != dirsok and args.dirs:
                if args.verbose:
                    sys.stderr.write('\n')
                sys.stderr.write('Incomplete directories: SPDX in Files\n')
                for f in sorted(parser.spdx_dirs.keys()):
                    di = parser.spdx_dirs[f]
                    if di.missing:
                        valid = di.total - di.missing
                        pc = int(100 * valid / di.total)
                        sys.stderr.write('    %-80s: %5d of %5d  %3d%%\n' %(f, valid, di.total, pc))

            if ndirs and ndirs != dirsok and args.files:
                if args.verbose or args.dirs:
                    sys.stderr.write('\n')
                sys.stderr.write('Files without SPDX:\n')
                for f in sorted(parser.spdx_dirs.keys()):
                    di = parser.spdx_dirs[f]
                    for f in sorted(di.files):
                        sys.stderr.write('    %s\n' %f)

            sys.exit(0)

    except Exception as ex:
        sys.stderr.write('FAIL: %s\n' %ex)
        sys.stderr.write('%s\n' %traceback.format_exc())
        sys.exit(1)