Adding upstream version 4:24.2.0.upstream/4%24.2.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 400 insertions, 0 deletions

diff --git a/bin/find-german-comments b/bin/find-german-comments
new file mode 100755
index 0000000000..f4ad23d9d2
--- /dev/null
+++ b/bin/find-german-comments
@@ -0,0 +1,400 @@
+#!/usr/bin/env python3
+########################################################################
+#
+#  Copyright (c) 2010 Jonas Jensen, Miklos Vajna
+#
+#  Permission is hereby granted, free of charge, to any person
+#  obtaining a copy of this software and associated documentation
+#  files (the "Software"), to deal in the Software without
+#  restriction, including without limitation the rights to use,
+#  copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the
+#  Software is furnished to do so, subject to the following
+#  conditions:
+#
+#  The above copyright notice and this permission notice shall be
+#  included in all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#  OTHER DEALINGS IN THE SOFTWARE.
+#
+########################################################################
+
+
+import sys
+import re
+import subprocess
+import os
+import argparse
+import string
+
+class Parser:
+    """
+    This parser extracts comments from source files, tries to guess
+    their language and then prints out the German ones.
+    """
+    def __init__(self):
+        self.strip = string.punctuation + " \n"
+        self.text_cat = self.start_text_cat()
+        parser = argparse.ArgumentParser(description='Searches for German comments in cxx/hxx source files inside a given root directory recursively.')
+        parser.add_argument("-f", "--filenames-only", action="store_true",
+            help="Only print the filenames of files containing German comments")
+        parser.add_argument("-v", "--verbose", action="store_true",
+            help="Turn on verbose mode (print only positives progress to stderr)")
+        parser.add_argument("-l", "--line-numbers", action="store_true",
+            help="Prints the filenames and line numbers only.")
+        parser.add_argument("-L", "--line-numbers-pos", action="store_true",
+            help="Prints the filenames and line numbers only (if positive).")
+        parser.add_argument("-t", "--threshold", action="store", default=0, type=int,
+            help="When used with '--line-numbers', only bothers outputting comment info if there are more than X number of flagged comments. Useful for weeding out false positives.")
+        parser.add_argument("directory", nargs='?', default='.', type=str, help='Give a directory to search in')
+        self.args = parser.parse_args()
+        self.check_source_files(self.args.directory)
+
+    def get_comments(self, filename):
+        """
+        Extracts the source code comments.
+        """
+        linenum = 0
+        if self.args.verbose:
+            print("processing file '%s'...\n" % filename)
+        sock = open(filename)
+        # add an empty line to trigger the output of collected oneliner
+        # comment group
+        lines = sock.readlines() + ["\n"]
+        sock.close()
+
+        in_comment = False
+        buf = []
+        count = 1
+        for i in lines:
+            if "//" in i and not in_comment:
+                # if we find a new //-style comment, then we
+                # just append it to a previous one if: there is
+                # only whitespace before the // mark that is
+                # necessary to make comments longer, giving
+                # more reliable output
+                if not len(re.sub("(.*)//.*", r"\1", i).strip(self.strip)):
+                    s = re.sub(".*// ?", "", i).strip(self.strip)
+                    if len(s):
+                        buf.append(s)
+                else:
+                    # otherwise it's an independent //-style comment in the next line
+                    yield (count, "\n    ".join(buf))
+                    buf = [re.sub(".*// ?", "", i.strip(self.strip))]
+            elif "//" not in i and not in_comment and len(buf) > 0:
+                # first normal line after a // block
+                yield (count, "\n    ".join(buf))
+                buf = []
+            elif "/*" in i and "*/" not in i and not in_comment:
+                # start of a real multiline comment
+                in_comment = True
+                linenum = count
+                s = re.sub(r".*/\*+", "", i.strip(self.strip))
+                if len(s):
+                    buf.append(s.strip(self.strip))
+            elif in_comment and not "*/" in i:
+                # in multiline comment
+                s = re.sub(r"^( |\|)*\*?", "", i)
+                if len(s.strip(self.strip)):
+                    buf.append(s.strip(self.strip))
+            elif "*/" in i and in_comment:
+                # end of multiline comment
+                in_comment = False
+                s = re.sub(r"\*+/.*", "", i.strip(self.strip))
+                if len(s):
+                    buf.append(s)
+                yield (count, "\n    ".join(buf))
+                buf = []
+            elif "/*" in i and "*/" in i:
+                # c-style oneliner comment
+                yield (count, re.sub(r".*/\*(.*)\*/.*", r"\1", i).strip(self.strip))
+            count += 1
+
+    def start_text_cat(self):
+        cwd = os.getcwd()
+        # change to our directory
+        os.chdir(os.path.split(os.path.abspath(sys.argv[0]))[0])
+        sock = subprocess.Popen(["text_cat/text_cat", "-s", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        os.chdir(cwd)
+        return sock
+
+    def get_lang(self, s):
+        """ the output is 'german' or 'english' or 'german or english'. When
+        unsure, just don't warn, there are strings where you just can't
+        determine the results reliably, like '#110680#' """
+
+        self.text_cat.stdin.write(bytes(s, 'utf-8'))
+        self.text_cat.stdin.write(bytes("\n", 'utf-8'))
+        self.text_cat.stdin.flush()
+        lang = self.text_cat.stdout.readline().strip()
+        return lang
+
+    def is_german(self, s):
+        """
+        determines if a string is German or not
+        """
+        # for short strings we can't do reliable recognition, so skip
+        # short strings and less than 4 words
+        s = s.replace('\n', ' ')
+        if len(s) < 32 or len(s.split()) < 4:
+            return False
+        return self.get_lang(s) == b"german"
+
+    def check_file(self, path):
+        """
+        checks each comment in a file
+        """
+        def tab_calc(path):
+            START = 40 #Default of 10 tabs
+            if len(path) >= START:
+                return 1
+            diff = START - len(path)
+            if diff % 4 is not 0:
+                padding = 1
+            else:
+                padding = 0
+            return (diff/4)+padding
+
+        if self.args.line_numbers or self.args.line_numbers_pos:
+            TABS = "\t"*10
+            path_linenums = []
+            for linenum, s in self.get_comments(path):
+                if self.is_german(s):
+                    path_linenums.append(linenum)
+            valid = len(path_linenums) > int(self.args.threshold)
+            if self.args.line_numbers:
+                print("%s ... %s positives -- %s\n" % (path, str(len(path_linenums)), str(valid)))
+            if valid:
+                if self.args.line_numbers_pos:
+                    print("%s ... %s positives\n" % (path, str(len(path_linenums))))
+                    return
+                if len(path) + (len(path_linenums)*4) > 75:
+                    print("%s:\n" % path)
+                    while path_linenums:
+                        i = 0
+                        numline = []
+                        while i < 10:
+                            try:
+                                numline.append(path_linenums[0])
+                                path_linenums.remove(path_linenums[0])
+                            except IndexError:
+                                i = 10
+                            i += 1
+                        numline = [str(i) for i in numline]
+                        print("%s%s" % (TABS, ",".join(numline)))
+                else:
+                    if self.args.line_numbers:
+                        path_linenums = [str(i) for i in path_linenums]
+                        print("%s:%s%s" % (path, "\t"*int(tab_calc(path)), ",".join(path_linenums)))
+
+        elif not self.args.filenames_only:
+            for linenum, s in self.get_comments(path):
+                if self.is_german(s):
+                    print("%s:%s: %s" % (path, linenum, s))
+        else:
+            fnames = set([])
+            for linenum, s in self.get_comments(path):
+                if self.is_german(s):
+                    # Make sure we print each filename only once
+                    fnames.add(path)
+            # Print the filenames
+            for f in fnames:
+                print(f)
+
+    def first_elem(self, path):
+        """
+        Returns the root directory in our repo of a given path, so we can check against the allowlist.
+        """
+        lastElem = os.path.dirname(path)
+        done = False
+        while not done:
+            nextElem = os.path.split(lastElem)[0]
+            if nextElem is not '':
+                lastElem = nextElem
+            else:
+                done = True
+        return lastElem
+
+    def check_source_files(self, directory):
+        """
+        checks each _tracked_ file in a directory recursively
+        """
+
+	# top-level project directory -> use allowlist.
+        globalscan = False
+        if os.path.exists(directory + "/.git/config"):
+           globalscan = True
+
+        # Change into the given dir, so "git ls-tree" does work.
+        os.chdir(directory)
+
+        sock = os.popen(r"git ls-tree -r HEAD --name-only | grep -E '\.(c|cc|cpp|cxx|h|hxx|mm)$'")
+        lines = sock.readlines()
+        sock.close()
+
+        # Helps to speedup a global scan
+        directory_allowlist = {
+            "ure" : 1,
+            "ios" : 1,
+            "bean" : 1,
+            "apple_remote" : 1,
+            "UnoControls" : 1,
+            "accessibility" : 1,
+            "android" : 1,
+            "animations" : 1,
+            "avmedia" : 1,
+            "basctl" : 1,
+            "basegfx" : 1,
+            "basic" : 1,
+            "binaryurp" : 1,
+            "bridges" : 1,
+            "canvas" : 1,
+            "chart2" : 1,
+            "cli_ure" : 1,
+            "codemaker" : 1,
+            "comphelper" : 1,
+            "compilerplugins" : 1,
+            "configmgr" : 1,
+            "connectivity" : 1,
+            "cppcanvas" : 1,
+            "cppu" : 1,
+            "cppuhelper" : 1,
+            "cpputools" : 1,
+            "cui" : 1,
+            "dbaccess" : 1,
+            "desktop" : 1,
+            "drawinglayer" : 1,
+            "editeng" : 1,
+            "embeddedobj" : 1,
+            "embedserv" : 1,
+            "eventattacher" : 1,
+            "extensions" : 1,
+            "external" : 1,
+            "filter" : 1,
+            "forms" : 1,
+            "formula" : 1,
+            "fpicker" : 1,
+            "framework" : 1,
+            "helpcompiler" : 1,
+            "hwpfilter" : 1,
+            "i18npool" : 1,
+            "i18nlangtag" : 1,
+            "i18nutil" : 1,
+            "idl" : 1,
+            "idlc" : 1,
+            "include" : 1,
+            "io" : 1,
+            "javaunohelper" : 1,
+            "jvmaccess" : 1,
+            "jvmfwk" : 1,
+            "jurt" : 1,
+            "l10ntools" : 1,
+            "libreofficekit" : 1,
+            "lingucomponent" : 1,
+            "linguistic" : 1,
+            "lotuswordpro" : 1,
+            "mysqlc" : 1,
+            "o3tl" : 1,
+            "odk" : 1,
+            "officecfg" : 1,
+            "opencl" : 1,
+            "oox" : 1,
+            "package" : 1,
+            "postprocess" : 1,
+            "pyuno" : 1,
+            "registry" : 1,
+            "remotebridges" : 1,
+            "reportdesign" : 1,
+            "rsc" : 1,
+            "sal" : 1,
+            "salhelper" : 1,
+            "sax" : 1,
+            "sc" : 1,
+            "scaddins" : 1,
+            "sccomp" : 1,
+            "scripting" : 1,
+            "sd" : 1,
+            "sdext" : 1,
+            "sfx2" : 1,
+            "shell" : 1,
+            "setup_native" : 1,
+            "sot" : 1,
+            "slideshow" : 1,
+            "smoketest" : 1,
+            "solenv" : 1,
+            "soltools" : 1,
+            "starmath" : 1,
+            "stoc" : 1,
+            "store" : 1,
+            "svgio" : 1,
+            "svl" : 1,
+            "svtools" : 1,
+            "svx" : 1,
+            "sw" : 1,
+            "test" : 1,
+            "testtools" : 1,
+            "toolkit" : 1,
+            "tools" : 1,
+            "touch" : 1,
+            "ucb" : 1,
+            "ucbhelper" : 1,
+            "unodevtools" : 1,
+            "unotest" : 1,
+            "unoidl" : 1,
+            "unotools" : 1,
+            "unoxml" : 1,
+            "uui" : 1,
+            "vbahelper" : 1,
+            "vcl" : 1,
+            "winaccessibility" : 1,
+            "writerfilter" : 1,
+            "writerperfect" : 1,
+            "xmlhelp" : 1,
+            "xmloff" : 1,
+            "xmlreader" : 1,
+            "xmlsecurity" : 1,
+            "xmlscript" : 1,
+        }
+
+        if globalscan:
+            print("Scanning all files globally:")
+        elif directory == '.':
+            print("Scanning all files in our current directory:")
+        else:
+            print("Scanning all files in", directory + ":")
+
+        num_checked = 0
+
+        for path in lines:
+            baseDir = self.first_elem(path)
+            # If we have a globalscan use the allowlist.
+            if globalscan:
+                if not baseDir in directory_allowlist:
+                    sys.stderr.write("\n - Error: Missing path %s -\n\n" % baseDir)
+                    sys.exit(1)
+                elif directory_allowlist[baseDir] is 0:
+                    self.check_file(path.strip())
+                    num_checked = num_checked + 1
+                elif directory_allowlist[baseDir] is 1:
+                    sys.stderr.write("Skipping excluded directory %s\n" % baseDir)
+                    directory_allowlist[baseDir] = 2
+            elif not globalscan:
+                self.check_file(path.strip())
+                num_checked = num_checked + 1
+
+        print("Scanned %s files\n" % num_checked)
+
+try:
+    Parser()
+except KeyboardInterrupt:
+    print("Interrupted!")
+    sys.exit(0)
+
+# vim:set shiftwidth=4 softtabstop=4 expandtab: