diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 16:51:28 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 16:51:28 +0000 |
| commit | 940b4d1848e8c70ab7642901a68594e8016caffc (patch) | |
| tree | eb72f344ee6c3d9b80a7ecc079ea79e9fba8676d /bin | |
| parent | Initial commit. (diff) | |
| download | libreoffice-upstream.tar.xz libreoffice-upstream.zip | |
Adding upstream version 1:7.0.4.upstream/1%7.0.4upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'bin')
104 files changed, 19314 insertions, 0 deletions
diff --git a/bin/README b/bin/README new file mode 100644 index 000000000..d5d0829ce --- /dev/null +++ b/bin/README @@ -0,0 +1,9 @@ +Tools and scripts mostly not used during the build + +This direction has a number of key pieces (?) that are used during the +build, or are simply generally useful. One example is + +bin/find-german-comments <directory> + +which will try to detect and extract all the German comments in a +given source code hierarchy / directory. diff --git a/bin/bash-completion.in b/bin/bash-completion.in new file mode 100644 index 000000000..77087c593 --- /dev/null +++ b/bin/bash-completion.in @@ -0,0 +1,90 @@ +# Programmable bash_completion file for the main office applications +# It is based on /etc/profile.d/complete.bash from SUSE Linux 10.1 + +_def=; _dir=; _file=; _nosp= +if complete -o default _nullcommand &> /dev/null ; then + _def="-o default" + _dir="-o dirnames" + _file="-o filenames" +fi +_minusdd="-d ${_dir}" +_minusdf="-d ${_file}" +if complete -o nospace _nullcommand &> /dev/null ; then + _nosp="-o nospace" + _minusdd="${_nosp} ${_dir}" + _minusdf="${_nosp} ${_dir}" +fi +complete -r _nullcommand &> /dev/null + +# General expanding shell function +@OFFICE_SHELL_FUNCTION@ () +{ + # bash `complete' is broken because you can not combine + # -d, -f, and -X pattern without missing directories. + local c=${COMP_WORDS[COMP_CWORD]} + local a="${COMP_LINE}" + local e s g=0 cd dc t="" + local IFS + + shopt -q extglob && g=1 + test $g -eq 0 && shopt -s extglob + # Don't be fooled by the bash parser if extglob is off by default + cd='*-?(c)d*' + dc='*-d?(c)*' + + case "${1##*/}" in +@BASH_COMPLETION_SUFFIXES_CHECKS@ + *) e='!*' + esac + + case "$(complete -p ${1##*/} 2> /dev/null)" in + *-d*) ;; + *) s="-S/" + esac + + IFS=' +' + case "$c" in + \$\(*\)) eval COMPREPLY=\(${c}\) ;; + \$\(*) COMPREPLY=($(compgen -c -P '$(' -S ')' -- ${c#??})) ;; + \`*\`) eval COMPREPLY=\(${c}\) ;; + \`*) COMPREPLY=($(compgen -c -P '\`' -S '\`' -- ${c#?})) ;; + \$\{*\}) eval COMPREPLY=\(${c}\) ;; + \$\{*) COMPREPLY=($(compgen -v -P '${' -S '}' -- ${c#??})) ;; + \$*) COMPREPLY=($(compgen -v -P '$' -- ${c#?})) ;; + \~*/*) COMPREPLY=($(compgen -f -X "$e" -- ${c})) ;; + \~*) COMPREPLY=($(compgen -u ${s} -- ${c})) ;; + *@*) COMPREPLY=($(compgen -A hostname -P '@' -S ':' -- ${c#*@})) ;; + *[*?[]*) COMPREPLY=($(compgen -G "${c}")) ;; + *[?*+\!@]\(*\)*) + if test $g -eq 0 ; then + COMPREPLY=($(compgen -f -X "$e" -- $c)) + test $g -eq 0 && shopt -u extglob + return + fi + COMPREPLY=($(compgen -G "${c}")) ;; + *) + if test "$c" = ".." ; then + COMPREPLY=($(compgen -d -X "$e" -S / ${_nosp} -- $c)) + else + for s in $(compgen -f -X "$e" -- $c) ; do + if test -d $s ; then + COMPREPLY=(${COMPREPLY[@]} $(compgen -f -X "$e" -S / -- $s)) + elif test -z "$t" ; then + COMPREPLY=(${COMPREPLY[@]} $s) + else + case "$(file -b $s 2> /dev/null)" in + $t) COMPREPLY=(${COMPREPLY[@]} $s) ;; + esac + fi + done + fi ;; + esac + test $g -eq 0 && shopt -u extglob +} + + +complete -d -X '.[^./]*' -F @OFFICE_SHELL_FUNCTION@ ${_file} \ +@BASH_COMPLETION_OOO_APPS@ + +unset _def _dir _file _nosp _minusdd _minusdf diff --git a/bin/benchmark-document-loading b/bin/benchmark-document-loading new file mode 100644 index 000000000..11611a2b2 --- /dev/null +++ b/bin/benchmark-document-loading @@ -0,0 +1,486 @@ +#!/usr/bin/env python # -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# Version: MPL 1.1 / GPLv3+ / LGPLv3+ +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License or as specified alternatively below. You may obtain a copy of +# the License at http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# Major Contributor(s): +# Copyright (C) 2012 Red Hat, Inc., Michael Stahl <mstahl@redhat.com> +# (initial developer) +# +# All Rights Reserved. +# +# For minor contributions see the git repository. +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 3 or later (the "GPLv3+"), or +# the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"), +# in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable +# instead of those above. + +# Simple script to load a bunch of documents and export them as Flat ODF +# +# Personally I run it like this: +# ~/lo/master-suse/instdir/program/python ~/lo/master-suse/bin/benchmark-document-loading --soffice=path:/home/tml/lo/master-suse/instdir/program/soffice --outdir=file://$PWD/out --userdir=file:///tmp/test $PWD/docs +# + +import argparse +import datetime +import os +import subprocess +import sys +import threading +import time +import urllib +try: + from urllib.parse import quote +except ImportError: + from urllib import quote +import uuid + +try: + import pyuno + import uno + import unohelper +except ImportError: + print("pyuno not found: try to set PYTHONPATH and URE_BOOTSTRAP variables") + print("PYTHONPATH=/installation/opt/program") + print("URE_BOOTSTRAP=file:///installation/opt/program/fundamentalrc") + raise + +try: + from com.sun.star.beans import PropertyValue + from com.sun.star.document import XDocumentEventListener + from com.sun.star.io import IOException, XOutputStream +except ImportError: + print("UNO API class not found: try to set URE_BOOTSTRAP variable") + print("URE_BOOTSTRAP=file:///installation/opt/program/fundamentalrc") + raise + +validCalcFileExtensions = [ ".xlsx", ".xls", ".ods", ".fods" ] +validWriterFileExtensions = [ ".docx" , ".rtf", ".odt", ".fodt", ".doc" ] +validImpressFileExtensions = [ ".ppt", ".pptx", ".odp", ".fodp" ] +validDrawFileExtensions = [ ".odg", ".fodg" ] +validRevereseFileExtensions = [ ".vsd", ".vdx", ".cdr", ".pub", ".wpd" ] +validFileExtensions = {"calc": validCalcFileExtensions, + "writer": validWriterFileExtensions, + "impress": validImpressFileExtensions, + "draw": validDrawFileExtensions, + "reverse": validRevereseFileExtensions} +flatODFTypes = {"calc": (".fods", "OpenDocument Spreadsheet Flat XML"), + "writer": (".fodt", "OpenDocument Text Flat XML"), + "impress": (".fodp", "OpenDocument Presentation Flat XML"), + "draw": (".fodg", "OpenDocument Drawing Flat XML")} + +outdir = "" + +def partition(list, pred): + left = [] + right = [] + for e in list: + if pred(e): + left.append(e) + else: + right.append(e) + return (left, right) + +def filelist(directory, suffix): + if not directory: + raise Exception("filelist: empty directory") + if directory[-1] != "/": + directory += "/" + files = [directory + f for f in os.listdir(directory)] +# print(files) + return [f for f in files + if os.path.isfile(f) and os.path.splitext(f)[1] == suffix] + +def getFiles(dirs, suffix): +# print( dirs ) + files = [] + for d in dirs: + files += filelist(d, suffix) + return files + +### UNO utilities ### + +class OutputStream( unohelper.Base, XOutputStream ): + def __init__( self ): + self.closed = 0 + + def closeOutput(self): + self.closed = 1 + + def writeBytes( self, seq ): + sys.stdout.write( seq.value ) + + def flush( self ): + pass + +class OfficeConnection: + def __init__(self, args): + self.args = args + self.soffice = None + self.socket = None + self.xContext = None + self.pro = None + def setUp(self): + (method, sep, rest) = self.args.soffice.partition(":") + if sep != ":": + raise Exception("soffice parameter does not specify method") + if method == "path": + socket = "pipe,name=pytest" + str(uuid.uuid1()) + userdir = self.args.userdir + if not userdir: + raise Exception("'path' method requires --userdir") + if not userdir.startswith("file://"): + raise Exception("--userdir must be file URL") + self.soffice = self.bootstrap(rest, userdir, socket) + elif method == "connect": + socket = rest + else: + raise Exception("unsupported connection method: " + method) + self.xContext = self.connect(socket) + + def bootstrap(self, soffice, userdir, socket): + argv = [ soffice, "--accept=" + socket + ";urp", + "-env:UserInstallation=" + userdir, + "--quickstart=no", + "--norestore", "--nologo", "--headless" ] + if self.args.valgrind: + argv.append("--valgrind") + os.putenv("SAL_LOG", "-INFO-WARN") + os.putenv("LIBO_ONEWAY_STABLE_ODF_EXPORT", "YES") + self.pro = subprocess.Popen(argv) +# print(self.pro.pid) + + def connect(self, socket): + xLocalContext = uno.getComponentContext() + xUnoResolver = xLocalContext.ServiceManager.createInstanceWithContext("com.sun.star.bridge.UnoUrlResolver", xLocalContext) + url = "uno:" + socket + ";urp;StarOffice.ComponentContext" +# print("OfficeConnection: connecting to: " + url) + while True: + try: + xContext = xUnoResolver.resolve(url) + return xContext +# except com.sun.star.connection.NoConnectException + except pyuno.getClass("com.sun.star.connection.NoConnectException"): +# print("NoConnectException: sleeping...") + time.sleep(1) + + def tearDown(self): + if self.soffice: + if self.xContext: + try: +# print("tearDown: calling terminate()...") + xMgr = self.xContext.ServiceManager + xDesktop = xMgr.createInstanceWithContext("com.sun.star.frame.Desktop", self.xContext) + xDesktop.terminate() +# print("...done") +# except com.sun.star.lang.DisposedException: + except pyuno.getClass("com.sun.star.beans.UnknownPropertyException"): +# print("caught UnknownPropertyException while TearDown") + pass # ignore, also means disposed + except pyuno.getClass("com.sun.star.lang.DisposedException"): +# print("caught DisposedException while TearDown") + pass # ignore + else: + self.soffice.terminate() + ret = self.soffice.wait() + self.xContext = None + self.socket = None + self.soffice = None + if ret != 0: + raise Exception("Exit status indicates failure: " + str(ret)) +# return ret + def kill(self): + command = "kill " + str(self.pro.pid) + with open("killFile.log", "a") as killFile: + killFile.write(command + "\n") +# print("kill") +# print(command) + os.system(command) + +class PersistentConnection: + def __init__(self, args): + self.args = args + self.connection = None + def getContext(self): + return self.connection.xContext + def setUp(self): + assert(not self.connection) + conn = OfficeConnection(self.args) + conn.setUp() + self.connection = conn + def preTest(self): + assert(self.connection) + def postTest(self): + assert(self.connection) + def tearDown(self): + if self.connection: + try: + self.connection.tearDown() + finally: + self.connection = None + def kill(self): + if self.connection: + self.connection.kill() + +def simpleInvoke(connection, test): + try: + connection.preTest() + test.run(connection.getContext(), connection) + finally: + connection.postTest() + +def runConnectionTests(connection, invoker, tests): + try: + connection.setUp() + for test in tests: + invoker(connection, test) + finally: + pass + #connection.tearDown() + +class EventListener(XDocumentEventListener,unohelper.Base): + def __init__(self): + self.layoutFinished = False + def documentEventOccured(self, event): +# print(str(event.EventName)) + if event.EventName == "OnLayoutFinished": + self.layoutFinished = True + def disposing(event): + pass + +def mkPropertyValue(name, value): + return uno.createUnoStruct("com.sun.star.beans.PropertyValue", + name, 0, value, 0) + +### tests ### + +def logTimeSpent(url, startTime): + print(os.path.basename(urllib.parse.urlparse(url).path) + "\t" + str(time.time()-startTime)) + +def loadFromURL(xContext, url, t, component): + xDesktop = xContext.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", xContext) + props = [("Hidden", True), ("ReadOnly", True)] # FilterName? + loadProps = tuple([mkPropertyValue(name, value) for (name, value) in props]) + xListener = None + if component == "writer": + xListener = EventListener() + xGEB = xContext.getValueByName( + "/singletons/com.sun.star.frame.theGlobalEventBroadcaster") + xGEB.addDocumentEventListener(xListener) + try: + xDoc = None + startTime = time.time() + xDoc = xDesktop.loadComponentFromURL(url, "_blank", 0, loadProps) + if component == "calc": + try: + if xDoc: + xDoc.calculateAll() + except AttributeError: + pass + t.cancel() + logTimeSpent(url, startTime) + return xDoc + elif component == "writer": + time_ = 0 + t.cancel() + while time_ < 30: + if xListener.layoutFinished: + logTimeSpent(url, startTime) + return xDoc +# print("delaying...") + time_ += 1 + time.sleep(1) + else: + t.cancel() + logTimeSpent(url, startTime) + return xDoc + with open("file.log", "a") as fh: + fh.write("layout did not finish\n") + return xDoc + except pyuno.getClass("com.sun.star.beans.UnknownPropertyException"): + xListener = None + raise # means crashed, handle it later + except pyuno.getClass("com.sun.star.lang.DisposedException"): + xListener = None + raise # means crashed, handle it later + except pyuno.getClass("com.sun.star.lang.IllegalArgumentException"): + pass # means could not open the file, ignore it + except: + if xDoc: +# print("CLOSING") + xDoc.close(True) + raise + finally: + if xListener: + xGEB.removeDocumentEventListener(xListener) + +def exportToODF(xContext, xDoc, baseName, t, component): + exportFileName = outdir + "/" + os.path.splitext(baseName)[0] + flatODFTypes[component][0] + print("exportToODF " + baseName + " => " + exportFileName) + props = [("FilterName", flatODFTypes[component][1]), + ("Overwrite", True)] + storeProps = tuple([mkPropertyValue(name, value) for (name, value) in props]) + xDoc.storeToURL(exportFileName, tuple(storeProps)) + +def handleCrash(file, disposed): +# print("File: " + file + " crashed") + with open("crashlog.txt", "a") as crashLog: + crashLog.write('Crash:' + file + ' ') + if disposed == 1: + crashLog.write('through disposed\n') +# crashed_files.append(file) +# add here the remaining handling code for crashed files + +def alarm_handler(args): + args.kill() + +class HandleFileTest: + def __init__(self, file, state, component): + self.file = file + self.state = state + self.component = component + def run(self, xContext, connection): +# print("Loading document: " + self.file) + t = None + args = None + try: + url = "file://" + quote(self.file) + with open("file.log", "a") as fh: + fh.write(url + "\n") + xDoc = None + args = [connection] + t = threading.Timer(60, alarm_handler, args) + t.start() + xDoc = loadFromURL(xContext, url, t, self.component) + self.state.goodFiles.append(self.file) + exportToODF(xContext, xDoc, os.path.basename(urllib.parse.urlparse(url).path), t, self.component) + except pyuno.getClass("com.sun.star.beans.UnknownPropertyException"): +# print("caught UnknownPropertyException " + self.file) + if not t.is_alive(): +# print("TIMEOUT!") + self.state.timeoutFiles.append(self.file) + else: + t.cancel() + handleCrash(self.file, 0) + self.state.badPropertyFiles.append(self.file) + connection.tearDown() + connection.setUp() + except pyuno.getClass("com.sun.star.lang.DisposedException"): +# print("caught DisposedException " + self.file) + if not t.is_alive(): +# print("TIMEOUT!") + self.state.timeoutFiles.append(self.file) + else: + t.cancel() + handleCrash(self.file, 1) + self.state.badDisposedFiles.append(self.file) + connection.tearDown() + connection.setUp() + finally: + if t.is_alive(): + t.cancel() + try: + if xDoc: + t = threading.Timer(10, alarm_handler, args) + t.start() + xDoc.close(True) + t.cancel() + except pyuno.getClass("com.sun.star.beans.UnknownPropertyException"): + print("caught UnknownPropertyException while closing") + self.state.badPropertyFiles.append(self.file) + connection.tearDown() + connection.setUp() + except pyuno.getClass("com.sun.star.lang.DisposedException"): + print("caught DisposedException while closing") + if t.is_alive(): + t.cancel() + else: + self.state.badDisposedFiles.append(self.file) + connection.tearDown() + connection.setUp() +# print("...done with: " + self.file) + +class State: + def __init__(self): + self.goodFiles = [] + self.badDisposedFiles = [] + self.badPropertyFiles = [] + self.timeoutFiles = [] + + +def write_state_report(files_list, start_time, report_filename, description): + with open(report_filename, "w") as fh: + fh.write("%s:\n" % description) + fh.write("Starttime: %s\n" % start_time.isoformat()) + for f in files_list: + fh.write("%s\n" % f) + + +def writeReport(state, startTime): + write_state_report(state.goodFiles, startTime, "goodFiles.log", + "Files which loaded perfectly") + write_state_report(state.badDisposedFiles, startTime, "badDisposedFiles.log", + "Files which crashed with DisposedException") + write_state_report(state.badPropertyFiles, startTime, "badPropertyFiles.log", + "Files which crashed with UnknownPropertyException") + write_state_report(state.timeoutFiles, startTime, "timeoutFiles.log", + "Files which timed out") + +def runHandleFileTests(opts): + startTime = datetime.datetime.now() + connection = PersistentConnection(opts) + global outdir + outdir = os.path.join(opts.outdir, startTime.strftime('%Y%m%d.%H%M%S')) + try: + tests = [] + state = State() +# print("before map") + for component, validExtension in validFileExtensions.items(): + files = [] + for suffix in validExtension: + files.extend(getFiles(opts.dirs, suffix)) + files.sort() + tests.extend( (HandleFileTest(file, state, component) for file in files) ) + runConnectionTests(connection, simpleInvoke, tests) + finally: + connection.kill() + writeReport(state, startTime) + +def parseArgs(argv): + epilog = "'location' is a pathname, not a URL. 'outdir' and 'userdir' are URLs.\n" \ + "The 'directory' parameters should be full absolute pathnames, not URLs." + + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, + epilog=epilog) + parser.add_argument('--soffice', metavar='method:location', required=True, + help="specify soffice instance to connect to\n" + "supported methods: 'path', 'connect'") + parser.add_argument('--outdir', metavar='URL', required=True, + help="specify the output directory for flat ODF exports") + parser.add_argument('--userdir', metavar='URL', + help="specify user installation directory for 'path' method") + parser.add_argument('--valgrind', action='store_true', + help="pass --valgrind to soffice for 'path' method") + parser.add_argument('dirs', metavar='directory', nargs='+') + + args = parser.parse_args(argv[1:]) + + return args + + +if __name__ == "__main__": + opts = parseArgs(sys.argv) + runHandleFileTests(opts) + +# vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/bffvalidator.sh.in b/bin/bffvalidator.sh.in new file mode 100644 index 000000000..e43522764 --- /dev/null +++ b/bin/bffvalidator.sh.in @@ -0,0 +1,3 @@ +#!/bin/sh + +wine @BFFVALIDATOR_EXE@ `winepath -w $1` diff --git a/bin/check-elf-dynamic-objects b/bin/check-elf-dynamic-objects new file mode 100755 index 000000000..11f587363 --- /dev/null +++ b/bin/check-elf-dynamic-objects @@ -0,0 +1,262 @@ +#!/bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# verify that ELF NEEDED entries are known-good so hopefully builds run on +# lots of different GNU/Linux distributions + +set -euo pipefail + +PARA=1 +check_path="${INSTDIR:-.}" + +help() +{ + cat << "EOF" + -d <dir> directory to check + -p run unbound parallel checks + -h help +EOF + [ -z "${1:-}" ] && exit 0 +} + +die() +{ + echo "$1" + echo + help 1 + exit 1 +} + +while [ "${1:-}" != "" ]; do + parm=${1%%=*} + arg=${1#*=} + has_arg= + if [ "${1}" != "${parm?}" ] ; then + has_arg=1 + else + arg="" + fi + + case "${parm}" in + --dir|-d) + if [ "$has_arg" ] ; then + check_path="$arg" + else + shift + check_path="$1" + fi + if [ ! -d "$check_path" ]; then + die "Invalid directory '$check_path'" + fi + ;; + -h) + help + ;; + -p) + # this sounds counter intuitive but the idea + # is to possibly support -p <n> + # in the meantime: 0 = nolimit and -p 1 would mean + # the current default: serialize + PARA=0 + ;; + -*) + die "Invalid option $1" + ;; + *) + if [ "$DO_NEW" = 1 ] ; then + REPO="$1" + else + die "Invalid argument $1" + fi + ;; + esac + shift +done + + +files=$(find "${check_path}/program" "${check_path}/sdk/bin" -type f) +# all RPATHs should point to ${INSTDIR}/program so that's the files they find +programfiles=$(echo ${files} | grep -o '/program/[^/]* ' | xargs -n 1 basename) + +# whitelists should contain only system libraries that have a good reputation +# of maintaining ABI stability +# allow extending the whitelist using the environment variable to be able to work +# on the installer stuff without the need for a baseline setup +globalwhitelist="ld-linux-x86-64.so.2 ld-linux.so.2 libc.so.6 libm.so.6 libdl.so.2 libpthread.so.0 librt.so.1 libutil.so.1 libnsl.so.1 libcrypt.so.1 libgcc_s.so.1 libstdc++.so.6 libz.so.1 libfontconfig.so.1 libfreetype.so.6 libxml2.so.2 libxslt.so.1 libexslt.so.0 ${LO_ELFCHECK_WHITELIST-}" +x11whitelist="libX11.so.6 libX11-xcb.so.1 libXext.so.6 libSM.so.6 libICE.so.6 libXinerama.so.1 libXrender.so.1 libXrandr.so.2 libcairo.so.2" +openglwhitelist="libGL.so.1" +giowhitelist="libgio-2.0.so.0 libgobject-2.0.so.0 libgmodule-2.0.so.0 libgthread-2.0.so.0 libglib-2.0.so.0 libdbus-glib-1.so.2 libdbus-1.so.3" +gstreamerwhitelist="libgstaudio-1.0.so.0 libgstpbutils-1.0.so.0 libgstvideo-1.0.so.0 libgstbase-1.0.so.0 libgstreamer-1.0.so.0" +gtk3whitelist="libgtk-3.so.0 libgdk-3.so.0 libcairo-gobject.so.2 libpangocairo-1.0.so.0 libfribidi.so.0 libatk-1.0.so.0 libcairo.so.2 libgio-2.0.so.0 libpangoft2-1.0.so.0 libpango-1.0.so.0 libfontconfig.so.1 libfreetype.so.6 libgdk_pixbuf-2.0.so.0 libgobject-2.0.so.0 libglib-2.0.so.0 libgmodule-2.0.so.0 libgthread-2.0.so.0 libdbus-glib-1.so.2 libdbus-1.so.3 libharfbuzz.so.0" +qt5whitelist="libQt5Core.so.5 libQt5Gui.so.5 libQt5Network.so.5 libQt5Widgets.so.5 libQt5X11Extras.so.5 libcairo.so.2 libglib-2.0.so.0 libgobject-2.0.so.0 libxcb.so.1 libxcb-icccm.so.4" +kf5whitelist="libKF5ConfigCore.so.5 libKF5CoreAddons.so.5 libKF5I18n.so.5 libKF5KIOCore.so.5 libKF5KIOFileWidgets.so.5 libKF5KIOWidgets.so.5 libKF5WindowSystem.so.5" +avahiwhitelist="libdbus-glib-1.so.2 libdbus-1.so.3 libgobject-2.0.so.0 libgmodule-2.0.so.0 libgthread-2.0.so.0 libglib-2.0.so.0 libavahi-common.so.3 libavahi-client.so.3" +kerberoswhitelist="libgssapi_krb5.so.2 libcom_err.so.2 libkrb5.so.3" +dconfwhitelist="libdconf.so.1 libgio-2.0.so.0 libglib-2.0.so.0 libgobject-2.0.so.0" + +check_one_file() +{ +local file="$1" + + skip=0 + whitelist="${globalwhitelist}" + case "${file}" in + */sdk/docs/*) + # skip the majority of files, no ELF binaries here + skip=1 + ;; + */_uuid.cpython-*.so) + whitelist="${whitelist} libuuid.so.1" + ;; + */libcairo.so.2) + whitelist="${whitelist} ${x11whitelist} libxcb-shm.so.0 libxcb.so.1 libxcb-render.so.0" + ;; + */libcairocanvaslo.so) + whitelist="${whitelist} libcairo.so.2" + ;; + */libucpgio1lo.so|*/liblosessioninstalllo.so|*/libevoablo.so) + whitelist="${whitelist} ${giowhitelist}" + ;; + */libavmediagst.so) + whitelist="${whitelist} ${gtk3whitelist} ${gstreamerwhitelist}" + ;; + */libvclplug_kf5lo.so|*/libkf5be1lo.so) + if [ "$ENABLE_KF5" = TRUE ]; then + whitelist="${whitelist} ${qt5whitelist} ${kf5whitelist}" + fi + ;; + */libvclplug_gtk3lo.so|*/updater) + whitelist="${whitelist} ${x11whitelist} ${gtk3whitelist}" + ;; + */libvclplug_qt5lo.so) + if [ "$ENABLE_QT5" = TRUE ]; then + whitelist="${whitelist} ${qt5whitelist}" + fi + ;; + */libvclplug_gtk3_kde5lo.so) + if [ "$ENABLE_GTK3_KDE5" = TRUE ]; then + whitelist="${whitelist} ${x11whitelist} ${gtk3whitelist} ${qt5whitelist} ${kf5whitelist}" + fi + ;; + */lo_kde5filepicker) + if [ "$ENABLE_GTK3_KDE5" = TRUE ]; then + whitelist="${whitelist} ${x11whitelist} ${gtk3whitelist} ${qt5whitelist} \ + ${kf5whitelist}" + fi + ;; + */libdesktop_detectorlo.so|*/ui-previewer|*/oosplash|*/gengal.bin) + whitelist="${whitelist} ${x11whitelist}" + ;; + */libvclplug_genlo.so|*/libchartcorelo.so|*/libavmediaogl.so|*/libOGLTranslo.so|*/liboglcanvaslo.so) + whitelist="${whitelist} ${x11whitelist} ${openglwhitelist}" + ;; + */libvcllo.so) + whitelist="${whitelist} ${x11whitelist} ${openglwhitelist} ${giowhitelist} libcups.so.2" + ;; + */libsofficeapp.so) + whitelist="${whitelist} ${x11whitelist} ${openglwhitelist} ${giowhitelist} libcups.so.2" + ;; + */liblibreofficekitgtk.so) + whitelist="${whitelist} ${gtk3whitelist}" + ;; + */libsdlo.so) + whitelist="${whitelist} ${avahiwhitelist}" + ;; + */libskialo.so) + whitelist="${whitelist} ${openglwhitelist} ${x11whitelist}" + ;; + */libofficebean.so) + whitelist="${whitelist} libjawt.so" + ;; + */libpostgresql-sdbc-impllo.so) + whitelist="${whitelist} ${kerberoswhitelist}" + ;; + */libconfigmgrlo.so) + if [ "$ENABLE_DCONF" = TRUE ]; then + whitelist="${whitelist} ${dconfwhitelist}" + fi + ;; + */libmergedlo.so) + whitelist="${whitelist} ${x11whitelist} ${openglwhitelist} ${giowhitelist} libcups.so.2 libcairo.so.2" + ;; + esac + if test "${skip}" = 0 && readelf -d "${file}" &> /dev/null ; then + rpath=$(readelf -d "${file}" | grep '(\(RPATH\|RUNPATH\))' || true) + neededs=$(readelf -d "${file}" | grep '(NEEDED)' | sed -e 's/.*\[\(.*\)\]$/\1/') + neededsinternal= + for needed in ${neededs} + do + if ! echo ${whitelist} | grep -q -w "${needed}" ; then + neededsinternal="${neededsinternal} ${needed}" + if ! echo ${programfiles} | grep -q -w "${needed}" ; then + echo "${file}" has suspicious NEEDED: "${needed}" + status=1 + fi + fi + done + if test -z "${rpath}" ; then + case "${file}" in + */python-core-*/lib/lib-dynload/*) + # python modules don't have RPATH + ;; + */share/extensions/*) + # extension libraries don't have RPATH + ;; + *) + # no NEEDED from instdir, no RPATH needed + if test -n "${neededsinternal}" ; then + echo "${file}" has no RPATH + status=1 + fi + ;; + esac + else + case "$file" in + */sdk/bin/*) + if echo "${rpath}" | grep -q -v '\[\$ORIGIN/../../program\]$' ; then + echo "${file}" has unexpected RPATH "${rpath}" + status=1 + fi + ;; + *) + if echo "${rpath}" | grep -q -v '\[\$ORIGIN\]$' ; then + echo "${file}" has unexpected RPATH "${rpath}" + status=1 + fi + ;; + esac + fi + fi +} +status=0 + +if [ "$PARA" = "1" ] ; then + for file in ${files} + do + check_one_file $file + done +else + rm -f check_elf.out + for file in ${files} + do + ( + check_one_file $file + )>> check_elf.out & + done + + wait + + if [ -s check_elf.out ] ; then + cat check_elf.out + status=1 + fi + rm check_elf.out +fi +exit ${status} + diff --git a/bin/check-icon-sizes.py b/bin/check-icon-sizes.py new file mode 100755 index 000000000..535caa3ef --- /dev/null +++ b/bin/check-icon-sizes.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import os + +from PIL import Image + +""" +This script walks through all icon files and checks whether the sc_* and lc_* files have the correct size. +""" + +icons_folder = os.path.abspath(os.path.join(__file__, '..', '..', 'icon-themes')) + +def check_size(filename, size): + image = Image.open(filename) + width, height = image.size + if width != size or height != size: + print("%s has size %dx%d but should have %dx%d" % (filename, width, height, size, size)) + +for root, dirs, files in os.walk(icons_folder): + for filename in files: + if not filename.endswith('png'): + continue + if filename.startswith('lc_'): + check_size(os.path.join(root, filename), 24) + elif filename.startswith('sc_'): + check_size(os.path.join(root, filename), 16) + diff --git a/bin/check-implementer-notes.py b/bin/check-implementer-notes.py new file mode 100755 index 000000000..10b7c168b --- /dev/null +++ b/bin/check-implementer-notes.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +import json, re, subprocess, sys, urllib3 + +http = urllib3.PoolManager() + +# TDF implementer notes pages for LibreOffice +wiki_pages = [ + 'https://wiki.documentfoundation.org/api.php?action=parse&format=json&page=Development/ODF_Implementer_Notes/List_of_LibreOffice_ODF_Extensions&prop=wikitext', + 'https://wiki.documentfoundation.org/api.php?action=parse&format=json&page=Development/ODF_Implementer_Notes/List_of_LibreOffice_OpenFormula_Extensions&prop=wikitext'] + +# get all commit hashes mentioned in implementer notes +wiki_commit_hashes = {} +query = re.compile('\{\{commit\|(\\w+)\|\\w*\|\\w*\}\}', re.IGNORECASE) +for page in wiki_pages: + r = http.request('GET', page) + data = json.loads(r.data.decode('utf-8')) + for line in data['parse']['wikitext']['*'].split('\n'): + for res in query.finditer(line): + wiki_commit_hashes[res.group(1)] = '' + +# get all commits that change core/schema/* - and are _not_ mentioned +# in the wiki page +# Cut-off is May 18th 2020, when Michael Stahl had finished cleaning this up +for commit in subprocess.check_output( + ['git', '--no-pager', '-C', sys.path[0]+'/..', 'log', + '--since=2020-05-18', '--format=%H', '--', 'schema/'], + stderr=subprocess.STDOUT).decode("utf-8").split("\n"): + if commit != '' and commit not in wiki_commit_hashes: + print('missing commit: %s' % commit) + diff --git a/bin/check-missing-unittests.py b/bin/check-missing-unittests.py new file mode 100755 index 000000000..9a81b3ab7 --- /dev/null +++ b/bin/check-missing-unittests.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import datetime +import subprocess +import sys + +def main(ignoredBugs): + results = { + 'export': { + 'docx': {}, + 'doc': {}, + 'pptx': {}, + 'xlsx': {}, + 'xhtml': {}, + 'html': {}, + } + } + hasTestSet = set() + + repoPath = os.path.dirname(os.path.abspath(__file__)) + '/..' + branch = subprocess.check_output( + ['git', '-C', repoPath, 'rev-parse', '--abbrev-ref', 'HEAD'], + stderr=subprocess.DEVNULL) + last_hash = subprocess.check_output( + ['git', '-C', repoPath, 'rev-parse', 'HEAD'], + stderr=subprocess.DEVNULL) + output = subprocess.check_output( + ['git', '-C', repoPath, 'log', '--since="2012-01-01', '--name-only' ,'--pretty=format:"%s"'], + stderr=subprocess.DEVNULL) + commits = output.decode('utf-8', 'ignore').split('\n\n') + + for commit in reversed(commits): + + summary = commit.split('\n', 1)[0].lower() + + #Check summary has a bug id + if 'tdf#' in summary or 'fdo#' in summary: + + isIgnored = False + for i in ignoredBugs: + if i in summary: + isIgnored = True + if isIgnored: + continue + + if 'tdf#' in summary: + if not summary.split('tdf#')[1][0].isdigit(): + continue + bugId = ''.join(filter(str.isdigit, summary.split('tdf#')[1].split(' ')[0])) + elif 'fdo#' in summary: + if not summary.split('fdo#')[1][0].isdigit(): + continue + bugId = ''.join(filter(str.isdigit, summary.split('fdo#')[1].split(' ')[0])) + + + if bugId in hasTestSet: + continue + + changedFiles = commit.split('\n', 1)[1] + if 'qa' in changedFiles: + hasTestSet.add(bugId) + continue + + elif 'sw/source/filter/ww8/docx' in changedFiles or \ + 'writerfilter/source/dmapper' in changedFiles or \ + 'starmath/source/ooxmlimport' in changedFiles: + results['export']['docx'][bugId] = summary + + elif 'sw/source/filter/ww8/ww8' in changedFiles: + results['export']['doc'][bugId] = summary + + elif 'sc/source/filter/excel/xe' in changedFiles: + results['export']['xlsx'][bugId] = summary + + elif 'oox/source/export/' in changedFiles: + results['export']['pptx'][bugId] = summary + + elif 'filter/source/xslt/odf2xhtml/export' in changedFiles: + results['export']['xhtml'][bugId] = summary + + elif 'sw/source/filter/html/' in changedFiles: + results['export']['html'][bugId] = summary + + # Add others here + + print() + print('{{TopMenu}}') + print('{{Menu}}') + print('{{Menu.Development}}') + print() + print('Date: ' + str(datetime.datetime.now())) + print() + print('Commits: ' + str(len(commits))) + print() + print('Branch: ' + branch.decode().strip()) + print() + print('Hash: ' + str(last_hash.decode().strip())) + for k,v in results.items(): + print('\n== ' + k + ' ==') + for k1, v1 in v.items(): + print('\n=== ' + k1 + ' ===') + for bugId, summary in v1.items(): + if bugId not in hasTestSet: + print( + "* {} - [https://bugs.documentfoundation.org/show_bug.cgi?id={} tdf#{}]".format( + summary, bugId, bugId)) + print('\n== ignored bugs ==') + print(' '.join(ignoredBugs)) + print() + print('[[Category:QA]][[Category:Development]]') + +def usage(): + message = """usage: {program} [bugs to ignore (each one is one argument)] + +Sample: {program} 10000 10001 10002""" + print(message.format(program = os.path.basename(sys.argv[0]))) + +if __name__ == '__main__': + + args = set() + if len(sys.argv) > 1: + arg1 = sys.argv[1] + if arg1 == '-h' or arg1 == "--help": + usage() + sys.exit(1) + for i in sys.argv: + if i.isdigit(): + args.add(i) + + main(sorted(args)) diff --git a/bin/convwatch.py b/bin/convwatch.py new file mode 100644 index 000000000..d88d43d64 --- /dev/null +++ b/bin/convwatch.py @@ -0,0 +1,462 @@ +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Conversion watch, initially intended to detect if document layout changed since the last time it was run. +# +# Print a set of docs, compare the pdf against the old run and highlight the differences +# + +import getopt +import os +import subprocess +import sys +import time +import uuid +import datetime +import traceback +import threading +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + +try: + import pyuno + import uno + import unohelper +except ImportError: + print("pyuno not found: try to set PYTHONPATH and URE_BOOTSTRAP variables") + print("PYTHONPATH=/installation/opt/program") + print("URE_BOOTSTRAP=file:///installation/opt/program/fundamentalrc") + raise + +try: + from com.sun.star.document import XDocumentEventListener +except ImportError: + print("UNO API class not found: try to set URE_BOOTSTRAP variable") + print("URE_BOOTSTRAP=file:///installation/opt/program/fundamentalrc") + raise + +### utilities ### + +def log(*args): + print(*args, flush=True) + +def partition(list, pred): + left = [] + right = [] + for e in list: + if pred(e): + left.append(e) + else: + right.append(e) + return (left, right) + +def filelist(dir, suffix): + if len(dir) == 0: + raise Exception("filelist: empty directory") + if not(dir[-1] == "/"): + dir += "/" + files = [dir + f for f in os.listdir(dir)] +# log(files) + return [f for f in files + if os.path.isfile(f) and os.path.splitext(f)[1] == suffix] + +def getFiles(dirs, suffix): + files = [] + for dir in dirs: + files += filelist(dir, suffix) + return files + +### UNO utilities ### + +class OfficeConnection: + def __init__(self, args): + self.args = args + self.soffice = None + self.socket = None + self.xContext = None + def setUp(self): + (method, sep, rest) = self.args["--soffice"].partition(":") + if sep != ":": + raise Exception("soffice parameter does not specify method") + if method == "path": + self.socket = "pipe,name=pytest" + str(uuid.uuid1()) + try: + userdir = self.args["--userdir"] + except KeyError: + raise Exception("'path' method requires --userdir") + if not(userdir.startswith("file://")): + raise Exception("--userdir must be file URL") + self.soffice = self.bootstrap(rest, userdir, self.socket) + elif method == "connect": + self.socket = rest + else: + raise Exception("unsupported connection method: " + method) + self.xContext = self.connect(self.socket) + + def bootstrap(self, soffice, userdir, socket): + argv = [ soffice, "--accept=" + socket + ";urp", + "-env:UserInstallation=" + userdir, + "--quickstart=no", + "--norestore", "--nologo", "--headless" ] + if "--valgrind" in self.args: + argv.append("--valgrind") + return subprocess.Popen(argv) + + def connect(self, socket): + xLocalContext = uno.getComponentContext() + xUnoResolver = xLocalContext.ServiceManager.createInstanceWithContext( + "com.sun.star.bridge.UnoUrlResolver", xLocalContext) + url = "uno:" + socket + ";urp;StarOffice.ComponentContext" + log("OfficeConnection: connecting to: " + url) + while True: + try: + xContext = xUnoResolver.resolve(url) + return xContext +# except com.sun.star.connection.NoConnectException + except pyuno.getClass("com.sun.star.connection.NoConnectException"): + log("NoConnectException: sleeping...") + time.sleep(1) + + def tearDown(self): + if self.soffice: + if self.xContext: + try: + log("tearDown: calling terminate()...") + xMgr = self.xContext.ServiceManager + xDesktop = xMgr.createInstanceWithContext( + "com.sun.star.frame.Desktop", self.xContext) + xDesktop.terminate() + log("...done") +# except com.sun.star.lang.DisposedException: + except pyuno.getClass("com.sun.star.beans.UnknownPropertyException"): + log("caught UnknownPropertyException") + pass # ignore, also means disposed + except pyuno.getClass("com.sun.star.lang.DisposedException"): + log("caught DisposedException") + pass # ignore + else: + self.soffice.terminate() + ret = self.soffice.wait() + self.xContext = None + self.socket = None + self.soffice = None + if ret != 0: + raise Exception("Exit status indicates failure: " + str(ret)) +# return ret + +class WatchDog(threading.Thread): + def __init__(self, connection): + threading.Thread.__init__(self, name="WatchDog " + connection.socket) + self.connection = connection + def run(self): + try: + if self.connection.soffice: # not possible for "connect" + self.connection.soffice.wait(timeout=120) # 2 minutes? + except subprocess.TimeoutExpired: + log("WatchDog: TIMEOUT -> killing soffice") + self.connection.soffice.terminate() # actually killing oosplash... + self.connection.xContext = None + log("WatchDog: killed soffice") + +class PerTestConnection: + def __init__(self, args): + self.args = args + self.connection = None + self.watchdog = None + def getContext(self): + return self.connection.xContext + def setUp(self): + assert(not(self.connection)) + def preTest(self): + conn = OfficeConnection(self.args) + conn.setUp() + self.connection = conn + self.watchdog = WatchDog(self.connection) + self.watchdog.start() + def postTest(self): + if self.connection: + try: + self.connection.tearDown() + finally: + self.connection = None + self.watchdog.join() + def tearDown(self): + assert(not(self.connection)) + +class PersistentConnection: + def __init__(self, args): + self.args = args + self.connection = None + def getContext(self): + return self.connection.xContext + def setUp(self): + conn = OfficeConnection(self.args) + conn.setUp() + self.connection = conn + def preTest(self): + assert(self.connection) + def postTest(self): + assert(self.connection) + def tearDown(self): + if self.connection: + try: + self.connection.tearDown() + finally: + self.connection = None + +def simpleInvoke(connection, test): + try: + connection.preTest() + test.run(connection.getContext()) + finally: + connection.postTest() + +def retryInvoke(connection, test): + tries = 5 + while tries > 0: + try: + tries -= 1 + try: + connection.preTest() + test.run(connection.getContext()) + return + finally: + connection.postTest() + except KeyboardInterrupt: + raise # Ctrl+C should work + except: + log("retryInvoke: caught exception") + raise Exception("FAILED retryInvoke") + +def runConnectionTests(connection, invoker, tests): + try: + connection.setUp() + failed = [] + for test in tests: + try: + invoker(connection, test) + except KeyboardInterrupt: + raise # Ctrl+C should work + except: + failed.append(test.file) + estr = traceback.format_exc() + log("... FAILED with exception:\n" + estr) + return failed + finally: + connection.tearDown() + +class EventListener(XDocumentEventListener,unohelper.Base): + def __init__(self): + self.layoutFinished = False + def documentEventOccured(self, event): +# log(str(event.EventName)) + if event.EventName == "OnLayoutFinished": + self.layoutFinished = True + def disposing(event): + pass + +def mkPropertyValue(name, value): + return uno.createUnoStruct("com.sun.star.beans.PropertyValue", + name, 0, value, 0) + +### tests ### + +def loadFromURL(xContext, url): + xDesktop = xContext.ServiceManager.createInstanceWithContext( + "com.sun.star.frame.Desktop", xContext) + props = [("Hidden", True), ("ReadOnly", True)] # FilterName? + loadProps = tuple([mkPropertyValue(name, value) for (name, value) in props]) + xListener = EventListener() + xGEB = xContext.getValueByName( + "/singletons/com.sun.star.frame.theGlobalEventBroadcaster") + xGEB.addDocumentEventListener(xListener) + xDoc = None + try: + xDoc = xDesktop.loadComponentFromURL(url, "_blank", 0, loadProps) + if xDoc is None: + raise Exception("No document loaded?") + time_ = 0 + while time_ < 30: + if xListener.layoutFinished: + return xDoc + log("delaying...") + time_ += 1 + time.sleep(1) + log("timeout: no OnLayoutFinished received") + return xDoc + except: + if xDoc: + log("CLOSING") + xDoc.close(True) + raise + finally: + if xListener: + xGEB.removeDocumentEventListener(xListener) + +def printDoc(xContext, xDoc, url): + props = [ mkPropertyValue("FileName", url) ] +# xDoc.print(props) + uno.invoke(xDoc, "print", (tuple(props),)) # damn, that's a keyword! + busy = True + while busy: + log("printing...") + time.sleep(1) + prt = xDoc.getPrinter() + for value in prt: + if value.Name == "IsBusy": + busy = value.Value + log("...done printing") + +class LoadPrintFileTest: + def __init__(self, file, prtsuffix): + self.file = file + self.prtsuffix = prtsuffix + def run(self, xContext): + start = datetime.datetime.now() + log("Time: " + str(start) + " Loading document: " + self.file) + xDoc = None + try: + if os.name == 'nt' and self.file[1] == ':': + url = "file:///" + self.file[0:2] + quote(self.file[2:]) + else: + url = "file://" + quote(self.file) + xDoc = loadFromURL(xContext, url) + printDoc(xContext, xDoc, url + self.prtsuffix) + finally: + if xDoc: + xDoc.close(True) + end = datetime.datetime.now() + log("...done with: " + self.file + " in: " + str(end - start)) + +def runLoadPrintFileTests(opts, dirs, suffix, reference): + if reference: + prtsuffix = ".pdf.reference" + else: + prtsuffix = ".pdf" + files = getFiles(dirs, suffix) + tests = (LoadPrintFileTest(file, prtsuffix) for file in files) +# connection = PersistentConnection(opts) + connection = PerTestConnection(opts) + failed = runConnectionTests(connection, simpleInvoke, tests) + print("all printed: FAILURES: " + str(len(failed))) + for fail in failed: + print(fail) + return failed + +def mkImages(file, resolution): + argv = [ "gs", "-r" + resolution, "-sOutputFile=" + file + ".%04d.jpeg", + "-dNOPROMPT", "-dNOPAUSE", "-dBATCH", "-sDEVICE=jpeg", file ] + ret = subprocess.check_call(argv) + +def mkAllImages(dirs, suffix, resolution, reference, failed): + if reference: + prtsuffix = ".pdf.reference" + else: + prtsuffix = ".pdf" + for dir in dirs: + files = filelist(dir, suffix) + log(files) + for f in files: + if f in failed: + log("Skipping failed: " + f) + else: + mkImages(f + prtsuffix, resolution) + +def identify(imagefile): + argv = ["identify", "-format", "%k", imagefile] + process = subprocess.Popen(argv, stdout=subprocess.PIPE) + result, _ = process.communicate() + if process.wait() != 0: + raise Exception("identify failed") + if result.partition(b"\n")[0] != b"1": + log("identify result: " + result.decode('utf-8')) + log("DIFFERENCE in " + imagefile) + +def compose(refimagefile, imagefile, diffimagefile): + argv = [ "composite", "-compose", "difference", + refimagefile, imagefile, diffimagefile ] + subprocess.check_call(argv) + +def compareImages(file): + allimages = [f for f in filelist(os.path.dirname(file), ".jpeg") + if f.startswith(file)] +# refimages = [f for f in filelist(os.path.dirname(file), ".jpeg") +# if f.startswith(file + ".reference")] +# log("compareImages: allimages:" + str(allimages)) + (refimages, images) = partition(sorted(allimages), + lambda f: f.startswith(file + ".pdf.reference")) +# log("compareImages: images" + str(images)) + for (image, refimage) in zip(images, refimages): + compose(image, refimage, image + ".diff") + identify(image + ".diff") + if (len(images) != len(refimages)): + log("DIFFERENT NUMBER OF IMAGES FOR: " + file) + +def compareAllImages(dirs, suffix): + log("compareAllImages...") + for dir in dirs: + files = filelist(dir, suffix) +# log("compareAllImages:" + str(files)) + for f in files: + compareImages(f) + log("...compareAllImages done") + + +def parseArgs(argv): + (optlist,args) = getopt.getopt(argv[1:], "hr", + ["help", "soffice=", "userdir=", "reference", "valgrind"]) +# print optlist + return (dict(optlist), args) + +def usage(): + message = """usage: {program} [option]... [directory]..." + -h | --help: print usage information + -r | --reference: generate new reference files (otherwise: compare) + --soffice=method:location + specify soffice instance to connect to + supported methods: 'path', 'connect' + --userdir=URL specify user installation directory for 'path' method + --valgrind pass --valgrind to soffice for 'path' method""" + print(message.format(program = os.path.basename(sys.argv[0]))) + +def checkTools(): + try: + subprocess.check_output(["gs", "--version"]) + except: + print("Cannot execute 'gs'. Please install ghostscript.") + sys.exit(1) + try: + subprocess.check_output(["composite", "-version"]) + subprocess.check_output(["identify", "-version"]) + except: + print("Cannot execute 'composite' or 'identify'.") + print("Please install ImageMagick.") + sys.exit(1) + +if __name__ == "__main__": + checkTools() + (opts,args) = parseArgs(sys.argv) + if len(args) == 0: + usage() + sys.exit(1) + if "-h" in opts or "--help" in opts: + usage() + sys.exit() + elif "--soffice" in opts: + reference = "-r" in opts or "--reference" in opts + failed = runLoadPrintFileTests(opts, args, ".odt", reference) + mkAllImages(args, ".odt", "200", reference, failed) + if not(reference): + compareAllImages(args, ".odt") + else: + usage() + sys.exit(1) + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/distro-install-clean-up b/bin/distro-install-clean-up new file mode 100755 index 000000000..701c9ffa0 --- /dev/null +++ b/bin/distro-install-clean-up @@ -0,0 +1,92 @@ +#!/bin/sh + +if test -z "${SRC_ROOT}"; then + echo "distro-install-clean-up: No environment set!" + exit 1 +fi + +echo "Cleaning up ..."; + +remove_help_localization() +{ + lang=$1 + + # nothing to be done if the localization is en-US if it does not exist + # or if it is already removed + test "$lang" = "en-US" -o \ + ! -e $DESTDIR$INSTALLDIR/help/$lang -o \ + -L $DESTDIR$INSTALLDIR/help/$lang && return; + + echo "... remove \"$lang\"" + + rm -rf $DESTDIR$INSTALLDIR/help/$lang + grep -v "$INSTALLDIR/help/$lang" $DESTDIR/gid_Module_Root.$lang >$DESTDIR/gid_Module_Root.$lang.new + mv -f $DESTDIR/gid_Module_Root.$lang.new $DESTDIR/gid_Module_Root.$lang + # FIXME: the following code could be used without the condition + # and should replace the lines above after only the milestones + # providing gid_Module_Helppack_Help and fixed gid_Module_Root.$lang + # are supported + # Note: The problem with gid_Module_Root.$lang is that it still includes + # %dir */help/* entries. + # Note: It was still necessary on ppc with gcj (OOo-2.0.2). Strange. Have to + # investigate it later. + if test -f $DESTDIR/gid_Module_Helppack_Help.$lang ; then + grep -v "$INSTALLDIR/help/$lang" $DESTDIR/gid_Module_Helppack_Help.$lang >$DESTDIR/gid_Module_Helppack_Help.$lang.new + mv -f $DESTDIR/gid_Module_Helppack_Help.$lang.new $DESTDIR/gid_Module_Helppack_Help.$lang + fi + + # Note: We created a compat symlink in the past. It is no longer necessary. + # We do not want it because RPM has problems with update when we remove + # poor localizations in never packages +} + +# Check if the English help is installed and is in the main package (is first on the list) +# Note that Java-disabled builds do not create help at all. +if test -f $DESTDIR$INSTALLDIR/help/en/sbasic.cfg -a \ + "`for lang in $WITH_LANG_LIST ; do echo $lang ; break ; done`" = "en-US" ; then + + echo "Removing duplicated English help..." + + for lang in $WITH_LANG_LIST ; do + test ! -f $DESTDIR$INSTALLDIR/help/en/sbasic.cfg -o ! -f $DESTDIR$INSTALLDIR/help/$lang/sbasic.cfg && continue; + if diff $DESTDIR$INSTALLDIR/help/en/sbasic.cfg $DESTDIR$INSTALLDIR/help/$lang/sbasic.cfg >/dev/null 2>&1 ; then + remove_help_localization $lang + fi + done + + echo "Removing poor help localizations..." + + for lang in $WITH_POOR_HELP_LOCALIZATIONS ; do + remove_help_localization $lang + done +fi + +echo "Fixing permissions..." +for dir in $DESTDIR$DOCDIR $DESTDIR$INSTALLDIR/sdk/examples ; do + if test -d $dir -a -w $dir ; then + find "$dir" -type f \( -name "*.txt" -o -name "*.java" -o -name "*.xml" -o \ + -name "*.xcu" -o -name "*.xcs" -o -name "*.html" -o \ + -name "*.pdf" -o -name "*.ps" -o -name "*.gif" -o \ + -name "*.png" -o -name "*.jpg" -o -name "Makefile" -o \ + -name "manifest.mf" \) -exec chmod 644 {} \; + fi +done + +if test "z$DESTDIR" != "z" ; then + echo "Checking for DESTDIR inside installed files..." + found_destdir= + for file in `find $DESTDIR -type f` ; do + grep -q "$DESTDIR" $file && echo "$file: includes the string \"$DESTDIR\"" && found_destdir=1 + done + if test "z$found_destdir" != "z" ; then + echo "!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!" + echo "The path DESTDIR:$DESTDIR was found inside some" + echo "installed files. It is probably a bug." + echo + echo "Especially, if the DESTDIR is set to \$RPM_BUILD_ROOT" + echo "when creating RPM packages. Even it could be a security hole" + echo "if the application searches /var/tmp for binaries or" + echo "config files because the directory is world-writable." + echo "!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!" + fi +fi diff --git a/bin/distro-install-desktop-integration b/bin/distro-install-desktop-integration new file mode 100755 index 000000000..1da104e47 --- /dev/null +++ b/bin/distro-install-desktop-integration @@ -0,0 +1,192 @@ +#!/bin/sh + +if test -z "${SRC_ROOT}"; then + echo "distro-install-clean-up: No environment set!" + exit 1 +fi + +PRODUCTVERSION_NODOT=`echo $PRODUCTVERSION | sed -e "s/\.//"` + +mkdir -p "$DESTDIR$BINDIR" + +create_wrapper() +{ + echo "Install $BINDIR/$1" + + if test -L "$DESTDIR$BINDIR/$1" ; then + # do not overwrite $BINDIR/libreoffice symlink created by create_tree.sh + # the symlink is necessary by java UNO components to find + # the UNO installation using $PATH; this function used to be provided + # by $BINDIR/soffice symlink, see + # http://udk.openoffice.org/common/man/spec/transparentofficecomponents.html + # Note: if you want to support parallel installation of more OOo versions + # you cannot include this link directly into the package + # For example, the Novell package mark this symlink as %ghost + # and update it in %post and %postun + echo " skip already existing symlink $BINDIR/$1" + else + mkdir -p "$DESTDIR$BINDIR" + cat <<EOT >"$DESTDIR$BINDIR/$1" +#!/bin/sh +$INSTALLDIR/program/$2 $3 "\$@" +EOT + chmod 755 "$DESTDIR$BINDIR/$1" + fi + # put into file list + test -f "$DESTDIR/$4" && echo "$BINDIR/$1" >>$DESTDIR/$4 +} + +create_man_link() +{ + echo "Install $MANDIR/man1/$1.1.gz" + + mkdir -p $DESTDIR$MANDIR/man1 + echo ".so man1/$2.1" >| $DESTDIR$MANDIR/man1/$1.1 + gzip -f $DESTDIR$MANDIR/man1/$1.1 + test -f "$DESTDIR/$3" && echo "$MANDIR/man1/$1.1.gz" >>"$DESTDIR/$3" +} + +install_man() +{ + echo "Install $MANDIR/man1/$1.1.gz" + + mkdir -p $DESTDIR$MANDIR/man1 + cp "${SRCDIR?}"/sysui/desktop/man/$1.1 $DESTDIR$MANDIR/man1 || exit 1; + gzip -f $DESTDIR$MANDIR/man1/$1.1 + test -f "$DESTDIR/$2" && echo "$MANDIR/man1/$1.1.gz" >>"$DESTDIR/$2" +} + + +add_wrapper() +{ + lowrapper_name="$1" + target_binary="$2" + target_option_1="$3" + used_man_page="$4" + desktop_file="$5" + file_list="$6" + + # do we want compat oowrapper? + oowrapper_name="" + if test "$WITH_COMPAT_OOWRAPPERS" = 'TRUE' ; then + oowrapper_name=`echo "$lowrapper_name" | sed -e "s/^lo/oo/"` + # "oo" prefix only for wrappers stating with "lo" prefix + test "$oowrapper_name" = "$lowrapper_name" && oowrapper_name= + fi + + # wrappers + create_wrapper "$lowrapper_name" "$target_binary" "$target_option_1" "$file_list" + test -n "$oowrapper_name" && create_wrapper "$oowrapper_name" "$target_binary" "$target_option_1" "$file_list" + + # man pages + if test "$used_man_page" = "$lowrapper_name" ; then + # need to install the manual page + install_man "$lowrapper_name" "$file_list" + else + # just link the manual page + create_man_link "$lowrapper_name" "$used_man_page" "$file_list" + fi + test -n "$oowrapper_name" && create_man_link "$oowrapper_name" "$used_man_page" "$file_list" + + # add desktop file to the right file list + test -n "$desktop_file" -a -f "$DESTDIR/$file_list" && echo "$PREFIXDIR/share/applications/$desktop_file" >>"$DESTDIR/$file_list" +} + +# install desktop integration from plain packages +sysui_temp=`mktemp -d -t distro-pack-desktop-integration-XXXXXX` +cp -a workdir/CustomTarget/sysui/share/libreoffice/* "$sysui_temp" +cp -a "${SRCDIR?}"/sysui/desktop/share/create_tree.sh "$sysui_temp" +cd $sysui_temp +# we want non-versioned stuff in the distro packages +sed -i \ + -e "s/\($INSTALLDIRNAME\)$PRODUCTVERSION_NODOT/\1/" \ + -e "s/\($INSTALLDIRNAME\)$PRODUCTVERSION/\1/" \ + -e "s/\($PRODUCTNAME\) $PRODUCTVERSION/\1/" \ + * +# call in subshell to do not malform PRODUCTVERSION, ... +( + export OFFICE_PREFIX=$LIBDIR + export PREFIX=$INSTALLDIRNAME + export ICON_PREFIX=$INSTALLDIRNAME + export ICON_SOURCE_DIR="${SRCDIR?}"/sysui/desktop/icons + export PRODUCTVERSION= + export KDEMAINDIR=$PREFIXDIR + export PREFIXDIR=${PREFIXDIR} + export GNOMEDIR=$PREFIXDIR + export GNOME_MIME_THEME=hicolor + export APPDATA_SOURCE_DIR="${SRCDIR?}"/sysui/desktop/appstream-appdata + bash ./create_tree.sh +) +cd - +rm -rf $sysui_temp + +# we do not want some stuff from the plain packages +if test -d $DESTDIR/opt ; then + rm -f $DESTDIR/opt/$INSTALLDIRNAME + rmdir $DESTDIR/opt 2>/dev/null || true +fi + +# we want non-versioned desktop files +cd $DESTDIR/$INSTALLDIR/share/xdg +# we want non-versioned stuff in the distro packages +sed -i \ + -e "s/\($INSTALLDIRNAME\)$PRODUCTVERSION_NODOT/\1/" \ + -e "s/\($INSTALLDIRNAME\)$PRODUCTVERSION/\1/" \ + -e "s/\($PRODUCTNAME\) $PRODUCTVERSION/\1/" \ + *.desktop +cd - + +# put the stuff installed by create_tree.sh into the right file lists +# desktop files will be added by the corresponding add_wrapper command +if test -f $DESTDIR/gid_Module_Root_Brand ; then + for dir in $PREFIXDIR/share/application-registry \ + $PREFIXDIR/share/mime/packages \ + $PREFIXDIR/share/mime-info \ + $PREFIXDIR/share/icons ; do + find "$DESTDIR$dir" \( -type f -o -type l \) -printf "$dir/%P\n" >>$DESTDIR/gid_Module_Root_Brand + done +fi + +# wrappers and man pages +# FIXME: do not have desktop file and MIME icon for unopkg +add_wrapper lobase soffice "--base" "libreoffice" "libreoffice-base.desktop" "gid_Module_Brand_Prg_Base" +add_wrapper localc soffice "--calc" "libreoffice" "libreoffice-calc.desktop" "gid_Module_Brand_Prg_Calc" +add_wrapper lodraw soffice "--draw" "libreoffice" "libreoffice-draw.desktop" "gid_Module_Brand_Prg_Draw" +add_wrapper lomath soffice "--math" "libreoffice" "libreoffice-math.desktop" "gid_Module_Brand_Prg_Math" +add_wrapper loimpress soffice "--impress" "libreoffice" "libreoffice-impress.desktop" "gid_Module_Brand_Prg_Impress" +add_wrapper loweb soffice "--web" "libreoffice" "" "gid_Module_Brand_Prg_Wrt" +add_wrapper lowriter soffice "--writer" "libreoffice" "libreoffice-writer.desktop" "gid_Module_Brand_Prg_Wrt" +add_wrapper lofromtemplate soffice ".uno:NewDoc" "libreoffice" "" "gid_Module_Root_Brand" +add_wrapper libreoffice soffice "" "libreoffice" "libreoffice-startcenter.desktop" "gid_Module_Root_Brand" +add_wrapper loffice soffice "" "libreoffice" "" "gid_Module_Root_Brand" +add_wrapper unopkg unopkg "" "unopkg" "" "gid_Module_Root_Brand" + +# there are two more desktop files for optional filters +test -f $DESTDIR/gid_Module_Optional_Xsltfiltersamples && echo "$PREFIXDIR/share/applications/libreoffice-xsltfilter.desktop" >>"$DESTDIR/gid_Module_Optional_Xsltfiltersamples" + +# $BINDIR/ooffice symlink is necessary by java UNO components to find +# the UNO installation using $PATH, see +# http://udk.openoffice.org/common/man/spec/transparentofficecomponents.html +# Note: if you want to support parallel installation of more OOo versions +# you cannot include this link directly into the package +# For example, the Novell package mark this symlink as %ghost +# and update it in %post and %postun +ln -sf "$INSTALLDIR/program/soffice" "$DESTDIR$BINDIR/soffice" +test -f $DESTDIR/gid_Module_Root_Brand && echo "$BINDIR/soffice" >>$DESTDIR/gid_Module_Root_Brand + +# create bash completion +mkdir -p $DESTDIR/usr/share/bash-completion/completions +"${SRCDIR?}"/bin/generate-bash-completion.py bin/bash-completion.in $DESTDIR/usr/share/bash-completion/completions/$INSTALLDIRNAME.sh +test -f $DESTDIR/gid_Module_Root_Brand && echo "/usr/share/bash-completion/completions/$INSTALLDIRNAME.sh" >>$DESTDIR/gid_Module_Root_Brand +if test "$WITH_COMPAT_OOWRAPPERS" = "TRUE" ; then + "${SRCDIR?}"/bin/generate-bash-completion.py --compat-oowrappers bin/bash-completion.in $DESTDIR/usr/share/bash-completion/completions/ooffice.sh + test -f $DESTDIR/gid_Module_Root_Brand && echo "/usr/share/bash-completion/completions/ooffice.sh" >>$DESTDIR/gid_Module_Root_Brand +fi + +echo "Install $OOINSTDIR/basis$VERSION/program/java-set-classpath"; +mkdir -p $DESTDIR$INSTALLDIR/program +sed -e "s|@INSTALLDIR@|$INSTALLDIR|g" "${SRCDIR?}"/bin/java-set-classpath.in >| "$DESTDIR$INSTALLDIR/program/java-set-classpath" || exit 1; +chmod 755 "$DESTDIR$INSTALLDIR/program/java-set-classpath" +test -f $DESTDIR/gid_Module_Root_Brand && echo "$INSTALLDIR/program/java-set-classpath" >>$DESTDIR/gid_Module_Root_Brand + +exit 0 diff --git a/bin/distro-install-file-lists b/bin/distro-install-file-lists new file mode 100755 index 000000000..34db93e0a --- /dev/null +++ b/bin/distro-install-file-lists @@ -0,0 +1,517 @@ +#!/bin/sh + +if test -z "${SRC_ROOT}"; then + echo "distro-install-clean-up: No environment set!" + exit 1 +fi + +BUILDDIR=`pwd` +FILELISTSDIR="$BUILDDIR/file-lists" + +# remove installed file even from the file list +# Params: file_list file_to_remove +remove_file() +{ + rm -f "$DESTDIR/$2" + perl -pi -e "s|^$2$||" "$1" +} + +# move one file from one list of files to a second one +# Params: target_file_list source_file_list file_to_move +mv_file_between_flists() +{ + if grep "^$3\$" $2 >/dev/null 2>&1 ; then + # \$3 can be regular expression + grep "^$3\$" $2 >>$1 + perl -pi -e "s|^$3$||" $2 + fi +} +# add the directories from the source list of files to the target list of +# file which are used in the target list of files but are missing there +# Params: target_file_list source_file_list +add_used_directories() +{ + sort -u -r $2 | sed -n "s|^%dir \(.*\)\$|s%^\\\\(\1\\\\).*%\\\\1%p|p" >$2.pattern + sed -n -f $2.pattern $1 | sort -u | sed "s|^|%dir |" >>$1 + rm $2.pattern + sort -u $1 >$1.unique + mv $1.unique $1 +} + +# remove a duplicity between two filelist +# Params: filelist_with_original filelist_with_duplicity duplicit_path +remove_duplicity_from_flists() +{ + if grep "$3" "$1" >/dev/null 2>&1 && \ + grep "$3" "$2" >/dev/null 2>&1 ; then + perl -pi -e "s|^$3$||" $2 + fi +} + +# merges one file list into another one +# Params: source_filelist dest_filelist replace_dest +merge_flists() +{ + if test -f "$1" ; then + cat "$1" >>"$2" + sort -u "$2" >"$2".sorted + mv "$2".sorted "$2" + fi +} + +if ! test -f $DESTDIR/gid_Module_Root; then + echo "Error: Failed to generate package file lists"; + echo " Have you defined DESTDIR?" + exit +fi + + +rm -rf "$FILELISTSDIR" +mkdir -p "$FILELISTSDIR" + +cd $DESTDIR + +if test "z$OOO_VENDOR" != "zDebian" ; then + + echo "Generating package file lists for $OOO_VENDOR..." + + rm -f common_list.txt + for module in gid_Module_Root gid_Module_Root_Brand \ + gid_Module_Root_Files_[0-9] \ + gid_Module_Root_Hack \ + gid_Module_Oo_Linguistic \ + gid_Module_Root_Extension_Dictionary_* \ + gid_Module_Root_Ure_Hidden ; do + merge_flists $module $FILELISTSDIR/common_list.txt + done + + # it is not a real extension; it used to be in the main package... + merge_flists gid_Module_Optional_Extensions_Script_Provider_For_JS $FILELISTSDIR/common_list.txt + + if test "$SPLIT_APP_MODULES" = "TRUE" ; then + merge_flists gid_Module_Prg_Base_Bin $FILELISTSDIR/base_list.txt + merge_flists gid_Module_Prg_Calc_Bin $FILELISTSDIR/calc_list.txt + merge_flists gid_Module_Prg_Draw_Bin $FILELISTSDIR/draw_list.txt + merge_flists gid_Module_Prg_Math_Bin $FILELISTSDIR/math_list.txt + merge_flists gid_Module_Prg_Impress_Bin $FILELISTSDIR/impress_list.txt + merge_flists gid_Module_Prg_Wrt_Bin $FILELISTSDIR/writer_list.txt + merge_flists gid_Module_Brand_Prg_Base $FILELISTSDIR/base_list.txt + merge_flists gid_Module_Brand_Prg_Calc $FILELISTSDIR/calc_list.txt + merge_flists gid_Module_Brand_Prg_Draw $FILELISTSDIR/draw_list.txt + merge_flists gid_Module_Brand_Prg_Math $FILELISTSDIR/math_list.txt + merge_flists gid_Module_Brand_Prg_Impress $FILELISTSDIR/impress_list.txt + merge_flists gid_Module_Brand_Prg_Wrt $FILELISTSDIR/writer_list.txt + merge_flists gid_Module_Reportbuilder $FILELISTSDIR/base_list.txt + merge_flists gid_Module_Pdfimport $FILELISTSDIR/draw_list.txt + + # FIXME: small; low dependencies; why optional module? + merge_flists gid_Module_Optional_OGLTrans $FILELISTSDIR/impress_list.txt + else + merge_flists gid_Module_Prg_Base_Bin $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Prg_Calc_Bin $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Prg_Draw_Bin $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Prg_Math_Bin $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Prg_Impress_Bin $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Prg_Wrt_Bin $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Brand_Prg_Base $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Brand_Prg_Calc $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Brand_Prg_Draw $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Brand_Prg_Math $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Brand_Prg_Impress $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Brand_Prg_Wrt $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Reportbuilder $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Pdfimport $FILELISTSDIR/common_list.txt + # FIXME: small; low dependencies; why optional module? + merge_flists gid_Module_Optional_OGLTrans $FILELISTSDIR/common_list.txt + fi + + if test "$SPLIT_APP_MODULES" = "TRUE" -a "$OOO_VENDOR" = "SUSE" ; then + # move the prebuilt icons into a hacky temporary package + # we want to repack them into a noarch package as soon as possible + # without the build dependency on the huge devel package + merge_flists gid_Module_Root_Files_Images $FILELISTSDIR/icon_themes_prebuilt.txt + else + merge_flists gid_Module_Root_Files_Images $FILELISTSDIR/common_list.txt + fi + + if test "$SPLIT_OPT_FEATURES" = "TRUE" ; then + if test "z$OOO_VENDOR" = "zMandriva" ; then + merge_flists gid_Module_Optional_Grfflt $FILELISTSDIR/draw_list.txt + merge_flists gid_Module_Optional_Headless $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Optional_Pymailmerge $FILELISTSDIR/pyuno_list.txt + merge_flists gid_Module_Pyuno $FILELISTSDIR/pyuno_list.txt + merge_flists gid_Module_Script_Provider_For_Python $FILELISTSDIR/pyuno_list.txt + merge_flists gid_Module_Optional_Pyuno_LibreLogo $FILELISTSDIR/pyuno_list.txt + merge_flists gid_Module_Optional_Xsltfiltersamples $FILELISTSDIR/common_list.txt + else + merge_flists gid_Module_Optional_Grfflt $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Optional_Headless $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Optional_Pymailmerge $FILELISTSDIR/mailmerge_list.txt + merge_flists gid_Module_Pyuno $FILELISTSDIR/pyuno_list.txt + merge_flists gid_Module_Optional_Pyuno_LibreLogo $FILELISTSDIR/pyuno_list.txt + merge_flists gid_Module_Script_Provider_For_Python $FILELISTSDIR/pyuno_list.txt + merge_flists gid_Module_Optional_Xsltfiltersamples $FILELISTSDIR/filters_list.txt + fi + else + merge_flists gid_Module_Optional_Grfflt $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Optional_Headless $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Optional_Pymailmerge $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Pyuno $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Optional_Pyuno_LibreLogo $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Script_Provider_For_Python $FILELISTSDIR/common_list.txt + merge_flists gid_Module_Optional_Xsltfiltersamples $FILELISTSDIR/common_list.txt + fi + + # lang packs + for lang in `echo $WITH_LANG_LIST | sed -e s/-/_/g`; do + lang_lists= + if test "$OOO_VENDOR" = "Mandriva" -o \( "$OOO_VENDOR" = "SUSE" -a "$SPLIT_APP_MODULES" = "TRUE" \) ; then + test -f gid_Module_Langpack_Basis_$lang && lang_lists="$lang_lists gid_Module_Langpack_Basis_$lang" || : + test -f gid_Module_Langpack_Brand_$lang && lang_lists="$lang_lists gid_Module_Langpack_Brand_$lang" || : + test -f gid_Module_Langpack_Resource_$lang && lang_lists="$lang_lists gid_Module_Langpack_Resource_$lang" || : + test -f gid_Module_Langpack_Impress_$lang && lang_lists="$lang_lists gid_Module_Langpack_Impress_$lang" || : + test -f gid_Module_Langpack_Draw_$lang && lang_lists="$lang_lists gid_Module_Langpack_Draw_$lang" || : + test -f gid_Module_Langpack_Math_$lang && lang_lists="$lang_lists gid_Module_Langpack_Math_$lang" || : + test -f gid_Module_Langpack_Calc_$lang && lang_lists="$lang_lists gid_Module_Langpack_Calc_$lang" || : + test -f gid_Module_Langpack_Base_$lang && lang_lists="$lang_lists gid_Module_Langpack_Base_$lang" || : + test -f gid_Module_Langpack_Writer_$lang && lang_lists="$lang_lists gid_Module_Langpack_Writer_$lang" || : + # Place helps on dedicated packages. + test -f gid_Module_Helppack_Help_$lang && sort -u gid_Module_Helppack_Help_$lang > $FILELISTSDIR/help_${lang}_list.txt || : + else + test -f gid_Module_Langpack_Basis_$lang && lang_lists="$lang_lists gid_Module_Langpack_Basis_$lang" || : + test -f gid_Module_Langpack_Brand_$lang && lang_lists="$lang_lists gid_Module_Langpack_Brand_$lang" || : + test -f gid_Module_Langpack_Resource_$lang && lang_lists="$lang_lists gid_Module_Langpack_Resource_$lang" || : + test -f gid_Module_Langpack_Impress_$lang && lang_lists="$lang_lists gid_Module_Langpack_Impress_$lang" || : + test -f gid_Module_Langpack_Draw_$lang && lang_lists="$lang_lists gid_Module_Langpack_Draw_$lang" || : + test -f gid_Module_Langpack_Math_$lang && lang_lists="$lang_lists gid_Module_Langpack_Math_$lang" || : + test -f gid_Module_Langpack_Calc_$lang && lang_lists="$lang_lists gid_Module_Langpack_Calc_$lang" || : + test -f gid_Module_Langpack_Base_$lang && lang_lists="$lang_lists gid_Module_Langpack_Base_$lang" || : + test -f gid_Module_Langpack_Writer_$lang && lang_lists="$lang_lists gid_Module_Langpack_Writer_$lang" || : + test -f gid_Module_Helppack_Help_$lang && lang_lists="$lang_lists gid_Module_Helppack_Help_$lang" || : + fi + if test -n "$lang_lists" ; then + # all files are installed below $INSTALLDIR/basis; we want to own also $INSTALLDIR + echo "%dir $INSTALLDIR" >$FILELISTSDIR/lang_${lang}_list.txt + cat $lang_lists | sort -u >>$FILELISTSDIR/lang_${lang}_list.txt + fi + # some help files are in _Langpack_{Writer,Impress,...}_<lang> + # move them from -l10n to -help + if test "$OOO_VENDOR" = "Mandriva" -o \( "$OOO_VENDOR" = "SUSE" -a "$SPLIT_APP_MODULES" = "TRUE" \) ; then + for lang in `echo $WITH_LANG_LIST | sed -e s/-/_/g`; do + test -f $FILELISTSDIR/help_${lang}_list.txt || continue; + mv_file_between_flists $FILELISTSDIR/help_${lang}_list.txt $FILELISTSDIR/lang_${lang}_list.txt $INSTALLDIR/help/.* + add_used_directories $FILELISTSDIR/help_${lang}_list.txt $FILELISTSDIR/lang_${lang}_list.txt + done + fi + done + + if test -f $FILELISTSDIR/lang_en_US_list.txt -a "$OOO_VENDOR" = "SUSE" -a "$SPLIT_APP_MODULES" != "TRUE" ; then + cat $FILELISTSDIR/lang_en_US_list.txt >>$FILELISTSDIR/common_list.txt + rm $FILELISTSDIR/lang_en_US_list.txt + fi + + if test -f gid_Module_Root_SDK ; then + cp gid_Module_Root_SDK $FILELISTSDIR/sdk_list.txt + fi + + cd $FILELISTSDIR + + # gnome subpackage + test -f $DESTDIR/gid_Module_Optional_Gnome && cp $DESTDIR/gid_Module_Optional_Gnome gnome_list.txt || : + mv_file_between_flists gnome_list.txt common_list.txt $INSTALLDIR/program/libevoab2.so + mv_file_between_flists gnome_list.txt common_list.txt $INSTALLDIR/program/libvclplug_gtk[0-9]*l..so + add_used_directories gnome_list.txt common_list.txt + + # mono subpackage + mv_file_between_flists mono_list.txt common_list.txt $INSTALLDIR/program/cli_.*.dll + mv_file_between_flists mono_list.txt common_list.txt $INSTALLDIR/program/cli_.*.dll.config + mv_file_between_flists mono_list.txt common_list.txt $INSTALLDIR/program/policy.*.cli_.*.dll + mv_file_between_flists mono_list.txt common_list.txt $INSTALLDIR/program/cli_.*.dll + mv_file_between_flists mono_list.txt common_list.txt $INSTALLDIR/program/cli_.*.dll.config + mv_file_between_flists mono_list.txt common_list.txt $INSTALLDIR/program/policy.*.cli_.*.dll + mv_file_between_flists mono_list.txt common_list.txt $INSTALLDIR/program/libcli_.*.so + add_used_directories mono_list.txt common_list.txt + # add the files from GAC if it was installed + test -f mono_gac && cat mono_gac >>mono_list.txt + + # postgresql subpackage + test -f $DESTDIR/gid_Module_Optional_PostgresqlSdbc && cp $DESTDIR/gid_Module_Optional_PostgresqlSdbc postgresql_list.txt || : + + # mailmerge + if test "$SPLIT_OPT_FEATURES" = "TRUE" ; then + if test "z$OOO_VENDOR" = "zMandriva" ; then + flist=pyuno_list.txt + else + flist=mailmerge_list.txt + fi + mv_file_between_flists $flist common_list.txt $INSTALLDIR/program/mailmerge.py + add_used_directories $flist common_list.txt + fi + + if test "z$OOO_VENDOR" = "zSUSE" ; then + # officebean subpackage + test -f $DESTDIR/gid_Module_Optional_Extensions_Script_Provider_For_BS && cp $DESTDIR/gid_Module_Optional_Extensions_Script_Provider_For_BS officebean_list.txt || : + mv_file_between_flists officebean_list.txt common_list.txt $INSTALLDIR/program/classes/officebean.jar + mv_file_between_flists officebean_list.txt common_list.txt $INSTALLDIR/program/libofficebean.so + add_used_directories officebean_list.txt common_list.txt + fi + + if test -f sdk_list.txt ; then + # in this case we move all entries including directories + mv_file_between_flists sdk_doc_list.txt sdk_list.txt "%dir $DOCDIR/sdk/docs.*" + mv_file_between_flists sdk_doc_list.txt sdk_list.txt "$DOCDIR/sdk/docs.*" + mv_file_between_flists sdk_doc_list.txt sdk_list.txt "$DOCDIR/sdk/examples" + mv_file_between_flists sdk_doc_list.txt sdk_list.txt "$DOCDIR/sdk/index.html" + mv_file_between_flists sdk_doc_list.txt sdk_list.txt "%dir $INSTALLDIR/sdk/examples.*" + mv_file_between_flists sdk_doc_list.txt sdk_list.txt "$INSTALLDIR/sdk/docs" + mv_file_between_flists sdk_doc_list.txt sdk_list.txt "$INSTALLDIR/sdk/examples.*" + mv_file_between_flists sdk_doc_list.txt sdk_list.txt "$INSTALLDIR/sdk/index.html" + add_used_directories sdk_doc_list.txt sdk_list.txt + fi + + + # Mandriva packaging + if test "$OOO_VENDOR" = "Mandriva"; then + # Not used + remove_file common_list.txt $INSTALLDIR/share/gallery/htmltheme.orig + remove_file common_list.txt $INSTALLDIR/share/dict/ooo/dictionary.lst + + # And these are in -draw package + mv_file_between_flists draw_list.txt common_list.txt $INSTALLDIR/share/registry/modules/org/openoffice/TypeDetection/Filter/fcfg_drawgraphics_filters.xcu + mv_file_between_flists draw_list.txt common_list.txt $INSTALLDIR/share/registry/modules/org/openoffice/TypeDetection/Filter/fcfg_drawgraphics_types.xcu + + # And these are in -impress package + mv_file_between_flists impress_list.txt common_list.txt $INSTALLDIR/share/registry/modules/org/openoffice/TypeDetection/Filter/fcfg_impressgraphics_filters.xcu + mv_file_between_flists impress_list.txt common_list.txt $INSTALLDIR/share/registry/modules/org/openoffice/TypeDetection/Types/fcfg_impressgraphics_types.xcu + + # Split out the gallery + mv_file_between_flists gallery_list.txt common_list.txt "$INSTALLDIR/share/gallery.*" + test -r galleries.txt && cat galleries.txt >> gallery_list.txt + + # Split out dtd-officedocument1.0 + mv_file_between_flists dtd_list.txt common_list.txt "$INSTALLDIR/share/dtd/officedocument.*" + + # Split out java stuff + mv_file_between_flists java_common_list.txt common_list.txt $INSTALLDIR/program/JREProperties.class + mv_file_between_flists java_common_list.txt common_list.txt "$INSTALLDIR/program/classes.*" + mv_file_between_flists java_common_list.txt common_list.txt $INSTALLDIR/program/libofficebean.so + mv_file_between_flists java_common_list.txt common_list.txt "$INSTALLDIR/share/Scripts/java.*" + mv_file_between_flists java_common_list.txt writer_list.txt $INSTALLDIR/program/classes/writer2latex.jar + + # Move arch-dependent/dup files from common to core + for f in \ + ".*\.so" \ + ".*\.so\..*" \ + "program/.*\.rdb" \ + program/configimport.bin \ + program/javaldx \ + program/msfontextract \ + program/oosplash.bin \ + program/pagein \ + program/pagein-calc \ + program/pagein-common \ + program/pagein-draw \ + program/pagein-impress \ + program/pagein-writer \ + program/pkgchk.bin \ + program/pluginapp.bin \ + program/setofficelang.bin \ + program/soffice.bin \ + program/uno.bin \ + program/unopkg.bin \ + program/uri-encode + do + mv_file_between_flists core_list.txt common_list.txt "$INSTALLDIR/$f" + done + + # themes are included in other packages + # don't use remove_file as we don't want them removed from the buildroot. + mv_file_between_flists /dev/null common_list.txt $INSTALLDIR/share/config/images_crystal.zip + mv_file_between_flists /dev/null common_list.txt $INSTALLDIR/share/config/images_hicontrast.zip + mv_file_between_flists /dev/null common_list.txt $INSTALLDIR/share/config/images.zip + fi + + # remove known duplicities to do not have files packaged in two packages + # the Bulgarian fixes can be removed after the issue #54110 is fixed + remove_duplicity_from_flists common_list.txt lang_bg_list.txt $INSTALLDIR/presets/config/arrowhd.soe + remove_duplicity_from_flists common_list.txt lang_bg_list.txt $INSTALLDIR/presets/config/classic.sog + remove_duplicity_from_flists common_list.txt lang_bg_list.txt $INSTALLDIR/presets/config/hatching.soh + remove_duplicity_from_flists common_list.txt lang_bg_list.txt $INSTALLDIR/presets/config/modern.sog + remove_duplicity_from_flists common_list.txt lang_bg_list.txt $INSTALLDIR/presets/config/palette.soc + remove_duplicity_from_flists common_list.txt lang_bg_list.txt $INSTALLDIR/presets/config/styles.sod + # the British fixes can be removed after the issue #54113 is fixed + remove_duplicity_from_flists common_list.txt lang_en-GB_list.txt $INSTALLDIR/presets/config/standard.sog + +else + + echo "Creating package directories..." + + test -d pkg && rm -r pkg || : + + # Create package tree (needed by Debian's dpkg) + # create_package_directory <list_file> <directory_name> + create_package_directory() + { + listfile=$1 + directory="$2" + perl -nl \ + -e " if(/^%dir (.*)/) + {system('mkdir', '-p', '-m', '755', \"$directory\".\$1);} + else + {rename('./'.\$_, \"$directory\".\$_);} + " \ + $listfile + } + + create_package_directory gid_Module_Root_Ure_Hidden pkg/ure + + create_package_directory gid_Module_Root pkg/libreoffice-common + create_package_directory gid_Module_Root_Brand pkg/libreoffice-common + create_package_directory gid_Module_Root_Files_Images pkg/libreoffice-common + create_package_directory gid_Module_Oo_Linguistic pkg/libreoffice-common + create_package_directory gid_Module_Optional_Xsltfiltersamples pkg/libreoffice-common + create_package_directory gid_Module_Optional_Grfflt pkg/libreoffice-draw + create_package_directory gid_Module_Prg_Calc_Bin pkg/libreoffice-calc + create_package_directory gid_Module_Prg_Math_Bin pkg/libreoffice-math + create_package_directory gid_Module_Prg_Draw_Bin pkg/libreoffice-draw + create_package_directory gid_Module_Prg_Wrt_Bin pkg/libreoffice-writer + create_package_directory gid_Module_Prg_Impress_Bin pkg/libreoffice-impress + create_package_directory gid_Module_Prg_Base_Bin pkg/libreoffice-base + create_package_directory gid_Module_Brand_Prg_Calc pkg/libreoffice-calc + create_package_directory gid_Module_Brand_Prg_Math pkg/libreoffice-math + create_package_directory gid_Module_Brand_Prg_Draw pkg/libreoffice-draw + create_package_directory gid_Module_Brand_Prg_Wrt pkg/libreoffice-writer + create_package_directory gid_Module_Brand_Prg_Impress pkg/libreoffice-impress + create_package_directory gid_Module_Brand_Prg_Base pkg/libreoffice-base + create_package_directory gid_Module_Pyuno pkg/python-uno + create_package_directory gid_Module_Optional_Gnome pkg/libreoffice-gnome + + create_package_directory gid_Module_Root_Files_2 pkg/libreoffice-common + create_package_directory gid_Module_Root_Files_3 pkg/libreoffice-common + create_package_directory gid_Module_Root_Files_4 pkg/libreoffice-common + create_package_directory gid_Module_Root_Files_5 pkg/libreoffice-common + create_package_directory gid_Module_Root_Files_6 pkg/libreoffice-common + create_package_directory gid_Module_Root_Files_7 pkg/libreoffice-common + if [ -e gid_Module_Optional_Pymailmerge ]; then + create_package_directory gid_Module_Optional_Pymailmerge pkg/libreoffice-emailmerge + else # post m26 + mkdir -p pkg/libreoffice-emailmerge/$INSTALLDIR/program + mv pkg/libreoffice-common/$INSTALLDIR/program/mailmerge.py \ + pkg/libreoffice-emailmerge/$INSTALLDIR/program/mailmerge.py + fi + create_package_directory gid_Module_Optional_OGLTrans pkg/libreoffice-ogltrans + + create_package_directory gid_Module_Root_SDK pkg/libreoffice-dev + + for l in `echo $WITH_LANG_LIST`; do + for p in Impress Draw Math Calc Base Writer; do + create_package_directory gid_Module_Langpack_${p}_`echo $l | sed -e s/-/_/g` pkg/libreoffice-l10n-$l + done + create_package_directory gid_Module_Langpack_Basis_`echo $l | sed -e s/-/_/g` pkg/libreoffice-l10n-$l + create_package_directory gid_Module_Langpack_Brand_`echo $l | sed -e s/-/_/g` pkg/libreoffice-l10n-$l + create_package_directory gid_Module_Langpack_Resource_`echo $l | sed -e s/-/_/g` pkg/libreoffice-l10n-$l + create_package_directory gid_Module_Helppack_Help_`echo $l | sed -e s/-/_/g` pkg/libreoffice-help-$l + + # some help files are in _Langpack_{Writer,Impress,...}_<lang> + # move them from -l10n to -help + if [ "$l" = "en-US" ]; then d=en; else d=$l; fi + mv pkg/libreoffice-l10n-$l/$INSTALLDIR/help/$d/* \ + pkg/libreoffice-help-$l/$INSTALLDIR/help/$d && \ + rmdir pkg/libreoffice-l10n-$l/$INSTALLDIR/help/$d + done + + # move_wrappers <directory_name> <name> [...] + move_wrappers() + { + directory=$1 + shift + mkdir -m755 -p "$directory"/usr/bin + while test -n "$1"; do + mv usr/*bin/"$1$BINSUFFIX" "$directory"/usr/bin + shift + done + } + move_wrappers pkg/libreoffice-common soffice unopkg + if test "$COMPAT_OOWRAPPERS" = "YES" ; then + move_wrappers pkg/libreoffice-common ooffice oofromtemplate + move_wrappers pkg/libreoffice-base oobase + move_wrappers pkg/libreoffice-writer oowriter ooweb + move_wrappers pkg/libreoffice-calc oocalc + move_wrappers pkg/libreoffice-impress ooimpress + move_wrappers pkg/libreoffice-math oomath + move_wrappers pkg/libreoffice-draw oodraw + fi + move_wrappers pkg/libreoffice-common libreoffice lofromtemplate + move_wrappers pkg/libreoffice-base lobase + move_wrappers pkg/libreoffice-writer lowriter loweb + move_wrappers pkg/libreoffice-calc localc + move_wrappers pkg/libreoffice-impress loimpress + move_wrappers pkg/libreoffice-math lomath + move_wrappers pkg/libreoffice-draw lodraw + + # Move all libraries, binaries, *.rdb from -common to -core + for d in $INSTALLDIR/program $INSTALLDIR/program; do \ + if [ ! -d $DESTDIR/pkg/libreoffice-core/$d ]; then \ + mkdir -p $DESTDIR/pkg/libreoffice-core/$d; \ + fi && + ( cd pkg/libreoffice-common/$d + find -maxdepth 1 \ + -regex '\./\(.*\.so.*\|.*\.bin\|pagein\|msfontextract\|.*\.rdb\|javaldx\|uri-encode\)' \ + -exec mv {} $DESTDIR/pkg/libreoffice-core/$d \; + ); \ + done + + # install additional ooo-build scripts & misc stuff + mkdir -p pkg/libreoffice-common/usr/share/man/man1 + if test "$COMPAT_OOWRAPPERS" = "YES" ; then + mv usr/share/man/man1/openoffice$BINSUFFIX.1 \ + pkg/libreoffice-common/usr/share/man/man1 + fi + mv usr/share/man/man1/libreoffice$BINSUFFIX.1 \ + pkg/libreoffice-common/usr/share/man/man1 + mkdir -p pkg/libreoffice-common/etc/bash_completion.d + if test "$COMPAT_OOWRAPPERS" = "YES" ; then + mv etc/bash_completion.d/ooffice$BINSUFFIX.sh \ + pkg/libreoffice-common/etc/bash_completion.d + fi + mv etc/bash_completion.d/libreoffice$BINSUFFIX.sh \ + pkg/libreoffice-common/etc/bash_completion.d + mv .$INSTALLDIR/program/java-set-classpath \ + pkg/libreoffice-common/$INSTALLDIR/program + if echo $WITH_LANG_LIST | grep -q en-US; then + for i in forms/resume.ott officorr/project-proposal.ott; do \ + mkdir -p pkg/libreoffice-common/$INSTALLDIR/share/template/en-US/`dirname $i`; \ + mv .$INSTALLDIR/share/template/en-US/$i \ + pkg/libreoffice-common/$INSTALLDIR/share/template/en-US/$i; \ + done; \ + fi + # Warn for any remaining files + find . -path './pkg' -prune -o -not -name 'gid_Module_*' -not -type d -exec echo "File not packaged: {}" \; +fi + +# mark the config files +RPM_CONFIG_FILE_TAGS= +if test "$OOO_VENDOR" = "SUSE" -o "$OOO_VENDOR" = "RedHat"; then + RPM_CONFIG_FILE_TAGS="%config" +elif test "$OOO_VENDOR" = "PLD" ; then + RPM_CONFIG_FILE_TAGS="%config(noreplace) %verify(not md5 size mtime)" +fi + +if test "z$RPM_CONFIG_FILE_TAGS" != "z" ; then + cd $FILELISTSDIR + perl -pi -e "s|^($INSTALLDIR/help/.*\.xsl)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/help/.*\.css)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/program/[a-zA-Z0-9_\.]*rc)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/program/.*\.xsl)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/share/config/[a-zA-Z0-9]*rc)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/share/dict/ooo/.*\.lst)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/share/psprint/.*\.conf)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/share/registry/.*\.xcu)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/share/registry/.*\.properties)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/share/registry/.*\.xcs)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + -e "s|^($INSTALLDIR/user/config/.*\.so.)\$|$RPM_CONFIG_FILE_TAGS \\1|;" \ + *_list.txt +fi + +mkdir -p $FILELISTSDIR/orig +mv -f $DESTDIR/gid_Module_* $FILELISTSDIR/orig diff --git a/bin/distro-install-sdk b/bin/distro-install-sdk new file mode 100755 index 000000000..e8cf28d61 --- /dev/null +++ b/bin/distro-install-sdk @@ -0,0 +1,84 @@ +#!/bin/sh + +if test -z "${SRC_ROOT}"; then + echo "distro-install-clean-up: No environment set!" + exit 1 +fi + +if test -d $DESTDIR$INSTALLDIR/sdk ; then + + echo "SDK installation clean up" + + # bin potential .orig files + find $DESTDIR$INSTALLDIR/sdk -name "*.orig" -exec rm -f {} \; + + # move some SDK directories to the right place according to FHS + # note that examples must stay in $DESTDIR$INSTALLDIR/sdk because there are used + # relative paths to $DESTDIR$INSTALLDIR/sdk/setting and it does not work via + # a symlink + mkdir -p $DESTDIR$PREFIXDIR/include + mkdir -p $DESTDIR$DATADIR/idl + mkdir -p $DESTDIR$DATADIR/$INSTALLDIRNAME/sdk + mkdir -p $DESTDIR$DOCDIR/sdk + mv $DESTDIR$INSTALLDIR/sdk/include $DESTDIR$PREFIXDIR/include/$INSTALLDIRNAME + if [ -d $DESTDIR$INSTALLDIR/sdk/classes ]; then + mv $DESTDIR$INSTALLDIR/sdk/classes $DESTDIR$DATADIR/$INSTALLDIRNAME/sdk/classes + fi + mv $DESTDIR$INSTALLDIR/sdk/idl $DESTDIR$DATADIR/idl/$INSTALLDIRNAME + mv $DESTDIR$INSTALLDIR/sdk/docs $DESTDIR$DOCDIR/sdk + mv $DESTDIR$INSTALLDIR/sdk/share/readme $DESTDIR$DOCDIR/sdk/readme + mv $DESTDIR$INSTALLDIR/sdk/index.html $DESTDIR$DOCDIR/sdk + + # compat symlinks + ln -sf $PREFIXDIR/include/$INSTALLDIRNAME $DESTDIR$INSTALLDIR/sdk/include + ln -sf $DATADIR/$INSTALLDIRNAME/sdk/classes $DESTDIR$INSTALLDIR/sdk/classes + ln -sf $DATADIR/idl/$INSTALLDIRNAME $DESTDIR$INSTALLDIR/sdk/idl + ln -sf $DOCDIR/sdk/docs $DESTDIR$INSTALLDIR/sdk/ + ln -sf $DOCDIR/sdk/index.html $DESTDIR$INSTALLDIR/sdk/index.html + ln -sf $INSTALLDIR/sdk/examples $DESTDIR$DOCDIR/sdk/examples + + # fix file list + sed -e "s|^\(%dir \)\?$INSTALLDIR/sdk/include|\1$PREFIXDIR/include/$INSTALLDIRNAME|" \ + -e "s|^\(%dir \)\?$INSTALLDIR/sdk/classes|\1$DATADIR/$INSTALLDIRNAME/sdk/classes|" \ + -e "s|^\(%dir \)\?$INSTALLDIR/sdk/idl|\1$DATADIR/idl/$INSTALLDIRNAME|" \ + -e "s|^\(%dir \)\?$INSTALLDIR/sdk/docs|\1$DOCDIR/sdk/docs|" \ + -e "s|^\(%dir \)\?$INSTALLDIR/sdk/share/readme|\1$DOCDIR/sdk/readme|" \ + -e "s|^$INSTALLDIR/sdk/index.html$|$DOCDIR/sdk/index.html|" \ + -e "s|^\(%dir \)\?$INSTALLDIR/sdk/share.*$||" \ + -e "/\.orig$/D" \ + -e "/^$/D" \ + $DESTDIR/gid_Module_Root_SDK | sort -u \ + >$DESTDIR/gid_Module_Root_SDK.new + mv $DESTDIR/gid_Module_Root_SDK.new $DESTDIR/gid_Module_Root_SDK + # + echo "%dir $DATADIR/$INSTALLDIRNAME/sdk" >>$DESTDIR/gid_Module_Root_SDK + echo "%dir $DATADIR/$INSTALLDIRNAME" >>$DESTDIR/gid_Module_Root_SDK + echo "%dir $DATADIR/idl" >>$DESTDIR/gid_Module_Root_SDK + echo "%dir $DOCDIR/sdk/docs" >>$DESTDIR/gid_Module_Root_SDK + echo "%dir $DOCDIR/sdk" >>$DESTDIR/gid_Module_Root_SDK + echo "%dir $DOCDIR" >>$DESTDIR/gid_Module_Root_SDK + echo "$INSTALLDIR/sdk/include" >>$DESTDIR/gid_Module_Root_SDK + echo "$INSTALLDIR/sdk/classes" >>$DESTDIR/gid_Module_Root_SDK + echo "$INSTALLDIR/sdk/idl" >>$DESTDIR/gid_Module_Root_SDK + echo "$INSTALLDIR/sdk/docs" >>$DESTDIR/gid_Module_Root_SDK + echo "$INSTALLDIR/sdk/index.html" >>$DESTDIR/gid_Module_Root_SDK + echo "$DOCDIR/sdk/examples" >>$DESTDIR/gid_Module_Root_SDK + + # generate default profiles + sed -e "s,@OO_SDK_NAME@,libreoffice${PRODUCTVERSION}_sdk," \ + -e "s,@OO_SDK_HOME@,$INSTALLDIR/sdk," \ + -e "s,@OFFICE_HOME@,$INSTALLDIR," \ + -e "s,@OO_SDK_MAKE_HOME@,$(dirname $(command -v make))," \ + -e "s,@OO_SDK_ZIP_HOME@,$(dirname $(command -v zip))," \ + -e "s,@OO_SDK_CPP_HOME@,$(dirname $(command -v cpp))," \ + -e "s,@OO_SDK_SED_HOME@,$(dirname $(command -v sed))," \ + -e "s,@OO_SDK_CAT_HOME@,$(dirname $(command -v cat))," \ + -e "s,@OO_SDK_JAVA_HOME@,$JAVA_HOME," \ + -e "s,@OO_SDK_OUTPUT_DIR@,\$HOME," \ + -e "s,@SDK_AUTO_DEPLOYMENT@,NO," \ + $DESTDIR$INSTALLDIR/sdk/setsdkenv_unix.sh.in \ + > $DESTDIR$INSTALLDIR/sdk/setsdkenv_unix.sh + chmod 755 $DESTDIR$INSTALLDIR/sdk/setsdkenv_unix.sh + echo $INSTALLDIR/sdk/setsdkenv_unix.sh >>$DESTDIR/gid_Module_Root_SDK + +fi diff --git a/bin/dump-poolitems-values.py b/bin/dump-poolitems-values.py new file mode 100755 index 000000000..c2c5f357e --- /dev/null +++ b/bin/dump-poolitems-values.py @@ -0,0 +1,91 @@ +#!/usr/bin/python + + +# Produce a dump of name->constant of the poolitem values, to make interpreting things in the debugger easier +# + +import subprocess +import sys + +macroNameToValue = dict() +macroNameToOriginalLine = dict() + + +def extractMacroValue(macroValue): + if isinstance(macroValue, int): + return macroValue + elif macroValue.isdigit(): + return int(macroValue) + elif macroValue[0:2] == "0x": + return int(macroValue, 16) + elif macroValue.find("+") != -1: + tokens = macroValue.split("+") + tokens1 = tokens[0].strip() + tokens2 = tokens[1].strip() + return extractMacroValue(tokens1) + extractMacroValue(tokens2) + elif macroValue.find("-") != -1: + tokens = macroValue.split("-") + tokens1 = tokens[0].strip() + tokens2 = tokens[1].strip() + return extractMacroValue(tokens1) - extractMacroValue(tokens2) + rv = extractMacroValue(macroNameToValue[macroValue]) + macroNameToValue[macroValue] = rv + return rv + + +a = subprocess.Popen("cpp -E -dD -Iinclude/ include/editeng/eeitem.hxx", stdout=subprocess.PIPE, shell=True) + +with a.stdout as txt: + for line in txt: + line = line.strip() + originalLine = line + if not line.startswith("#define "): continue + # strip the '#define' off the front + idx1 = line.find(" ") + line = line[idx1 : len(line)].strip() + # extract the name + idx1 = line.find(" ") + if (idx1 == -1): continue + macroName = line[0 : idx1].strip() + line = line[idx1 : len(line)].strip() + # ignore internal stuff + if macroName.startswith("_"): continue + # strip any trailing comments + idx1 = line.find("//") + if (idx1 != -1): + line = line[0 : idx1].strip() + idx1 = line.find("/*") + if (idx1 != -1): + line = line[0 : idx1].strip() + if len(line) == 0: continue + # strip brackets + if line[0] == "(": line = line[1:] + if line[len(line)-1] == ")": line = line[0:len(line)-1] + macroValue = line.strip() + # ignore macros that #define strings, not interested in those + if (macroValue.find("\"") != -1): continue + # ignore the multiline macros + if (macroValue.find("\\") != -1): continue + # check for redefinitions + if macroNameToValue.has_key(macroName): + print "Redefinition:\n\t", macroNameToOriginalLine[macroName], "\n\t" , originalLine + else: + macroNameToValue[macroName] = macroValue + macroNameToOriginalLine[macroName] = originalLine + +# decode the constants into their numeric values recursively +macroValueToName = dict() +for macroName in macroNameToValue: + macroValue = macroNameToValue[macroName] + try: + macroValue = extractMacroValue(macroName) + macroValueToName[macroValue] = macroName + except KeyError: + print "warning: could not decode macro ", macroName + +for macroValue in sorted(macroValueToName): + macroName = macroValueToName[macroValue] + print repr(macroNameToValue[macroName]).rjust(5), " ", macroName + + + diff --git a/bin/extract-tooltip.py b/bin/extract-tooltip.py new file mode 100755 index 000000000..5397c718f --- /dev/null +++ b/bin/extract-tooltip.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +import sys +import os +import re +import urlparse + +def usage(): + message = """ usage: {program} inDir outDir +inDir: directory containing .ht files +outDir: target for the new files""" + print(message.format(program = os.path.basename(sys.argv[0]))) + +def parseFile(filename): + file = open(filename, "r") + data = file.readlines() + data = [line.rstrip('\n') for line in data] + + pairs = {} + regEx = re.compile("^(\S+)\s(\S+)\s(\S+)\s((?:\s*\S*)+)$") + old_line = None + for line in data: + if len(line) > 0: + if(old_line != None): + print filename + #print("failed to parse line") + #print(old_line) + line = old_line + line + print line + old_line = None + split_line = regEx.split(line) + #print(split_line) + #print(urlparse.unquote(split_line[2])) + #print(split_line[4]) + if(old_line == None and split_line[4] == "" and split_line[3] != "0"): + print(line) + print(split_line) + old_line = line + else: + pairs[urlparse.unquote(split_line[2])] = split_line[4] + assert(len(split_line) == 6) + #print data + #print(pairs) + return pairs + +def parseFiles(dir): + strings = [] + for files in os.listdir(dir): + if files.endswith(".ht"): + string = parseFile(os.path.join(dir,files)) + print(files) + #print string + strings.append([files, string]) + return strings + +def extractSharedEntries(strings): + first_dict = strings[0][1] + shared_dict = {} + #print(first_dict) + for key, value in first_dict.iteritems(): + # check that the entry in the same in all dics + is_in_all_dicts = True + for dict_file_pair in strings: + dict = dict_file_pair[1] + if not dict.has_key(key): + is_in_all_dicts = False + elif not dict[key] == value: + print("Element with different values") + print(key) + is_in_all_dicts = False + if is_in_all_dicts: + shared_dict[key] = value + #print(shared_dict) + for dict_file_pair in strings: + for key in shared_dict.iterkeys(): + dict_file_pair[1].pop(key) + + strings.append(["shared.ht", shared_dict]) + return strings + +def writeOutFiles(dir, strings): + for string in strings: + file_name_base = string[0] + file_name_base = file_name_base.replace(".ht", ".properties") + file_name = os.path.join(dir, file_name_base) + file = open(file_name, "w") + for key, value in string[1].iteritems(): + try: + file.write(key) + file.write("=") + file.write(value) + file.write("\n") + except UnicodeDecodeError: + print key + print value + file.close() + +def main (args): + if(len(args) != 3): + usage() + sys.exit(1) + + strings = parseFiles(args[1]) + new_strings = extractSharedEntries(strings) + writeOutFiles(args[2], new_strings) + +if __name__ == "__main__": + main(sys.argv) diff --git a/bin/fake_pom.xml b/bin/fake_pom.xml new file mode 100644 index 000000000..50599f3ab --- /dev/null +++ b/bin/fake_pom.xml @@ -0,0 +1,6 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"> + <modelVersion>4.0.0</modelVersion> + <groupId>org.libreoffice</groupId> + <artifactId>LibreOffice-Maven</artifactId> + <version>1</version> +</project> diff --git a/bin/find-can-be-private-symbols.classes.results b/bin/find-can-be-private-symbols.classes.results new file mode 100644 index 000000000..05defda91 --- /dev/null +++ b/bin/find-can-be-private-symbols.classes.results @@ -0,0 +1,283 @@ +BitmapAlphaClampFilter +BitmapColorQuantizationFilter +BitmapConvolutionMatrixFilter +BitmapEmbossGreyFilter +BitmapMedianFilter +BitmapMonochromeFilter +BitmapMonochromeMatrixFilter +BitmapMosaicFilter +BitmapPalette +BitmapPopArtFilter +BitmapSepiaFilter +BitmapSimpleColorQuantizationFilter +BitmapSmoothenFilter +BitmapSobelGreyFilter +BitmapSolarizeFilter +ConditionEditDropTarget +CurrencyFormatter +DdeGetPutItem +DdeLink +DdeService +DdeTopic +E3dCompoundObject +EditUndo +FmDesignModeChangedHint +FocusListenerMultiplexer +FontSelectPattern +FontSubsetInfo +FreetypeManager::IFSD_Equal +GrBackendFormat +GrBackendRenderTarget +GrBackendTexture +GrContext +GrContextThreadSafeProxy +GrContext_Base +GrGLExtensions +GrGLInterface +GrGpuResource +GrGpuResource::ProxyAccess +GrImageContext +GrVkExtensions +GrVkSecondaryCBDrawContext +HelpLinker +Hunspell +Hunzip +ImplJobSetup +IndexerPreProcessor +KeyListenerMultiplexer +MetaAction +MetaGradientExAction +MorkParser +MouseListenerMultiplexer +MouseMotionListenerMultiplexer +MyThes +OpenGLFramebuffer +OpenGLZone +PackedTextureAtlasManager +PaintListenerMultiplexer +PhysicalFontFamily +ProcessData +RenderList +SalData +SalDisplay +SalInfoPrinter +SalPrinter +SalSystem +SbClassModuleObject +ScChart2DataProvider +ScFormatEntry +ScPaintHint +ScPreviewShell +ScRefreshTimer +SdAbstractDialogFactory +SdOptionsItem +SdOptionsLayout +SdOptionsMisc +SdOptionsPrint +SdOptionsSnap +SdXImpressDocument +SdrCaptionEscDirItem +SdrCaptionTypeItem +SdrEdgeNode1HorzDistItem +SdrEdgeNode1VertDistItem +SdrEdgeNode2HorzDistItem +SdrEdgeNode2VertDistItem +SdrEmbedObjectLink +SdrGrafBlueItem +SdrGrafContrastItem +SdrGrafCropItem +SdrGrafGamma100Item +SdrGrafGreenItem +SdrGrafLuminanceItem +SdrGrafModeItem +SdrGrafRedItem +SdrGrafTransparenceItem +SdrMeasureTextHPosItem +SdrMeasureTextVPosItem +SdrMeasureUnitItem +SdrOnOffItem +SdrPercentItem +SdrSignedPercentItem +SdrTextAniAmountItem +SdrTextAniDelayItem +SdrTextAniDirectionItem +SdrTextAniKindItem +SdrTextHorzAdjustItem +SdrUndoInsertObj +SdrUndoNewPage +SdrUndoPageMasterPage +SdrYesNoItem +SfxNavigator +SfxStyleSheetModifiedHint +SfxViewFrameItem +SfxVisibilityItem +SpinListenerMultiplexer +SvxGraphicObject +SvxMetricField +SvxPrintItem +SvxRsidItem +SvxShowText +SvxTPage +SwAnchoredObject +SwAuthenticator +SwColExample +SwConnectionListener +SwContrastGrf +SwDrawFrameFormat +SwDrawModeGrf +SwExtraRedline +SwFltRedline +SwFormatEditInReadonly +SwFormatEndAtTextEnd +SwFormatFollowTextFlow +SwFormatFootnoteAtTextEnd +SwFormatLayoutSplit +SwFormatNoBalancedColumns +SwFormatRowSplit +SwGammaGrf +SwHeaderAndFooterEatSpacingItem +SwLayoutFrame +SwLuminanceGrf +SwMirrorGrf +SwNumRuleItem +SwPagePreview +SwRedlineExtraData +SwRedlineExtraData_FormatColl +SwShellCursor +SwTableCellInfo::Impl +SwTableCellRedline +SwTableRowRedline +SwTestItem +SwWebDocShell +SwWebView +SwWrtShellItem +SwXTextRange::Impl +SwXTextTableCursor +SyntaxHighlighter::Tokenizer +SystemWindow::ImplData +TBCExtraInfo +TBCGeneralInfo +TreeEditListenerMultiplexer +TreeExpansionListenerMultiplexer +TreeSelectionListenerMultiplexer +VclAlignment +VclBin +VclBuilder::MenuAndId +VclBuilder::ParserState +VclBuilder::sortIntoBestTabTraversalOrder +VclDrawingArea +VclGrid +VclWindowEvent +WString +WindowListenerMultiplexer +X11SalObject +X11SkiaSalGraphicsImpl +XMLCellStyleExport +XMLConstantsPropertyHandler +XMLEnumPropertyHdl +XMLShapeStyleContext +basegfx::BColorModifier +basegfx::MinimalSystemDependentDataManager +canvas +chart::PopupRequest +comphelper::RefCountedMutex +comphelper::service_decl::ServiceDecl::Factory +connectivity::component::OComponentPreparedStatement +connectivity::component::OComponentStatement +connectivity::file::OBoolOperator +connectivity::file::OOp_ISNOTNULL +connectivity::file::OOp_ISNULL +connectivity::file::OOp_LIKE +connectivity::odbc::OConnection +connectivity::odbc::ODBCDriver +connectivity::odbc::ODatabaseMetaData +connectivity::odbc::ODatabaseMetaDataResultSet +connectivity::odbc::OPreparedStatement +connectivity::odbc::OResultSet +connectivity::odbc::OResultSetMetaData +connectivity::odbc::OStatement +connectivity::odbc::OStatement_BASE2 +connectivity::odbc::OStatement_Base +connectivity::odbc::OTools +connectivity::sdbcx::IObjectCollection +connectivity::sdbcx::OGroup +connectivity::sdbcx::OKey +cppu::BootstrapException +cppu::ClassData +cppu::ClassDataBase +dbtools::param::ParameterWrapper +desktop::CallbackFlushHandler::CallbackData +dp_misc::AbortChannel +drawinglayer::animation::AnimationEntry +drawinglayer::primitive2d::AnimatedSwitchPrimitive2D +drawinglayer::primitive2d::ObjectAndViewTransformationDependentPrimitive2D +drawinglayer::primitive2d::SdrFrameBorderData::SdrConnectStyleData +drawinglayer::primitive2d::ViewTransformationDependentPrimitive2D +drawinglayer::primitive3d +drawinglayer::primitive3d::BasePrimitive3D +drawinglayer::primitive3d::BufferedDecompositionPrimitive3D +drawinglayer::primitive3d::GroupPrimitive3D +drawinglayer::primitive3d::PolyPolygonMaterialPrimitive3D +drawinglayer::primitive3d::PolygonHairlinePrimitive3D +drawinglayer::primitive3d::SdrPrimitive3D +formula::FormulaByteToken +formula::FormulaDoubleToken +formula::FormulaErrorToken +formula::FormulaExternalToken +formula::FormulaFAPToken +formula::FormulaIndexToken +formula::FormulaJumpToken +formula::FormulaMissingToken +formula::FormulaTokenIterator::Item +formula::FormulaTypedDoubleToken +formula::FormulaUnknownToken +jvmaccess::UnoVirtualMachine::CreationException +jvmaccess::VirtualMachine::AttachGuard::CreationException +linguistic::PropertyChgHelper +linguistic::PropertyHelper_Spell +oox::IProgressBar +oox::ole::AxContainerModelBase +oox::ole::AxControlModelBase +oox::ole::AxFontDataModel +oox::ole::AxImageModel +oox::ole::AxMorphDataModelBase +oox::ole::AxMultiPageModel +oox::ole::AxPageModel +oox::ole::AxTabStripModel +oox::ole::AxToggleButtonModel +oox::ole::AxUserFormModel +psp::PrintFontManager::PrintFont +salhelper::ORealDynamicLoader +sc::FormulaGroupInterpreter +sd::DrawView +sdr::SelectionController +sdr::ViewSelection +sdr::animation::primitiveAnimator +sdr::contact::ObjectContactPainter +sdr::properties::BaseProperties +sfx2::sidebar::Panel +sfx2::sidebar::SidebarToolBox +sfx2::sidebar::TabBar::Item +skjpeg_destination_mgr +svt::MultiLineEditImplementation +svt::MultiLineTextCell +svx::CommonStyleManager +svx::DialControl::DialControl_Impl +svx::PropertyValueProvider +sw::BroadcastingModify +sw::UnoCursorHint +ucbhelper::ActiveDataSink +ucbhelper::InteractionAbort +ucbhelper::InteractionApprove +ucbhelper::InteractionDisapprove +ucbhelper::InteractionRetry +ucbhelper::InteractionSupplyAuthentication +utl::Bootstrap::Impl +utl::OInputStreamHelper +vcl::ExtOutDevData +vcl::test::OutputDeviceTestGradient +void OpenGLTexture +writerperfect::DirectoryStream::Impl +xmloff::OControlBorderHandler +xmloff::OFontWidthHandler +xmloff::ORotationAngleHandler diff --git a/bin/find-can-be-private-symbols.functions.results b/bin/find-can-be-private-symbols.functions.results new file mode 100644 index 000000000..59453f8ea --- /dev/null +++ b/bin/find-can-be-private-symbols.functions.results @@ -0,0 +1,39 @@ +GrGLAssembleInterface(void*, void (*(*)(void*, char const*))()) +ImplCallPreNotify(NotifyEvent&) +ImplDestroyHelpWindow(bool) +ImplFastBitmapConversion(BitmapBuffer&, BitmapBuffer const&, SalTwoRect const&) +ImplGetSalSystem() +ImplHideSplash() +ImplSVMain() +ScFilterCreate +SdResId(char const*, int) +Simplify(SkPath const&, SkPath*) +clewErrorString +component_getImplementationEnvironment +createLink +ddot +dl_cairo_surface_set_device_scale(_cairo_surface*, double, double) +endlu(SvStream&) +explain +fieldlen(char const*) +getDataArea +heuristics +invert +libreofficekit_hook +libreofficekit_hook_2 +main +mod +privateSnippetExecutor +reg_closeKey(void*) +reg_closeRegistry(void*) +reg_dumpRegistry(void*) +reg_openKey(void*, _rtl_uString*, void**) +reg_openRegistry(_rtl_uString*, void**) +reg_openRootKey(void*, void**) +report +scale +setLink +set_column +set_title +spaces +vcl_crc64 diff --git a/bin/find-can-be-private-symbols.py b/bin/find-can-be-private-symbols.py new file mode 100755 index 000000000..f5ff83fd1 --- /dev/null +++ b/bin/find-can-be-private-symbols.py @@ -0,0 +1,226 @@ +#!/usr/bin/python2 +# +# Find exported symbols that can be made non-exported. +# +# Noting that (a) parsing these commands is a pain, the output is quite irregular and (b) I'm fumbling in the +# dark here, trying to guess what exactly constitutes an "import" vs an "export" of a symbol, linux linking +# is rather complex. +# +# Takes about 5min to run on a decent machine. +# +# The standalone function analysis is reasonable reliable, but the class/method analysis is less so +# (something to do with destructor thunks not showing up in my results?) +# +# Also, the class/method analysis will not catch problems like +# 'dynamic_cast from 'Foo' with hidden type visibility to 'Bar' with default type visibility' +# but loplugin:dyncastvisibility will do that for you +# + +import subprocess +import sys +import re + +exported_symbols = set() +imported_symbols = set() +# standalone functions that are exported but not imported +unused_function_exports = set() +classes_with_exported_symbols = set() +classes_with_imported_symbols = set() +# all names that exist in the source code +all_source_names = set() + + +# look for imported symbols in executables +subprocess_find_all_source_names = subprocess.Popen("git grep -oh -P '\\b\\w\\w\\w+\\b' -- '*.h*'", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) +with subprocess_find_all_source_names.stdout as txt: + for line in txt: + line = line.strip() + all_source_names.add(line) +subprocess_find_all_source_names.terminate() + +subprocess_find = subprocess.Popen("find ./instdir -name *.so && find ./workdir/LinkTarget/CppunitTest -name *.so", stdout=subprocess.PIPE, shell=True) +with subprocess_find.stdout as txt: + for line in txt: + sharedlib = line.strip() + # look for exported symbols + subprocess_nm = subprocess.Popen("nm -D " + sharedlib, stdout=subprocess.PIPE, shell=True) + with subprocess_nm.stdout as txt2: + # We are looking for lines something like: + # 0000000000036ed0 T flash_component_getFactory + line_regex = re.compile(r'^[0-9a-fA-F]+ T ') + for line2 in txt2: + line2 = line2.strip() + if line_regex.match(line2): + exported_symbols.add(line2.split(" ")[2]) + # look for imported symbols + subprocess_objdump = subprocess.Popen("objdump -T " + sharedlib, stdout=subprocess.PIPE, shell=True) + with subprocess_objdump.stdout as txt2: + # ignore some header bumpf + txt2.readline() + txt2.readline() + txt2.readline() + txt2.readline() + # We are looking for lines something like: + # 0000000000000000 DF *UND* 0000000000000000 _ZN16FilterConfigItem10WriteInt32ERKN3rtl8OUStringEi + for line2 in txt2: + line2 = line2.strip() + tokens = line2.split(" ") + if len(tokens) < 7 or not(tokens[7].startswith("*UND*")): continue + sym = tokens[len(tokens)-1] + imported_symbols.add(sym) +subprocess_find.terminate() + +# look for imported symbols in executables +subprocess_find = subprocess.Popen("find ./instdir -name *.bin", stdout=subprocess.PIPE, shell=True) +with subprocess_find.stdout as txt: + for line in txt: + executable = line.strip() + # look for exported symbols + subprocess_nm = subprocess.Popen("nm -D " + executable + " | grep -w U", stdout=subprocess.PIPE, shell=True) + with subprocess_nm.stdout as txt2: + # We are looking for lines something like: + # U sal_detail_deinitialize + for line2 in txt2: + line2 = line2.strip() + sym = line2.split(" ")[1] + imported_symbols.add(sym) +subprocess_find.terminate() + +diff = exported_symbols - imported_symbols +print("exported = " + str(len(exported_symbols))) +print("imported = " + str(len(imported_symbols))) +print("diff = " + str(len(diff))) + +for sym in exported_symbols: + filtered_sym = subprocess.check_output(["c++filt", sym]).strip() + if filtered_sym.startswith("non-virtual thunk to "): filtered_sym = filtered_sym[21:] + elif filtered_sym.startswith("virtual thunk to "): filtered_sym = filtered_sym[17:] + i = filtered_sym.find("(") + i = filtered_sym.rfind("::", 0, i) + if i != -1: + classname = filtered_sym[:i] + # find classes where all of the exported symbols are not imported + classes_with_exported_symbols.add(classname) + else: + func = filtered_sym + # find standalone functions which are exported but not imported + if not(sym in imported_symbols): unused_function_exports.add(func) + +for sym in imported_symbols: + filtered_sym = subprocess.check_output(["c++filt", sym]).strip() + if filtered_sym.startswith("non-virtual thunk to "): filtered_sym = filtered_sym[21:] + elif filtered_sym.startswith("virtual thunk to "): filtered_sym = filtered_sym[17:] + i = filtered_sym.find("(") + i = filtered_sym.rfind("::", 0, i) + if i != -1: + classname = filtered_sym[:i] + classes_with_imported_symbols.add(classname) + +def extractFunctionNameFromSignature(sym): + i = sym.find("(") + if i == -1: return sym + return sym[:i] + +with open("bin/find-can-be-private-symbols.functions.results", "wt") as f: + for sym in sorted(unused_function_exports): + # Filter out most of the noise. + # No idea where these are coming from, but not our code. + if sym.startswith("CERT_"): continue + elif sym.startswith("DER_"): continue + elif sym.startswith("FORM_"): continue + elif sym.startswith("FPDF"): continue + elif sym.startswith("HASH_"): continue + elif sym.startswith("Hunspell_"): continue + elif sym.startswith("LL_"): continue + elif sym.startswith("LP_"): continue + elif sym.startswith("LU"): continue + elif sym.startswith("MIP"): continue + elif sym.startswith("MPS"): continue + elif sym.startswith("NSS"): continue + elif sym.startswith("NSC_"): continue + elif sym.startswith("PK11"): continue + elif sym.startswith("PL_"): continue + elif sym.startswith("PQ"): continue + elif sym.startswith("PBE_"): continue + elif sym.startswith("PORT_"): continue + elif sym.startswith("PRP_"): continue + elif sym.startswith("PR_"): continue + elif sym.startswith("PT_"): continue + elif sym.startswith("QS_"): continue + elif sym.startswith("REPORT_"): continue + elif sym.startswith("RSA_"): continue + elif sym.startswith("SEC"): continue + elif sym.startswith("SGN"): continue + elif sym.startswith("SOS"): continue + elif sym.startswith("SSL_"): continue + elif sym.startswith("VFY_"): continue + elif sym.startswith("_PR_"): continue + elif sym.startswith("_"): continue + elif sym.startswith("ber_"): continue + elif sym.startswith("bfp_"): continue + elif sym.startswith("ldap_"): continue + elif sym.startswith("ne_"): continue + elif sym.startswith("opj_"): continue + elif sym.startswith("pg_"): continue + elif sym.startswith("pq"): continue + elif sym.startswith("presolve_"): continue + elif sym.startswith("sqlite3_"): continue + # dynamically loaded + elif sym.endswith("get_implementation"): continue + elif sym.endswith("component_getFactory"): continue + elif sym == "CreateDialogFactory": continue + elif sym == "CreateUnoWrapper": continue + elif sym == "CreateWindow": continue + elif sym == "ExportDOC": continue + elif sym == "ExportPPT": continue + elif sym == "ExportRTF": continue + elif sym == "GetSaveWarningOfMSVBAStorage_ww8": continue + elif sym == "GetSpecialCharsForEdit": continue + elif sym.startswith("Import"): continue + elif sym.startswith("Java_com_sun_star_"): continue + elif sym.startswith("TestImport"): continue + elif sym.startswith("getAllCalendars_"): continue + elif sym.startswith("getAllCurrencies_"): continue + elif sym.startswith("getAllFormats"): continue + elif sym.startswith("getBreakIteratorRules_"): continue + elif sym.startswith("getCollationOptions_"): continue + elif sym.startswith("getCollatorImplementation_"): continue + elif sym.startswith("getContinuousNumberingLevels_"): continue + elif sym.startswith("getDateAcceptancePatterns_"): continue + elif sym.startswith("getForbiddenCharacters_"): continue + elif sym.startswith("getIndexAlgorithm_"): continue + elif sym.startswith("getLCInfo_"): continue + elif sym.startswith("getLocaleItem_"): continue + elif sym.startswith("getOutlineNumberingLevels_"): continue + elif sym.startswith("getReservedWords_"): continue + elif sym.startswith("getSTC_"): continue + elif sym.startswith("getSearchOptions_"): continue + elif sym.startswith("getTransliterations_"): continue + elif sym.startswith("getUnicodeScripts_"): continue + elif sym.startswith("lok_"): continue + # UDK API + elif sym.startswith("osl_"): continue + elif sym.startswith("rtl_"): continue + elif sym.startswith("typelib_"): continue + elif sym.startswith("typereg_"): continue + elif sym.startswith("uno_"): continue + # remove things we found that do not exist in our source code, they're not ours + if not(extractFunctionNameFromSignature(sym) in all_source_names): continue + f.write(sym + "\n") + +with open("bin/find-can-be-private-symbols.classes.results", "wt") as f: + for sym in sorted(classes_with_exported_symbols - classes_with_imported_symbols): + # externals + if sym.startswith("libcdr"): continue + elif sym.startswith("libabw"): continue + elif sym.startswith("libebook"): continue + elif sym.startswith("libepubgen"): continue + elif sym.startswith("libfreehand"): continue + elif sym.startswith("libmspub"): continue + elif sym.startswith("libpagemaker"): continue + elif sym.startswith("libqxp"): continue + elif sym.startswith("libvisio"): continue + elif sym.startswith("libzmf"): continue + elif sym.startswith("lucene::"): continue + elif sym.startswith("Sk"): continue + f.write(sym + "\n") diff --git a/bin/find-clang-format.py b/bin/find-clang-format.py new file mode 100755 index 000000000..38c9aac10 --- /dev/null +++ b/bin/find-clang-format.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from difflib import unified_diff +from pathlib import Path +from subprocess import PIPE, Popen + +BLACKLIST = Path("solenv/clang-format/blacklist") +THRESHOLD = os.getenv("CLANG_THRESHOLD", 5) +CLANG_BINARY = Path(os.getenv("CLANG_FORMAT", "/opt/lo/bin/clang-format")) + + +def calculate_diff_size(diff): + additions, removals = 0, 0 + # ignore first 2 item in the sequence + # which are +++ and --- + for line in diff[2:]: + if line.startswith("+"): + additions += 1 + elif line.startswith("-"): + removals += 1 + return max((additions, removals)) + + +def format_stream(path, *extra_args): + process = Popen( + [CLANG_BINARY, *extra_args], stdout=PIPE, stderr=PIPE, stdin=PIPE, + ) + stdout, stderr = process.communicate(input=path.read_bytes()) + if stderr: + print("<FAIL>", str(path)) + print(stderr.decode()) + print("<FAIL>") + exit(1) + stdout = stdout.decode() + return stdout + + +def main(*args): + if not CLANG_BINARY.exists(): + print("Couldn't find clang-format on {!s}".format(CLANG_BINARY)) + exit(1) + + for path in BLACKLIST.read_text().splitlines(): + path = Path(path) + if not path.exists(): + continue + + original = path.read_text() + new = format_stream(path, *args) + originalsize = len(original.splitlines()) + diff = unified_diff(original.splitlines(), new.splitlines(), n=0) + diffsize = calculate_diff_size(tuple(diff)) + if diffsize <= (originalsize * 5) // 100: + print(path, "(size: {}/{})".format(diffsize, originalsize)) + + +if __name__ == "__main__": + import sys + + main(*sys.argv[1:]) diff --git a/bin/find-duplicated-files.py b/bin/find-duplicated-files.py new file mode 100755 index 000000000..08d90076c --- /dev/null +++ b/bin/find-duplicated-files.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import os +import sys + +from filecmp import dircmp + +""" +This script compares two directories and lists the files which are the same in both directories. +Intended to find duplicate icons among icon themes. + +Adopted from the example at https://docs.python.org/3.5/library/filecmp.html + +Usage: ./bin/findduplicatefiles dir1 dir2 +""" + +def print_diff_files(dcmp): + for name in dcmp.same_files: + print("%s found in %s and %s" % (name, dcmp.left, dcmp.right)) + for sub_dcmp in dcmp.subdirs.values(): + print_diff_files(sub_dcmp) + +if len(sys.argv) != 3: + print("Usage: %s dir1 dir2" % sys.argv[0]) + exit() + +dir1 = sys.argv[1] +dir2 = sys.argv[2] + +if not os.path.isdir(dir1) or not os.path.isdir(dir2): + print("Arguments must be directories!") + exit() + +dcmp = dircmp(dir1, dir2) +print_diff_files(dcmp) + diff --git a/bin/find-duplicated-sids.py b/bin/find-duplicated-sids.py new file mode 100755 index 000000000..8f5e4ff92 --- /dev/null +++ b/bin/find-duplicated-sids.py @@ -0,0 +1,92 @@ +#!/usr/bin/python + + +# Scan .hrc files for conflicting SID constants +# +# This is not as easy as it sounds because some of the constants depend on other constants whose names do not start with SID_ +# + +import subprocess +import sys + +sidNameToValue = dict() +sidNameToOriginalLine = dict() + + +def extractSidValue(sidValue): + if isinstance(sidValue, int): + return sidValue + if sidValue.isdigit(): + return int(sidValue) + if sidValue[0:2] == "0x": + return int(sidValue, 16) + if sidValue.find("+") != -1: + tokens = sidValue.split("+") + tokens1 = tokens[0].strip() + tokens2 = tokens[1].strip() + return extractSidValue(tokens1) + extractSidValue(tokens2) + rv = extractSidValue(sidNameToValue[sidValue]) + sidNameToValue[sidValue] = rv + return rv + + +#a = subprocess.Popen("git grep -P '#define\s+(SID_|SC_|DETECTIVE_|DRAWTEXTBAR_|DRAW_BAR_|RID_|OBJBAR_FORMAT_|TAB_POPUP_|DATA_MENU_|EXTRA_MENU_|FORMAT_MENU_|INSERT_MENU_|VIEW_MENU_|EDIT_MENU_|FILE_MENU_|SC_FUNCTION_|RC_)'", stdout=subprocess.PIPE, shell=True) +a = subprocess.Popen("git grep -Pn '#define\s+(\S+)' -- *.hrc", stdout=subprocess.PIPE, shell=True) + +with a.stdout as txt: + for line in txt: + originalLine = line.strip() + # strip the '#define' off the front + idx1 = line.find(" ") + line = line[idx1 : len(line)].strip() + # extract the name + idx1 = line.find(" ") + if (idx1 == -1): continue + sidName = line[0 : idx1].strip() + line = line[idx1 : len(line)].strip() + # strip any trailing comments + idx1 = line.find("//") + if (idx1 != -1): + line = line[0 : idx1].strip() + idx1 = line.find("/*") + if (idx1 != -1): + line = line[0 : idx1].strip() + if len(line) == 0: continue + # strip brackets + if line[0] == "(": line = line[1:] + if line[len(line)-1] == ")": line = line[0:len(line)-1] + sidTextValue = line.strip() + # ignore the #define strings + if (sidTextValue.find("\"") != -1): continue + # ignore the multiline macros + if (sidTextValue.find("\\") != -1): continue + # check for redefinitions + if sidName[0:4] == "SID_" and sidNameToValue.has_key(sidName): + print "Redefinition:\n\t", sidNameToOriginalLine[sidName], "\n\t" , originalLine + else: + sidNameToValue[sidName] = sidTextValue + sidNameToOriginalLine[sidName] = originalLine + + # decode the constants into their numeric values recursively + sidNamesToIgnore = set() + for sidName in sidNameToValue: + sidTextValue = sidNameToValue[sidName] + try: + sidValueNum = extractSidValue(sidTextValue) + sidNameToValue[sidName] = sidValueNum + except KeyError: + sidNamesToIgnore.add(sidName) + + # check for conflicts + sidValueToName = dict() + for sidName in sidNameToValue: + if sidName in sidNamesToIgnore: continue + if sidName[0:4] != "SID_": continue + sidValue = sidNameToValue[sidName] + if sidValueToName.has_key(sidValue): + print "conflict:\n\t", sidNameToOriginalLine[sidName], "\n\t", sidNameToOriginalLine[sidValueToName[sidValue]] + else: + sidValueToName[sidValue] = sidName + + + diff --git a/bin/find-files-not-referenced-by-makefile.py b/bin/find-files-not-referenced-by-makefile.py new file mode 100755 index 000000000..70232ed1c --- /dev/null +++ b/bin/find-files-not-referenced-by-makefile.py @@ -0,0 +1,53 @@ +#!/usr/bin/python2 + +# Look for CXX files that are not referenced by any makefile + +import subprocess +import sys + +sourceFiles = set() + +a = subprocess.Popen("git ls-files", stdout=subprocess.PIPE, shell=True) +with a.stdout as txt: + for filename in txt: + if filename.find(".cxx") != -1 \ + and filename.find("precompiled") == -1 \ + and filename.find("/workben") == -1 \ + and not filename.startswith("odk/examples/") \ + and not filename.startswith("bridges/") \ + and not filename.startswith("compilerplugins/") \ + and filename.find("/qa/") == -1 \ + and filename.find("/test/") == -1 \ + and not filename.startswith("testtools/") \ + and not filename.startswith("vcl/") \ + and not filename.startswith("cli_ure/"): + sourceFiles.add(filename.strip()) + +a = subprocess.Popen("git ls-files */*.mk", stdout=subprocess.PIPE, shell=True) +with a.stdout as txt: + for makefilename in txt: + makefilename = makefilename.strip() + with open(makefilename, "r") as makefile: + moduleName = makefilename[:makefilename.find("/")] + state = 0 + for line in makefile: + line = line.strip() + if state == 0 and "_add_exception_objects" in line: + state = 1 + elif state == 1 and line != "))": + s = line.replace("\\","").replace(")", "").strip() + # parse line like: $(call gb_Helper_optional,AVMEDIA,svx/source/sidebar/media/MediaPlaybackPanel) \ + idx = s.rfind(",") + if idx != -1: + s = s[idx+1:].strip() + sourceFiles.discard(s + ".cxx") + elif state == 1: + state = 0 + + + + +print "files not listed in makefile" +print "----------------------------" +for x in sorted(sourceFiles): + print x diff --git a/bin/find-german-comments b/bin/find-german-comments new file mode 100755 index 000000000..bb76941c1 --- /dev/null +++ b/bin/find-german-comments @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +######################################################################## +# +# Copyright (c) 2010 Jonas Jensen, Miklos Vajna +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +######################################################################## + + +import sys +import re +import subprocess +import os +import argparse +import string + +class Parser: + """ + This parser extracts comments from source files, tries to guess + their language and then prints out the German ones. + """ + def __init__(self): + self.strip = string.punctuation + " \n" + self.text_cat = self.start_text_cat() + parser = argparse.ArgumentParser(description='Searches for German comments in cxx/hxx source files inside a given root directory recursively.') + parser.add_argument("-f", "--filenames-only", action="store_true", + help="Only print the filenames of files containing German comments") + parser.add_argument("-v", "--verbose", action="store_true", + help="Turn on verbose mode (print only positives progress to stderr)") + parser.add_argument("-l", "--line-numbers", action="store_true", + help="Prints the filenames and line numbers only.") + parser.add_argument("-L", "--line-numbers-pos", action="store_true", + help="Prints the filenames and line numbers only (if positive).") + parser.add_argument("-t", "--threshold", action="store", default=0, type=int, + help="When used with '--line-numbers', only bothers outputting comment info if there are more than X number of flagged comments. Useful for weeding out false positives.") + parser.add_argument("directory", nargs='?', default='.', type=str, help='Give a directory to search in') + self.args = parser.parse_args() + self.check_source_files(self.args.directory) + + def get_comments(self, filename): + """ + Extracts the source code comments. + """ + linenum = 0 + if self.args.verbose: + print("processing file '%s'...\n" % filename) + sock = open(filename) + # add an empty line to trigger the output of collected oneliner + # comment group + lines = sock.readlines() + ["\n"] + sock.close() + + in_comment = False + buf = [] + count = 1 + for i in lines: + if "//" in i and not in_comment: + # if we find a new //-style comment, then we + # just append it to a previous one if: there is + # only whitespace before the // mark that is + # necessary to make comments longer, giving + # more reliable output + if not len(re.sub("(.*)//.*", r"\1", i).strip(self.strip)): + s = re.sub(".*// ?", "", i).strip(self.strip) + if len(s): + buf.append(s) + else: + # otherwise it's an independent //-style comment in the next line + yield (count, "\n ".join(buf)) + buf = [re.sub(".*// ?", "", i.strip(self.strip))] + elif "//" not in i and not in_comment and len(buf) > 0: + # first normal line after a // block + yield (count, "\n ".join(buf)) + buf = [] + elif "/*" in i and "*/" not in i and not in_comment: + # start of a real multiline comment + in_comment = True + linenum = count + s = re.sub(".*/\*+", "", i.strip(self.strip)) + if len(s): + buf.append(s.strip(self.strip)) + elif in_comment and not "*/" in i: + # in multiline comment + s = re.sub("^( |\|)*\*?", "", i) + if len(s.strip(self.strip)): + buf.append(s.strip(self.strip)) + elif "*/" in i and in_comment: + # end of multiline comment + in_comment = False + s = re.sub(r"\*+/.*", "", i.strip(self.strip)) + if len(s): + buf.append(s) + yield (count, "\n ".join(buf)) + buf = [] + elif "/*" in i and "*/" in i: + # c-style oneliner comment + yield (count, re.sub(".*/\*(.*)\*/.*", r"\1", i).strip(self.strip)) + count += 1 + + def start_text_cat(self): + cwd = os.getcwd() + # change to our directory + os.chdir(os.path.split(os.path.abspath(sys.argv[0]))[0]) + sock = subprocess.Popen(["text_cat/text_cat", "-s", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + os.chdir(cwd) + return sock + + def get_lang(self, s): + """ the output is 'german' or 'english' or 'german or english'. When + unsure, just don't warn, there are strings where you just can't + determine the results reliably, like '#110680#' """ + + self.text_cat.stdin.write(bytes(s, 'utf-8')) + self.text_cat.stdin.write(bytes("\n", 'utf-8')) + self.text_cat.stdin.flush() + lang = self.text_cat.stdout.readline().strip() + return lang + + def is_german(self, s): + """ + determines if a string is German or not + """ + # for short strings we can't do reliable recognition, so skip + # short strings and less than 4 words + s = s.replace('\n', ' ') + if len(s) < 32 or len(s.split()) < 4: + return False + return self.get_lang(s) == b"german" + + def check_file(self, path): + """ + checks each comment in a file + """ + def tab_calc(path): + START = 40 #Default of 10 tabs + if len(path) >= START: + return 1 + diff = START - len(path) + if diff % 4 is not 0: + padding = 1 + else: + padding = 0 + return (diff/4)+padding + + if self.args.line_numbers or self.args.line_numbers_pos: + TABS = "\t"*10 + path_linenums = [] + for linenum, s in self.get_comments(path): + if self.is_german(s): + path_linenums.append(linenum) + valid = len(path_linenums) > int(self.args.threshold) + if self.args.line_numbers: + print("%s ... %s positives -- %s\n" % (path, str(len(path_linenums)), str(valid))) + if valid: + if self.args.line_numbers_pos: + print("%s ... %s positives\n" % (path, str(len(path_linenums)))) + return + if len(path) + (len(path_linenums)*4) > 75: + print("%s:\n" % path) + while path_linenums: + i = 0 + numline = [] + while i < 10: + try: + numline.append(path_linenums[0]) + path_linenums.remove(path_linenums[0]) + except IndexError: + i = 10 + i += 1 + numline = [str(i) for i in numline] + print("%s%s" % (TABS, ",".join(numline))) + else: + if self.args.line_numbers: + path_linenums = [str(i) for i in path_linenums] + print("%s:%s%s" % (path, "\t"*int(tab_calc(path)), ",".join(path_linenums))) + + elif not self.args.filenames_only: + for linenum, s in self.get_comments(path): + if self.is_german(s): + print("%s:%s: %s" % (path, linenum, s)) + else: + fnames = set([]) + for linenum, s in self.get_comments(path): + if self.is_german(s): + # Make sure we print each filename only once + fnames.add(path) + # Print the filenames + for f in fnames: + print(f) + + def first_elem(self, path): + """ + Returns the root directory in our repo of a given path, so we can check against the whitelist. + """ + lastElem = os.path.dirname(path) + done = False + while not done: + nextElem = os.path.split(lastElem)[0] + if nextElem is not '': + lastElem = nextElem + else: + done = True + return lastElem + + def check_source_files(self, directory): + """ + checks each _tracked_ file in a directory recursively + """ + + # top-level project directory -> use whitelist. + globalscan = False + if os.path.exists(directory + "/.git/config"): + globalscan = True + + # Change into the given dir, so "git ls-tree" does work. + os.chdir(directory) + + sock = os.popen(r"git ls-tree -r HEAD --name-only |egrep '\.(c|cc|cpp|cxx|h|hxx|mm)$'") + lines = sock.readlines() + sock.close() + + # Helps to speedup a global scan + directory_whitelist = { + "ure" : 1, + "ios" : 1, + "bean" : 1, + "apple_remote" : 1, + "UnoControls" : 1, + "accessibility" : 1, + "android" : 1, + "animations" : 1, + "avmedia" : 1, + "basctl" : 1, + "basegfx" : 1, + "basic" : 1, + "binaryurp" : 1, + "bridges" : 1, + "canvas" : 1, + "chart2" : 1, + "cli_ure" : 1, + "codemaker" : 1, + "comphelper" : 1, + "compilerplugins" : 1, + "configmgr" : 1, + "connectivity" : 1, + "cppcanvas" : 1, + "cppu" : 1, + "cppuhelper" : 1, + "cpputools" : 1, + "cui" : 1, + "dbaccess" : 1, + "desktop" : 1, + "drawinglayer" : 1, + "dtrans" : 1, + "editeng" : 1, + "embeddedobj" : 1, + "embedserv" : 1, + "eventattacher" : 1, + "extensions" : 1, + "external" : 1, + "filter" : 1, + "forms" : 1, + "formula" : 1, + "fpicker" : 1, + "framework" : 1, + "helpcompiler" : 1, + "hwpfilter" : 1, + "i18npool" : 1, + "i18nlangtag" : 1, + "i18nutil" : 1, + "idl" : 1, + "idlc" : 1, + "include" : 1, + "io" : 1, + "javaunohelper" : 1, + "jvmaccess" : 1, + "jvmfwk" : 1, + "jurt" : 1, + "l10ntools" : 1, + "libreofficekit" : 1, + "lingucomponent" : 1, + "linguistic" : 1, + "lotuswordpro" : 1, + "mysqlc" : 1, + "o3tl" : 1, + "odk" : 1, + "officecfg" : 1, + "onlineupdate" : 1, + "opencl" : 1, + "oox" : 1, + "package" : 1, + "postprocess" : 1, + "pyuno" : 1, + "registry" : 1, + "remotebridges" : 1, + "reportdesign" : 1, + "rsc" : 1, + "sal" : 1, + "salhelper" : 1, + "sax" : 1, + "sc" : 1, + "scaddins" : 1, + "sccomp" : 1, + "scripting" : 1, + "sd" : 1, + "sdext" : 1, + "sfx2" : 1, + "shell" : 1, + "setup_native" : 1, + "sot" : 1, + "slideshow" : 1, + "smoketest" : 1, + "solenv" : 1, + "soltools" : 1, + "starmath" : 1, + "stoc" : 1, + "store" : 1, + "svgio" : 1, + "svl" : 1, + "svtools" : 1, + "svx" : 1, + "sw" : 1, + "test" : 1, + "testtools" : 1, + "toolkit" : 1, + "tools" : 1, + "touch" : 1, + "ucb" : 1, + "ucbhelper" : 1, + "unodevtools" : 1, + "unotest" : 1, + "unoidl" : 1, + "unotools" : 1, + "unoxml" : 1, + "uui" : 1, + "vbahelper" : 1, + "vcl" : 1, + "winaccessibility" : 1, + "writerfilter" : 1, + "writerperfect" : 1, + "xmlhelp" : 1, + "xmloff" : 1, + "xmlreader" : 1, + "xmlsecurity" : 1, + "xmlscript" : 1, + } + + if globalscan: + print("Scanning all files globally:") + elif directory == '.': + print("Scanning all files in our current directory:") + else: + print("Scanning all files in", directory + ":") + + num_checked = 0 + + for path in lines: + baseDir = self.first_elem(path) + # If we have a globalscan use the whitelist. + if globalscan: + if not baseDir in directory_whitelist: + sys.stderr.write("\n - Error: Missing path %s -\n\n" % baseDir) + sys.exit(1) + elif directory_whitelist[baseDir] is 0: + self.check_file(path.strip()) + num_checked = num_checked + 1 + elif directory_whitelist[baseDir] is 1: + sys.stderr.write("Skipping whitelisted directory %s\n" % baseDir) + directory_whitelist[baseDir] = 2 + elif not globalscan: + self.check_file(path.strip()) + num_checked = num_checked + 1 + + print("Scanned %s files\n" % num_checked) + +try: + Parser() +except KeyboardInterrupt: + print("Interrupted!") + sys.exit(0) + +# vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/find-headers-to-move-inside-modules.py b/bin/find-headers-to-move-inside-modules.py new file mode 100755 index 000000000..313e30762 --- /dev/null +++ b/bin/find-headers-to-move-inside-modules.py @@ -0,0 +1,52 @@ +#!/usr/bin/python2 + +# Look for headers inside include/ that can be moved into their respective modules. + +import subprocess +import sys + +headerSet = set() +a = subprocess.Popen("git ls-files include/", stdout=subprocess.PIPE, shell=True) +with a.stdout as txt: + for line in txt: + header = line[8:].strip(); + if "README" in header: continue + if header == "version.hrc": continue + if header == "svtools/editimplementation.hxx": continue + # ignore URE headers + if header.startswith("IwyuFilter_include.yaml"): continue + if header.startswith("cppu/"): continue + if header.startswith("cppuhelper/"): continue + if header.startswith("osl/"): continue + if header.startswith("sal/"): continue + if header.startswith("salhelper/"): continue + if header.startswith("uno/"): continue + # these are direct copies of mozilla code + if header.startswith("onlineupdate/mozilla/"): continue + headerSet.add(header) + +headerSetUnused = headerSet.copy() +headerSetOnlyInOwnModule = headerSet.copy() +a = subprocess.Popen("git grep '^#include <'", stdout=subprocess.PIPE, shell=True) +with a.stdout as txt: + for line in txt: + idx1 = line.find("#include <") + include = line[idx1 + 10 : len(line)-2] + headerSetUnused.discard(include) + # + idx1 = line.find("/") + includedFromModule = line[0 : idx1] + idx1 = include.find("/") + module = include[0 : idx1] + if module != includedFromModule: + headerSetOnlyInOwnModule.discard(include) + +print "completely unused" +print "----------------------------" +for x in sorted(headerSetUnused): + print x +print "" +print "only used in own module" +print "----------------------------" +for x in sorted(headerSetOnlyInOwnModule): + print x diff --git a/bin/find-mergedlib-can-be-private.classes.results b/bin/find-mergedlib-can-be-private.classes.results new file mode 100644 index 000000000..b75fe09fb --- /dev/null +++ b/bin/find-mergedlib-can-be-private.classes.results @@ -0,0 +1,426 @@ +Accelerator +B3dCamera +B3dTransformationSet +B3dViewport +BitmapMedianFilter +BitmapMonochromeMatrixFilter +BitmapPalette +BitmapPopArtFilter +BitmapSobelGreyFilter +CalendarField +CodeCompleteDataCache +ConvertChar +CurrencyBox +CurrencyFormatter +CursorWrapper +DateBox +DateField +DdeGetPutItem +DdeHotLink +DdeItem +DdeLink +DdeService +DdeTopic +DockingAreaWindow +DockingManager +DoubleCurrencyField +DoubleNumericField +E3dCompoundObject +E3dDefaultAttributes +E3dExtrudeObj +E3dPolygonObj +EditAbstractDialogFactory +EditUndo +EditUndoManager +EditViewCallbacks +EnhancedCustomShape +EnhancedCustomShape::FunctionParser +FileChangedChecker +FilterMatch +FixedBitmap +FixedHyperlink +FmDesignModeChangedHint +FmFormObj +FmFormPageImpl +FmXFormShell +FontSelectPattern +FontSizeNames +FontSubsetInfo +FormattedField::StaticFormatter +FormatterBase +FreetypeManager::IFSD_Equal +GroupBox +HelpLinker +Hunspell +Hunzip +ImageControl +ImplJobSetup +IndexerPreProcessor +IntroWindow +ListenerMultiplexerBase +LongCurrencyBox +LongCurrencyField +LongCurrencyFormatter +MenuToggleButton +MetaAction +MetaArcAction +MetaBmpAction +MetaBmpExScalePartAction +MetaBmpScaleAction +MetaBmpScalePartAction +MetaChordAction +MetaClipRegionAction +MetaEllipseAction +MetaFontAction +MetaGradientExAction +MetaISectRectClipRegionAction +MetaISectRegionClipRegionAction +MetaLayoutModeAction +MetaMapModeAction +MetaMoveClipRegionAction +MetaOverlineColorAction +MetaPieAction +MetaPixelAction +MetaPolyLineAction +MetaPolyPolygonAction +MetaPolygonAction +MetaPopAction +MetaPushAction +MetaRasterOpAction +MetaRefPointAction +MetaRoundRectAction +MetaTextAlignAction +MetaTextArrayAction +MetaTextColorAction +MetaTextFillColorAction +MetaTextLanguageAction +MetaTextLineColorAction +MetaWallpaperAction +MetafileAccessor +ModuleSizeExceeded +MoreButton +MultiListBox +MyThes +NativeNumberWrapper +NfCurrencyEntry +NotebookbarTabControlBase +NotifyEvent +NumericBox +NumericField +OFlowChainedText +OpenFileDropTargetListener +OpenGLFramebuffer +OpenGLZone +PackedTextureAtlasManager +PatternBox +PatternField +PatternFormatter +PhysicalFontFamily +PlaceEditDialog +PrinterOptions +ProgressBar +QueueInfo +RenderList +SalData +SalInfoPrinter +SalPrinter +SalSystem +SbClassModuleObject +SbMethod +SbxInfo +SbxObject +SdrEmbedObjectLink +SdrGrafBlueItem +SdrGrafContrastItem +SdrGrafCropItem +SdrGrafGamma100Item +SdrGrafGreenItem +SdrGrafLuminanceItem +SdrGrafModeItem +SdrGrafRedItem +SdrGrafTransparenceItem +SdrMeasureField +SdrMeasureObj +SdrSignedPercentItem +SdrTextFixedCellHeightItem +SdrUndoPageMasterPage +SelectionListenerMultiplexer +SfxAllEnumItem +SfxDocumentInfoItem +SfxItemSetHint +SfxMetricItem +SfxNavigator +SfxObjectItem +SfxStatusListener +SfxStyleSheetModifiedHint +SfxTemplatePanelControl +SfxViewFrameItem +SgaObject +SkiaPackedSurfaceAtlasManager +SkiaZone +SpinButton +SpinListenerMultiplexer +SvParser<HtmlTokenId>::TokenStackType +SvParser<int>::TokenStackType +SvtBasePrintOptions +SvtPrintFileOptions +SvtPrinterOptions +Svx3DCloseBackItem +Svx3DCloseFrontItem +Svx3DNormalsKindItem +Svx3DPerspectiveItem +Svx3DShadeModeItem +Svx3DTextureKindItem +Svx3DTextureModeItem +Svx3DTextureProjectionXItem +Svx3DTextureProjectionYItem +SvxCurrencyToolBoxControl +SvxEditSourceAdapter +SvxPasswordDialog +SvxPropertySetInfoPool +SvxTPage +SvxTextRotateItem +SyntaxHighlighter::Tokenizer +SystemWindow::ImplData +TETextDataObject +TabDialog +TabPaneValue +TextListenerMultiplexer +Throbber +TimeBox +TimeFormatter +UFlowChainedText +UnoEditControl +UnoWrapperBase +VCLXDateField +VCLXEdit +VCLXMenuBar +VCLXSpinField +ValueSet +VclAlignment +VclBin +VclBuilder::MenuAndId +VclBuilder::ParserState +VclBuilder::sortIntoBestTabTraversalOrder +VclDrawingArea +VclGrid +VclWindowEvent +XMLDashStyleExport +XMLDashStyleImport +XMLGradientStyleExport +XMLGradientStyleImport +XMLHatchStyleExport +XMLHatchStyleImport +XMLImageStyle +XMLMarkerStyleExport +XMLMarkerStyleImport +XMLShapeStyleContext +accessibility::AccessibleEditableTextPara +accessibility::AccessibleParaManager +avmedia::MediaControlBase +avmedia::MediaFloater +basegfx::B2DTrapezoid +basegfx::B3DPoint +basegfx::B3DTuple +basegfx::BColorModifier +basegfx::BColorModifierStack +basegfx::BColorModifier_RGBLuminanceContrast +basegfx::BColorModifier_black_and_white +basegfx::BColorModifier_gamma +basegfx::BColorModifier_gray +basegfx::BColorModifier_invert +basegfx::BColorModifier_replace +basegfx::MinimalSystemDependentDataManager +basegfx::ODFGradientInfo +basegfx::RasterConverter3D +basegfx::SystemDependentDataHolder +basegfx::SystemDependentDataManager +basegfx::triangulator +canvas +char& std::vector<char, std::allocator<char> > +comphelper::IndexAccessIterator +comphelper::OAccessibleSelectionHelper +comphelper::OEventListenerHelper +comphelper::OPropertySetAggregationHelper +comphelper::OPropertyStateHelper +comphelper::OSequenceOutputStream +comphelper::OStatefulPropertySet +comphelper::OStreamSection +comphelper::OWeakEventListenerAdapter +comphelper::OWrappedAccessibleChildrenManager +comphelper::PropertyBag +comphelper::StillReadWriteInteraction +comphelper::service_decl::ServiceDecl::Factory +connectivity::sdbcx::IObjectCollection +connectivity::sdbcx::OGroup +connectivity::sdbcx::OKey +dbtools::param::ParameterWrapper +desktop::CallbackFlushHandler::CallbackData +dp_misc::AbortChannel +drawinglayer::animation::AnimationEntry +drawinglayer::animation::AnimationEntryFixed +drawinglayer::animation::AnimationEntryLinear +drawinglayer::animation::AnimationEntryList +drawinglayer::animation::AnimationEntryLoop +drawinglayer::attribute::FillGraphicAttribute +drawinglayer::attribute::FillHatchAttribute +drawinglayer::attribute::LineStartEndAttribute +drawinglayer::attribute::MaterialAttribute3D +drawinglayer::attribute::Sdr3DLightAttribute +drawinglayer::attribute::Sdr3DObjectAttribute +drawinglayer::attribute::SdrFillGraphicAttribute +drawinglayer::attribute::SdrGlowAttribute +drawinglayer::attribute::SdrLightingAttribute +drawinglayer::attribute::SdrLineAttribute +drawinglayer::attribute::SdrLineFillShadowAttribute3D +drawinglayer::attribute::SdrLineStartEndAttribute +drawinglayer::attribute::SdrSceneAttribute +drawinglayer::attribute::SdrShadowAttribute +drawinglayer::primitive2d::AnimatedBlinkPrimitive2D +drawinglayer::primitive2d::AnimatedInterpolatePrimitive2D +drawinglayer::primitive2d::AnimatedSwitchPrimitive2D +drawinglayer::primitive2d::BackgroundColorPrimitive2D +drawinglayer::primitive2d::ControlPrimitive2D +drawinglayer::primitive2d::DiscreteShadowPrimitive2D +drawinglayer::primitive2d::Embedded3DPrimitive2D +drawinglayer::primitive2d::FillGraphicPrimitive2D +drawinglayer::primitive2d::GlowPrimitive2D +drawinglayer::primitive2d::GridPrimitive2D +drawinglayer::primitive2d::GroupPrimitive2D +drawinglayer::primitive2d::HelplinePrimitive2D +drawinglayer::primitive2d::InvertPrimitive2D +drawinglayer::primitive2d::MarkerArrayPrimitive2D +drawinglayer::primitive2d::MediaPrimitive2D +drawinglayer::primitive2d::MetafilePrimitive2D +drawinglayer::primitive2d::ObjectAndViewTransformationDependentPrimitive2D +drawinglayer::primitive2d::PagePreviewPrimitive2D +drawinglayer::primitive2d::PolyPolygonGradientPrimitive2D +drawinglayer::primitive2d::PolyPolygonGraphicPrimitive2D +drawinglayer::primitive2d::PolyPolygonHatchPrimitive2D +drawinglayer::primitive2d::PolyPolygonSelectionPrimitive2D +drawinglayer::primitive2d::PolygonMarkerPrimitive2D +drawinglayer::primitive2d::PolygonStrokeArrowPrimitive2D +drawinglayer::primitive2d::ScenePrimitive2D +drawinglayer::primitive2d::SdrFrameBorderData::SdrConnectStyleData +drawinglayer::primitive2d::ShadowPrimitive2D +drawinglayer::primitive2d::TextHierarchyBlockPrimitive2D +drawinglayer::primitive2d::TextHierarchyBulletPrimitive2D +drawinglayer::primitive2d::TextHierarchyEditPrimitive2D +drawinglayer::primitive2d::TextHierarchyFieldPrimitive2D +drawinglayer::primitive2d::TextHierarchyLinePrimitive2D +drawinglayer::primitive2d::TextHierarchyParagraphPrimitive2D +drawinglayer::primitive2d::ViewTransformationDependentPrimitive2D +drawinglayer::primitive2d::ViewportDependentPrimitive2D +drawinglayer::primitive2d::WrongSpellPrimitive2D +drawinglayer::primitive3d +drawinglayer::primitive3d::BasePrimitive3D +drawinglayer::primitive3d::BufferedDecompositionPrimitive3D +drawinglayer::primitive3d::GroupPrimitive3D +drawinglayer::primitive3d::ModifiedColorPrimitive3D +drawinglayer::primitive3d::PolyPolygonMaterialPrimitive3D +drawinglayer::primitive3d::PolygonHairlinePrimitive3D +drawinglayer::primitive3d::Primitive3DContainer +drawinglayer::primitive3d::SdrCubePrimitive3D +drawinglayer::primitive3d::SdrExtrudePrimitive3D +drawinglayer::primitive3d::SdrLathePrimitive3D +drawinglayer::primitive3d::SdrPolyPolygonPrimitive3D +drawinglayer::primitive3d::SdrPrimitive3D +drawinglayer::primitive3d::SdrSpherePrimitive3D +drawinglayer::primitive3d::TransformPrimitive3D +drawinglayer::processor2d::HitTestProcessor2D +drawinglayer::processor3d::BaseProcessor3D +drawinglayer::processor3d::CutFindProcessor +emfio::WinMtfFontStyle +formula::FormulaTokenIterator::Item +framework +framework::AddonMenuManager +framework::AddonsOptions +framework::ConfigAccess +framework::ConstItemContainer +framework::Converter +framework::DispatchHelper +framework::FrameListAnalyzer +framework::HandlerCache +framework::InteractionRequest +framework::MenuAttributes +framework::MenuConfiguration +framework::RequestFilterSelect +framework::RootItemContainer +framework::SaxNamespaceFilter +framework::StatusBarConfiguration +framework::ToolBoxConfiguration +framework::TransactionManager +framework::UIConfigurationImporterOOo1x +legacy::CntInt32 +legacy::SfxBool +legacy::SvxAdjust +legacy::SvxBox +legacy::SvxBrush +legacy::SvxColor +legacy::SvxCrossedOut +legacy::SvxFont +legacy::SvxFontHeight +legacy::SvxHorJustify +legacy::SvxLine +legacy::SvxPosture +legacy::SvxTextLine +legacy::SvxVerJustify +legacy::SvxWeight +psp::PrintFontManager::PrintFont +sdr::SelectionController +sdr::ViewSelection +sdr::animation::primitiveAnimator +sdr::contact::ObjectContactPainter +sdr::properties::BaseProperties +sdr::table::Cell +sfx2::sidebar::Panel +sfx2::sidebar::SidebarToolBox +sfx2::sidebar::TabBar +sfx2::sidebar::TabBar::Item +svt +svt::AddressBookSourceDialog +svt::GenericToolboxController +svt::GraphicAccess +svt::IEditImplementation +svt::MultiLineEditImplementation +svt::MultiLineTextCell +svt::OStringTransferable +svt::PopupMenuControllerBase +svt::SpinCellController +svt::TemplateFolderCache +svtools::AsynchronLink +svtools::ToolbarPopup +svx::DialControl::DialControl_Impl +svx::IPropertyValueProvider +svx::sidebar::GalleryControl +svxform +svxform::DataNavigatorManager +svxform::NavigatorFrameManager +svxform::OLocalExchange +svxform::OLocalExchangeHelper +svxform::OSQLParserClient +toolkitform +ucbhelper::ActiveDataSink +ucbhelper::InteractionApprove +ucbhelper::InteractionDisapprove +ucbhelper::InteractionSupplyAuthentication +ucbhelper::InterceptedInteraction +ucbhelper::SimpleNameClashResolveRequest +utl::Bootstrap::Impl +utl::DefaultFontConfiguration +utl::DesktopTerminationObserver +utl::FontSubstConfiguration +utl::OConfigurationValueContainer +utl::ProgressHandlerWrap +utl::ZipPackageHelper +utl::detail::Options +vcl::AccessibleFactoryAccess +vcl::EventPoster +vcl::ExtOutDevData +vcl::ILibreOfficeKitNotifier +vcl::ORoadmap +vcl::OldStylePrintAdaptor +vcl::PDFWriter::AnyWidget +vcl::test::OutputDeviceTestGradient +void OpenGLTexture +wchar_t& std::vector<wchar_t, std::allocator<wchar_t> > diff --git a/bin/find-mergedlib-can-be-private.py b/bin/find-mergedlib-can-be-private.py new file mode 100755 index 000000000..ac9d96712 --- /dev/null +++ b/bin/find-mergedlib-can-be-private.py @@ -0,0 +1,154 @@ +#!/usr/bin/python2 +# +# Generate a custom linker script/map file for the --enabled-mergedlibs merged library +# which reduces the startup time and enables further optimisations with --enable-lto because 60% or more +# of the symbols become internal only. +# + +import subprocess +import sys +import re +import multiprocessing + +exported_symbols = set() +imported_symbols = set() + + +# Copied from solenv/gbuild/extensions/pre_MergedLibsList.mk +# TODO there has to be a way to run gmake and get it to dump this list for me +merged_libs = { \ + "avmedia" \ + ,"basctl" \ + ,"basprov" \ + ,"basegfx" \ + ,"canvasfactory" \ + ,"canvastools" \ + ,"comphelper" \ + ,"configmgr" \ + ,"cppcanvas" \ + ,"crashreport)" \ + ,"dbtools" \ + ,"deployment" \ + ,"deploymentmisc" \ + ,"desktopbe1)" \ + ,"desktop_detector)" \ + ,"drawinglayer" \ + ,"editeng" \ + ,"expwrap" \ + ,"filterconfig" \ + ,"fsstorage" \ + ,"fwe" \ + ,"fwi" \ + ,"fwk" \ + ,"helplinker)" \ + ,"i18npool" \ + ,"i18nutil" \ + ,"lng" \ + ,"localebe1" \ + ,"mcnttype" \ + ,"msfilter" \ + ,"mtfrenderer" \ + ,"opencl" \ + ,"package2" \ + ,"sax" \ + ,"sb" \ + ,"simplecanvas" \ + ,"sfx" \ + ,"sofficeapp" \ + ,"sot" \ + ,"spl" \ + ,"stringresource" \ + ,"svl" \ + ,"svt" \ + ,"svx" \ + ,"svxcore" \ + ,"tk" \ + ,"tl" \ + ,"ucb1" \ + ,"ucbhelper" \ + ,"ucpexpand1" \ + ,"ucpfile1" \ + ,"unoxml" \ + ,"utl" \ + ,"uui" \ + ,"vcl" \ + ,"xmlscript" \ + ,"xo" \ + ,"xstor" } + +# look for symbols exported by libmerged +subprocess_nm = subprocess.Popen("nm -D instdir/program/libmergedlo.so", stdout=subprocess.PIPE, shell=True) +with subprocess_nm.stdout as txt: + # We are looking for lines something like: + # 0000000000036ed0 T flash_component_getFactory + line_regex = re.compile(r'^[0-9a-fA-F]+ T ') + for line in txt: + line = line.strip() + if line_regex.match(line): + exported_symbols.add(line.split(" ")[2]) +subprocess_nm.terminate() + +# look for symbols imported from libmerged +subprocess_find = subprocess.Popen("(find instdir/program/ -type f; ls ./workdir/LinkTarget/CppunitTest/*.so) | xargs grep -l mergedlo", + stdout=subprocess.PIPE, shell=True) +with subprocess_find.stdout as txt: + for line in txt: + sharedlib = line.strip() + s = sharedlib[sharedlib.find("/lib") + 4 : len(sharedlib) - 3] + if s in merged_libs: continue + # look for imported symbols + subprocess_objdump = subprocess.Popen("objdump -T " + sharedlib, stdout=subprocess.PIPE, shell=True) + with subprocess_objdump.stdout as txt2: + # ignore some header bumpf + txt2.readline() + txt2.readline() + txt2.readline() + txt2.readline() + # We are looking for lines something like (noting that one of them uses spaces, and the other tabs) + # 0000000000000000 DF *UND* 0000000000000000 _ZN16FilterConfigItem10WriteInt32ERKN3rtl8OUStringEi + for line2 in txt2: + line2 = line2.strip() + if line2.find("*UND*") == -1: continue + tokens = line2.split(" ") + sym = tokens[len(tokens)-1].strip() + imported_symbols.add(sym) + subprocess_objdump.terminate() +subprocess_find.terminate() + +intersec_symbols = exported_symbols.intersection(imported_symbols) +print("no symbols exported from libmerged = " + str(len(exported_symbols))) +print("no symbols that can be made internal = " + str(len(intersec_symbols))) + +# Now look for classes where none of the class symbols are imported, +# i.e. we can mark the whole class as hidden + +def extract_class(sym): + filtered_sym = subprocess.check_output(["c++filt", sym]).strip() + if filtered_sym.startswith("vtable for "): + classname = filtered_sym[11:] + return classname + if filtered_sym.startswith("non-virtual thunk to "): + filtered_sym = filtered_sym[21:] + elif filtered_sym.startswith("virtual thunk to "): + filtered_sym = filtered_sym[17:] + i = filtered_sym.find("(") + if i != -1: + i = filtered_sym.rfind("::", 0, i) + if i != -1: + classname = filtered_sym[:i] + return classname + return "" + +pool = multiprocessing.Pool(multiprocessing.cpu_count()) +classes_with_exported_symbols = set(pool.map(extract_class, list(exported_symbols))) +classes_with_imported_symbols = set(pool.map(extract_class, list(imported_symbols))) + +# Some stuff is particular to Windows, so won't be found by a Linux analysis, so remove +# those classes. +can_be_private_classes = classes_with_exported_symbols - classes_with_imported_symbols; +can_be_private_classes.discard("SpinField") + +with open("bin/find-mergedlib-can-be-private.classes.results", "wt") as f: + for sym in sorted(can_be_private_classes): + if sym.startswith("std::") or sym.startswith("void std::"): continue + f.write(sym + "\n") diff --git a/bin/find-most-common-warn-messages.py b/bin/find-most-common-warn-messages.py new file mode 100755 index 000000000..dc2ecf8ab --- /dev/null +++ b/bin/find-most-common-warn-messages.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 + +# A script to search our test logs and sort the messages by how common they are so we can start to +# reduce the noise a little. + +import sys +import re +import io +import subprocess + +# find . -name '*.log' | xargs grep -h 'warn:' | sort | uniq -c | sort -n --field-separator=: --key=5,6 + +process = subprocess.Popen("find workdir -name '*.log' | xargs grep -h 'warn:' | sort", + shell=True, stdout=subprocess.PIPE, universal_newlines=True) + +messages = dict() # dict of sourceAndLine->count +sampleOfMessage = dict() # dict of sourceAndLine->string +for line in process.stdout: + line = line.strip() + # a sample line is: + # warn:sw:18790:1:sw/source/core/doc/DocumentRedlineManager.cxx:98: redline table corrupted: overlapping redlines + tokens = line.split(":") + sourceAndLine = tokens[4] + ":" + tokens[5] + if (sourceAndLine in messages): + messages[sourceAndLine] = messages[sourceAndLine] + 1 + else: + messages[sourceAndLine] = 1 + sampleOfMessage[sourceAndLine] = line[line.find(tokens[6]):] + +tmplist = list() # set of tuple (count, sourceAndLine) +for key, value in messages.items(): + tmplist.append([value,key]) + +print( "The top 20 warnings" ) +print("") +for i in sorted(tmplist, key=lambda v: v[0])[-20:]: + print( "%6d %s %s" % (i[0], i[1], sampleOfMessage[i[1]]) ) + + diff --git a/bin/find-most-repeated-functions.py b/bin/find-most-repeated-functions.py new file mode 100755 index 000000000..767f80240 --- /dev/null +++ b/bin/find-most-repeated-functions.py @@ -0,0 +1,42 @@ +#!/usr/bin/python +# +# Find the top 100 functions that are repeated in multiple .o files, so we can out-of-line those +# +# + +import subprocess +from collections import defaultdict + +# the odd bash construction here is because some of the .o files returned by find are not object files +# and I don't want xargs to stop when it hits an error +a = subprocess.Popen("find instdir/program/ -name *.so | xargs echo nm --radix=d --size-sort --demangle | bash", stdout=subprocess.PIPE, shell=True) + +#xargs sh -c "somecommand || true" + +nameDict = defaultdict(int) +with a.stdout as txt: + for line in txt: + line = line.strip() + idx1 = line.find(" ") + idx2 = line.find(" ", idx1 + 1) + name = line[idx2:] + nameDict[name] += 1 + +sizeDict = defaultdict(set) +for k, v in nameDict.iteritems(): + sizeDict[v].add(k) + +cnt = 0 +for k in sorted(list(sizeDict), reverse=True): + print k + for v in sizeDict[k]: + print v + cnt += 1 + if cnt > 100 : break + +#first = sorted(list(sizeDict))[-1] +#print first + + +#include/vcl/ITiledRenderable.hxx +# why is gaLOKPointerMap declared inside this header? diff --git a/bin/find-undocumented-classes b/bin/find-undocumented-classes new file mode 100755 index 000000000..8bab72bc9 --- /dev/null +++ b/bin/find-undocumented-classes @@ -0,0 +1,33 @@ +#!/bin/bash + +# finds undocumented classes in the current directory (recursive) + +type -p doxygen >/dev/null || exit + +filter= +quiet=n +if [ "$1" = "-q" ]; then + filter=">/dev/null" + quiet=y + shift +fi + +doxygen=$(mktemp -d) +eval doxygen -g $doxygen/doxygen.cfg $filter +sed -i "/HTML_OUTPUT/s|html|$doxygen/html|" $doxygen/doxygen.cfg +sed -i '/GENERATE_LATEX/s/= YES/= NO/' $doxygen/doxygen.cfg +sed -i '/RECURSIVE/s/= NO/= YES/' $doxygen/doxygen.cfg +# do we have any arguments? +if [ -n "$*" ]; then + sed -i "/^INPUT[^_]/s|=.*|= $*|" $doxygen/doxygen.cfg +fi +eval doxygen $doxygen/doxygen.cfg $filter 2> $doxygen/errors.txt +if [ "$quiet" == "n" ]; then + echo + echo "The following classes are undocumented:" + echo +fi +cat $doxygen/errors.txt|grep -i 'Warning: Compound.*is not documented' +rm -rf $doxygen + +# vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/find-unneeded-includes b/bin/find-unneeded-includes new file mode 100755 index 000000000..fcbabad87 --- /dev/null +++ b/bin/find-unneeded-includes @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This parses the output of 'include-what-you-use', focusing on just removing +# not needed includes and providing a relatively conservative output by +# filtering out a number of LibreOffice-specific false positives. +# +# It assumes you have a 'compile_commands.json' around (similar to clang-tidy), +# you can generate one with 'make vim-ide-integration'. +# +# Design goals: +# - blacklist mechanism, so a warning is either fixed or blacklisted +# - works in a plugins-enabled clang build +# - no custom configure options required +# - no need to generate a dummy library to build a header + +import glob +import json +import multiprocessing +import os +import queue +import re +import subprocess +import sys +import threading +import yaml + + +def ignoreRemoval(include, toAdd, absFileName, moduleRules): + # global rules + + # Avoid replacing .hpp with .hdl in the com::sun::star and ooo::vba namespaces. + if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"): + hdl = include.replace(".hpp", ".hdl") + if hdl in toAdd: + return True + + # Avoid debug STL. + debugStl = { + "array": ("debug/array", ), + "bitset": ("debug/bitset", ), + "deque": ("debug/deque", ), + "forward_list": ("debug/forward_list", ), + "list": ("debug/list", ), + "map": ("debug/map.h", "debug/multimap.h"), + "set": ("debug/set.h", "debug/multiset.h"), + "unordered_map": ("debug/unordered_map", ), + "unordered_set": ("debug/unordered_set", ), + "vector": ("debug/vector", ), + } + for k, values in debugStl.items(): + if include == k: + for value in values: + if value in toAdd: + return True + + # Avoid proposing to use libstdc++ internal headers. + bits = { + "exception": "bits/exception.h", + "memory": "bits/shared_ptr.h", + "functional": "bits/std_function.h", + "cmath": "bits/std_abs.h", + "ctime": "bits/types/clock_t.h", + "cstdint": "bits/stdint-uintn.h", + } + for k, v in bits.items(): + if include == k and v in toAdd: + return True + + # Avoid proposing o3tl fw declaration + o3tl = { + "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }", + "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }", + "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }", + } + for k, v, in o3tl.items(): + if include == k and v in toAdd: + return True + + # Follow boost documentation. + if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd: + return True + if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd: + return True + if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd: + return True + if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd: + return True + if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd: + return True + + # Avoid .hxx to .h proposals in basic css/uno/* API + unoapi = { + "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h", + "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h", + "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h", + "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h" + } + for k, v in unoapi.items(): + if include == k and v in toAdd: + return True + + # 3rd-party, non-self-contained headers. + if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd: + return True + if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd: + return True + + noRemove = ( + # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not + # removing this. + "sal/config.h", + # Works around a build breakage specific to the broken Android + # toolchain. + "android/compatibility.hxx", + ) + if include in noRemove: + return True + + # Ignore when <foo> is to be replaced with "foo". + if include in toAdd: + return True + + fileName = os.path.relpath(absFileName, os.getcwd()) + + # Skip headers used only for compile test + if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx": + if include.endswith(".hpp"): + return True + + # yaml rules + + if "blacklist" in moduleRules.keys(): + blacklistRules = moduleRules["blacklist"] + if fileName in blacklistRules.keys(): + if include in blacklistRules[fileName]: + return True + + return False + + +def unwrapInclude(include): + # Drop <> or "" around the include. + return include[1:-1] + + +def processIWYUOutput(iwyuOutput, moduleRules, fileName): + inAdd = False + toAdd = [] + inRemove = False + toRemove = [] + currentFileName = None + + for line in iwyuOutput: + line = line.strip() + + # Bail out if IWYU gave an error due to non self-containedness + if re.match ("(.*): error: (.*)", line): + return -1 + + if len(line) == 0: + if inRemove: + inRemove = False + continue + if inAdd: + inAdd = False + continue + + shouldAdd = fileName + " should add these lines:" + match = re.match(shouldAdd, line) + if match: + currentFileName = match.group(0).split(' ')[0] + inAdd = True + continue + + shouldRemove = fileName + " should remove these lines:" + match = re.match(shouldRemove, line) + if match: + currentFileName = match.group(0).split(' ')[0] + inRemove = True + continue + + if inAdd: + match = re.match('#include ([^ ]+)', line) + if match: + include = unwrapInclude(match.group(1)) + toAdd.append(include) + else: + # Forward declaration. + toAdd.append(line) + + if inRemove: + match = re.match("- #include (.*) // lines (.*)-.*", line) + if match: + # Only suggest removals for now. Removing fwd decls is more complex: they may be + # indeed unused or they may removed to be replaced with an include. And we want to + # avoid the later. + include = unwrapInclude(match.group(1)) + lineno = match.group(2) + if not ignoreRemoval(include, toAdd, currentFileName, moduleRules): + toRemove.append("%s:%s: %s" % (currentFileName, lineno, include)) + + for remove in sorted(toRemove): + print("ERROR: %s: remove not needed include" % remove) + return len(toRemove) + + +def run_tool(task_queue, failed_files): + while True: + invocation, moduleRules = task_queue.get() + if not len(failed_files): + print("[IWYU] " + invocation.split(' ')[-1]) + p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1]) + if retcode == -1: + print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation) + elif retcode > 0: + print("ERROR: The following command found unused includes:\n" + invocation) + failed_files.append(invocation) + task_queue.task_done() + + +def isInUnoIncludeFile(path): + return path.startswith("include/com/") \ + or path.startswith("include/cppu/") \ + or path.startswith("include/cppuhelper/") \ + or path.startswith("include/osl/") \ + or path.startswith("include/rtl/") \ + or path.startswith("include/sal/") \ + or path.startswith("include/salhelper/") \ + or path.startswith("include/systools/") \ + or path.startswith("include/typelib/") \ + or path.startswith("include/uno/") + + +def tidy(compileCommands, paths): + return_code = 0 + try: + max_task = multiprocessing.cpu_count() + task_queue = queue.Queue(max_task) + failed_files = [] + for _ in range(max_task): + t = threading.Thread(target=run_tool, args=(task_queue, failed_files)) + t.daemon = True + t.start() + + for path in sorted(paths): + if isInUnoIncludeFile(path): + continue + + moduleName = path.split("/")[0] + + rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml") + moduleRules = {} + if os.path.exists(rulePath): + moduleRules = yaml.load(open(rulePath)) + assume = None + pathAbs = os.path.abspath(path) + compileFile = pathAbs + matches = [i for i in compileCommands if i["file"] == compileFile] + if not len(matches): + if "assumeFilename" in moduleRules.keys(): + assume = moduleRules["assumeFilename"] + if assume: + assumeAbs = os.path.abspath(assume) + compileFile = assumeAbs + matches = [i for i in compileCommands if i["file"] == compileFile] + if not len(matches): + print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'") + continue + else: + print("WARNING: no compile commands for '" + path + "'") + continue + + _, _, args = matches[0]["command"].partition(" ") + if assume: + args = args.replace(assumeAbs, "-x c++ " + pathAbs) + + invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args + task_queue.put((invocation, moduleRules)) + + task_queue.join() + if len(failed_files): + return_code = 1 + + except KeyboardInterrupt: + print('\nCtrl-C detected, goodbye.') + os.kill(0, 9) + + sys.exit(return_code) + + +def main(argv): + if not len(argv): + print("usage: find-unneeded-includes [FILE]...") + return + + try: + with open("compile_commands.json", 'r') as compileCommandsSock: + compileCommands = json.load(compileCommandsSock) + except FileNotFoundError: + print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration") + sys.exit(-1) + + tidy(compileCommands, paths=argv) + +if __name__ == '__main__': + main(sys.argv[1:]) + +# vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/find-unused-defines.py b/bin/find-unused-defines.py new file mode 100755 index 000000000..8e708c4a0 --- /dev/null +++ b/bin/find-unused-defines.py @@ -0,0 +1,170 @@ +#!/usr/bin/python2 + +# Search for unused constants in header files. +# +# Note that sometimes these constants are calculated, so some careful checking of the output is necessary. +# +# Takes about 4 hours to run this on a fast machine with an SSD +# + +import subprocess +import sys +import re + +exclusionSet = set([ + # List of RID constants where we compute a value using a base before calling one of the RESSTR methods + # Found with: git grep -P 'RID_\w+\s*\+' -- :/ ':!*.hrc' ':!*.src' ':!*.java' ':!*.py' ':!*.xba' + "RID_SVXSTR_KEY_", + "RID_UPDATE_BUBBLE_TEXT_", + "RID_UPDATE_BUBBLE_T_TEXT_", + "RID_SVXSTR_TBLAFMT_", + "RID_BMP_CONTENT_", + "RID_DROPMODE_", + "RID_BMP_LEVEL", + "RID_SVXSTR_BULLET_DESCRIPTION", + "RID_SVXSTR_SINGLENUM_DESCRIPTION", + "RID_SVXSTR_OUTLINENUM_DESCRIPTION", + "RID_SVXSTR_RULER_", + "RID_GALLERYSTR_THEME_", + "RID_SVXSTR_BULLET_DESCRIPTION", + "RID_SVXSTR_SINGLENUM_DESCRIPTION", + "RID_SVXSTR_OUTLINENUM_DESCRIPTION", + # doing some weird stuff in svx/source/unodraw/unoprov.cxx involving mapping of UNO api names to translated names and back again + "RID_SVXSTR_GRDT", + "RID_SVXSTR_HATCH", + "RID_SVXSTR_BMP", + "RID_SVXSTR_DASH", + "RID_SVXSTR_LEND", + "RID_SVXSTR_TRASNGR", + # other places doing calculations + "RID_SVXSTR_DEPTH", + "RID_SUBSETSTR_", + "ANALYSIS_", + "FLD_DOCINFO_CHANGE", + "FLD_EU_", + "FLD_INPUT_", + "FLD_PAGEREF_", + "FLD_STAT_", + "FMT_AUTHOR_", + "FMT_CHAPTER_", + "FMT_DBFLD_", + "FMT_FF_", + "FMT_GETVAR_", + "FMT_MARK_", + "FMT_REF_", + "FMT_SETVAR_", + "STR_AUTH_FIELD_ADDRESS_", + "STR_AUTH_TYPE_", + "STR_AUTOFMTREDL_", + "STR_CONTENT_TYPE_", + "STR_UPDATE_ALL", + "STR_UPDATE_INDEX", + "STR_UPDATE_LINK", + "BMP_PLACEHOLDER_", + "STR_RPT_HELP_", + "STR_TEMPLATE_NAME", + "UID_BRWEVT_", + "HID_EVT_", + "HID_PROP_", + "STR_VOBJ_MODE_", + "STR_COND_", + "SCSTR_CONTENT_", + "DATE_FUNCDESC_", + "DATE_FUNCNAME_", + "DATE_DEFFUNCNAME_", + "PRICING_DEFFUNCNAME_", + "PRICING_FUNCDESC_", + "PRICING_FUNCNAME_", + "STR_ItemValCAPTION", + "STR_ItemValCIRC", + "STR_ItemValEDGE", + "STR_ItemValFITTOSIZE", + "STR_ItemValMEASURE_", + "STR_ItemValMEASURETEXT_", + "STR_ItemValTEXTANI_", + "STR_ItemValTEXTHADJ", + "STR_ItemValTEXTVADJ", + "RID_SVXITEMS_VERJUST", + "RID_SVXITEMS_ORI", + "RID_SVXITEMS_JUSTMETHOD", + "RID_SVXITEMS_HORJUST", + "MM_PART", + ]) + + +def in_exclusion_set( a ): + for f in exclusionSet: + if a.startswith(f): + return True; + return False; + +# find defines, excluding the externals folder +a = subprocess.Popen("git grep -hP '^#define\s+\w\w\w\w+\s*' -- \"[!e][!x][!t]*\" | sort -u", stdout=subprocess.PIPE, shell=True) + +name_re = re.compile("#define\s+(\w+)") +with a.stdout as txt: + for line in txt: + idName = name_re.match(line).group(1) + if idName.startswith("INCLUDED_"): continue + # the various _START and _END constants are normally unused outside of the .hrc and .src files, and that's fine + if idName.endswith("_START"): continue + if idName.endswith("_BEGIN"): continue + if idName.endswith("_END"): continue + if idName == "RID_SVX_FIRSTFREE": continue + if idName == "": continue + if idName.startswith("__com"): continue # these are the include/header macros for the UNO stuff + if in_exclusion_set(idName): continue + # search for the constant + b = subprocess.Popen(["git", "grep", "-w", idName], stdout=subprocess.PIPE) + found_reason_to_exclude = False + with b.stdout as txt2: + cnt = 0 + for line2 in txt2: + line2 = line2.strip() # otherwise the comparisons below will not work + # ignore if/undef magic, does not indicate an actual use (most of the time) + if "ifdef" in line2: continue + if "undef" in line2: continue + # ignore commented out code + if line2.startswith("//"): continue + if line2.startswith("/*"): continue + # check if we found one in actual code + if idName.startswith("SID_"): + if not ".hrc:" in line2 and not ".src:" in line2 and not ".sdi:" in line2: found_reason_to_exclude = True + else: + if not ".hrc:" in line2 and not ".src:" in line2: found_reason_to_exclude = True + if idName.startswith("RID_"): + # is the constant being used as an identifier by entries in .src files? + if ".src:" in line2 and "Identifier = " in line2: found_reason_to_exclude = True + # is the constant being used by the property controller extension or reportdesigner inspection, + # which use macros to declare constants, hiding them from a search + if "extensions/source/propctrlr" in line2: found_reason_to_exclude = True + if "reportdesign/source/ui/inspection/inspection.src" in line2: found_reason_to_exclude = True + if idName.startswith("HID_"): + # is the constant being used as an identifier by entries in .src files + if ".src:" in line2 and "HelpId = " in line2: found_reason_to_exclude = True + # is it being used as a constant in an ItemList in .src files? + if ".src:" in line2 and (";> ;" in line2 or "; >;" in line2): found_reason_to_exclude = True + # these are used in calculations in other .hrc files + if "sw/inc/rcid.hrc:" in line2: found_reason_to_exclude = True + # calculations + if "sw/source/uibase/inc/ribbar.hrc:" in line2 and "ST_" in idName: found_reason_to_exclude = True + if "sw/source/uibase/inc/ribbar.hrc:" in line2 and "STR_IMGBTN_" in idName: found_reason_to_exclude = True + if "sw/source/core/undo/undo.hrc:" in line2: found_reason_to_exclude = True + if "sw/inc/poolfmt.hrc:" in line2: found_reason_to_exclude = True + # used via a macro that hides them from search + if "dbaccess/" in line2 and idName.startswith("PROPERTY_ID_"): found_reason_to_exclude = True + if "reportdesign/" in line2 and idName.startswith("HID_RPT_PROP_"): found_reason_to_exclude = True + if "reportdesign/" in line2 and idName.startswith("RID_STR_"): found_reason_to_exclude = True + if "forms/" in line2 and idName.startswith("PROPERTY_"): found_reason_to_exclude = True + if "svx/source/tbxctrls/extrusioncontrols.hrc:" in line2 and idName.startswith("DIRECTION_"): found_reason_to_exclude = True + if "svx/source/tbxctrls/extrusioncontrols.hrc:" in line2 and idName.startswith("FROM_"): found_reason_to_exclude = True + # if we see more than a few lines then it's probably one of the BASE/START/BEGIN things + cnt = cnt + 1 + if cnt > 2: found_reason_to_exclude = True + if not found_reason_to_exclude: + print(idName) + # otherwise the previous line of output will be incorrectly mixed into the below git output, because of buffering + sys.stdout.flush() + # search again, so we log the location and filename of stuff we want to remove + subprocess.call(["git", "grep", "-wn", idName]) + diff --git a/bin/find-unused-sid-commands.py b/bin/find-unused-sid-commands.py new file mode 100755 index 000000000..32f45e0f8 --- /dev/null +++ b/bin/find-unused-sid-commands.py @@ -0,0 +1,53 @@ +#!/usr/bin/python +# +# Find potentially unused UNO command entries in SDI files. +# +# Note that this is not foolproof, some extra checking is required because some command names might be +# constructed at runtime. +# + +import subprocess + +# search for entries in .sdi files that declare UNO/SID commands +a = subprocess.Popen("git grep -P '^\s*\w+Item\s+\w+\s+SID_\w+$' -- *.sdi", stdout=subprocess.PIPE, shell=True) + +# parse out the UNO command names +commandSet = list() +with a.stdout as txt: + for line in txt: + line = line.strip() + idx1 = line.find(" ") + idx2 = line.find(" ", idx1 + 1) + commandName = line[idx1+1 : idx2].strip() + sidName = line[idx2+1:].strip() + commandSet.append((commandName,sidName)) + +# now check to see if that UNO command is called anywhere in the codebase. +for pair in commandSet: + commandName = pair[0] + sidName = pair[1] + + # check to see if that UNO command is called anywhere in the codebase. + a = subprocess.Popen("git grep -wFn '.uno:" + commandName + "'", stdout=subprocess.PIPE, shell=True) + cnt = 0 + with a.stdout as txt2: + for line2 in txt2: + cnt = cnt + 1 + if cnt > 0: continue + + # check to see if the SID is used programmatically + foundLines = "" + a = subprocess.Popen("git grep -wn " + sidName, stdout=subprocess.PIPE, shell=True) + with a.stdout as txt2: + for line2 in txt2: + foundLines = foundLines + line2 + if foundLines.find("ExecuteList") != -1: continue + if foundLines.find("GetDispatcher()->Execute") != -1: continue + if foundLines.find("ExecuteScenarioSlot") != -1: continue + # TODO not sure about this, but let's tackle the easy ones first + if foundLines.find("Invalidate(") != -1: continue + + # dump any lines that contain the SID, so we can eyeball the results + print("remove: " + commandName) + print(foundLines) + print("----------------------------------------------------------------------------") diff --git a/bin/find-unused-typedefs.py b/bin/find-unused-typedefs.py new file mode 100755 index 000000000..b07c16d2b --- /dev/null +++ b/bin/find-unused-typedefs.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2 + +import subprocess + +# find typedefs, excluding the externals folder +a = subprocess.Popen("git grep -P 'typedef\s+.+\s+\w+;' -- \"[!e][!x][!t]*\"", stdout=subprocess.PIPE, shell=True) + +# parse out the typedef names +typedefSet = set() +with a.stdout as txt: + for line in txt: + idx2 = line.rfind(";") + idx1 = line.rfind(" ", 0, idx2) + typedefName = line[idx1+1 : idx2] + if typedefName.startswith("*"): + typedefName = typedefName[1:] + # ignore anything less than 5 characters, it's probably a parsing error + if len(typedefName) < 5: continue + typedefSet.add(typedefName) + +for typedefName in sorted(typedefSet): + print("checking: " + typedefName) + a = subprocess.Popen(["git", "grep", "-wn", typedefName], stdout=subprocess.PIPE) + foundLine2 = "" + cnt = 0 + with a.stdout as txt2: + for line2 in txt2: + cnt = cnt + 1 + foundLine2 += line2 + if cnt == 1: + print("remove: " + foundLine2) + elif cnt == 2: + print("inline: " + foundLine2) + diff --git a/bin/find-unusedheaders.py b/bin/find-unusedheaders.py new file mode 100755 index 000000000..7ca9bea4b --- /dev/null +++ b/bin/find-unusedheaders.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Find dirs in: +workdir/Dep/CObject +workdir/Dep/CxxObject + +Concat these files and compare them with the output of +`git ls-tree HEAD -r --name-only` and report files in the git ls-tree that aren't in the first. +""" + +import os +import subprocess + + +def get_files_dict_recursively(directory): + data = {} + for root, _, files in os.walk(directory, topdown=False): + for f in files: + basename = os.path.splitext(f)[0] + data[basename] = os.path.join(root, f) + return data + + +def main(): + data = {} + for d in ('workdir/Dep/CObject', 'workdir/Dep/CxxObject'): + tmp = get_files_dict_recursively(d) + data.update(tmp) + + gitfiles = subprocess.check_output(['git', 'ls-tree', 'HEAD', '-r', '--name-only']).decode('utf-8').split('\n') + + for f in gitfiles: + ext = os.path.splitext(f)[1] + if ext[1:] in ('c', 'cxx', 'h', 'hxx'): + tmp = os.path.basename(f) + tmp = os.path.splitext(tmp)[0] + if tmp not in data: + print(f) + +if __name__ == '__main__': + main() diff --git a/bin/fixincludeguards.sh b/bin/fixincludeguards.sh new file mode 100755 index 000000000..2655534aa --- /dev/null +++ b/bin/fixincludeguards.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# corrects include guards for hxx/h files automatically by its path. + +# Usage: +# a) fixincludeguards.sh header.hxx +# b) find . -name *.hxx -or -name *.h | xargs bash ./bin/fixincludeguards.sh + +# TODO: This doesn't fix wrong #endif comments, like: +# #ifndef FOO_BAR_HXX +# #define FOO_BAR_HXX +# ... +# #endif // OTHER_BAR_HXX + +# TODO: Make this portable. As it is now, it likely only works on Linux, or +# other platforms with a purely GNU toolset. + +guard_prefix="INCLUDED_" + +for fn in "$@"; do + # remove leading ./, if invoked with find + fn=`echo "$fn" | sed 's/^.\///g'` + + # global header in include/ top level dir: + # drop the project dir + fnfixed=`echo $fn | sed 's,include/,,g'` + # add examples prefix to headers in odk/examples + fnfixed=`echo $fnfixed | sed 's,odk/examples/\(cpp\|DevelopersGuide\|OLE\)/,examples_,g'` + + # convert file path to header guard + guard=`echo "$fnfixed" | sed 's/[\/\.-]/_/g' | tr 'a-z' 'A-Z'` + + if [ aa"`git grep -h "^\s*#ifndef ${guard_prefix}$guard" "$fn" | wc -l`" != "aa1" ] || + [ aa"`git grep -h "^\s*#define ${guard_prefix}$guard" "$fn" | wc -l`" != "aa1" ]; then + + # pattern which identifies guards, common one look like + # _MODULE_FILE_HXX, FILE_H, FILE_INC + pattern=".*\(_HXX\|_H\|_INC\|_hxx\|_h\|_inc\)" + + ### extract guard definition + # head to take only the first match + old_guard=`git grep -h "#ifndef $pattern" "$fn" | head -n1 | sed "s/.*\s\($pattern.*\)/\1/"` + + if [ aa"$old_guard" == aa"" ]; then + echo -e "$fn: \e[00;31mwarning:\e[00m guard not detectable" + continue + fi + + + if [ aa"`git grep -w "$old_guard" | cut -d ':' -f1 | sort -u | wc -l `" != aa"1" ]; then + echo -e "$fn: \e[00;31mwarning:\e[00m $old_guard guard definition used in other files" + continue + fi + + ### skip some special files... + + # skip this comphelper stuff: + # INCLUDED_COMPHELPER_IMPLBASE_VAR_HXX_14 + if [ aa"INCLUDED_COMPHELPER_IMPLBASE_" == aa"`echo $old_guard | sed "s/VAR_HXX_[0-9]\+//g"`" ]; then + continue + fi + + # skip files like xmloff/source/forms/elementimport_impl.hxx + if [ aa"`git grep -h "#error.*directly" "$fn" | wc -l`" != "aa0" ]; then + continue + fi + + + ### replace old guard with new scheme guard + echo "$fn: $old_guard" + + # includes leading whitespace removal + sed -i "s/\s*${old_guard}/ ${guard_prefix}${guard}/g" "$fn" + + + ### clean up endif + sed -i "s/#endif\s*\(\/\/\|\/\*\)\s*\#\?\(ifndef\)\?\s*!\?\s*\(${guard_prefix}${guard}\).*/#endif \/\/ \3/g" "$fn" + + fi +done diff --git a/bin/fuzzfiles b/bin/fuzzfiles new file mode 100755 index 000000000..ed0432d23 --- /dev/null +++ b/bin/fuzzfiles @@ -0,0 +1,41 @@ +#! /bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +#check that zzuf is installed +hash zzuf &> /dev/null +if [ $? -eq 1 ];then + echo >&2 "zzuf not found. Please install and/or fix the PATH environment variable. Aborting" + exit -1 +fi + +#check that file(s) to fuzz are mentioned +if [[ $# -eq 0 ]]; then + echo "Usage: fuzzfiles.sh <list of seed files to fuzz>" + echo "The generated fuzzed files will be output to the current working directory" + echo "The fuzzed files will be named XYZ-ratio-NNNN where:" + echo -e "\tXYZ: the original file name" + echo -e "\tratio: the fuzz ratio (what % of bytes were fuzzed)" + echo -e "\tNNNN: the mutation # for that file and ratio combo" + exit -1 +fi + +for file in $@; do + if [ -d $file ]; then + echo "$file is a directory. Only files are allowed" + elif [ -e $file ]; then + basename=${file##*/} + #Sequence from 0.001 to 0.5 + for ratio in `seq -w 1 2 500 | sed -e 's/^/0./'`; do + echo "Fuzzing $file with ratio $ratio" + for i in {1..1000}; do + zzuf -r $ratio < $file > "$basename-$ratio-$i" + done #end of for i in {1.. + done #end of for ratio in ... + fi #end if of file validity check +done #end for file in $@ diff --git a/bin/gbuild-to-ide b/bin/gbuild-to-ide new file mode 100755 index 000000000..f74712caf --- /dev/null +++ b/bin/gbuild-to-ide @@ -0,0 +1,1911 @@ +#! /usr/bin/env python3 +# -*- Mode: python; tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import argparse +import ntpath +import os +import os.path +import shutil +import re +import sys +import uuid +import json +import xml.etree.ElementTree as ET +import xml.dom.minidom as minidom +import traceback +import subprocess +from sys import platform +import collections + +class GbuildLinkTarget: + def __init__(self, name, location, include, include_sys, defs, cxxobjects, cxxflags, cobjects, cflags, linked_libs): + (self.name, self.location, self.include, self.include_sys, self.defs, self.cxxobjects, self.cxxflags, self.cobjects, self.cflags, self.linked_libs) = ( + name, location, include, include_sys, defs, cxxobjects, cxxflags, cobjects, cflags, linked_libs) + + def short_name(self): + return self.name + + def is_empty(self): + return not self.include and not self.defs and not self.cxxobjects and not self.cobjects and not self.linked_libs + + def __str__(self): + return '%s at %s with include path: %s, isystem includes: %s, defines: %s, objects: %s, cxxflags: %s, cobjects: %s, cflags: %s and linked libs: %s' % ( + self.short_name(), self.location, self.include, self.include_sys, self.defs, self.cxxobjects, + self.cxxflags, self.cobjects, self.cflags, self.linked_libs) + + +class GbuildLib(GbuildLinkTarget): + def __init__(self, name, location, include, include_sys, defs, cxxobjects, cxxflags, cobjects, cflags, linked_libs): + GbuildLinkTarget.__init__(self, name, location, include, include_sys, defs, cxxobjects, cxxflags, cobjects, cflags, linked_libs) + + def short_name(self): + """Return the short name of target based on the Library_* makefile name""" + return 'Library %s' % self.name + + def target_name(self): + return 'Library_%s' % self.name + + def library_name(self): + return self.name + +class GbuildTest(GbuildLinkTarget): + def __init__(self, name, location, include, include_sys, defs, cxxobjects, cxxflags, cobjects, cflags, linked_libs): + GbuildLinkTarget.__init__(self, name, location, include, include_sys, defs, cxxobjects, cxxflags, cobjects, cflags, linked_libs) + + def short_name(self): + """Return the short name of target based n the CppunitTest_* makefile names""" + return 'CppunitTest %s' % self.name + + def target_name(self): + return 'CppunitTest_%s' % self.name + +class GbuildExe(GbuildLinkTarget): + def __init__(self, name, location, include, include_sys, defs, cxxobjects, cxxflags, cobjects, cflags, linked_libs): + GbuildLinkTarget.__init__(self, name, location, include, include_sys, defs, cxxobjects, cxxflags, cobjects, cflags, linked_libs) + + def short_name(self): + """Return the short name of target based on the Executable_* makefile name""" + return 'Executable %s' % self.name + + def target_name(self): + return 'Executable_%s' % self.name + + +class GbuildParser: + """Main data model object. + + Attributes: + target_by_path : dict[path:string, set(target)] + where target is one of the GbuildLinkTarget subclasses + target_by_location : dict[path:string, set(target)] + where target is one of the GbuildLinkTarget subclasses + """ + def __init__(self, makecmd): + self.makecmd = makecmd + self.binpath = os.path.dirname(os.environ['GPERF']) # woha, this is quite a hack + (self.srcdir, self.builddir, self.instdir, self.workdir) = (os.environ['SRCDIR'], os.environ['BUILDDIR'], os.environ['INSTDIR'], os.environ['WORKDIR']) + (self.libs, self.exes, self.tests, self.modulenamelist) = ([], [], [], []) + (self.target_by_path, self.target_by_location) = ({}, {}) + + includepattern = re.compile('-I(\S+)') + isystempattern = re.compile('-isystem\s*(\S+)') + warningpattern = re.compile('-W\S+') + libpattern = re.compile('Library_(.*)\.mk') + exepattern = re.compile('Executable_(.*)\.mk') + testpattern = re.compile('CppunitTest_(.*)\.mk') + + @staticmethod + def __split_includes(includes): + foundisystem = GbuildParser.isystempattern.findall(includes) + foundincludes = [includeswitch.strip() for includeswitch in GbuildParser.includepattern.findall(includes) if + len(includeswitch) > 2] + return (foundincludes, foundisystem) + + @staticmethod + def __split_objs(objsline): + return [obj for obj in objsline.strip().split(' ') if len(obj) > 0 and obj != 'CXXOBJECTS' and obj != 'COBJECTS' and obj != '+='] + + @staticmethod + def __split_defs(defsline): + defs = {} + alldefs = [defswitch.strip() for defswitch in defsline.strip().lstrip('-D').split(' -D') if len(defswitch) > 2] + for d in alldefs: + dparts = d.split(' -U') + """after dparts.pop(0), dparts will contain only undefs""" + defparts = dparts.pop(0).strip().split('=') + if len(defparts) == 1: + defparts.append(None) + defs[defparts[0]] = defparts[1] + """Drop undefed items (if any) from previous defs""" + for u in dparts: + defs.pop(u.strip(), '') + defs["LIBO_INTERNAL_ONLY"] = None + return defs + + @staticmethod + def __split_flags(flagsline, flagslineappend): + return [cxxflag.strip() for cxxflag in GbuildParser.warningpattern.sub('', '%s %s' % (flagsline, flagslineappend)).split(' ') if len(cxxflag) > 1] + + @staticmethod + def __lib_from_json(json): + (foundincludes, foundisystem) = GbuildParser.__split_includes(json['INCLUDE']) + return GbuildLib( + GbuildParser.libpattern.match(os.path.basename(json['MAKEFILE'])).group(1), + os.path.dirname(json['MAKEFILE']), + foundincludes, + foundisystem, + GbuildParser.__split_defs(json['DEFS']), + GbuildParser.__split_objs(json['CXXOBJECTS']), + GbuildParser.__split_flags(json['CXXFLAGS'], json['CXXFLAGSAPPEND']), + GbuildParser.__split_objs(json['COBJECTS']), + GbuildParser.__split_flags(json['CFLAGS'], json['CFLAGSAPPEND']), + json['LINKED_LIBS'].strip().split(' ')) + + @staticmethod + def __test_from_json(json): + (foundincludes, foundisystem) = GbuildParser.__split_includes(json['INCLUDE']) + testname_match = GbuildParser.testpattern.match(os.path.basename(json['MAKEFILE'])) + + # Workaround strange writer test makefile setup + if testname_match is None: + testname = "StrangeWriterMakefiles" + else: + testname = testname_match.group(1) + + return GbuildTest( + testname, + os.path.dirname(json['MAKEFILE']), + foundincludes, + foundisystem, + GbuildParser.__split_defs(json['DEFS']), + GbuildParser.__split_objs(json['CXXOBJECTS']), + GbuildParser.__split_flags(json['CXXFLAGS'], json['CXXFLAGSAPPEND']), + GbuildParser.__split_objs(json['COBJECTS']), + GbuildParser.__split_flags(json['CFLAGS'], json['CFLAGSAPPEND']), + json['LINKED_LIBS'].strip().split(' ')) + + @staticmethod + def __exe_from_json(json): + (foundincludes, foundisystem) = GbuildParser.__split_includes(json['INCLUDE']) + return GbuildExe( + GbuildParser.exepattern.match(os.path.basename(json['MAKEFILE'])).group(1), + os.path.dirname(json['MAKEFILE']), + foundincludes, + foundisystem, + GbuildParser.__split_defs(json['DEFS']), + GbuildParser.__split_objs(json['CXXOBJECTS']), + GbuildParser.__split_flags(json['CXXFLAGS'], json['CXXFLAGSAPPEND']), + GbuildParser.__split_objs(json['COBJECTS']), + GbuildParser.__split_flags(json['CFLAGS'], json['CFLAGSAPPEND']), + json['LINKED_LIBS'].strip().split(' ')) + + def parse(self): + for jsonfilename in os.listdir(os.path.join(self.workdir, 'GbuildToJson', 'Library')): + with open(os.path.join(self.workdir, 'GbuildToJson', 'Library', jsonfilename), 'r') as f: + lib = self.__lib_from_json(json.load(f)) + self.libs.append(lib) + for jsonfilename in os.listdir(os.path.join(self.workdir, 'GbuildToJson', 'Executable')): + with open(os.path.join(self.workdir, 'GbuildToJson', 'Executable', jsonfilename), 'r') as f: + exe = self.__exe_from_json(json.load(f)) + self.exes.append(exe) + for jsonfilename in os.listdir(os.path.join(self.workdir, 'GbuildToJson', 'CppunitTest')): + with open(os.path.join(self.workdir, 'GbuildToJson', 'CppunitTest', jsonfilename), 'r') as f: + test = self.__test_from_json(json.load(f)) + self.tests.append(test) + for target in set(self.libs) | set(self.exes) | set(self.tests): + if target.location not in self.target_by_location: + self.target_by_location[target.location] = set() + self.target_by_location[target.location] |= set([target]) + for cxx in target.cxxobjects: + path = '/'.join(cxx.split('/')[:-1]) + if path not in self.target_by_path: + self.target_by_path[path] = set() + self.target_by_path[path] |= set([target]) + for c in target.cobjects: + path = '/'.join(c.split('/')[:-1]) + if path not in self.target_by_path: + self.target_by_path[path] = set() + self.target_by_path[path] |= set([target]) + for location in self.target_by_location: + self.modulenamelist.append(os.path.split(location)[1]) + return self + + +class IdeIntegrationGenerator: + + def __init__(self, gbuildparser, ide): + self.gbuildparser = gbuildparser + self.ide = ide + + def emit(self): + pass + +class EclipseCDTIntegrationGenerator(IdeIntegrationGenerator): + + def __init__(self, gbuildparser, ide): + IdeIntegrationGenerator.__init__(self, gbuildparser, ide) + + def create_include_paths(self): + for module in self.gbuildparser.modulenamelist: + modulepath = os.path.join(self.gbuildparser.builddir, module) + includedirfile = open(os.path.join(modulepath, '.eclipsesettingfile'), 'w') + modulelibs = [] + for lib in self.gbuildparser.target_by_path.keys(): + if lib.startswith(module+'/'): + modulelibs.append(lib) + include = set() + for lib in modulelibs: + for target in self.gbuildparser.target_by_path[lib]: + include |= set(target.include) + includedirfile.write('\n'.join(include)) + includedirfile.close() + + + def create_macros(self): + for module in self.gbuildparser.modulenamelist: + modulepath = os.path.join(self.gbuildparser.builddir, module) + macrofile = open(os.path.join(modulepath, '.macros'), 'w') + modulelibs = [] + for lib in self.gbuildparser.target_by_path.keys(): + if lib.startswith(module+'/'): + modulelibs.append(lib) + define = [] + defineset = set() + for lib in modulelibs: + for target in self.gbuildparser.target_by_path[lib]: + for i in target.defs.keys(): + tmp = str(i) +','+str(target.defs[i]) + if tmp not in defineset: + defineset.add(tmp) + macrofile.write('\n'.join(defineset)) + macrofile.close() + + + def create_settings_file(self): + + settingsfiletemplate = """\ +<?xml version="1.0" encoding="UTF-8"?> +<cdtprojectproperties> +<section name="org.eclipse.cdt.internal.ui.wizards.settingswizards.IncludePaths"> +<language name="C++ Source File"> + + +</language> +<language name="C Source File"> + +</language> +<language name="Object File"> + +</language> +<language name="Assembly Source File"> + +</language> +</section> +<section name="org.eclipse.cdt.internal.ui.wizards.settingswizards.Macros"> +<language name="C++ Source File"> + +</language> +<language name="C Source File"> + +</language> +<language name="Object File"> + +</language> +<language name="Assembly Source File"> + +</language> +</section> +</cdtprojectproperties> +""" + + for module in self.gbuildparser.modulenamelist: + tempxml = [] + modulepath = os.path.join(self.gbuildparser.builddir, module) + + settingsfile = open(os.path.join(modulepath, 'eclipsesettingfile.xml'), 'w') + settingsfile.write(settingsfiletemplate) + settingsfile.close() + + settingsfile = open(os.path.join(modulepath, 'eclipsesettingfile.xml'), 'r') + tempxml = settingsfile.readlines() + tempinclude = open(os.path.join(modulepath, '.eclipsesettingfile'), 'r') + tempmacro = open(os.path.join(modulepath, '.macros'), 'r') + for includepath in tempinclude: + if includepath[-1:] == "\n": + includepath = includepath[:-1] + templine = "<includepath>%s</includepath>\n" % includepath + tempxml.insert(5, templine) + + for line in tempmacro: + macroskeyvalue = line.split(',') + macrokey = macroskeyvalue[0] + macrovalue = macroskeyvalue[1] + if macrovalue[-1:] == "\n": + macrovalue = macrovalue[:-1] + templine = "<macro><name>%s</name><value>%s</value></macro>\n" %(macrokey, macrovalue) + tempxml.insert(-13, templine) + tempxml="".join(tempxml) + settingsfile.close + + settingsfile = open(os.path.join(modulepath, 'eclipsesettingfile.xml'), 'w') + settingsfile.write(tempxml) + settingsfile.close() + os.remove(os.path.join(modulepath, '.eclipsesettingfile')) + os.remove(os.path.join(modulepath, '.macros')) + + def emit(self): + self.create_include_paths() + self.create_macros() + self.create_settings_file() + +class CodeliteIntegrationGenerator(IdeIntegrationGenerator): + + def __init__(self, gbuildparser, ide): + IdeIntegrationGenerator.__init__(self, gbuildparser, ide) + + def emit(self): + self.create_workspace_file() + for module in self.gbuildparser.modulenamelist: + self.create_project_file(module) + #self.create_project_file('vcl') + + def create_workspace_file(self): + root_node = ET.Element('CodeLite_Workspace', Name='libo2', Database='./libo2.tags', Version='10.0.0') + for module in self.gbuildparser.modulenamelist: + ET.SubElement(root_node, 'Project', Name=module, Path='%s/%s.project' % (module, module), Active='No') + build_matrix_node = ET.SubElement(root_node, 'BuildMatrix') + workspace_config_node = ET.SubElement(build_matrix_node, 'WorkspaceConfiguration', Name='Debug', Selected='yes') + ET.SubElement(workspace_config_node, 'Environment') + for module in self.gbuildparser.modulenamelist: + ET.SubElement(workspace_config_node, 'Project', Name=module, ConfigName='Debug') + workspace_config_node = ET.SubElement(build_matrix_node, 'WorkspaceConfiguration', Name='Release', Selected='yes') + ET.SubElement(workspace_config_node, 'Environment') + for module in self.gbuildparser.modulenamelist: + ET.SubElement(workspace_config_node, 'Project', Name=module, ConfigName='Release') + + self.write_pretty_xml(root_node, os.path.join(self.gbuildparser.builddir, 'libo2.workspace')) + + def create_project_file(self, module_name): + root_node = ET.Element('CodeLite_Project', Name=module_name, InternalType='') + ET.SubElement(root_node, 'Plugins') + + # add CXX files + virtual_dirs = collections.defaultdict(set) + for target_path in self.gbuildparser.target_by_path.keys(): + if target_path.startswith(module_name+'/'): + for target in self.gbuildparser.target_by_path[target_path]: + for file in target.cxxobjects: + relative_file = '/'.join(file.split('/')[1:]) + path = '/'.join(file.split('/')[1:-1]) + virtual_dirs[path].add(relative_file + '.cxx') + # add HXX files + all_libs = set(self.gbuildparser.libs) | set(self.gbuildparser.exes) + for lib in all_libs: + if lib.name == module_name: + for hdir in lib.include: + # only want the module-internal ones + if hdir.startswith(module_name+'/'): + for hf in os.listdir(hdir): + if hf.endswith(('.h', '.hxx', '.hpp', '.hrc')): + path = '/'.join(hf.split('/')[1:-1]) + virtual_dirs[path].add(hf) + # add HXX files from the root/include/** folders + module_include = os.path.join(self.gbuildparser.builddir, 'include', module_name) + if os.path.exists(module_include): + for hf in os.listdir(module_include): + if hf.endswith(('.h', '.hxx', '.hpp', '.hrc')): + path = '../include/' + ('/'.join(hf.split('/')[1:-1])) + virtual_dirs['include/' + module_name].add('../include/' + module_name + '/' + hf) + + for vd_name in sorted(virtual_dirs.keys()): + vd_files = sorted(virtual_dirs[vd_name]) + parent_node = root_node + for subname in vd_name.split('/'): + parent_node = ET.SubElement(parent_node, 'VirtualDirectory', Name=subname) + for file in vd_files: + ET.SubElement(parent_node, 'File', Name=file) + + ET.SubElement(root_node, 'Description') + ET.SubElement(root_node, 'Dependencies') + ET.SubElement(root_node, 'Dependencies', Name='Debug') + ET.SubElement(root_node, 'Dependencies', Name='Release') + + settingstemplate = """\ + <Settings Type="Dynamic Library"> + <GlobalSettings> + <Compiler Options="" C_Options="" Assembler=""> + <IncludePath Value="."/> + </Compiler> + <Linker Options=""> + <LibraryPath Value="."/> + </Linker> + <ResourceCompiler Options=""/> + </GlobalSettings> + <Configuration Name="Debug" CompilerType="clang( based on LLVM 3.5.0 )" DebuggerType="GNU gdb debugger" Type="Dynamic Library" BuildCmpWithGlobalSettings="append" BuildLnkWithGlobalSettings="append" BuildResWithGlobalSettings="append"> + <Compiler Options="-g" C_Options="-g" Assembler="" Required="yes" PreCompiledHeader="" PCHInCommandLine="no" PCHFlags="" PCHFlagsPolicy="0"> + <IncludePath Value="."/> + </Compiler> + <Linker Options="" Required="yes"/> + <ResourceCompiler Options="" Required="no"/> + <General OutputFile="" IntermediateDirectory="./Debug" Command="" CommandArguments="" UseSeparateDebugArgs="no" DebugArguments="" WorkingDirectory="$(IntermediateDirectory)" PauseExecWhenProcTerminates="yes" IsGUIProgram="no" IsEnabled="yes"/> + <BuildSystem Name="Default"/> + <Environment EnvVarSetName="<Use Defaults>" DbgSetName="<Use Defaults>"> + <![CDATA[]]> + </Environment> + <Debugger IsRemote="no" RemoteHostName="" RemoteHostPort="" DebuggerPath="" IsExtended="no"> + <DebuggerSearchPaths/> + <PostConnectCommands/> + <StartupCommands/> + </Debugger> + <PreBuild/> + <PostBuild/> + <CustomBuild Enabled="yes"> + <RebuildCommand/> + <CleanCommand>make %s.clean</CleanCommand> + <BuildCommand>make %s.build</BuildCommand> + <PreprocessFileCommand/> + <SingleFileCommand/> + <MakefileGenerationCommand/> + <ThirdPartyToolName>None</ThirdPartyToolName> + <WorkingDirectory>$(WorkspacePath)</WorkingDirectory> + </CustomBuild> + <AdditionalRules> + <CustomPostBuild/> + <CustomPreBuild/> + </AdditionalRules> + <Completion EnableCpp11="no" EnableCpp14="no"> + <ClangCmpFlagsC/> + <ClangCmpFlags/> + <ClangPP/> + <SearchPaths/> + </Completion> + </Configuration> + <Configuration Name="Release" CompilerType="clang( based on LLVM 3.5.0 )" DebuggerType="GNU gdb debugger" Type="Dynamic Library" BuildCmpWithGlobalSettings="append" BuildLnkWithGlobalSettings="append" BuildResWithGlobalSettings="append"> + <Compiler Options="" C_Options="" Assembler="" Required="yes" PreCompiledHeader="" PCHInCommandLine="no" PCHFlags="" PCHFlagsPolicy="0"> + <IncludePath Value="."/> + </Compiler> + <Linker Options="-O2" Required="yes"/> + <ResourceCompiler Options="" Required="no"/> + <General OutputFile="" IntermediateDirectory="./Release" Command="" CommandArguments="" UseSeparateDebugArgs="no" DebugArguments="" WorkingDirectory="$(IntermediateDirectory)" PauseExecWhenProcTerminates="yes" IsGUIProgram="no" IsEnabled="yes"/> + <BuildSystem Name="Default"/> + <Environment EnvVarSetName="<Use Defaults>" DbgSetName="<Use Defaults>"> + <![CDATA[]]> + </Environment> + <Debugger IsRemote="no" RemoteHostName="" RemoteHostPort="" DebuggerPath="" IsExtended="no"> + <DebuggerSearchPaths/> + <PostConnectCommands/> + <StartupCommands/> + </Debugger> + <PreBuild/> + <PostBuild/> + <CustomBuild Enabled="yes"> + <RebuildCommand/> + <CleanCommand>make %s.clean</CleanCommand> + <BuildCommand>make %s.build</BuildCommand> + <PreprocessFileCommand/> + <SingleFileCommand/> + <MakefileGenerationCommand/> + <ThirdPartyToolName>None</ThirdPartyToolName> + <WorkingDirectory>$(WorkspacePath)</WorkingDirectory> + </CustomBuild> + <AdditionalRules> + <CustomPostBuild/> + <CustomPreBuild/> + </AdditionalRules> + <Completion EnableCpp11="no" EnableCpp14="no"> + <ClangCmpFlagsC/> + <ClangCmpFlags/> + <ClangPP/> + <SearchPaths/> + </Completion> + </Configuration> + </Settings> +""" + root_node.append(ET.fromstring(settingstemplate % (module_name, module_name, module_name, module_name))) + + self.write_pretty_xml(root_node, os.path.join(self.gbuildparser.builddir, module_name, '%s.project' % module_name)) + + def write_pretty_xml(self, node, file_path): + xml_str = ET.tostring(node, encoding='unicode') + pretty_str = minidom.parseString(xml_str).toprettyxml(encoding='utf-8') + with open(file_path, 'w') as f: + f.write(pretty_str.decode()) + +class DebugIntegrationGenerator(IdeIntegrationGenerator): + + def __init__(self, gbuildparser, ide): + IdeIntegrationGenerator.__init__(self, gbuildparser, ide) + + def emit(self): + print(self.gbuildparser.srcdir) + print(self.gbuildparser.builddir) + for lib in self.gbuildparser.libs: + print(lib) + for exe in self.gbuildparser.exes: + print(exe) + for test in self.gbuildparser.tests: + print(test) + + +class VimIntegrationGenerator(IdeIntegrationGenerator): + + def __init__(self, gbuildparser, ide): + IdeIntegrationGenerator.__init__(self, gbuildparser, ide) + + def emit(self): + global_list = [] + for lib in set(self.gbuildparser.libs) | set(self.gbuildparser.tests) | set(self.gbuildparser.exes): + entries = [] + for file in lib.cxxobjects: + filePath = os.path.join(self.gbuildparser.srcdir, file) + ".cxx" + entry = {'directory': lib.location, 'file': filePath, 'command': self.generateCommand(lib, filePath)} + entries.append(entry) + global_list.extend(entries) + export_file = open('compile_commands.json', 'w') + json.dump(global_list, export_file) + + def generateCommand(self, lib, file): + command = 'clang++ -Wall' + for key, value in lib.defs.items(): + command += ' -D' + command += key + if value is not None: + command += '=' + command += value + + for include in lib.include: + command += ' -I' + command += include + for isystem in lib.include_sys: + command += ' -isystem ' + command += isystem + for cxxflag in lib.cxxflags: + command += ' ' + command += cxxflag + command += ' -c ' + command += file + return command + + +class KdevelopIntegrationGenerator(IdeIntegrationGenerator): + + def encode_int(self, i): + temp = '%08x' % i + return '\\x%s\\x%s\\x%s\\x%s' % (temp[0:2], temp[2:4], temp[4:6], temp[6:8]) + + def encode_string(self, string): + result = self.encode_int(len(string) * 2) + for c in string.encode('utf-16-be'): + if c in range(32, 126): + result += chr(c) + else: + result += '\\x%02x' % c + return result + + def generate_buildsystemconfigtool(self, configid, tool, args, exe, typenr): + return KdevelopIntegrationGenerator.buildsystemconfigtooltemplate % {'configid': configid, 'tool': tool, + 'args': args, 'exe': exe, 'typenr': typenr} + + buildsystemconfigtooltemplate = """ +[CustomBuildSystem][BuildConfig%(configid)d][Tool%(tool)s] +Arguments=%(args)s +Enabled=true +Environment= +Executable=%(exe)s +Type=%(typenr)d + +""" + + def generate_buildsystemconfig(self, configid, moduledir, builddir, title, buildparms=''): + result = KdevelopIntegrationGenerator.buildsystemconfigtemplate % {'configid': configid, 'builddir': builddir, + 'title': title} + result += self.generate_buildsystemconfigtool(configid, 'Clean', 'clean %s' % buildparms, + self.gbuildparser.makecmd, 3) + result += self.generate_buildsystemconfigtool(configid, 'Build', 'all %s' % buildparms, + self.gbuildparser.makecmd, 0) + return result + + buildsystemconfigtemplate = """ +[CustomBuildSystem][BuildConfig%(configid)d] +BuildDir=file://%(builddir)s +Title=%(title)s + +""" + + def generate_buildsystem(self, moduledir): + result = KdevelopIntegrationGenerator.buildsystemtemplate % {'defaultconfigid': 0} + result += self.generate_buildsystemconfig(0, moduledir, moduledir, 'Module Build -- Release') + result += self.generate_buildsystemconfig(1, moduledir, self.gbuildparser.builddir, 'Full Build -- Release') + result += self.generate_buildsystemconfig(2, moduledir, moduledir, 'Module Build -- Debug', 'debug=T') + result += self.generate_buildsystemconfig(3, moduledir, self.gbuildparser.builddir, 'Full Build -- Debug', + 'debug=T') + return result + + buildsystemtemplate = """ +[CustomBuildSystem] +CurrentConfiguration=BuildConfig%(defaultconfigid)d + +""" + + def generate_launch(self, launchid, launchname, executablepath, args, workdir): + return KdevelopIntegrationGenerator.launchtemplate % {'launchid': launchid, 'launchname': launchname, + 'executablepath': executablepath, 'args': args, + 'workdir': workdir} + + launchtemplate = """ +[Launch][Launch Configuration %(launchid)d] +Configured Launch Modes=execute +Configured Launchers=nativeAppLauncher +Name=%(launchname)s +Type=Native Application + +[Launch][Launch Configuration %(launchid)d][Data] +Arguments=%(args)s +Dependencies=@Variant(\\x00\\x00\\x00\\t\\x00\\x00\\x00\\x00\\x00) +Dependency Action=Nothing +EnvironmentGroup=default +Executable=file://%(executablepath)s +External Terminal=konsole --noclose --workdir %%workdir -e %%exe +Project Target= +Use External Terminal=false +Working Directory=file://%(workdir)s +isExecutable=true + +""" + + def generate_launches(self, moduledir): + launches = ','.join(['Launch Configuration %d' % i for i in range(7)]) + result = KdevelopIntegrationGenerator.launchestemplate % {'launches': launches} + result += self.generate_launch(0, 'Local tests -- quick tests (unitcheck)', self.gbuildparser.makecmd, + 'unitcheck', moduledir) + result += self.generate_launch(1, 'Local tests -- slow tests (unitcheck, slowcheck, screenshot)', self.gbuildparser.makecmd, + 'unitcheck slowcheck screenshot', moduledir) + result += self.generate_launch(2, 'Local tests -- integration tests (unitcheck, slowcheck, screenshot, subsequentcheck)', + self.gbuildparser.makecmd, 'unitcheck slowcheck screenshot subsequentcheck', moduledir) + result += self.generate_launch(3, 'Global tests -- quick tests (unitcheck)', self.gbuildparser.makecmd, + 'unitcheck', self.gbuildparser.builddir) + result += self.generate_launch(4, 'Global tests -- slow tests (unitcheck, slowcheck, screenshot)', + self.gbuildparser.makecmd, 'unitcheck slowcheck screenshot', self.gbuildparser.builddir) + result += self.generate_launch(5, 'Global tests -- integration tests (unitcheck, slowcheck, screenshot, subsequentcheck)', + self.gbuildparser.makecmd, 'unitcheck slowcheck screenshot subsequentcheck', + self.gbuildparser.builddir) + result += self.generate_launch(6, 'Run LibreOffice', + os.path.join(self.gbuildparser.instdir, 'program/soffice.bin'), '', + self.gbuildparser.instdir) + return result + + launchestemplate = """ +[Launch] +Launch Configurations=%(launches)s + +""" + + def write_modulebeef(self, moduledir, modulename): + beefdir = os.path.join(moduledir, '.kdev4') + os.mkdir(beefdir) + beeffile = open(os.path.join(beefdir, 'Module_%s.kdev4' % modulename), 'w') + beeffile.write(self.generate_buildsystem(moduledir)) + beeffile.write(self.generate_launches(moduledir)) + beeffile.close() + + def write_modulestub(self, moduledir, modulename): + stubfile = open(os.path.join(moduledir, 'Module_%s.kdev4' % modulename), 'w') + stubfile.write(KdevelopIntegrationGenerator.modulestubtemplate % {'modulename': modulename, + 'builditem': self.encode_string( + 'Module_%s' % modulename)}) + stubfile.close() + + modulestubtemplate = """ +[Buildset] +BuildItems=@Variant(\\x00\\x00\\x00\\t\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00\\x00\\x00\\x00\\x01%(builditem)s) + +[Project] +Name=Module_%(modulename)s +Manager=KDevCustomBuildSystem +VersionControl=kdevgit +""" + + def write_includepaths(self, path): + includedirfile = open(os.path.join(path, '.kdev_include_paths'), 'w') + include = set() + for target in self.gbuildparser.target_by_path[path]: + include |= set(target.include) + includedirfile.write('\n'.join(include)) + includedirfile.close() + + def __init__(self, gbuildparser, ide): + IdeIntegrationGenerator.__init__(self, gbuildparser, ide) + + def emit(self): + for path in self.gbuildparser.target_by_path: + self.write_includepaths(path) + for location in self.gbuildparser.target_by_location: + for f in os.listdir(location): + if f.endswith('.kdev4'): + try: + os.remove(os.path.join(location, f)) + except OSError: + shutil.rmtree(os.path.join(location, f)) + for location in self.gbuildparser.target_by_location: + modulename = os.path.split(location)[1] + self.write_modulestub(location, modulename) + self.write_modulebeef(location, modulename) + + +class XcodeIntegrationGenerator(IdeIntegrationGenerator): + + def indent(self, file, level): + if level == 0: + return + for i in range(0, level): + file.write(' ') + + def write_object(self, object, file, indent): + if isinstance(object, int): + file.write('%d' % object) + elif isinstance(object, str) and not re.search('[^A-Za-z0-9_]', object): + file.write('%s' % object) + elif isinstance(object, str): + file.write('"%s"' % object) + elif isinstance(object, dict): + self.write_dict(object, file, indent) + + # Write a dictionary out as an "old-style (NeXT) ASCII plist" + def write_dict(self, dict, file, indent): + file.write('{') + file.write('\n') + for key in sorted(dict.keys()): + self.indent(file, indent + 1) + file.write('%s = ' % key) + self.write_object(dict[key], file, indent + 1) + file.write(';\n') + self.indent(file, indent) + file.write('}') + + def write_dict_to_plist(self, dict, file): + file.write('// !$*UTF8*$!\n') + self.write_dict(dict, file, 0) + + def get_product_type(self, modulename): + if modulename in self.gbuildparser.libs: + return 'com.apple.product-type.library.dynamic' + elif modulename in self.gbuildparser.exes: + return 'com.apple.product-type.something' + + counter = 0 + + def generate_id(self): + XcodeIntegrationGenerator.counter = XcodeIntegrationGenerator.counter + 1 + return str('X%07x' % XcodeIntegrationGenerator.counter) + + def generate_build_phases(self, modulename): + result = [self.sourcesBuildPhaseId] + return result + + def generate_root_object(self, modulename): + result = {'isa': 'PBXProject', + 'attributes': {'LastUpgradeCheck': '0500', + 'ORGANIZATIONNAME': 'LibreOffice'}, + 'buildConfigurationList': self.generate_id(), + 'compatibilityVersion': 'Xcode 3.2', + 'hasScannedForEncodings': 0, + 'knownRegions': ['en'], + 'mainGroup': self.mainGroupId, + 'productRefGroup': self.productRefGroupId, + 'projectDirPath': '', + 'projectRoot': '', + 'targets': self.targetId} + return result + + def generate_target(self, modulename): + result = {'isa': 'PBXNativeTarget', + 'buildConfigurationList': self.generate_id(), + 'buildPhases': self.generate_build_phases(modulename), + 'buildRules': [], + 'dependencies': [], + 'name': modulename, + 'productName': modulename, + 'productReference': self.productReferenceId, + 'productType': self.get_product_type(modulename)} + return result + + def generate_main_group(self, modulename): + result = {'isa': 'PBXGroup', + 'children': [self.subMainGroupId, self.productGroupId], + 'sourceTree': '<group>'} + return result + + def generate_sub_main_children(self, modulename): + return {} + + def generate_sub_main_group(self, modulename): + result = {'isa': 'PBXGroup', + 'children': self.generate_sub_main_children(modulename), + 'path': modulename, + 'sourceTree': '<group>'} + return result + + def generate_product_group(self, modulename): + result = {'isa': 'PBXGroup', + 'children': [self.productReferenceId], + 'name': 'Products', + 'sourceTree': '<group>'} + return result + + def build_source_list(self, module): + self.sourceRefList = {} + self.sourceList = {} + + for i in module.cxxobjects: + ref = self.generate_id() + self.sourceList[self.generate_id()] = ref + self.sourceRefList[ref] = {'lastKnownFileType': 'sourcecode.cpp.cpp', + 'path': i + '.cxx', + 'sourceTree': '<group>'} + + def generate_sources_build_phase(self, modulename): + result = {'isa': 'PBXSourcesBuildPhase', + 'buildActionMask': 2147483647, + 'files': self.sourceList.keys(), + 'runOnlyForDeploymentPostprocessing': 0} + return result + + def generate_project(self, target): + self.rootObjectId = self.generate_id() + self.mainGroupId = self.generate_id() + self.subMainGroupId = self.generate_id() + self.productReferenceId = self.generate_id() + self.productRefGroupId = self.generate_id() + self.productGroupId = self.generate_id() + self.targetId = self.generate_id() + self.build_source_list(target) + self.sourcesBuildPhaseId = self.generate_id() + objects = {self.rootObjectId: self.generate_root_object(target), + self.targetId: self.generate_target(target), + self.mainGroupId: self.generate_main_group(target), + self.subMainGroupId: self.generate_sub_main_group(target), + self.productGroupId: self.generate_product_group(target), + self.sourcesBuildPhaseId: self.generate_sources_build_phase(target) + } + for i in self.sourceList.keys(): + ref = self.sourceList[i] + objects[i] = {'isa': 'PBXBuildFile', + 'fileRef': ref} + objects[ref] = {'isa': 'PBXFileReference', + 'lastKnownFileType': self.sourceRefList[ref]['lastKnownFileType'], + 'path': self.sourceRefList[ref]['path']} + project = {'archiveVersion': 1, + 'classes': {}, + 'objectVersion': 46, + 'objects': objects, + 'rootObject': self.rootObjectId} + return project + + # For some reverse-engineered documentation on the project.pbxproj format, + # see http://www.monobjc.net/xcode-project-file-format.html . + def write_xcodeproj(self, moduledir, target): + xcodeprojdir = os.path.join(moduledir, '%s.xcodeproj' % target.target_name()) + try: + os.mkdir(xcodeprojdir) + except: + pass + self.write_dict_to_plist(self.generate_project(target), + open(os.path.join(xcodeprojdir, 'project.pbxproj'), 'w')) + + def __init__(self, gbuildparser, ide): + IdeIntegrationGenerator.__init__(self, gbuildparser, ide) + + def emit(self): + self.rootlocation = './' + for location in self.gbuildparser.target_by_location: + # module = location.split('/')[-1] + # module_directory = os.path.join(self.rootlocation, module) + for target in self.gbuildparser.target_by_location[location]: + # project_path = os.path.join(module_directory, '%s.pbxroj' % target.target_name()) + self.write_xcodeproj(location, target) + + +class VisualStudioIntegrationGenerator(IdeIntegrationGenerator): + + def __init__(self, gbuildparser, ide): + IdeIntegrationGenerator.__init__(self, gbuildparser, ide) + self.toolset = self.retrieve_toolset(ide) + self.solution_directory = self.gbuildparser.builddir + self.configurations = { + 'Build': { + 'build': self.module_make_command('%(target)s'), + 'clean': self.module_make_command('%(target)s.clean'), + 'rebuild': self.module_make_command('%(target)s.clean %(target)s') + }, + 'Unit Tests': { + 'build': self.module_make_command('unitcheck'), + 'clean': self.module_make_command('clean'), + 'rebuild': self.module_make_command('clean unitcheck'), + }, + 'Integration tests': { + 'build': self.module_make_command('unitcheck slowcheck screenshot subsequentcheck'), + 'clean': self.module_make_command('clean'), + 'rebuild': self.module_make_command('clean unitcheck slowcheck screenshot subsequentcheck') + } + } + + def retrieve_toolset(self, ide): + ide_toolset_map = {'vs2017': 'v141', 'vs2019': 'v142'} + return ide_toolset_map[ide] + + def module_make_command(self, targets): + return '%(sh)s -c "PATH=\\"/bin:$PATH\\";BUILDDIR=\\"%(builddir)s\\" %(makecmd)s -rsC %(location)s ' + targets + '"' + + class Project: + + def __init__(self, guid, target, project_path): + self.guid = guid + self.target = target + self.path = project_path + + def emit(self): + all_projects = [] + for location in self.gbuildparser.target_by_location: + projects = [] + module = location.split('/')[-1] + module_directory = os.path.join(self.solution_directory, module) + for target in self.gbuildparser.target_by_location[location]: + project_path = os.path.join(module_directory, '%s.vcxproj' % target.target_name()) + project_guid = self.write_project(project_path, target) + p = VisualStudioIntegrationGenerator.Project(project_guid, target, project_path) + projects.append(p) + self.write_solution(os.path.join(module_directory, '%s.sln' % module), projects) + all_projects += projects + + self.write_solution(os.path.join(self.solution_directory, 'LibreOffice.sln'), all_projects) + + nmake_project_guid = '8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942' + + def get_dependency_libs(self, linked_libs, library_projects): + dependency_libs = {} + for linked_lib in linked_libs: + for library_project in library_projects: + if library_project.target.library_name() == linked_lib: + dependency_libs[library_project.guid] = library_project + return dependency_libs + + def write_solution(self, solution_path, projects): + print('Solution %s:' % os.path.splitext(os.path.basename(solution_path))[0], end='') + library_projects = [project for project in projects if project.target in self.gbuildparser.libs] + with open(solution_path, 'w') as f: + f.write('Microsoft Visual Studio Solution File, Format Version 12.00\n') + for project in projects: + target = project.target + print(' %s' % target.target_name(), end='') + proj_path = os.path.relpath(project.path, os.path.abspath(os.path.dirname(solution_path))) + f.write('Project("{%s}") = "%s", "%s", "{%s}"\n' % + (VisualStudioIntegrationGenerator.nmake_project_guid, + target.short_name(), proj_path, project.guid)) + libs_in_solution = self.get_dependency_libs(target.linked_libs, + library_projects) + if libs_in_solution: + f.write('\tProjectSection(ProjectDependencies) = postProject\n') + for lib_guid in libs_in_solution.keys(): + f.write('\t\t{%(guid)s} = {%(guid)s}\n' % {'guid': lib_guid}) + f.write('\tEndProjectSection\n') + f.write('EndProject\n') + f.write('Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B9292527-A979-4D13-A598-C75A33222174}"\n') + f.write('\tProjectSection(SolutionItems) = preProject\n') + # The natvis file gives pretty-printed variable values when debugging + natvis_path = os.path.join(gbuildparser.srcdir, 'solenv/vs/LibreOffice.natvis') + f.write('\t\t%(natvis)s = %(natvis)s\n' % {'natvis': natvis_path}) + f.write('\tEndProjectSection\n') + f.write('EndProject\n') + f.write('Global\n') + platform = 'Win32' + f.write('\tGlobalSection(SolutionConfigurationPlatforms) = preSolution\n') + for cfg in self.configurations: + f.write('\t\t%(cfg)s|%(platform)s = %(cfg)s|%(platform)s\n' % {'cfg': cfg, 'platform': platform}) + f.write('\tEndGlobalSection\n') + f.write('\tGlobalSection(ProjectConfigurationPlatforms) = postSolution\n') + # Specifies project configurations for solution configuration + for project in projects: + for cfg in self.configurations: + params = {'guid': project.guid, 'sol_cfg': cfg, 'proj_cfg': cfg, 'platform': platform} + f.write('\t\t{%(guid)s}.%(sol_cfg)s|%(platform)s.ActiveCfg = %(proj_cfg)s|%(platform)s\n' % params) + # Build.0 is basically 'Build checkbox' in configuration manager + f.write('\t\t{%(guid)s}.%(sol_cfg)s|%(platform)s.Build.0 = %(proj_cfg)s|%(platform)s\n' % params) + f.write('\tEndGlobalSection\n') + f.write('EndGlobal\n') + print('') + + @staticmethod + def to_long_names(shortnames): + if platform == "cygwin": + return (subprocess.check_output(["cygpath", "-wal"] + shortnames).decode("utf-8", "strict").rstrip()).split("\n") + else: + return shortnames + + @staticmethod + def defs_list(defs): + defines_list = [] + # List defines + for key, value in defs.items(): + define = key + if value is not None: + define += '=' + value + defines_list.append(define) + return defines_list + + def write_project(self, project_path, target): + # See info at http://blogs.msdn.com/b/visualstudio/archive/2010/05/14/a-guide-to-vcxproj-and-props-file-structure.aspx + folder = os.path.dirname(project_path) + if not os.path.exists(folder): + os.makedirs(folder) + project_guid = str(uuid.uuid4()).upper() + cxxflags = ' '.join(target.cxxflags) + ns = 'http://schemas.microsoft.com/developer/msbuild/2003' + ET.register_namespace('', ns) + proj_node = ET.Element('{%s}Project' % ns, DefaultTargets='Build', ToolsVersion='4.0') + proj_confs_node = ET.SubElement(proj_node, '{%s}ItemGroup' % ns, Label='ProjectConfigurations') + platform = 'Win32' + for configuration in self.configurations: + proj_conf_node = ET.SubElement(proj_confs_node, + '{%s}ProjectConfiguration' % ns, + Include='%s|%s' % (configuration, platform)) + conf_node = ET.SubElement(proj_conf_node, '{%s}Configuration' % ns) + conf_node.text = configuration + platform_node = ET.SubElement(proj_conf_node, '{%s}Platform' % ns) + platform_node.text = platform + + globals_node = ET.SubElement(proj_node, '{%s}PropertyGroup' % ns, Label='Globals') + proj_guid_node = ET.SubElement(globals_node, '{%s}ProjectGuid' % ns) + proj_guid_node.text = '{%s}' % project_guid + proj_keyword_node = ET.SubElement(globals_node, '{%s}Keyword' % ns) + proj_keyword_node.text = 'MakeFileProj' + proj_name_node = ET.SubElement(globals_node, '{%s}ProjectName' % ns) + proj_name_node.text = target.short_name() + + ET.SubElement(proj_node, '{%s}Import' % ns, Project='$(VCTargetsPath)\Microsoft.Cpp.Default.props') + for configuration in self.configurations: + conf_node = ET.SubElement(proj_node, '{%s}PropertyGroup' % ns, Label="Configuration", + Condition="'$(Configuration)|$(Platform)'=='%s|%s'" % (configuration, platform)) + # Type of project used by the MSBuild to determine build process, see Microsoft.Makefile.targets + conf_type_node = ET.SubElement(conf_node, '{%s}ConfigurationType' % ns) + conf_type_node.text = 'Makefile' + # This defines the version of Visual Studio which can show next to project names in the Solution Explorer + platform_toolset_node = ET.SubElement(conf_node, '{%s}PlatformToolset' % ns) + platform_toolset_node.text = self.toolset + + ET.SubElement(proj_node, '{%s}Import' % ns, Project='$(VCTargetsPath)\Microsoft.Cpp.props') + ET.SubElement(proj_node, '{%s}ImportGroup' % ns, Label='ExtensionSettings') + for configuration in self.configurations: + prop_sheets_node = ET.SubElement(proj_node, '{%s}ImportGroup' % ns, Label='Configuration', + Condition="'$(Configuration)|$(Platform)'=='%s|%s'" % (configuration, platform)) + ET.SubElement(prop_sheets_node, '{%s}Import' % ns, + Project='$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props', + Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')", + Label='LocalAppDataPlatform') + + ET.SubElement(proj_node, '{%s}PropertyGroup' % ns, Label='UserMacros') + # VS IDE (at least "Peek definition") is allergic to paths like "C:/PROGRA~2/WI3CF2~1/10/Include/10.0.14393.0/um"; see + # https://developercommunity.visualstudio.com/content/problem/139659/vc-peek-definition-fails-to-navigate-to-windows-ki.html + # We need to convert to long paths here. Do this once, since it's time-consuming operation. + include_path_node_text = ';'.join(self.to_long_names(target.include)) + for cfg_name, cfg_targets in self.configurations.items(): + conf_node = ET.SubElement(proj_node, '{%s}PropertyGroup' % ns, + Condition="'$(Configuration)|$(Platform)'=='%s|%s'" % (cfg_name, platform)) + nmake_params = { + 'sh': os.path.join(self.gbuildparser.binpath, 'dash.exe'), + 'builddir': self.gbuildparser.builddir, + 'location': target.location, + 'makecmd': self.gbuildparser.makecmd, + 'target': target.target_name()} + nmake_build_node = ET.SubElement(conf_node, '{%s}NMakeBuildCommandLine' % ns) + nmake_build_node.text = cfg_targets['build'] % nmake_params + nmake_clean_node = ET.SubElement(conf_node, '{%s}NMakeCleanCommandLine' % ns) + nmake_clean_node.text = cfg_targets['clean'] % nmake_params + nmake_rebuild_node = ET.SubElement(conf_node, '{%s}NMakeReBuildCommandLine' % ns) + nmake_rebuild_node.text = cfg_targets['rebuild'] % nmake_params + nmake_output_node = ET.SubElement(conf_node, '{%s}NMakeOutput' % ns) + nmake_output_node.text = os.path.join(self.gbuildparser.instdir, 'program', 'soffice.bin') + nmake_defs_node = ET.SubElement(conf_node, '{%s}NMakePreprocessorDefinitions' % ns) + nmake_defs_node.text = ';'.join(self.defs_list(target.defs) + ['$(NMakePreprocessorDefinitions)']) + include_path_node = ET.SubElement(conf_node, '{%s}IncludePath' % ns) + include_path_node.text = include_path_node_text + additional_options_node = ET.SubElement(conf_node, '{%s}AdditionalOptions' % ns) + additional_options_node.text = cxxflags + + ET.SubElement(proj_node, '{%s}ItemDefinitionGroup' % ns) + + cxxobjects_node = ET.SubElement(proj_node, '{%s}ItemGroup' % ns) + for cxxobject in target.cxxobjects: + cxxabspath = os.path.join(self.gbuildparser.srcdir, cxxobject) + cxxfile = cxxabspath + '.cxx' + if os.path.isfile(cxxfile): + ET.SubElement(cxxobjects_node, '{%s}ClCompile' % ns, Include=cxxfile) + else: + print('Source %s in project %s does not exist' % (cxxfile, target.target_name())) + + cobjects_node = ET.SubElement(proj_node, '{%s}ItemGroup' % ns) + for cobject in target.cobjects: + cabspath = os.path.join(self.gbuildparser.srcdir, cobject) + cfile = cabspath + '.c' + if os.path.isfile(cfile): + ET.SubElement(cobjects_node, '{%s}ClCompile' % ns, Include=cfile) + else: + print('Source %s in project %s does not exist' % (cfile, target.target_name())) + + includes_node = ET.SubElement(proj_node, '{%s}ItemGroup' % ns) + for cxxobject in target.cxxobjects: + include_abs_path = os.path.join(self.gbuildparser.srcdir, cxxobject) + hxxfile = include_abs_path + '.hxx' + if os.path.isfile(hxxfile): + ET.SubElement(includes_node, '{%s}ClInclude' % ns, Include=hxxfile) + # Few files have corresponding .h files + hfile = include_abs_path + '.h' + if os.path.isfile(hfile): + ET.SubElement(includes_node, '{%s}ClInclude' % ns, Include=hfile) + for cobject in target.cobjects: + include_abs_path = os.path.join(self.gbuildparser.srcdir, cobject) + hfile = include_abs_path + '.h' + if os.path.isfile(hfile): + ET.SubElement(includes_node, '{%s}ClInclude' % ns, Include=hfile) + ET.SubElement(proj_node, '{%s}Import' % ns, Project='$(VCTargetsPath)\Microsoft.Cpp.targets') + ET.SubElement(proj_node, '{%s}ImportGroup' % ns, Label='ExtensionTargets') + self.write_pretty_xml(proj_node, project_path) + self.write_filters(project_path + '.filters', + os.path.join(self.gbuildparser.srcdir, os.path.basename(target.location)), + [cxx_node.get('Include') for cxx_node in cxxobjects_node.findall('{%s}ClCompile' % ns)], + [c_node.get('Include') for c_node in cobjects_node.findall('{%s}ClCompile' % ns)], + [include_node.get('Include') for include_node in includes_node.findall('{%s}ClInclude' % ns)]) + return project_guid + + def get_filter(self, module_dir, proj_file): + return '\\'.join(os.path.relpath(proj_file, module_dir).split('/')[:-1]) + + def get_subfilters(self, proj_filter): + parts = proj_filter.split('\\') + subfilters = set([proj_filter]) if proj_filter else set() + for i in range(1, len(parts)): + subfilters.add('\\'.join(parts[:i])) + return subfilters + + def write_pretty_xml(self, node, file_path): + xml_str = ET.tostring(node, encoding='unicode') + pretty_str = minidom.parseString(xml_str).toprettyxml(encoding='utf-8') + with open(file_path, 'w') as f: + f.write(pretty_str.decode()) + + def add_nodes(self, files_node, module_dir, tag, project_files): + ns = 'http://schemas.microsoft.com/developer/msbuild/2003' + filters = set() + for project_file in project_files: + file_node = ET.SubElement(files_node, tag, Include=project_file) + if os.path.commonprefix([module_dir, project_file]) == module_dir: + project_filter = self.get_filter(module_dir, project_file) + filter_node = ET.SubElement(file_node, '{%s}Filter' % ns) + filter_node.text = project_filter + filters |= self.get_subfilters(project_filter) + return filters + + def write_filters(self, filters_path, module_dir, cxx_files, c_files, include_files): + ns = 'http://schemas.microsoft.com/developer/msbuild/2003' + ET.register_namespace('', ns) + proj_node = ET.Element('{%s}Project' % ns, ToolsVersion='4.0') + filters = set() + compiles_node = ET.SubElement(proj_node, '{%s}ItemGroup' % ns) + filters |= self.add_nodes(compiles_node, module_dir, '{%s}ClCompile' % ns, cxx_files) + filters |= self.add_nodes(compiles_node, module_dir, '{%s}ClCompile' % ns, c_files) + include_node = ET.SubElement(proj_node, '{%s}ItemGroup' % ns) + filters |= self.add_nodes(include_node, module_dir, '{%s}ClInclude' % ns, include_files) + + filters_node = ET.SubElement(proj_node, '{%s}ItemGroup' % ns) + for proj_filter in filters: + filter_node = ET.SubElement(filters_node, '{%s}Filter' % ns, Include=proj_filter) + filter_id_node = ET.SubElement(filter_node, '{%s}UniqueIdentifier' % ns) + filter_id_node.text = '{%s}' % str(uuid.uuid4()) + self.write_pretty_xml(proj_node, filters_path) + + +class QtCreatorIntegrationGenerator(IdeIntegrationGenerator): + + def __init__(self, gbuildparser, ide): + IdeIntegrationGenerator.__init__(self, gbuildparser, ide) + self.target_by_location = {} + for target in set(self.gbuildparser.libs) | set(self.gbuildparser.exes) | set(self.gbuildparser.tests): + if target.location not in self.target_by_location: + self.target_by_location[target.location] = set() + self.target_by_location[target.location] |= set([target]) + + self._do_log = False # set to 'True' to activate log of QtCreatorIntegrationGenerator + if self._do_log: + qtlog_path = os.path.abspath('../qtlog_.txt') + self.qtlog = open(qtlog_path, 'w') + + def _log(self, message): + if self._do_log: + self.qtlog.write(message) + + def log_close(self): + if self._do_log: + self.qtlog.close() + + def generate_build_configs(self, lib_folder): + module_folder = os.path.join(self.base_folder, lib_folder) + xml = "" + # In QtCreator UI, build configs are listed alphabetically, + # so it can be different from the creation order. + # So we prefix the names with the index. + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '0', + 'base_folder': module_folder, + 'arg': "", + 'name': "1-Build %s" % lib_folder, + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '1', + 'base_folder': module_folder, + 'arg': "unitcheck", + 'name': "2-Local tests -- quick tests (unitcheck)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '2', + 'base_folder': module_folder, + 'arg': "unitcheck slowcheck screenshot", + 'name': "3-Local tests -- slow tests (unitcheck, slowcheck, screenshot)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '3', + 'base_folder': module_folder, + 'arg': "unitcheck slowcheck screenshot subsequentcheck", + 'name': "4-Local tests -- integration tests (unitcheck, slowcheck, screenshot, subsequentcheck)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '4', + 'base_folder': self.base_folder, + 'arg': "unitcheck", + 'name': "5-Global tests -- quick tests (unitcheck)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '5', + 'base_folder': self.base_folder, + 'arg': "unitcheck slowcheck screenshot", + 'name': "6-Global tests -- slow tests (unitcheck, slowcheck, screenshot)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '6', + 'base_folder': self.base_folder, + 'arg': "unitcheck slowcheck screenshot subsequentcheck", + 'name': "7-Global tests -- integration tests (unitcheck, slowcheck, screenshot, subsequentcheck)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '7', + 'base_folder': self.base_folder, + 'arg': "build-nocheck", + 'name': "8-Global build -- nocheck", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '8', + 'base_folder': self.base_folder, + 'arg': "", + 'name': "9-Global build", + } + + xml += QtCreatorIntegrationGenerator.build_configs_count_template % { + 'nb': '9', + } + return xml + + def generate_meta_build_configs(self): + xml = "" + # In QtCreator UI, build configs are listed alphabetically, + # so it can be different from the creation order. + # So we prefix the names with the index. + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '0', + 'base_folder': self.base_folder, + 'arg': "", + 'name': "01-Global Build", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '1', + 'base_folder': self.base_folder, + 'arg': "unitcheck", + 'name': "02-Global tests -- quick tests (unitcheck)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '2', + 'base_folder': self.base_folder, + 'arg': "unitcheck slowcheck screenshot", + 'name': "03-Global tests -- slow tests (unitcheck, slowcheck, screenshot)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '3', + 'base_folder': self.base_folder, + 'arg': "unitcheck slowcheck screenshot subsequentcheck", + 'name': "04-Global tests -- integration tests (unitcheck, slowcheck, screenshot, subsequentcheck)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '4', + 'base_folder': self.base_folder, + 'arg': "perfcheck", + 'name': "05-Global tests -- performance tests (perfcheck)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '5', + 'base_folder': self.base_folder, + 'arg': "check", + 'name': "06-Global tests -- tests (check)", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '6', + 'base_folder': self.base_folder, + 'arg': "build-nocheck", + 'name': "07-Global build -- nocheck", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '7', + 'base_folder': self.base_folder, + 'arg': "build-l10n-only", + 'name': "08-Global build -- build-l10n-only", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '8', + 'base_folder': self.base_folder, + 'arg': "build-non-l10n-only", + 'name': "09-Global build -- build-non-l10n-only", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '9', + 'base_folder': self.base_folder, + 'arg': "clean", + 'name': "10-Global build -- clean", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '10', + 'base_folder': self.base_folder, + 'arg': "clean-build", + 'name': "11-Global build -- clean-build", + } + xml += QtCreatorIntegrationGenerator.build_configs_template % { + 'index': '11', + 'base_folder': self.base_folder, + 'arg': "clean-host", + 'name': "12-Global build -- clean-host", + } + xml += QtCreatorIntegrationGenerator.build_configs_count_template % { + 'nb': '12', + } + return xml + + # By default, QtCreator creates 2 BuildStepList : "Build" et "Clean" + # but the "clean" can be empty. + build_configs_template = """ + <valuemap type="QVariantMap" key="ProjectExplorer.Target.BuildConfiguration.%(index)s"> + <value type="QString" key="ProjectExplorer.BuildConfiguration.BuildDirectory">%(base_folder)s</value> + + <valuemap type="QVariantMap" key="ProjectExplorer.BuildConfiguration.BuildStepList.0"> + + <valuemap type="QVariantMap" key="ProjectExplorer.BuildStepList.Step.0"> + <value type="bool" key="ProjectExplorer.BuildStep.Enabled">true</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Make</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">Qt4ProjectManager.MakeStep</value> + <valuelist type="QVariantList" key="Qt4ProjectManager.MakeStep.AutomaticallyAddedMakeArguments"> + <value type="QString">-w</value> + <value type="QString">-r</value> + </valuelist> + <value type="bool" key="Qt4ProjectManager.MakeStep.Clean">false</value> + <value type="QString" key="Qt4ProjectManager.MakeStep.MakeArguments">%(arg)s</value> + <value type="QString" key="Qt4ProjectManager.MakeStep.MakeCommand"></value> + </valuemap> + + <value type="int" key="ProjectExplorer.BuildStepList.StepsCount">1</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Build</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">ProjectExplorer.BuildSteps.Build</value> + </valuemap> + + <value type="int" key="ProjectExplorer.BuildConfiguration.BuildStepListCount">1</value> + <value type="bool" key="ProjectExplorer.BuildConfiguration.ClearSystemEnvironment">false</value> + <valuelist type="QVariantList" key="ProjectExplorer.BuildConfiguration.UserEnvironmentChanges"/> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">%(name)s</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">Qt4ProjectManager.Qt4BuildConfiguration</value> + <value type="int" key="Qt4ProjectManager.Qt4BuildConfiguration.BuildConfiguration">%(index)s</value> + <value type="bool" key="Qt4ProjectManager.Qt4BuildConfiguration.UseShadowBuild">true</value> + </valuemap> + """ + + build_configs_count_template = """ + <!-- nb build configurations --> + <value type="int" key="ProjectExplorer.Target.BuildConfigurationCount">%(nb)s</value> + """ + + def generate_deploy_configs(self, lib_folder): + xml = QtCreatorIntegrationGenerator.deploy_configs_template % {} + return xml + + deploy_configs_template = """ + <valuemap type="QVariantMap" key="ProjectExplorer.Target.DeployConfiguration.0"> + <valuemap type="QVariantMap" key="ProjectExplorer.BuildConfiguration.BuildStepList.0"> + <value type="int" key="ProjectExplorer.BuildStepList.StepsCount">0</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Deploy</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">ProjectExplorer.BuildSteps.Deploy</value> + </valuemap> + <value type="int" key="ProjectExplorer.BuildConfiguration.BuildStepListCount">1</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Deploy locally</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">ProjectExplorer.DefaultDeployConfiguration</value> + </valuemap> + <value type="int" key="ProjectExplorer.Target.DeployConfigurationCount">1</value> + """ + + def generate_run_configs(self, lib_folder): + + # If we use 'soffice', it's ok only for "Run", not for "Debug". + # So we put "soffice.bin" that is ok for both. + loexec = "%s/instdir/program/soffice.bin" % self.base_folder + xml = QtCreatorIntegrationGenerator.run_configs_template % { + 'loexec': loexec, + 'workdir': self.base_folder + } + return xml + + run_configs_template = """ + <valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.0"> + <valuelist type="QVariantList" key="Analyzer.Valgrind.AddedSuppressionFiles"/> + <value type="bool" key="Analyzer.Valgrind.Callgrind.CollectBusEvents">false</value> + <value type="bool" key="Analyzer.Valgrind.Callgrind.CollectSystime">false</value> + <value type="bool" key="Analyzer.Valgrind.Callgrind.EnableBranchSim">false</value> + <value type="bool" key="Analyzer.Valgrind.Callgrind.EnableCacheSim">false</value> + <value type="bool" key="Analyzer.Valgrind.Callgrind.EnableEventToolTips">true</value> + <value type="double" key="Analyzer.Valgrind.Callgrind.MinimumCostRatio">0.01</value> + <value type="double" key="Analyzer.Valgrind.Callgrind.VisualisationMinimumCostRatio">10</value> + <value type="bool" key="Analyzer.Valgrind.FilterExternalIssues">true</value> + <value type="int" key="Analyzer.Valgrind.LeakCheckOnFinish">1</value> + <value type="int" key="Analyzer.Valgrind.NumCallers">25</value> + <valuelist type="QVariantList" key="Analyzer.Valgrind.RemovedSuppressionFiles"/> + <value type="int" key="Analyzer.Valgrind.SelfModifyingCodeDetection">1</value> + <value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value> + <value type="bool" key="Analyzer.Valgrind.ShowReachable">false</value> + <value type="bool" key="Analyzer.Valgrind.TrackOrigins">true</value> + <value type="QString" key="Analyzer.Valgrind.ValgrindExecutable">valgrind</value> + <valuelist type="QVariantList" key="Analyzer.Valgrind.VisibleErrorKinds"> + <value type="int">0</value> + <value type="int">1</value> + <value type="int">2</value> + <value type="int">3</value> + <value type="int">4</value> + <value type="int">5</value> + <value type="int">6</value> + <value type="int">7</value> + <value type="int">8</value> + <value type="int">9</value> + <value type="int">10</value> + <value type="int">11</value> + <value type="int">12</value> + <value type="int">13</value> + <value type="int">14</value> + </valuelist> + <value type="int" key="PE.EnvironmentAspect.Base">2</value> + <valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/> + + <value type="QString" key="ProjectExplorer.CustomExecutableRunConfiguration.Arguments"></value> + <value type="QString" key="ProjectExplorer.CustomExecutableRunConfiguration.Executable">%(loexec)s</value> + <value type="bool" key="ProjectExplorer.CustomExecutableRunConfiguration.UseTerminal">false</value> + <value type="QString" key="ProjectExplorer.CustomExecutableRunConfiguration.WorkingDirectory">%(workdir)s</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Run libreoffice/instdir/program/soffice</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">ProjectExplorer.CustomExecutableRunConfiguration</value> + <value type="uint" key="RunConfiguration.QmlDebugServerPort">3768</value> + <value type="bool" key="RunConfiguration.UseCppDebugger">false</value> + <value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value> + <value type="bool" key="RunConfiguration.UseMultiProcess">false</value> + <value type="bool" key="RunConfiguration.UseQmlDebugger">false</value> + <value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value> + + </valuemap> + <value type="int" key="ProjectExplorer.Target.RunConfigurationCount">1</value> + """ + + def generate_pro_user_content(self, lib_folder): + + build_configs = self.generate_build_configs(lib_folder) + deploy_configs = self.generate_deploy_configs(lib_folder) + run_configs = self.generate_run_configs(lib_folder) + + xml = QtCreatorIntegrationGenerator.pro_user_template % { + 'build_configs': build_configs, + 'deploy_configs': deploy_configs, + 'run_configs': run_configs, + } + return xml + + def generate_meta_pro_user_content(self): + + build_configs = self.generate_meta_build_configs() + deploy_configs = self.generate_deploy_configs("") + run_configs = self.generate_run_configs("") + + xml = QtCreatorIntegrationGenerator.pro_user_template % { + 'build_configs': build_configs, + 'deploy_configs': deploy_configs, + 'run_configs': run_configs, + } + return xml + + pro_user_template = """<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE QtCreatorProject> +<!-- Written by QtCreator 3.1.1, 2015-05-14T15:54:34. --> +<qtcreator> + <data> + <variable>ProjectExplorer.Project.ActiveTarget</variable> + <value type="int">0</value> + </data> + + <!-- editor settings --> + <data> + <variable>ProjectExplorer.Project.EditorSettings</variable> + <valuemap type="QVariantMap"> + <value type="bool" key="EditorConfiguration.AutoIndent">true</value> + <value type="bool" key="EditorConfiguration.AutoSpacesForTabs">false</value> + <value type="bool" key="EditorConfiguration.CamelCaseNavigation">true</value> + <valuemap type="QVariantMap" key="EditorConfiguration.CodeStyle.0"> + <value type="QString" key="language">Cpp</value> + <valuemap type="QVariantMap" key="value"> + <value type="QByteArray" key="CurrentPreferences">CppGlobal</value> + </valuemap> + </valuemap> + <valuemap type="QVariantMap" key="EditorConfiguration.CodeStyle.1"> + <value type="QString" key="language">QmlJS</value> + <valuemap type="QVariantMap" key="value"> + <value type="QByteArray" key="CurrentPreferences">QmlJSGlobal</value> + </valuemap> + </valuemap> + <value type="int" key="EditorConfiguration.CodeStyle.Count">2</value> + <value type="QByteArray" key="EditorConfiguration.Codec">UTF-8</value> + <value type="bool" key="EditorConfiguration.ConstrainTooltips">false</value> + <value type="int" key="EditorConfiguration.IndentSize">4</value> + <value type="bool" key="EditorConfiguration.KeyboardTooltips">false</value> + <value type="int" key="EditorConfiguration.MarginColumn">80</value> + <value type="bool" key="EditorConfiguration.MouseHiding">true</value> + <value type="bool" key="EditorConfiguration.MouseNavigation">true</value> + <value type="int" key="EditorConfiguration.PaddingMode">1</value> + <value type="bool" key="EditorConfiguration.ScrollWheelZooming">true</value> + <value type="bool" key="EditorConfiguration.ShowMargin">false</value> + <value type="int" key="EditorConfiguration.SmartBackspaceBehavior">1</value> + <value type="bool" key="EditorConfiguration.SpacesForTabs">true</value> + <value type="int" key="EditorConfiguration.TabKeyBehavior">0</value> + <value type="int" key="EditorConfiguration.TabSize">8</value> + <value type="bool" key="EditorConfiguration.UseGlobal">true</value> + <value type="int" key="EditorConfiguration.Utf8BomBehavior">1</value> + <value type="bool" key="EditorConfiguration.addFinalNewLine">true</value> + <value type="bool" key="EditorConfiguration.cleanIndentation">true</value> + <value type="bool" key="EditorConfiguration.cleanWhitespace">true</value> + <value type="bool" key="EditorConfiguration.inEntireDocument">false</value> + </valuemap> + </data> + + <data> + <variable>ProjectExplorer.Project.PluginSettings</variable> + <valuemap type="QVariantMap"/> + </data> + + <!-- target --> + <data> + <variable>ProjectExplorer.Project.Target.0</variable> + <valuemap type="QVariantMap"> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Desktop</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Desktop</value> + <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">{0701de51-c96e-4e4f-85c3-e70b223c5076}</value> + <value type="int" key="ProjectExplorer.Target.ActiveBuildConfiguration">0</value> + <value type="int" key="ProjectExplorer.Target.ActiveDeployConfiguration">0</value> + <value type="int" key="ProjectExplorer.Target.ActiveRunConfiguration">0</value> + + <!-- build configurations --> + %(build_configs)s + + <!-- deploy configurations --> + %(deploy_configs)s + + <!-- plugin settings --> + <valuemap type="QVariantMap" key="ProjectExplorer.Target.PluginSettings"/> + + <!-- run configurations --> + %(run_configs)s + + </valuemap> + </data> + <!-- nb targets : 1 --> + <data> + <variable>ProjectExplorer.Project.TargetCount</variable> + <value type="int">1</value> + </data> + <data> + <variable>ProjectExplorer.Project.Updater.EnvironmentId</variable> + <value type="QByteArray">{5abcafed-86f6-49f6-b1cb-380fadd21211}</value> + </data> + <data> + <variable>ProjectExplorer.Project.Updater.FileVersion</variable> + <value type="int">15</value> + </data> +</qtcreator> +""" + + def remove_qt_files(self): + + def do_remove_file(loc, afile): + try: + os.remove(os.path.join(loc, afile)) + self._log("removed %s\n" % afile) + except OSError: + self._log("unable to remove %s\n" % afile) + + do_remove_file(self.base_folder, "lo.pro") + do_remove_file(self.base_folder, "lo.pro.user") + for location in self.target_by_location: + for f in os.listdir(location): + if f.endswith('.pro') or f.endswith('.pro.user'): + do_remove_file(location, f) + + def get_source_extension(self, src_file): + path = os.path.join(self.base_folder, src_file) + for ext in (".cxx", ".cpp", ".c", ".mm"): + if os.path.isfile(path + ext): + return ext + return "" + + def get_header_extension(self, src_file): + path = os.path.join(self.base_folder, src_file) + for ext in (".hxx", ".hpp", ".h"): + if os.path.isfile(path + ext): + return ext + return "" + + def build_data_libs(self): + + self.data_libs = {} + + all_libs = set(self.gbuildparser.libs) | set(self.gbuildparser.exes) | set(self.gbuildparser.tests) + for lib in all_libs: + self._log("\nlibrary : %s, loc=%s" % (lib.short_name(), lib.location)) + lib_name = os.path.basename(lib.location) + lib_folder = os.path.relpath(lib.location, self.base_folder) + + def lopath(path): + if platform =="cygwin": + # absolute paths from GbuildToJson are Windows paths, + # so convert everything to such ones + abs_path = path + if not ntpath.isabs(abs_path): + abs_path = ntpath.join(self.gbuildparser.srcdir, path) + return ntpath.relpath(abs_path, lib.location).replace('\\', '/') + + return os.path.relpath(path, lib.location) + + defines_list = [] + sources_list = [] + includepath_list = [] + # The explicit headers list is not mandatory : + # QtCreator just needs 'include_path_list' to find all headers files. + # But files listed in 'header_list' will be shown + # in a specific "Headers" folder in QtCreator's Project panel. + # We will list here only headers files of current lib. + headers_list = [] + for file_ in lib.cxxobjects: + # the file has no extension : search it + # self._log("\n file : %s" % file_) + ext = self.get_source_extension(file_) + if ext: + sources_list.append(lopath(file_ + ext)) + + # few cxxobject files have a header beside + ext = self.get_header_extension(file_) + if ext: + headers_list.append(lopath(file_ + ext)) + + cxxflags_list = [] + for cxxflag in lib.cxxflags: + # extract flag for C++ standard version + if cxxflag.startswith('-std'): + cxxflags_list.append(cxxflag) + + # List all include paths + for hdir in (lib.include + lib.include_sys): + hf_lopath = lopath(hdir) + includepath_list.append(hf_lopath) + + # List headers files from current lib + for hdir in lib.include: + if hdir.startswith(lib.location): + for dirpath, _, files in os.walk(hdir): + for hf in files: + if hf.endswith(('.h', '.hxx', '.hpp', '.hrc')): + hf_lopath = lopath(os.path.join(dirpath, hf)) + headers_list.append(hf_lopath) + + # List defines + for key, value in lib.defs.items(): + define = key + if value is not None: + define += '=' + value + defines_list.append(define) + + # All data are prepared, store them for the lib. + if lib_folder in self.data_libs: + self.data_libs[lib_folder]['sources'] |= set(sources_list) + self.data_libs[lib_folder]['headers'] |= set(headers_list) + self.data_libs[lib_folder]['cxxflags'] |= set(cxxflags_list) + self.data_libs[lib_folder]['includepath'] |= set(includepath_list) + self.data_libs[lib_folder]['defines'] |= set(defines_list) + else: + self.data_libs[lib_folder] = { + 'sources': set(sources_list), + 'headers': set(headers_list), + 'cxxflags': set(cxxflags_list), + 'includepath': set(includepath_list), + 'defines': set(defines_list), + 'loc': lib.location, + 'name': lib_name + } + + def emit(self): + + self.base_folder = self.gbuildparser.builddir + + # we remove existing '.pro' and '.pro.user' files + self.remove_qt_files() + + # for .pro files, we must explicitly list all files (.c, .h) + # so we can't reuse directly the same method than for kde integration. + self.build_data_libs() + + subdirs_list = self.data_libs.keys() + # Now we can create Qt files + for lib_folder in subdirs_list: + sources_list = sorted(self.data_libs[lib_folder]['sources']) + headers_list = sorted(self.data_libs[lib_folder]['headers']) + cxxflags_list = sorted(self.data_libs[lib_folder]['cxxflags']) + includepath_list = sorted(self.data_libs[lib_folder]['includepath']) + defines_list = sorted(self.data_libs[lib_folder]['defines']) + lib_loc = self.data_libs[lib_folder]['loc'] + lib_name = self.data_libs[lib_folder]['name'] + + sources = " \\\n".join(sources_list) + headers = " \\\n".join(headers_list) + cxxflags = " \\\n".join(cxxflags_list) + includepath = " \\\n".join(includepath_list) + defines = " \\\n".join(defines_list) + + # create .pro file + qt_pro_file = '%s/%s.pro' % (lib_loc, lib_name) + try: + content = QtCreatorIntegrationGenerator.pro_template % {'sources': sources, 'headers': headers, + 'cxxflags': cxxflags, 'includepath': includepath, 'defines': defines} + mode = 'w+' + with open(qt_pro_file, mode) as fpro: + fpro.write(content) + self._log("created %s\n" % qt_pro_file) + + except Exception as e: + print("ERROR : creating pro file=" + qt_pro_file, file=sys.stderr) + print(e, file=sys.stderr) + temp = traceback.format_exc() # .decode('utf8') + print(temp, file=sys.stderr) + print("\n\n", file=sys.stderr) + + # create .pro.user file + qt_pro_user_file = '%s/%s.pro.user' % (lib_loc, lib_name) + try: + with open(qt_pro_user_file, mode) as fprouser: + fprouser.write(self.generate_pro_user_content(lib_folder)) + self._log("created %s\n" % qt_pro_user_file) + + except Exception as e: + print("ERROR : creating pro.user file=" + qt_pro_user_file, file=sys.stderr) + print(e, file=sys.stderr) + temp = traceback.format_exc() + print(temp, file=sys.stderr) + print("\n\n", file=sys.stderr) + + # create meta .pro file (lists all sub projects) + qt_meta_pro_file = 'lo.pro' + try: + subdirs = " \\\n".join(sorted(subdirs_list)) + content = QtCreatorIntegrationGenerator.pro_meta_template % {'subdirs': subdirs} + with open(qt_meta_pro_file, 'w+') as fmpro: + fmpro.write(content) + + except Exception as e: + print("ERROR : creating lo.pro file=" + qt_meta_pro_file, file=sys.stderr) + print(e, file=sys.stderr) + temp = traceback.format_exc() + print(temp, file=sys.stderr) + print("\n\n", file=sys.stderr) + + # create meta .pro.user file + qt_meta_pro_user_file = 'lo.pro.user' + try: + with open(qt_meta_pro_user_file, mode) as fmprouser: + fmprouser.write(self.generate_meta_pro_user_content()) + self._log("created %s\n" % qt_meta_pro_user_file) + + except Exception as e: + print("ERROR : creating lo.pro.user file=" + qt_meta_pro_user_file, file=sys.stderr) + print(e, file=sys.stderr) + temp = traceback.format_exc() + print(temp, file=sys.stderr) + print("\n\n", file=sys.stderr) + + self.log_close() + + pro_template = """TEMPLATE = app +CONFIG += console +CONFIG -= app_bundle +CONFIG -= qt + +QMAKE_CXXFLAGS += %(cxxflags)s + +INCLUDEPATH += %(includepath)s + +SOURCES += %(sources)s + +HEADERS += %(headers)s + +DEFINES += %(defines)s + +""" + pro_meta_template = """TEMPLATE = subdirs + +SUBDIRS = %(subdirs)s +""" + + +def get_options(): + parser = argparse.ArgumentParser( + description='LibreOffice gbuild IDE project generator') + parser.add_argument('--ide', dest='ide', required=True, + help='the IDE to generate project files for') + parser.add_argument('--make', dest='makecmd', required=True, + help='the command to execute make') + return parser.parse_args() + + +if __name__ == '__main__': + args = get_options() + # FIXME: Hack + if args.makecmd == 'make': + args.makecmd = '/usr/bin/make' + + paths = {} + generators = { + 'codelite': CodeliteIntegrationGenerator, + 'eclipsecdt': EclipseCDTIntegrationGenerator, + 'kdevelop': KdevelopIntegrationGenerator, + 'xcode': XcodeIntegrationGenerator, + 'vs2017': VisualStudioIntegrationGenerator, + 'vs2019': VisualStudioIntegrationGenerator, + 'vim': VimIntegrationGenerator, + 'debug': DebugIntegrationGenerator, + 'qtcreator': QtCreatorIntegrationGenerator, + } + + if args.ide not in generators.keys(): + print("Invalid ide. valid values are %s" % ','.join(generators.keys())) + sys.exit(1) + + gbuildparser = GbuildParser(args.makecmd).parse() + + generators[args.ide](gbuildparser, args.ide).emit() + print("Successfully created the project files.") + +# Local Variables: +# indent-tabs-mode: nil +# End: +# +# vim: set et sw=4 ts=4: diff --git a/bin/gen-boost-headers b/bin/gen-boost-headers new file mode 100755 index 000000000..5fe0e3a20 --- /dev/null +++ b/bin/gen-boost-headers @@ -0,0 +1,67 @@ +#!/bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# generate a bunch of dummy headers that wrap the crappy boost headers and +# suppress a myriad of warnings; requires GCC's #include_next extension + +set -euo pipefail +IFS=$'\n\t' + +GENDIR=${SRCDIR}/external/boost/include + +rm -rf ${GENDIR} +mkdir ${GENDIR} + +# note: clucene contains a copy of half of boost, so ignore it too +# note: firebird contains a copy of half of boost, so ignore it too + +cat <(cd ${SRCDIR} && git grep -h '^# *include') \ + <(find ${WORKDIR}/UnpackedTarball/ -mindepth 1 -maxdepth 1 -type d \ + | grep -v boost \ + | grep -v clucene \ + | grep -v firebird \ + | xargs grep -hr '^# *include') \ + | grep -o '\bboost.*\.\(h\|hpp\|ipp\)' \ + | sort | uniq \ + | while read -r HEADER; do + mkdir -p "$(dirname ${GENDIR}/${HEADER})" + cat > "${GENDIR}/${HEADER}" << _EOF +/* generated by $0, do not edit! */ +#pragma once +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpragmas" /* first! for GCC */ +#pragma GCC diagnostic ignored "-Wunknown-warning-option" // second! for Clang 5 +#pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#pragma GCC diagnostic ignored "-Wdeprecated-copy" +#pragma GCC diagnostic ignored "-Wdeprecated-copy-dtor" +#pragma GCC diagnostic ignored "-Wextra" +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wimplicit-fallthrough" +#pragma GCC diagnostic ignored "-Winvalid-constexpr" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic ignored "-Wmicrosoft-unqualified-friend" +#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" +#pragma GCC diagnostic ignored "-Wparentheses" +#pragma GCC diagnostic ignored "-Wplacement-new" +#pragma GCC diagnostic ignored "-Wreturn-type" +#pragma GCC diagnostic ignored "-Wshadow" +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#pragma GCC diagnostic ignored "-Wtautological-constant-out-of-range-compare" +#pragma GCC diagnostic ignored "-Wtype-limits" +#pragma GCC diagnostic ignored "-Wundef" +#pragma GCC diagnostic ignored "-Wunused-local-typedefs" +#pragma GCC diagnostic ignored "-Wunused-macros" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wunused-variable" +#include_next <${HEADER}> +#pragma GCC diagnostic pop +_EOF + done + diff --git a/bin/gen-iwyu-dummy-lib b/bin/gen-iwyu-dummy-lib new file mode 100755 index 000000000..c7d64817d --- /dev/null +++ b/bin/gen-iwyu-dummy-lib @@ -0,0 +1,79 @@ +#!/bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Create a makefile that builds every non-generated header as a source file. +# This should help to ensure the headers are self-contained and don't +# impose unnecessary requirements (unnecessary includes) on client code. +# +# This script is fully compliant with the UNIX philosophy +# (and if you can't read it you are clearly not worthy) + +set -e + +iwyu_INCLUDES=$(grep -h -r ":$" "$BUILDDIR"/workdir/Dep/*Object* \ + | grep -v 'workdir\|config_host' | grep -v "^/usr" \ + | sed -e "s,^${SRCDIR}/,," | sed -e "s/:$//" | sort -u) + +iwyu_INCLUDEDIRS=$(echo "$iwyu_INCLUDES" | sed -e "s,/[^/]*$,," | grep -v "^include" | sort -u) + +iwyu_EXTERNALS=$(ls "$SRCDIR"/*/*Library*mk "$SRCDIR"/*/*Executable*mk \ + | xargs awk -f "$SRCDIR"/bin/gen-iwyu-dummy-lib.awk \ + | grep -v '$(\|)\|\\$\|apr\|breakpad\|bzip2\|expat_x64\|mDNSResponder\|serf\|zlib_x64') + +iwyu_DIR="$BUILDDIR"/iwyudummy/ +mkdir -p "$iwyu_DIR" + +{ + echo 'module_directory:=$(dir $(realpath $(firstword $(MAKEFILE_LIST))))' + echo "include ${SRCDIR}/solenv/gbuild/partial_build.mk" +} > "$iwyu_DIR"Makefile + +{ + echo '$(eval $(call gb_Module_Module,iwyudummy))' + echo '$(eval $(call gb_Module_add_targets,iwyudummy,StaticLibrary_iwyudummy))' +} > "$iwyu_DIR"Module_iwyudummy.mk + +{ + # prevent some common configuration errors + echo 'ifneq ($(COMPILER_PLUGINS),)' + echo ' $(call gb_Output_error,--enable-compiler-plugins does not work well with this: bailing out)' + echo 'endif' + + echo '$(eval $(call gb_StaticLibrary_StaticLibrary,iwyudummy))' + # clang will "compile" headers to .gch by default + echo '$(eval $(call gb_StaticLibrary_add_cxxflags,iwyudummy,-x c++ -D__cplusplus=201402L -D__STDC_VERSION__=201112L -Wno-unused-macros -Wno-unused-const-variable))' + echo '$(eval $(call gb_StaticLibrary_use_custom_headers,iwyudummy,officecfg/registry))' + echo '$(eval $(call gb_StaticLibrary_use_sdk_api,iwyudummy))' + echo '$(eval $(call gb_StaticLibrary_use_externals,iwyudummy,\' + for ext in ${iwyu_EXTERNALS}; do + echo "${ext} \\"; + done + echo '))' + + echo '$(eval $(call gb_StaticLibrary_set_include,iwyudummy,\' + echo '$$(INCLUDE) \' + for dir in ${iwyu_INCLUDEDIRS}; do + if echo "$dir" | grep ".*/inc/" &>/dev/null; then + iwyu_INCLUDEDIRS_EXTRA+=" ${dir%/inc/*}/inc" + fi + done + for dir in $(echo ${iwyu_INCLUDEDIRS_EXTRA} | sed -e "s/ /\n/g" | uniq) ${iwyu_INCLUDEDIRS}; do + echo "-I${SRCDIR}/${dir} \\"; + done + # it fails to find stddef.h? + echo "-I/usr/lib/clang/$(llvm-config --version)/include \\" + echo "))" + + echo '$(eval $(call gb_StaticLibrary__add_iwyu_headers,iwyudummy,\' + for hdr in ${iwyu_INCLUDES}; do + echo "${hdr} \\"; + done + echo '))' +} > "$iwyu_DIR"StaticLibrary_iwyudummy.mk + diff --git a/bin/gen-iwyu-dummy-lib.awk b/bin/gen-iwyu-dummy-lib.awk new file mode 100644 index 000000000..464d9515c --- /dev/null +++ b/bin/gen-iwyu-dummy-lib.awk @@ -0,0 +1,34 @@ +BEGIN { domatch = 0; } + +{ + if ($0 ~ /use_external(s)?,/ ) + { + if (index($0, "))")) + { + gsub(/.*,/, ""); + gsub(/\)+/, ""); + if (!($0 in exts)) + { + exts[$0]; + print $0; + } + } + else + { + domatch = 1; + } + } + else if ($0 ~ /\)\)/ ) + { + domatch = 0; + } + else if (domatch == 1) + { + if (!($1 in exts)) + { + exts[$1]; + print $1; + } + } +} + diff --git a/bin/generate-bash-completion.py b/bin/generate-bash-completion.py new file mode 100755 index 000000000..0702a3635 --- /dev/null +++ b/bin/generate-bash-completion.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +""" +Script to generate LibreOffice bash_completion file for the main applications +""" + +import argparse +import sys + +MASTERDOCS = ["sxg", "odm", "sgl"] + +BASEDOCS = ["odb"] + +CALCDOCS = ["sxc", "stc", "dif", "dbf", "xls", "xlw", "xlt", "rtf", "sdc", "vor", + "slk", "txt", "htm", "html", "wk1", "wks", "123", "xml", "ods", "ots", + "fods", "csv", "xlsb", "xlsm", "xlsx", "xltm", "xltx"] + +DRAWDOCS = ["sxd", "std", "dxf", "emf", "eps", "met", "pct", "sgf", "sgv", "sda", + "sdd", "vor", "svm", "wmf", "bmp", "gif", "jpg", "jpeg", "jfif", "fif", + "jpe", "pcd", "pcx", "pgm", "png", "ppm", "psd", "ras", "tga", "tif", + "tiff", "xbm", "xpm", "odg", "otg", "fodg", "odc", "odi", "sds", + "wpg", "svg", "vdx", "vsd", "vsdm", "vsdx"] + +IMPRESSDOCS = ["sxi", "sti", "ppt", "pps", "pot", "sxd", "sda", "sdd", "sdp", + "vor", "cgm", "odp", "otp", "fodp", "ppsm", "ppsx", "pptm", "pptx", + "potm", "potx"] + +MATHDOCS = ["sxm", "smf", "mml", "odf"] + +WEBDOCS = ["htm", "html", "stw", "txt", "vor", "oth"] + +WRITERDOCS = ["doc", "dot", "rtf", "sxw", "stw", "sdw", "vor", "txt", "htm?", + "xml", "wp", "wpd", "wps", "odt", "ott", "fodt", "docm", "docx", + "dotm", "dotx"] + +TEMPLATES = ["stw", "dot", "vor", "stc", "xlt", "sti", "pot", "std", "stw", + "dotm", "dotx", "potm", "potx", "xltm", "xltx"] + +ALLDOCS = MASTERDOCS + BASEDOCS + CALCDOCS + DRAWDOCS + IMPRESSDOCS + MATHDOCS + WEBDOCS + WRITERDOCS + TEMPLATES + +EXTENSIONS = ["oxt"] + + +class App(object): + def __init__(self, name, compat_name, suffix_list): + self.name = name + self.compat_name = compat_name + self.suffix_list = suffix_list + + +class SetAppCompatName(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, True) + for app in APPS.values(): + app.name = app.compat_name + + +class SetAppName(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + APPS[self.dest].name = values + + +# default names of lowrappers +# use "" for name if you want to disable any wrapper +APPS = { + 'office': App("libreoffice", 'openoffice', ALLDOCS), # libreoffice should contain all + 'office_short': App("loffice", 'ooffice', ALLDOCS), # libreoffice should contain all + 'master': App("", '', MASTERDOCS), + 'base': App("lobase", 'oobase', BASEDOCS), + 'calc': App("localc", 'oocalc', CALCDOCS), + 'draw': App("lodraw", 'oodraw', DRAWDOCS), + 'impress': App("loimpress", 'ooimpress', IMPRESSDOCS), + 'math': App("lomath", 'oomath', MATHDOCS), + 'template': App("lofromtemplate", 'oofromtemplate', TEMPLATES), + 'unopkg': App("unopkg", 'unopkg', EXTENSIONS), # unopkg is a standalone tool + 'web': App("loweb", 'ooweb', WEBDOCS), + 'writer': App("lowriter", 'oowriter', WRITERDOCS + MASTERDOCS) +} + + +def check_open(filename, mode): + try: + with open(filename, mode): + pass + except OSError as e: + mode = 'reading' if mode == 'r' else 'writing' + sys.exit("Error: can't open %s for %s: %s" % (filename, mode, e)) + + +def print_app_suffixes_check(out, app): + if not app.suffix_list: + sys.exit('Error: No suffix defined for %s' % app.name) + + suffix_str = '|'.join(['%s|%s' % (s, s.upper()) for s in app.suffix_list]) + out.write(" %s)\t\te=\'!*.+(%s)\' ;;\n" % (app.name, suffix_str)) + + +def print_suffixes_check(out): + for app in APPS.values(): + if not app.name: # skip the disabled wrapper + continue + print_app_suffixes_check(out, app) + + +def main(): + parser = argparse.ArgumentParser(description='Script to Generate bash completion for LO wrappers', + epilog='The other options allows to redefine the wrapper names.\n' + 'The value "" can be used to disable any wrapper.', + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('input_file') + parser.add_argument('output_file') + parser.add_argument('--binsuffix', metavar='suffix', + help='defines a suffix that is added after each wrapper') + parser.add_argument('--compat-oowrappers', metavar='', nargs=0, action=SetAppCompatName, default=False, + help='set wrapper names to the old default oo* wrapper names') + for app in APPS: + parser.add_argument('--%s' % app, metavar='wrapper_name', action=SetAppName) + + args = parser.parse_args() + + check_open(args.input_file, 'r') + check_open(args.output_file, 'w') + + # add binsuffix + if args.binsuffix: + for app in APPS.values(): + if app.name: + app.name += args.binsuffix + + if args.compat_oowrappers: + office_shell_function = '_ooexp_' + else: + office_shell_function = '_loexp_' + + # the last app will be printed without the final backslash + apps_to_print = ' \\\n'.join(['\t\t\t\t\t%s' % app.name for app in APPS.values() if app.name]) + + with open(args.input_file, 'r') as in_fh, open(args.output_file, 'w') as out_fh: + for line in in_fh: + line = line.replace('@OFFICE_SHELL_FUNCTION@', office_shell_function) + if '@BASH_COMPLETION_SUFFIXES_CHECKS@' in line: + print_suffixes_check(out_fh) + elif '@BASH_COMPLETION_OOO_APPS@' in line: + if not apps_to_print: + sys.exit('Error: No LO wrapper was selected') + out_fh.write('%s\n' % apps_to_print) + else: + out_fh.write(line) + + +if __name__ == '__main__': + main() + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/get-bugzilla-attachments-by-mimetype b/bin/get-bugzilla-attachments-by-mimetype new file mode 100755 index 000000000..1d1f45165 --- /dev/null +++ b/bin/get-bugzilla-attachments-by-mimetype @@ -0,0 +1,584 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# This digs through a pile of bugzilla's and populates the cwd with a big +# collection of bug-docs in per-filetype dirs with bug-ids as names with +# prefixes to indicate which bug-tracker, e.g. +# +# fdo-bugid-X.suffix +# rhbz-bugid-X.suffix +# moz-bugid-X.suffix +# +# where X is the n'th attachment of that type in the bug +# +# The results are stored in the current directory, categorized by the +# extension of the downloaded file. When a file already exists, it is assumed +# it is already downloaded by a previous run, and up-to-date. + +from __future__ import print_function +import feedparser +import base64 +import datetime +import glob +import re +import os, os.path +import stat +import sys +import threading +try: + import queue +except: + import Queue as queue +try: + from urllib.request import urlopen +except: + from urllib import urlopen +try: + import xmlrpc.client as xmlrpclib +except: + import xmlrpclib +from xml.dom import minidom +from xml.sax.saxutils import escape + +def urlopen_retry(url): + maxretries = 3 + for i in range(maxretries + 1): + try: + return urlopen(url) + except IOError as e: + print("caught IOError: " + str(e)) + if maxretries == i: + raise + print("retrying...") + +def get_from_bug_url_via_xml(url, mimetype, prefix, suffix): + id = url.rsplit('=', 2)[1] + print("id is " + prefix + id + " " + suffix) + print("parsing " + id) + sock = urlopen_retry(url+"&ctype=xml") + dom = minidom.parse(sock) + sock.close() + attachmentid=0 + for attachment in dom.getElementsByTagName('attachment'): + attachmentid += 1 + print(" mimetype is", end=' ') + for node in attachment.childNodes: + if node.nodeName == 'type': + # check if attachment is deleted + if not node.firstChild: + print('deleted attachment, skipping') + continue + + print(node.firstChild.nodeValue, end=' ') + if node.firstChild.nodeValue.lower() != mimetype.lower(): + print('skipping') + break + elif node.nodeName == 'data': + # check if attachment is deleted (i.e. https://bugs.kde.org/show_bug.cgi?id=53343&ctype=xml) + if not node.firstChild: + print('deleted attachment, skipping') + continue + + download = suffix + '/' +prefix + id + '-' + str(attachmentid) + '.' + suffix + if os.path.isfile(download): + print("assuming " + download + " is up to date") + continue + + # prevent re-downloading FDO attachments from TDF + if prefix == "tdf" and int(id) < 88776: + fdodownload = download.replace("tdf", "fdo") + if os.path.isfile(fdodownload): + print("assuming FDO " + fdodownload + " is up to date") + continue + + print('downloading as ' + download) + tmpfile = download + ".tmp" + f = open(tmpfile, 'wb') + f.write(base64.b64decode(node.firstChild.nodeValue)) + f.close() + os.rename(tmpfile, download) + break + +def get_novell_bug_via_xml(url, mimetype, prefix, suffix): + id = url.rsplit('=', 2)[1] + print("id is " + prefix + id + " " + suffix) + print("parsing " + id) + sock = urlopen_retry(url+"&ctype=xml") + dom = minidom.parse(sock) + sock.close() + attachmentid=0 + for comment in dom.getElementsByTagName('thetext'): + commentText = comment.firstChild.nodeValue + match = re.search(r".*Created an attachment \(id=([0-9]+)\)", commentText) + if not match: + continue + + attachmentid += 1 + + download = suffix + '/' + prefix + id + '-' + str(attachmentid) + '.' + suffix + if os.path.isfile(download): + print("assuming " + download + " is up to date") + continue + + realAttachmentId = match.group(1) + handle = urlopen_retry(novellattach + realAttachmentId) + if not handle: + print("attachment %s is not accessible" % realAttachmentId) + continue + print(" mimetype is", end=' ') + + info = handle.info() + if info.get_content_type: + remoteMime = info.get_content_type() + else: + remoteMime = info.gettype() + print(remoteMime, end=' ') + if remoteMime != mimetype: + print("skipping") + continue + + print('downloading as ' + download) + tmpfile = download + ".tmp" + f = open(tmpfile, 'wb') + f.write(handle.read()) + f.close() + os.rename(tmpfile, download) + +def create_query(mimetype): + query = dict() + query['query_format']='advanced' + query['field0-0-0']='attachments.mimetype' + query['type0-0-0']='equals' + query['value0-0-0']=mimetype + return query + +def get_downloaded_files(prefix, suffix): + return glob.glob(os.path.join(suffix, '%s*.%s' % (prefix, suffix))) + +def get_file_bz_ids(files, prefix): + return set([os.path.basename(f).split('-')[0].replace(prefix, '', 1) for f in files]) + +def get_changed_date(files): + newest = max([os.stat(f)[stat.ST_MTIME] for f in files]) + # Subtract a day to avoid timezone differences. The worst thing that + # can happen is that we are going to process more bugs than necessary. + return datetime.date.fromtimestamp(newest - 24 * 60 * 60) + +def get_through_rpc_query(rpcurl, showurl, mimetype, prefix, suffix): + try: + os.mkdir(suffix) + except: + pass + + def process(query, full, have=[]): + try: + proxy = xmlrpclib.ServerProxy(rpcurl) + result = proxy.Bug.search(query) + bugs = result['bugs'] + print(str(len(bugs)) + ' bugs to process') + + if full: + available = set([str(bug['id']) for bug in bugs]) + # we already have files from all available bugs + if available.difference(set(have)) == set(): + print("assuming all downloaded files are up to date") + return + + for bug in bugs: + url = showurl + str(bug['id']) + get_from_bug_url_via_xml(url, mimetype, prefix, suffix) + except xmlrpclib.Fault as err: + print("A fault occurred") + print("Fault code: %s" % err.faultCode) + print(err.faultString) + + query = create_query(mimetype) + query['column_list']='bug_id' + + files = get_downloaded_files(prefix, suffix) + + if files != []: + print('looking for updated bugs having %s attachment(s)' % mimetype) + query_changed = query.copy() + query_changed['field0-1-0'] = 'days_elapsed' + query_changed['type0-1-0'] = 'lessthaneq' + query_changed['value0-1-0'] = str((datetime.date.today() - get_changed_date(files)).days) + process(query_changed, False) + + print('looking for all bugs having %s attachment(s)' % mimetype) + process(query, True, get_file_bz_ids(files, prefix)) + +def get_through_rss_query(queryurl, mimetype, prefix, suffix): + try: + os.mkdir(suffix) + except: + pass + + #Getting detailed bug information and downloading an attachment body is not possible without logging in to Novell bugzilla + #get_novell_bug_via_xml function is a workaround for that situation + get_bug_function = get_novell_bug_via_xml if prefix == "novell" else get_from_bug_url_via_xml + + def process(query, full, have=[]): + url = queryurl + '?' + '&'.join(['='.join(kv) for kv in query.items()]) + print('url is ' + url) + d = feedparser.parse(url) + print(str(len(d['entries'])) + ' bugs to process') + + entries = [] + for entry in d['entries']: + bugid = entry['id'].split('=')[-1] + entries.append(entry) + + if full: + available = set([str(entry['id'].split('=')[-1]) for entry in entries]) + # we already have files from all available bugs + if available.difference(set(have)) == set(): + print("assuming all downloaded files are up to date") + return + + for entry in entries: + try: + get_bug_function(entry['id'], mimetype, prefix, suffix) + except KeyboardInterrupt: + raise # Ctrl+C should work + except: + print(entry['id'] + " failed: " + str(sys.exc_info()[0])) + pass + + query = create_query(escape(mimetype.replace("+","%2B"))) + query['ctype'] = 'rss' + + files = get_downloaded_files(prefix, suffix) + + if files != []: + print('looking for updated bugs having %s attachment(s)' % mimetype) + query_changed = query.copy() + query_changed['field0-1-0'] = 'delta_ts' + query_changed['type0-1-0'] = 'greaterthaneq' + query_changed['value0-1-0'] = get_changed_date(files).isoformat() + process(query_changed, False) + + print('looking for all bugs having %s attachment(s)' % mimetype) + process(query, True, get_file_bz_ids(files, prefix)) + +#since searching bugs having attachments with specific mimetypes is not available in launchpad API +#we're iterating over all bugs of the most interesting source packages +launchpad_pkgs = ( + "abiword", + "calibre", + "calligra", + "gnumeric", + "inkscape", + "koffice", + "libabw", + "libcdr", + "libe-book", + "libetonyek", + "libfreehand", + "libmspub", + "libmwaw", + "liborcus", + "libpagemaker", + "libreoffice", + "libvisio", + "libwpd", + "libwpg", + "libwps", + "openoffice.org", + "python-uniconvertor", + "scribus", + "sk1", + "unoconv", +) + +def get_launchpad_bugs(prefix): + #launchpadlib python module is required to download launchpad attachments + from launchpadlib.launchpad import Launchpad + + launchpad = Launchpad.login_anonymously("attachmentdownload", "production") + ubuntu = launchpad.distributions["ubuntu"] + + for pkg in launchpad_pkgs: + srcpkg = ubuntu.getSourcePackage(name=pkg) + pkgbugs = srcpkg.searchTasks(status=["New", "Fix Committed", "Invalid", "Won't Fix", "Confirmed", "Triaged", "In Progress", "Incomplete", "Incomplete (with response)", "Incomplete (without response)", "Fix Released", "Opinion", "Expired"]) + + for bugtask in pkgbugs: + bug = bugtask.bug + id = str(bug.id) + print("parsing " + id + " status: " + bugtask.status + " title: " + bug.title[:50]) + attachmentid = 0 + for attachment in bug.attachments: + attachmentid += 1 + handle = attachment.data.open() + if not handle.content_type in mimetypes: + #print "skipping" + continue + + suffix = mimetypes[handle.content_type] + if not os.path.isdir(suffix): + try: + os.mkdir(suffix) + except: + pass + + download = suffix + '/' + prefix + id + '-' + str(attachmentid) + '.' + suffix + + if os.path.isfile(download): + print("assuming " + id + " is up to date") + break + + print('mimetype is ' + handle.content_type + ' downloading as ' + download) + + tmpfile = download + ".tmp" + f = open(tmpfile, "wb") + f.write(handle.read()) + f.close() + os.rename(tmpfile, download) + +rss_bugzillas = ( + ( 'abi', 'http://bugzilla.abisource.com/buglist.cgi' ), #added for abiword + ( 'fdo', 'http://bugs.freedesktop.org/buglist.cgi' ), + ( 'gentoo', 'http://bugs.gentoo.org/buglist.cgi' ), + ( 'gnome', 'http://bugzilla.gnome.org/buglist.cgi' ), # added for gnumeric + ( 'kde', 'http://bugs.kde.org/buglist.cgi' ), # added for koffice/calligra + ( 'mandriva', 'https://qa.mandriva.com/buglist.cgi' ), + ( 'moz', 'https://bugzilla.mozilla.org/buglist.cgi' ), + # It seems something has changed and it is no longer possible to + # download any files from there. + # NOTE: I am leaving it in the list, commented out, just so someone + # does not add it back immediately .-) + # 'novell': 'https://bugzilla.novell.com/buglist.cgi', +# note: running this script against bz.apache.org apparently causes one's IP +# to be blacklisted or something; you won't get new files in any case... +# ( 'ooo', 'https://bz.apache.org/ooo/buglist.cgi' ), + ( 'tdf', 'http://bugs.documentfoundation.org/buglist.cgi' ), +) + +redhatrpc = 'https://bugzilla.redhat.com/xmlrpc.cgi' +redhatbug = 'https://bugzilla.redhat.com/show_bug.cgi?id=' + +#Novell Bugzilla requires users to log in, in order to get details of the bugs such as attachment bodies etc. +#As a dirty workaround, we parse comments containing "Created an attachment (id=xxxxxx)" and download attachments manually +#python-bugzilla claims that it supports Novell bugzilla login but it's not working right now and novell bugzilla login +#system is a nightmare +novellattach = 'https://bugzilla.novell.com/attachment.cgi?id=' + +mimetypes = { +# ODF + 'application/vnd.oasis.opendocument.base': 'odb', + 'application/vnd.oasis.opendocument.database': 'odb', + 'application/vnd.oasis.opendocument.chart': 'odc', + 'application/vnd.oasis.opendocument.chart-template': 'otc', + 'application/vnd.oasis.opendocument.formula': 'odf', + 'application/vnd.oasis.opendocument.formula-template': 'otf', + 'application/vnd.oasis.opendocument.graphics': 'odg', + 'application/vnd.oasis.opendocument.graphics-template': 'otg', + 'application/vnd.oasis.opendocument.graphics-flat-xml': 'fodg', + 'application/vnd.oasis.opendocument.presentation': 'odp', + 'application/vnd.oasis.opendocument.presentation-template': 'otp', + 'application/vnd.oasis.opendocument.presentation-flat-xml': 'fodp', + 'application/vnd.oasis.opendocument.spreadsheet': 'ods', + 'application/vnd.oasis.opendocument.spreadsheet-template': 'ots', + 'application/vnd.oasis.opendocument.spreadsheet-flat-xml': 'fods', + 'application/vnd.oasis.opendocument.text': 'odt', + 'application/vnd.oasis.opendocument.text-flat-xml': 'fodt', + 'application/vnd.oasis.opendocument.text-master': 'odm', + 'application/vnd.oasis.opendocument.text-template': 'ott', + 'application/vnd.oasis.opendocument.text-master-template': 'otm', + 'application/vnd.oasis.opendocument.text-web': 'oth', +# OOo XML + 'application/vnd.sun.xml.base': 'odb', + 'application/vnd.sun.xml.calc': 'sxc', + 'application/vnd.sun.xml.calc.template': 'stc', + 'application/vnd.sun.xml.chart': 'sxs', + 'application/vnd.sun.xml.draw': 'sxd', + 'application/vnd.sun.xml.draw.template': 'std', + 'application/vnd.sun.xml.impress': 'sxi', + 'application/vnd.sun.xml.impress.template': 'sti', + 'application/vnd.sun.xml.math': 'sxm', + 'application/vnd.sun.xml.writer': 'sxw', + 'application/vnd.sun.xml.writer.global': 'sxg', + 'application/vnd.sun.xml.writer.template': 'stw', + 'application/vnd.sun.xml.writer.web': 'stw', +# MSO + 'application/rtf': 'rtf', + 'text/rtf': 'rtf', + 'application/msword': 'doc', + 'application/vnd.ms-powerpoint': 'ppt', + 'application/vnd.ms-excel': 'xls', + 'application/vnd.ms-excel.sheet.binary.macroEnabled.12': 'xlsb', + 'application/vnd.ms-excel.sheet.macroEnabled.12': 'xlsm', + 'application/vnd.ms-excel.template.macroEnabled.12': 'xltm', + 'application/vnd.ms-powerpoint.presentation.macroEnabled.12': 'pptm', + 'application/vnd.ms-powerpoint.slide.macroEnabled.12': 'sldm', + 'application/vnd.ms-powerpoint.slideshow.macroEnabled.12': 'ppsm', + 'application/vnd.ms-powerpoint.template.macroEnabled.12': 'potm', + 'application/vnd.ms-word.document.macroEnabled.12': 'docm', + 'application/vnd.ms-word.template.macroEnabled.12': 'dotm', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.template': 'xltx', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx', + 'application/vnd.openxmlformats-officedocument.presentationml.template': 'potx', + 'application/vnd.openxmlformats-officedocument.presentationml.slideshow': 'ppsx', + 'application/vnd.openxmlformats-officedocument.presentationml.slide': 'sldx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.template': 'dotx', + 'application/vnd.visio': 'vsd', + 'application/visio.drawing': 'vsd', + 'application/vnd.visio2013': 'vsdx', + 'application/vnd.visio.xml': 'vdx', + 'application/x-mspublisher': 'pub', +#WPS Office + 'application/wps-office.doc': 'doc', + 'application/wps-office.docx': 'docx', + 'application/wps-office.xls': 'xls', + 'application/wps-office.xlsx': 'xlsx', + 'application/wps-office.ppt': 'ppt', + 'application/wps-office.pptx': 'pptx', +# W3C + 'application/xhtml+xml': 'xhtml', + 'application/mathml+xml': 'mml', + 'text/html': 'html', + 'application/docbook+xml': 'docbook', +# misc + 'text/csv': 'csv', + 'text/spreadsheet': 'slk', + 'application/x-qpro': 'qpro', + 'application/x-dbase': 'dbf', + 'application/vnd.corel-draw': 'cdr', + 'application/vnd.lotus-wordpro': 'lwp', + 'application/vnd.lotus-1-2-3': 'wks', + 'application/vnd.wordperfect': 'wpd', + 'application/wordperfect5.1': 'wpd', + 'application/vnd.ms-works': 'wps', + 'application/clarisworks' : 'cwk', + 'application/macwriteii' : 'mw', + 'application/vnd.apple.keynote': 'key', + 'application/vnd.apple.numbers': 'numbers', + 'application/vnd.apple.pages': 'pages', + 'application/x-iwork-keynote-sffkey': 'key', + 'application/x-iwork-numbers-sffnumbers': 'numbers', + 'application/x-iwork-pages-sffpages': 'pages', + 'application/x-hwp': 'hwp', + 'application/x-aportisdoc': 'pdb', + 'application/prs.plucker' : 'pdb_plucker', + 'application/vnd.palm' : 'pdb_palm', + 'application/x-sony-bbeb' : 'lrf', + 'application/x-pocket-word': 'psw', + 'application/x-t602': '602', + 'application/x-fictionbook+xml': 'fb2', + 'application/x-abiword': 'abw', + 'application/x-pagemaker': 'pmd', + 'application/x-gnumeric': 'gnumeric', + 'application/vnd.stardivision.calc': 'sdc', + 'application/vnd.stardivision.draw': 'sda', + 'application/vnd.stardivision.writer': 'sdw', + 'application/x-starcalc': 'sdc', + 'application/x-stardraw': 'sdd', + 'application/x-starwriter': 'sdw', +# relatively uncommon image mimetypes + 'image/x-freehand': 'fh', + 'image/cgm': 'cgm', + 'image/tif': 'tiff', + 'image/tiff': 'tiff', + 'image/vnd.dxf': 'dxf', + 'image/emf': 'emf', + 'image/x-emf': 'emf', + 'image/x-targa': 'tga', + 'image/x-sgf': 'sgf', + 'image/x-svm': 'svm', + 'image/wmf': 'wmf', + 'image/x-wmf': 'wmf', + 'image/x-pict': 'pict', + 'image/x-cmx': 'cmx', + 'image/svg+xml': 'svg', + 'image/bmp': 'bmp', + 'image/x-ms-bmp': 'bmp', + 'image/x-MS-bmp': 'bmp', + 'image/x-wpg': 'wpg', + 'image/x-eps': 'eps', + 'image/x-met': 'met', + 'image/x-portable-bitmap': 'pbm', + 'image/x-photo-cd': 'pcd', + 'image/x-pcx': 'pcx', + 'image/x-portable-graymap': 'pgm', + 'image/x-portable-pixmap': 'ppm', + 'image/vnd.adobe.photoshop': 'psd', + 'image/x-cmu-raster': 'ras', + 'image/x-sun-raster': 'ras', + 'image/x-xbitmap': 'xbm', + 'image/x-xpixmap': 'xpm', +} + +# disabled for now, this would download gigs of pngs/jpegs... +common_noncore_mimetypes = { +# graphics + 'image/gif': 'gif', + 'image/jpeg': 'jpeg', + 'image/png': 'png', +# pdf, etc. + 'application/pdf': 'pdf', +} + +class manage_threads(threading.Thread): + def run(self): + #print(threading.current_thread().get_ident()) + while 1: + # Try to receive a job from queue + try: + # Get job from queue + # Use job parameters to call our query + # Then let the queue know we are done with this job + (uri, mimetype, prefix, extension) = jobs.get(True,6) + try: + get_through_rss_query(uri, mimetype, prefix, extension) + finally: + jobs.task_done() + except KeyboardInterrupt: + raise # Ctrl+C should work + except queue.Empty: + break + +def generate_multi_threading(): + for (prefix, uri) in rss_bugzillas: + + # Initialize threads + for i in range(max_threads): + manage_threads().start() + + # Create a job for every mimetype for a bugzilla + for (mimetype,extension) in mimetypes.items(): + # It seems that bugzilla has problems returning that many results + # (10000 results is probably a limit set somewhere) so we always + # end processing the complete list. + if mimetype == 'text/html' and prefix == 'moz': + continue + + jobs.put([uri, mimetype, prefix, extension], block=True) + print("successfully placed a job in the queue searching for " + mimetype + " in bugtracker " + prefix) + + # Continue when all mimetypes are done for a bugzilla + jobs.join() + print("DONE with bugtracker " + prefix) + +max_threads = 20 # Number of threads to create, (1 = without multi-threading) +jobs = queue.Queue() + +generate_multi_threading() + +for (mimetype,extension) in mimetypes.items(): + get_through_rpc_query(redhatrpc, redhatbug, mimetype, "rhbz", extension) + +try: + get_launchpad_bugs("lp") +except ImportError: + print("launchpadlib unavailable, skipping Ubuntu tracker") + +# vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/get_config_variables b/bin/get_config_variables new file mode 100644 index 000000000..60a2bdc04 --- /dev/null +++ b/bin/get_config_variables @@ -0,0 +1,23 @@ +#!/bin/sh +#set -x + +glv_var="$1" +glv_config="config_host.mk" + +if [ "$glv_var" = "--build" ] ; then + glv_config="config_build.mk" + shift +elif [ "$glv_var" = "--host" ] ; then + shift +fi + +while [ -n "$1" ] ; do + glv_var="$1" + shift + glv_value=$(grep "^ *export ${glv_var}=" ${glv_config} | sed -e "s/[^=]*=//") + export ${glv_var}="${glv_value}" +done + +unset glv_var +unset glv_value +unset glv_config diff --git a/bin/git-ps1 b/bin/git-ps1 new file mode 100755 index 000000000..8a0980091 --- /dev/null +++ b/bin/git-ps1 @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +r= +b= +g="$(git rev-parse --git-dir 2>/dev/null)" + +if [ -n "$g" ]; then + if [ -d "$g/../.dotest" ] + then + if test -f "$g/../.dotest/rebasing" + then + r="|REBASE" + elif test -f "$g/../.dotest/applying" + then + r="|AM" + else + r="|AM/REBASE" + fi + b="$(git symbolic-ref HEAD 2>/dev/null)" + elif [ -f "$g/.dotest-merge/interactive" ] + then + r="|REBASE-i" + b="$(cat "$g/.dotest-merge/head-name")" + elif [ -d "$g/.dotest-merge" ] + then + r="|REBASE-m" + b="$(cat "$g/.dotest-merge/head-name")" + elif [ -f "$g/MERGE_HEAD" ] + then + r="|MERGING" + b="$(git symbolic-ref HEAD 2>/dev/null)" + else + if [ -f "$g/BISECT_LOG" ] + then + r="|BISECTING" + fi + if ! b="$(git symbolic-ref HEAD 2>/dev/null)" + then + if ! b="$(git describe --exact-match HEAD 2>/dev/null)" + then + b="$(cut -c1-7 "$g/HEAD")..." + fi + fi + fi + + if [ -n "$1" ]; then + printf "$1" "${b##refs/heads/}$r" + else + printf "%s" "${b##refs/heads/}$r" + fi +else + printf "not-in-git" +fi diff --git a/bin/gla11y b/bin/gla11y new file mode 100755 index 000000000..b1d98c7c0 --- /dev/null +++ b/bin/gla11y @@ -0,0 +1,1401 @@ +#!/usr/bin/env python +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This file incorporates work covered by the following license notice: +# +# Copyright (c) 2018 Martin Pieuchot +# Copyright (c) 2018-2020 Samuel Thibault <sthibault@hypra.fr> +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# Take LibreOffice (glade) .ui files and check for non accessible widgets + +from __future__ import print_function + +import os +import sys +import getopt +try: + import lxml.etree as ET + lxml = True +except ImportError: + if sys.version_info < (2,7): + print("gla11y needs lxml or python >= 2.7") + exit() + import xml.etree.ElementTree as ET + lxml = False + +# Toplevel widgets +widgets_toplevel = [ + 'GtkWindow', + 'GtkOffscreenWindow', + 'GtkApplicationWindow', + 'GtkDialog', + 'GtkFileChooserDialog', + 'GtkColorChooserDialog', + 'GtkFontChooserDialog', + 'GtkMessageDialog', + 'GtkRecentChooserDialog', + 'GtkAssistant', + 'GtkAppChooserDialog', + 'GtkPrintUnixDialog', + 'GtkShortcutsWindow', +] + +widgets_ignored = widgets_toplevel + [ + # Containers + 'GtkBox', + 'GtkGrid', + 'GtkNotebook', + 'GtkFrame', + 'GtkAspectFrame', + 'GtkListBox', + 'GtkFlowBox', + 'GtkOverlay', + 'GtkMenuBar', + 'GtkToolbar', + 'GtkToolpalette', + 'GtkPaned', + 'GtkHPaned', + 'GtkVPaned', + 'GtkButtonBox', + 'GtkHButtonBox', + 'GtkVButtonBox', + 'GtkLayout', + 'GtkFixed', + 'GtkEventBox', + 'GtkExpander', + 'GtkViewport', + 'GtkScrolledWindow', + 'GtkAlignment', + 'GtkRevealer', + 'GtkSearchBar', + 'GtkHeaderBar', + 'GtkStack', + 'GtkStackSwticher', + 'GtkPopover', + 'GtkPopoverMenu', + 'GtkActionBar', + 'GtkHandleBox', + 'GtkShortcutsSection', + 'GtkShortcutsGroup', + 'GtkTable', + + 'GtkVBox', + 'GtkHBox', + 'GtkToolItem', + 'GtkMenu', + + # Invisible actions + 'GtkSeparator', + 'GtkHSeparator', + 'GtkVSeparator', + 'GtkAction', + 'GtkToggleAction', + 'GtkActionGroup', + 'GtkCellRendererGraph', + 'GtkCellRendererPixbuf', + 'GtkCellRendererProgress', + 'GtkCellRendererSpin', + 'GtkCellRendererText', + 'GtkCellRendererToggle', + 'GtkSeparatorMenuItem', + 'GtkSeparatorToolItem', + + # Storage objects + 'GtkListStore', + 'GtkTreeStore', + 'GtkTreeModelFilter', + 'GtkTreeModelSort', + + 'GtkEntryBuffer', + 'GtkTextBuffer', + 'GtkTextTag', + 'GtkTextTagTable', + + 'GtkSizeGroup', + 'GtkWindowGroup', + 'GtkAccelGroup', + 'GtkAdjustment', + 'GtkEntryCompletion', + 'GtkIconFactory', + 'GtkStatusIcon', + 'GtkFileFilter', + 'GtkRecentFilter', + 'GtkRecentManager', + 'GThemedIcon', + + 'GtkTreeSelection', + + 'GtkListBoxRow', + 'GtkTreeViewColumn', + + # Useless to label + 'GtkScrollbar', + 'GtkHScrollbar', + 'GtkStatusbar', + 'GtkInfoBar', + + # These are actually labels + 'GtkLinkButton', + + # This precisely give a11y information :) + 'AtkObject', +] + +widgets_suffixignored = [ +] + +# These widgets always need a label +widgets_needlabel = [ + 'GtkEntry', + 'GtkSearchEntry', + 'GtkScale', + 'GtkHScale', + 'GtkVScale', + 'GtkSpinButton', + 'GtkSwitch', +] + +# These widgets normally have their own label +widgets_buttons = [ + 'GtkButton', + 'GtkToolButton', + 'GtkToggleButton', + 'GtkToggleToolButton', + 'GtkRadioButton', + 'GtkRadioToolButton', + 'GtkCheckButton', + 'GtkModelButton', + 'GtkLockButton', + 'GtkColorButton', + 'GtkMenuButton', + + 'GtkMenuItem', + 'GtkImageMenuItem', + 'GtkMenuToolButton', + 'GtkRadioMenuItem', + 'GtkCheckMenuItem', +] + +# These widgets are labels that can label other widgets +widgets_labels = [ + 'GtkLabel', + 'GtkAccelLabel', +] + +# The rest should probably be labelled if there are orphan labels + +# GtkSpinner +# GtkProgressBar +# GtkLevelBar + +# GtkComboBox +# GtkComboBoxText +# GtkFileChooserButton +# GtkAppChooserButton +# GtkFontButton +# GtkCalendar +# GtkColorChooserWidget + +# GtkCellView +# GtkTreeView +# GtkTextView +# GtkIconView + +# GtkImage +# GtkArrow +# GtkDrawingArea + +# GtkScaleButton +# GtkVolumeButton + + +# TODO: +# GtkColorPlane ? +# GtkColorScale ? +# GtkColorSwatch ? +# GtkFileChooserWidget ? +# GtkFishbowl ? +# GtkFontChooserWidget ? +# GtkIcon ? +# GtkInspector* ? +# GtkMagnifier ? +# GtkPathBar ? +# GtkPlacesSidebar ? +# GtkPlacesView ? +# GtkPrinterOptionWidget ? +# GtkStackCombo ? +# GtkStackSidebar ? +# GtkStackSwitcher ? + +progname = os.path.basename(sys.argv[0]) + +suppressions = {} +suppressions_to_line = {} +false_positives = {} +ids = {} +ids_dup = {} +labelled_by_elm = {} +label_for_elm = {} +mnemonic_for_elm = {} + +gen_suppr = None +gen_supprfile = None +suppr_prefix = "" +outfile = None + +pflag = False + +warn_orphan_labels = True + +errors = 0 +errexists = 0 +warnings = 0 +warnexists = 0 +fatals = 0 +fatalexists = 0 + +enables = [ ] +dofatals = [ ] + +# +# XML browsing and printing functions +# + +def elm_parent(root, elm): + """ + Return the parent of the element. + """ + if lxml: + return elm.getparent() + else: + def find_parent(cur, elm): + for o in cur: + if o == elm: + return cur + parent = find_parent(o, elm) + if parent is not None: + return parent + return None + return find_parent(root, elm) + +def step_elm(elm): + """ + Return the XML class path step corresponding to elm. + This can be empty if the elm does not have any class or id. + """ + step = elm.attrib.get('class') + if step is None: + step = "" + oid = elm.attrib.get('id') + if oid is not None: + oid = oid.encode('ascii','ignore').decode('ascii') + step += "[@id='%s']" % oid + if len(step) > 0: + step += '/' + return step + +def find_elm(root, elm): + """ + Return the XML class path of the element from the given root. + This is the slow version used when getparent is not available. + """ + if root == elm: + return "" + for o in root: + path = find_elm(o, elm) + if path is not None: + step = step_elm(o) + return step + path + return None + +def errpath(filename, tree, elm): + """ + Return the XML class path of the element + """ + if elm is None: + return "" + path = "" + if 'class' in elm.attrib: + path += elm.attrib['class'] + oid = elm.attrib.get('id') + if oid is not None: + oid = oid.encode('ascii','ignore').decode('ascii') + path = "//" + path + "[@id='%s']" % oid + else: + if lxml: + elm = elm.getparent() + while elm is not None: + step = step_elm(elm) + path = step + path + elm = elm.getparent() + else: + path = find_elm(tree.getroot(), elm)[:-1] + path = filename + ':' + path + return path + +# +# Warning/Error printing functions +# + +def elm_prefix(filename, elm): + """ + Return the display prefix of the element + """ + if elm == None or not lxml: + return "%s:" % filename + else: + return "%s:%u" % (filename, elm.sourceline) + +def elm_name(elm): + """ + Return a display name of the element + """ + if elm is not None: + name = "" + if 'class' in elm.attrib: + name = "'%s' " % elm.attrib['class'] + if 'id' in elm.attrib: + id = elm.attrib['id'].encode('ascii','ignore').decode('ascii') + name += "'%s' " % id + if not name: + name = "'" + elm.tag + "'" + if lxml: + name += " line " + str(elm.sourceline) + return name + return "" + +def elm_name_line(elm): + """ + Return a display name of the element with line number + """ + if elm is not None: + name = elm_name(elm) + if lxml and " line " not in name: + name += "line " + str(elm.sourceline) + " " + return name + return "" + +def elm_line(elm): + """ + Return the line for the given element. + """ + if lxml: + return " line " + str(elm.sourceline) + else: + return "" + +def elms_lines(elms): + """ + Return the list of lines for the given elements. + """ + if lxml: + return " lines " + ', '.join([str(l.sourceline) for l in elms]) + else: + return "" + +def elms_names_lines(elms): + """ + Return the list of names and lines for the given elements. + """ + return ', '.join([elm_name_line(elm) for elm in elms]) + +def elm_suppr(filename, tree, elm, msgtype, dogen): + """ + Return the prefix to be displayed to the user and the suppression line for + the warning type "msgtype" for element "elm" + """ + global gen_suppr, gen_supprfile, suppr_prefix, pflag + + if suppressions or false_positives or gen_suppr is not None or pflag: + prefix = errpath(filename, tree, elm) + if prefix[0:len(suppr_prefix)] == suppr_prefix: + prefix = prefix[len(suppr_prefix):] + + if suppressions or false_positives or gen_suppr is not None: + suppr = '%s %s' % (prefix, msgtype) + + if gen_suppr is not None and msgtype is not None and dogen: + if gen_supprfile is None: + gen_supprfile = open(gen_suppr, 'w') + print(suppr, file=gen_supprfile) + else: + suppr = None + + if not pflag: + # Use user-friendly line numbers + prefix = elm_prefix(filename, elm) + if prefix[0:len(suppr_prefix)] == suppr_prefix: + prefix = prefix[len(suppr_prefix):] + + return (prefix, suppr) + +def is_enabled(elm, msgtype, l, default): + """ + Test whether warning type msgtype is enabled for elm in l + """ + enabled = default + for (enable, thetype, klass) in l: + # Match warning type + if thetype is not None: + if thetype != msgtype: + continue + # Match elm class + if klass is not None and elm is not None: + if klass != elm.attrib.get('class'): + continue + enabled = enable + return enabled + +def err(filename, tree, elm, msgtype, msg, error = True): + """ + Emit a warning or error for an element + """ + global errors, errexists, warnings, warnexists, fatals, fatalexists + + # Let user tune whether a warning or error + fatal = is_enabled(elm, msgtype, dofatals, error) + + # By default warnings and errors are enabled, but let user tune it + if not is_enabled(elm, msgtype, enables, True): + return + + (prefix, suppr) = elm_suppr(filename, tree, elm, msgtype, True) + if suppr in false_positives: + # That was actually expected + return + if suppr in suppressions: + # Suppressed + suppressions[suppr] = False + if fatal: + fatalexists += 1 + if error: + errexists += 1 + else: + warnexists += 1 + return + + if error: + errors += 1 + else: + warnings += 1 + if fatal: + fatals += 1 + + msg = "%s %s%s: %s%s" % (prefix, + "FATAL " if fatal else "", + "ERROR" if error else "WARNING", + elm_name(elm), msg) + print(msg) + if outfile is not None: + print(msg, file=outfile) + +def warn(filename, tree, elm, msgtype, msg): + """ + Emit a warning for an element + """ + err(filename, tree, elm, msgtype, msg, False) + +# +# Labelling testing functions +# + +def find_button_parent(root, elm): + """ + Find a parent which is a button + """ + if lxml: + parent = elm.getparent() + if parent is not None: + if parent.attrib.get('class') in widgets_buttons: + return parent + return find_button_parent(root, parent) + else: + def find_parent(cur, elm): + for o in cur: + if o == elm: + if cur.attrib.get('class') in widgets_buttons: + # we are the button, immediately above the target + return cur + else: + # we aren't the button, but target is over there + return True + parent = find_parent(o, elm) + if parent == True: + # It is over there, but didn't find a button yet + if cur.attrib.get('class') in widgets_buttons: + # we are the button + return cur + else: + return True + if parent is not None: + # we have the button parent over there + return parent + return None + parent = find_parent(root, elm) + if parent == True: + parent = None + return parent + + +def is_labelled_parent(elm): + """ + Return whether this element is a labelled parent + """ + klass = elm.attrib.get('class') + if klass in widgets_toplevel: + return True + if klass == 'GtkShortcutsGroup': + children = elm.findall("property[@name='title']") + if len(children) >= 1: + return True + if klass == 'GtkFrame' or klass == 'GtkNotebook': + children = elm.findall("child[@type='tab']") + elm.findall("child[@type='label']") + if len(children) >= 1: + return True + return False + +def elm_labelled_parent(root, elm): + """ + Return the first labelled parent of the element, which can thus be used as + the root of widgets with common labelled context + """ + + if lxml: + def find_labelled_parent(elm): + if is_labelled_parent(elm): + return elm + parent = elm.getparent() + if parent is None: + return None + return find_labelled_parent(parent) + parent = elm.getparent() + if parent is None: + return None + return find_labelled_parent(elm.getparent()) + else: + def find_labelled_parent(cur, elm): + if cur == elm: + # the target element is over there + return True + for o in cur: + parent = find_labelled_parent(o, elm) + if parent == True: + # target element is over there, check ourself + if is_labelled_parent(cur): + # yes, and we are the first ancestor of the target element + return cur + else: + # no, but target element is over there. + return True + if parent != None: + # the first ancestor of the target element was over there + return parent + return None + parent = find_labelled_parent(root, elm) + if parent == True: + parent = None + return parent + +def is_orphan_label(filename, tree, root, obj, orphan_root, doprint = False): + """ + Check whether this label has no accessibility relation, or doubtful relation + because another label labels the same target + """ + global label_for_elm, labelled_by_elm, mnemonic_for_elm, warnexists + + # label-for + label_for = obj.findall("accessibility/relation[@type='label-for']") + for rel in label_for: + target = rel.attrib['target'] + l = label_for_elm[target] + if len(l) > 1: + return True + + # mnemonic_widget + mnemonic_for = obj.findall("property[@name='mnemonic_widget']") + \ + obj.findall("property[@name='mnemonic-widget']") + for rel in mnemonic_for: + target = rel.text + l = mnemonic_for_elm[target] + if len(l) > 1: + return True + + if len(label_for) > 0: + # At least one label-for, we are not orphan. + return False + + if len(mnemonic_for) > 0: + # At least one mnemonic_widget, we are not orphan. + return False + + labelled_by = obj.findall("accessibility/relation[@type='labelled-by']") + if len(labelled_by) > 0: + # Oh, a labelled label, probably not to be labelling anything + return False + + # explicit role? + roles = [x.text for x in obj.findall("child[@internal-child='accessible']/object[@class='AtkObject']/property[@name='AtkObject::accessible-role']")] + roles += [x.attrib.get("type") for x in obj.findall("accessibility/role")] + if len(roles) > 1 and doprint: + err(filename, tree, obj, "multiple-role", "has multiple <child internal-child='accessible'><object class='AtkObject'><property name='AtkBoject::accessible-role'>" + "%s" % elms_lines(children)) + for role in roles: + if role == 'static' or role == 'ATK_ROLE_STATIC': + # This is static text, not meant to label anything + return False + + parent = elm_parent(root, obj) + if parent is not None: + childtype = parent.attrib.get('type') + if childtype is None: + childtype = parent.attrib.get('internal-child') + if parent.tag == 'child' and childtype == 'label' \ + or childtype == 'tab': + # This is a frame or a notebook label, not orphan. + return False + + if find_button_parent(root, obj) is not None: + # This label is part of a button + return False + + oid = obj.attrib.get('id') + if oid is not None: + if oid in labelled_by_elm: + # Some widget is labelled by us, we are not orphan. + # We should have had a label-for, will warn about it later. + return False + + # No label-for, no mnemonic-for, no labelled-by, we are orphan. + (_, suppr) = elm_suppr(filename, tree, obj, "orphan-label", False) + if suppr in false_positives: + # That was actually expected + return False + if suppr in suppressions: + # Warning suppressed for this label + if suppressions[suppr]: + warnexists += 1 + suppressions[suppr] = False + return False + + if doprint: + context = elm_name(orphan_root) + if context: + context = " within " + context + warn(filename, tree, obj, "orphan-label", "does not specify what it labels" + context) + return True + +def is_orphan_widget(filename, tree, root, obj, orphan, orphan_root, doprint = False): + """ + Check whether this widget has no accessibility relation. + """ + global warnexists + if obj.tag != 'object': + return False + + oid = obj.attrib.get('id') + klass = obj.attrib.get('class') + + # "Don't care" special case + if klass in widgets_ignored: + return False + for suffix in widgets_suffixignored: + if klass[-len(suffix):] == suffix: + return False + + # Widgets usual do not strictly require a label, i.e. a labelled parent + # is enough for context, but some do always need one. + requires_label = klass in widgets_needlabel + + labelled_by = obj.findall("accessibility/relation[@type='labelled-by']") + + # Labels special case + if klass in widgets_labels: + return False + + # Case 1: has an explicit <child internal-child="accessible"> sub-element + children = obj.findall("child[@internal-child='accessible']") + if len(children) > 1 and doprint: + err(filename, tree, obj, "multiple-accessible", "has multiple <child internal-child='accessible'>" + "%s" % elms_lines(children)) + if len(children) >= 1: + return False + + # Case 2: has an <accessibility> sub-element with a "labelled-by" + # <relation> pointing to an existing element. + if len(labelled_by) > 0: + return False + + # Case 3: has a label-for + if oid in label_for_elm: + return False + + # Case 4: has a mnemonic + if oid in mnemonic_for_elm: + return False + + # Case 5: Has a <property name="tooltip_text"> + tooltips = obj.findall("property[@name='tooltip_text']") + \ + obj.findall("property[@name='tooltip-text']") + if len(tooltips) > 1 and doprint: + err(filename, tree, obj, "multiple-tooltip", "has multiple tooltip_text properties") + if len(tooltips) >= 1 and klass != 'GtkCheckButton': + return False + + # Case 6: Has a <property name="placeholder_text"> + placeholders = obj.findall("property[@name='placeholder_text']") + \ + obj.findall("property[@name='placeholder-text']") + if len(placeholders) > 1 and doprint: + err(filename, tree, obj, "multiple-placeholder", "has multiple placeholder_text properties") + if len(placeholders) >= 1: + return False + + # Buttons usually don't need an external label, their own is enough, (but they do need one) + if klass in widgets_buttons: + + labels = obj.findall("property[@name='label']") + if len(labels) > 1 and doprint: + err(filename, tree, obj, "multiple-label", "has multiple label properties") + if len(labels) >= 1: + # Has a <property name="label"> + return False + + actions = obj.findall("property[@name='action_name']") + if len(actions) > 1 and doprint: + err(filename, tree, obj, "multiple-action_name", "has multiple action_name properties") + if len(actions) >= 1: + # Has a <property name="action_name"> + return False + + # Uses id as an action_name + if 'id' in obj.attrib: + if obj.attrib['id'].startswith(".uno:"): + return False + + gtklabels = obj.findall(".//object[@class='GtkLabel']") + obj.findall(".//object[@class='GtkAccelLabel']") + if len(gtklabels) >= 1: + # Has a custom label + return False + + # no label for a button, warn + if doprint: + warn(filename, tree, obj, "button-no-label", "does not have its own label"); + if not is_enabled(obj, "button-no-label", enables, True): + # Warnings disabled + return False + (_, suppr) = elm_suppr(filename, tree, obj, "button-no-label", False) + if suppr in false_positives: + # That was actually expected + return False + if suppr in suppressions: + # Warning suppressed for this widget + if suppressions[suppr]: + warnexists += 1 + suppressions[suppr] = False + return False + return True + + # GtkImages special case + if klass == "GtkImage": + uses = [u for u in tree.iterfind(".//object/property[@name='image']") if u.text == oid] + if len(uses) > 0: + # This image is just used by another element, don't warn + # about the image itself, we probably want the warning on + # the element instead. + return False + + if find_button_parent(root, obj) is not None: + # This image is part of a button, we want the warning on the button + # instead, if any. + return False + + # GtkEntry special case + if klass == 'GtkEntry' or klass == 'GtkSearchEntry': + parent = elm_parent(root, obj) + if parent is not None: + if parent.tag == 'child' and \ + parent.attrib.get('internal-child') == "entry": + # This is an internal entry of another widget. Relations + # will be handled by that widget. + return False + + # GtkShortcutsShortcut special case + if klass == 'GtkShortcutsShortcut': + children = obj.findall("property[@name='title']") + if len(children) >= 1: + return False + + + # Really no label, perhaps emit a warning + if not is_enabled(obj, "no-labelled-by", enables, True): + # Warnings disabled for this class of widgets + return False + (_, suppr) = elm_suppr(filename, tree, obj, "no-labelled-by", False) + if suppr in false_positives: + # That was actually expected + return False + if suppr in suppressions: + # Warning suppressed for this widget + if suppressions[suppr]: + warnexists += 1 + suppressions[suppr] = False + return False + + if not orphan: + # No orphan label, so probably the labelled parent provides enough + # context. + if requires_label: + # But these always need a label. + if doprint: + warn(filename, tree, obj, "no-labelled-by", "has no accessibility label") + return True + return False + + if doprint: + context = elm_name(orphan_root) + if context: + context = " within " + context + warn(filename, tree, obj, "no-labelled-by", "has no accessibility label while there are orphan labels" + context) + return True + +def orphan_items(filename, tree, root, elm): + """ + Check whether from some element there exists orphan labels and orphan widgets + """ + orphan_labels = False + orphan_widgets = False + if elm.attrib.get('class') in widgets_labels: + orphan_labels = is_orphan_label(filename, tree, root, elm, None) + else: + orphan_widgets = is_orphan_widget(filename, tree, root, elm, True, None) + for obj in elm: + # We are not interested in orphan labels under another labelled + # parent. This also allows to keep linear complexity. + if not is_labelled_parent(obj): + label, widget = orphan_items(filename, tree, root, obj) + if label: + orphan_labels = True + if widget: + orphan_widgets = True + if orphan_labels and orphan_widgets: + # No need to look up more + break + return orphan_labels, orphan_widgets + +# +# UI accessibility checks +# + +def check_props(filename, tree, root, elm, forward): + """ + Check the given list of relation properties + """ + props = elm.findall("property[@name='" + forward + "']") + for prop in props: + if prop.text not in ids: + err(filename, tree, elm, "undeclared-target", forward + " uses undeclared target '%s'" % prop.text) + return props + +def is_visible(obj): + visible = False + visible_prop = obj.findall("property[@name='visible']") + visible_len = len(visible_prop) + if visible_len: + visible_txt = visible_prop[visible_len - 1].text + if visible_txt.lower() == "true": + visible = True + elif visible_txt.lower() == "false": + visible = False + return visible + +def check_rels(filename, tree, root, elm, forward, backward = None): + """ + Check the relations given by forward + """ + oid = elm.attrib.get('id') + rels = elm.findall("accessibility/relation[@type='" + forward + "']") + for rel in rels: + target = rel.attrib['target'] + if target not in ids: + err(filename, tree, elm, "undeclared-target", forward + " uses undeclared target '%s'" % target) + elif backward is not None: + widget = ids[target] + backrels = widget.findall("accessibility/relation[@type='" + backward + "']") + if len([x for x in backrels if x.attrib['target'] == oid]) == 0: + err(filename, tree, elm, "missing-" + backward, "has " + forward + \ + ", but is not " + backward + " by " + elm_name_line(widget)) + return rels + +def check_a11y_relation(filename, tree): + """ + Emit an error message if any of the 'object' elements of the XML + document represented by `root' doesn't comply with Accessibility + rules. + """ + global widgets_ignored, ids, label_for_elm, labelled_by_elm, mnemonic_for_elm + + def check_elm(orphan_root, obj, orphan_labels, orphan_widgets): + """ + Check one element, knowing that orphan_labels/widgets tell whether + there are orphan labels and widgets within orphan_root + """ + + oid = obj.attrib.get('id') + klass = obj.attrib.get('class') + + # "Don't care" special case + if klass in widgets_ignored: + return + for suffix in widgets_suffixignored: + if klass[-len(suffix):] == suffix: + return + + # Widgets usual do not strictly require a label, i.e. a labelled parent + # is enough for context, but some do always need one. + requires_label = klass in widgets_needlabel + + if oid is not None: + # Check that ids are unique + if oid in ids_dup: + if ids[oid] == obj: + # We are the first, warn + duplicates = tree.findall(".//object[@id='" + oid + "']") + err(filename, tree, obj, "duplicate-id", "has the same id as other elements " + elms_names_lines(duplicates)) + + # Check label-for and their dual labelled-by + label_for = check_rels(filename, tree, root, obj, "label-for", "labelled-by") + + # Check labelled-by and its dual label-for + labelled_by = check_rels(filename, tree, root, obj, "labelled-by", "label-for") + + visible = is_visible(obj) + + # Should have only one label + if len(labelled_by) >= 1: + if oid in mnemonic_for_elm: + warn(filename, tree, obj, "labelled-by-and-mnemonic", + "has both a mnemonic " + elm_name_line(mnemonic_for_elm[oid][0]) + "and labelled-by relation") + if len(labelled_by) > 1: + warn(filename, tree, obj, "multiple-labelled-by", "has multiple labelled-by relations") + if oid in label_for_elm: + if len(label_for_elm[oid]) > 1: + warn(filename, tree, obj, "duplicate-label-for", "is referenced by multiple label-for " + elms_names_lines(label_for_elm[oid])) + elif len(label_for_elm[oid]) == 1: + paired = label_for_elm[oid][0] + if visible != is_visible(paired): + warn(filename, tree, obj, "visibility-conflict", "visibility conflicts with paired " + elm_name_line(paired)) + if oid in mnemonic_for_elm: + if len(mnemonic_for_elm[oid]) > 1: + warn(filename, tree, obj, "duplicate-mnemonic", "is referenced by multiple mnemonic_widget " + elms_names_lines(mnemonic_for_elm[oid])) + + # Check member-of + member_of = check_rels(filename, tree, root, obj, "member-of") + + # Labels special case + if klass in widgets_labels: + properties = check_props(filename, tree, root, obj, "mnemonic_widget") + \ + check_props(filename, tree, root, obj, "mnemonic-widget") + if len(properties) > 1: + err(filename, tree, obj, "multiple-mnemonic", "has multiple mnemonic_widgets properties" + "%s" % elms_lines(properties)) + + # Emit orphaning warnings + if warn_orphan_labels or orphan_widgets: + is_orphan_label(filename, tree, root, obj, orphan_root, True) + + # We are done with the label + return + + # Not a label, will perhaps need one + + # Emit orphaning warnings + is_orphan_widget(filename, tree, root, obj, orphan_labels, orphan_root, True) + + root = tree.getroot() + + # Flush ids and relations from previous files + ids = {} + ids_dup = {} + labelled_by_elm = {} + label_for_elm = {} + mnemonic_for_elm = {} + + # First pass to get links into hash tables, no warning, just record duplicates + for obj in root.iter('object'): + oid = obj.attrib.get('id') + if oid is not None: + if oid not in ids: + ids[oid] = obj + else: + ids_dup[oid] = True + + labelled_by = obj.findall("accessibility/relation[@type='labelled-by']") + for rel in labelled_by: + target = rel.attrib.get('target') + if target is not None: + if target not in labelled_by_elm: + labelled_by_elm[target] = [ obj ] + else: + labelled_by_elm[target].append(obj) + + label_for = obj.findall("accessibility/relation[@type='label-for']") + for rel in label_for: + target = rel.attrib.get('target') + if target is not None: + if target not in label_for_elm: + label_for_elm[target] = [ obj ] + else: + label_for_elm[target].append(obj) + + mnemonic_for = obj.findall("property[@name='mnemonic_widget']") + \ + obj.findall("property[@name='mnemonic-widget']") + for rel in mnemonic_for: + target = rel.text + if target is not None: + if target not in mnemonic_for_elm: + mnemonic_for_elm[target] = [ obj ] + else: + mnemonic_for_elm[target].append(obj) + + # Second pass, recursive depth-first, to be able to efficiently know whether + # there are orphan labels within a part of the tree. + def recurse(orphan_root, obj, orphan_labels, orphan_widgets): + if obj == root or is_labelled_parent(obj): + orphan_root = obj + orphan_labels, orphan_widgets = orphan_items(filename, tree, root, obj) + + if obj.tag == 'object': + check_elm(orphan_root, obj, orphan_labels, orphan_widgets) + + for o in obj: + recurse(orphan_root, o, orphan_labels, orphan_widgets) + + recurse(root, root, False, False) + +# +# Main +# + +def usage(fatal = True): + print("`%s' checks accessibility of glade .ui files" % progname) + print("") + print("Usage: %s [-p] [-g SUPPR_FILE] [-s SUPPR_FILE] [-f SUPPR_FILE] [-P PREFIX] [-o LOG_FILE] [file ...]" % progname) + print("") + print(" -p Print XML class path instead of line number") + print(" -g Generate suppression file SUPPR_FILE") + print(" -s Suppress warnings given by file SUPPR_FILE, but count them") + print(" -f Suppress warnings given by file SUPPR_FILE completely") + print(" -P Remove PREFIX from file names in warnings") + print(" -o Also prints errors and warnings to given file") + print("") + print(" --widgets-FOO [+][CLASS1[,CLASS2[,...]]]") + print(" Give or extend one of the lists of widget classes, where FOO can be:") + print(" - toplevel : widgets to be considered toplevel windows") + print(" - ignored : widgets which do not need labelling (e.g. GtkBox)") + print(" - suffixignored : suffixes of widget classes which do not need labelling") + print(" - needlabel : widgets which always need labelling (e.g. GtkEntry)") + print(" - buttons : widgets which need their own label but not more") + print(" (e.g. GtkButton)") + print(" - labels : widgets which provide labels (e.g. GtkLabel)") + print(" --widgets-print print default widgets lists") + print("") + print(" --enable-all enable all warnings/dofatals (default)") + print(" --disable-all disable all warnings/dofatals") + print(" --fatal-all make all warnings dofatals") + print(" --not-fatal-all do not make all warnings dofatals (default)") + print("") + print(" --enable-type=TYPE enable warning/fatal type TYPE") + print(" --disable-type=TYPE disable warning/fatal type TYPE") + print(" --fatal-type=TYPE make warning type TYPE a fatal") + print(" --not-fatal-type=TYPE make warning type TYPE not a fatal") + print("") + print(" --enable-widgets=CLASS enable warning/fatal type CLASS") + print(" --disable-widgets=CLASS disable warning/fatal type CLASS") + print(" --fatal-widgets=CLASS make warning type CLASS a fatal") + print(" --not-fatal-widgets=CLASS make warning type CLASS not a fatal") + print("") + print(" --enable-specific=TYPE.CLASS enable warning/fatal type TYPE for widget") + print(" class CLASS") + print(" --disable-specific=TYPE.CLASS disable warning/fatal type TYPE for widget") + print(" class CLASS") + print(" --fatal-specific=TYPE.CLASS make warning type TYPE a fatal for widget") + print(" class CLASS") + print(" --not-fatal-specific=TYPE.CLASS make warning type TYPE not a fatal for widget") + print(" class CLASS") + print("") + print(" --disable-orphan-labels only warn about orphan labels when there are") + print(" orphan widgets in the same context") + print("") + print("Report bugs to <bugs@hypra.fr>") + sys.exit(2 if fatal else 0) + +def widgets_opt(widgets_list, arg): + """ + Replace or extend `widgets_list' with the list of classes contained in `arg' + """ + append = arg and arg[0] == '+' + if append: + arg = arg[1:] + + if arg: + widgets = arg.split(',') + else: + widgets = [] + + if not append: + del widgets_list[:] + + widgets_list.extend(widgets) + + +def main(): + global pflag, gen_suppr, gen_supprfile, suppressions, suppr_prefix, false_positives, dofatals, enables, dofatals, warn_orphan_labels + global widgets_toplevel, widgets_ignored, widgets_suffixignored, widgets_needlabel, widgets_buttons, widgets_labels + global outfile + + try: + opts, args = getopt.getopt(sys.argv[1:], "hpiIg:s:f:P:o:L:", [ + "help", + "version", + + "widgets-toplevel=", + "widgets-ignored=", + "widgets-suffixignored=", + "widgets-needlabel=", + "widgets-buttons=", + "widgets-labels=", + "widgets-print", + + "enable-all", + "disable-all", + "fatal-all", + "not-fatal-all", + + "enable-type=", + "disable-type=", + "fatal-type=", + "not-fatal-type=", + + "enable-widgets=", + "disable-widgets=", + "fatal-widgets=", + "not-fatal-widgets=", + + "enable-specific=", + "disable-specific=", + "fatal-specific=", + "not-fatal-specific=", + + "disable-orphan-labels", + ] ) + except getopt.GetoptError: + usage() + + suppr = None + false = None + out = None + filelist = None + + for o, a in opts: + if o == "--help" or o == "-h": + usage(False) + if o == "--version": + print("0.1") + sys.exit(0) + elif o == "-p": + pflag = True + elif o == "-g": + gen_suppr = a + elif o == "-s": + suppr = a + elif o == "-f": + false = a + elif o == "-P": + suppr_prefix = a + elif o == "-o": + out = a + elif o == "-L": + filelist = a + + elif o == "--widgets-toplevel": + widgets_opt(widgets_toplevel, a) + elif o == "--widgets-ignored": + widgets_opt(widgets_ignored, a) + elif o == "--widgets-suffixignored": + widgets_opt(widgets_suffixignored, a) + elif o == "--widgets-needlabel": + widgets_opt(widgets_needlabel, a) + elif o == "--widgets-buttons": + widgets_opt(widgets_buttons, a) + elif o == "--widgets-labels": + widgets_opt(widgets_labels, a) + elif o == "--widgets-print": + print("--widgets-toplevel '" + ','.join(widgets_toplevel) + "'") + print("--widgets-ignored '" + ','.join(widgets_ignored) + "'") + print("--widgets-suffixignored '" + ','.join(widgets_suffixignored) + "'") + print("--widgets-needlabel '" + ','.join(widgets_needlabel) + "'") + print("--widgets-buttons '" + ','.join(widgets_buttons) + "'") + print("--widgets-labels '" + ','.join(widgets_labels) + "'") + sys.exit(0) + + elif o == '--enable-all': + enables.append( (True, None, None) ) + elif o == '--disable-all': + enables.append( (False, None, None) ) + elif o == '--fatal-all': + dofatals.append( (True, None, None) ) + elif o == '--not-fatal-all': + dofatals.append( (False, None, None) ) + + elif o == '--enable-type': + enables.append( (True, a, None) ) + elif o == '--disable-type': + enables.append( (False, a, None) ) + elif o == '--fatal-type': + dofatals.append( (True, a, None) ) + elif o == '--not-fatal-type': + dofatals.append( (False, a, None) ) + + elif o == '--enable-widgets': + enables.append( (True, None, a) ) + elif o == '--disable-widgets': + enables.append( (False, None, a) ) + elif o == '--fatal-widgets': + dofatals.append( (True, None, a) ) + elif o == '--not-fatal-widgets': + dofatals.append( (False, None, a) ) + + elif o == '--enable-specific': + (thetype, klass) = a.split('.', 1) + enables.append( (True, thetype, klass) ) + elif o == '--disable-specific': + (thetype, klass) = a.split('.', 1) + enables.append( (False, thetype, klass) ) + elif o == '--fatal-specific': + (thetype, klass) = a.split('.', 1) + dofatals.append( (True, thetype, klass) ) + elif o == '--not-fatal-specific': + (thetype, klass) = a.split('.', 1) + dofatals.append( (False, thetype, klass) ) + + elif o == '--disable-orphan-labels': + warn_orphan_labels = False + + # Read suppression file before overwriting it + if suppr is not None: + try: + supprfile = open(suppr, 'r') + line_no = 1; + for line in supprfile.readlines(): + prefix = line.rstrip() + suppressions[prefix] = True + suppressions_to_line[prefix] = line_no + line_no = line_no + 1; + supprfile.close() + except IOError: + pass + + # Read false positives file + if false is not None: + try: + falsefile = open(false, 'r') + for line in falsefile.readlines(): + prefix = line.rstrip() + false_positives[prefix] = True + falsefile.close() + except IOError: + pass + + if out is not None: + outfile = open(out, 'w') + + if filelist is not None: + try: + filelistfile = open(filelist, 'r') + for line in filelistfile.readlines(): + line = line.strip() + if line: + args += line.split(' ') + filelistfile.close() + except IOError: + err(filelist, None, None, "unable to read file list file") + + for filename in args: + try: + tree = ET.parse(filename) + except ET.ParseError: + err(filename, None, None, "parse", "malformatted xml file") + continue + except IOError: + err(filename, None, None, None, "unable to read file") + continue + + try: + check_a11y_relation(filename, tree) + except Exception as error: + import traceback + traceback.print_exc() + err(filename, None, None, "parse", "error parsing file") + + if errors > 0 or errexists > 0: + estr = "%s new error%s" % (errors, 's' if errors > 1 else '') + if errexists > 0: + estr += " (%s suppressed by %s)" % (errexists, suppr) + print(estr) + + if warnings > 0 or warnexists > 0: + wstr = "%s new warning%s" % (warnings, 's' if warnings > 1 else '') + if warnexists > 0: + wstr += " (%s suppressed by %s)" % (warnexists, suppr) + print(wstr) + + if fatals > 0 or fatalexists > 0: + wstr = "%s new fatal%s" % (fatals, 's' if fatals > 1 else '') + if fatalexists > 0: + wstr += " (%s suppressed by %s)" % (fatalexists, suppr) + print(wstr) + + n = 0 + for (suppr,unused) in suppressions.items(): + if unused: + n += 1 + + if n > 0: + print("%s suppression%s unused:" % (n, 's' if n > 1 else '')) + for (suppr,unused) in suppressions.items(): + if unused: + print(" %s:%s" % (suppressions_to_line[suppr], suppr)) + + if gen_supprfile is not None: + gen_supprfile.close() + if outfile is not None: + outfile.close() + if fatals > 0 and gen_suppr is None: + print("Explanations are available on https://wiki.documentfoundation.org/Development/Accessibility") + sys.exit(1) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + pass + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/includebloat.awk b/bin/includebloat.awk new file mode 100755 index 000000000..3792ef950 --- /dev/null +++ b/bin/includebloat.awk @@ -0,0 +1,51 @@ +#!/usr/bin/gawk -f +# -*- tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Generate a list of files included by the C++ compiler during the build +# sorted by the total bytes an included file contributed to preprocessor input. +# usage: first do a full build with "make check", then run this from $BUILDDIR + +# NOTE: by default gbuild does not generate dependencies for system headers +# (in particular the C++ standard library), so those will NOT be counted + +BEGIN { + cmd = "find workdir/Dep/CxxObject/ -name *.d | xargs cat" + while ((cmd | getline) > 0) { + if ($0 ~ /^ .*\\$/) { + gsub(/^ /, ""); + gsub(/ *\\$/, ""); + includes[$1]++ + if ($2) { + # GCC emits 2 per line if short enough! + includes[$2]++ + } + } + } + exit +} + +END { + for (inc in includes) { + cmd = "wc -c " inc + if ((cmd | getline) < 0) + print "ERROR on: " cmd + sizes[inc] = $1 # $0 is wc -c output, $1 is size + totals[inc] = $1 * includes[inc] + totalsize += totals[inc] + close(cmd) + } + PROCINFO["sorted_in"] = "@val_num_desc" + print "sum total bytes included (excluding system headers): " totalsize + for (inc in totals) { + print totals[inc], sizes[inc], includes[inc], inc + } +} + +# vim: set noet sw=4 ts=4: diff --git a/bin/ios-mapfile-statistics b/bin/ios-mapfile-statistics new file mode 100755 index 000000000..07f3f0aa6 --- /dev/null +++ b/bin/ios-mapfile-statistics @@ -0,0 +1,78 @@ +#!/usr/bin/perl -w + +use strict; + +use Getopt::Std; +$Getopt::Std::STANDARD_HELP_VERSION++; + +my %args; + +getopts('f:s', \%args); + +sub VERSION_MESSAGE { + # Nothing +} + +sub HELP_MESSAGE { + print <<EOS +This program parses a linker map file, especially one produced when linking an iOS executable. + +Input is read from a map file provided as command-line argument + +By default a list of libraries used and the size of code and data +linked in from each library is printed, in reverse order of size. + +The following options are available: +-s Print a list of symbols instead. +-f 'filter' Filter which libraries are handled. The filter can be + a regular expression, typically several library names + combined with the '|' operator. Makes sense only when + -s is used too. +EOS +} + +die "The -f switch makes sense only if -s is also used\n" if defined($args{'f'}) && !defined($args{'s'}); + +die "Please provide one map file name\n" if !defined($ARGV[0]); + +die "Just one argument please\n" if defined($ARGV[1]); + +my $state = 0; +my %libofnumber; +my %sizeoflib; +my %sizeofsym; + +open(INPUT, '<', $ARGV[0]) || die "Could not open $ARGV[0]: $!\n"; + +while (<INPUT>) { + if ($state == 0 && m!^# Object files:!) { + $state = 1; + } elsif ($state == 1 && m!^\[ *([0-9]+)\] .*/([-_a-z0-9]+\.a)\(.*!i) { + $libofnumber{$1} = $2; + } elsif ($state == 1 && m!^# Sections:!) { + $state = 2; + } elsif ($state == 2 && m!^# Address\s+Size\s+File\s+Name!) { + $state = 3; + } elsif ($state == 3 && m!^0x[0-9A-F]+\s+(0x[0-9A-F]+)\s+\[ *([0-9]+)\] (.*)!) { + my ($size,$libnum,$symbol) = ($1, $2, $3); + if (defined($libofnumber{$libnum})) { + $sizeoflib{$libofnumber{$libnum}} += hex($size); + if (!defined($args{'f'}) || $libofnumber{$libnum} =~ /$args{'f'}/) { + $sizeofsym{$symbol} = hex($size); + } + } + } +} + +if ($args{'s'}) { + # Print symbols in reverse size order + foreach (sort { $sizeofsym{$b} <=> $sizeofsym{$a} } keys(%sizeofsym)) { + print $_, ": ", $sizeofsym{$_}, "\n"; + } +} else { + # Print libraries in reverse size order + foreach (sort { $sizeoflib{$b} <=> $sizeoflib{$a} } keys(%sizeoflib)) { + print $_, ": ", $sizeoflib{$_}, "\n"; + } +} + diff --git a/bin/java-set-classpath.in b/bin/java-set-classpath.in new file mode 100644 index 000000000..507264a3d --- /dev/null +++ b/bin/java-set-classpath.in @@ -0,0 +1,53 @@ +#!/bin/sh +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# java-set-classpath - Utility to update the default +# CLASSPATH for LibreOffice + +if test "z$1" = "z" ; then + echo "Update the default CLASSPATH for LibreOffice" + echo "" + echo "Usage: $0 [dir|jar]..." + echo "" + echo "The utility updates the LibreOffice system setting. It adds or removes" + echo "the given directories and jar-files to or from the default CLASSPATH" + echo "depending on if they are available on the system or not." + echo "" + echo "Parameters:" + echo " dir - absolute path to a directory" + echo " jar - absolute path to a jar-file" + exit 0; +fi + +JVM_CONFIG_FILE=@INSTALLDIR@/program/fundamentalrc + +for path in $@ ; do + if test "z${path%%/*}" != "z" ; then + echo "Warning: the path "$path" is not absolute and will be ignored" + continue + fi + if test -e $path ; then + # the file exist + grep "URE_MORE_JAVA_CLASSPATH_URLS.*file:/*$path\([[:space:]].*\)\?$" $JVM_CONFIG_FILE >/dev/null && continue + # it is not registered + TMP_FILE=`mktemp /tmp/ooset-java-class.XXXXXXXXXX` || exit 1 + sed -e "s|^\(.*URE_MORE_JAVA_CLASSPATH_URLS.*\)$|\1 file://$path|" $JVM_CONFIG_FILE >$TMP_FILE + mv -f $TMP_FILE $JVM_CONFIG_FILE + chmod 644 $JVM_CONFIG_FILE + else + # the file does not exist, remove it from the configuration + TMP_FILE=`mktemp /tmp/ooset-java-class.XXXXXXXXXX` || exit 1; + sed -e "s|^\(.*URE_MORE_JAVA_CLASSPATH_URLS.*\)file:/*$path\([[:space:]].*\)\?$|\1\2|" \ + -e "s/\(URE_MORE_JAVA_CLASSPATH_URLS=\)[[:space:]]\+/\1/" \ + -e "/^.*URE_MORE_JAVA_CLASSPATH_URLS/s/[[:space:]]\+/ /g" \ + -e "/^.*URE_MORE_JAVA_CLASSPATH_URLS/s/[[:space:]]*$//" $JVM_CONFIG_FILE >$TMP_FILE + mv -f $TMP_FILE $JVM_CONFIG_FILE + chmod 644 $JVM_CONFIG_FILE + fi +done diff --git a/bin/lint-ui.py b/bin/lint-ui.py new file mode 100755 index 000000000..d9d0784df --- /dev/null +++ b/bin/lint-ui.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. +# +# Takes a LibreOffice .ui file and provides linting tips for maintaining +# a consistent look for dialogs + +import sys +import xml.etree.ElementTree as ET +import re + +DEFAULT_WARNING_STR = 'Lint assertion failed' + +POSSIBLE_TOP_LEVEL_WIDGETS = ['GtkDialog', 'GtkMessageDialog', 'GtkBox', 'GtkFrame', 'GtkGrid'] +IGNORED_TOP_LEVEL_WIDGETS = ['GtkAdjustment', 'GtkImage', 'GtkListStore', 'GtkSizeGroup', 'GtkMenu', 'GtkTextBuffer'] +BORDER_WIDTH = '6' +BUTTON_BOX_SPACING = '12' +ALIGNMENT_TOP_PADDING = '6' +#https://developer.gnome.org/hig-book/3.0/windows-alert.html.en#alert-spacing +MESSAGE_BOX_SPACING = '24' +MESSAGE_BORDER_WIDTH = '12' + +IGNORED_WORDS = ['the', 'of', 'to', 'for', 'a', 'and', 'as', 'from', 'on', 'into', 'by', 'at', 'or', 'do', 'in', 'when'] + +def lint_assert(predicate, warning=DEFAULT_WARNING_STR): + if not predicate: + print(" * " + warning) + +def check_top_level_widget(element): + # check widget type + widget_type = element.attrib['class'] + lint_assert(widget_type in POSSIBLE_TOP_LEVEL_WIDGETS, + "Top level widget should be 'GtkDialog', 'GtkFrame', 'GtkBox', or 'GtkGrid'") + + # check border_width property + border_width_properties = element.findall("property[@name='border_width']") + if len(border_width_properties) < 1: + lint_assert(False, "No border_width set on top level widget. Should probably be " + BORDER_WIDTH) + if len(border_width_properties) == 1: + border_width = border_width_properties[0] + if widget_type == "GtkMessageDialog": + lint_assert(border_width.text == MESSAGE_BORDER_WIDTH, + "Top level 'border_width' property should be " + MESSAGE_BORDER_WIDTH) + else: + lint_assert(border_width.text == BORDER_WIDTH, + "Top level 'border_width' property should be " + BORDER_WIDTH) + + # check that any widget which has 'has-default' also has 'can-default' + for widget in element.findall('.//object'): + if not widget.attrib['class']: + continue + widget_type = widget.attrib['class'] + has_defaults = widget.findall("./property[@name='has_default']") + if len(has_defaults) > 0 and has_defaults[0].text == "True": + can_defaults = widget.findall("./property[@name='can_default']") + lint_assert(len(can_defaults)>0 and can_defaults[0].text == "True", + "has_default without can_default in " + widget_type + " with id = '" + widget.attrib['id'] + "'", widget) + +def check_button_box_spacing(element): + spacing = element.findall("property[@name='spacing']")[0] + lint_assert(spacing.text == BUTTON_BOX_SPACING, + "Button box 'spacing' should be " + BUTTON_BOX_SPACING) + +def check_message_box_spacing(element): + spacing = element.findall("property[@name='spacing']")[0] + lint_assert(spacing.text == MESSAGE_BOX_SPACING, + "Button box 'spacing' should be " + MESSAGE_BOX_SPACING) + +def check_radio_buttons(root): + radios = [element for element in root.findall('.//object') if element.attrib['class'] == 'GtkRadioButton'] + for radio in radios: + radio_underlines = radio.findall("./property[@name='use_underline']") + assert len(radio_underlines) <= 1 + if len(radio_underlines) < 1: + lint_assert(False, "No use_underline in GtkRadioButton with id = '" + radio.attrib['id'] + "'") + +def check_check_buttons(root): + radios = [element for element in root.findall('.//object') if element.attrib['class'] == 'GtkCheckButton'] + for radio in radios: + radio_underlines = radio.findall("./property[@name='use_underline']") + assert len(radio_underlines) <= 1 + if len(radio_underlines) < 1: + lint_assert(False, "No use_underline in GtkCheckButton with id = '" + radio.attrib['id'] + "'") + +def check_frames(root): + frames = [element for element in root.findall('.//object') if element.attrib['class'] == 'GtkFrame'] + for frame in frames: + frame_alignments = frame.findall("./child/object[@class='GtkAlignment']") + assert len(frame_alignments) <= 1 + if len(frame_alignments) < 1: + lint_assert(False, "No GtkAlignment in GtkFrame with id = '" + frame.attrib['id'] + "'") + if len(frame_alignments) == 1: + alignment = frame_alignments[0] + check_alignment_top_padding(alignment) + +def check_alignment_top_padding(alignment): + top_padding_properties = alignment.findall("./property[@name='top_padding']") + assert len(top_padding_properties) <= 1 + if len(top_padding_properties) < 1: + lint_assert(False, "No GtkAlignment 'top_padding' set. Should probably be " + ALIGNMENT_TOP_PADDING) + if len(top_padding_properties) == 1: + top_padding = top_padding_properties[0] + lint_assert(top_padding.text == ALIGNMENT_TOP_PADDING, + "GtkAlignment 'top_padding' should be " + ALIGNMENT_TOP_PADDING) + +def check_title_labels(root): + labels = root.findall(".//child[@type='label']") + titles = [label.find(".//property[@name='label']") for label in labels] + for title in titles: + if title is None: + continue + words = re.split(r'[^a-zA-Z0-9:_-]', title.text) + first = True + for word in words: + if word[0].islower() and (word not in IGNORED_WORDS or first): + lint_assert(False, "The word '" + word + "' should be capitalized") + first = False + +def main(): + print(" == " + sys.argv[1] + " ==") + tree = ET.parse(sys.argv[1]) + root = tree.getroot() + + lint_assert('domain' in root.attrib, "interface needs to specific translation domain") + + top_level_widgets = [element for element in root.findall('object') if element.attrib['class'] not in IGNORED_TOP_LEVEL_WIDGETS] + assert len(top_level_widgets) == 1 + + top_level_widget = top_level_widgets[0] + check_top_level_widget(top_level_widget) + + # TODO - only do this if we have a GtkDialog? + # check button box spacing + button_box = top_level_widget.findall("./child/object[@id='dialog-vbox1']") + if len(button_box) > 0: + element = button_box[0] + check_button_box_spacing(element) + + message_box = top_level_widget.findall("./child/object[@id='messagedialog-vbox']") + if len(message_box) > 0: + element = message_box[0] + check_message_box_spacing(element) + + check_frames(root) + + check_radio_buttons(root) + + check_check_buttons(root) + + check_title_labels(root) + +if __name__ == "__main__": + main() diff --git a/bin/list-dispatch-commands.py b/bin/list-dispatch-commands.py new file mode 100755 index 000000000..0b13f89e8 --- /dev/null +++ b/bin/list-dispatch-commands.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 + +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Script to generate https://wiki.documentfoundation.org/Development/DispatchCommands +""" + +import argparse +import os +import sys + + +def get_files_list(directory, extension): + array_items = [] + + dh = os.scandir(directory) + for entry in dh: + if entry.is_dir(): + array_items += get_files_list(entry.path, extension) + elif entry.is_file(): + if entry.name.endswith(extension): + array_items.append(entry.path) + + return array_items + + +def analyze_file(filename, all_slots): + with open(filename) as fh: + for line in fh: + if not line.startswith('// Slot Nr. '): + continue + + tmp = line.split(':') + slot_id = tmp[1].strip() + + line = next(fh) + tmp = line.split(',') + slot_rid = tmp[1] + + next(fh) + next(fh) + line = next(fh) + mode = 'C' if 'CACHABLE' in line else ' ' + mode += 'U' if 'AUTOUPDATE' in line else ' ' + mode += 'M' if 'MENUCONFIG' in line else ' ' + mode += 'T' if 'TOOLBOXCONFIG' in line else ' ' + mode += 'A' if 'ACCELCONFIG' in line else ' ' + + next(fh) + next(fh) + line = next(fh) + if '"' not in line: + line = next(fh) + tmp = line.split('"') + try: + slot_name = '.uno:' + tmp[1] + except IndexError: + print("Warning: expected \" in line '%s' from file %s" % (line.strip(), filename), + file=sys.stderr) + slot_name = '.uno:' + + if slot_name not in all_slots: + all_slots[slot_name] = {'slot_id': slot_id, + 'slot_rid': slot_rid, + 'mode': mode, + 'slot_description': ''} + + +def analyze_xcu(filename, all_slots): + with open(filename) as fh: + for line in fh: + if '<node oor:name=".uno:' not in line: + continue + + tmp = line.split('"') + slot_name = tmp[1] + + while '<value xml:lang="en-US">' not in line: + try: + line = next(fh) + except StopIteration: + print("Warning: couldn't find '<value xml:lang=\"en-US\">' line in %s" % filename, + file=sys.stderr) + break + + line = line.replace('<value xml:lang="en-US">', '') + line = line.replace('</value>', '').strip() + + if slot_name in all_slots: + all_slots[slot_name]['slot_description'] = line.replace('~', '') + + +def main(): + modules = ['basslots', 'scslots', 'sdgslots', 'sdslots', 'sfxslots', 'smslots', 'svxslots', 'swslots'] + sdi_dir = './workdir/SdiTarget' + sdi_ext = '.hxx' + xcu_dir = 'officecfg/registry/data/org/openoffice/Office/UI' + xcu_ext = '.xcu' + all_slots = {} + + parser = argparse.ArgumentParser() + parser.add_argument('module', choices=modules) + args = parser.parse_args() + + module_filename = args.module + sdi_ext + + sdi_files = get_files_list(sdi_dir, sdi_ext) + for sdi_file in sdi_files: + sdi_file_basename = os.path.basename(sdi_file) + if sdi_file_basename == module_filename: + analyze_file(sdi_file, all_slots) + + xcu_files = get_files_list(xcu_dir, xcu_ext) + for xcu_file in xcu_files: + analyze_xcu(xcu_file, all_slots) + + for name in sorted(all_slots.keys()): + props = all_slots[name] + print('|-\n| %s' % name) + print('| %(slot_rid)s\n| %(slot_id)s\n| %(mode)s\n| %(slot_description)s' % props) + + print("|-") + +if __name__ == '__main__': + main() diff --git a/bin/list-uitest.py b/bin/list-uitest.py new file mode 100755 index 000000000..da6703556 --- /dev/null +++ b/bin/list-uitest.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import datetime + +def analyze_file(filename): + class_name = "" + method_list = [] + with open(filename, encoding='utf-8') as fh: + for line in fh: + if line.lstrip().startswith('class '): + class_name = line.lstrip().split(" ")[1].split("(")[0] + elif line.lstrip().startswith('def test_'): + method_list.append( + line.lstrip().split("test_")[1].split("(")[0]) + else: + continue + return class_name, method_list + +def get_files_list(directory, extension): + array_items = [] + + dh = os.scandir(directory) + for entry in dh: + if entry.is_dir(): + array_items += get_files_list(entry.path, extension) + elif entry.is_file(): + if entry.name.endswith(extension): + array_items.append(entry.path) + + return array_items + +def linkFormat(name): + if name.startswith('tdf'): + return "[https://bugs.documentfoundation.org/show_bug.cgi?id={} {}]"\ + .format(name.split('tdf')[1], name) + else: + return name + + +def main(): + uitest_ext = '.py' + uitest_dirs = { + 'Writer' : ['../uitest/writer_tests/', '../writerperfect/qa/uitest/', '../sw/qa/uitest/'], + 'Calc' : ['../uitest/calc_tests', '../sc/qa/uitest/'], + 'Impress' : ['../uitest/impress_tests/', '../sd/qa/uitest/'], + 'Math': ['../uitest/math_tests/'], + 'Draw': [''], + 'Manual_tests': ['../uitest/manual_tests/']} + + print('{{TopMenu}}') + print('{{Menu}}') + print('{{Menu.Development}}') + print() + print('Generated on ' + str(datetime.datetime.now())) + for k,v in uitest_dirs.items(): + print('\n=== ' + k + ' ===') + for uitest_dir in v: + if uitest_dir: + uitest_files = get_files_list(uitest_dir, uitest_ext) + for uitest_file in uitest_files: + class_name, method_names = analyze_file(uitest_file) + if class_name: + print("* {} ({})".format( + linkFormat(class_name),uitest_file[3:])) + for m in method_names: + print('**' + linkFormat(m)) + print() + print('[[Category:QA]][[Category:Development]]') + +if __name__ == '__main__': + main() diff --git a/bin/lo-all-static-libs b/bin/lo-all-static-libs new file mode 100755 index 000000000..0fcea02eb --- /dev/null +++ b/bin/lo-all-static-libs @@ -0,0 +1,91 @@ +#!/bin/sh + +# Output a list of all our (static) libraries, to be used when +# building the single executable or single dynamic object that is used +# in an LO-based iOS or Android app. (All our libraries and bundled +# 3rd-party ones are built as static archives for these platforms.) + +# This script is to be run once a full "make" for iOS or Android has +# otherwise completed, when just building the actual apps is left. + +if test -z "$INSTDIR" ; then + echo This script should be invoked only in a build. + exit 1 +fi + +if test "$OS" != ANDROID -a "$OS" != iOS; then + echo This script makes sense only in Android or iOS builds. +fi + +foolibs= +for var in EBOOK_LIBS FREEHAND_LIBS HARFBUZZ_LIBS HUNSPELL_LIBS HYPHEN_LIB MYTHES_LIBS; do + dirs= + libs= + for i in `eval echo '$'$var`; do + case "$i" in + -L*) dirs="$dirs ${i#-L}";; + -l*) libs="$libs ${i#-l}";; + esac + done + for l in $libs; do + for d in $dirs; do + test -f $d/lib$l.a && foolibs="$foolibs $d/lib$l.a" + done + done +done + +case $OS in +ANDROID) + oslibs="$WORKDIR/UnpackedTarball/curl/lib/.libs/*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/fontconfig/src/.libs/libfontconfig.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/freetype/objs/.libs/libfreetype.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/pixman/pixman/.libs/libpixman-1.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/cairo/src/.libs/libcairo.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/xmlsec/src/.libs/libxmlsec1.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/xmlsec/src/nss/.libs/libxmlsec1-nss.a" + # Only liblo-bootstrap.a ends up here: + oslibs="$oslibs $WORKDIR/LinkTarget/Library/lib*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/openssl/*.a" + + # coinmp not used for iOS + oslibs="$oslibs $WORKDIR/UnpackedTarball/coinmp/Cbc/src/.libs/*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/coinmp/Cgl/src/.libs/*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/coinmp/Clp/src/.libs/*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/coinmp/Clp/src/OsiClp/.libs/*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/coinmp/CoinMP/src/.libs/*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/coinmp/CoinUtils/src/.libs/*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/coinmp/Osi/src/Osi/.libs/*.a" + ;; +iOS) + oslibs="$WORKDIR/UnpackedTarball/icu/source/stubdata/*.a" + oslibs="$oslibs $WORKDIR/UnpackedTarball/cppunit/src/cppunit/.libs/*.a" + ;; +*) + oslibs= + ;; +esac + +echo $INSTDIR/$LIBO_LIB_FOLDER/lib*.a \ + $foolibs \ + $WORKDIR/LinkTarget/StaticLibrary/lib*.a \ + $oslibs \ + $WORKDIR/UnpackedTarball/icu/source/lib/*.a \ + $WORKDIR/UnpackedTarball/libjpeg-turbo/.libs/*.a \ + $WORKDIR/UnpackedTarball/liblangtag/liblangtag/.libs/*.a \ + $WORKDIR/UnpackedTarball/lcms2/src/.libs/*.a \ + $WORKDIR/UnpackedTarball/libabw/src/lib/.libs/*.a \ + $WORKDIR/UnpackedTarball/libcdr/src/lib/.libs/*.a \ + $WORKDIR/UnpackedTarball/libepubgen/src/lib/.libs/*.a \ + $WORKDIR/UnpackedTarball/libexttextcat/src/.libs/*.a \ + $WORKDIR/UnpackedTarball/libmspub/src/lib/.libs/*.a \ + $WORKDIR/UnpackedTarball/libmwaw/src/lib/.libs/*.a \ + $WORKDIR/UnpackedTarball/libodfgen/src/.libs/*.a \ + $WORKDIR/UnpackedTarball/liborcus/src/*/.libs/*.a \ + $WORKDIR/UnpackedTarball/librevenge/src/*/.libs/*.a \ + $WORKDIR/UnpackedTarball/libvisio/src/lib/.libs/*.a \ + $WORKDIR/UnpackedTarball/libwp?/src/lib/.libs/*.a \ + $WORKDIR/UnpackedTarball/raptor/src/.libs/*.a \ + $WORKDIR/UnpackedTarball/rasqal/src/.libs/*.a \ + $WORKDIR/UnpackedTarball/redland/src/.libs/*.a \ + $WORKDIR/UnpackedTarball/libxml2/.libs/*.a \ + $WORKDIR/UnpackedTarball/libxslt/libxslt/.libs/*.a diff --git a/bin/lo-commit-stat b/bin/lo-commit-stat new file mode 100755 index 000000000..08e8a1785 --- /dev/null +++ b/bin/lo-commit-stat @@ -0,0 +1,584 @@ +#!/usr/bin/perl + eval 'exec /usr/bin/perl -S $0 ${1+"$@"}' + if $running_under_some_shell; +#!/usr/bin/perl + +use strict; +use warnings; +use LWP::UserAgent; +use utf8; +use File::Temp; +use Encode; +use open ':encoding(utf8)'; +use open ':std' => ':encoding(utf8)'; + +my %module_dirname = ( + "core" => "", + "dictionaries" => "dictionaries", + "help" => "helpcontent2", + "translations" => "translations" +); + + +my %bugzillas = ( + fdo => "https://bugs.documentfoundation.org/show_bug.cgi?id=", + tdf => "https://bugs.documentfoundation.org/show_bug.cgi?id=", + bnc => "https://bugzilla.novell.com/show_bug.cgi?id=", + rhbz => "https://bugzilla.redhat.com/show_bug.cgi?id=", + i => "https://bz.apache.org/ooo/show_bug.cgi?id=", + fate => "https://features.opensuse.org/", +); + +sub search_bugs($$$$) +{ + my ($pdata, $module, $commit_id, $line) = @_; + + my $bug = ""; + my $bug_orig; + while (defined $bug) { + + # match fdo#123, rhz#123, i#123, #123 + # but match only bug number with >= 4 digits + if ( $line =~ m/(\w+\#+\d{4,})/ ) { + $bug_orig = $1; + $bug = $1; + # default to issuezilla for the #123 variant + # but match only bug number with >= 4 digits + } elsif ( $line =~ m/(\#)(\d{4,})/ ) { + $bug_orig = $1 . $2; + $bug = "i#$2"; + # match #i123# + } elsif ( $line =~ m/(\#i)(\d+)(\#)/ ) { + $bug_orig = $1 . $2 . $3; + $bug = "i#$2"; + } else { + $bug = undef; + next; + } + +# print " found $bug\n"; + # remove bug number from the comment; it will be added later a standardized way + $bug_orig =~ s/\#/\\#/; + $line =~ s/(,\s)*[Rr](elated|esolve[ds]):?\s*$bug_orig\s?:?\s*//; + $line =~ s/\s*-\s*$bug_orig\s*//; + $line =~ s/\(?$bug_orig\)?\s*[:,-]?\s*//; + + # bnc# is preferred over n# for novell bugs + $bug =~ s/^n\#/bnc#/; + # deb# is preferred over debian# for debian bugs + $bug =~ s/^debian\#/deb#/; + # easyhack# is sometimes used for fdo# - based easy hacks + $bug =~ s/^easyhack\#/fdo#/; + # someone mistyped fdo as fd0 + $bug =~ s/^fd0\#/fdo#/; + # save the bug number + $pdata->{$module}{$commit_id}{'bugs'}{$bug} = 1; + } + + return $line; +} + +sub standardize_summary($) +{ + my $line = shift; + + $line =~ s/^\s*//; + $line =~ s/\s*$//; + + # lower first letter if the word contains only lowercase letter + if ( $line =~ m/(^.[a-z]+\b)/ ) { + $line =~ m/(^.)/; + my $first_char = lc($1); + $line =~ s/^./$first_char/; + } + + # FIXME: remove do at the end of line + # remove bug numbers + return $line; +} + +sub generate_git_cherry_ids_log($$$$$) +{ + my ($pdata, $repo_dir, $module, $branch_name, $git_args) = @_; + + my $commit_ids_log; + my $commit_ids_log_fh; + $commit_ids_log_fh = File::Temp->new(TEMPLATE => 'lo-commit-stat-ids-XXXXXX', + DIR => '/tmp', + UNLINK => 0); + $commit_ids_log = $commit_ids_log_fh->filename; + + print STDERR "Filtering cherry-picked commits in the git repo: $module...\n"; + + my $cmd = "cd $repo_dir; git cherry $git_args"; + open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!"; + + while (my $line = <GIT>) { + + # skip cherry-picked commits + next if ( $line =~ m/^\-/ ); + + if ( $line =~ m/^\+ / ) { + $line =~ s/^\+ //; + print $commit_ids_log_fh $line; + } + } + + close GIT; + close $commit_ids_log_fh; + + return $commit_ids_log; +} + +sub load_git_log($$$$$$$) +{ + my ($pdata, $repo_dir, $module, $branch_name, $git_command, $git_cherry, $git_args) = @_; + + my $cmd = "cd $repo_dir;"; + my $commit_ids_log; + + if ($git_cherry) { + $commit_ids_log = generate_git_cherry_ids_log($pdata, $repo_dir, $module, $branch_name, $git_args); + $cmd .= " cat $commit_ids_log | xargs -n 1 $git_command -1"; + } else { + $cmd .= " $git_command $git_args"; + } + + my $commit_id; + my $summary; + + print STDERR "Analyzing log from the git repo: $module...\n"; + +# FIXME: ./g pull move submodules in unnamed branches +# my $repo_branch_name = get_branch_name($repo_dir); +# if ( $branch_name ne $repo_branch_name ) { +# die "Error: mismatch of branches:\n" . +# " main repo is on the branch: $branch_name\n" . +# " $module repo is on the branch: $repo_branch_name\n"; +# } + + open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!"; + + while (my $line = <GIT>) { + chomp $line; + + if ( $line =~ m/^commit ([0-9a-z]{20})/ ) { + $commit_id = $1; + $summary=undef; + next; + } + + if ( $line =~ /^Author:\s*([^\<]*)\<([^\>]*)>/ ) { + # get rid of extra empty spaces; + my $name = $1; + my $email = $2; + $name =~ s/\s+$//; + die "Error: Author already defined for the commit {$commit_id}\n" if defined ($pdata->{$module}{$commit_id}{'author'}); + $pdata->{$module}{$commit_id}{'author'}{'name'} = $name; + $pdata->{$module}{$commit_id}{'author'}{'email'} = $email; + next; + } + + if ( $line =~ /^Date:\s+/ ) { + # ignore date line + next; + } + + if ( $line =~ /^\s*$/ ) { + # ignore empty line + next; + } + + unless (defined $pdata->{$module}{$commit_id}{'summary'}) { + $line = search_bugs($pdata, $module, $commit_id, $line); + # FIXME: need to be implemented + # search_keywords($pdata, $line); + + $summary = standardize_summary($line); + $pdata->{$module}{$commit_id}{'summary'} = $summary; + } + } + + close GIT; + unlink $commit_ids_log if ($git_cherry); +} + +sub get_repo_name($) +{ + my $repo_dir = shift; + + open (GIT_CONFIG, "$repo_dir/.git/config") || + die "can't open \"$$repo_dir/.git/config\" for reading: $!\n"; + + while (my $line = <GIT_CONFIG>) { + chomp $line; + + if ( $line =~ /^\s*url\s*=\s*(\S+)$/ ) { + my $repo_name = "$1"; + $repo_name = s/.*\///g; + return "$repo_name"; + } + } + die "Error: can't find repo name in \"$$repo_dir/.git/config\"\n"; +} + +sub load_data($$$$$$$) +{ + my ($pdata, $top_dir, $p_module_dirname, $branch_name, $git_command, $git_cherry, $git_args) = @_; + + foreach my $module (sort { $a cmp $b } keys %{$p_module_dirname}) { + load_git_log($pdata, "$top_dir/$p_module_dirname->{$module}", $module, $branch_name, $git_command, $git_cherry, $git_args); + } +} + +sub get_branch_name($) +{ + my ($top_dir) = @_; + + my $branch; + my $cmd = "cd $top_dir && git branch"; + + open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!"; + + while (my $line = <GIT>) { + chomp $line; + + if ( $line =~ m/^\*\s*(\S+)/ ) { + $branch = "$1"; + } + } + + close GIT; + + die "Error: did not detect git branch name in $top_dir\n" unless defined ($branch); + + return $branch; +} + +sub get_bug_list($$$) +{ + my ($pdata, $pbugs, $check_bugzilla) = @_; + + # associate bugs with their summaries and fixers + foreach my $module ( keys %{$pdata}) { + foreach my $id ( keys %{$pdata->{$module}}) { + foreach my $bug (keys %{$pdata->{$module}{$id}{'bugs'}}) { + my $author = $pdata->{$module}{$id}{'author'}{'name'}; + my $summary = $pdata->{$module}{$id}{'summary'}; + $pbugs->{$bug}{'summary'} = $summary; + $pbugs->{$bug}{'author'}{$author} = 1; + } + } + } + + # try to replace summaries with bug names from bugzilla + if ($check_bugzilla) { + print "Getting bug titles:\n"; + foreach my $bug ( sort { $a cmp $b } keys %{$pbugs}) { + $pbugs->{$bug}{'summary'} = get_bug_name($bug, $pbugs->{$bug}{'summary'}); + } + } +} + +sub open_log_file($$$$$$) +{ + my ($log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki) = @_; + + my $logfilename = "$log_prefix-$branch_name-$log_suffix"; + $logfilename = "$log_dir/$logfilename" if (defined $log_dir); + if ($wiki) { + $logfilename .= ".wiki"; + } else { + $logfilename .= ".log"; + } + + if (-f $logfilename) { + print "WARNING: The log file already exists: $logfilename\n"; + print "Do you want to overwrite it? (Y/n)?\n"; + my $answer = <STDIN>; + chomp $answer; + $answer = "y" unless ($answer); + die "Please, rename the file or choose another log suffix\n" if ( lc($answer) ne "y" ); + } + + my $log; + open($log, '>', $logfilename) || die "Can't open \"$logfilename\" for writing: $!\n"; + + return $log; +} + +sub print_commit_summary($$$$$$) +{ + my ($summary, $pmodule_title, $pbugs, $pauthors, $prefix, $log) = @_; + + return if ( $summary eq "" ); + + # print module title if not done yet + if ( defined ${$pmodule_title} ) { + print $log "${$pmodule_title}\n"; + ${$pmodule_title} = undef; + } + + # finally print the summary line + my $bugs = ""; + if ( %{$pbugs} ) { + $bugs = " (" . join (", ", keys %{$pbugs}) . ")"; + } + + my $authors = ""; + if ( %{$pauthors} ) { + $authors = " [" . join (", ", keys %{$pauthors}) . "]"; + } + + print $log $prefix, $summary, $bugs, $authors, "\n"; +} + +sub print_commits($$$) +{ + my ($pdata, $log, $wiki) = @_; + + foreach my $module ( sort { $a cmp $b } keys %{$pdata}) { + # check if this module has any entries at all + my $module_title = "+ $module"; + if ( %{$pdata->{$module}} ) { + my $old_summary=""; + my %authors = (); + my %bugs = (); + foreach my $id ( sort { lc $pdata->{$module}{$a}{'summary'} cmp lc $pdata->{$module}{$b}{'summary'} } keys %{$pdata->{$module}}) { + my $summary = $pdata->{$module}{$id}{'summary'}; + if ($summary ne $old_summary) { + print_commit_summary($old_summary, \$module_title, \%bugs, \%authors, " + ", $log); + $old_summary = $summary; + %authors = (); + %bugs = (); + } + # collect bug numbers + if (defined $pdata->{$module}{$id}{'bugs'}) { + foreach my $bug (keys %{$pdata->{$module}{$id}{'bugs'}}) { + $bugs{$bug} = 1; + } + } + # collect author names + my $author = $pdata->{$module}{$id}{'author'}{'name'}; + $authors{$author} = 1; + } + print_commit_summary($old_summary, \$module_title, \%bugs, \%authors, " + ", $log); + } + } +} + +sub get_bug_name($$) +{ + my ($bug, $summary) = @_; + print "$bug: "; + + $bug =~ m/(?:(\w*)\#+(\d+))/; # fdo#12345 + my $bugzilla = $1; # fdo + my $bug_number = $2; # 12345 + + if ( $bugzillas{$bugzilla} ) { + my $url = $bugzillas{$bugzilla} . $bug_number; + my $ua = LWP::UserAgent->new; + $ua->timeout(10); + $ua->env_proxy; + my $response = $ua->get($url); + if ($response->is_success) { + my $title = decode('utf8', $response->title); + if ( $title =~ s/^(?:Bug $bug_number \S+|$bug_number –) // ) { + print "$title\n"; + return $title; + } else { + print "warning: not found; using commit message (only got $title)"; + } + } + } + print "\n"; + + return $summary; +} + +sub print_bugs($$$$) +{ + my ($pbugs, $log, $wiki) = @_; + + # sort alphabetically by bugzilla-type, but within that numerically + foreach my $bug ( sort { ($a =~ /(\D+)/)[0] cmp ($b =~ /(\D+)/)[0] || + ($a =~ /(\d+)/)[0] <=> ($b =~ /(\d+)/)[0] } keys %{$pbugs}) { + my $summary = $pbugs->{$bug}{'summary'}; + + my $authors = ""; + if ( %{$pbugs->{$bug}{'author'}} ) { + $authors = " [" . join (", ", keys %{$pbugs->{$bug}{'author'}}) . "]"; + } + + $bug =~ s/(.*)\#(.*)/# {{$1|$2}}/ if ($wiki); + print $log $bug, " ", $summary, $authors, "\n"; + } +} + +sub print_bugs_changelog($$$$) +{ + my ($pbugs, $log, $wiki) = @_; + + foreach my $bug ( sort { $a cmp $b } keys %{$pbugs}) { + my $summary = $pbugs->{$bug}{'summary'}; + + my $authors = ""; + if ( %{$pbugs->{$bug}{'author'}} ) { + $authors = " [" . join (", ", keys %{$pbugs->{$bug}{'author'}}) . "]"; + } + + print $log " + $summary ($bug)$authors\n"; + } +} + +sub print_bugnumbers($$$$) +{ + my ($pbugs, $log, $wiki) = @_; + + print $log join ("\n", sort { $a cmp $b } keys %{$pbugs}), "\n"; +} + +sub generate_log($$$$$$$$) +{ + my ($pused_data, $print_func, $log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki) = @_; + + my $log = open_log_file($log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki); + & {$print_func} ($pused_data, $log, $wiki); + close $log; +} + +######################################################################## +# help + +sub usage() +{ + print "This script generates LO git commit summary\n\n" . + + "Usage: lo-commit-stat [--help] [--no-submodules] [--module=<module>] --log-dir=<dir> --log-suffix=<string> topdir [git_arg...]\n\n" . + + "Options:\n" . + " --help print this help\n" . + " --no-submodule read changes just from the main repository, ignore submodules\n" . + " --module=<module> summarize just changes from the given module, use \"core\"\n" . + " for the main module\n" . + " --log-dir=<dir> directory where to put the generated log\n" . + " --log-suffix=<string> suffix of the log file name; the result will be\n" . + " commit-log-<branch>-<log-name-suffix>.log; the branch name\n" . + " is detected automatically\n" . + " --commits generate log with all commits (default)\n" . + " --bugs generate log with bugzilla entries\n" . + " --bugs-changelog generate log with bugzilla entries, use changelog style\n" . + " --bugs-wiki generate log with bugzilla entries, use wiki markup\n" . + " --bugs-numbers generate log with bugzilla numbers\n" . + " --rev-list use \"git rev-list\" instead of \"git log\"; useful to check\n" . + " differences between branches\n" . + " --cherry use \"git cherry\" instead of \"git log\"; detects cherry-picked\n" . + " commits between branches\n" . + " topdir directory with the libreoffice/core clone\n" . + " git_arg extra parameters passed to the git command to define\n" . + " the area of interest; The default command is \"git log\" and\n" . + " parameters might be, for example, --after=\"2010-09-27\" or\n" . + " TAG..HEAD; with the option --rev-list, useful might be, for\n" . + " example origin/master ^origin/libreoffice-3-3; with the option\n" . + " --rev-list, useful might be, for example libreoffice-3.6.3.2\n" . + " libreoffice-3.6.4.1\n"; +} + + +####################################################################### +####################################################################### +# MAIN +####################################################################### +####################################################################### + + +my $module; +my %generate_log = (); +my $top_dir; +my $log_dir; +my $log_suffix; +my $log; +my $list_bugs = 0; +my $check_bugzilla = 0; +my $branch_name; +my $git_command = "git log"; +my $git_cherry; +my $git_args = ""; +my %data; +my %bugs = (); + + +foreach my $arg (@ARGV) { + if ($arg eq '--help') { + usage(); + exit; + } elsif ($arg eq '--no-submodule') { + $module = "core"; + } elsif ($arg =~ m/--module=(.*)/) { + $module = $1; + } elsif ($arg =~ m/--log-suffix=(.*)/) { + $log_suffix = "$1"; + } elsif ($arg =~ m/--log-dir=(.*)/) { + $log_dir = "$1"; + } elsif ($arg eq '--commits') { + $generate_log{"commits"} = 1; + } elsif ($arg eq '--bugs') { + $generate_log{"bugs"} = 1; + $check_bugzilla = 1; + $list_bugs = 1; + } elsif ($arg eq '--bugs-changelog') { + $generate_log{"bugs-changelog"} = 1; + $check_bugzilla = 1; + $list_bugs = 1; + } elsif ($arg eq '--bugs-wiki' || $arg eq '--wikibugs') { + $generate_log{"bugs-wiki"} = 1; + $check_bugzilla = 1; + $list_bugs = 1; + } elsif ($arg eq '--bugs-numbers' || $arg eq '--bug-numbers') { + $generate_log{"bugs-numbers"} = 1; + $list_bugs = 1; + } elsif ($arg eq '--rev-list') { + $git_command = "git rev-list --pretty=medium" + } elsif ($arg eq '--cherry') { + $git_command = "git log"; + $git_cherry = 1; + } else { + if (! defined $top_dir) { + $top_dir=$arg; + } else { + $git_args .= " $arg"; + } + } +} + +# default log +unless (%generate_log) { + $generate_log{"commits"} = 1; +} + +# we want only one module +if ($module) { + my $name = $module_dirname{$module}; + %module_dirname = (); + $module_dirname{$module} = $name; +} + +(defined $top_dir) || die "Error: top directory is not defined\n"; +(-d "$top_dir") || die "Error: not a directory: $top_dir\n"; +(-f "$top_dir/.git/config") || die "Error: can't find $top_dir/.git/config\n"; + +(!defined $log_dir) || (-d $log_dir) || die "Error: directory does no exist: $log_dir\n"; + +(defined $log_suffix) || die "Error: define log suffix using --log-suffix=<string>\n"; + +$branch_name = get_branch_name($top_dir); + +load_data(\%data, $top_dir, \%module_dirname, $branch_name, $git_command, $git_cherry, $git_args); +get_bug_list(\%data, \%bugs, $check_bugzilla) if ($list_bugs); + +generate_log(\%data, \&print_commits, $log_dir, "commits", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"commits"}); +generate_log(\%bugs, \&print_bugs, $log_dir, "bugs", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs"}); +generate_log(\%bugs, \&print_bugs, $log_dir, "bugs", $log_suffix, $top_dir, $branch_name, 1) if (defined $generate_log{"bugs-wiki"}); +generate_log(\%bugs, \&print_bugs_changelog, $log_dir, "bugs-changelog", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs-changelog"}); +generate_log(\%bugs, \&print_bugnumbers, $log_dir, "bug-numbers", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs-numbers"}); diff --git a/bin/lo-pack-sources b/bin/lo-pack-sources new file mode 100755 index 000000000..8c795dc17 --- /dev/null +++ b/bin/lo-pack-sources @@ -0,0 +1,485 @@ +#!/usr/bin/perl + eval 'exec /usr/bin/perl -S $0 ${1+"$@"}' + if $running_under_some_shell; +#!/usr/bin/perl + +use strict; +use File::Copy; +use File::Temp qw/ tempfile tempdir /; + +my %module_dirname = ( + "core" => "", + "dictionaries" => "dictionaries", + "help" => "helpcontent2", + "translations" => "translations" +); +my $lo_topdir_name; + +# get libreoffice-build version from the given libreoffice-build sources +sub get_config_version($) +{ + my ($lo_core_dir) = @_; + my $version; + + open (CONFIGURE, "$lo_core_dir/configure.ac") || + die "can't open \"$lo_core_dir/configure.ac\" for reading: $!\n"; + + while (my $line = <CONFIGURE>) { + chomp $line; + + if ($line =~ /AC_INIT\s*\(\s*libreoffice-build\s*,\s*([\w\.]*)\)/) { + $version="$1"; + } + } + close (CONFIGURE); + return $version; +} + +# increment the version for a test build: +# + add 'a' if the version ended with a number +# + bump the letter otherwise +sub inc_test_version($) +{ + my ($version) = @_; + + my $lastchar = chop $version; + my $new_version; + + if ($lastchar =~ /\d/) { + return "$version" . "$lastchar" . "a"; + } elsif ($lastchar =~ /\w/) { + # select next letter alphabetically: a->b, b->c, ... + $lastchar =~ tr/0a-zA-Z/a-zA-Z0/; + return "$version" . "$lastchar"; + } else { + die "Can't generate test version from \"$version$lastchar\n"; + } +} + +sub get_release_version($$$$) +{ + my ($config_version, $state_config_version, $state_release_version, $inc_version) = @_; + my $release_version; + + if (defined $state_config_version && + defined $state_release_version && + "$state_config_version" eq "$config_version") { + $release_version = "$state_release_version"; + } else { + $release_version = "$config_version"; + } + + if ( defined $inc_version ) { + $release_version = inc_test_version($release_version); + } + + return $release_version; +} + +# copy files to temp dir; showing a progress; using a black list +sub copy_dir_filter_and_show_progress($$) +{ + my ($source_dir, $target_dir) = @_; + + print "Copying \"$source_dir\" -> \"$target_dir\"..."; + # copy sources from git and show progress + system ("cd $source_dir && " . + "git archive --format=tar HEAD | " . + " tar -xf - -C $target_dir ") && + die "Error: copying failed: $!\n"; + print "\n"; +} + +# copy files to temp dir; showing a progress; using a black list +sub remove_empty_submodules($) +{ + my ($target_topdir) = @_; + + foreach my $submodule (sort values %module_dirname) { + next unless ($submodule); + print "Removing empty submodule: $submodule...\n"; + rmdir "$target_topdir/$submodule" || die "Error: Can't remove submodule directory: $target_topdir/$submodule"; + } +} + +# copy the source directory into a tmp directory +# omit the .git subdirectories +sub copy_lo_module_to_tempdir($$$) +{ + my ($source_dir, $module, $lo_topdir_name) = @_; + my $tempdir = tempdir( 'libreoffice-XXXXXX', DIR => File::Spec->tmpdir ); + + mkdir "$tempdir/$lo_topdir_name" || die "Can't create directory \"$tempdir/$lo_topdir_name\": $!\n"; + mkdir "$tempdir/$lo_topdir_name/$module_dirname{$module}" || die "Can't create directory \"$tempdir/$lo_topdir_name/$module_dirname{$module}\": $!\n" if ($module_dirname{$module}); + + copy_dir_filter_and_show_progress("$source_dir/$module_dirname{$module}", "$tempdir/$lo_topdir_name/$module_dirname{$module}"); + remove_empty_submodules("$tempdir/$lo_topdir_name/") if ($module eq "core"); + + return $tempdir; +} + +sub generate_lo_module_changelog($$$) +{ + my ($source_dir, $lo_module_release_topdir, $module) = @_; + + my $log_name = "ChangeLog"; + $log_name .= "-$module_dirname{$module}" if ($module_dirname{$module}); + print "Generating changelog for $module...\n"; + system ("cd $source_dir/$module_dirname{$module} && " . + "git log --date=short --pretty='format:%cd %an <%ae> [%H]%n%n%w(0,8,8)%s%n%e%+b' " . + ">$lo_module_release_topdir/$log_name" ) && + die "Error: generating failed: $!\n"; +} + +sub run_autogen($$) +{ + my ($dir, $module) = @_; + + print "Running autogen for $module...\n"; + system ("cd $dir && " . + "NOCONFIGURE=1 ./autogen.sh && " . + "rm -rf autom4te.cache && " . + "cd - >/dev/null 2>&1") && die "Error: autogen failed: $!\n"; +} + +sub generate_sources_version_file($$) +{ + my ($dir, $release_version) = @_; + + open (VERFILE, ">$dir/sources.ver") || die "Can't open $dir/sources.ver: $!\n"; + + print VERFILE "lo_sources_ver=$release_version\n"; + + close VERFILE; +} + +sub generate_tarball($$$) +{ + my ($dir, $tarball, $tar_compress_option) = @_; + + print "Creating $tarball..."; + # generate the tarball in the current directory; avoid "./" prefix in the stored paths; show progress + system ("tar -c $tar_compress_option -f $tarball -C $dir $lo_topdir_name") && + die "Error: releasing failed: $!\n"; + print "\n"; +} + +sub generate_md5($) +{ + my ($filename) = @_; + + print "Generating MD5...\n"; + system ("md5sum $filename >$filename.md5") && + die "Error: releasing failed: $!\n"; +} + +sub default_releases_state_file($) +{ + my ($lo_core_dir) = @_; + + my $rootdir = $lo_core_dir; + $rootdir =~ s/^(.*?)\/?[^\/]+\/?$/$1/; + + my $releases_state_file; + if ($rootdir) { + $releases_state_file = "$rootdir/.releases"; + } else { + $releases_state_file = ".releases"; + } + + return "$releases_state_file"; +} + +sub load_releases_state($) +{ + my ($releases_state_file) = @_; + + my $state_config_version; + my $state_release_version; + + if (open (STATE, "$releases_state_file")) { + + while (my $line = <STATE>) { + chomp $line; + + if ($line =~ /^\s*configure_version\s*=\s*(.*)$/) { + $state_config_version = "$1"; + } elsif ($line =~ /^\s*released_version\s*=\s*(.*)$/) { + $state_release_version = "$1"; + } + } + close (STATE); + } + + return $state_config_version, $state_release_version; +} + +sub save_releases_state($$$) +{ + my ($releases_state_file, $config_version, $release_version) = @_; + + open (STATE, '>', "$releases_state_file") || + die "Can't open \"$releases_state_file\" for writing: $!\n"; + + print STATE "configure_version = $config_version\n"; + print STATE "released_version = $release_version\n"; + + close (STATE); +} + +sub remove_tempdir($) +{ + my ($tempdir) = @_; + +# print "Cleaning $tempdir...\n"; + system ("rm -rf $tempdir") && die "Error: rm failed: $!\n"; +} + +sub check_if_file_exists($$) +{ + my ($file, $force) = @_; + + if (-e $file) { + if (defined $force) { + print "Warning: $file already exists and will be replaced!\n"; + } else { + die "Error: $file already exists.\n". + " Use --force if you want to replace it.\n"; + } + } +} + +sub check_if_already_released($$$$$$) +{ + my ($p_module_tarball_name, $force, $bzip2, $xz, $pack_lo_core, $pack_lo_modules) = @_; + + foreach my $tarball_name ( sort values %{$p_module_tarball_name} ) { + check_if_file_exists("$tarball_name.tar.bz2", $force) if (defined $bzip2); + check_if_file_exists("$tarball_name.tar.xz", $force) if (defined $xz); + } +} + +sub prepare_module_sources($$$$) +{ + my ($source_dir, $release_version, $module, $lo_topdir_name) = @_; + + # prepare sources + my $temp_dir = copy_lo_module_to_tempdir($source_dir, $module, $lo_topdir_name); + generate_lo_module_changelog($source_dir, "$temp_dir/$lo_topdir_name", $module); + run_autogen("$temp_dir/$lo_topdir_name", $module) if ($module eq 'core'); + generate_sources_version_file("$temp_dir/$lo_topdir_name", $release_version) if ($module eq 'core'); + + return $temp_dir; +} + +sub pack_module_sources($$$$) +{ + my ($temp_dir, $md5, $tarball, $tar_compress_option) = @_; + + generate_tarball($temp_dir, $tarball, $tar_compress_option); + generate_md5($tarball) if (defined $md5); +} + +sub generate_module_tarball($$$$$$$$) +{ + my ($source_dir, $release_version, $module, $md5, $bzip2, $xz, $lo_topdir_name, $module_tarball_name) = @_; + + my $temp_dir = prepare_module_sources($source_dir, $release_version, $module, $lo_topdir_name); + pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.bz2", "--bzip2") if (defined $bzip2); + pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.xz", "--xz") if (defined $xz); + remove_tempdir($temp_dir); +} + + +sub generate_tarballs($$$$$$$$$) +{ + my ($source_dir, $release_version, $md5, $bzip2, $xz, $lo_topdir_name, $p_module_tarball_name, $pack_lo_core, $pack_lo_modules) = @_; + + foreach my $module (sort keys %{$p_module_tarball_name} ) { + print "\n--- Generating $module ---\n"; + generate_module_tarball($source_dir, $release_version, $module, $md5, $bzip2, $xz, $lo_topdir_name, $p_module_tarball_name->{$module}); + } +} + + +sub usage() +{ + print "This tool helps to pack the libreoffice-build and module sources\n\n" . + + "Usage:\n". + "\tlo-pack-sources [--help]\n" . + "\t [--force] [--md5] [--bzip2] [--xz]\n" . + "\t [--version][--set-version=<ver>] [--inc-version]\n" . + "\t [--no-submodule] [--module=<module>]\n" . + "\t [dir]\n\n" . + + "Options:\n\n" . + "\t--help: print this help\n" . + "\t--force: replace an already existing release of the same version\n" . + "\t--md5: generate md5 sum for the final tarball\n" . + "\t--bzip2: generate tarballs compressed by bzip2\n" . + "\t--xz: generate tarballs compressed by xz (default)\n" . + "\t--version: just print version of the released package but do not\n" . + "\t\trelease it; the version is affected by the other options, e.g.\n" . + "\t\t--inc-version\n" . + "\t--set-version: force another version\n" . + "\t--inc-version: increment the latest version; there is a difference\n" . + "\t\tbetween test release (default) and final (not yet supported)\n" . + "\t--no-submodule: do not pack sources from git submodules\n" . + "\t--module=<module>: pack just a single module, use \"core\"\n" . + "\t\tfor the main git repo,\n" . + "\tdir: path of the source directory, either libreoffice-build or module\n"; +} + + +my $module; +my $ptf; +my $md5; +my $bzip2; +my $xz; +my $inc_version; +my $config_version; +my $set_version; +my $get_config_version; +my $release_version; +my $pack_lo_core=1; +my $pack_lo_modules=1; +my $source_dir; +my $releases_state_file; +my $state_config_version; +my $state_release_version; +my $lo_core_tempdir; +my $force; +my $verbose=1; +my %module_tarball_name; + +################### +# Arguments parsing +################### + +for my $arg (@ARGV) { + if ($arg eq '--help' || $arg eq '-h') { + usage; + exit 0; + } elsif ($arg eq '--force') { + $force=1; + } elsif ($arg eq '--md5') { + $md5=1; + } elsif ($arg eq '--bzip2') { + $bzip2=1; + } elsif ($arg eq '--xz') { + $xz=1; + } elsif ($arg eq '--version') { + $get_config_version=1; + $verbose = undef; + } elsif ($arg eq '--inc-version') { + $inc_version=1 + } elsif ($arg =~ m/--set-version=(.*)/) { + $set_version="$1"; + } elsif ($arg eq '--no-submodule') { + $module = "core"; + } elsif ($arg =~ m/--module=(.*)/) { + # process just one module and do not pack libreoffice-build + die("Error: unknown module: $1") unless (defined $module_dirname{$1}); + $module = $1; + } elsif ($arg =~ /^-/ ) { + die "Error: unknown option: $arg\n"; + } else { + if (! defined $source_dir) { + $source_dir = $arg; + } else { + die "Error: Too many arguments $arg\n"; + } + } +} + +# ugly hack; we want only one module +if ($module) { + my $name = $module_dirname{$module}; + %module_dirname = (); + $module_dirname{$module} = $name; +} + +################### +# Initial checks +################### + +unless ( defined $source_dir ) { + die "Error: undefined source directory, try --help\n"; +} + +unless ( -d "$source_dir" ) { + die "Error: is not a directory: $source_dir\n"; +} + +# check if it is a valid libreoffice-core directory +unless (-f "$source_dir/autogen.sh" && -f "$source_dir/config_host.mk.in") { + die "Error: \"$source_dir\" is not a valid libreoffice-core directory\n"; +} + +if (defined $set_version && defined $inc_version) { + die "Error: --set-version and --inc-version options can't be used together\n"; +} + +# default compression +$xz = 1 unless (defined $xz || defined $bzip2); + + +################### +# Main logic +################### + + +print "Source: $source_dir\n" if ($verbose); + +# detect some paths +$releases_state_file = default_releases_state_file($source_dir) unless (defined $releases_state_file); + +# detect versions +$config_version = get_config_version($source_dir); +($state_config_version, $state_release_version) = load_releases_state($releases_state_file); +if (defined $set_version) { + $release_version = "$set_version"; +} else { + $release_version = get_release_version($config_version, $state_config_version, $state_release_version, $inc_version); +} + +# define tarball names +print "Detected module:\n"; +foreach my $module (sort keys %module_dirname) { + if (-e "$source_dir/$module_dirname{$module}/.git") { + print " $module\n"; + if ($module eq "core") { + $module_tarball_name{$module} = "libreoffice-$release_version"; + } else { + $module_tarball_name{$module} = "libreoffice-$module-$release_version"; + } + } else { + print "did not found: $source_dir/$module_dirname{$module}/.git\n"; + print "Warning: $module sources are not available -> skipping\n"; + } +} + +# top directory inside the source tarballs +$lo_topdir_name = "libreoffice-$release_version"; + +print "Default version : $config_version\n" if ($verbose && defined $config_version); +print "Last used version : $state_release_version\n" if ($verbose && defined $state_release_version); +print "New version : $release_version\n" if ($verbose); + +# do the real job +if ( defined $get_config_version ) { + print "$release_version\n"; +} else { + check_if_already_released(\%module_tarball_name, $force, $bzip2, $xz, $pack_lo_core, $pack_lo_modules); + + # give a chance to stop the process + print ("\nWaiting 3 seconds...\n"); + sleep 3; + + generate_tarballs($source_dir, $release_version, $md5, $bzip2, $xz, $lo_topdir_name, \%module_tarball_name, $pack_lo_core, $pack_lo_modules); + + if ( defined $releases_state_file ) { + save_releases_state($releases_state_file, $config_version, $release_version); + } +} diff --git a/bin/lo-xlate-lang b/bin/lo-xlate-lang new file mode 100755 index 000000000..d158b3fd5 --- /dev/null +++ b/bin/lo-xlate-lang @@ -0,0 +1,213 @@ +#!/usr/bin/env perl + +use strict; + +my $progname=$0; $progname = $& if $progname =~ m,[^/]+$,; + +my %PREFIX; # used to search for prefix numbers +my %ISOCODE; # used to search for iso codes +my %LANGUAGE; # used to search for language names + +#======================================================================= +# initialisation code - stuff the DATA into the CODES hash +#======================================================================= +sub init { + + my $prefix; + my $code; + my $name; + + + while (<DATA>) + { + next unless /\S/; + chop; + ($prefix, $code, $name ) = split(/:/, $_, 3); + $PREFIX{$prefix} = $prefix; + $PREFIX{$code} = $prefix; + $PREFIX{$name} = $prefix; + + $ISOCODE{$prefix} = $code; + $ISOCODE{$code} = $code; + $ISOCODE{$name} = $code; + + $LANGUAGE{$prefix} = $name; + $LANGUAGE{$code} = $name; + $LANGUAGE{$name} = $name; + } +} + + +#======================================================================= +# usage - error message +#======================================================================= +sub usage { + my $errmsg = shift; + my $errcode = shift; + print STDERR "$progname: $errmsg\n" if $errmsg; + print STDERR "$progname: Converts between prefix codes, iso codes and langnames\n"; + print STDERR " Usage: $progname (-i|-l|-p|-h) <code>|all\n"; + print STDERR " -i <code>: convert prefix to iso code (ex: 03 -> pt)\n"; + print STDERR " -l <code>: convert iso code to language name (ex: pt -> portuguese)\n"; + print STDERR " -p <code>: convert iso code to prefix (ex: pt -> 03)\n"; + print STDERR " the code can either be an iso code, a prefix or even a language name\n"; + print STDERR " The special code \"all\" asks for all possible values.\n\n"; + print STDERR " -h : print this help\n"; + exit $errcode; +} + +#======================================================================= +# main - +#======================================================================= +init(); + +my ($LanguageCode, $LanguageMap); + +while ($ARGV[0] =~ /^-/) { + $_ = shift; + if (m/^-i/) { + $LanguageMap = \%ISOCODE; + } + elsif (m/^-l/) { + $LanguageMap = \%LANGUAGE; + } + elsif (m/^-p/) { + $LanguageMap = \%PREFIX; + } + elsif (m/^-h/) { + usage("",0); + } + else { + usage ("unknown option $_",1); + } +} + +usage ("no operation specified on command line",1) + if (!$LanguageMap); + +usage ("no language code specified on command line",1) + if (!($LanguageCode = shift)); + +if ($LanguageCode =~ (m/^all$/)) { + # Asked for all codes + my $old=""; + foreach my $key (sort values %$LanguageMap) { + if ($key ne $old) { + print "$key "; + $old=$key; + } + } + print "\n"; + exit 0; +} + +usage ("no mapping found for $LanguageCode\n",1) + if (!($LanguageMap->{$LanguageCode})); + +print $LanguageMap->{$LanguageCode}, "\n"; + +1; + +# keep third column names here with openoffice-dir/share/*/<long lang name>/ + +__DATA__ +:be:belarusian +:bg:bulgarian +:bn:bengali +:bs:bosnian +:en-GB:english_british +:gu:gujarati +:hr:croatian +:km:khmer +:kmr-Latn:Kurmanji +:pa-IN:punjabi +:rw:kinarwanda +:xh:xhosa +:lt:lithuanian +:ne:nepali +:vi:vietnamese +:nso:northern_sotho +:ss:swazi +:sr:serbian +:ve:venda +:ts:tsonga +:st:southern_sotho +:tn:tswana +:br:breton +:ga:gaelic +:gd:scottish_gaelic +:th:thai +:hi:hindi +:bs-BA:bosnian +:en-ZA:english_southafrican +:mk:macedonian +:as:assamese +:ml:malayalam +:mr:marathi +:or:odia +:ur:urdu +:fa:farsi +:lv:latvian +:nr:ndebele +:ne:nepalese +:sh:serbian +:te:telugu +:ta:tamil +:tg:tajik +:ka:georgian +:eo:esperanto +:uk:ukrainian +:kk:kazakh +:dz:dzongkha +:kn:kannada +:gl:galician +:uz:uzbek +:oc:occitan +:ro:romanian +:eu:basque +:mn:mongolian +:om:oromo +:bo:tibetan +:ast:asturian +:is:icelandic +:ug:uighur +:si:sinhala +:id:indonesian +:my:burmese +:am:amharic +:gug:guarani +:szl:upper_silesian +01:en-US:english_american +03:pt:portuguese +07:ru:russian +26:ns:northernsotho +27:af:afrikaans +28:zu:zulu +30:el:greek +31:nl:dutch +33:fr:french +34:es:spanish +35:fi:finnish +36:hu:hungarian +37:ca:catalan +39:it:italian +42:cs:czech +43:sk:slovak +45:da:danish +46:sv:swedish +47:nb:norwegian +48:pl:polish +49:de:german +50:sl:slovenian +53:cy:welsh +55:pt-BR:portuguese_brazilian +77:et:estonian +79:nn:norwegian_nynorsk +81:ja:japanese +82:ko:korean +86:zh-CN:chinese_simplified +88:zh-TW:chinese_traditional +90:tr:turkish +91:hi:hindi +96:ar:arabic +97:he:hebrew diff --git a/bin/lolcat b/bin/lolcat new file mode 100755 index 000000000..27bb32624 --- /dev/null +++ b/bin/lolcat @@ -0,0 +1,21 @@ +#!/usr/bin/perl -w + +use strict; +use IO::Handle; + +die "Usage: $0 identifier\n" . + "(identifier is for example org.libreoffice)" unless $#ARGV == 0; + +my $id = $ARGV[0]; + +open (LOGCAT, "adb logcat |") || die "Could not open pipe from adb logcat"; +my $pid = ''; + +while (<LOGCAT>) { + if (m!^I/ActivityManager\( *\d+\): Start proc $id for activity .*: pid=(\d+)!) { + $pid = $1; + } elsif (m!^[EIWD]/[^(]+\( *$pid\)!) { + print $_; + STDOUT->flush(); + } +} diff --git a/bin/module-deps.pl b/bin/module-deps.pl new file mode 100755 index 000000000..abec124e4 --- /dev/null +++ b/bin/module-deps.pl @@ -0,0 +1,556 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Getopt::Long qw(GetOptions VersionMessage); +use Pod::Usage; + +my $gnumake; +my $src_root; +my $makefile_build; +my $verbose = 0; +my $no_leaf; +my $from_file; +my $to_file; +my $output_file; +my $preserve_libs = 0; +my $toposort = 0; +my %merged_libs; + +sub logit($) +{ + print STDERR shift if ($verbose); +} + +sub read_deps() +{ + my $p; + my $to; + my $invalid_tolerance = 100; + my $line_count = 0; + my %deps; + if (defined $to_file) + { + open($to, ">$to_file") or die "can not open file for writing $to_file"; + } + if (defined $from_file) { + open ($p, $from_file) || die "can't read deps from cache file: $!"; + } else { + open ($p, "ENABLE_PRINT_DEPS=1 $gnumake -qrf $makefile_build|") || die "can't launch make: $!"; + } + $|=1; + print STDERR "reading deps "; + while (<$p>) { + my $line = $_; + $line_count++; + print STDERR '.' if ($line_count % 10 == 0); + logit($line); + print $to $line if defined $to_file; + chomp ($line); + if ($line =~ m/^MergeLibContents:\s+(\S+.*)\s*$/) { + for my $dep (split / /, $1) { + $merged_libs{$dep} = 1 if $dep ne ''; + } + } elsif ($line =~ m/^LibraryDep:\s+(\S+) links against (.*)$/) { +# if ($line =~ m/^LibraryDep:\s+(\S+)\s+links against/) { + $deps{$1} = ' ' if (!defined $deps{$1}); + $deps{$1} = $deps{$1} . ' ' . $2; + } elsif ($line =~ m/^LibraryDep:\s+links against/) { +# these need fixing, we call gb_LinkTarget__use_$... +# and get less than normal data back to gb_LinkTarget_use_libraries +# print STDERR "ignoring unhelpful external dep\n"; + } elsif ($invalid_tolerance < 0) { +# print "read all dependencies to: '$line'\n"; + last; + } else { +# print "no match '$line'\n"; + $invalid_tolerance--; + } + } + close ($p); + print STDERR " done\n"; + + return \%deps; +} + +# graphviz etc. don't like some names +sub clean_name($) +{ + my $name = shift; + $name =~ s/[\-\/\.]/_/g; + return $name; +} + +# first create nodes for each entry +sub clean_tree($) +{ + my $deps = shift; + my %tree; + for my $name (sort keys %{$deps}) { + my $need_str = $deps->{$name}; + $need_str =~ s/^\s+//g; + $need_str =~ s/\s+$//g; + my @needs = split /\s+/, $need_str; + $name =~ m/^([^_]+)_(\S+)$/ || die "invalid target name: '$name'"; + my $type = $1; + my $target = clean_name ($2); + $type eq 'Executable' || $type eq 'Library' || + $type eq 'CppunitTest' || die "Unknown type '$type'"; + + my %result; + $result{type} = $type; + $result{target} = $target; + $result{merged} = 0; + my @clean_needs; + for my $need (@needs) { + push @clean_needs, clean_name($need); + } + $result{deps} = \@clean_needs; + if (defined $tree{$target}) { + logit("warning -duplicate target: '$target'\n"); + delete($tree{$target}); + } + $tree{$target} = \%result; + + logit("$target ($type): " . join (',', @clean_needs) . "\n"); + } + return \%tree; +} + +sub has_child_dep($$$) +{ + my ($tree,$search,$name) = @_; + my $node = $tree->{$name}; + return defined $node->{flat_deps}->{$search}; +} + +# flatten deps recursively into a single hash per module +sub build_flat_dep_hash($$); +sub build_flat_dep_hash($$) +{ + my ($tree, $name) = @_; + my %flat_deps; + + my $node = $tree->{$name}; + return if (defined $node->{flat_deps}); + + # build flat deps for children + for my $child (@{$node->{deps}}) { + build_flat_dep_hash($tree, $child) + } + + for my $child (@{$node->{deps}}) { + $flat_deps{$child} = 1; + for my $dep (@{$tree->{$child}->{deps}}) { + $flat_deps{$dep} = 1; + } + } + $node->{flat_deps} = \%flat_deps; + + # useful debugging ... + if (defined $ENV{DEP_CACHE_FILE}) { + logit("node '$name' has flat-deps: '" . join(',', keys %flat_deps) . "' " . + "vs. '" . join(',', @{$node->{deps}}) . "'\n"); + } +} + +# many modules depend on vcl + sal, but vcl depends on sal +# so we want to strip sal out - and the same for many +# similar instances +sub prune_redundant_deps($) +{ + my $tree = shift; + for my $name (sort keys %{$tree}) { + build_flat_dep_hash($tree, $name); + } +} + +# glob on libo directory +sub create_lib_module_map() +{ + my %l2m; + # hardcode the libs that don't have a directory + $l2m{'merged'} = 'merged'; + + for (glob($src_root."/*/Library_*.mk")) + { + /.*\/(.*)\/Library_(.*)\.mk/; + # add module -> module + $l2m{$1} = $1; + # add lib -> module + $l2m{$2} = $1; + } + return \%l2m; +} + +# call prune redundant_deps +# rewrite the deps array +sub optimize_tree($) +{ + my $tree = shift; + prune_redundant_deps($tree); + for my $name (sort keys %{$tree}) { + my $result = $tree->{$name}; + logit("minimising deps for $result->{target}\n"); + my @newdeps; + for my $dep (@{$result->{deps}}) { + # is this implied by any other child ? + logit("checking if '$dep' is redundant\n"); + my $preserve = 1; + for my $other_dep (@{$result->{deps}}) { + next if ($other_dep eq $dep); + if (has_child_dep($tree,$dep,$other_dep)) { + logit("$dep is implied by $other_dep - ignoring\n"); + $preserve = 0; + last; + } + } + push @newdeps, $dep if ($preserve); + } + # re-write the shrunk set to accelerate things + $result->{deps} = \@newdeps; + } + return $tree; +} + +# walking through the library based graph and creating a module based graph. +sub collapse_lib_to_module($) +{ + my $tree = shift; + my %digraph; + my $l2m = create_lib_module_map(); + my %unknown_libs; + for my $lib_name (sort keys %{$tree}) { + my $result = $tree->{$lib_name}; + $unknown_libs{$lib_name} = 1 && next if (!grep {/$lib_name/} keys %$l2m); + + # new collapsed name. + my $name = $l2m->{$lib_name}; + + # sal has no dependencies, take care of it + # otherwise it doesn't have target key + if (!@{$result->{deps}}) { + if (!exists($digraph{$name})) { + my @empty; + $digraph{$name}{deps} = \@empty; + $digraph{$name}{target} = $result->{target}; + $digraph{$name}{merged} = $result->{merged}; + } + } + for my $dep (@{$result->{deps}}) { + my $newdep; + $newdep = $l2m->{$dep}; + + die "Mis-named */Library_*.mk file - should match rules: '$dep'" if (!defined $newdep); + $dep = $newdep; + + # ignore: two libraries from the same module depend on each other + next if ($name eq $dep); + if (exists($digraph{$name})) + { + my @deps = @{$digraph{$name}{deps}}; + # only add if we haven't seen already that edge? + if (!grep {/$dep/} @deps) + { + push @deps, $dep; + $digraph{$name}{deps} = \@deps; + } + } + else + { + my @deps; + push @deps, $dep; + $digraph{$name}{deps} = \@deps; + $digraph{$name}{target} = $result->{target}; + $digraph{$name}{merged} = $result->{merged}; + } + } + } + logit("warn: no module for libs were found and dropped: [" . + join(",", (sort (keys(%unknown_libs)))) . "]\n"); + return optimize_tree(\%digraph); +} + +sub prune_leaves($) +{ + my $tree = shift; + my %newtree; + my %name_has_deps; + + # we like a few leaves around: + for my $nonleaf ('desktop', 'sw', 'sc', 'sd', 'starmath') { + $name_has_deps{$nonleaf} = 1; + } + + # find which modules are depended on by others + for my $name (keys %{$tree}) { + for my $dep (@{$tree->{$name}->{deps}}) { + $name_has_deps{$dep} = 1; + } + } + + # prune modules with no deps + for my $name (keys %{$tree}) { + delete $tree->{$name} if (!defined $name_has_deps{$name}); + } + + return optimize_tree($tree); +} + +sub annotate_mergelibs($) +{ + my $tree = shift; + print STDERR "annotating mergelibs\n"; + for my $name (keys %{$tree}) { + if (defined $merged_libs{$name}) { + $tree->{$name}->{merged} = 1; +# print STDERR "mark $name as merged\n"; + } + } +} + +sub dump_graphviz($) +{ + my $tree = shift; + my $to = \*STDOUT; + open($to, ">$output_file") if defined($output_file); + print $to <<END; +digraph LibreOffice { +edge [color="#31CEF0", len=0.4] +edge [fontname=Arial, fontsize=10, fontcolor="#31CEF0"] +END +; + + my @merged_names; + my @normal_names; + for my $name (sort keys %{$tree}) { + if ($tree->{$name}->{merged}) { + push @merged_names, $name; + } else { + push @normal_names, $name; + } + } + print $to "node [fontname=Verdana, fontsize=10, height=0.02, width=0.02,". + 'shape=Mrecord,color="#BBBBBB"' . + "];" . join(';', @normal_names) . "\n"; + print $to "node [fontname=Verdana, fontsize=10, height=0.02, width=0.02,". + 'shape=box,style=filled,color="#CCCCCC"' . + "];" . join(';', @merged_names) . "\n"; + + for my $name (sort keys %{$tree}) { + my $result = $tree->{$name}; + logit("minimising deps for $result->{target}\n"); + for my $dep (@{$result->{deps}}) { + print $to "$name -> $dep;\n" ; + } + } + print $to "}\n"; +} + +sub toposort_visit($$$$); +sub toposort_visit($$$$) +{ + my $tree = shift; + my $list = shift; + my $tags = shift; + my $name = shift; + die "dependencies don't form a DAG" + if (defined($tags->{$name}) && $tags->{$name} == 1); + if (!$tags->{$name}) { + $tags->{$name} = 1; + my $result = $tree->{$name}; + for my $dep (@{$result->{deps}}) { + toposort_visit($tree, $list, $tags, $dep); + } + $tags->{$name} = 2; + push @{$list}, $name; + } +} + +sub dump_toposort($) +{ + my $tree = shift; + my @list; + my %tags; + for my $name (sort keys %{$tree}) { + toposort_visit($tree, \@list, \%tags, $name); + } + my $to = \*STDOUT; + open($to, ">$output_file") if defined($output_file); + for (my $i = 0; $i <= $#list; ++$i) { + print $to "$list[$i]\n"; + } +} + +sub filter_targets($) +{ + my $tree = shift; + for my $name (sort keys %{$tree}) + { + my $result = $tree->{$name}; + if ($result->{type} eq 'CppunitTest' || + ($result->{type} eq 'Executable' && + $result->{target} ne 'soffice_bin')) + { + delete($tree->{$name}); + } + } +} + +sub parse_options() +{ + my %h = ( + 'verbose|v' => \$verbose, + 'help|h' => \my $help, + 'man|m' => \my $man, + 'version|r' => sub { + VersionMessage(-msg => "You are using: 1.0 of "); + }, + 'preserve-libs|p' => \$preserve_libs, + 'toposort|t' => \$toposort, + 'write-dep-file|w=s' => \$to_file, + 'read-dep-file|f=s' => \$from_file, + 'no-leaf|l' => \$no_leaf, + 'output-file|o=s' => \$output_file); + GetOptions(%h) or pod2usage(2); + pod2usage(1) if $help; + pod2usage(-exitstatus => 0, -verbose => 2) if $man; + ($gnumake, $makefile_build) = @ARGV if $#ARGV == 1; + $gnumake = 'make' if (!defined $gnumake); + $makefile_build = 'Makefile.gbuild' if (!defined $makefile_build); + $src_root = defined $ENV{SRC_ROOT} ? $ENV{SRC_ROOT} : "."; +} + +sub main() +{ + parse_options(); + my $deps = read_deps(); + my $tree = clean_tree($deps); + filter_targets($tree); + optimize_tree($tree); + annotate_mergelibs($tree); + if (!$preserve_libs && !defined($ENV{PRESERVE_LIBS})) { + $tree = collapse_lib_to_module($tree); + } + if ($no_leaf) { + $tree = prune_leaves($tree); + } + if ($toposort) { + dump_toposort($tree); + } else { + dump_graphviz($tree); + } +} + +main() + + __END__ + +=head1 NAME + +module-deps - Generate module dependencies for LibreOffice build system + +=head1 SYNOPSIS + +module_deps [options] [gnumake] [makefile] + +=head1 OPTIONS + +=over 8 + +=item B<--help> + +=item B<-h> + +Print a brief help message and exits. + +=item B<--man> + +=item B<-m> + +Prints the manual page and exits. + +=item B<--version> + +=item B<-v> + +Prints the version and exits. + +=item B<--preserve-libs> + +=item B<-p> + +Don't collapse libs to modules + +=item B<--toposort> + +=item B<-t> + +Output a topological sorting instead of a graph + +=item B<--read-dep-file file> + +=item B<-f> + +Read dependency from file. + +=item B<--write-dep-file file> + +=item B<-w> + +Write dependency to file. + +=item B<--output-file file> + +=item B<-o> + +Write graph or sort output to file + +=back + +=head1 DESCRIPTION + +B<This program> parses the output of LibreOffice make process +(or cached input file) and generates the digraph build dependency, +that must be piped to B<graphviz> program (typically B<dot>). + +B<Hacking on it>: + +The typical (optimized) B<workflow> includes 3 steps: + +=over 3 + +=item 1 +Create cache dependency file: module_deps --write-dep-file lo.dep + +=item 2 +Use cache dependency file: module_deps --read-dep-file lo.dep -o lo.graphviz + +=item 3 +Pipe the output to graphviz: cat lo.graphviz | dot -Tpng -o lo.png + +=back + +=head1 TODO + +=over 2 + +=item 1 +Add soft (include only) dependency + +=item 2 +Add dependency on external modules + +=back + +=head1 AUTHOR + +=over 2 + +=item Michael Meeks + +=item David Ostrovsky + +=back + +=cut diff --git a/bin/moveglobalheaders.sh b/bin/moveglobalheaders.sh new file mode 100755 index 000000000..ca202832b --- /dev/null +++ b/bin/moveglobalheaders.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +topdirs=`find . -mindepth 1 -maxdepth 1 -type d -not -name sal` +mkdir -p include/ +for dir in $topdirs +do + dir=`echo "$dir"| sed -e 's/^..//'` + if test -d $dir/inc/$dir + then + if test -f $dir/Package_inc.mk + then + if test -f $dir/Module_$dir.mk + then + git mv $dir/inc/$dir include/$dir + git rm $dir/Package_inc.mk + grep -v Package_inc $dir/Module_$dir.mk > $dir/Module_dir.mk.new + mv -f $dir/Module_dir.mk.new $dir/Module_$dir.mk + git add $dir/Module_$dir.mk + else + echo "WARN: no $dir/Module_$dir.mk" + fi + else + echo "WARN: no file $dir/Package_inc.mk" + fi + fi +done +#grep -v Package_inc.mk sal/CustomTarget_sal_allheaders.mk > sal/CustomTarget_sal_allheaders.mk.new +#mv sal/CustomTarget_sal_allheaders.mk.new sal/CustomTarget_sal_allheaders.mk +#git add sal/CustomTarget_sal_allheaders.mk + +# we like to be special ... +sed -ie 's/\/svtools\/inc\/svtools/\/include\/svtools\//' svtools/Library_svt.mk +sed -ie 's/\/sfx2\/inc\/sfx2/\/include\/sfx2\//' sfx2/Library_sfx.mk +git add svtools/Library_svt.mk sfx2/Library_sfx.mk + +# urgh +sed -ie 's/\.\.\/svx\//svx\//' svx/source/svdraw/svdoashp.cxx +git add svx/source/svdraw/svdoashp.cxx diff --git a/bin/odfvalidator.sh.in b/bin/odfvalidator.sh.in new file mode 100644 index 000000000..605e74731 --- /dev/null +++ b/bin/odfvalidator.sh.in @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +java -Djavax.xml.validation.SchemaFactory:http://relaxng.org/ns/structure/1.0=org.iso_relax.verifier.jaxp.validation.RELAXNGSchemaFactoryImpl -Dorg.iso_relax.verifier.VerifierFactoryLoader=com.sun.msv.verifier.jarv.FactoryLoaderImpl -jar @TARFILE_LOCATION@/@ODFVALIDATOR_JAR@ "$@" diff --git a/bin/officeotron.sh.in b/bin/officeotron.sh.in new file mode 100644 index 000000000..7281f1bcd --- /dev/null +++ b/bin/officeotron.sh.in @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +java -jar @TARFILE_LOCATION@/@OFFICEOTRON_JAR@ "$@" diff --git a/bin/oss-fuzz-build.sh b/bin/oss-fuzz-build.sh new file mode 100755 index 000000000..646accc8a --- /dev/null +++ b/bin/oss-fuzz-build.sh @@ -0,0 +1,55 @@ +#!/bin/bash -e + +if [ -z "${OUT}" ] || [ -z "${SRC}" ] || [ -z "${WORK}" ]; then + echo "OUT, SRC or WORK not set - script expects to be called inside oss-fuzz build env" + exit 1 +fi + +#shuffle CXXFLAGS -stdlib=libc++ arg into CXX as well because we use +#the CXX as the linker and need to pass -stdlib=libc++ to build +export CXX="$CXX -stdlib=libc++ -fsanitize-blacklist=$SRC/libreoffice/bin/sanitize-blacklist.txt" +#similarly force the -fsanitize etc args in as well as pthread to get +#things to link successfully during the build +export LDFLAGS="$CFLAGS -Wl,--compress-debug-sections,zlib -lpthread" + +df -h $OUT $WORK + +cd $WORK +$SRC/libreoffice/autogen.sh --with-distro=LibreOfficeOssFuzz --with-external-tar=$SRC + +make clean + +#build-time rsc tool leaks a titch +export ASAN_OPTIONS="detect_leaks=0" + +make fuzzers + +pushd instdir/program +head -c -14 services.rdb > templateservices.rdb +tail -c +85 ./services/services.rdb >> templateservices.rdb +for a in *fuzzer; do + #some minimal fonts required + mv $a $OUT + mkdir -p $OUT/$a.fonts + cp $SRC/884ed41809687c3e168fc7c19b16585149ff058eca79acbf3ee784f6630704cc-opens___.ttf ../share/fonts/truetype/Liberation* $OUT/$a.fonts + #minimal runtime requirements + cp templateservices.rdb $OUT/$a.services.rdb + cp types.rdb $OUT/$a.types.rdb + cp types/offapi.rdb $OUT/$a.moretypes.rdb + cat > $OUT/$a.unorc << EOF +[Bootstrap] +URE_INTERNAL_LIB_DIR=\${ORIGIN} +UNO_TYPES=\${ORIGIN}/$a.types.rdb \${ORIGIN}/$a.moretypes.rdb +UNO_SERVICES=\${ORIGIN}/$a.services.rdb +EOF +done +popd + +df -h $OUT $WORK + +#starting corpuses +cp $SRC/*_seed_corpus.zip $OUT +#fuzzing dictionaries +cp $SRC/*.dict $OUT +#options files +cp $SRC/libreoffice/vcl/workben/*.options $OUT diff --git a/bin/parse-perfcheck.py b/bin/parse-perfcheck.py new file mode 100755 index 000000000..158ef62fe --- /dev/null +++ b/bin/parse-perfcheck.py @@ -0,0 +1,258 @@ +#!/usr/bin/python + +# This file is part of the LibreOffice project. +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import os +import getopt +import csv + + +colsResult = {} +allTests = [] + +def parseFile(dirname, filename, lastCommit): + + curTestComment, total = None, None + + path = os.path.join(dirname, filename) + + trigger = "desc: Trigger: Client Request: " + trigger_len = len(trigger) + totals = "totals: " + totals_len = len(totals) + + with open(path,'r') as callgrindFile: + lines = callgrindFile.readlines() + + for line in lines: + if line.startswith(trigger): + curTestComment = line[trigger_len:].replace("\n","") + elif line.startswith(totals): + total = line[totals_len:].replace("\n","") + + if curTestComment is None or total is None: + return None + + testName = os.path.basename(dirname).replace(".test.core","") + + lastCommitId, lastCommitDate = lastCommit + if lastCommitId not in colsResult: + colsResult[lastCommitId] = {} + colsResult[lastCommitId]['date'] = lastCommitDate + colsResult[lastCommitId]['values'] = {} + + colsResult[lastCommitId]['values'][curTestComment] = total + + return [lastCommitId, lastCommitDate, testName, curTestComment, total, filename] + +def processDirectory(rootDir, needsCsvHeader, lastCommit): + + results = [] + + if needsCsvHeader: + results.append(["lastCommit", "lastCommitDate", "test filename", "dump comment", "count", "filename"]) + + for dirName, subdirList, fileList in os.walk(rootDir): + files = [f for f in fileList if f.startswith("callgrind.out.")] + for fname in files: + found = parseFile(dirName, fname, lastCommit) + if found is not None: + results.append(found) + return results + +def getLastCommitInfo(): + + stream = os.popen("git log --date=iso") + line = stream.readline() + commitId = line.replace("commit ","").replace("\n","") + line = stream.readline() + line = stream.readline() + commitDate = line.replace("Date: ","").replace("\n","").strip() + + return commitId, commitDate + +def displayUsage(): + + usage = """ + +Parses the callgrind results of make perfcheck + +Arguments : + + --csv-file\t\t the target CSV file - new or containing previous tests - default : perfcheckResult.csv + --source-directory\t directory containing make perfcheck output - default : ./workdir/CppunitTest + --alert-type\t\t mode for calculating alerts - valid values : previous first + --alert-value\t\t alert threshold in % - default = 10 + + --help\t\t this message + +Columned output is dumped into csv-file + ".col" + +Alerts, if any, are displayed in standard output + +""" + print(usage) + +class WrongArguments(Exception): + pass + +def analyzeArgs(args): + + try: + opts, args = getopt.getopt(args, 'x', [ + 'csv-file=', 'source-directory=', 'alert-type=', 'alert-value=', 'help']) + except getopt.GetoptError: + raise WrongArguments + + targetFileName = "perfcheckResult.csv" + sourceDirectory = "./workdir/CppunitTest" + alertType = "" + alertValue = 10 + + for o, a in opts: + if o == '--help': + displayUsage() + sys.exit() + elif o == "--csv-file": + targetFileName = a + elif o == "--source-directory": + sourceDirectory = a + elif o == "--alert-type": + alertType = a + elif o == "--alert-value": + alertValue = float(a) + else: + raise WrongArguments + + return targetFileName, sourceDirectory, alertType, alertValue + +def readCsvFile(targetFilename): + + with open(targetFilename, 'r') as csvfile: + reader = csv.reader(csvfile, delimiter="\t") + # skip header + next(reader) + for line in reader: + + # do not process empty lines + if not line: + continue + + curId, curDate, curTestName, curTestComment, curValue, currCallgrindFile = line + + if curTestComment not in allTests: + allTests.append(curTestComment) + + if curId not in colsResult: + colsResult[curId] = {} + colsResult[curId]['date'] = curDate + colsResult[curId]['values'] = {} + + colsResult[curId]['values'][curTestComment] = curValue + +if __name__ == '__main__': + + #check args + try: + targetFileName, sourceDirectory, alertType, alertValue = analyzeArgs(sys.argv[1:]) + except WrongArguments: + displayUsage() + sys.exit(1) + + # check if sourceDirectory exists + if not os.path.isdir(sourceDirectory): + print("sourceDirectory %s not found - Aborting" % (sourceDirectory)) + sys.exit(1) + + # read the complete CSV file + if os.path.isfile(targetFileName): + readCsvFile(targetFileName) + needsCsvHeader = False + else: + needsCsvHeader = True + + # last commit Id + lastCommitId, lastCommitDate = getLastCommitInfo() + + # walker through directory + if lastCommitId not in colsResult: + + lastCommit = (lastCommitId, lastCommitDate) + results = processDirectory(sourceDirectory, needsCsvHeader, lastCommit) + ppResults = "\n".join(["\t".join(row) for row in results]) + + print('\nNew results\n' + ppResults) + + # append raw result + with open(targetFileName,'a') as csvfile: + writer = csv.writer(csvfile, delimiter='\t') + writer.writerows(results) + print("\nCSV file written at " + targetFileName + '\n') + + else: + print("\nCSV file up to date " + targetFileName + '\n') + + + # build columned output + + # header + mLine = '\t'.join(["commit", "date"] + allTests) + '\n' + + alertTest = {} + + with open(targetFileName + '.col','w') as fileResult: + for k in colsResult: + mLine += k + "\t" + colsResult[k]['date'] + "\t" + for t in allTests: + if t in colsResult[k]['values']: + mValue= colsResult[k]['values'][t] + if t not in alertTest: + alertTest[t] = {} + alertTest[t][colsResult[k]['date']] = mValue + else: + mValue = "" + mLine += mValue + "\t" + mLine += "\n" + + # write columned result + fileResult.write(mLine) + + print("Columned file written at " + targetFileName + '.col\n') + + # check for Alerts + + if alertType == "": + sys.exit(1) + + alertResult = "" + + for t in alertTest: + + testDict = alertTest[t] + + # sort + keylist = sorted(testDict.keys()) + maxVal = float(testDict[keylist[-1]]) + minVal = 0 + + if alertType == "previous": + if len(keylist) > 1: + minVal = float(testDict[keylist[-2]]) + else: + minVal = float(testDict[keylist[0]]) + + if minVal != 0: + delta = 100 * ((maxVal-minVal)/minVal) + else: + delta = 0 + + if delta > float(alertValue): + alertResult += t + "\t" + "{:.2f}".format(delta) + " %\n" + + if alertResult != "": + print("!!!!!!!! ALERT !!!!!!!\n") + print(alertResult) diff --git a/bin/refcount_leak.py b/bin/refcount_leak.py new file mode 100755 index 000000000..2a24cb51e --- /dev/null +++ b/bin/refcount_leak.py @@ -0,0 +1,179 @@ +#!/usr/bin/python3 +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +### script to help debug leaks of reference counted objects + +## I. to use it, first override acquire() and release() + +# Foo * g_pTrackedFoo = 0; + +# Foo::Foo() +# static int nFoos = 0; +# if (++nFoos == 42) // track instance #42 +# g_pTrackedFoo = this; + +# void Foo::acquire() +# if (this == g_pTrackedFoo) +# ; // set gdb breakpoint here +# Foo_Base::acquire() + +# void Foo::release() +# if (this == g_pTrackedFoo) +# ; // set gdb breakpoint here +# Foo_Base::release() + +## II. run test/soffice in gdb and set breakpoints in acquire/release +## with a command to print the backtrace + +# set logging on +# break foo.cxx:123 +# break foo.cxx:234 + +# command 1 2 +# bt +# c +# end +# run + +## III. now feed logfile gdb.txt into this script + +# bin/refcount_leak.py < gdb.txt + +### + +from operator import itemgetter +import re +import sys + +threshold = 2 + +class Trace: + clock = 0 # global counter + # frames: list of stack frames, beginning with outermost + def __init__(self, lines): + lines.reverse() + self.frames = lines + Trace.clock += 1 + self.clock = Trace.clock + +def addTrace(traces, lines): + if not(traces is None) and len(lines) > 0: + traces.append(Trace(lines)) + +def readGdbLog(infile): + traces_acquire = [] + traces_release = [] + current = None + lines = [] + apattern = re.compile("^Breakpoint.*::acquire") + rpattern = re.compile("^Breakpoint.*::release") + for line in infile: + if apattern.match(line): + addTrace(current, lines) + lines = [] + current = traces_acquire + if rpattern.match(line): + addTrace(current, lines) + lines = [] + current = traces_release + if line.startswith("#"): + # strip #123 stack frame number, and newline + lines.append(line[line.index("0x"):-1]) + addTrace(current, lines) + print("# parsed traces acquire: ", len(traces_acquire)) + print("# parsed traces release: ", len(traces_release)) + return (traces_acquire, traces_release) + +def getFunction(frame): + start = frame.index(" in ") + len(" in ") + try: + end = frame.index(" at ", start) + except ValueError as e: + # argh... stack frames may be split across multiple lines if + # a parameter has a fancy pretty printer + return frame[start:] + return frame[start:end] + + +def matchStack(trace_acquire, trace_release): + if trace_release.clock < trace_acquire.clock: + return None # acquire must precede release + common = 0 + refpattern = re.compile(r"::Reference<.*>::Reference\(") + for (frame1, frame2) in zip(trace_release.frames, trace_acquire.frames): + if frame1 == frame2: + common += 1 + else: + if getFunction(frame1) == getFunction(frame2): + common += 1 + acquireframes = len(trace_acquire.frames) + # there is sometimes a dozen frames of UNO type related junk + # on the stack where the acquire() happens, which breaks the + # matching; try to avoid that + for i in range(common, acquireframes): + if refpattern.search(trace_acquire.frames[i]): + acquireframes = i+1 # cut off junk above Reference ctor + break + score = max(len(trace_release.frames), acquireframes) - common + # smaller score is better + return (score, trace_release.clock - trace_acquire.clock) + +# brute force greedy n^2 matching +def matchStacks(traces_acquire, traces_release): + matches = [] + for release in traces_release: + for acquire in traces_acquire: + score = matchStack(acquire, release) + if score is not None: + matches.append((score, acquire, release)) + matches.sort(key=itemgetter(0)) + return matches + +def bestMatches(traces_acquire, traces_release, matches): + traces_aunmatched = traces_acquire + traces_runmatched = traces_release + bestmatches = [] + for (score,acquire,release) in matches: + if not(acquire in traces_aunmatched and release in traces_runmatched): + continue + traces_aunmatched.remove(acquire) + traces_runmatched.remove(release) + bestmatches.append((score,acquire,release)) + print("# unmatched acquire: ", len(traces_aunmatched)) + print("# unmatched release: ", len(traces_runmatched)) + return (bestmatches,traces_aunmatched,traces_runmatched) + +def printTrace(trace): + for frame in reversed(trace.frames): + print(" ", frame) + +def printMatched(bestmatches): + for (score,acquire,release) in reversed(bestmatches): + print("\n*** Matched trace with score: ", score) + print(" acquire: ") + printTrace(acquire) + print(" release: ") + printTrace(release) + +def printUnmatched(traces, prefix): + for trace in traces: + print("\n*** Unmatched trace (", prefix, "):") + printTrace(trace) + +if __name__ == "__main__": + (traces_acquire, traces_release) = readGdbLog(sys.stdin) + matches = matchStacks(traces_acquire, traces_release) + (bestmatches,traces_au,traces_ru) = bestMatches(traces_acquire, traces_release, matches) + # print output, sorted with the most suspicious stuff first: + printUnmatched(traces_au, "acquire") + printUnmatched(traces_ru, "release") + printMatched(bestmatches) + +# vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/removetooltip_markups.sh b/bin/removetooltip_markups.sh new file mode 100755 index 000000000..5699fce99 --- /dev/null +++ b/bin/removetooltip_markups.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Run the script in the core directory to remove all tooltip_markup +# properties from the .ui files + +SED_BIN=`which sed` +CUT_BIN=`which cut` +LOG_FILE="modified-$(date +%s).log" + +removeTooltipMarkup() +{ + LINE=$(grep -n "<property name=\"tooltip_markup\"" $1 | $CUT_BIN -f 1 -d ':') + TEXT=$(grep "<property name=\"tooltip_markup\"" $1) + grep -v "<property name=\"tooltip_markup\"" $1 > temp && mv temp $1 + echo "removed $TEXT from $1 at line $LINE" >> $LOG_FILE +} + +changeTooltipMarkup() +{ + LINE=$(grep -n "<property name=\"tooltip_markup\"" $1 | $CUT_BIN -f 1 -d ':') + $SED_BIN "s/tooltip_markup/tooltip_text/g" $i > temp && mv temp $1 + echo "renamed tooltip_markup from $1 at line $LINE" >> $LOG_FILE +} + +checkTooltipMarkup() +{ + TEXT=`grep "<property name=\"tooltip_text\"" $1` + MARKUP=`grep "<property name=\"tooltip_markup\"" $1` + + if [[ $MARKUP ]] && [[ $TEXT ]] + then + removeTooltipMarkup "$1" + fi + if [[ $MARKUP ]] && [[ ! $TEXT ]] + then + changeTooltipMarkup "$1" + fi +} + +shopt -s globstar +echo " " > $LOG_FILE +for i in **/*.ui; do + echo -n "." + checkTooltipMarkup "$i" +done + +echo +echo "Done!" diff --git a/bin/rename-sw-abbreviations.sh b/bin/rename-sw-abbreviations.sh new file mode 100755 index 000000000..219b4f269 --- /dev/null +++ b/bin/rename-sw-abbreviations.sh @@ -0,0 +1,43 @@ +#! /bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# This script renames the most annoying abbreviations in Writer (and partially +# in the shared code too). Just run it in the source directory. + +# sw only: + +for I in "FrmFmt/FrameFormat" "Fmt/Format" "Cntnt/Content" "Txt/Text" "Tbl/Table" "GotoFld/GotoFormatField" "Fld/Field" "Ftn/Footnote" "Updt/Update" "Fml/Formula" "Hnt/Hint" "CurCrsr/CurrentCursor" "VisCrsr/VisibleCursor" "Crsr/Cursor" "CntFrm/ContentFrame" "Frm/Frame" "Stk/Stack" +do + S="${I%/*}" + # change all except the filenames (in the .mk and in #include) + # also avoid numFmt (OOXML token) and other stuff that must stay verbatim + git grep -l "$S" sw/ | grep -v -e '\.mk' -e '/data/' -e '/testdocuments/' | xargs sed -i '/\(#include\|numFmt\|ForeignTxt\)/ !{ s/'"$I"'/g }' +done + +# global: + +for I in "SvxSwAutoFmtFlags/SvxSwAutoFormatFlags" "GetCharFmtName/GetCharFormatName" \ + "SvxFmtBreakItem/SvxFormatBreakItem" "SvxFmtKeepItem/SvxFormatKeepItem" \ + "SvxFmtSplitItem/SvxFormatSplitItem" "etTxtLeft/etTextLeft" \ + "etTxtFirstLineOfst/etTextFirstLineOfst" "CntntProtected/ContentProtected" \ + "etTxtColor/etTextColor" "ClearFldColor/ClearFieldColor" \ + "etCntntProtect/etContentProtect" "etPropTxtFirstLineOfst/etPropTextFirstLineOfst" \ + "etCharFmtName/etCharFormatName" "HasMergeFmtTbl/HasMergeFormatTable" \ + "etMergeFmtIndex/etMergeFormatIndex" "bAFmtByInput/bAFormatByInput" \ + "bAFmt/bAFormat" "IsTxtFmt/IsTextFormat" "BuildWhichTbl/BuildWhichTable" \ + "etFld/etField" "IsAutoFmtByInput/IsAutoFormatByInput" \ + "etAutoFmtByInput/etAutoFormatByInput" "etMacroTbl/etMacroTable" \ + "SvxClipboardFmtItem/SvxClipboardFormatItem" "SwFlyFrmFmt/SwFlyFrameFormat" \ + "etTxtSize/etTextSize" +do + S="${I%/*}" + git grep -l "$S" | grep -v -e '\.mk' -e 'rename-sw-abbreviations.sh' | xargs sed -i "s/$I/g" +done + +# vim: set et sw=4 ts=4 textwidth=0: diff --git a/bin/run b/bin/run new file mode 100755 index 000000000..523da3c0e --- /dev/null +++ b/bin/run @@ -0,0 +1,76 @@ +#!/bin/sh +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# simple wrapper script to run non-installed executables from workdir + +setdefaults() +{ + dir=$(realpath "$(pwd)") + + while test ! -d "${dir}/instdir/program" ; do + if test "${dir}" = "/"; then + echo "error: cannot find \"program\" dir from \"$(pwd)\"" + exit 1 + fi + dir=$(realpath "${dir}/..") + done + + exedir="${dir}"/workdir/LinkTarget/Executable + export URE_BOOTSTRAP=file://"${dir}"/instdir/program/fundamentalrc +} + +if uname | grep -i CYGWIN >/dev/null; then + + setdefaults + + exedir=$(cygpath -m "${dir}"/workdir/LinkTarget/Executable) + export URE_BOOTSTRAP=file:///$(cygpath -m "${dir}")/instdir/program/fundamental.ini + export PATH=${PATH:+$PATH:}"${dir}"/instdir/program + SEARCH_PATH="${PATH}" + +elif [ $(uname) = Darwin ]; then + + dir=$(pwd) + + # Get PRODUCTNAME from config_host.mk, LibreOffice or LibreOfficeDev + eval `grep 'export PRODUCTNAME=' config_host.mk` + + if [ ! -d "${dir}/instdir/$PRODUCTNAME.app" ]; then + echo "error: cannot find \"instdir/$PRODUCTNAME.app\" dir in \"$(pwd)\"" + exit 1 + fi + + exedir="$dir"/workdir/LinkTarget/Executable + export URE_BOOTSTRAP=file://"${dir}"/instdir/$PRODUCTNAME.app/Contents/Resources/fundamentalrc + export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:+$DYLD_LIBRARY_PATH:}"${dir}"/instdir/$PRODUCTNAME.app/Contents/Frameworks + SEARCH_PATH="${DYLD_LIBRARY_PATH}" + +elif [ $(uname) = Haiku ]; then + + setdefaults + + export LIBRARY_PATH=${LIBRARY_PATH:+$LIBRARY_PATH:}"${dir}"/instdir/program + SEARCH_PATH="${LIBRARY_PATH}" + +else + + setdefaults + + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}"${dir}"/instdir/program + SEARCH_PATH="${LD_LIBRARY_PATH}" + +fi + +# echo "setting URE_BOOTSTRAP to: ${URE_BOOTSTRAP}" +# echo "setting search path to: ${SEARCH_PATH}" +# echo "execing: ${exedir}/$1" + +exec ${LO_TRACE} "${exedir}/$@" + +# vi:set shiftwidth=4 expandtab: diff --git a/bin/sanitize-blacklist.txt b/bin/sanitize-blacklist.txt new file mode 100644 index 000000000..e3e995f1a --- /dev/null +++ b/bin/sanitize-blacklist.txt @@ -0,0 +1,12 @@ +[float-divide-by-zero] +src:*/sc/source/core/tool/interpr1.cxx +src:*/sc/source/core/tool/interpr2.cxx +src:*/scaddins/source/analysis/analysis.cxx +src:*/scaddins/source/analysis/financial.cxx +[signed-integer-overflow] +src:*/boost/boost/rational.hpp +src:*/include/tools/gen.hxx +src:*/tools/source/generic/gen.cxx +[vptr] +fun:_ZN4cppu14throwExceptionERKN3com3sun4star3uno3AnyE +src:*/include/com/sun/star/uno/Reference.hxx diff --git a/bin/sanitize-image-links b/bin/sanitize-image-links new file mode 100755 index 000000000..6b5a2ec48 --- /dev/null +++ b/bin/sanitize-image-links @@ -0,0 +1,38 @@ +#!/bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# This will reorder icon-themes/*/links.txt to the right order + +for I in icon-themes/*/links.txt ; do + D="${I%/links.txt}" + cat "$I" | while read LINK ORIG + do + if [ -f "$D/$LINK" -a -f "$D/$ORIG" ] ; then + if diff "$D/$LINK" "$D/$ORIG" >/dev/null 2>&1 ; then + echo "$I: removing $LINK from git: both $LINK and $ORIG are the same files" 1>&2 + git rm "$D/$LINK" 1>/dev/null + echo $LINK $ORIG + else + echo "$I: link and orig differs, check the images, and remove manually: $LINK $ORIG" 1>&2 + echo $LINK $ORIG + fi + elif [ -f "$D/$LINK" ] ; then + echo "$I: swapping to right order: $ORIG $LINK" 1>&2 + echo $ORIG $LINK + elif [ -n "$LINK" -a "${LINK:0:1}" != "#" -a ! -f "$D/$LINK" -a ! -f "$D/$ORIG" ] ; then + echo "$I: neither exists, removing the line: $LINK $ORIG" 1>&2 + else + echo $LINK $ORIG + fi + done > "$I-fixed" + + mv "$I-fixed" "$I" +done + +# vim: set expandtab sw=4 ts=4: diff --git a/bin/striplanguagetags.sh b/bin/striplanguagetags.sh new file mode 100755 index 000000000..0df4b0be5 --- /dev/null +++ b/bin/striplanguagetags.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# take a .zip containing a flat hierarchy of odf files and strip out the +# language and country tags in each of them and repack it all up +# should convert templates so that documents based on them use +# the default-document-language rather than the hardcoded lang-tag +# +# All a bit hacky, but it should work + +if [ -z "$CALLXSLTPROC" ]; then + echo "$0: \$CALLXSLTPROC not defined!" + echo "$0: Apparently we are not called from the build process, bailing out." + exit 1 +fi + +tempfoo=`basename $0` + +XSL=`mktemp /tmp/${tempfoo}.XXXXXX` +if [ $? -ne 0 ]; then + echo "$0: Can't create temp file, exiting..." + exit 1 +fi + +# On Windows, xsltproc is a non-Cygwin program, so we can't pass +# a Cygwin /tmp path to it +[ "$COM" == MSC ] && XSL=`cygpath -m -s $XSL` + +WRKDIR=`mktemp -d /tmp/${tempfoo}.XXXXXX` +if [ $? -ne 0 ]; then + echo "$0: Can't create temp dir, exiting..." + exit 1 +fi + +cat > $XSL << EOF +<?xml version="1.0" encoding="UTF-8"?> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0"> + +<xsl:template match="node()|@*"> + <xsl:copy> + <xsl:apply-templates select="@*|node()"/> + </xsl:copy> +</xsl:template> + +<xsl:template match="@fo:language"/> +<xsl:template match="@fo:country"/> +<xsl:template match="@fo:script"/> +<xsl:template match="@number:rfc-language-tag"/> +<xsl:template match="@style:rfc-language-tag"/> +<xsl:template match="@table:rfc-language-tag"/> +<xsl:template match="@style:rfc-language-tag-asian"/> +<xsl:template match="@style:rfc-language-tag-complex"/> + +</xsl:stylesheet> +EOF + +unzip -q $1 -d $WRKDIR +pushd $WRKDIR +for a in *; do + unzip -qc $a styles.xml > styles.tmp + eval "$CALLXSLTPROC -o styles.xml $XSL styles.tmp" + zip -qr $a styles.xml + rm styles.xml styles.tmp +done +popd +zip -qrj $1 $WRKDIR +rm -rf $WRKDIR +rm -f $XSL diff --git a/bin/stubify.pl b/bin/stubify.pl new file mode 100755 index 000000000..c61bc531e --- /dev/null +++ b/bin/stubify.pl @@ -0,0 +1,262 @@ +#!/usr/bin/env perl + +use Fcntl; +use POSIX; +use strict; + +# simple pkgconfig goodness +my $destdir; +my $recursive = 0; +my $assembler_out = 0; +my %pkg_configs = (); +my @pkg_config_paths = split(/:/, $ENV{PKG_CONFIG_PATH}); +push @pkg_config_paths, "/usr"; + +# Stubify a shared library ... +sub read_gen_symbols($$) +{ + my ($shlib, $fh) = @_; + my $obj; + + print $fh "\t.file \"$shlib\"\n"; + open $obj, "objdump -T $shlib|" || die "Can't objdump $shlib: $!"; + + while (my $line = <$obj>) { + $line =~ /([0-9a-f]*)\s+([gw ])\s+..\s+(\S*)\s*([0-9a-f]+)..............(.*)/ || next; + my ($address, $linkage, $type, $size, $symbol) = ($1, $2, $3, $4, $5); + + next if ($type eq '*UND*' || $type eq '*ABS*'); + +# print "Symbol '$symbol' type '$type' '$linkage' addr $address, size $size\n"; + + $symbol || die "no symbol for line $line"; + + next if ($symbol eq '_init' || $symbol eq '_fini'); + + $linkage =~ s/g//g; + + my $progbits = '@progbits'; + $progbits = '@nobits' if ($type eq '.bss'); + print $fh "\t.section $type.$symbol,\"a".$linkage."G\",$progbits,$symbol,comdat\n"; + print $fh ".globl $symbol\n"; + print $fh "\t.type $symbol,"; + if ($type eq '.text') { + print $fh "\@function\n"; + } else { + print $fh "\@object\n"; + } + print $fh "$symbol:\n"; + if ($type eq '.text') { + print $fh "\tret\n"; + } else { + my $isize = hex($size); + print $fh "\t.size $symbol, $isize\n"; + for (my $i = 0; $i < $isize; $i++) { + print $fh "\t.byte 0\n"; + } + } + print $fh "\n"; + } + + close $obj; +} + +sub stubify($$) +{ + my $shlib = shift; + my $output = shift; + my ($pipe, $tmpf); + + my $tmpname; + do { + $tmpname = tmpnam(); + } until sysopen($tmpf, $tmpname, O_RDWR|O_CREAT|O_EXCL, 0666); + close($tmpf); + + if ($assembler_out) { + open ($pipe, ">-"); + } else { + open ($pipe, "| as -o $tmpname") || die "can't start assembler: $!"; + } + read_gen_symbols ($shlib, $pipe); + close ($pipe) || die "Failed to assemble to: $tmpname: $!"; + + system ("gcc -shared -o $output $tmpname") && die "failed to exec gcc: $!"; + unlink $tmpname; +} + +sub help_exit() +{ + print "Usage: stubify <destdir> <pkg-config-names>\n"; + print "Converts libraries into stubs, and bundles them and their pkg-config files\n"; + print "into destdir\n"; + print " -R stubbify and include all dependent pkgconfig files\n"; + exit 1; +} + +sub parse_pkgconfig($$) +{ + my $name = shift; + my $file = shift; + my $fh; + my %hash; + my @hashes; + + print "parse $file\n"; + open ($fh, $file) || die "Can't open $file: $!"; + while (<$fh>) { + my ($key, $value); + if (/^\s*([^=]+)\s*=\s*([^=]+)\s*$/) { + $key = $1; $value = $2; + } elsif (/^\s*([^:]+)\s*:\s*([^:]+)\s*$/) { + $key = $1; $value = $2; + } elsif (/^\s*$/) { + next; + } else { + die "invalid pkgconfig line: $_\n"; + } + chomp ($key); chomp ($value); + $hash{$key} = $value; + } + close ($fh); + for my $key (keys (%hash)) { + print "\t'$key'\t=\t'$hash{$key}'\n"; + } + + $hash{_Name} = $name; + $hash{_File} = $file; + + push @hashes, \%hash; + if ($recursive && + !defined $pkg_configs{$name} && + defined $hash{Requires}) { + my @reqs = (); + for my $req (split (/[ ,]/, $hash{Requires})) { + print "parse $req of $name\n"; + push @reqs, get_pc_files($req); + } + $hash{_Requires} = \@reqs; + push @hashes, @reqs; + } + $pkg_configs{$name} = \%hash; + return @hashes; +} + +sub get_pc_files($) +{ + my $name = shift; + for my $prefix (@pkg_config_paths) { + my $path = "$prefix/lib/pkgconfig/$name.pc"; + return parse_pkgconfig ($name,$path) if (-f $path); + } + die "Failed to find pkg-config file for $name"; +} + +# primitive substitution +sub get_var($$) +{ + my ($pc, $var) = @_; + my $val = $pc->{"$var"}; + while ($val =~ m/^(.*)\$\{\s*(\S+)\s*\}(.*)$/) { + $val = $1 . get_var($pc, $2). $3; + } + return $val; +} + +sub copy_lib($@) +{ + my $lib = shift; + while (my $path = shift) { + my $name = "$path/$lib"; + next if (! -f $name); + + # need to run ldconfig post install ... + while (-l $name) { + my $dir = $name; + $dir =~ s/\/[^\/]*$//; + my $link = readlink($name); + if ($link =~ m/^\//) { + $name = $link; + } else { + $name = "$dir/$link"; + } + } + + # ignore /lib - they use monstrous symbol versioning + if ($name =~ m/^\/lib/) { + print "\tskipping system library: $lib in $name\n"; + return; + } + + stubify ($name, "$destdir/$name"); + } +} + +sub copy_and_stubify ($) +{ + my $pc = shift; + + `mkdir -p $destdir/usr/lib/pkgconfig`; + `mkdir -p $destdir/$pc->{libdir}` if (defined $pc->{libdir}); + `mkdir -p $destdir/$pc->{includedir}` if (defined $pc->{includedir}); + + # copy .pc across - FIXME, may need to re-write paths + `cp -a $pc->{_File} $destdir/usr/lib/pkgconfig`; + + # copy includes across + my @includes = split (/ /, get_var ($pc, "Cflags")); + for my $arg (@includes) { + if ($arg =~ m/^-I(.*)$/) { + my $srcdir = $1; + if (! -d $srcdir || $srcdir eq '/usr/include') { + print "Warning: bogus include of '$srcdir' for pkg $pc->{_Name}\n"; + } else { + `mkdir -p $destdir/$srcdir`; + `cp -a $srcdir/* $destdir/$srcdir`; + } + } + } + + # stubify libraries + my @libs = split (/ /, get_var ($pc, "Libs")); + my @libpath = ( "/lib", "/usr/lib" ); + for my $arg (@libs) { + if ($arg =~ m/^-l(.*)$/) { + my $lib = "lib".$1.".so"; +# print "lib $lib @libpath?\n"; + copy_lib ($lib, @libpath); + } elsif ($arg =~ m/^-L(.*)$/) { + my $path = $1; + push (@libpath, $path) if (! grep ($path, @libpath)); + } + } +} + +my @pcnames = (); +my @tostub; + +for my $arg (@ARGV) { + if ($arg eq '--help' || $arg eq '-h') { + help_exit(); + } elsif ($arg eq '-r' || $arg eq '-R') { + $recursive = 1; + } elsif (!defined $destdir) { + $destdir = $arg; + } else { + push @pcnames, $arg; + } +} + +help_exit() if (!defined $destdir); +`mkdir -p $destdir`; + +for my $name (@pcnames) { + push @tostub, get_pc_files($name); +} +print "stubify: "; +select STDERR; $| = 1; +for my $pc (@tostub) { + print " " . $pc->{_Name} . "\n"; + copy_and_stubify ($pc); +} +print "\n"; diff --git a/bin/symbolstore.py b/bin/symbolstore.py new file mode 100755 index 000000000..7ddd8d2ac --- /dev/null +++ b/bin/symbolstore.py @@ -0,0 +1,644 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is mozilla.org code. +# +# The Initial Developer of the Original Code is +# The Mozilla Foundation +# Portions created by the Initial Developer are Copyright (C) 2007 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Ted Mielczarek <ted.mielczarek@gmail.com> +# Ben Turner <mozilla@songbirdnest.com> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +# +# Usage: symbolstore.py <params> <dump_syms path> <symbol store path> +# <debug info files or dirs> +# Runs dump_syms on each debug info file specified on the command line, +# then places the resulting symbol file in the proper directory +# structure in the symbol store path. Accepts multiple files +# on the command line, so can be called as part of a pipe using +# find <dir> | xargs symbolstore.pl <dump_syms> <storepath> +# But really, you might just want to pass it <dir>. +# +# Parameters accepted: +# -c : Copy debug info files to the same directory structure +# as sym files +# -a "<archs>" : Run dump_syms -a <arch> for each space separated +# cpu architecture in <archs> (only on macOS) +# -s <srcdir> : Use <srcdir> as the top source directory to +# generate relative filenames. + +import sys +import os +import re +import shutil +from optparse import OptionParser + +# Utility classes + +class VCSFileInfo: + """ A base class for version-controlled file information. Ensures that the + following attributes are generated only once (successfully): + + self.root + self.clean_root + self.revision + self.filename + + The attributes are generated by a single call to the GetRoot, + GetRevision, and GetFilename methods. Those methods are explicitly not + implemented here and must be implemented in derived classes. """ + + def __init__(self, file): + if not file: + raise ValueError + self.file = file + + def __getattr__(self, name): + """ __getattr__ is only called for attributes that are not set on self, + so setting self.[attr] will prevent future calls to the GetRoot, + GetRevision, and GetFilename methods. We don't set the values on + failure on the off chance that a future call might succeed. """ + + if name == "root": + root = self.GetRoot() + if root: + self.root = root + return root + + elif name == "clean_root": + clean_root = self.GetCleanRoot() + if clean_root: + self.clean_root = clean_root + return clean_root + + elif name == "revision": + revision = self.GetRevision() + if revision: + self.revision = revision + return revision + + elif name == "filename": + filename = self.GetFilename() + if filename: + self.filename = filename + return filename + + raise AttributeError + + def GetRoot(self): + """ This method should return the unmodified root for the file or 'None' + on failure. """ + raise NotImplementedError + + def GetCleanRoot(self): + """ This method should return the repository root for the file or 'None' + on failure. """ + raise NotImplementedErrors + + def GetRevision(self): + """ This method should return the revision number for the file or 'None' + on failure. """ + raise NotImplementedError + + def GetFilename(self): + """ This method should return the repository-specific filename for the + file or 'None' on failure. """ + raise NotImplementedError + +class CVSFileInfo(VCSFileInfo): + """ A class to maintain version information for files in a CVS repository. + Derived from VCSFileInfo. """ + + def __init__(self, file, srcdir): + VCSFileInfo.__init__(self, file) + self.srcdir = srcdir + + def GetRoot(self): + (path, filename) = os.path.split(self.file) + root = os.path.join(path, "CVS", "Root") + if not os.path.isfile(root): + return None + f = open(root, "r") + root_name = f.readline().strip() + f.close() + if root_name: + return root_name + print >> sys.stderr, "Failed to get CVS Root for %s" % filename + return None + + def GetCleanRoot(self): + parts = self.root.split('@') + if len(parts) > 1: + # we don't want the extra colon + return parts[1].replace(":","") + print >> sys.stderr, "Failed to get CVS Root for %s" % filename + return None + + def GetRevision(self): + (path, filename) = os.path.split(self.file) + entries = os.path.join(path, "CVS", "Entries") + if not os.path.isfile(entries): + return None + f = open(entries, "r") + for line in f: + parts = line.split("/") + if len(parts) > 1 and parts[1] == filename: + return parts[2] + print >> sys.stderr, "Failed to get CVS Revision for %s" % filename + return None + + def GetFilename(self): + file = self.file + if self.revision and self.clean_root: + if self.srcdir: + # strip the base path off + # but we actually want the last dir in srcdir + file = os.path.normpath(file) + # the lower() is to handle win32+vc8, where + # the source filenames come out all lowercase, + # but the srcdir can be mixed case + if file.lower().startswith(self.srcdir.lower()): + file = file[len(self.srcdir):] + (head, tail) = os.path.split(self.srcdir) + if tail == "": + tail = os.path.basename(head) + file = tail + file + return "cvs:%s:%s:%s" % (self.clean_root, file, self.revision) + return file + +class SVNFileInfo(VCSFileInfo): + url = None + repo = None + svndata = {} + + # This regex separates protocol and optional username/password from a url. + # For instance, all the following urls will be transformed into + # 'foo.com/bar': + # + # http://foo.com/bar + # svn+ssh://user@foo.com/bar + # svn+ssh://user:pass@foo.com/bar + # + rootRegex = re.compile(r'^\S+?:/+(?:[^\s/]*@)?(\S+)$') + + def __init__(self, file): + """ We only want to run subversion's info tool once so pull all the data + here. """ + + VCSFileInfo.__init__(self, file) + + if os.path.isfile(file): + command = os.popen("svn info %s" % file, "r") + for line in command: + # The last line of the output is usually '\n' + if line.strip() == '': + continue + # Split into a key/value pair on the first colon + key, value = line.split(':', 1) + if key in ["Repository Root", "Revision", "URL"]: + self.svndata[key] = value.strip() + + exitStatus = command.close() + if exitStatus: + print >> sys.stderr, "Failed to get SVN info for %s" % file + + def GetRoot(self): + key = "Repository Root" + if key in self.svndata: + match = self.rootRegex.match(self.svndata[key]) + if match: + return match.group(1) + print >> sys.stderr, "Failed to get SVN Root for %s" % self.file + return None + + # File bug to get this teased out from the current GetRoot, this is temporary + def GetCleanRoot(self): + return self.root + + def GetRevision(self): + key = "Revision" + if key in self.svndata: + return self.svndata[key] + print >> sys.stderr, "Failed to get SVN Revision for %s" % self.file + return None + + def GetFilename(self): + if self.root and self.revision: + if "URL" in self.svndata and "Repository Root" in self.svndata: + url, repo = self.svndata["URL"], self.svndata["Repository Root"] + file = url[len(repo) + 1:] + return "svn:%s:%s:%s" % (self.root, file, self.revision) + print >> sys.stderr, "Failed to get SVN Filename for %s" % self.file + return self.file + +# Utility functions + +# A cache of files for which VCS info has already been determined. Used to +# prevent extra filesystem activity or process launching. +vcsFileInfoCache = {} + +def GetVCSFilename(file, srcdir): + """Given a full path to a file, and the top source directory, + look for version control information about this file, and return + a tuple containing + 1) a specially formatted filename that contains the VCS type, + VCS location, relative filename, and revision number, formatted like: + vcs:vcs location:filename:revision + For example: + cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36 + 2) the unmodified root information if it exists""" + (path, filename) = os.path.split(file) + if path == '' or filename == '': + return (file, None) + + fileInfo = None + root = '' + if file in vcsFileInfoCache: + # Already cached this info, use it. + fileInfo = vcsFileInfoCache[file] + else: + if os.path.isdir(os.path.join(path, "CVS")): + fileInfo = CVSFileInfo(file, srcdir) + if fileInfo: + root = fileInfo.root + elif os.path.isdir(os.path.join(path, ".svn")) or \ + os.path.isdir(os.path.join(path, "_svn")): + fileInfo = SVNFileInfo(file); + vcsFileInfoCache[file] = fileInfo + + if fileInfo: + file = fileInfo.filename + + # we want forward slashes on win32 paths + return (file.replace("\\", "/"), root) + +def GetPlatformSpecificDumper(**kwargs): + """This function simply returns a instance of a subclass of Dumper + that is appropriate for the current platform.""" + return {'win32': Dumper_Win32, + 'cygwin': Dumper_Win32, + 'linux2': Dumper_Linux, + 'sunos5': Dumper_Solaris, + 'darwin': Dumper_Mac}[sys.platform](**kwargs) + +def SourceIndex(fileStream, outputPath, cvs_root): + """Takes a list of files, writes info to a data block in a .stream file""" + # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing + # Create the srcsrv data block that indexes the pdb file + result = True + pdbStreamFile = open(outputPath, "w") + pdbStreamFile.write('''SRCSRV: ini ------------------------------------------------\r\nVERSION=1\r\nSRCSRV: variables ------------------------------------------\r\nCVS_EXTRACT_CMD=%fnchdir%(%targ%)cvs.exe -d %fnvar%(%var2%) checkout -r %var4% -d %var4% -N %var3%\r\nMYSERVER=''') + pdbStreamFile.write(cvs_root) + pdbStreamFile.write('''\r\nSRCSRVTRG=%targ%\%var4%\%fnbksl%(%var3%)\r\nSRCSRVCMD=%CVS_EXTRACT_CMD%\r\nSRCSRV: source files ---------------------------------------\r\n''') + pdbStreamFile.write(fileStream) # can't do string interpolation because the source server also uses this and so there are % in the above + pdbStreamFile.write("SRCSRV: end ------------------------------------------------\r\n\n") + pdbStreamFile.close() + return result + +class Dumper: + """This class can dump symbols from a file with debug info, and + store the output in a directory structure that is valid for use as + a Breakpad symbol server. Requires a path to a dump_syms binary-- + |dump_syms| and a directory to store symbols in--|symbol_path|. + Optionally takes a list of processor architectures to process from + each debug file--|archs|, the full path to the top source + directory--|srcdir|, for generating relative source file names, + and an option to copy debug info files alongside the dumped + symbol files--|copy_debug|, mostly useful for creating a + Microsoft Symbol Server from the resulting output. + + You don't want to use this directly if you intend to call + ProcessDir. Instead, call GetPlatformSpecificDumper to + get an instance of a subclass.""" + def __init__(self, dump_syms, symbol_path, + archs=None, srcdir=None, copy_debug=False, vcsinfo=False, srcsrv=False): + # popen likes absolute paths, at least on windows + self.dump_syms = dump_syms + self.symbol_path = symbol_path + if archs is None: + # makes the loop logic simpler + self.archs = [''] + else: + self.archs = ['-a %s' % a for a in archs.split()] + if srcdir is not None: + self.srcdir = os.path.normpath(srcdir) + else: + self.srcdir = None + self.copy_debug = copy_debug + self.vcsinfo = vcsinfo + self.srcsrv = srcsrv + + # subclasses override this + def ShouldProcess(self, file): + return False + + def RunFileCommand(self, file): + """Utility function, returns the output of file(1)""" + try: + # we use -L to read the targets of symlinks, + # and -b to print just the content, not the filename + return os.popen("file -Lb " + file).read() + except: + return "" + + # This is a no-op except on Win32 + def FixFilenameCase(self, file): + return file + + # This is a no-op except on Win32 + def SourceServerIndexing(self, debug_file, guid, sourceFileStream, cvs_root): + return "" + + # subclasses override this if they want to support this + def CopyDebug(self, file, debug_file, guid): + pass + + def Process(self, file_or_dir): + "Process a file or all the (valid) files in a directory." + if os.path.isdir(file_or_dir): + return self.ProcessDir(file_or_dir) + elif os.path.isfile(file_or_dir): + return self.ProcessFile(file_or_dir) + # maybe it doesn't exist? + return False + + def ProcessDir(self, dir): + """Process all the valid files in this directory. Valid files + are determined by calling ShouldProcess.""" + result = True + for root, dirs, files in os.walk(dir): + for f in files: + fullpath = os.path.join(root, f) + if self.ShouldProcess(fullpath): + if not self.ProcessFile(fullpath): + result = False + return result + + def ProcessFile(self, file): + """Dump symbols from this file into a symbol file, stored + in the proper directory structure in |symbol_path|.""" + result = False + sourceFileStream = '' + # tries to get cvsroot from the .mozconfig first - if it's not set + # the tinderbox cvs_path will be assigned further down + cvs_root = os.environ.get("SRCSRV_ROOT") + for arch in self.archs: + try: + cmd = os.popen("%s %s %s" % (self.dump_syms, arch, file), "r") + module_line = cmd.next() + if module_line.startswith("MODULE"): + # MODULE os cpu guid debug_file + (guid, debug_file) = (module_line.split())[3:5] + # strip off .pdb extensions, and append .sym + sym_file = re.sub("\.pdb$", "", debug_file) + ".sym" + # we do want forward slashes here + rel_path = os.path.join(debug_file, + guid, + sym_file).replace("\\", "/") + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + try: + os.makedirs(os.path.dirname(full_path)) + except OSError: # already exists + pass + f = open(full_path, "w") + f.write(module_line) + # now process the rest of the output + for line in cmd: + if line.startswith("FILE"): + # FILE index filename + (x, index, filename) = line.split(None, 2) + if sys.platform == "sunos5": + start = filename.find(self.srcdir) + if start == -1: + start = 0 + filename = filename[start:] + filename = self.FixFilenameCase(filename.rstrip()) + sourcepath = filename + if self.vcsinfo: + (filename, rootname) = GetVCSFilename(filename, self.srcdir) + # sets cvs_root in case the loop through files were to end on an empty rootname + if cvs_root is None: + if rootname: + cvs_root = rootname + # gather up files with cvs for indexing + if filename.startswith("cvs"): + (ver, checkout, source_file, revision) = filename.split(":", 3) + sourceFileStream += sourcepath + "*MYSERVER*" + source_file + '*' + revision + "\r\n" + f.write("FILE %s %s\n" % (index, filename)) + else: + # pass through all other lines unchanged + f.write(line) + f.close() + cmd.close() + # we output relative paths so callers can get a list of what + # was generated + print rel_path + if self.copy_debug: + self.CopyDebug(file, debug_file, guid) + if self.srcsrv: + # Call on SourceServerIndexing + result = self.SourceServerIndexing(debug_file, guid, sourceFileStream, cvs_root) + result = True + except StopIteration: + pass + except: + print >> sys.stderr, "Unexpected error: ", sys.exc_info()[0] + raise + return result + +# Platform-specific subclasses. For the most part, these just have +# logic to determine what files to extract symbols from. + +class Dumper_Win32(Dumper): + fixedFilenameCaseCache = {} + + def ShouldProcess(self, file): + """This function will allow processing of pdb files that have dll + or exe files with the same base name next to them.""" + if file.endswith(".pdb"): + (path,ext) = os.path.splitext(file) + if os.path.isfile(path + ".exe") or os.path.isfile(path + ".dll") or os.path.isfile(path + ".bin"): + return True + return False + + def FixFilenameCase(self, file): + """Recent versions of Visual C++ put filenames into + PDB files as all lowercase. If the file exists + on the local filesystem, fix it.""" + + # Use a cached version if we have one. + if file in self.fixedFilenameCaseCache: + return self.fixedFilenameCaseCache[file] + + result = file + + (path, filename) = os.path.split(file) + if os.path.isdir(path): + lc_filename = filename.lower() + for f in os.listdir(path): + if f.lower() == lc_filename: + result = os.path.join(path, f) + break + + # Cache the corrected version to avoid future filesystem hits. + self.fixedFilenameCaseCache[file] = result + return result + + def CopyDebug(self, file, debug_file, guid): + rel_path = os.path.join(debug_file, + guid, + debug_file).replace("\\", "/") + print rel_path + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + shutil.copyfile(file, full_path) + pass + + def SourceServerIndexing(self, debug_file, guid, sourceFileStream, cvs_root): + # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing + cwd = os.getcwd() + streamFilename = debug_file + ".stream" + stream_output_path = os.path.join(cwd, streamFilename) + # Call SourceIndex to create the .stream file + result = SourceIndex(sourceFileStream, stream_output_path, cvs_root) + + if self.copy_debug: + pdbstr_path = os.environ.get("PDBSTR_PATH") + pdbstr = os.path.normpath(pdbstr_path) + pdb_rel_path = os.path.join(debug_file, guid, debug_file) + pdb_filename = os.path.normpath(os.path.join(self.symbol_path, pdb_rel_path)) + # move to the dir with the stream files to call pdbstr + os.chdir(os.path.dirname(stream_output_path)) + os.spawnv(os.P_WAIT, pdbstr, [pdbstr, "-w", "-p:" + pdb_filename, "-i:" + streamFilename, "-s:srcsrv"]) + # clean up all the .stream files when done + os.remove(stream_output_path) + return result + +class Dumper_Linux(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .so extension, and additionally + file(1) reports as being ELF files. It expects to find the file + command in PATH.""" + if file.endswith(".so") or file.endswith(".bin") or os.access(file, os.X_OK): + return self.RunFileCommand(file).startswith("ELF") + return False + + def CopyDebug(self, file, debug_file, guid): + # We want to strip out the debug info, and add a + # .gnu_debuglink section to the object, so the debugger can + # actually load our debug info later. + file_dbg = file + ".dbg" + os.system("objcopy --only-keep-debug %s %s" % (file, file_dbg)) + os.system("objcopy --add-gnu-debuglink=%s %s" % (file_dbg, file)) + + rel_path = os.path.join(debug_file, + guid, + debug_file + ".dbg") + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + shutil.copyfile(file_dbg, full_path) + # gzip the shipped debug files + os.system("gzip %s" % full_path) + print rel_path + ".gz" + +class Dumper_Solaris(Dumper): + def RunFileCommand(self, file): + """Utility function, returns the output of file(1)""" + try: + output = os.popen("file " + file).read() + return output.split('\t')[1]; + except: + return "" + + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .so extension, and additionally + file(1) reports as being ELF files. It expects to find the file + command in PATH.""" + if file.endswith(".so") or os.access(file, os.X_OK): + return self.RunFileCommand(file).startswith("ELF") + return False + +class Dumper_Mac(Dumper): + def ShouldProcess(self, file): + """This function will allow processing of files that are + executable, or end with the .dylib extension, and additionally + file(1) reports as being Mach-O files. It expects to find the file + command in PATH.""" + if file.endswith(".dylib") or os.access(file, os.X_OK): + return self.RunFileCommand(file).startswith("Mach-O") + return False + +# Entry point if called as a standalone program +def main(): + parser = OptionParser(usage="usage: %prog [options] <dump_syms binary> <symbol store path> <debug info files>") + parser.add_option("-c", "--copy", + action="store_true", dest="copy_debug", default=False, + help="Copy debug info files into the same directory structure as symbol files") + parser.add_option("-a", "--archs", + action="store", dest="archs", + help="Run dump_syms -a <arch> for each space separated cpu architecture in ARCHS (only on macOS)") + parser.add_option("-s", "--srcdir", + action="store", dest="srcdir", + help="Use SRCDIR to determine relative paths to source files") + parser.add_option("-v", "--vcs-info", + action="store_true", dest="vcsinfo", + help="Try to retrieve VCS info for each FILE listed in the output") + parser.add_option("-i", "--source-index", + action="store_true", dest="srcsrv", default=False, + help="Add source index information to debug files, making them suitable for use in a source server.") + (options, args) = parser.parse_args() + + #check to see if the pdbstr.exe exists + if options.srcsrv: + pdbstr = os.environ.get("PDBSTR_PATH") + if not os.path.exists(pdbstr): + print >> sys.stderr, "Invalid path to pdbstr.exe - please set/check PDBSTR_PATH.\n" + sys.exit(1) + + if len(args) < 3: + parser.error("not enough arguments") + exit(1) + + dumper = GetPlatformSpecificDumper(dump_syms=args[0], + symbol_path=args[1], + copy_debug=options.copy_debug, + archs=options.archs, + srcdir=options.srcdir, + vcsinfo=options.vcsinfo, + srcsrv=options.srcsrv) + for arg in args[2:]: + dumper.Process(arg) + +# run main if run directly +if __name__ == "__main__": + main() diff --git a/bin/symstore.sh b/bin/symstore.sh new file mode 100755 index 000000000..532efb166 --- /dev/null +++ b/bin/symstore.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +# Files listed here would not be store in the symbolestore-server. +# The format is a string with files e.g. +# BLACKLIST="python.exe +# file.dll +# next_file.exe" +# +# It removes "python.exe", "file.dll", and "next_file.exe" from what's +# added to the symstore. Separator is the newline +BLACK_LIST="python.exe" + +# List files here where it's ok for this script to find more than one +# occurrence in the build tree. Files _not_ included here will generate +# an error, if duplicates are found. +# +# Same format as for BLACK_LIST above +MOREPDBS_OKLIST="libcurl.dll +freebl3.dll +libeay32.dll +nspr4.dll +nss3.dll +nssckbi.dll +nssdbm3.dll +nssutil3.dll +plc4.dll +plds4.dll +smime3.dll +softokn3.dll +sqlite3.dll +ssl3.dll +ssleay32.dll" + +verbose_none() +{ + do_none= +} + +add_pdb() +{ + extension=$1 + pdbext=$2 + list=$3 + stats_notfound=0 + stats_found=0 + stats_morefound=0 + declare -a pdball + echo "Collect $extension" + ret=$(find "${INSTDIR}/" -type f -name "*.${extension}" | grep -vF "$BLACK_LIST") + while IFS= read -r file + do + ${VERBOSE} -n "Found: $file" + # store dll/exe itself (needed for minidumps) + if [ $WITHEXEC == 1 ] ; then + cygpath -w "$file" >> "$list" + ${VERBOSE} " insert" + else + ${VERBOSE} " " + fi + + # store pdb file + filename=$(basename "$file" ".${extension}") + pdball+=($(grep -i "/${filename}${pdbext}" <<< ${ALL_PDBS})) + if [ -n "${pdball[0]}" ]; then + cygpath -w "${pdball[0]}" >> "$list" + fi + case ${#pdball[@]} in + 0) ((++stats_notfound)) + ${VERBOSE} " PDB not found" + ;; + 1) ((++stats_found)) + ${VERBOSE} " ${pdball[0]} insert" + ;; + *) ((++stats_morefound)) + if [ -z "$(echo $file | grep -F "$MOREPDBS_OKLIST")" ]; then + echo "Error: found duplicate PDBs:" + for morepdbs in ${pdball[@]} ; do + echo " $morepdbs" + done + exit 1 + else + ${VERBOSE} " ${pdball[0]} insert (is in more okay list)" + fi + ;; + esac + unset pdball + done <<EOF +${ret} +EOF + + echo " Found PDBs : $stats_found" + echo " Missing PDBs : $stats_notfound" + echo " Multiple PDBs : $stats_morefound" +} + +# check preconditions +if [ -z "${INSTDIR}" -o -z "${WORKDIR}" ]; then + echo "INSTDIR or WORKDIR not set - script expects calling inside buildenv" + exit 1 +fi +if [ ! -d "${INSTDIR}" -o ! -d "${WORKDIR}" ]; then + echo "INSTDIR or WORKDIR not present - script expects calling after full build" + exit 1 +fi +which symstore.exe > /dev/null 2>&1 || { + echo "symstore.exe is expected in the PATH" + exit 1 +} + +# defaults +MAX_KEEP=5 +SYM_PATH=${WORKDIR}/symstore +COMMENT="" +COMCMD="" +WITHEXEC=1 +VERBOSE=verbose_none + +USAGE="Usage: $0 [-h|-k <keep_num_versions>|-p <symbol_store_path>|-c <comment>|-n|-v] + -h: this cruft + -c <comment> specifies a comment for the transaction + -n do not store exe/dll on the symbole server + -k <int>: keep this number of old symbol versions around + (default: ${MAX_KEEP}. Set to 0 for unlimited) + -v verbose mode, output detail report of files + -p <path>: specify full path to symbol store tree +If no path is specified, defaults to ${SYM_PATH}. +" + +# process args +while : +do + case "$1" in + -k|--keep) MAX_KEEP="$2"; shift 2;; + -p|--path) SYM_PATH="$2"; shift 2;; + -c|--comment) COMCMD="/c"; COMMENT="$2"; shift 2;; + -n|--noexec) WITHEXEC=0; shift ;; + -v|--verbose) VERBOSE=echo; shift ;; + -h|--help) echo "${USAGE}"; exit 0;; + -*) echo "${USAGE}" >&2; exit 1;; + *) break;; + esac +done + +if [ $# -gt 0 ]; then + echo "${USAGE}" >&2 + exit 1 +fi + +# populate symbol store from here +TMPFILE=$(mktemp) || exit 1 +trap '{ rm -f ${TMPFILE}; }' EXIT + +# collect all PDBs +ALL_PDBS=$(find "${WORKDIR}/" -type f -name "*.pdb") + +# add dlls and executables +add_pdb dll .pdb "${TMPFILE}" +add_pdb exe .pdb "${TMPFILE}" +add_pdb bin .bin.pdb "${TMPFILE}" + +# stick all of it into symbol store +symstore.exe add /compress /f "@$(cygpath -w "${TMPFILE}")" /s "$(cygpath -w "${SYM_PATH}")" /t "${PRODUCTNAME}" /v "${LIBO_VERSION_MAJOR}.${LIBO_VERSION_MINOR}.${LIBO_VERSION_MICRO}.${LIBO_VERSION_PATCH}${LIBO_VERSION_SUFFIX}${LIBO_VERSION_SUFFIX_SUFFIX}" "${COMCMD}" "${COMMENT}" +rm -f "${TMPFILE}" + +# Cleanup symstore, older revisions will be removed. Unless the +# .dll/.exe changes, the .pdb should be shared, so with incremental +# tinderbox several revisions should not be that space-demanding. +if [ "${MAX_KEEP}" -gt 0 -a -d "${SYM_PATH}/000Admin" ]; then + to_remove=$(ls -1 "${SYM_PATH}/000Admin" | grep -v '\.txt' | grep -v '\.deleted' | sort | head -n "-${MAX_KEEP}") + for revision in $to_remove; do + echo "Remove $revision from symstore" + symstore.exe del /i "${revision}" /s "$(cygpath -w "${SYM_PATH}")" + done +fi diff --git a/bin/test-hid-vs-ui.py b/bin/test-hid-vs-ui.py new file mode 100755 index 000000000..635a121ad --- /dev/null +++ b/bin/test-hid-vs-ui.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. +# +# Parses all help files (.xhp) to check that hids referencing .ui are up-to-date +# From fdo#67350 + + +import sys +import argparse +import os +import subprocess +import xml.etree.ElementTree as ET +import collections +import re +import smtplib +import email +import email.mime.text +import time +import datetime + +# retrieve all hids related to .ui files +def init_hids(): + global args, local_repo + if local_repo: + repo_dir = os.path.join(core_repo_dir,'helpcontent2') + os.chdir(repo_dir) + return subprocess.check_output(['git','grep','hid="[^"]*/[^"]*">','.']) + else: + repo_dir = '/var/tmp/help.git' + if not os.path.exists(repo_dir):os.makedirs(repo_dir) + os.chdir(repo_dir) + + if not os.path.exists(os.path.join(repo_dir,'config')): + subprocess.call(['git','clone','--bare','git://gerrit.libreoffice.org/help',repo_dir]) + elif not args['git_static']: + subprocess.call(['git','fetch','origin']) + return subprocess.check_output(['git','grep','hid="[^"]*/[^"]*">','master','--']) + +# retrieve .ui files list from the core +def init_core_files(): + global core_repo_dir, local_repo + core_repo_dir = args['core_repo_dir'] + if core_repo_dir is None: + core_repo_dir = os.path.dirname(os.path.abspath(os.path.dirname(sys.argv[0]))) + local_repo = True + + if not os.path.exists(core_repo_dir):os.makedirs(core_repo_dir) + os.chdir(core_repo_dir) + + if not os.path.exists(os.path.join(core_repo_dir,'.git')): + subprocess.call(['git','clone','git://gerrit.libreoffice.org/core',core_repo_dir]) + elif not args['git_static']: + subprocess.call(['git','fetch','origin']) + allfiles = subprocess.check_output(['git','ls-tree','--name-only','--full-name','-r','master']) + return re.findall('.*\.ui',allfiles) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser('hid for ui consistency parser') + parser.add_argument('-s', '--send-to', action='append', help='email address to send the report to. Use one flag per address.', required=False) + parser.add_argument('-g', '--git-static', action='store_true', help='to avoid contacting remote server to refresh repositories.', required=False) + parser.add_argument('-r', '--core-repo-dir', help='enforce path to core repository when analyzing .ui files.', required=False) + args=vars(parser.parse_args()) + + uifileslist = init_core_files() # play it early to gain the local repo identification + + rows = init_hids().splitlines() + #<tree>:<relative_file>:<text> + # handled as sets to remove duplicates (and we don't need an iterator) + targets = collections.defaultdict(set) + origin = collections.defaultdict(set) + + # fill all matching hids and their parent file + for row in rows: + fname, rawtext = row.split(':',1)[0:] + hid = rawtext.split('hid="')[1].split('"')[0] + if hid.startswith('.uno'): continue + uifileraw, compname = hid.rsplit('/',1) + uifile = uifileraw + ".ui" + # map modules/ etc, which exist only in install + # back to their source location + if uifile.startswith("modules/scalc"): + uifile = "sc/scalc" + uifile[13:] + elif uifile.startswith("modules/swriter"): + uifile = "sw/swriter" + uifile[15:] + elif uifile.startswith("modules/schart"): + uifile = "chart2" + uifile[14:] + elif uifile.startswith("modules/smath"): + uifile = "starmath/smath" + uifile[13:] + elif uifile.startswith("modules/sdraw"): + uifile = "sd/sdraw" + uifile[13:] + elif uifile.startswith("modules/simpress"): + uifile = "sd/simpress" + uifile[16:] + elif uifile.startswith("modules/BasicIDE"): + uifile = "basctl/basicide" + uifile[16:] + elif uifile.startswith("modules/sabpilot"): + uifile = "extensions/sabpilot" + uifile[16:] + elif uifile.startswith("modules/sbibliography"): + uifile = "extensions/sbibliography" + uifile[21:] + elif uifile.startswith("modules/scanner"): + uifile = "extensions/scanner" + uifile[15:] + elif uifile.startswith("modules/spropctrlr"): + uifile = "extensions/spropctrlr" + uifile[18:] + elif uifile.startswith("sfx"): + uifile = "sfx2" + uifile[3:] + elif uifile.startswith("svt"): + uifile = "svtools" + uifile[3:] + elif uifile.startswith("fps"): + uifile = "fpicker" + uifile[3:] + components = uifile.split('/',1); + uifile = components[0] + '/uiconfig/' + components[1] + targets[uifile].add(compname.split(':')[0]) + origin[uifile].add(fname) # help file(s) + + errors = '' + # search in all .ui files referenced in help + # 2 possible errors: file not found in repo, id not found in file + for uikey in dict.keys(targets): + if uikey not in uifileslist: + if len(origin[uikey]) == 1: + errors += '\nFrom ' + origin[uikey].pop() + else: + errors += '\nFrom one of ' + str(origin[uikey]).replace('set(','').replace(')','') + errors += ', we did not find file '+ uikey+'.' + continue + + full_path = os.path.join(core_repo_dir,uikey) + # print full_path + root = ET.parse(full_path).getroot() + ids = [element.attrib['id'].split(':')[0] for element in root.findall('.//object[@id]')] + # print targets[uikey] + missing_ids = [ element for element in targets[uikey] if element not in ids ] + if missing_ids: + if len(origin[uikey]) == 1: + errors += '\nFrom ' + origin[uikey].pop() + else: + errors += '\nFrom one of ' + str(origin[uikey]).replace('set(','').replace(')','') + errors += ', referenced items '+ str(missing_ids) + ' were not found inside '+ uikey+'.' + + if not errors: + errors = '\nall is clean\n' + + if args['send_to']: + msg_from = os.path.basename(sys.argv[0]) + '@libreoffice.org' + if isinstance(args['send_to'], basestring): + msg_to = [args['send_to']] + else: + msg_to = args['send_to'] + print "send to array " + msg_to[0] + + server = smtplib.SMTP('localhost') + body = ''' +Hello, + +Here is the report for wrong hids from help related to .ui files + +''' + body += errors + body += ''' + +Best, + +Your friendly LibreOffice Help-ids Checker + +Note: The bot generating this message can be found and improved here: + https://gerrit.libreoffice.org/gitweb?p=dev-tools.git;a=blob;f=scripts/test-hid-vs-ui.py''' + now = datetime.datetime.now() + msg = email.mime.text.MIMEText(body, 'plain', 'UTF-8') + msg['From'] = msg_from + msg['To'] = msg_to[0] + msg['Cc'] = ', '.join(msg_to[1:]) # Works only if at least 2 items in tuple + msg['Date'] = email.utils.formatdate(time.mktime(now.timetuple())) + msg['Subject'] = 'LibreOffice Gerrit News for python on %s' % (now.date().isoformat()) + msg['Reply-To'] = msg_to[0] + msg['X-Mailer'] = 'LibreOfficeGerritDigestMailer 1.1' + + server.sendmail(msg_from, msg_to, str(msg)) + else: + print errors + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/text_cat/COPYING b/bin/text_cat/COPYING new file mode 100644 index 000000000..5ab7695ab --- /dev/null +++ b/bin/text_cat/COPYING @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + <one line to give the library's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + <signature of Ty Coon>, 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/bin/text_cat/Copyright b/bin/text_cat/Copyright new file mode 100644 index 000000000..c1e75d3af --- /dev/null +++ b/bin/text_cat/Copyright @@ -0,0 +1,21 @@ +Copyright (c) 1994, 1995, 1996, 1997 by Gertjan van Noord. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the + Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301 USA + +cf. the file COPYING + + diff --git a/bin/text_cat/LM/english.lm b/bin/text_cat/LM/english.lm new file mode 100644 index 000000000..ab71632c6 --- /dev/null +++ b/bin/text_cat/LM/english.lm @@ -0,0 +1,400 @@ +_ 20326 +e 6617 +t 4843 +o 3834 +n 3653 +i 3602 +a 3433 +s 2945 +r 2921 +h 2507 +e_ 2000 +d 1816 +_t 1785 +c 1639 +l 1635 +th 1535 +he 1351 +_th 1333 +u 1309 +f 1253 +m 1175 +p 1151 +_a 1145 +the 1142 +_the 1060 +s_ 978 +er 968 +_o 967 +he_ 928 +d_ 888 +t_ 885 +the_ 844 +_the_ 843 +on 842 +in 817 +y 783 +n_ 773 +b 761 +re 754 +, 734 +,_ 732 +an 732 +g 728 +w 718 +_i 707 +en 676 +f_ 599 +y_ 595 +of 594 +_of 592 +es 589 +ti 587 +v 580 +_of_ 575 +of_ 575 +nd 568 +at 549 +r_ 540 +_w 534 +it 522 +ed 496 +_p 494 +nt 485 +_c 462 +o_ 457 +io 450 +_an 439 +te 432 +or 425 +_b 418 +nd_ 407 +to 406 +st 402 +is 401 +_s 396 +_in 389 +ion 385 +and 385 +de 384 +ve 382 +ha 375 +ar 366 +_m 361 +and_ 360 +_and 360 +_and_ 358 +se 353 +_to 347 +me 346 +to_ 344 +ed_ 339 +. 330 +be 329 +_f 329 +._ 329 +_to_ 320 +co 317 +ic 316 +ns 308 +al 307 +le 304 +ou 304 +ce 293 +ent 279 +l_ 278 +_co 277 +tio 275 +on_ 274 +_d 274 +tion 268 +ri 266 +_e 264 +ng 253 +hi 251 +er_ 249 +ea 246 +as 245 +_be 242 +pe 242 +h_ 234 +_r 232 +ec 227 +ch 223 +ro 222 +ct 220 +_h 219 +pr 217 +in_ 217 +ne 214 +ll 214 +rt 213 +s,_ 210 +s, 210 +li 209 +ra 208 +T 207 +wh 204 +a_ 203 +ac 201 +_wh 199 +_n 196 +ts 196 +di 196 +es_ 195 +si 194 +re_ 193 +at_ 192 +nc 192 +ie 190 +_a_ 188 +_in_ 185 +ing 184 +us 182 +_re 182 +g_ 179 +ng_ 178 +op 178 +con 177 +tha 175 +_l 174 +_tha 174 +ver 173 +ma 173 +ion_ 171 +_con 171 +ci 170 +ons 170 +_it 170 +po 169 +ere 168 +is_ 167 +ta 167 +la 166 +_pr 165 +fo 164 +ho 164 +ir 162 +ss 161 +men 160 +be_ 160 +un 159 +ty 159 +_be_ 158 +ing_ 157 +om 156 +ot 156 +hat 155 +ly 155 +_g 155 +em 153 +_T 151 +rs 150 +mo 148 +ch_ 148 +wi 147 +we 147 +ad 147 +ts_ 145 +res 143 +_wi 143 +I 143 +hat_ 142 +ei 141 +ly_ 141 +ni 140 +os 140 +ca 139 +ur 139 +A 138 +ut 138 +that 138 +_that 137 +ati 137 +_fo 137 +st_ 137 +il 136 +or_ 136 +for 136 +pa 136 +ul 135 +ate 135 +ter 134 +it_ 134 +nt_ 133 +that_ 132 +_ha 129 +al_ 128 +el 128 +as_ 127 +ll_ 127 +_ma 125 +no 124 +ment 124 +an_ 124 +tion_ 122 +su 122 +bl 122 +_de 122 +nce 120 +pl 120 +fe 119 +tr 118 +so 118 +int 115 +ov 114 +e, 114 +e,_ 114 +_u 113 +ent_ 113 +Th 113 +her 113 +j 112 +atio 112 +ation 112 +_Th 111 +le_ 110 +ai 110 +_it_ 110 +_on 110 +_for 109 +ect 109 +k 109 +hic 108 +est 108 +der 107 +tu 107 +na 106 +_by_ 106 +by_ 106 +E 106 +by 106 +_by 106 +ve_ 106 +_di 106 +en_ 104 +vi 104 +m_ 103 +_whi 102 +iv 102 +whi 102 +ns_ 102 +_A 101 +ich 100 +ge 100 +pro 99 +ess 99 +_whic 99 +ers 99 +hich 99 +ce_ 99 +which 99 +whic 99 +all 98 +ove 98 +_is 98 +ich_ 97 +ee 97 +hich_ 97 +n,_ 96 +n, 96 +im 95 +ir_ 94 +hei 94 +ions 94 +sti 94 +se_ 94 +per 93 +The 93 +_pa 93 +heir 93 +id 93 +eir 93 +eir_ 93 +ig 93 +heir_ 93 +_no 93 +ev 93 +era 92 +_int 92 +ted 91 +_The 91 +ies 91 +art 91 +thei 90 +_ar 90 +_thei 90 +their 90 +_pro 90 +et 89 +_pe 88 +_mo 88 +ther 88 +x 87 +gh 87 +S 87 +_is_ 87 +ol 87 +ty_ 87 +_I 86 +nde 86 +am 86 +rn 86 +nte 86 +mp 85 +_su 84 +_we 84 +par 84 +_v 84 +pu 82 +his 82 +ow 82 +mi 82 +go 81 +N 81 +ue 81 +ple 81 +ep 80 +ab 80 +;_ 80 +; 80 +ex 80 +ain 80 +over 80 +_un 79 +q 79 +qu 79 +pp 79 +ith 79 +ry 79 +_as 79 +ber 79 +ub 78 +av 78 +uc 78 +s._ 77 +s. 77 +enc 77 +are 77 +iti 77 +gr 76 +his_ 76 +ua 76 +part 76 +ff 75 +eve 75 +O 75 +rea 74 +ous 74 +ia 74 +The_ 73 +ag 73 +mb 73 +_go 73 +fa 72 +on,_ 72 +ern 72 +t,_ 72 +on, 72 +t, 72 +_me 71 diff --git a/bin/text_cat/LM/german.lm b/bin/text_cat/LM/german.lm new file mode 100644 index 000000000..6f14f51ef --- /dev/null +++ b/bin/text_cat/LM/german.lm @@ -0,0 +1,400 @@ +_ 31586 +e 15008 +n 9058 +i 7299 +r 6830 +t 5662 +s 5348 +a 4618 +h 4176 +d 4011 +er 3415 +en 3412 +u 3341 +l 3266 +n_ 2848 +c 2636 +ch 2460 +g 2407 +o 2376 +e_ 2208 +r_ 2128 +m 2077 +_d 1948 +de 1831 +en_ 1786 +ei 1718 +er_ 1570 +in 1568 +te 1505 +ie 1505 +b 1458 +t_ 1425 +f 1306 +k 1176 +ge 1144 +s_ 1137 +un 1113 +, 1104 +,_ 1099 +w 1099 +z 1060 +nd 1039 +he 1004 +st 989 +_s 952 +_de 949 +. 909 +_e 906 +ne 906 +der 880 +._ 847 +be 841 +es 829 +ic 796 +_a 791 +ie_ 779 +is 769 +ich 763 +an 755 +re 749 +di 732 +ein 730 +se 730 +" 720 +ng 709 +_i 706 +sc 683 +sch 681 +it 673 +der_ 652 +h_ 651 +ch_ 642 +S 630 +le 609 +p 609 +ä 607 +ü 603 +au 603 +v 602 +che 599 +_w 596 +d_ 585 +die 576 +_di 572 +m_ 562 +_die 559 +el 548 +_S 540 +_der 529 +li 527 +_der_ 523 +si 515 +al 514 +ns 507 +on 501 +or 495 +ti 490 +ten 487 +ht 486 +die_ 485 +_die_ 483 +D 479 +rt 478 +nd_ 476 +_u 470 +nt 468 +A 466 +in_ 464 +den 461 +cht 447 +und 443 +me 440 +_z 429 +ung 426 +ll 423 +_un 421 +_ei 419 +_n 415 +hr 412 +ine 412 +_A 408 +_ein 405 +ar 404 +ra 403 +_v 400 +_g 400 +as 395 +zu 392 +et 389 +em 385 +_D 380 +eine 376 +gen 376 +g_ 376 +da 368 +we 366 +K 365 +lt 360 +B 354 +_" 353 +nde 349 +ni 347 +und_ 345 +E 345 +ur 345 +_m 342 +ri 341 +ha 340 +eh 339 +ten_ 338 +es_ 336 +_K 336 +_und 335 +ig 335 +_b 335 +hen 334 +_und_ 332 +_au 329 +_B 327 +_da 325 +_zu 324 +_in 322 +at 321 +us 318 +wi 307 +n, 305 +n,_ 304 +nn 304 +te_ 301 +eit 301 +_h 300 +ter 299 +M 298 +n. 295 +ß 294 +ng_ 289 +sche 289 +- 283 +rs 282 +den_ 282 +_si 280 +G 280 +im 278 +_ge 277 +chen 276 +rd 273 +_E 273 +n._ 270 +icht 270 +rn 268 +uf 267 +isch 264 +isc 264 +nen 263 +_in_ 262 +_M 260 +_er 257 +ich_ 255 +ac 253 +lic 252 +_G 252 +ber 252 +la 251 +vo 251 +eb 250 +ke 249 +F 248 +as_ 248 +hen_ 248 +ach 245 +en, 244 +ung_ 243 +lich 243 +ste 243 +en,_ 243 +_k 241 +ben 241 +_f 241 +en. 241 +_be 239 +it_ 239 +L 238 +_se 237 +mi 236 +ve 236 +na 236 +on_ 236 +P 235 +ss 234 +ist 234 +ö 234 +ht_ 233 +ru 233 +st_ 229 +_F 229 +ts 227 +ab 226 +W 226 +ol 225 +_eine 225 +hi 225 +so 224 +em_ 223 +"_ 223 +ren 222 +en._ 221 +chen_ 221 +R 221 +ta 221 +ere 220 +ische 219 +ers 218 +ert 217 +_P 217 +tr 217 +ed 215 +ze 215 +eg 215 +ens 215 +ür 213 +ah 212 +_vo 212 +ne_ 211 +cht_ 210 +uc 209 +_wi 209 +nge 208 +lle 208 +fe 207 +_L 207 +ver 206 +hl 205 +V 204 +ma 203 +wa 203 +auf 201 +H 198 +_W 195 +T 195 +nte 193 +uch 193 +l_ 192 +sei 192 +nen_ 190 +u_ 189 +_den 189 +_al 189 +_V 188 +t. 188 +lte 187 +ut 186 +ent 184 +sich 183 +sic 183 +il 183 +ier 182 +am 181 +gen_ 180 +sen 179 +fü 178 +um 178 +t._ 177 +f_ 174 +he_ 174 +ner 174 +nst 174 +ls 174 +_sei 173 +ro 173 +ir 173 +ebe 173 +mm 173 +ag 172 +ern 169 +t,_ 169 +t, 169 +eu 169 +ft 168 +icht_ 167 +hre 167 +Be 166 +nz 165 +nder 165 +_T 164 +_den_ 164 +iche 163 +tt 163 +zu_ 162 +and 162 +J 161 +rde 160 +rei 160 +_we 159 +_H 159 +ige 159 +_Be 158 +rte 157 +hei 156 +das 155 +aus 155 +che_ 154 +_das 154 +_zu_ 154 +tz 154 +_ni 153 +das_ 153 +_R 153 +N 153 +des 153 +_ve 153 +_J 152 +I 152 +_das_ 152 +men 151 +_so 151 +_ver 151 +_auf 150 +ine_ 150 +_ha 150 +rg 149 +ind 148 +eben 148 +kt 147 +mit 147 +_an 147 +her 146 +Ge 146 +Sc 145 +_sich 145 +U 145 +Sch 145 +_sic 145 +end 145 +Di 144 +abe 143 +ck 143 +sse 142 +ür_ 142 +ell 142 +ik 141 +o_ 141 +nic 141 +nich 141 +sa 141 +_fü 140 +hn 140 +zi 140 +no 140 +nicht 140 +im_ 139 +von_ 139 +von 139 +_nic 139 +_nich 139 +eine_ 139 +oc 138 +wei 138 +io 138 +schen 138 +gt 138 diff --git a/bin/text_cat/text_cat b/bin/text_cat/text_cat new file mode 100755 index 000000000..74dae861d --- /dev/null +++ b/bin/text_cat/text_cat @@ -0,0 +1,242 @@ +#!/usr/bin/perl -w +# © Gertjan van Noord, 1997. +# mailto:vannoord@let.rug.nl + +use strict; +use vars qw($opt_d $opt_f $opt_h $opt_i $opt_l $opt_n $opt_s $opt_t $opt_v $opt_u $opt_a); +use Getopt::Std; +use Benchmark; + +my $non_word_characters='0-9\s'; +my @languages; # languages (sorted by name) +my %ngram_for; # map language x ngram => rang + +# OPTIONS +getopts('a:d:f:hi:lnst:u:v'); + +# defaults: set $opt_X unless already defined (Perl Cookbook p. 6): +$opt_a ||= 10; +$opt_d ||= '/users1/vannoord/Perl/TextCat/LM'; +$opt_f ||= 0; +$opt_t ||= 400; +$opt_u ||= 1.05; + +$| = 1; # auto-flush stdout + +sub help { + print <<HELP +Text Categorization. Typically used to determine the language of a +given document. + +Usage +----- + +* print help message: + +$0 -h + +* for guessing: + +$0 [-a Int] [-d Dir] [-f Int] [-i N] [-l] [-t Int] [-u Int] [-v] + + -a the program returns the best-scoring language together + with all languages which are $opt_u times worse (cf option -u). + If the number of languages to be printed is larger than the value + of this option (default: $opt_a) then no language is returned, but + instead a message that the input is of an unknown language is + printed. Default: $opt_a. + -d indicates in which directory the language models are + located (files ending in .lm). Currently only a single + directory is supported. Default: $opt_d. + -f Before sorting is performed the Ngrams which occur this number + of times or less are removed. This can be used to speed up + the program for longer inputs. For short inputs you should use + -f 0. + Default: $opt_f. + -i N only read first N lines + -l indicates that input is given as an argument on the command line, + e.g. text_cat -l "this is english text" + Cannot be used in combination with -n. + -s Determine language of each line of input. Not very efficient yet, + because language models are re-loaded after each line. + -t indicates the topmost number of ngrams that should be used. + If used in combination with -n this determines the size of the + output. If used with categorization this determines + the number of ngrams that are compared with each of the language + models (but each of those models is used completely). + -u determines how much worse result must be in order not to be + mentioned as an alternative. Typical value: 1.05 or 1.1. + Default: $opt_u. + -v verbose. Continuation messages are written to standard error. + +* for creating new language model, based on text read from standard input: + +$0 -n [-v] + + -v verbose. Continuation messages are written to standard error. + + +HELP +} + +if ($opt_h) { help(); exit 0; }; + +if ($opt_n) { + my %ngram=(); + my @result = create_lm(input(),\%ngram); + print join("\n",map { "$_\t $ngram{$_}" ; } @result),"\n"; +} elsif ($opt_l) { + classify($ARGV[0]); +} elsif ($opt_s) { + while (<>) { + chomp; + classify($_); + } +} else { + classify(input()); +} + +sub read_model { + my ($file) = @_; + open(LM,"$file") or die "cannot open $file: $!\n"; + my %ngram; + my $rang = 1; + while (<LM>) { + chomp; + # only use lines starting with appropriate character. Others are + # ignored. + if (/^[^$non_word_characters]+/o) { + $ngram{$&} = $rang++; + } + } + return \%ngram; +} + +sub read_models { + # open directory to find which languages are supported + opendir DIR, "$opt_d" or die "directory $opt_d: $!\n"; + @languages = sort(grep { s/\.lm// && -r "$opt_d/$_.lm" } readdir(DIR)); + closedir DIR; + @languages or die "sorry, can't read any language models from $opt_d\n" . + "language models must reside in files with .lm ending\n"; + + foreach my $language (@languages) { + $ngram_for{$language} = read_model("$opt_d/$language.lm"); + } +} + +# CLASSIFICATION +sub classify { + my ($input)=@_; + my %results=(); + my $maxp = $opt_t; + read_models() if !@languages; + + # create ngrams for input. Note that hash %unknown is not used; + # it contains the actual counts which are only used under -n: creating + # new language model (and even then they are not really required). + my @unknown=create_lm($input); + + my $t1 = new Benchmark; + foreach my $language (@languages) { + # compares the language model with input ngrams list + my $ngram = $ngram_for{$language} or die "no ngrams for $language"; + + my ($i,$p)=(0,0); + while ($i < @unknown) { + if ($ngram->{$unknown[$i]}) { + $p=$p+abs($ngram->{$unknown[$i]}-$i); + } else { + $p=$p+$maxp; + } + ++$i; + } + #print STDERR "$language: $p\n" if $opt_v; + + $results{$language} = $p; + } + print STDERR "read language models done (" . + timestr(timediff(new Benchmark, $t1)) . + ".\n" if $opt_v; + my @results = sort { $results{$a} <=> $results{$b} } keys %results; + + print join("\n",map { "$_\t $results{$_}"; } @results),"\n" if $opt_v; + my $a = $results{$results[0]}; + + my @answers=(shift(@results)); + while (@results && $results{$results[0]} < ($opt_u *$a)) { + @answers=(@answers,shift(@results)); + } + if (@answers > $opt_a) { + print "I don't know; " . + "Perhaps this is a language I haven't seen before?\n"; + } else { + print join(" or ", @answers), "\n"; + } +} + +# first and only argument is reference to hash. +# this hash is filled, and a sorted list (opt_n elements) +# is returned. +sub input { + my $read=""; + if ($opt_i) { + while(<>) { + if ($. == $opt_i) { + return $read . $_; + } + $read = $read . $_; + } + return $read; + } else { + local $/; # so it doesn't affect $/ elsewhere + undef $/; + $read = <>; # swallow input. + $read || die "determining the language of an empty file is hard...\n"; + return $read; + } +} + + +sub create_lm { + my $t1 = new Benchmark; + my $ngram; + ($_,$ngram) = @_; #$ngram contains reference to the hash we build + # then add the ngrams found in each word in the hash + my $word; + foreach $word (split("[$non_word_characters]+")) { + $word = "_" . $word . "_"; + my $len = length($word); + my $flen=$len; + my $i; + for ($i=0;$i<$flen;$i++) { + $$ngram{substr($word,$i,5)}++ if $len > 4; + $$ngram{substr($word,$i,4)}++ if $len > 3; + $$ngram{substr($word,$i,3)}++ if $len > 2; + $$ngram{substr($word,$i,2)}++ if $len > 1; + $$ngram{substr($word,$i,1)}++; + $len--; + } + } + ###print "@{[%$ngram]}"; + my $t2 = new Benchmark; + print STDERR "count_ngrams done (". + timestr(timediff($t2, $t1)) .").\n" if $opt_v; + + # as suggested by Karel P. de Vos, k.vos@elsevier.nl, we speed up + # sorting by removing singletons + map { my $key=$_; if ($$ngram{$key} <= $opt_f) + { delete $$ngram{$key}; }; } keys %$ngram; + #however I have very bad results for short inputs, this way + + + # sort the ngrams, and spit out the $opt_t frequent ones. + # adding `or $a cmp $b' in the sort block makes sorting five + # times slower..., although it would be somewhat nicer (unique result) + my @sorted = sort { $$ngram{$b} <=> $$ngram{$a} } keys %$ngram; + splice(@sorted,$opt_t) if (@sorted > $opt_t); + print STDERR "sorting done (" . + timestr(timediff(new Benchmark, $t2)) . + ").\n" if $opt_v; + return @sorted; +} diff --git a/bin/text_cat/version b/bin/text_cat/version new file mode 100644 index 000000000..e6ba9d571 --- /dev/null +++ b/bin/text_cat/version @@ -0,0 +1,2 @@ +1.10 + diff --git a/bin/ui-checkdomain.sh b/bin/ui-checkdomain.sh new file mode 100755 index 000000000..30e0c5b0f --- /dev/null +++ b/bin/ui-checkdomain.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script finds .ui files with incorrect translation domain set +# and prints the domain, the file name and the expected domain +# See also the discussion at https://gerrit.libreoffice.org/#/c/72973/ + +declare -A modules + +# List of modules with .ui files and their expected translation domain +modules+=( \ + [basctl]=basctl \ + [chart2]=chart \ + [cui]=cui \ + [dbaccess]=dba \ + [desktop]=dkt \ + [editeng]=editeng \ + [extensions]=pcr \ + [filter]=flt \ + [formula]="for" \ + [fpicker]=fps \ + [framework]=fwk \ + [reportdesign]=rpt \ + [sc]=sc \ + [sd]=sd \ + [sfx2]=sfx \ + [starmath]=sm \ + [svtools]=svt \ + [svx]=svx \ + [sw]=sw \ + [uui]=uui \ + [vcl]=vcl \ + [writerperfect]=wpt \ + [xmlsecurity]=xsc \ +) + +# Iterate the keys, i.e. modules with a uiconfig subdir +for key in ${!modules[@]}; do + # Enumerate all .ui files in each module + for uifile in $(git ls-files ${key}/uiconfig/*\.ui); do + # Check that they contain the expected domain in double quotation marks, print the line if they don't + grep "\<interface domain=" $uifile | grep -v "\"${modules[${key}]}\""; + if [ "$?" -eq 0 ] ; + # Report the file name and the expected domain + then echo "^Problematic interface domain in file: $uifile ; should be: "${modules[${key}]}""; + fi + done +done diff --git a/bin/ui-translatable.sh b/bin/ui-translatable.sh new file mode 100755 index 000000000..d8188778b --- /dev/null +++ b/bin/ui-translatable.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script prints lines from .ui files, where the translatable="yes" attribute +# was not set -- presumably by mistake. It prints a few false positives though. + +for i in `git ls-files *.ui`; do + for j in "label" "title" "text" "format" "copyright" "comments" "preview_text" "tooltip" "message" ; do + grep -s "\<property name\=\"$j\"" $i | grep -v "translatable\=\"yes" | grep -v "translatable\=\"no" | grep -v gtk\- | grep ">.*[A-Za-z].*<"; + if [ "$?" -eq 0 ] ; + then echo "Source: $i^"; + fi + done + grep -s "<item" $i | grep -v "translatable\=\"yes" | grep -v "translatable\=\"no" | grep ">.*[A-Za-z].*<"; + if [ "$?" -eq 0 ] ; + then echo "Source: $i^"; + fi +done diff --git a/bin/unpack-sources b/bin/unpack-sources new file mode 100755 index 000000000..2408eda1e --- /dev/null +++ b/bin/unpack-sources @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +usage() +{ + echo "Helper script to unpack the LO source tarballs" + echo + echo "Usage: ${0##*/} [--help] start-dir tarball..." + echo + echo "Options:" + echo + echo " --help this help" + echo " start-dir path where the sources are unpacked (bootstrap directory)" + echo " tarball list of LO source tarball that need to be unpacked" +} + +start_dir= +tarballs= + +while test -n "$1" ; do + case "$1" in + --help) + usage + exit 0; + ;; + --download) + download="yes" + ;; + -*) + echo "Error: unknown option: $1" + exit 1; + ;; + *) + if test -z "$start_dir" ; then + start_dir="$1" + else + tarballs="$tarballs $1" + fi + ;; + esac + shift +done + +if test -z "$start_dir" ; then + echo "Error: Please, define where to unpack sources, try --help" +fi + +if ! test -f $start_dir/Repository.mk ; then + echo "Error: $start_dir is not a valid LibreOffice core source directory" + exit 1; +fi + +if test ! -f $start_dir/sources.ver -o -d $start_dir/.git ; then + echo "Warning: sources are from git and not from tarball" + echo " Do nothing." + exit 0; +fi + +lo_src_dir="$start_dir/src" +mkdir -p "$lo_src_dir" + +for tarball in $tarballs ; do + tarname=`basename $tarball | sed -e "s/\.tar\..*//"` + if test -d $lo_src_dir/$tarname ; then + echo "Warning: $lo_src_dir/$tarname already exists => skipping" + continue; + fi + + echo "Unpacking $tarname..." + echo mkdir -p "$lo_src_dir/$tarname" + if ! mkdir -p "$lo_src_dir/$tarname" ; then + echo "Error: could not create directory $lo_src_dir/$tarname" + fi + echo tar -xf "$tarball" -C "$lo_src_dir/$tarname" --strip-components=1 + if ! tar -xf "$tarball" -C "$lo_src_dir/$tarname" --strip-components=1; then + echo "Error: could not unpack $tarname" + exit 1 + fi + + # create symlinks for module directories; ignore git-hooks directory + for dir in `find "$lo_src_dir/$tarname" -mindepth 1 -maxdepth 1 -type d -path $lo_src_dir/$tarname/git-hooks -o -printf "$tarname/%f\n"` ; do + ln -sf "src/$dir" "$start_dir" + done +done diff --git a/bin/update/common.sh b/bin/update/common.sh new file mode 100644 index 000000000..5bba576c7 --- /dev/null +++ b/bin/update/common.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Code shared by update packaging scripts. +# Author: Darin Fisher +# + +# ----------------------------------------------------------------------------- +# By default just assume that these tools exist on our path +MAR=${MAR:-mar} +BZIP2=${BZIP2:-bzip2} +MBSDIFF=${MBSDIFF:-mbsdiff} + +# ----------------------------------------------------------------------------- +# Helper routines + +notice() { + echo "$*" 1>&2 +} + +get_file_size() { + info=($(ls -ln "$1")) + echo ${info[4]} +} + +check_externals() { + + # check whether we can call the mar executable + "$MAR" --version > /dev/null 2>&1 + if [ $? != 0 ]; then + notice "Could not find a valid mar executable in the path or in the MAR environment variable" + exit 1 + fi + + # check whether we can access the bzip2 executable + "$BZIP2" --help > /dev/null 2>&1 + if [ $? != 0 ]; then + notice "Could not find a valid bzip2 executable in the PATH or in the BZIP2 environment variable" + exit 1 + fi +} + +copy_perm() { + reference="$1" + target="$2" + + if [ -x "$reference" ]; then + chmod 0755 "$target" + else + chmod 0644 "$target" + fi +} + +make_add_instruction() { + f="$1" + filev2="$2" + # The third param will be an empty string when a file add instruction is only + # needed in the version 2 manifest. This only happens when the file has an + # add-if-not instruction in the version 3 manifest. This is due to the + # precomplete file prior to the version 3 manifest having a remove instruction + # for this file so the file is removed before applying a complete update. + filev3="$3" + + # Used to log to the console + if [ $4 ]; then + forced=" (forced)" + else + forced= + fi + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " add-if \"$testdir\" \"$f\"" + echo "add-if \"$testdir\" \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add-if \"$testdir\" \"$f\"" >> $filev3 + fi + else + notice " add \"$f\"$forced" + echo "add \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add \"$f\"" >> $filev3 + fi + fi +} + +check_for_add_if_not_update() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ + `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +check_for_add_to_manifestv2() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +make_add_if_not_instruction() { + f="$1" + filev3="$2" + + notice " add-if-not \"$f\" \"$f\"" + echo "add-if-not \"$f\" \"$f\"" >> $filev3 +} + +make_patch_instruction() { + f="$1" + filev2="$2" + filev3="$3" + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 + else + notice " patch \"$f.patch\" \"$f\"" + echo "patch \"$f.patch\" \"$f\"" >> $filev2 + echo "patch \"$f.patch\" \"$f\"" >> $filev3 + fi +} + +append_remove_instructions() { + dir="$1" + filev2="$2" + filev3="$3" + + if [ -f "$dir/removed-files" ]; then + listfile="$dir/removed-files" + elif [ -f "$dir/Contents/Resources/removed-files" ]; then + listfile="$dir/Contents/Resources/removed-files" + fi + if [ -n "$listfile" ]; then + # Map spaces to pipes so that we correctly handle filenames with spaces. + files=($(cat "$listfile" | tr " " "|" | sort -r)) + num_files=${#files[*]} + for ((i=0; $i<$num_files; i=$i+1)); do + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. + if [ -n "$f" ]; then + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + notice " rmdir \"$f\"" + echo "rmdir \"$f\"" >> $filev2 + echo "rmdir \"$f\"" >> $filev3 + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + notice " rmrfdir \"$f\"" + echo "rmrfdir \"$f\"" >> $filev2 + echo "rmrfdir \"$f\"" >> $filev3 + else + notice " remove \"$f\"" + echo "remove \"$f\"" >> $filev2 + echo "remove \"$f\"" >> $filev3 + fi + fi + fi + done + fi +} + +# List all files in the current directory, stripping leading "./" +# Pass a variable name and it will be filled as an array. +list_files() { + count=0 + + find . -type f \ + ! -name "update.manifest" \ + ! -name "updatev2.manifest" \ + ! -name "updatev3.manifest" \ + ! -name "temp-dirlist" \ + ! -name "temp-filelist" \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "temp-filelist" + while read file; do + eval "${1}[$count]=\"$file\"" + (( count++ )) + done < "temp-filelist" + rm "temp-filelist" +} + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "temp-dirlist" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "temp-dirlist" + rm "temp-dirlist" +} diff --git a/bin/update/config.py b/bin/update/config.py new file mode 100644 index 000000000..0bc60a07f --- /dev/null +++ b/bin/update/config.py @@ -0,0 +1,28 @@ + +import configparser +import os + +class Config(object): + + def __init__(self): + self.certificate_path = None + self.certificate_name = None + self.channel = None + self.base_url = None + self.upload_url = None + self.server_url = None + +def parse_config(config_file): + config = configparser.ConfigParser() + config.read(os.path.expanduser(config_file)) + + data = Config() + updater_data = config['Updater'] + data.base_url = updater_data['base-url'] + data.certificate_name = updater_data['certificate-name'] + data.certificate_path = updater_data['certificate-path'] + data.channel = updater_data['channel'] + data.upload_url = updater_data['upload-url'] + data.server_url = updater_data["ServerURL"] + + return data diff --git a/bin/update/create_build_config.py b/bin/update/create_build_config.py new file mode 100755 index 000000000..7cc8ac4be --- /dev/null +++ b/bin/update/create_build_config.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python3 + +import json +import sys +import os + +from config import parse_config + +from tools import replace_variables_in_string + +def update_all_url_entries(data, **kwargs): + data['complete']['url'] = replace_variables_in_string(data['complete']['url'], **kwargs) + + if sys.platform != "cygwin": + for language in data['languages']: + language['complete']['url'] = replace_variables_in_string(language['complete']['url'], **kwargs) + + if 'partials' in data: + for partial in data['partials']: + partial['file']['url'] = replace_variables_in_string(partial['file']['url'], **kwargs) + + if sys.platform == "cygwin": + continue + + for lang, lang_file in partial['languages'].items(): + lang_file['url'] = replace_variables_in_string(lang_file['url'], **kwargs) + +def main(argv): + if len(argv) < 7: + print("Usage: create_build_config.py $PRODUCTNAME $VERSION $BUILDID $PLATFORM $TARGETDIR $UPDATE_CONFIG") + sys.exit(1) + + config = parse_config(argv[6]) + + data = { 'productName' : argv[1], + 'version' : argv[2], + 'buildNumber' : argv[3], + 'updateChannel' : config.channel, + 'platform' : argv[4] + } + + extra_data_files = ['complete_info.json', 'partial_update_info.json'] + if sys.platform != "cygwin": + extra_data_files.append('complete_lang_info.json') + + for extra_file in extra_data_files: + extra_file_path = os.path.join(argv[5], extra_file) + if not os.path.exists(extra_file_path): + continue + with open(extra_file_path, "r") as f: + extra_data = json.load(f) + data.update(extra_data) + + update_all_url_entries(data, channel=config.channel, platform=argv[4], buildid=argv[3], version=argv[2]) + + with open(os.path.join(argv[5], "build_config.json"), "w") as f: + json.dump(data, f, indent=4) + +if __name__ == "__main__": + main(sys.argv) diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py new file mode 100755 index 000000000..48686be21 --- /dev/null +++ b/bin/update/create_full_mar.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import sys +import os +import subprocess +import json + +from tools import uncompress_file_to_dir, get_file_info, make_complete_mar_name +from config import parse_config +from signing import sign_mar_file +from path import UpdaterPath, convert_to_unix, convert_to_native + +current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__))) + +def main(): + if len(sys.argv) < 5: + print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $FILENAMEPREFIX $UPDATE_CONFIG") + sys.exit(1) + + update_config = sys.argv[4] + filename_prefix = sys.argv[3] + workdir = sys.argv[2] + product_name = sys.argv[1] + + if len(update_config) == 0: + print("missing update config") + sys.exit(1) + + update_path = UpdaterPath(workdir) + update_path.ensure_dir_exist() + + target_dir = update_path.get_update_dir() + temp_dir = update_path.get_current_build_dir() + + config = parse_config(update_config) + + tar_dir = os.path.join(update_path.get_workdir(), "installation", product_name, "archive", "install", "en-US") + tar_file = os.path.join(tar_dir, os.listdir(tar_dir)[0]) + + uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir) + + mar_file = make_complete_mar_name(target_dir, filename_prefix) + path = os.path.join(current_dir_path, 'make_full_update.sh') + subprocess.call([path, convert_to_native(mar_file), convert_to_native(uncompress_dir)]) + + sign_mar_file(target_dir, config, mar_file, filename_prefix) + + file_info = { 'complete' : get_file_info(mar_file, config.base_url) } + + with open(os.path.join(target_dir, 'complete_info.json'), "w") as complete_info_file: + json.dump(file_info, complete_info_file, indent = 4) + +if __name__ == '__main__': + main() diff --git a/bin/update/create_full_mar_for_languages.py b/bin/update/create_full_mar_for_languages.py new file mode 100755 index 000000000..039521dd1 --- /dev/null +++ b/bin/update/create_full_mar_for_languages.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import subprocess +import json + +from tools import uncompress_file_to_dir, get_file_info + +from config import parse_config +from path import UpdaterPath +from signing import sign_mar_file + +current_dir_path = os.path.dirname(os.path.realpath(__file__)) + +def make_complete_mar_name(target_dir, filename_prefix, language): + filename = filename_prefix + "_" + language + "_complete_langpack.mar" + return os.path.join(target_dir, filename) + +def create_lang_infos(mar_file_name, language, url): + data = {'lang' : language, + 'complete' : get_file_info(mar_file_name, url) + } + return data + +def main(): + if len(sys.argv) < 5: + print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $UPDATE_CONFIG") + sys.exit(1) + + update_config = sys.argv[4] + filename_prefix = sys.argv[3] + workdir = sys.argv[2] + product_name = sys.argv[1] + + updater_path = UpdaterPath(workdir) + target_dir = updater_path.get_update_dir() + temp_dir = updater_path.get_language_dir() + + config = parse_config(update_config) + + language_pack_dir = os.path.join(workdir, "installation", product_name + "_languagepack", "archive", "install") + language_packs = os.listdir(language_pack_dir) + lang_infos = [] + for language in language_packs: + if language == 'log': + continue + + language_dir = os.path.join(language_pack_dir, language) + language_file = os.path.join(language_dir, os.listdir(language_dir)[0]) + + directory = uncompress_file_to_dir(language_file, os.path.join(temp_dir, language)) + + mar_file_name = make_complete_mar_name(target_dir, filename_prefix, language) + + subprocess.call([os.path.join(current_dir_path, 'make_full_update.sh'), mar_file_name, directory]) + + sign_mar_file(target_dir, config, mar_file_name, filename_prefix) + + lang_infos.append(create_lang_infos(mar_file_name, language, config.base_url)) + + with open(os.path.join(target_dir, "complete_lang_info.json"), "w") as language_info_file: + json.dump({'languages' : lang_infos}, language_info_file, indent=4) + +if __name__ == '__main__': + main() diff --git a/bin/update/create_partial_update.py b/bin/update/create_partial_update.py new file mode 100755 index 000000000..9412bcd6e --- /dev/null +++ b/bin/update/create_partial_update.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +import requests +import json +import sys +import hashlib +import os +import subprocess +import errno +import json + +from config import parse_config +from uncompress_mar import extract_mar +from tools import get_file_info, get_hash +from signing import sign_mar_file + +from path import UpdaterPath, mkdir_p, convert_to_unix, convert_to_native + +BUF_SIZE = 1024 +current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__))) + +class InvalidFileException(Exception): + + def __init__(self, *args, **kwargs): + super().__init__(self, *args, **kwargs) + +def download_file(filepath, url, hash_string): + with open(filepath, "wb") as f: + response = requests.get(url, stream=True) + + if not response.ok: + return + + for block in response.iter_content(1024): + f.write(block) + + file_hash = get_hash(filepath) + + if file_hash != hash_string: + raise InvalidFileException("file hash does not match for file %s: Expected %s, Got: %s" % (url, hash_string, file_hash)) + +def handle_language(lang_entries, filedir): + mar = os.environ.get('MAR', 'mar') + langs = {} + for lang, data in lang_entries.items(): + lang_dir = os.path.join(filedir, lang) + lang_file = os.path.join(lang_dir, "lang.mar") + mkdir_p(lang_dir) + download_file(lang_file , data["url"], data["hash"]) + dir_path = os.path.join(lang_dir, "lang") + mkdir_p(dir_path) + extract_mar(lang_file, dir_path) + langs[lang] = dir_path + + return langs + +def download_mar_for_update_channel_and_platform(config, platform, temp_dir): + mar = os.environ.get('MAR', 'mar') + base_url = config.server_url + "update/partial-targets/1/" + url = base_url + platform + "/" + config.channel + r = requests.get(url) + if r.status_code != 200: + print(r.content) + raise Exception("download failed") + + update_info = json.loads(r.content.decode("utf-8")) + update_files = update_info['updates'] + downloaded_updates = {} + for update_file in update_files: + build = update_file["build"] + filedir = os.path.join(temp_dir, build) + + mkdir_p(filedir) + + filepath = filedir + "/complete.mar" + url = update_file["update"]["url"] + expected_hash = update_file["update"]["hash"] + download_file(filepath, url, expected_hash) + + dir_path = os.path.join(filedir, "complete") + mkdir_p(dir_path) + extract_mar(filepath, dir_path) + + downloaded_updates[build] = {"complete": dir_path} + + langs = handle_language(update_file["languages"], filedir) + downloaded_updates[build]["languages"] = langs + + return downloaded_updates + +def generate_file_name(current_build_id, old_build_id, mar_name_prefix): + name = "%s_from_%s_partial.mar" %(mar_name_prefix, old_build_id) + return name + +def generate_lang_file_name(current_build_id, old_build_id, mar_name_prefix, lang): + name = "%s_%s_from_%s_partial.mar" %(mar_name_prefix, lang, old_build_id) + return name + +def add_single_dir(path): + dir_name = [os.path.join(path, name) for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))] + return dir_name[0] + +def main(): + workdir = sys.argv[1] + + updater_path = UpdaterPath(workdir) + updater_path.ensure_dir_exist() + + mar_name_prefix = sys.argv[2] + update_config = sys.argv[3] + platform = sys.argv[4] + build_id = sys.argv[5] + + current_build_path = updater_path.get_current_build_dir() + mar_dir = updater_path.get_mar_dir() + temp_dir = updater_path.get_previous_build_dir() + update_dir = updater_path.get_update_dir() + + current_build_path = add_single_dir(current_build_path) + if sys.platform == "cygwin": + current_build_path = add_single_dir(current_build_path) + + config = parse_config(update_config) + + updates = download_mar_for_update_channel_and_platform(config, platform, temp_dir) + + data = {"partials": []} + + for build, update in updates.items(): + file_name = generate_file_name(build_id, build, mar_name_prefix) + mar_file = os.path.join(update_dir, file_name) + subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(mar_file), convert_to_native(update["complete"]), convert_to_native(current_build_path)]) + sign_mar_file(update_dir, config, mar_file, mar_name_prefix) + + partial_info = {"file":get_file_info(mar_file, config.base_url), "from": build, "to": build_id, "languages": {}} + + # on Windows we don't use language packs + if sys.platform != "cygwin": + for lang, lang_info in update["languages"].items(): + lang_name = generate_lang_file_name(build_id, build, mar_name_prefix, lang) + + # write the file into the final directory + lang_mar_file = os.path.join(update_dir, lang_name) + + # the directory of the old language file is of the form + # workdir/mar/language/en-US/LibreOffice_<version>_<os>_archive_langpack_<lang>/ + language_dir = add_single_dir(os.path.join(mar_dir, "language", lang)) + subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(lang_mar_file), convert_to_native(lang_info), convert_to_native(language_dir)]) + sign_mar_file(update_dir, config, lang_mar_file, mar_name_prefix) + + # add the partial language info + partial_info["languages"][lang] = get_file_info(lang_mar_file, config.base_url) + + data["partials"].append(partial_info) + + with open(os.path.join(update_dir, "partial_update_info.json"), "w") as f: + json.dump(data, f) + + +if __name__ == '__main__': + main() diff --git a/bin/update/get_update_channel.py b/bin/update/get_update_channel.py new file mode 100755 index 000000000..f94507d64 --- /dev/null +++ b/bin/update/get_update_channel.py @@ -0,0 +1,23 @@ +#!/usr/bin/python3 +# -*- Mode: python; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import sys +from config import parse_config + +def main(): + if len(sys.argv) < 2: + sys.exit(1) + + update_config = sys.argv[1] + config = parse_config(update_config) + print(config.channel) + +if __name__ == "__main__": + main() + +# vim:set shiftwidth=4 softtabstop=4 expandtab: */ diff --git a/bin/update/make_full_update.sh b/bin/update/make_full_update.sh new file mode 100755 index 000000000..cb7de49b2 --- /dev/null +++ b/bin/update/make_full_update.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates full update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +if [ $1 = -h ]; then + print_usage + notice "" + notice "The contents of DIRECTORY will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice "" + exit 1 +fi + +check_externals +# ----------------------------------------------------------------------------- + +archive="$1" +targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi +workdir="$targetdir.work" +updatemanifestv2="$workdir/updatev2.manifest" +updatemanifestv3="$workdir/updatev3.manifest" +targetfiles="updatev2.manifest updatev3.manifest" + +mkdir -p "$workdir" +echo "updatev2.manifest" >> $workdir/files.txt +echo "updatev3.manifest" >> $workdir/files.txt + +# Generate a list of all files in the target directory. +pushd "$targetdir" +if test $? -ne 0 ; then + exit 1 +fi + +# if [ ! -f "precomplete" ]; then +# if [ ! -f "Contents/Resources/precomplete" ]; then +# notice "precomplete file is missing!" +# exit 1 +# fi +# fi + +list_files files + +popd + +# Add the type of update to the beginning of the update manifests. +> $updatemanifestv2 +> $updatemanifestv3 +notice "" +notice "Adding type instruction to update manifests" +notice " type complete" +echo "type \"complete\"" >> $updatemanifestv2 +echo "type \"complete\"" >> $updatemanifestv3 + +notice "" +notice "Adding file add instructions to update manifests" +num_files=${#files[*]} + +for ((i=0; $i<$num_files; i=$i+1)); do + f="${files[$i]}" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + if check_for_add_to_manifestv2 "$f"; then + make_add_instruction "$f" "$updatemanifestv2" "" 1 + fi + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + fi + + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" + echo $f >> $workdir/files.txt +done + +# Append remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" + +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" +$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" + +eval "$MAR -C \"$workdir\" -c output.mar -f $workdir/files.txt" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/bin/update/make_incremental_update.sh b/bin/update/make_incremental_update.sh new file mode 100755 index 000000000..e76f2159f --- /dev/null +++ b/bin/update/make_incremental_update.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates incremental update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" + notice "" + notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice " -f clobber this file in the installation" + notice " Must be a path to a file to clobber in the partial update." + notice "" +} + +check_for_forced_update() { + force_list="$1" + forced_file_chk="$2" + + local f + + if [ "$forced_file_chk" = "precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "${forced_file_chk##*.}" = "chk" ]; then + ## "true" *giggle* + return 0; + fi + + for f in $force_list; do + #echo comparing $forced_file_chk to $f + if [ "$forced_file_chk" = "$f" ]; then + ## "true" *giggle* + return 0; + fi + done + ## 'false'... because this is bash. Oh yay! + return 1; +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +requested_forced_updates='Contents/MacOS/firefox' + +while getopts "hf:" flag +do + case "$flag" in + h) print_usage; exit 0 + ;; + f) requested_forced_updates="$requested_forced_updates $OPTARG" + ;; + ?) print_usage; exit 1 + ;; + esac +done + +# ----------------------------------------------------------------------------- + +let arg_start=$OPTIND-1 +shift $arg_start + +archive="$1" +olddir="$2" +newdir="$3" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then + # Remove the / + newdir=$(echo "$newdir" | sed -e 's:\/$::') +fi +workdir="$newdir.work" +updatemanifestv2="$workdir/updatev2.manifest" +updatemanifestv3="$workdir/updatev3.manifest" + +mkdir -p "$workdir" +echo "updatev2.manifest" >> $workdir/files.txt +echo "updatev3.manifest" >> $workdir/files.txt + +# Generate a list of all files in the target directory. +pushd "$olddir" +if test $? -ne 0 ; then + exit 1 +fi + +list_files oldfiles +list_dirs olddirs + +popd + +pushd "$newdir" +if test $? -ne 0 ; then + exit 1 +fi + +# if [ ! -f "precomplete" ]; then +# if [ ! -f "Contents/Resources/precomplete" ]; then +# notice "precomplete file is missing!" +# exit 1 +# fi +# fi + +list_dirs newdirs +list_files newfiles + +popd + +# Add the type of update to the beginning of the update manifests. +notice "" +notice "Adding type instruction to update manifests" +> $updatemanifestv2 +> $updatemanifestv3 +notice " type partial" +echo "type \"partial\"" >> $updatemanifestv2 +echo "type \"partial\"" >> $updatemanifestv3 + +notice "" +notice "Adding file patch and add instructions to update manifests" + +num_oldfiles=${#oldfiles[*]} +remove_array= +num_removes=0 + +for ((i=0; $i<$num_oldfiles; i=$i+1)); do + f="${oldfiles[$i]}" + + # If this file exists in the new directory as well, then check if it differs. + if [ -f "$newdir/$f" ]; then + + if check_for_add_if_not_update "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p `dirname "$workdir/$f"` + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_if_not_instruction "$f" "$updatemanifestv3" + echo $f >> $workdir/files.txt + continue 1 + fi + + if check_for_forced_update "$requested_forced_updates" "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p `dirname "$workdir/$f"` + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 + echo $f >> $workdir/files.txt + continue 1 + fi + + if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then + # Compute both the compressed binary diff and the compressed file, and + # compare the sizes. Then choose the smaller of the two to package. + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + notice "diffing \"$f\"" + # MBSDIFF_HOOK represents the communication interface with funsize and, + # if enabled, caches the intermediate patches for future use and + # compute avoidance + # + # An example of MBSDIFF_HOOK env variable could look like this: + # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" + # where myscript.sh has the following usage: + # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ + # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL + # + # Note: patches are bzipped stashed in funsize to gain more speed + + # if service is not enabled then default to old behavior + if [ -z "$MBSDIFF_HOOK" ]; then + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $BZIP2 -z9 "$workdir/$f.patch" + else + # if service enabled then check patch existence for retrieval + if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then + notice "file \"$f\" found in funsize, diffing skipped" + else + # if not found already - compute it and cache it for future use + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $BZIP2 -z9 "$workdir/$f.patch" + $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" + fi + fi + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + patchfile="$workdir/$f.patch.bz2" + patchsize=$(get_file_size "$patchfile") + fullsize=$(get_file_size "$workdir/$f") + + if [ $patchsize -lt $fullsize ]; then + make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + mv -f "$patchfile" "$workdir/$f.patch" + rm -f "$workdir/$f" + echo $f.patch >> $workdir/files.txt + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + rm -f "$patchfile" + echo $f >> $workdir/files.txt + fi + fi + else + # remove instructions are added after add / patch instructions for + # consistency with make_incremental_updates.py + remove_array[$num_removes]=$f + (( num_removes++ )) + fi +done + +# Newly added files +notice "" +notice "Adding file add instructions to update manifests" +num_newfiles=${#newfiles[*]} + +for ((i=0; $i<$num_newfiles; i=$i+1)); do + f="${newfiles[$i]}" + + # If we've already tested this file, then skip it + for ((j=0; $j<$num_oldfiles; j=$j+1)); do + if [ "$f" = "${oldfiles[j]}" ]; then + continue 2 + fi + done + + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + fi + + + echo $f >> $workdir/files.txt +done + +notice "" +notice "Adding file remove instructions to update manifests" +for ((i=0; $i<$num_removes; i=$i+1)); do + f="${remove_array[$i]}" + notice " remove \"$f\"" + echo "remove \"$f\"" >> $updatemanifestv2 + echo "remove \"$f\"" >> $updatemanifestv3 +done + +# Add remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" + +notice "" +notice "Adding directory remove instructions for directories that no longer exist" +num_olddirs=${#olddirs[*]} + +for ((i=0; $i<$num_olddirs; i=$i+1)); do + f="${olddirs[$i]}" + # If this dir doesn't exist in the new directory remove it. + if [ ! -d "$newdir/$f" ]; then + notice " rmdir $f/" + echo "rmdir \"$f/\"" >> $updatemanifestv2 + echo "rmdir \"$f/\"" >> $updatemanifestv3 + fi +done + +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" +$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" + +mar_command="$MAR" +if [[ -n $PRODUCT_VERSION ]] +then + mar_command="$mar_command -V $PRODUCT_VERSION" +fi +if [[ -n $CHANNEL_ID ]] +then + mar_command="$mar_command -H $CHANNEL_ID" +fi +mar_command="$mar_command -C \"$workdir\" -c output.mar -f $workdir/files.txt" +eval "$mar_command" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/bin/update/path.py b/bin/update/path.py new file mode 100644 index 000000000..0fe0fd5eb --- /dev/null +++ b/bin/update/path.py @@ -0,0 +1,69 @@ +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import os +import errno +import subprocess +from sys import platform + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def convert_to_unix(path): + if platform == "cygwin": + return subprocess.check_output(["cygpath", "-u", path]).decode("utf-8", "strict").rstrip() + else: + return path + +def convert_to_native(path): + if platform == "cygwin": + return subprocess.check_output(["cygpath", "-m", path]).decode("utf-8", "strict").rstrip() + else: + return path + +class UpdaterPath(object): + + def __init__(self, workdir): + self._workdir = convert_to_unix(workdir) + + def get_workdir(self): + return self._workdir + + def get_update_dir(self): + return os.path.join(self._workdir, "update-info") + + def get_current_build_dir(self): + return os.path.join(self._workdir, "mar", "current-build") + + def get_mar_dir(self): + return os.path.join(self._workdir, "mar") + + def get_previous_build_dir(self): + return os.path.join(self._workdir, "mar", "previous-build") + + def get_language_dir(self): + return os.path.join(self.get_mar_dir(), "language") + + def get_workdir(self): + return self._workdir + + def ensure_dir_exist(self): + mkdir_p(self.get_update_dir()) + mkdir_p(self.get_current_build_dir()) + mkdir_p(self.get_mar_dir()) + mkdir_p(self.get_previous_build_dir()) + mkdir_p(self.get_language_dir()) + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/update/signing.py b/bin/update/signing.py new file mode 100644 index 000000000..c0b43ce91 --- /dev/null +++ b/bin/update/signing.py @@ -0,0 +1,12 @@ +from tools import make_complete_mar_name + +import os +import subprocess +import path + +def sign_mar_file(target_dir, config, mar_file, filename_prefix): + signed_mar_file = make_complete_mar_name(target_dir, filename_prefix + '_signed') + mar_executable = os.environ.get('MAR', 'mar') + subprocess.check_call([mar_executable, '-C', path.convert_to_native(target_dir), '-d', path.convert_to_native(config.certificate_path), '-n', config.certificate_name, '-s', path.convert_to_native(mar_file), path.convert_to_native(signed_mar_file)]) + + os.rename(signed_mar_file, mar_file) diff --git a/bin/update/tools.py b/bin/update/tools.py new file mode 100644 index 000000000..8cd786635 --- /dev/null +++ b/bin/update/tools.py @@ -0,0 +1,64 @@ +import os +import hashlib +import zipfile +import tarfile + +def uncompress_file_to_dir(compressed_file, uncompress_dir): + command = None + extension = os.path.splitext(compressed_file)[1] + + try: + os.mkdir(uncompress_dir) + except FileExistsError as e: + pass + + if extension == '.gz': + tar = tarfile.open(compressed_file) + tar.extractall(uncompress_dir) + tar.close() + elif extension == '.zip': + zip_file = zipfile.ZipFile(compressed_file) + zip_file.extractall(uncompress_dir) + zip_file.close() + + uncompress_dir = os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]) + if " " in os.listdir(uncompress_dir)[0]: + print("replacing whitespace in directory name") + os.rename(os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]), + os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0].replace(" ", "_"))) + else: + print("Error: unknown extension " + extension) + + return os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]) + +BUF_SIZE = 1048576 + +def get_hash(file_path): + sha512 = hashlib.sha512() + with open(file_path, 'rb') as f: + while True: + data = f.read(BUF_SIZE) + if not data: + break + sha512.update(data) + return sha512.hexdigest() + +def get_file_info(mar_file, url): + filesize = os.path.getsize(mar_file) + data = { 'hash' : get_hash(mar_file), + 'hashFunction' : 'sha512', + 'size' : filesize, + 'url' : url + os.path.basename(mar_file)} + + return data + +def replace_variables_in_string(string, **kwargs): + new_string = string + for key, val in kwargs.items(): + new_string = new_string.replace('$(%s)'%key, val) + + return new_string + +def make_complete_mar_name(target_dir, filename_prefix): + filename = filename_prefix + "_complete.mar" + return os.path.join(target_dir, filename) diff --git a/bin/update/uncompress_mar.py b/bin/update/uncompress_mar.py new file mode 100755 index 000000000..0989c7e92 --- /dev/null +++ b/bin/update/uncompress_mar.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Extract a mar file and uncompress the content + +import os +import re +import sys +import subprocess +from path import convert_to_native + +def uncompress_content(file_path): + bzip2 = os.environ.get('BZIP2', 'bzip2') + file_path_compressed = file_path + ".bz2" + os.rename(file_path, file_path_compressed) + subprocess.check_call(["bzip2", "-d", convert_to_native(file_path_compressed)]) + +def extract_mar(mar_file, target_dir): + mar = os.environ.get('MAR', 'mar') + subprocess.check_call([mar, "-C", convert_to_native(target_dir), "-x", convert_to_native(mar_file)]) + file_info = subprocess.check_output([mar, "-t", convert_to_native(mar_file)]) + lines = file_info.splitlines() + prog = re.compile("\d+\s+\d+\s+(.+)") + for line in lines: + match = prog.match(line.decode("utf-8", "strict")) + if match is None: + continue + info = match.groups()[0] + # ignore header line + if info == b'NAME': + continue + + uncompress_content(os.path.join(target_dir, info)) + +def main(): + if len(sys.argv) != 3: + print("Help: This program takes exactly two arguments pointing to a mar file and a target location") + sys.exit(1) + + mar_file = sys.argv[1] + target_dir = sys.argv[2] + extract_mar(mar_file, target_dir) + +if __name__ == "__main__": + main() + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/update/upload_build_config.py b/bin/update/upload_build_config.py new file mode 100755 index 000000000..9a87661ee --- /dev/null +++ b/bin/update/upload_build_config.py @@ -0,0 +1,42 @@ +#! /usr/bin/env python3 + +import sys +import os +import configparser +import requests + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +def main(argv): + + updater_config = sys.argv[2] + + config = configparser.ConfigParser() + config.read(os.path.expanduser(updater_config)) + + user = config["Updater"]["User"] + password = config["Updater"]["Password"] + base_address = config["Updater"]["ServerURL"] + + login_url = base_address + "accounts/login/" + + session = requests.session() + r1 = session.get(login_url) + csrftoken = session.cookies['csrftoken'] + + login_data = { 'username': user,'password': password, + 'csrfmiddlewaretoken': csrftoken } + r1 = session.post(login_url, data=login_data, headers={"Referer": login_url}) + + url = base_address + "update/upload/release" + data = {} + data['csrfmiddlewaretoken'] = csrftoken + + build_config = os.path.join(sys.argv[1], "build_config.json") + r = session.post(url, files={'release_config': open(build_config, "r")}, data=data) + print(r.content) + if r.status_code != 200: + sys.exit(1) + +if __name__ == "__main__": + main(sys.argv) diff --git a/bin/update/upload_builds.py b/bin/update/upload_builds.py new file mode 100755 index 000000000..210668e0d --- /dev/null +++ b/bin/update/upload_builds.py @@ -0,0 +1,32 @@ +#! /usr/bin/env python3 + +import sys +import os +import subprocess + +from config import parse_config +from path import convert_to_unix + +from tools import replace_variables_in_string + +def main(): + product_name = sys.argv[1] + buildid = sys.argv[2] + platform = sys.argv[3] + update_dir = sys.argv[4] + update_config = sys.argv[5] + + config = parse_config(update_config) + upload_url = replace_variables_in_string(config.upload_url, channel=config.channel, buildid=buildid, platform=platform) + + target_url, target_dir = upload_url.split(':') + + command = "ssh %s 'mkdir -p %s'"%(target_url, target_dir) + print(command) + subprocess.call(command, shell=True) + for file in os.listdir(update_dir): + if file.endswith('.mar'): + subprocess.call(['scp', convert_to_unix(os.path.join(update_dir, file)), upload_url]) + +if __name__ == '__main__': + main() diff --git a/bin/update_pch b/bin/update_pch new file mode 100755 index 000000000..00cd50681 --- /dev/null +++ b/bin/update_pch @@ -0,0 +1,1308 @@ +#! /usr/bin/env python3 +# -*- Mode: python; tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +""" +This script generates precompiled headers for a given +module and library. + +Given a gmake makefile that belongs to some LO module: +1) Process the makefile to find source files (process_makefile). +2) For every source file, find all includes (process_source). +3) Uncommon and rare includes are filtered (remove_rare). +4) Conflicting headers are excluded (filter_ignore). +5) Local files to the source are excluded (Filter_Local). +6) Fixup missing headers that sources expect (fixup). +7) The resulting includes are sorted by category (sort_by_category). +8) The pch file is generated (generate). +""" + +import sys +import re +import os +import unittest +import glob + +CUTOFF = 1 +EXCLUDE_MODULE = False +EXCLUDE_LOCAL = False +EXCLUDE_SYSTEM = True +SILENT = False +WORKDIR = 'workdir' + +# System includes: oox, sal, sd, svl, vcl + +INCLUDE = False +EXCLUDE = True +DEFAULTS = \ +{ +# module.library : (min, system, module, local), best time + 'accessibility.acc' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 7.8 + 'basctl.basctl' : ( 3, EXCLUDE, INCLUDE, EXCLUDE), # 11.9 + 'basegfx.basegfx' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 3.8 + 'basic.sb' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 10.7 + 'chart2.chartcontroller' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 18.4 + 'chart2.chartcore' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 22.5 + 'comphelper.comphelper' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 7.6 + 'configmgr.configmgr' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 6.0 + 'connectivity.ado' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 6.4 + 'connectivity.calc' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 4.6 + 'connectivity.dbase' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # 5.2 + 'connectivity.dbpool2' : ( 5, EXCLUDE, INCLUDE, EXCLUDE), # 3.0 + 'connectivity.dbtools' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 0.8 + 'connectivity.file' : ( 2, EXCLUDE, INCLUDE, EXCLUDE), # 5.1 + 'connectivity.firebird_sdbc' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 5.1 + 'connectivity.flat' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # 4.6 + 'connectivity.mysql' : ( 4, EXCLUDE, INCLUDE, EXCLUDE), # 3.4 + 'connectivity.odbc' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 5.0 + 'connectivity.postgresql-sdbc-impl' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 6.7 + 'cppcanvas.cppcanvas' : (11, EXCLUDE, INCLUDE, INCLUDE), # 4.8 + 'cppuhelper.cppuhelper' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 4.6 + 'cui.cui' : ( 8, EXCLUDE, INCLUDE, EXCLUDE), # 19.7 + 'dbaccess.dba' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 13.8 + 'dbaccess.dbaxml' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 6.5 + 'dbaccess.dbu' : (12, EXCLUDE, EXCLUDE, EXCLUDE), # 23.6 + 'dbaccess.sdbt' : ( 1, EXCLUDE, INCLUDE, EXCLUDE), # 2.9 + 'desktop.deployment' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 6.1 + 'desktop.deploymentgui' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 5.7 + 'desktop.deploymentmisc' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 3.4 + 'desktop.sofficeapp' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 6.5 + 'drawinglayer.drawinglayer' : ( 4, EXCLUDE, EXCLUDE, EXCLUDE), # 7.4 + 'editeng.editeng' : ( 5, EXCLUDE, INCLUDE, EXCLUDE), # 13.0 + 'forms.frm' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 14.2 + 'framework.fwe' : (10, EXCLUDE, INCLUDE, EXCLUDE), # 5.5 + 'framework.fwi' : ( 9, EXCLUDE, INCLUDE, EXCLUDE), # 3.4 + 'framework.fwk' : ( 7, EXCLUDE, INCLUDE, INCLUDE), # 14.8 + 'framework.fwl' : ( 5, EXCLUDE, INCLUDE, INCLUDE), # 5.1 + 'hwpfilter.hwp' : ( 3, EXCLUDE, INCLUDE, INCLUDE), # 6.0 + 'lotuswordpro.lwpft' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 11.6 + 'oox.oox' : ( 6, EXCLUDE, EXCLUDE, INCLUDE), # 28.2 + 'package.package2' : ( 3, EXCLUDE, INCLUDE, INCLUDE), # 4.5 + 'package.xstor' : ( 2, EXCLUDE, INCLUDE, EXCLUDE), # 3.8 + 'reportdesign.rpt' : ( 9, EXCLUDE, INCLUDE, INCLUDE), # 9.4 + 'reportdesign.rptui' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 13.1 + 'reportdesign.rptxml' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 7.6 + 'sal.sal' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 4.2 + 'sc.sc' : (12, EXCLUDE, INCLUDE, INCLUDE), # 92.6 + 'sc.scfilt' : ( 4, EXCLUDE, EXCLUDE, INCLUDE), # 39.9 + 'sc.scui' : ( 1, EXCLUDE, EXCLUDE, INCLUDE), # 15.0 + 'sc.vbaobj' : ( 1, EXCLUDE, EXCLUDE, INCLUDE), # 17.3 + 'sd.sd' : ( 4, EXCLUDE, EXCLUDE, INCLUDE), # 47.4 + 'sd.sdui' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 9.4 + 'sdext.PresentationMinimizer' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # 4.1 + 'sdext.PresenterScreen' : ( 2, EXCLUDE, INCLUDE, EXCLUDE), # 7.1 + 'sfx2.sfx' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 27.4 + 'slideshow.slideshow' : ( 4, EXCLUDE, INCLUDE, EXCLUDE), # 10.8 + 'sot.sot' : ( 5, EXCLUDE, EXCLUDE, INCLUDE), # 3.1 + 'starmath.sm' : ( 5, EXCLUDE, EXCLUDE, INCLUDE), # 10.9 + 'svgio.svgio' : ( 8, EXCLUDE, EXCLUDE, INCLUDE), # 4.3 + 'emfio.emfio' : ( 8, EXCLUDE, EXCLUDE, INCLUDE), # 4.3 + 'svl.svl' : ( 6, EXCLUDE, EXCLUDE, EXCLUDE), # 7.6 + 'svtools.svt' : ( 4, EXCLUDE, INCLUDE, EXCLUDE), # 17.6 + 'svx.svx' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 20.7 + 'svx.svxcore' : ( 7, EXCLUDE, INCLUDE, EXCLUDE), # 37.0 + 'sw.msword' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 22.4 + 'sw.sw' : ( 7, EXCLUDE, EXCLUDE, INCLUDE), # 129.6 + 'sw.swui' : ( 3, EXCLUDE, INCLUDE, INCLUDE), # 26.1 + 'sw.vbaswobj' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 13.1 + 'tools.tl' : ( 5, EXCLUDE, EXCLUDE, EXCLUDE), # 4.2 + 'unotools.utl' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 7.0 + 'unoxml.unoxml' : ( 1, EXCLUDE, EXCLUDE, EXCLUDE), # 4.6 + 'uui.uui' : ( 4, EXCLUDE, EXCLUDE, EXCLUDE), # 4.9 + 'vbahelper.msforms' : ( 3, EXCLUDE, INCLUDE, INCLUDE), # 5.2 + 'vbahelper.vbahelper' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 7.0 + 'vcl.vcl' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 35.7 + 'writerfilter.writerfilter' : ( 5, EXCLUDE, EXCLUDE, EXCLUDE), # 19.7/27.3 + 'xmloff.xo' : ( 7, EXCLUDE, INCLUDE, INCLUDE), # 22.1 + 'xmloff.xof' : ( 1, EXCLUDE, EXCLUDE, INCLUDE), # 4.4 + 'xmlscript.xmlscript' : ( 4, EXCLUDE, EXCLUDE, INCLUDE), # 3.6 + 'xmlsecurity.xmlsecurity' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 5.1 + 'xmlsecurity.xsec_xmlsec' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # 4.4 + 'xmlsecurity.xsec_gpg' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # ? +} + +def remove_rare(raw, min_use=-1): + """ Remove headers not commonly included. + The minimum threshold is min_use. + """ + # The minimum number of times a header + # must be included to be in the PCH. + min_use = min_use if min_use >= 0 else CUTOFF + + out = [] + if not raw or not len(raw): + return out + + inc = sorted(raw) + last = inc[0] + count = 1 + for x in range(1, len(inc)): + i = inc[x] + if i == last: + count += 1 + else: + if count >= min_use: + out.append(last) + last = i + count = 1 + + # Last group. + if count >= min_use: + out.append(last) + + return out + +def process_list(list, callable): + """ Given a list and callable + we pass each entry through + the callable and only add to + the output if not blank. + """ + out = [] + for i in list: + line = callable(i) + if line and len(line): + out.append(line) + return out + +def find_files(path, recurse=True): + list = [] + for root, dir, files in os.walk(path): + list += map(lambda x: os.path.join(root, x), files) + return list + +def get_filename(line): + """ Strips the line from the + '#include' and angled brakets + and return the filename only. + """ + if not len(line) or line[0] != '#': + return line + return re.sub(r'(.*#include\s*)<(.*)>(.*)', r'\2', line) + +def is_c_runtime(inc, root, module): + """ Heuristic-based detection of C/C++ + runtime headers. + They are all-lowercase, with .h or + no extension, filename only. + Try to check that they are not LO headers. + """ + inc = get_filename(inc) + + if inc.endswith('.hxx') or inc.endswith('.hpp'): + return False + + if inc.endswith('.h') and inc.startswith( 'config_' ): + return False + + hasdot = False + for c in inc: + if c == '/': + return False + if c == '.' and not inc.endswith('.h'): + return False + if c == '.': + hasdot = True + if c.isupper(): + return False + if not hasdot: # <memory> etc. + return True + + if glob.glob(os.path.join(root, module, '**', inc), recursive=True): + return False; + + return True + +def sanitize(raw): + """ There are two forms of includes, + those with <> and "". + Technically, the difference is that + the compiler can use an internal + representation for an angled include, + such that it doesn't have to be a file. + For our purposes, there is no difference. + Here, we convert everything to angled. + """ + if not raw or not len(raw): + return '' + raw = raw.strip() + if not len(raw): + return '' + return re.sub(r'(.*#include\s*)\"(.*)\"(.*)', r'#include <\2>', raw) + +class Filter_Local(object): + """ Filter headers local to a module. + allow_public: allows include/module/file.hxx + #include <module/file.hxx> + allow_module: allows module/inc/file.hxx + #include <file.hxx> + allow_locals: allows module/source/file.hxx and + module/source/inc/file.hxx + #include <file.hxx> + """ + def __init__(self, root, module, allow_public=True, allow_module=True, allow_locals=True): + self.root = root + self.module = module + self.allow_public = allow_public + self.allow_module = allow_module + self.allow_locals = allow_locals + self.public_prefix = '<' + self.module + '/' + + all = find_files(os.path.join(root, module)) + self.module_includes = [] + self.locals = [] + mod_prefix = module + '/inc/' + for i in all: + if mod_prefix in i: + self.module_includes.append(i) + else: + self.locals.append(i) + + def is_public(self, line): + return self.public_prefix in line + + def is_module(self, line): + """ Returns True if in module/inc/... """ + filename = get_filename(line) + for i in self.module_includes: + if i.endswith(filename): + return True + return False + + def is_local(self, line): + """ Returns True if in module/source/... """ + filename = get_filename(line) + for i in self.locals: + if i.endswith(filename): + return True + return False + + def is_external(self, line): + return is_c_runtime(line, self.root, self.module) and \ + not self.is_public(line) and \ + not self.is_module(line) and \ + not self.is_local(line) + + def find_local_file(self, line): + """ Finds the header file in the module dir, + but doesn't validate. + """ + filename = get_filename(line) + for i in self.locals: + if i.endswith(filename): + return i + for i in self.module_includes: + if i.endswith(filename): + return i + return None + + def proc(self, line): + assert line and len(line) + + if line[0] == '#': + if not SILENT: + sys.stderr.write('unhandled #include : {}\n'.format(line)) + return '' + + assert line[0] != '<' and line[0] != '#' + + filename = get_filename(line) + + # Local with relative path. + if filename.startswith('..'): + # Exclude for now as we don't have cxx path. + return '' + + # Locals are included first (by the compiler). + if self.is_local(filename): + # Use only locals that are in some /inc/ directory (either in <module>/inc or + # somewhere under <module>/source/**/inc/, compilations use -I for these paths + # and headers elsewhere would not be found when compiling the PCH. + if not self.allow_locals: + return '' + elif '/inc/' in filename: + return filename + elif glob.glob(os.path.join(self.root, self.module, '**', 'inc', filename), recursive=True): + return filename + else: + return '' + + # Module headers are next. + if self.is_module(filename): + return line if self.allow_module else '' + + # Public headers are last. + if self.is_public(line): + return line if self.allow_public else '' + + # Leave out potentially unrelated files local + # to some other module we can't include directly. + if '/' not in filename and not self.is_external(filename): + return '' + + # Unfiltered. + return line + +def filter_ignore(line, module): + """ Filters includes from known + problematic ones. + Expects sanitized input. + """ + assert line and len(line) + + # Always include files without extension. + if '.' not in line: + return line + + # Extract filenames for ease of comparison. + line = get_filename(line) + + # Filter out all files that are not normal headers. + if not line.endswith('.h') and \ + not line.endswith('.hxx') and \ + not line.endswith('.hpp') and \ + not line.endswith('.hdl'): + return '' + + ignore_list = [ + 'LibreOfficeKit/LibreOfficeKitEnums.h', # Needs special directives + 'LibreOfficeKit/LibreOfficeKitTypes.h', # Needs special directives + 'jerror.h', # c++ unfriendly + 'jpeglib.h', # c++ unfriendly + 'boost/spirit/include/classic_core.hpp', # depends on BOOST_SPIRIT_DEBUG + 'svtools/editimplementation.hxx' # no direct include + ] + + if module == 'accessibility': + ignore_list += [ + # STR_SVT_ACC_LISTENTRY_SELCTED_STATE redefined from svtools.hrc + 'accessibility/extended/textwindowaccessibility.hxx', + ] + if module == 'basic': + ignore_list += [ + 'basic/vbahelper.hxx', + ] + if module == 'connectivity': + ignore_list += [ + 'com/sun/star/beans/PropertyAttribute.hpp', # OPTIONAL defined via objbase.h + 'com/sun/star/sdbcx/Privilege.hpp', # DELETE defined via objbase.h + 'ado/*' , # some strange type conflict because of Window's adoctint.h + 'adoint.h', + 'adoctint.h', + ] + if module == 'sc': + ignore_list += [ + 'progress.hxx', # special directives + 'scslots.hxx', # special directives + ] + if module == 'sd': + ignore_list += [ + 'sdgslots.hxx', # special directives + 'sdslots.hxx', # special directives + ] + if module == 'sfx2': + ignore_list += [ + 'sfx2/recentdocsview.hxx', # Redefines ApplicationType defined in objidl.h + 'sfx2/sidebar/Sidebar.hxx', + 'sfx2/sidebar/UnoSidebar.hxx', + 'sfxslots.hxx', # externally defined types + ] + if module == 'sot': + ignore_list += [ + 'sysformats.hxx', # Windows headers + ] + if module == 'vcl': + ignore_list += [ + 'accmgr.hxx', # redefines ImplAccelList + 'image.h', + 'jobset.h', + 'opengl/gdiimpl.hxx', + 'opengl/salbmp.hxx', + 'openglgdiimpl', # ReplaceTextA + 'printdlg.hxx', + 'salinst.hxx', # GetDefaultPrinterA + 'salprn.hxx', # SetPrinterDataA + 'vcl/jobset.hxx', + 'vcl/oldprintadaptor.hxx', + 'vcl/opengl/OpenGLContext.hxx', + 'vcl/opengl/OpenGLHelper.hxx', # Conflicts with X header on *ix + 'vcl/print.hxx', + 'vcl/prntypes.hxx', # redefines Orientation from filter/jpeg/Exif.hxx + 'vcl/sysdata.hxx', + ] + if module == 'xmloff': + ignore_list += [ + 'SchXMLExport.hxx', # SchXMLAutoStylePoolP.hxx not found + 'SchXMLImport.hxx', # enums redefined in draw\sdxmlimp_impl.hxx + 'XMLEventImportHelper.hxx', # NameMap redefined in XMLEventExport.hxx + 'xmloff/XMLEventExport.hxx', # enums redefined + ] + if module == 'xmlsecurity': + ignore_list += [ + 'xmlsec/*', + 'xmlsecurity/xmlsec-wrapper.h', + ] + if module == 'external/pdfium': + ignore_list += [ + 'third_party/freetype/include/pstables.h', + ] + if module == 'external/clucene': + ignore_list += [ + '_bufferedstream.h', + '_condition.h', + '_gunichartables.h', + '_threads.h', + 'error.h', + 'CLucene/LuceneThreads.h', + 'CLucene/config/_threads.h', + ] + if module == 'external/skia': + ignore_list += [ + 'skcms_internal.h', + 'zlib.h', # causes crc32 conflict + 'dirent.h', # unix-specific + 'pthread.h', + 'unistd.h', + 'sys/stat.h', + 'ft2build.h', + 'fontconfig/fontconfig.h', + 'GL/glx.h', + 'src/Transform_inl.h', + 'src/c/sk_c_from_to.h', + 'src/c/sk_types_priv.h', + 'src/core/SkBlitBWMaskTemplate.h', + 'src/sfnt/SkSFNTHeader.h', + 'src/opts/', + 'src/core/SkCubicSolver.h', + 'src/sksl/SkSLCPP.h', + 'src/gpu/vk/GrVkAMDMemoryAllocator.h', + 'src/gpu/GrUtil.h', + ] + + for i in ignore_list: + if line.startswith(i): + return '' + if i[0] == '*' and line.endswith(i[1:]): + return '' + if i[-1] == '*' and line.startswith(i[:-1]): + return '' + + return line + +def fixup(includes, module): + """ Here we add any headers + necessary in the pch. + These could be known to be very + common but for technical reasons + left out of the pch by this generator. + Or, they could be missing from the + source files where they are used + (probably because they had been + in the old pch, they were missed). + Also, these could be headers + that make the build faster but + aren't added automatically. + """ + fixes = [] + def append(inc): + # Add a space to exclude from + # ignore bisecting. + line = ' #include <{}>'.format(inc) + try: + i = fixes.index(inc) + fixes[i] = inc + except: + fixes.append(inc) + + if module == 'basctl': + if 'basslots.hxx' in includes: + append('sfx2/msg.hxx') + + #if module == 'sc': + # if 'scslots.hxx' in includes: + # append('sfx2/msg.hxx') + return fixes + +def sort_by_category(list, root, module, filter_local): + """ Move all 'system' headers first. + Core files of osl, rtl, sal, next. + Everything non-module-specific third. + Last, module-specific headers. + """ + sys = [] + boo = [] + cor = [] + rst = [] + mod = [] + + prefix = '<' + module + '/' + for i in list: + if is_c_runtime(i, root, module): + sys.append(i) + elif '<boost/' in i: + boo.append(i) + elif prefix in i or not '/' in i: + mod.append(i) + elif '<sal/' in i or '<vcl/' in i: + cor.append(i) + elif '<osl/' in i or '<rtl/' in i: + if module == "sal": # osl and rtl are also part of sal + mod.append(i) + else: + cor.append(i) + # Headers from another module that is closely tied to the module. + elif module == 'sc' and '<formula' in i: + mod.append(i) + else: + rst.append(i) + + out = [] + out += [ "#if PCH_LEVEL >= 1" ] + out += sorted(sys) + out += sorted(boo) + out += [ "#endif // PCH_LEVEL >= 1" ] + out += [ "#if PCH_LEVEL >= 2" ] + out += sorted(cor) + out += [ "#endif // PCH_LEVEL >= 2" ] + out += [ "#if PCH_LEVEL >= 3" ] + out += sorted(rst) + out += [ "#endif // PCH_LEVEL >= 3" ] + out += [ "#if PCH_LEVEL >= 4" ] + out += sorted(mod) + out += [ "#endif // PCH_LEVEL >= 4" ] + return out + +def parse_makefile(groups, lines, lineno, lastif, ifstack): + + inobjects = False + ingeneratedobjects = False + inelse = False + suffix = 'cxx' + os_cond_re = re.compile('(ifeq|ifneq)\s*\(\$\(OS\)\,(\w*)\)') + + line = lines[lineno] + if line.startswith('if'): + lastif = line + if ifstack == 0: + # Correction if first line is an if. + lineno = parse_makefile(groups, lines, lineno, line, ifstack+1) + else: + lineno -= 1 + + while lineno + 1 < len(lines): + lineno += 1 + line = lines[lineno].strip() + line = line.rstrip('\\').strip() + #print('line #{}: {}'.format(lineno, line)) + if len(line) == 0: + continue + + if line == '))': + inobjects = False + ingeneratedobjects = False + elif 'add_exception_objects' in line or \ + 'add_cxxobject' in line: + inobjects = True + #print('inobjects') + #if ifstack and not SILENT: + #sys.stderr.write('Sources in a conditional, ignoring for now.\n') + elif 'add_generated_exception_objects' in line or \ + 'add_generated_cxxobject' in line: + ingeneratedobjects = True + elif 'set_generated_cxx_suffix' in line: + suffix_re = re.compile('.*set_generated_cxx_suffix,[^,]*,([^)]*).*') + match = suffix_re.match(line) + if match: + suffix = match.group(1) + elif line.startswith('if'): + lineno = parse_makefile(groups, lines, lineno, line, ifstack+1) + continue + elif line.startswith('endif'): + if ifstack: + return lineno + continue + elif line.startswith('else'): + inelse = True + elif inobjects or ingeneratedobjects: + if EXCLUDE_SYSTEM and ifstack: + continue + file = line + '.' + suffix + if ',' in line or '(' in line or ')' in line or file.startswith('-'): + #print('passing: ' + line) + pass # $if() probably, or something similar + else: + osname = '' + if lastif: + if 'filter' in lastif: + # We can't grok filter, yet. + continue + match = os_cond_re.match(lastif) + if not match: + # We only support OS conditionals. + continue + in_out = match.group(1) + osname = match.group(2) if match else '' + if (in_out == 'ifneq' and not inelse) or \ + (in_out == 'ifeq' and inelse): + osname = '!' + osname + + if osname not in groups: + groups[osname] = [] + if ingeneratedobjects: + file = WORKDIR + '/' + file + groups[osname].append(file) + + return groups + +def process_makefile(root, module, libname): + """ Parse a gmake makefile and extract + source filenames from it. + """ + + makefile = 'Library_{}.mk'.format(libname) + filename = os.path.join(os.path.join(root, module), makefile) + if not os.path.isfile(filename): + makefile = 'StaticLibrary_{}.mk'.format(libname) + filename = os.path.join(os.path.join(root, module), makefile) + if not os.path.isfile(filename): + sys.stderr.write('Error: Module {} has no makefile at {}.'.format(module, filename)) + + groups = {'':[], 'ANDROID':[], 'iOS':[], 'WNT':[], 'LINUX':[], 'MACOSX':[]} + + with open(filename, 'r') as f: + lines = f.readlines() + groups = parse_makefile(groups, lines, lineno=0, lastif=None, ifstack=0) + + return groups + +def is_allowed_if(line, module): + """ Check whether the given #if condition + is allowed for the given module or whether + its block should be ignored. + """ + + # remove trailing comments + line = re.sub(r'(.*) *//.*', r'\1', line) + line = line.strip() + + # Our sources always build with LIBO_INTERNAL_ONLY. + if line == "#if defined LIBO_INTERNAL_ONLY" or line == "#ifdef LIBO_INTERNAL_ONLY": + return True + if module == "external/skia": + # We always set these. + if line == "#ifdef SK_VULKAN" or line == "#if SK_SUPPORT_GPU": + return True + return False + +def process_source(root, module, filename, maxdepth=0): + """ Process a source file to extract + included headers. + For now, skip on compiler directives. + maxdepth is used when processing headers + which typically have protecting ifndef. + """ + + ifdepth = 0 + lastif = '' + raw_includes = [] + allowed_ifs = [] + ifsallowed = 0 + with open(filename, 'r') as f: + for line in f: + line = line.strip() + if line.startswith('#if'): + if is_allowed_if(line, module): + allowed_ifs.append(True) + ifsallowed += 1 + else: + allowed_ifs.append(False) + lastif = line + ifdepth += 1 + elif line.startswith('#endif'): + ifdepth -= 1 + if allowed_ifs[ ifdepth ]: + ifsallowed -= 1 + else: + lastif = '#if' + del allowed_ifs[ ifdepth ] + elif line.startswith('#include'): + if ifdepth - ifsallowed <= maxdepth: + line = sanitize(line) + if line: + line = get_filename(line) + if line and len(line): + raw_includes.append(line) + elif not SILENT: + sys.stderr.write('#include in {} : {}\n'.format(lastif, line)) + + return raw_includes + +def explode(root, module, includes, tree, filter_local, recurse): + incpath = os.path.join(root, 'include') + + for inc in includes: + filename = get_filename(inc) + if filename in tree or len(filter_local.proc(filename)) == 0: + continue + + try: + # Module or Local header. + filepath = filter_local.find_local_file(inc) + if filepath: + #print('trying loc: ' + filepath) + incs = process_source(root, module, filepath, maxdepth=1) + incs = map(get_filename, incs) + incs = process_list(incs, lambda x: filter_ignore(x, module)) + incs = process_list(incs, filter_local.proc) + tree[filename] = incs + if recurse: + tree = explode(root, module, incs, tree, filter_local, recurse) + #print('{} => {}'.format(filepath, tree[filename])) + continue + except: + pass + + try: + # Public header. + filepath = os.path.join(incpath, filename) + #print('trying pub: ' + filepath) + incs = process_source(root, module, filepath, maxdepth=1) + incs = map(get_filename, incs) + incs = process_list(incs, lambda x: filter_ignore(x, module)) + incs = process_list(incs, filter_local.proc) + tree[filename] = incs + if recurse: + tree = explode(root, module, incs, tree, filter_local, recurse) + #print('{} => {}'.format(filepath, tree[filename])) + continue + except: + pass + + # Failed, but remember to avoid searching again. + tree[filename] = [] + + return tree + +def make_command_line(): + args = sys.argv[:] + # Remove command line flags and + # use internal flags. + for i in range(len(args)-1, 0, -1): + if args[i].startswith('--'): + args.pop(i) + + args.append('--cutoff=' + str(CUTOFF)) + if EXCLUDE_SYSTEM: + args.append('--exclude:system') + else: + args.append('--include:system') + if EXCLUDE_MODULE: + args.append('--exclude:module') + else: + args.append('--include:module') + if EXCLUDE_LOCAL: + args.append('--exclude:local') + else: + args.append('--include:local') + + return ' '.join(args) + +def generate_includes(includes): + """Generates the include lines of the pch. + """ + lines = [] + for osname, group in includes.items(): + if not len(group): + continue + + if len(osname): + not_eq = '' + if osname[0] == '!': + not_eq = '!' + osname = osname[1:] + lines.append('') + lines.append('#if {}defined({})'.format(not_eq, osname)) + + for i in group: + lines.append(i) + + if len(osname): + lines.append('#endif') + + return lines + +def generate(includes, libname, filename, module): + header = \ +"""/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* + This file has been autogenerated by update_pch.sh. It is possible to edit it + manually (such as when an include file has been moved/renamed/removed). All such + manual changes will be rewritten by the next run of update_pch.sh (which presumably + also fixes all possible problems, so it's usually better to use it). +""" + + footer = \ +""" +/* vim:set shiftwidth=4 softtabstop=4 expandtab: */ +""" + import datetime + + with open(filename, 'w') as f: + f.write(header) + f.write('\n Generated on {} using:\n {}\n'.format( + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + make_command_line())) + f.write('\n If after updating build fails, use the following command to locate conflicting headers:\n ./bin/update_pch_bisect {} "make {}.build" --find-conflicts\n*/\n'.format( + filename, module)) + + # sal needs this for rand_s() + if module == 'sal' and libname == 'sal': + sal_define = """ +#if defined(_WIN32) +#if !defined _CRT_RAND_S +#define _CRT_RAND_S +#endif +#endif +""" + f.write(sal_define) + + # Dump the headers. + f.write('\n') + for i in includes: + f.write(i + '\n') + + # Some libraries pull windows headers that aren't self contained. + if (module == 'connectivity' and libname == 'ado') or \ + (module == 'xmlsecurity' and libname == 'xsec_xmlsec'): + ado_define = """ +// Cleanup windows header macro pollution. +#if defined(_WIN32) && defined(WINAPI) +#include <postwin.h> +#undef RGB +#endif +""" + f.write(ado_define) + + f.write(footer) + +def remove_from_tree(filename, tree): + # Remove this file, if top-level. + incs = tree.pop(filename, []) + for i in incs: + tree = remove_from_tree(i, tree) + + # Also remove if included from another. + for (k, v) in tree.items(): + if filename in v: + v.remove(filename) + + return tree + +def tree_to_list(includes, filename, tree): + if filename in includes: + return includes + includes.append(filename) + #incs = tree.pop(filename, []) + incs = tree[filename] if filename in tree else [] + for i in incs: + tree_to_list(includes, i, tree) + + return includes + +def promote(includes): + """ Common library headers are heavily + referenced, even if they are included + from a few places. + Here we separate them to promote + their inclusion in the final pch. + """ + promo = [] + for inc in includes: + if inc.startswith('boost') or \ + inc.startswith('sal') or \ + inc.startswith('osl') or \ + inc.startswith('rtl'): + promo.append(inc) + return promo + +def make_pch_filename(root, module, libname): + """ PCH files are stored here: + <root>/<module>/inc/pch/precompiled_<libname>.hxx + """ + + path = os.path.join(root, module) + path = os.path.join(path, 'inc') + path = os.path.join(path, 'pch') + path = os.path.join(path, 'precompiled_' + libname + '.hxx') + return path + +def main(): + + global CUTOFF + global EXCLUDE_MODULE + global EXCLUDE_LOCAL + global EXCLUDE_SYSTEM + global SILENT + global WORKDIR + + if os.getenv('WORKDIR'): + WORKDIR = os.getenv('WORKDIR') + + root = '.' + module = sys.argv[1] + libname = sys.argv[2] + header = make_pch_filename(root, module, libname) + + if not os.path.exists(os.path.join(root, module)): + raise Exception('Error: module [{}] not found.'.format(module)) + + key = '{}.{}'.format(module, libname) + if key in DEFAULTS: + # Load the module-specific defaults. + CUTOFF = DEFAULTS[key][0] + EXCLUDE_SYSTEM = DEFAULTS[key][1] + EXCLUDE_MODULE = DEFAULTS[key][2] + EXCLUDE_LOCAL = DEFAULTS[key][3] + + force_update = False + for x in range(3, len(sys.argv)): + i = sys.argv[x] + if i.startswith('--cutoff='): + CUTOFF = int(i.split('=')[1]) + elif i.startswith('--exclude:'): + cat = i.split(':')[1] + if cat == 'module': + EXCLUDE_MODULE = True + elif cat == 'local': + EXCLUDE_LOCAL = True + elif cat == 'system': + EXCLUDE_SYSTEM = True + elif i.startswith('--include:'): + cat = i.split(':')[1] + if cat == 'module': + EXCLUDE_MODULE = False + elif cat == 'local': + EXCLUDE_LOCAL = False + elif cat == 'system': + EXCLUDE_SYSTEM = False + elif i == '--silent': + SILENT = True + elif i == '--force': + force_update = True + else: + sys.stderr.write('Unknown option [{}].'.format(i)) + return 1 + + filter_local = Filter_Local(root, module, \ + not EXCLUDE_MODULE, \ + not EXCLUDE_LOCAL) + + # Read input. + groups = process_makefile(root, module, libname) + + generic = [] + for osname, group in groups.items(): + if not len(group): + continue + + includes = [] + for filename in group: + includes += process_source(root, module, filename) + + # Save unique top-level includes. + unique = set(includes) + promoted = promote(unique) + + # Process includes. + includes = remove_rare(includes) + includes = process_list(includes, lambda x: filter_ignore(x, module)) + includes = process_list(includes, filter_local.proc) + + # Remove the already included ones. + for inc in includes: + unique.discard(inc) + + # Explode the excluded ones. + tree = {i:[] for i in includes} + tree = explode(root, module, unique, tree, filter_local, not EXCLUDE_MODULE) + + # Remove the already included ones from the tree. + for inc in includes: + filename = get_filename(inc) + tree = remove_from_tree(filename, tree) + + extra = [] + for (k, v) in tree.items(): + extra += tree_to_list([], k, tree) + + promoted += promote(extra) + promoted = process_list(promoted, lambda x: filter_ignore(x, module)) + promoted = process_list(promoted, filter_local.proc) + promoted = set(promoted) + # If a promoted header includes others, remove the rest. + for (k, v) in tree.items(): + if k in promoted: + for i in v: + promoted.discard(i) + includes += [x for x in promoted] + + extra = remove_rare(extra) + extra = process_list(extra, lambda x: filter_ignore(x, module)) + extra = process_list(extra, filter_local.proc) + includes += extra + + includes = [x for x in set(includes)] + fixes = fixup(includes, module) + fixes = map(lambda x: '#include <' + x + '>', fixes) + + includes = map(lambda x: '#include <' + x + '>', includes) + sorted = sort_by_category(includes, root, module, filter_local) + includes = list(fixes) + sorted + + if len(osname): + for i in generic: + if i in includes: + includes.remove(i) + + groups[osname] = includes + if not len(osname): + generic = includes + + # Open the old pch and compare its contents + # with new includes. + # Clobber only if they are different. + with open(header, 'r') as f: + old_pch_lines = [x.strip() for x in f.readlines()] + new_lines = generate_includes(groups) + # Find the first include in the old pch. + start = -1 + for i in range(len(old_pch_lines)): + if old_pch_lines[i].startswith('#include') or old_pch_lines[i].startswith('#if PCH_LEVEL'): + start = i + break + # Clobber if there is a mismatch. + if force_update or start < 0 or (len(old_pch_lines) - start < len(new_lines)): + generate(new_lines, libname, header, module) + return 0 + else: + for i in range(len(new_lines)): + if new_lines[i] != old_pch_lines[start + i]: + generate(new_lines, libname, header, module) + return 0 + else: + # Identical, but see if new pch removed anything. + for i in range(start + len(new_lines), len(old_pch_lines)): + if '#include' in old_pch_lines[i]: + generate(new_lines, libname, header, module) + return 0 + + # Didn't update. + return 1 + +if __name__ == '__main__': + """ Process all the includes in a Module + to make into a PCH file. + Run without arguments for unittests, + and to see usage. + """ + + if len(sys.argv) >= 3: + status = main() + sys.exit(status) + + print('Usage: {} <Module name> <Library name> [options]'.format(sys.argv[0])) + print(' Always run from the root of LO repository.\n') + print(' Options:') + print(' --cutoff=<count> - Threshold to excluding headers.') + print(' --exclude:<category> - Exclude category-specific headers.') + print(' --include:<category> - Include category-specific headers.') + print(' --force - Force updating the pch even when nothing changes.') + print(' Categories:') + print(' module - Headers in /inc directory of a module.') + print(' local - Headers local to a source file.') + print(' system - Platform-specific headers.') + print(' --silent - print only errors.') + print('\nRunning unit-tests...') + + +class TestMethods(unittest.TestCase): + + def test_sanitize(self): + self.assertEqual(sanitize('#include "blah/file.cxx"'), + '#include <blah/file.cxx>') + self.assertEqual(sanitize(' #include\t"blah/file.cxx" '), + '#include <blah/file.cxx>') + self.assertEqual(sanitize(' '), + '') + + def test_filter_ignore(self): + self.assertEqual(filter_ignore('blah/file.cxx', 'mod'), + '') + self.assertEqual(filter_ignore('vector', 'mod'), + 'vector') + self.assertEqual(filter_ignore('file.cxx', 'mod'), + '') + + def test_remove_rare(self): + self.assertEqual(remove_rare([]), + []) + +class TestMakefileParser(unittest.TestCase): + + def setUp(self): + global EXCLUDE_SYSTEM + EXCLUDE_SYSTEM = False + + def test_parse_singleline_eval(self): + source = "$(eval $(call gb_Library_Library,sal))" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + + def test_parse_multiline_eval(self): + source = """$(eval $(call gb_Library_set_include,sal,\\ + $$(INCLUDE) \\ + -I$(SRCDIR)/sal/inc \\ +)) +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + + def test_parse_multiline_eval_with_if(self): + source = """$(eval $(call gb_Library_add_defs,sal,\\ + $(if $(filter $(OS),iOS), \\ + -DNO_CHILD_PROCESSES \\ + ) \\ +)) +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + + def test_parse_multiline_add_with_if(self): + source = """$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/time \\ + $(if $(filter DESKTOP,$(BUILD_TYPE)), sal/osl/unx/salinit) \\ +)) +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 1) + self.assertEqual(groups[''][0], 'sal/osl/unx/time.cxx') + + def test_parse_if_else(self): + source = """ifeq ($(OS),MACOSX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/mac/mac \\ +)) +else +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/uunxapi \\ +)) +endif +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 3) + self.assertEqual(len(groups['']), 0) + self.assertEqual(len(groups['MACOSX']), 1) + self.assertEqual(len(groups['!MACOSX']), 1) + self.assertEqual(groups['MACOSX'][0], 'sal/osl/mac/mac.cxx') + self.assertEqual(groups['!MACOSX'][0], 'sal/osl/unx/uunxapi.cxx') + + def test_parse_nested_if(self): + source = """ifeq ($(OS),MACOSX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/mac/mac \\ +)) +else +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/uunxapi \\ +)) + +ifeq ($(OS),LINUX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/textenc/context \\ +)) +endif +endif +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 4) + self.assertEqual(len(groups['']), 0) + self.assertEqual(len(groups['MACOSX']), 1) + self.assertEqual(len(groups['!MACOSX']), 1) + self.assertEqual(len(groups['LINUX']), 1) + self.assertEqual(groups['MACOSX'][0], 'sal/osl/mac/mac.cxx') + self.assertEqual(groups['!MACOSX'][0], 'sal/osl/unx/uunxapi.cxx') + self.assertEqual(groups['LINUX'][0], 'sal/textenc/context.cxx') + + def test_parse_exclude_system(self): + source = """ifeq ($(OS),MACOSX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/mac/mac \\ +)) +else +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/uunxapi \\ +)) + +ifeq ($(OS),LINUX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/textenc/context \\ +)) +endif +endif +""" + global EXCLUDE_SYSTEM + EXCLUDE_SYSTEM = True + + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + + def test_parse_filter(self): + source = """ifneq ($(filter $(OS),MACOSX iOS),) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/osxlocale \\ +)) +endif +""" + # Filter is still unsupported. + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + +unittest.main() + +# vim: set et sw=4 ts=4 expandtab: diff --git a/bin/update_pch.sh b/bin/update_pch.sh new file mode 100755 index 000000000..78b4a47e6 --- /dev/null +++ b/bin/update_pch.sh @@ -0,0 +1,65 @@ +#! /bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Usage: update_pch.sh [<module>/inc/pch/precompiled_xxx.hxx] +# Usage: update_pch.sh [<module>] +# Invoke: make cmd cmd="./bin/update_pch.sh [..]" + +if test -n "$SRC_DIR"; then + root="$SRC_DIR" +else + root=`dirname $0` + root=`cd $root/.. >/dev/null && pwd` +fi +root=`readlink -f $root` +cd $root + +if test -z "$1"; then + headers=`ls ./*/inc/pch/precompiled_*.hxx` +else + headers="$@" +fi + +# Split the headers into an array. +IFS=' ' read -a aheaders <<< $headers +hlen=${#aheaders[@]}; +if [ $hlen -gt 1 ]; then + if [ -z "$PARALLELISM" ]; then + PARALLELISM=0 # Let xargs decide + fi + echo $headers | xargs -n 1 -P $PARALLELISM $0 + exit $? +fi + +for x in $headers; do + if [ -d "$x" ]; then + # We got a directory, find pch files to update. + headers=`find $root/$x/ -type f -iname "precompiled_*.hxx"` + if test -n "$headers"; then + $0 "$headers" + fi + else + header=$x + update_msg=`echo $header | sed -e s%$root/%%` + module=`readlink -f $header | sed -e s%$root/%% -e s%/.*%%` + if [ "$module" = "pch" ]; then + continue # PCH's in pch/inc/pch/ are handled manually + fi + echo updating $update_msg + if [ "$module" = "external" ]; then + module=external/`readlink -f $header | sed -e s%$root/external/%% -e s%/.*%%` + fi + libname=`echo $header | sed -e s/.*precompiled_// -e s/\.hxx//` + + ./bin/update_pch "$module" "$libname" + fi +done + +#echo Done. +exit 0 diff --git a/bin/update_pch_autotune.sh b/bin/update_pch_autotune.sh new file mode 100755 index 000000000..ab9b0a688 --- /dev/null +++ b/bin/update_pch_autotune.sh @@ -0,0 +1,229 @@ +#! /bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Finds the optimal update_pch settings that results in, +# per module and library, the fastest build time and +# smallest intermediate files (.o/.obj) output. + +# Usage: update_pch_autotune.sh [<module1> <module2>] +# Invoke: /opt/lo/bin/make cmd cmd="./bin/update_pch_autotune.sh [..]" + +# The resulting values may be entered in update_pch +# to be use for generating PCH in the future. +# Run this script after major header changes. + +root=`dirname $0` +root=`cd $root/.. && pwd` +cd $root + +if test -z "$1"; then + modules=`ls ./*/inc/pch/precompiled_*.hxx | sed -e s%./%% -e s%/.*%% | uniq` +else + modules="$@" +fi + +if [[ "$OSTYPE" == "cygwin" ]]; then + MAKE=/opt/lo/bin/make +else + MAKE=make +fi + +function build() +{ + local START=$(date +%s.%N) + + $MAKE -sr "$module" > /dev/null + status=$? + if [ $status -ne 0 ]; + then + # Spurious failures happen. + $MAKE "$module.build" > /dev/null + status=$? + fi + + local END=$(date +%s.%N1) + build_time=$(printf %.1f $(echo "$END - $START" | bc)) + + size="FAILED" + score="FAILED" + if [ $status -eq 0 ]; + then + # The total size of the object files. + size="$(du -s workdir/CxxObject/$module/ | awk '{print $1}')" + # Add the pch file size. + filename_rel="workdir/PrecompiledHeader/nodebug/$(basename $header)*" + filename_dbg="workdir/PrecompiledHeader/debug/$(basename $header)*" + if [[ $filename_rel -nt $filename_dbg ]]; then + pch_size="$(du -s $filename_rel | awk '{print $1}' | paste -sd+ | bc)" + else + pch_size="$(du -s $filename_dbg | awk '{print $1}' | paste -sd+ | bc)" + fi + size="$(echo "$pch_size + $size" | bc)" + + # Compute a score based on the build time and size. + # The shorter the build time, and smaller disk usage, the higher the score. + score=$(printf %.2f $(echo "10000 / ($build_time * e($size/1048576))" | bc -l)) + fi +} + +function run() +{ + local msg="$module.$libname, ${@:3}, " + printf "$msg" + ./bin/update_pch "$module" "$libname" "${@:3}" --silent + status=$? + + if [ $status -eq 0 ]; + then + build + + summary="$build_time, $size, $score" + if [ $status -eq 0 ]; + then + new_best_for_cuttof=$(echo "$score > $best_score_for_cuttof" | bc -l) + if [ $new_best_for_cuttof -eq 1 ]; + then + best_score_for_cuttof=$score + fi + + new_best=$(echo "$score > $best_score" | bc -l) + if [ $new_best -eq 1 ]; + then + best_score=$score + best_args="${@:3}" + best_time=$build_time + best_cutoff=$cutoff + summary="$build_time, $size, $score,*" + fi + fi + else + # Skip if pch is not updated. + summary="0, 0, 0" + fi + + echo "$summary" +} + +function args_to_table() +{ + local sys="EXCLUDE" + local mod="EXCLUDE" + local loc="EXCLUDE" + local cutoff=0 + IFS=' ' read -r -a aargs <<< $best_args + for index in "${!aargs[@]}" + do + if [ "${aargs[index]}" = "--include:system" ]; + then + sys="INCLUDE" + elif [ "${aargs[index]}" = "--exclude:system" ]; + then + sys="EXCLUDE" + elif [ "${aargs[index]}" = "--include:module" ]; + then + mod="INCLUDE" + elif [ "${aargs[index]}" = "--exclude:module" ]; + then + mod="EXCLUDE" + elif [ "${aargs[index]}" = "--include:local" ]; + then + loc="INCLUDE" + elif [ "${aargs[index]}" = "--exclude:local" ]; + then + loc="EXCLUDE" + elif [[ "${aargs[index]}" == *"cutoff"* ]] + then + cutoff=$(echo "${aargs[index]}" | grep -Po '\-\-cutoff\=\K\d+') + fi + done + + local key=$(printf "'%s.%s'" $module $libname) + echo "$(printf " %-36s: (%2d, %s, %s, %s), # %5.1f" $key $cutoff $sys $mod $loc $best_time)" +} + +for module in $modules; do + + # Build without pch includes as sanity check. + #run "$root" "$module" --cutoff=999 + + # Build before updating pch. + $MAKE "$module.build" > /dev/null + if [ $? -ne 0 ]; + then + # Build with dependencies before updating pch. + echo "Failed to build $module, building known state with dependencies..." + ./bin/update_pch.sh "$module" > /dev/null + $MAKE "$module.clean" > /dev/null + $MAKE "$module.all" > /dev/null + if [ $? -ne 0 ]; + then + # Build all! + echo "Failed to build $module with dependencies, building all..." + $MAKE build-nocheck > /dev/null + if [ $? -ne 0 ]; + then + >&2 echo "Broken build. Please revert changes and try again." + exit 1 + fi + fi + fi + + # Find pch files in the module to update. + headers=`find $root/$module/ -type f -iname "precompiled_*.hxx"` + + # Each pch belongs to a library. + for header in $headers; do + libname=`echo $header | sed -e s/.*precompiled_// -e s/\.hxx//` + #TODO: Backup the header and restore when last tune fails. + + # Force update on first try below. + echo "Autotuning $module.$libname..." + ./bin/update_pch "$module" "$libname" --cutoff=999 --silent --force + + best_score=0 + best_args="" + best_time=0 + best_cutoff=0 + for i in {1..16}; do + cutoff=$i + best_score_for_cuttof=0 + #run "$root" "$module" "--cutoff=$i" --include:system --exclude:module --exclude:local + run "$root" "$module" "--cutoff=$i" --exclude:system --exclude:module --exclude:local + #run "$root" "$module" "--cutoff=$i" --include:system --include:module --exclude:local + run "$root" "$module" "--cutoff=$i" --exclude:system --include:module --exclude:local + #run "$root" "$module" "--cutoff=$i" --include:system --exclude:module --include:local + run "$root" "$module" "--cutoff=$i" --exclude:system --exclude:module --include:local + #run "$root" "$module" "--cutoff=$i" --include:system --include:module --include:local + run "$root" "$module" "--cutoff=$i" --exclude:system --include:module --include:local + + if [ $i -gt $((best_cutoff+2)) ]; + then + score_too_low=$(echo "$best_score_for_cuttof < $best_score / 1.10" | bc -l) + if [ $score_too_low -eq 1 ]; + then + echo "Score hit low of $best_score_for_cuttof, well below overall best of $best_score. Stopping." + break; + fi + fi + done + + ./bin/update_pch "$module" "$libname" $best_args --force --silent + echo "> $module.$libname, $best_args, $best_time, $size, $score" + echo + + table+=$'\n' + table+="$(args_to_table)" + done + +done + +echo "Update the relevant lines in ./bin/update_pch script:" +>&2 echo "$table" + +exit 0 diff --git a/bin/update_pch_bisect b/bin/update_pch_bisect new file mode 100755 index 000000000..8c86ac3cc --- /dev/null +++ b/bin/update_pch_bisect @@ -0,0 +1,354 @@ +#! /usr/bin/env python +# -*- Mode: python; tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +""" +This script is to fix precompiled headers. + +This script runs in two modes. +In one mode, it starts with a header +that doesn't compile. If finds the +minimum number of includes in the +header to remove to get a successful +run of the command (i.e. compile). + +In the second mode, it starts with a +header that compiles fine, however, +it contains one or more required +include without which it wouldn't +compile, which it identifies. + +Usage: ./bin/update_pch_bisect ./vcl/inc/pch/precompiled_vcl.hxx "make vcl.build" --find-required --verbose +""" + +import sys +import re +import os +import unittest +import subprocess + +SILENT = True +FIND_CONFLICTS = True + +IGNORE = 0 +GOOD = 1 +TEST_ON = 2 +TEST_OFF = 3 +BAD = 4 + +def run(command): + try: + cmd = command.split(' ', 1) + status = subprocess.call(cmd, stdout=open(os.devnull, 'w'), + stderr=subprocess.STDOUT, close_fds=True) + return True if status == 0 else False + except Exception as e: + sys.stderr.write('Error: {}\n'.format(e)) + return False + +def update_pch(filename, lines, marks): + with open(filename, 'w') as f: + for i in xrange(len(marks)): + mark = marks[i] + if mark <= TEST_ON: + f.write(lines[i]) + else: + f.write('//' + lines[i]) + +def log(*args, **kwargs): + global SILENT + if not SILENT: + print(*args, **kwargs) + +def bisect(lines, marks, min, max, update, command): + """ Disable half the includes and + calls the command. + Depending on the result, + recurse or return. + """ + global FIND_CONFLICTS + + log('Bisecting [{}, {}].'.format(min+1, max)) + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = TEST_ON if FIND_CONFLICTS else TEST_OFF + + assume_fail = False + if not FIND_CONFLICTS: + on_list = [x for x in marks if x in (TEST_ON, GOOD)] + assume_fail = (len(on_list) == 0) + + update(lines, marks) + if assume_fail or not command(): + # Failed + log('Failed [{}, {}].'.format(min+1, max)) + if min >= max - 1: + if not FIND_CONFLICTS: + # Try with this one alone. + marks[min] = TEST_ON + update(lines, marks) + if command(): + log(' Found @{}: {}'.format(min+1, lines[min].strip('\n'))) + marks[min] = GOOD + return marks + else: + log(' Found @{}: {}'.format(min+1, lines[min].strip('\n'))) + # Either way, this one is irrelevant. + marks[min] = BAD + return marks + + # Bisect + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = TEST_OFF if FIND_CONFLICTS else TEST_ON + + half = min + ((max - min) / 2) + marks = bisect(lines, marks, min, half, update, command) + marks = bisect(lines, marks, half, max, update, command) + else: + # Success + if FIND_CONFLICTS: + log(' Good [{}, {}].'.format(min+1, max)) + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = GOOD + + return marks + +def get_filename(line): + """ Strips the line from the + '#include' and angled brakets + and return the filename only. + """ + return re.sub(r'(.*#include\s*)<(.*)>(.*)', r'\2', line) + +def get_marks(lines): + marks = [] + min = -1 + max = -1 + for i in xrange(len(lines)): + line = lines[i] + if line.startswith('#include'): + marks.append(TEST_ON) + min = i if min < 0 else min + max = i + else: + marks.append(IGNORE) + + return (marks, min, max+1) + +def main(): + + global FIND_CONFLICTS + global SILENT + + filename = sys.argv[1] + command = sys.argv[2] + + for i in range(3, len(sys.argv)): + opt = sys.argv[i] + if opt == '--find-conflicts': + FIND_CONFLICTS = True + elif opt == '--find-required': + FIND_CONFLICTS = False + elif opt == '--verbose': + SILENT = False + else: + sys.stderr.write('Error: Unknown option [{}].\n'.format(opt)) + return 1 + + lines = [] + with open(filename) as f: + lines = f.readlines() + + (marks, min, max) = get_marks(lines) + + # Test preconditions. + log('Validating all-excluded state...') + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = TEST_OFF + update_pch(filename, lines, marks) + res = run(command) + + if FIND_CONFLICTS: + # Must build all excluded. + if not res: + sys.stderr.write("Error: broken state when all excluded, fix first and try again.") + return 1 + else: + # If builds all excluded, we can't bisect. + if res: + sys.stderr.write("Done: in good state when all excluded, nothing to do.") + return 1 + + # Must build all included. + log('Validating all-included state...') + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = TEST_ON + update_pch(filename, lines, marks) + if not run(command): + sys.stderr.write("Error: broken state without modifying, fix first and try again.") + return 1 + + marks = bisect(lines, marks, min, max+1, + lambda l, m: update_pch(filename, l, m), + lambda: run(command)) + if not FIND_CONFLICTS: + # Simplify further, as sometimes we can have + # false positives due to the benign nature + # of includes that are not absolutely required. + for i in xrange(len(marks)): + if marks[i] == GOOD: + marks[i] = TEST_OFF + update_pch(filename, lines, marks) + if not run(command): + # Revert. + marks[i] = GOOD + else: + marks[i] = BAD + elif marks[i] == TEST_OFF: + marks[i] = TEST_ON + + update_pch(filename, lines, marks) + + log('') + for i in xrange(len(marks)): + if marks[i] == (BAD if FIND_CONFLICTS else GOOD): + print("'{}',".format(get_filename(lines[i].strip('\n')))) + + return 0 + +if __name__ == '__main__': + + if len(sys.argv) in (3, 4, 5): + status = main() + sys.exit(status) + + print('Usage: {} <pch> <command> [--find-conflicts]|[--find-required] [--verbose]\n'.format(sys.argv[0])) + print(' --find-conflicts - Finds all conflicting includes. (Default)') + print(' Must compile without any includes.\n') + print(' --find-required - Finds all required includes.') + print(' Must compile with all includes.\n') + print(' --verbose - print noisy progress.') + print('Example: ./bin/update_pch_bisect ./vcl/inc/pch/precompiled_vcl.hxx "make vcl.build" --find-required --verbose') + print('\nRunning unit-tests...') + + +class TestBisectConflict(unittest.TestCase): + TEST = """ /* Test header. */ +#include <memory> +#include <set> +#include <algorithm> +#include <vector> +/* blah blah */ +""" + BAD_LINE = "#include <bad>" + + def setUp(self): + global FIND_CONFLICTS + FIND_CONFLICTS = True + + def _update_func(self, lines, marks): + self.lines = [] + for i in xrange(len(marks)): + mark = marks[i] + if mark <= TEST_ON: + self.lines.append(lines[i]) + else: + self.lines.append('//' + lines[i]) + + def _test_func(self): + """ Command function called by bisect. + Returns True on Success, False on failure. + """ + # If the bad line is still there, fail. + return self.BAD_LINE not in self.lines + + def test_success(self): + lines = self.TEST.split('\n') + (marks, min, max) = get_marks(lines) + marks = bisect(lines, marks, min, max, + lambda l, m: self._update_func(l, m), + lambda: self._test_func()) + self.assertTrue(BAD not in marks) + + def test_conflict(self): + lines = self.TEST.split('\n') + for pos in xrange(len(lines) + 1): + lines = self.TEST.split('\n') + lines.insert(pos, self.BAD_LINE) + (marks, min, max) = get_marks(lines) + + marks = bisect(lines, marks, min, max, + lambda l, m: self._update_func(l, m), + lambda: self._test_func()) + for i in xrange(len(marks)): + if i == pos: + self.assertEqual(BAD, marks[i]) + else: + self.assertNotEqual(BAD, marks[i]) + +class TestBisectRequired(unittest.TestCase): + TEST = """#include <algorithm> +#include <set> +#include <map> +#include <vector> +""" + REQ_LINE = "#include <req>" + + def setUp(self): + global FIND_CONFLICTS + FIND_CONFLICTS = False + + def _update_func(self, lines, marks): + self.lines = [] + for i in xrange(len(marks)): + mark = marks[i] + if mark <= TEST_ON: + self.lines.append(lines[i]) + else: + self.lines.append('//' + lines[i]) + + def _test_func(self): + """ Command function called by bisect. + Returns True on Success, False on failure. + """ + # If the required line is not there, fail. + found = self.REQ_LINE in self.lines + return found + + def test_success(self): + lines = self.TEST.split('\n') + (marks, min, max) = get_marks(lines) + marks = bisect(lines, marks, min, max, + lambda l, m: self._update_func(l, m), + lambda: self._test_func()) + self.assertTrue(GOOD not in marks) + + def test_required(self): + lines = self.TEST.split('\n') + for pos in xrange(len(lines) + 1): + lines = self.TEST.split('\n') + lines.insert(pos, self.REQ_LINE) + (marks, min, max) = get_marks(lines) + + marks = bisect(lines, marks, min, max, + lambda l, m: self._update_func(l, m), + lambda: self._test_func()) + for i in xrange(len(marks)): + if i == pos: + self.assertEqual(GOOD, marks[i]) + else: + self.assertNotEqual(GOOD, marks[i]) + +unittest.main() + +# vim: set et sw=4 ts=4 expandtab: diff --git a/bin/upload_symbols.py b/bin/upload_symbols.py new file mode 100755 index 000000000..277508da7 --- /dev/null +++ b/bin/upload_symbols.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import requests, sys +import platform, configparser + +def detect_platform(): + return platform.system() + +def main(): + if len(sys.argv) < 4: + print(sys.argv) + print("Invalid number of parameters") + print("Usage: upload-symbols.py symbols.zip config.ini \"long explanation\" [--system]") + sys.exit(1) + + base_url = "https://crashreport.libreoffice.org/" + upload_url = base_url + "upload/" + login_url = base_url + "accounts/login/" + + config = configparser.ConfigParser() + config.read(sys.argv[2]) + + user = config["CrashReport"]["User"] + password = config["CrashReport"]["Password"] + + platform = detect_platform() + files = {'symbols': open(sys.argv[1], 'rb')} + data = {'version': sys.argv[3], 'platform': platform} + + if len(sys.argv) > 4 and sys.argv[4] == "--system": + data['system'] = True + + session = requests.session() + session.get(login_url) + csrftoken = session.cookies['csrftoken'] + + login_data = { 'username': user,'password': password, + 'csrfmiddlewaretoken': csrftoken } + headers = { "Referer": base_url } + r1 = session.post(login_url, headers=headers, data=login_data) + + data['csrfmiddlewaretoken'] = csrftoken + + r = session.post(upload_url, headers=headers, files=files, data=data) + +if __name__ == "__main__": + main() + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/verify-custom-widgets-libs b/bin/verify-custom-widgets-libs new file mode 100755 index 000000000..7fad02f17 --- /dev/null +++ b/bin/verify-custom-widgets-libs @@ -0,0 +1,30 @@ +#!/bin/sh +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Run this from the source root dir of a completed build to +# verify that all customwidgets used in our .ui files have +# their factory method in the library they claim to be in +# +# Under Linux dlsym will search other locations and find +# them if they exist elsewhere, but not under windows, so +# its easy to put the wrong lib name in if developing +# under Linux + +ret=0 +FOO=`git grep -h -r lo- */uiconfig | sed -e "s/<object class=\"//g" | sed -e "s/\".*$//"| sed 's/^[ \t]*//;s/[ \t]*$//'|sort|uniq` +for foo in $FOO; do + lib=$(echo $foo | cut -f1 -d-) + symbol=$(echo $foo | cut -f2 -d-) + nm -D instdir/program/lib$lib.so | grep make$symbol > /dev/null + if [ $? != 0 ]; then + echo "$foo exists in a .ui file, but make$symbol is missing from lib$lib.so, Windows will fail to find the symbol and crash" + echo " typically make$symbol is in a different library and $foo should have the prefix of that library instead" + ret=1 + fi +done +exit $ret |