diff options
Diffstat (limited to '')
| -rw-r--r-- | bin/update/common.sh | 222 | ||||
| -rw-r--r-- | bin/update/config.py | 28 | ||||
| -rwxr-xr-x | bin/update/create_build_config.py | 60 | ||||
| -rwxr-xr-x | bin/update/create_full_mar.py | 54 | ||||
| -rwxr-xr-x | bin/update/create_full_mar_for_languages.py | 66 | ||||
| -rwxr-xr-x | bin/update/create_partial_update.py | 160 | ||||
| -rwxr-xr-x | bin/update/get_update_channel.py | 23 | ||||
| -rwxr-xr-x | bin/update/make_full_update.sh | 122 | ||||
| -rwxr-xr-x | bin/update/make_incremental_update.sh | 318 | ||||
| -rw-r--r-- | bin/update/path.py | 69 | ||||
| -rw-r--r-- | bin/update/signing.py | 12 | ||||
| -rw-r--r-- | bin/update/tools.py | 64 | ||||
| -rwxr-xr-x | bin/update/uncompress_mar.py | 54 | ||||
| -rwxr-xr-x | bin/update/upload_build_config.py | 42 | ||||
| -rwxr-xr-x | bin/update/upload_builds.py | 32 | ||||
| -rwxr-xr-x | bin/update_pch | 1322 | ||||
| -rwxr-xr-x | bin/update_pch.sh | 70 | ||||
| -rwxr-xr-x | bin/update_pch_autotune.sh | 229 | ||||
| -rwxr-xr-x | bin/update_pch_bisect | 354 |
19 files changed, 3301 insertions, 0 deletions
diff --git a/bin/update/common.sh b/bin/update/common.sh new file mode 100644 index 000000000..dcdbea8bb --- /dev/null +++ b/bin/update/common.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Code shared by update packaging scripts. +# Author: Darin Fisher +# + +# ----------------------------------------------------------------------------- +# By default just assume that these tools exist on our path +MAR=${MAR:-mar} +BZIP2=${BZIP2:-bzip2} +MBSDIFF=${MBSDIFF:-mbsdiff} + +# ----------------------------------------------------------------------------- +# Helper routines + +notice() { + echo "$*" 1>&2 +} + +get_file_size() { + info=($(ls -ln "$1")) + echo ${info[4]} +} + +check_externals() { + + # check whether we can call the mar executable + "$MAR" --version > /dev/null 2>&1 + if [ $? != 0 ]; then + notice "Could not find a valid mar executable in the path or in the MAR environment variable" + exit 1 + fi + + # check whether we can access the bzip2 executable + "$BZIP2" --help > /dev/null 2>&1 + if [ $? != 0 ]; then + notice "Could not find a valid bzip2 executable in the PATH or in the BZIP2 environment variable" + exit 1 + fi +} + +copy_perm() { + reference="$1" + target="$2" + + if [ -x "$reference" ]; then + chmod 0755 "$target" + else + chmod 0644 "$target" + fi +} + +make_add_instruction() { + f="$1" + filev2="$2" + # The third param will be an empty string when a file add instruction is only + # needed in the version 2 manifest. This only happens when the file has an + # add-if-not instruction in the version 3 manifest. This is due to the + # precomplete file prior to the version 3 manifest having a remove instruction + # for this file so the file is removed before applying a complete update. + filev3="$3" + + # Used to log to the console + if [ $4 ]; then + forced=" (forced)" + else + forced= + fi + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " add-if \"$testdir\" \"$f\"" + echo "add-if \"$testdir\" \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add-if \"$testdir\" \"$f\"" >> $filev3 + fi + else + notice " add \"$f\"$forced" + echo "add \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add \"$f\"" >> $filev3 + fi + fi +} + +check_for_add_if_not_update() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ + `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +check_for_add_to_manifestv2() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +make_add_if_not_instruction() { + f="$1" + filev3="$2" + + notice " add-if-not \"$f\" \"$f\"" + echo "add-if-not \"$f\" \"$f\"" >> $filev3 +} + +make_patch_instruction() { + f="$1" + filev2="$2" + filev3="$3" + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 + else + notice " patch \"$f.patch\" \"$f\"" + echo "patch \"$f.patch\" \"$f\"" >> $filev2 + echo "patch \"$f.patch\" \"$f\"" >> $filev3 + fi +} + +append_remove_instructions() { + dir="$1" + filev2="$2" + filev3="$3" + + if [ -f "$dir/removed-files" ]; then + listfile="$dir/removed-files" + elif [ -f "$dir/Contents/Resources/removed-files" ]; then + listfile="$dir/Contents/Resources/removed-files" + fi + if [ -n "$listfile" ]; then + # Map spaces to pipes so that we correctly handle filenames with spaces. + files=($(cat "$listfile" | tr " " "|" | sort -r)) + num_files=${#files[*]} + for ((i=0; $i<$num_files; i=$i+1)); do + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. + if [ -n "$f" ]; then + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + notice " rmdir \"$f\"" + echo "rmdir \"$f\"" >> $filev2 + echo "rmdir \"$f\"" >> $filev3 + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + notice " rmrfdir \"$f\"" + echo "rmrfdir \"$f\"" >> $filev2 + echo "rmrfdir \"$f\"" >> $filev3 + else + notice " remove \"$f\"" + echo "remove \"$f\"" >> $filev2 + echo "remove \"$f\"" >> $filev3 + fi + fi + fi + done + fi +} + +# List all files in the current directory, stripping leading "./" +# Pass a variable name and it will be filled as an array. +list_files() { + count=0 + + find . -type f \ + ! -name "update.manifest" \ + ! -name "updatev2.manifest" \ + ! -name "updatev3.manifest" \ + ! -name "temp-dirlist" \ + ! -name "temp-filelist" \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "temp-filelist" + while read file; do + eval "${1}[$count]=\"$file\"" + (( count++ )) + done < "temp-filelist" + rm "temp-filelist" +} + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "temp-dirlist" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "temp-dirlist" + rm "temp-dirlist" +} diff --git a/bin/update/config.py b/bin/update/config.py new file mode 100644 index 000000000..0bc60a07f --- /dev/null +++ b/bin/update/config.py @@ -0,0 +1,28 @@ + +import configparser +import os + +class Config(object): + + def __init__(self): + self.certificate_path = None + self.certificate_name = None + self.channel = None + self.base_url = None + self.upload_url = None + self.server_url = None + +def parse_config(config_file): + config = configparser.ConfigParser() + config.read(os.path.expanduser(config_file)) + + data = Config() + updater_data = config['Updater'] + data.base_url = updater_data['base-url'] + data.certificate_name = updater_data['certificate-name'] + data.certificate_path = updater_data['certificate-path'] + data.channel = updater_data['channel'] + data.upload_url = updater_data['upload-url'] + data.server_url = updater_data["ServerURL"] + + return data diff --git a/bin/update/create_build_config.py b/bin/update/create_build_config.py new file mode 100755 index 000000000..7cc8ac4be --- /dev/null +++ b/bin/update/create_build_config.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python3 + +import json +import sys +import os + +from config import parse_config + +from tools import replace_variables_in_string + +def update_all_url_entries(data, **kwargs): + data['complete']['url'] = replace_variables_in_string(data['complete']['url'], **kwargs) + + if sys.platform != "cygwin": + for language in data['languages']: + language['complete']['url'] = replace_variables_in_string(language['complete']['url'], **kwargs) + + if 'partials' in data: + for partial in data['partials']: + partial['file']['url'] = replace_variables_in_string(partial['file']['url'], **kwargs) + + if sys.platform == "cygwin": + continue + + for lang, lang_file in partial['languages'].items(): + lang_file['url'] = replace_variables_in_string(lang_file['url'], **kwargs) + +def main(argv): + if len(argv) < 7: + print("Usage: create_build_config.py $PRODUCTNAME $VERSION $BUILDID $PLATFORM $TARGETDIR $UPDATE_CONFIG") + sys.exit(1) + + config = parse_config(argv[6]) + + data = { 'productName' : argv[1], + 'version' : argv[2], + 'buildNumber' : argv[3], + 'updateChannel' : config.channel, + 'platform' : argv[4] + } + + extra_data_files = ['complete_info.json', 'partial_update_info.json'] + if sys.platform != "cygwin": + extra_data_files.append('complete_lang_info.json') + + for extra_file in extra_data_files: + extra_file_path = os.path.join(argv[5], extra_file) + if not os.path.exists(extra_file_path): + continue + with open(extra_file_path, "r") as f: + extra_data = json.load(f) + data.update(extra_data) + + update_all_url_entries(data, channel=config.channel, platform=argv[4], buildid=argv[3], version=argv[2]) + + with open(os.path.join(argv[5], "build_config.json"), "w") as f: + json.dump(data, f, indent=4) + +if __name__ == "__main__": + main(sys.argv) diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py new file mode 100755 index 000000000..48686be21 --- /dev/null +++ b/bin/update/create_full_mar.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import sys +import os +import subprocess +import json + +from tools import uncompress_file_to_dir, get_file_info, make_complete_mar_name +from config import parse_config +from signing import sign_mar_file +from path import UpdaterPath, convert_to_unix, convert_to_native + +current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__))) + +def main(): + if len(sys.argv) < 5: + print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $FILENAMEPREFIX $UPDATE_CONFIG") + sys.exit(1) + + update_config = sys.argv[4] + filename_prefix = sys.argv[3] + workdir = sys.argv[2] + product_name = sys.argv[1] + + if len(update_config) == 0: + print("missing update config") + sys.exit(1) + + update_path = UpdaterPath(workdir) + update_path.ensure_dir_exist() + + target_dir = update_path.get_update_dir() + temp_dir = update_path.get_current_build_dir() + + config = parse_config(update_config) + + tar_dir = os.path.join(update_path.get_workdir(), "installation", product_name, "archive", "install", "en-US") + tar_file = os.path.join(tar_dir, os.listdir(tar_dir)[0]) + + uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir) + + mar_file = make_complete_mar_name(target_dir, filename_prefix) + path = os.path.join(current_dir_path, 'make_full_update.sh') + subprocess.call([path, convert_to_native(mar_file), convert_to_native(uncompress_dir)]) + + sign_mar_file(target_dir, config, mar_file, filename_prefix) + + file_info = { 'complete' : get_file_info(mar_file, config.base_url) } + + with open(os.path.join(target_dir, 'complete_info.json'), "w") as complete_info_file: + json.dump(file_info, complete_info_file, indent = 4) + +if __name__ == '__main__': + main() diff --git a/bin/update/create_full_mar_for_languages.py b/bin/update/create_full_mar_for_languages.py new file mode 100755 index 000000000..039521dd1 --- /dev/null +++ b/bin/update/create_full_mar_for_languages.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import subprocess +import json + +from tools import uncompress_file_to_dir, get_file_info + +from config import parse_config +from path import UpdaterPath +from signing import sign_mar_file + +current_dir_path = os.path.dirname(os.path.realpath(__file__)) + +def make_complete_mar_name(target_dir, filename_prefix, language): + filename = filename_prefix + "_" + language + "_complete_langpack.mar" + return os.path.join(target_dir, filename) + +def create_lang_infos(mar_file_name, language, url): + data = {'lang' : language, + 'complete' : get_file_info(mar_file_name, url) + } + return data + +def main(): + if len(sys.argv) < 5: + print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $UPDATE_CONFIG") + sys.exit(1) + + update_config = sys.argv[4] + filename_prefix = sys.argv[3] + workdir = sys.argv[2] + product_name = sys.argv[1] + + updater_path = UpdaterPath(workdir) + target_dir = updater_path.get_update_dir() + temp_dir = updater_path.get_language_dir() + + config = parse_config(update_config) + + language_pack_dir = os.path.join(workdir, "installation", product_name + "_languagepack", "archive", "install") + language_packs = os.listdir(language_pack_dir) + lang_infos = [] + for language in language_packs: + if language == 'log': + continue + + language_dir = os.path.join(language_pack_dir, language) + language_file = os.path.join(language_dir, os.listdir(language_dir)[0]) + + directory = uncompress_file_to_dir(language_file, os.path.join(temp_dir, language)) + + mar_file_name = make_complete_mar_name(target_dir, filename_prefix, language) + + subprocess.call([os.path.join(current_dir_path, 'make_full_update.sh'), mar_file_name, directory]) + + sign_mar_file(target_dir, config, mar_file_name, filename_prefix) + + lang_infos.append(create_lang_infos(mar_file_name, language, config.base_url)) + + with open(os.path.join(target_dir, "complete_lang_info.json"), "w") as language_info_file: + json.dump({'languages' : lang_infos}, language_info_file, indent=4) + +if __name__ == '__main__': + main() diff --git a/bin/update/create_partial_update.py b/bin/update/create_partial_update.py new file mode 100755 index 000000000..9412bcd6e --- /dev/null +++ b/bin/update/create_partial_update.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +import requests +import json +import sys +import hashlib +import os +import subprocess +import errno +import json + +from config import parse_config +from uncompress_mar import extract_mar +from tools import get_file_info, get_hash +from signing import sign_mar_file + +from path import UpdaterPath, mkdir_p, convert_to_unix, convert_to_native + +BUF_SIZE = 1024 +current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__))) + +class InvalidFileException(Exception): + + def __init__(self, *args, **kwargs): + super().__init__(self, *args, **kwargs) + +def download_file(filepath, url, hash_string): + with open(filepath, "wb") as f: + response = requests.get(url, stream=True) + + if not response.ok: + return + + for block in response.iter_content(1024): + f.write(block) + + file_hash = get_hash(filepath) + + if file_hash != hash_string: + raise InvalidFileException("file hash does not match for file %s: Expected %s, Got: %s" % (url, hash_string, file_hash)) + +def handle_language(lang_entries, filedir): + mar = os.environ.get('MAR', 'mar') + langs = {} + for lang, data in lang_entries.items(): + lang_dir = os.path.join(filedir, lang) + lang_file = os.path.join(lang_dir, "lang.mar") + mkdir_p(lang_dir) + download_file(lang_file , data["url"], data["hash"]) + dir_path = os.path.join(lang_dir, "lang") + mkdir_p(dir_path) + extract_mar(lang_file, dir_path) + langs[lang] = dir_path + + return langs + +def download_mar_for_update_channel_and_platform(config, platform, temp_dir): + mar = os.environ.get('MAR', 'mar') + base_url = config.server_url + "update/partial-targets/1/" + url = base_url + platform + "/" + config.channel + r = requests.get(url) + if r.status_code != 200: + print(r.content) + raise Exception("download failed") + + update_info = json.loads(r.content.decode("utf-8")) + update_files = update_info['updates'] + downloaded_updates = {} + for update_file in update_files: + build = update_file["build"] + filedir = os.path.join(temp_dir, build) + + mkdir_p(filedir) + + filepath = filedir + "/complete.mar" + url = update_file["update"]["url"] + expected_hash = update_file["update"]["hash"] + download_file(filepath, url, expected_hash) + + dir_path = os.path.join(filedir, "complete") + mkdir_p(dir_path) + extract_mar(filepath, dir_path) + + downloaded_updates[build] = {"complete": dir_path} + + langs = handle_language(update_file["languages"], filedir) + downloaded_updates[build]["languages"] = langs + + return downloaded_updates + +def generate_file_name(current_build_id, old_build_id, mar_name_prefix): + name = "%s_from_%s_partial.mar" %(mar_name_prefix, old_build_id) + return name + +def generate_lang_file_name(current_build_id, old_build_id, mar_name_prefix, lang): + name = "%s_%s_from_%s_partial.mar" %(mar_name_prefix, lang, old_build_id) + return name + +def add_single_dir(path): + dir_name = [os.path.join(path, name) for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))] + return dir_name[0] + +def main(): + workdir = sys.argv[1] + + updater_path = UpdaterPath(workdir) + updater_path.ensure_dir_exist() + + mar_name_prefix = sys.argv[2] + update_config = sys.argv[3] + platform = sys.argv[4] + build_id = sys.argv[5] + + current_build_path = updater_path.get_current_build_dir() + mar_dir = updater_path.get_mar_dir() + temp_dir = updater_path.get_previous_build_dir() + update_dir = updater_path.get_update_dir() + + current_build_path = add_single_dir(current_build_path) + if sys.platform == "cygwin": + current_build_path = add_single_dir(current_build_path) + + config = parse_config(update_config) + + updates = download_mar_for_update_channel_and_platform(config, platform, temp_dir) + + data = {"partials": []} + + for build, update in updates.items(): + file_name = generate_file_name(build_id, build, mar_name_prefix) + mar_file = os.path.join(update_dir, file_name) + subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(mar_file), convert_to_native(update["complete"]), convert_to_native(current_build_path)]) + sign_mar_file(update_dir, config, mar_file, mar_name_prefix) + + partial_info = {"file":get_file_info(mar_file, config.base_url), "from": build, "to": build_id, "languages": {}} + + # on Windows we don't use language packs + if sys.platform != "cygwin": + for lang, lang_info in update["languages"].items(): + lang_name = generate_lang_file_name(build_id, build, mar_name_prefix, lang) + + # write the file into the final directory + lang_mar_file = os.path.join(update_dir, lang_name) + + # the directory of the old language file is of the form + # workdir/mar/language/en-US/LibreOffice_<version>_<os>_archive_langpack_<lang>/ + language_dir = add_single_dir(os.path.join(mar_dir, "language", lang)) + subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(lang_mar_file), convert_to_native(lang_info), convert_to_native(language_dir)]) + sign_mar_file(update_dir, config, lang_mar_file, mar_name_prefix) + + # add the partial language info + partial_info["languages"][lang] = get_file_info(lang_mar_file, config.base_url) + + data["partials"].append(partial_info) + + with open(os.path.join(update_dir, "partial_update_info.json"), "w") as f: + json.dump(data, f) + + +if __name__ == '__main__': + main() diff --git a/bin/update/get_update_channel.py b/bin/update/get_update_channel.py new file mode 100755 index 000000000..f94507d64 --- /dev/null +++ b/bin/update/get_update_channel.py @@ -0,0 +1,23 @@ +#!/usr/bin/python3 +# -*- Mode: python; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import sys +from config import parse_config + +def main(): + if len(sys.argv) < 2: + sys.exit(1) + + update_config = sys.argv[1] + config = parse_config(update_config) + print(config.channel) + +if __name__ == "__main__": + main() + +# vim:set shiftwidth=4 softtabstop=4 expandtab: */ diff --git a/bin/update/make_full_update.sh b/bin/update/make_full_update.sh new file mode 100755 index 000000000..4140ecae6 --- /dev/null +++ b/bin/update/make_full_update.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates full update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +if [ $1 = -h ]; then + print_usage + notice "" + notice "The contents of DIRECTORY will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice "" + exit 1 +fi + +check_externals +# ----------------------------------------------------------------------------- + +archive="$1" +targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi +workdir="$targetdir.work" +updatemanifestv2="$workdir/updatev2.manifest" +updatemanifestv3="$workdir/updatev3.manifest" +targetfiles="updatev2.manifest updatev3.manifest" + +mkdir -p "$workdir" +echo "updatev2.manifest" >> $workdir/files.txt +echo "updatev3.manifest" >> $workdir/files.txt + +# Generate a list of all files in the target directory. +pushd "$targetdir" +if test $? -ne 0 ; then + exit 1 +fi + +# if [ ! -f "precomplete" ]; then +# if [ ! -f "Contents/Resources/precomplete" ]; then +# notice "precomplete file is missing!" +# exit 1 +# fi +# fi + +list_files files + +popd + +# Add the type of update to the beginning of the update manifests. +> $updatemanifestv2 +> $updatemanifestv3 +notice "" +notice "Adding type instruction to update manifests" +notice " type complete" +echo "type \"complete\"" >> $updatemanifestv2 +echo "type \"complete\"" >> $updatemanifestv3 + +notice "" +notice "Adding file add instructions to update manifests" +num_files=${#files[*]} + +for ((i=0; $i<$num_files; i=$i+1)); do + f="${files[$i]}" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + if check_for_add_to_manifestv2 "$f"; then + make_add_instruction "$f" "$updatemanifestv2" "" 1 + fi + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + fi + + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" + echo $f >> $workdir/files.txt +done + +# Append remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" + +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" +$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" + +eval "$MAR -C \"$workdir\" -c output.mar -f $workdir/files.txt" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/bin/update/make_incremental_update.sh b/bin/update/make_incremental_update.sh new file mode 100755 index 000000000..31bddabdb --- /dev/null +++ b/bin/update/make_incremental_update.sh @@ -0,0 +1,318 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates incremental update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" + notice "" + notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice " -f clobber this file in the installation" + notice " Must be a path to a file to clobber in the partial update." + notice "" +} + +check_for_forced_update() { + force_list="$1" + forced_file_chk="$2" + + local f + + if [ "$forced_file_chk" = "precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "${forced_file_chk##*.}" = "chk" ]; then + ## "true" *giggle* + return 0; + fi + + for f in $force_list; do + #echo comparing $forced_file_chk to $f + if [ "$forced_file_chk" = "$f" ]; then + ## "true" *giggle* + return 0; + fi + done + ## 'false'... because this is bash. Oh yay! + return 1; +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +requested_forced_updates='Contents/MacOS/firefox' + +while getopts "hf:" flag +do + case "$flag" in + h) print_usage; exit 0 + ;; + f) requested_forced_updates="$requested_forced_updates $OPTARG" + ;; + ?) print_usage; exit 1 + ;; + esac +done + +# ----------------------------------------------------------------------------- + +let arg_start=$OPTIND-1 +shift $arg_start + +archive="$1" +olddir="$2" +newdir="$3" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then + # Remove the / + newdir=$(echo "$newdir" | sed -e 's:\/$::') +fi +workdir="$newdir.work" +updatemanifestv2="$workdir/updatev2.manifest" +updatemanifestv3="$workdir/updatev3.manifest" + +mkdir -p "$workdir" +echo "updatev2.manifest" >> $workdir/files.txt +echo "updatev3.manifest" >> $workdir/files.txt + +# Generate a list of all files in the target directory. +pushd "$olddir" +if test $? -ne 0 ; then + exit 1 +fi + +list_files oldfiles +list_dirs olddirs + +popd + +pushd "$newdir" +if test $? -ne 0 ; then + exit 1 +fi + +# if [ ! -f "precomplete" ]; then +# if [ ! -f "Contents/Resources/precomplete" ]; then +# notice "precomplete file is missing!" +# exit 1 +# fi +# fi + +list_dirs newdirs +list_files newfiles + +popd + +# Add the type of update to the beginning of the update manifests. +notice "" +notice "Adding type instruction to update manifests" +> $updatemanifestv2 +> $updatemanifestv3 +notice " type partial" +echo "type \"partial\"" >> $updatemanifestv2 +echo "type \"partial\"" >> $updatemanifestv3 + +notice "" +notice "Adding file patch and add instructions to update manifests" + +num_oldfiles=${#oldfiles[*]} +remove_array= +num_removes=0 + +for ((i=0; $i<$num_oldfiles; i=$i+1)); do + f="${oldfiles[$i]}" + + # If this file exists in the new directory as well, then check if it differs. + if [ -f "$newdir/$f" ]; then + + if check_for_add_if_not_update "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p `dirname "$workdir/$f"` + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_if_not_instruction "$f" "$updatemanifestv3" + echo $f >> $workdir/files.txt + continue 1 + fi + + if check_for_forced_update "$requested_forced_updates" "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p `dirname "$workdir/$f"` + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 + echo $f >> $workdir/files.txt + continue 1 + fi + + if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then + # Compute both the compressed binary diff and the compressed file, and + # compare the sizes. Then choose the smaller of the two to package. + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + notice "diffing \"$f\"" + # MBSDIFF_HOOK represents the communication interface with funsize and, + # if enabled, caches the intermediate patches for future use and + # compute avoidance + # + # An example of MBSDIFF_HOOK env variable could look like this: + # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" + # where myscript.sh has the following usage: + # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ + # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL + # + # Note: patches are bzipped stashed in funsize to gain more speed + + # if service is not enabled then default to old behavior + if [ -z "$MBSDIFF_HOOK" ]; then + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $BZIP2 -z9 "$workdir/$f.patch" + else + # if service enabled then check patch existence for retrieval + if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then + notice "file \"$f\" found in funsize, diffing skipped" + else + # if not found already - compute it and cache it for future use + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $BZIP2 -z9 "$workdir/$f.patch" + $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" + fi + fi + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + patchfile="$workdir/$f.patch.bz2" + patchsize=$(get_file_size "$patchfile") + fullsize=$(get_file_size "$workdir/$f") + + if [ $patchsize -lt $fullsize ]; then + make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + mv -f "$patchfile" "$workdir/$f.patch" + rm -f "$workdir/$f" + echo $f.patch >> $workdir/files.txt + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + rm -f "$patchfile" + echo $f >> $workdir/files.txt + fi + fi + else + # remove instructions are added after add / patch instructions for + # consistency with make_incremental_updates.py + remove_array[$num_removes]=$f + (( num_removes++ )) + fi +done + +# Newly added files +notice "" +notice "Adding file add instructions to update manifests" +num_newfiles=${#newfiles[*]} + +for ((i=0; $i<$num_newfiles; i=$i+1)); do + f="${newfiles[$i]}" + + # If we've already tested this file, then skip it + for ((j=0; $j<$num_oldfiles; j=$j+1)); do + if [ "$f" = "${oldfiles[j]}" ]; then + continue 2 + fi + done + + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + fi + + + echo $f >> $workdir/files.txt +done + +notice "" +notice "Adding file remove instructions to update manifests" +for ((i=0; $i<$num_removes; i=$i+1)); do + f="${remove_array[$i]}" + notice " remove \"$f\"" + echo "remove \"$f\"" >> $updatemanifestv2 + echo "remove \"$f\"" >> $updatemanifestv3 +done + +# Add remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" + +notice "" +notice "Adding directory remove instructions for directories that no longer exist" +num_olddirs=${#olddirs[*]} + +for ((i=0; $i<$num_olddirs; i=$i+1)); do + f="${olddirs[$i]}" + # If this dir doesn't exist in the new directory remove it. + if [ ! -d "$newdir/$f" ]; then + notice " rmdir $f/" + echo "rmdir \"$f/\"" >> $updatemanifestv2 + echo "rmdir \"$f/\"" >> $updatemanifestv3 + fi +done + +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" +$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" + +mar_command="$MAR" +if [[ -n $PRODUCT_VERSION ]] +then + mar_command="$mar_command -V $PRODUCT_VERSION" +fi +if [[ -n $CHANNEL_ID ]] +then + mar_command="$mar_command -H $CHANNEL_ID" +fi +mar_command="$mar_command -C \"$workdir\" -c output.mar -f $workdir/files.txt" +eval "$mar_command" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/bin/update/path.py b/bin/update/path.py new file mode 100644 index 000000000..0fe0fd5eb --- /dev/null +++ b/bin/update/path.py @@ -0,0 +1,69 @@ +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import os +import errno +import subprocess +from sys import platform + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def convert_to_unix(path): + if platform == "cygwin": + return subprocess.check_output(["cygpath", "-u", path]).decode("utf-8", "strict").rstrip() + else: + return path + +def convert_to_native(path): + if platform == "cygwin": + return subprocess.check_output(["cygpath", "-m", path]).decode("utf-8", "strict").rstrip() + else: + return path + +class UpdaterPath(object): + + def __init__(self, workdir): + self._workdir = convert_to_unix(workdir) + + def get_workdir(self): + return self._workdir + + def get_update_dir(self): + return os.path.join(self._workdir, "update-info") + + def get_current_build_dir(self): + return os.path.join(self._workdir, "mar", "current-build") + + def get_mar_dir(self): + return os.path.join(self._workdir, "mar") + + def get_previous_build_dir(self): + return os.path.join(self._workdir, "mar", "previous-build") + + def get_language_dir(self): + return os.path.join(self.get_mar_dir(), "language") + + def get_workdir(self): + return self._workdir + + def ensure_dir_exist(self): + mkdir_p(self.get_update_dir()) + mkdir_p(self.get_current_build_dir()) + mkdir_p(self.get_mar_dir()) + mkdir_p(self.get_previous_build_dir()) + mkdir_p(self.get_language_dir()) + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/update/signing.py b/bin/update/signing.py new file mode 100644 index 000000000..c0b43ce91 --- /dev/null +++ b/bin/update/signing.py @@ -0,0 +1,12 @@ +from tools import make_complete_mar_name + +import os +import subprocess +import path + +def sign_mar_file(target_dir, config, mar_file, filename_prefix): + signed_mar_file = make_complete_mar_name(target_dir, filename_prefix + '_signed') + mar_executable = os.environ.get('MAR', 'mar') + subprocess.check_call([mar_executable, '-C', path.convert_to_native(target_dir), '-d', path.convert_to_native(config.certificate_path), '-n', config.certificate_name, '-s', path.convert_to_native(mar_file), path.convert_to_native(signed_mar_file)]) + + os.rename(signed_mar_file, mar_file) diff --git a/bin/update/tools.py b/bin/update/tools.py new file mode 100644 index 000000000..8cd786635 --- /dev/null +++ b/bin/update/tools.py @@ -0,0 +1,64 @@ +import os +import hashlib +import zipfile +import tarfile + +def uncompress_file_to_dir(compressed_file, uncompress_dir): + command = None + extension = os.path.splitext(compressed_file)[1] + + try: + os.mkdir(uncompress_dir) + except FileExistsError as e: + pass + + if extension == '.gz': + tar = tarfile.open(compressed_file) + tar.extractall(uncompress_dir) + tar.close() + elif extension == '.zip': + zip_file = zipfile.ZipFile(compressed_file) + zip_file.extractall(uncompress_dir) + zip_file.close() + + uncompress_dir = os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]) + if " " in os.listdir(uncompress_dir)[0]: + print("replacing whitespace in directory name") + os.rename(os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]), + os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0].replace(" ", "_"))) + else: + print("Error: unknown extension " + extension) + + return os.path.join(uncompress_dir, os.listdir(uncompress_dir)[0]) + +BUF_SIZE = 1048576 + +def get_hash(file_path): + sha512 = hashlib.sha512() + with open(file_path, 'rb') as f: + while True: + data = f.read(BUF_SIZE) + if not data: + break + sha512.update(data) + return sha512.hexdigest() + +def get_file_info(mar_file, url): + filesize = os.path.getsize(mar_file) + data = { 'hash' : get_hash(mar_file), + 'hashFunction' : 'sha512', + 'size' : filesize, + 'url' : url + os.path.basename(mar_file)} + + return data + +def replace_variables_in_string(string, **kwargs): + new_string = string + for key, val in kwargs.items(): + new_string = new_string.replace('$(%s)'%key, val) + + return new_string + +def make_complete_mar_name(target_dir, filename_prefix): + filename = filename_prefix + "_complete.mar" + return os.path.join(target_dir, filename) diff --git a/bin/update/uncompress_mar.py b/bin/update/uncompress_mar.py new file mode 100755 index 000000000..02dafbaff --- /dev/null +++ b/bin/update/uncompress_mar.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Extract a mar file and uncompress the content + +import os +import re +import sys +import subprocess +from path import convert_to_native + +def uncompress_content(file_path): + bzip2 = os.environ.get('BZIP2', 'bzip2') + file_path_compressed = file_path + ".bz2" + os.rename(file_path, file_path_compressed) + subprocess.check_call(["bzip2", "-d", convert_to_native(file_path_compressed)]) + +def extract_mar(mar_file, target_dir): + mar = os.environ.get('MAR', 'mar') + subprocess.check_call([mar, "-C", convert_to_native(target_dir), "-x", convert_to_native(mar_file)]) + file_info = subprocess.check_output([mar, "-t", convert_to_native(mar_file)]) + lines = file_info.splitlines() + prog = re.compile(r"\d+\s+\d+\s+(.+)") + for line in lines: + match = prog.match(line.decode("utf-8", "strict")) + if match is None: + continue + info = match.groups()[0] + # ignore header line + if info == b'NAME': + continue + + uncompress_content(os.path.join(target_dir, info)) + +def main(): + if len(sys.argv) != 3: + print("Help: This program takes exactly two arguments pointing to a mar file and a target location") + sys.exit(1) + + mar_file = sys.argv[1] + target_dir = sys.argv[2] + extract_mar(mar_file, target_dir) + +if __name__ == "__main__": + main() + +# vim: set shiftwidth=4 softtabstop=4 expandtab: diff --git a/bin/update/upload_build_config.py b/bin/update/upload_build_config.py new file mode 100755 index 000000000..9a87661ee --- /dev/null +++ b/bin/update/upload_build_config.py @@ -0,0 +1,42 @@ +#! /usr/bin/env python3 + +import sys +import os +import configparser +import requests + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +def main(argv): + + updater_config = sys.argv[2] + + config = configparser.ConfigParser() + config.read(os.path.expanduser(updater_config)) + + user = config["Updater"]["User"] + password = config["Updater"]["Password"] + base_address = config["Updater"]["ServerURL"] + + login_url = base_address + "accounts/login/" + + session = requests.session() + r1 = session.get(login_url) + csrftoken = session.cookies['csrftoken'] + + login_data = { 'username': user,'password': password, + 'csrfmiddlewaretoken': csrftoken } + r1 = session.post(login_url, data=login_data, headers={"Referer": login_url}) + + url = base_address + "update/upload/release" + data = {} + data['csrfmiddlewaretoken'] = csrftoken + + build_config = os.path.join(sys.argv[1], "build_config.json") + r = session.post(url, files={'release_config': open(build_config, "r")}, data=data) + print(r.content) + if r.status_code != 200: + sys.exit(1) + +if __name__ == "__main__": + main(sys.argv) diff --git a/bin/update/upload_builds.py b/bin/update/upload_builds.py new file mode 100755 index 000000000..210668e0d --- /dev/null +++ b/bin/update/upload_builds.py @@ -0,0 +1,32 @@ +#! /usr/bin/env python3 + +import sys +import os +import subprocess + +from config import parse_config +from path import convert_to_unix + +from tools import replace_variables_in_string + +def main(): + product_name = sys.argv[1] + buildid = sys.argv[2] + platform = sys.argv[3] + update_dir = sys.argv[4] + update_config = sys.argv[5] + + config = parse_config(update_config) + upload_url = replace_variables_in_string(config.upload_url, channel=config.channel, buildid=buildid, platform=platform) + + target_url, target_dir = upload_url.split(':') + + command = "ssh %s 'mkdir -p %s'"%(target_url, target_dir) + print(command) + subprocess.call(command, shell=True) + for file in os.listdir(update_dir): + if file.endswith('.mar'): + subprocess.call(['scp', convert_to_unix(os.path.join(update_dir, file)), upload_url]) + +if __name__ == '__main__': + main() diff --git a/bin/update_pch b/bin/update_pch new file mode 100755 index 000000000..1cb5cb997 --- /dev/null +++ b/bin/update_pch @@ -0,0 +1,1322 @@ +#! /usr/bin/env python3 +# -*- Mode: python; tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +""" +This script generates precompiled headers for a given +module and library. + +Given a gmake makefile that belongs to some LO module: +1) Process the makefile to find source files (process_makefile). +2) For every source file, find all includes (process_source). +3) Uncommon and rare includes are filtered (remove_rare). +4) Conflicting headers are excluded (filter_ignore). +5) Local files to the source are excluded (Filter_Local). +6) Fixup missing headers that sources expect (fixup). +7) The resulting includes are sorted by category (sort_by_category). +8) The pch file is generated (generate). +""" + +import sys +import re +import os +import unittest +import glob + +CUTOFF = 1 +EXCLUDE_MODULE = False +EXCLUDE_LOCAL = False +EXCLUDE_SYSTEM = True +SILENT = False +WORKDIR = 'workdir' + +# System includes: oox, sal, sd, svl, vcl + +INCLUDE = False +EXCLUDE = True +DEFAULTS = \ +{ +# module.library : (min, system, module, local), best time + 'accessibility.acc' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 7.8 + 'basctl.basctl' : ( 3, EXCLUDE, INCLUDE, EXCLUDE), # 11.9 + 'basegfx.basegfx' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 3.8 + 'basic.sb' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 10.7 + 'chart2.chartcontroller' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 18.4 + 'chart2.chartcore' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 22.5 + 'comphelper.comphelper' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 7.6 + 'configmgr.configmgr' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 6.0 + 'connectivity.ado' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 6.4 + 'connectivity.calc' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 4.6 + 'connectivity.dbase' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # 5.2 + 'connectivity.dbpool2' : ( 5, EXCLUDE, INCLUDE, EXCLUDE), # 3.0 + 'connectivity.dbtools' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 0.8 + 'connectivity.file' : ( 2, EXCLUDE, INCLUDE, EXCLUDE), # 5.1 + 'connectivity.firebird_sdbc' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 5.1 + 'connectivity.flat' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # 4.6 + 'connectivity.mysql' : ( 4, EXCLUDE, INCLUDE, EXCLUDE), # 3.4 + 'connectivity.odbc' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 5.0 + 'connectivity.postgresql-sdbc-impl' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 6.7 + 'cppcanvas.cppcanvas' : (11, EXCLUDE, INCLUDE, INCLUDE), # 4.8 + 'cppuhelper.cppuhelper' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 4.6 + 'cui.cui' : ( 8, EXCLUDE, INCLUDE, EXCLUDE), # 19.7 + 'dbaccess.dba' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 13.8 + 'dbaccess.dbaxml' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 6.5 + 'dbaccess.dbu' : (12, EXCLUDE, EXCLUDE, EXCLUDE), # 23.6 + 'dbaccess.sdbt' : ( 1, EXCLUDE, INCLUDE, EXCLUDE), # 2.9 + 'desktop.deployment' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 6.1 + 'desktop.deploymentgui' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 5.7 + 'desktop.deploymentmisc' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 3.4 + 'desktop.sofficeapp' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 6.5 + 'drawinglayer.drawinglayer' : ( 4, EXCLUDE, EXCLUDE, EXCLUDE), # 7.4 + 'editeng.editeng' : ( 5, EXCLUDE, INCLUDE, EXCLUDE), # 13.0 + 'forms.frm' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 14.2 + 'framework.fwk' : ( 7, EXCLUDE, INCLUDE, INCLUDE), # 14.8 + 'hwpfilter.hwp' : ( 3, EXCLUDE, INCLUDE, INCLUDE), # 6.0 + 'lotuswordpro.lwpft' : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), # 11.6 + 'oox.oox' : ( 6, EXCLUDE, EXCLUDE, INCLUDE), # 28.2 + 'package.package2' : ( 3, EXCLUDE, INCLUDE, INCLUDE), # 4.5 + 'package.xstor' : ( 2, EXCLUDE, INCLUDE, EXCLUDE), # 3.8 + 'reportdesign.rpt' : ( 9, EXCLUDE, INCLUDE, INCLUDE), # 9.4 + 'reportdesign.rptui' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 13.1 + 'reportdesign.rptxml' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 7.6 + 'sal.sal' : ( 2, EXCLUDE, EXCLUDE, INCLUDE), # 4.2 + 'sc.sc' : (12, EXCLUDE, INCLUDE, INCLUDE), # 92.6 + 'sc.scfilt' : ( 4, EXCLUDE, EXCLUDE, INCLUDE), # 39.9 + 'sc.scui' : ( 1, EXCLUDE, EXCLUDE, INCLUDE), # 15.0 + 'sc.vbaobj' : ( 1, EXCLUDE, EXCLUDE, INCLUDE), # 17.3 + 'sd.sd' : ( 4, EXCLUDE, EXCLUDE, INCLUDE), # 47.4 + 'sd.sdui' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 9.4 + 'sdext.PresentationMinimizer' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # 4.1 + 'sdext.PresenterScreen' : ( 2, EXCLUDE, INCLUDE, EXCLUDE), # 7.1 + 'sfx2.sfx' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), # 27.4 + 'slideshow.slideshow' : ( 4, EXCLUDE, INCLUDE, EXCLUDE), # 10.8 + 'sot.sot' : ( 5, EXCLUDE, EXCLUDE, INCLUDE), # 3.1 + 'starmath.sm' : ( 5, EXCLUDE, EXCLUDE, INCLUDE), # 10.9 + 'svgio.svgio' : ( 8, EXCLUDE, EXCLUDE, INCLUDE), # 4.3 + 'emfio.emfio' : ( 8, EXCLUDE, EXCLUDE, INCLUDE), # 4.3 + 'svl.svl' : ( 6, EXCLUDE, EXCLUDE, EXCLUDE), # 7.6 + 'svtools.svt' : ( 4, EXCLUDE, INCLUDE, EXCLUDE), # 17.6 + 'svx.svx' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 20.7 + 'svx.svxcore' : ( 7, EXCLUDE, INCLUDE, EXCLUDE), # 37.0 + 'sw.msword' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 22.4 + 'sw.sw' : ( 7, EXCLUDE, EXCLUDE, INCLUDE), # 129.6 + 'sw.swui' : ( 3, EXCLUDE, INCLUDE, INCLUDE), # 26.1 + 'sw.vbaswobj' : ( 4, EXCLUDE, INCLUDE, INCLUDE), # 13.1 + 'tools.tl' : ( 5, EXCLUDE, EXCLUDE, EXCLUDE), # 4.2 + 'unotools.utl' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 7.0 + 'unoxml.unoxml' : ( 1, EXCLUDE, EXCLUDE, EXCLUDE), # 4.6 + 'uui.uui' : ( 4, EXCLUDE, EXCLUDE, EXCLUDE), # 4.9 + 'vbahelper.msforms' : ( 3, EXCLUDE, INCLUDE, INCLUDE), # 5.2 + 'vbahelper.vbahelper' : ( 3, EXCLUDE, EXCLUDE, INCLUDE), # 7.0 + 'vcl.vcl' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 35.7 + 'writerfilter.writerfilter' : ( 5, EXCLUDE, EXCLUDE, EXCLUDE), # 19.7/27.3 + 'xmloff.xo' : ( 7, EXCLUDE, INCLUDE, INCLUDE), # 22.1 + 'xmloff.xof' : ( 1, EXCLUDE, EXCLUDE, INCLUDE), # 4.4 + 'xmlscript.xmlscript' : ( 4, EXCLUDE, EXCLUDE, INCLUDE), # 3.6 + 'xmlsecurity.xmlsecurity' : ( 6, EXCLUDE, INCLUDE, INCLUDE), # 5.1 + 'xmlsecurity.xsec_xmlsec' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # 4.4 + 'xmlsecurity.xsec_gpg' : ( 2, EXCLUDE, INCLUDE, INCLUDE), # ? +} + +def remove_rare(raw, min_use=-1): + """ Remove headers not commonly included. + The minimum threshold is min_use. + """ + # The minimum number of times a header + # must be included to be in the PCH. + min_use = min_use if min_use >= 0 else CUTOFF + + out = [] + if not raw or not len(raw): + return out + + inc = sorted(raw) + last = inc[0] + count = 1 + for x in range(1, len(inc)): + i = inc[x] + if i == last: + count += 1 + else: + if count >= min_use: + out.append(last) + last = i + count = 1 + + # Last group. + if count >= min_use: + out.append(last) + + return out + +def process_list(list, callable): + """ Given a list and callable + we pass each entry through + the callable and only add to + the output if not blank. + """ + out = [] + for i in list: + line = callable(i) + if line and len(line): + out.append(line) + return out + +def find_files(path, recurse=True): + list = [] + for root, dir, files in os.walk(path): + list += map(lambda x: os.path.join(root, x), files) + return list + +def get_filename(line): + """ Strips the line from the + '#include' and angled brackets + and return the filename only. + """ + if not len(line) or line[0] != '#': + return line + return re.sub(r'(.*#include\s*)<(.*)>(.*)', r'\2', line) + +def is_c_runtime(inc, root, module): + """ Heuristic-based detection of C/C++ + runtime headers. + They are all-lowercase, with .h or + no extension, filename only. + Try to check that they are not LO headers. + """ + inc = get_filename(inc) + + if inc.endswith('.hxx') or inc.endswith('.hpp'): + return False + + if inc.endswith('.h') and inc.startswith( 'config_' ): + return False + + hasdot = False + for c in inc: + if c == '/': + return False + if c == '.' and not inc.endswith('.h'): + return False + if c == '.': + hasdot = True + if c.isupper(): + return False + if not hasdot: # <memory> etc. + return True + + if glob.glob(os.path.join(root, module, '**', inc), recursive=True): + return False; + + return True + +def sanitize(raw): + """ There are two forms of includes, + those with <> and "". + Technically, the difference is that + the compiler can use an internal + representation for an angled include, + such that it doesn't have to be a file. + For our purposes, there is no difference. + Here, we convert everything to angled. + """ + if not raw or not len(raw): + return '' + raw = raw.strip() + if not len(raw): + return '' + return re.sub(r'(.*#include\s*)\"(.*)\"(.*)', r'#include <\2>', raw) + +class Filter_Local(object): + """ Filter headers local to a module. + allow_public: allows include/module/file.hxx + #include <module/file.hxx> + allow_module: allows module/inc/file.hxx + #include <file.hxx> + allow_locals: allows module/source/file.hxx and + module/source/inc/file.hxx + #include <file.hxx> + """ + def __init__(self, root, module, allow_public=True, allow_module=True, allow_locals=True): + self.root = root + self.module = module + self.allow_public = allow_public + self.allow_module = allow_module + self.allow_locals = allow_locals + self.public_prefix = '<' + self.module + '/' + + all = find_files(os.path.join(root, module)) + self.module_includes = [] + self.locals = [] + mod_prefix = module + '/inc/' + for i in all: + if mod_prefix in i: + self.module_includes.append(i) + else: + self.locals.append(i) + + def is_public(self, line): + return self.public_prefix in line + + def is_module(self, line): + """ Returns True if in module/inc/... """ + filename = get_filename(line) + for i in self.module_includes: + if i.endswith(filename): + return True + return False + + def is_local(self, line): + """ Returns True if in module/source/... """ + filename = get_filename(line) + for i in self.locals: + if i.endswith(filename): + return True + return False + + def is_external(self, line): + return is_c_runtime(line, self.root, self.module) and \ + not self.is_public(line) and \ + not self.is_module(line) and \ + not self.is_local(line) + + def find_local_file(self, line): + """ Finds the header file in the module dir, + but doesn't validate. + """ + filename = get_filename(line) + for i in self.locals: + if i.endswith(filename): + return i + for i in self.module_includes: + if i.endswith(filename): + return i + return None + + def proc(self, line): + assert line and len(line) + + if line[0] == '#': + if not SILENT: + sys.stderr.write('unhandled #include : {}\n'.format(line)) + return '' + + assert line[0] != '<' and line[0] != '#' + + filename = get_filename(line) + + # Local with relative path. + if filename.startswith('..'): + # Exclude for now as we don't have cxx path. + return '' + + # Locals are included first (by the compiler). + if self.is_local(filename): + # Use only locals that are in some /inc/ directory (either in <module>/inc or + # somewhere under <module>/source/**/inc/, compilations use -I for these paths + # and headers elsewhere would not be found when compiling the PCH. + if not self.allow_locals: + return '' + elif '/inc/' in filename: + return filename + elif glob.glob(os.path.join(self.root, self.module, '**', 'inc', filename), recursive=True): + return filename + else: + return '' + + # Module headers are next. + if self.is_module(filename): + return line if self.allow_module else '' + + # Public headers are last. + if self.is_public(line): + return line if self.allow_public else '' + + # Leave out potentially unrelated files local + # to some other module we can't include directly. + if '/' not in filename and not self.is_external(filename): + return '' + + # Unfiltered. + return line + +def filter_ignore(line, module): + """ Filters includes from known + problematic ones. + Expects sanitized input. + """ + assert line and len(line) + + # Always include files without extension. + if '.' not in line: + return line + + # Extract filenames for ease of comparison. + line = get_filename(line) + + # Filter out all files that are not normal headers. + if not line.endswith('.h') and \ + not line.endswith('.hxx') and \ + not line.endswith('.hpp') and \ + not line.endswith('.hdl'): + return '' + + ignore_list = [ + 'LibreOfficeKit/LibreOfficeKitEnums.h', # Needs special directives + 'LibreOfficeKit/LibreOfficeKitTypes.h', # Needs special directives + 'jerror.h', # c++ unfriendly + 'jpeglib.h', # c++ unfriendly + 'boost/spirit/include/classic_core.hpp' # depends on BOOST_SPIRIT_DEBUG + ] + + if module == 'basic': + ignore_list += [ + 'basic/vbahelper.hxx', + ] + if module == 'connectivity': + ignore_list += [ + 'com/sun/star/beans/PropertyAttribute.hpp', # OPTIONAL defined via objbase.h + 'com/sun/star/sdbcx/Privilege.hpp', # DELETE defined via objbase.h + 'ado/*' , # some strange type conflict because of Window's adoctint.h + 'adoint.h', + 'adoctint.h', + ] + if module == 'sc': + ignore_list += [ + 'progress.hxx', # special directives + 'scslots.hxx', # special directives + ] + if module == 'sd': + ignore_list += [ + 'sdgslots.hxx', # special directives + 'sdslots.hxx', # special directives + ] + if module == 'sfx2': + ignore_list += [ + 'sfx2/recentdocsview.hxx', # Redefines ApplicationType defined in objidl.h + 'sfx2/sidebar/Sidebar.hxx', + 'sfx2/sidebar/UnoSidebar.hxx', + 'sfxslots.hxx', # externally defined types + ] + if module == 'sot': + ignore_list += [ + 'sysformats.hxx', # Windows headers + ] + if module == 'vcl': + ignore_list += [ + 'accmgr.hxx', # redefines ImplAccelList + 'image.h', + 'jobset.h', + 'opengl/gdiimpl.hxx', + 'opengl/salbmp.hxx', + 'openglgdiimpl', # ReplaceTextA + 'printdlg.hxx', + 'salinst.hxx', # GetDefaultPrinterA + 'salprn.hxx', # SetPrinterDataA + 'vcl/jobset.hxx', + 'vcl/oldprintadaptor.hxx', + 'vcl/opengl/OpenGLContext.hxx', + 'vcl/opengl/OpenGLHelper.hxx', # Conflicts with X header on *ix + 'vcl/print.hxx', + 'vcl/prntypes.hxx', # redefines Orientation from filter/jpeg/Exif.hxx + 'vcl/sysdata.hxx', + ] + if module == 'xmloff': + ignore_list += [ + 'SchXMLExport.hxx', # SchXMLAutoStylePoolP.hxx not found + 'SchXMLImport.hxx', # enums redefined in draw\sdxmlimp_impl.hxx + 'XMLEventImportHelper.hxx', # NameMap redefined in XMLEventExport.hxx + 'xmloff/XMLEventExport.hxx', # enums redefined + ] + if module == 'xmlsecurity': + ignore_list += [ + 'xmlsec/*', + 'xmlsecurity/xmlsec-wrapper.h', + ] + if module == 'external/pdfium': + ignore_list += [ + 'third_party/freetype/include/pstables.h', + ] + if module == 'external/clucene': + ignore_list += [ + '_bufferedstream.h', + '_condition.h', + '_gunichartables.h', + '_threads.h', + 'error.h', + 'CLucene/LuceneThreads.h', + 'CLucene/config/_threads.h', + ] + if module == 'external/skia': + ignore_list += [ + 'skcms_internal.h', + 'zlib.h', # causes crc32 conflict + 'dirent.h', # unix-specific + 'pthread.h', + 'unistd.h', + 'sys/stat.h', + 'ft2build.h', + 'fontconfig/fontconfig.h', + 'GL/glx.h', + 'src/Transform_inl.h', + 'src/c/sk_c_from_to.h', + 'src/c/sk_types_priv.h', + 'src/core/SkBlitBWMaskTemplate.h', + 'src/sfnt/SkSFNTHeader.h', + 'src/opts/', + 'src/core/SkCubicSolver.h', + 'src/sksl/SkSLCPP.h', + 'src/gpu/vk/GrVkAMDMemoryAllocator.h', + 'src/gpu/GrUtil.h', + 'src/sksl/', # conflict between SkSL::Expression and SkSL::dsl::Expression + 'include/sksl/', + 'src/gpu/vk/', + 'include/gpu/vk' + ] + if module == 'external/zxing': + ignore_list += [ + 'rss/ODRSSExpandedBinaryDecoder.h' + ] + + for i in ignore_list: + if line.startswith(i): + return '' + if i[0] == '*' and line.endswith(i[1:]): + return '' + if i[-1] == '*' and line.startswith(i[:-1]): + return '' + + return line + +def fixup(includes, module): + """ Here we add any headers + necessary in the pch. + These could be known to be very + common but for technical reasons + left out of the pch by this generator. + Or, they could be missing from the + source files where they are used + (probably because they had been + in the old pch, they were missed). + Also, these could be headers + that make the build faster but + aren't added automatically. + """ + fixes = [] + def append(inc): + # Add a space to exclude from + # ignore bisecting. + line = ' #include <{}>'.format(inc) + try: + i = fixes.index(inc) + fixes[i] = inc + except: + fixes.append(inc) + + append('sal/config.h') + + if module == 'basctl': + if 'basslots.hxx' in includes: + append('sfx2/msg.hxx') + + #if module == 'sc': + # if 'scslots.hxx' in includes: + # append('sfx2/msg.hxx') + return fixes + +def sort_by_category(list, root, module, filter_local): + """ Move all 'system' headers first. + Core files of osl, rtl, sal, next. + Everything non-module-specific third. + Last, module-specific headers. + """ + sys = [] + boo = [] + cor = [] + rst = [] + mod = [] + + prefix = '<' + module + '/' + for i in list: + if 'sal/config.h' in i: + continue # added unconditionally in fixup + if is_c_runtime(i, root, module): + sys.append(i) + elif '<boost/' in i: + boo.append(i) + elif prefix in i or not '/' in i: + mod.append(i) + elif '<sal/' in i or '<vcl/' in i: + cor.append(i) + elif '<osl/' in i or '<rtl/' in i: + if module == "sal": # osl and rtl are also part of sal + mod.append(i) + else: + cor.append(i) + # Headers from another module that is closely tied to the module. + elif module == 'sc' and '<formula' in i: + mod.append(i) + else: + rst.append(i) + + out = [] + out += [ "#if PCH_LEVEL >= 1" ] + out += sorted(sys) + out += sorted(boo) + out += [ "#endif // PCH_LEVEL >= 1" ] + out += [ "#if PCH_LEVEL >= 2" ] + out += sorted(cor) + out += [ "#endif // PCH_LEVEL >= 2" ] + out += [ "#if PCH_LEVEL >= 3" ] + out += sorted(rst) + out += [ "#endif // PCH_LEVEL >= 3" ] + out += [ "#if PCH_LEVEL >= 4" ] + out += sorted(mod) + out += [ "#endif // PCH_LEVEL >= 4" ] + return out + +def parse_makefile(groups, lines, lineno, lastif, ifstack): + + inobjects = False + ingeneratedobjects = False + inelse = False + suffix = 'cxx' + os_cond_re = re.compile(r'(ifeq|ifneq)\s*\(\$\(OS\)\,(\w*)\)') + + line = lines[lineno] + if line.startswith('if'): + lastif = line + if ifstack == 0: + # Correction if first line is an if. + lineno = parse_makefile(groups, lines, lineno, line, ifstack+1) + else: + lineno -= 1 + + while lineno + 1 < len(lines): + lineno += 1 + line = lines[lineno].strip() + line = line.rstrip('\\').strip() + #print('line #{}: {}'.format(lineno, line)) + if len(line) == 0: + continue + + if line == '))': + inobjects = False + ingeneratedobjects = False + elif 'add_exception_objects' in line or \ + 'add_cxxobject' in line: + inobjects = True + #print('inobjects') + #if ifstack and not SILENT: + #sys.stderr.write('Sources in a conditional, ignoring for now.\n') + elif 'add_generated_exception_objects' in line or \ + 'add_generated_cxxobject' in line: + ingeneratedobjects = True + elif 'set_generated_cxx_suffix' in line: + suffix_re = re.compile('.*set_generated_cxx_suffix,[^,]*,([^)]*).*') + match = suffix_re.match(line) + if match: + suffix = match.group(1) + elif line.startswith('if'): + lineno = parse_makefile(groups, lines, lineno, line, ifstack+1) + continue + elif line.startswith('endif'): + if ifstack: + return lineno + continue + elif line.startswith('else'): + inelse = True + elif inobjects or ingeneratedobjects: + if EXCLUDE_SYSTEM and ifstack: + continue + file = line + '.' + suffix + if ',' in line or '(' in line or ')' in line or file.startswith('-'): + #print('passing: ' + line) + pass # $if() probably, or something similar + else: + osname = '' + if lastif: + if 'filter' in lastif: + # We can't grok filter, yet. + continue + match = os_cond_re.match(lastif) + if not match: + # We only support OS conditionals. + continue + in_out = match.group(1) + osname = match.group(2) if match else '' + if (in_out == 'ifneq' and not inelse) or \ + (in_out == 'ifeq' and inelse): + osname = '!' + osname + + if osname not in groups: + groups[osname] = [] + if ingeneratedobjects: + file = WORKDIR + '/' + file + groups[osname].append(file) + + return groups + +def process_makefile(root, module, libname): + """ Parse a gmake makefile and extract + source filenames from it. + """ + + makefile = 'Library_{}.mk'.format(libname) + filename = os.path.join(os.path.join(root, module), makefile) + if not os.path.isfile(filename): + makefile = 'StaticLibrary_{}.mk'.format(libname) + filename = os.path.join(os.path.join(root, module), makefile) + if not os.path.isfile(filename): + sys.stderr.write('Error: Module {} has no makefile at {}.'.format(module, filename)) + + groups = {'':[], 'ANDROID':[], 'iOS':[], 'WNT':[], 'LINUX':[], 'MACOSX':[]} + + with open(filename, 'r') as f: + lines = f.readlines() + groups = parse_makefile(groups, lines, lineno=0, lastif=None, ifstack=0) + + return groups + +def is_allowed_if(line, module): + """ Check whether the given #if condition + is allowed for the given module or whether + its block should be ignored. + """ + + # remove trailing comments + line = re.sub(r'(.*) *//.*', r'\1', line) + line = line.strip() + + # Our sources always build with LIBO_INTERNAL_ONLY. + if line == "#if defined LIBO_INTERNAL_ONLY" or line == "#ifdef LIBO_INTERNAL_ONLY": + return True + # We use PCHs only for C++. + if line == "#if defined(__cplusplus)" or line == "#if defined __cplusplus": + return True + # Debug-specific code, it shouldn't hurt including it unconditionally. + if line == "#ifdef DBG_UTIL" or line == "#if OSL_DEBUG_LEVEL > 0": + return True + if module == "external/skia": + # We always set these. + if line == "#ifdef SK_VULKAN" or line == "#if SK_SUPPORT_GPU": + return True + return False + +def process_source(root, module, filename, maxdepth=0): + """ Process a source file to extract + included headers. + For now, skip on compiler directives. + maxdepth is used when processing headers + which typically have protecting ifndef. + """ + + ifdepth = 0 + lastif = '' + raw_includes = [] + allowed_ifs = [] + ifsallowed = 0 + with open(filename, 'r') as f: + for line in f: + line = line.strip() + if line.startswith('#if'): + if is_allowed_if(line, module): + allowed_ifs.append(True) + ifsallowed += 1 + else: + allowed_ifs.append(False) + lastif = line + ifdepth += 1 + elif line.startswith('#endif'): + ifdepth -= 1 + if allowed_ifs[ ifdepth ]: + ifsallowed -= 1 + else: + lastif = '#if' + del allowed_ifs[ ifdepth ] + elif line.startswith('#pragma once'): + # maxdepth == 1 means we are parsing a header file + # and are allowed one #ifdef block (the include guard), + # but in the #pragma once case do not allow that + assert maxdepth == 1 + maxdepth = 0 + elif line.startswith('#include'): + if ifdepth - ifsallowed <= maxdepth: + line = sanitize(line) + if line: + line = get_filename(line) + if line and len(line): + raw_includes.append(line) + elif not SILENT: + sys.stderr.write('#include in {} : {}\n'.format(lastif, line)) + + return raw_includes + +def explode(root, module, includes, tree, filter_local, recurse): + incpath = os.path.join(root, 'include') + + for inc in includes: + filename = get_filename(inc) + if filename in tree or len(filter_local.proc(filename)) == 0: + continue + + try: + # Module or Local header. + filepath = filter_local.find_local_file(inc) + if filepath: + #print('trying loc: ' + filepath) + incs = process_source(root, module, filepath, maxdepth=1) + incs = map(get_filename, incs) + incs = process_list(incs, lambda x: filter_ignore(x, module)) + incs = process_list(incs, filter_local.proc) + tree[filename] = incs + if recurse: + tree = explode(root, module, incs, tree, filter_local, recurse) + #print('{} => {}'.format(filepath, tree[filename])) + continue + except: + pass + + try: + # Public header. + filepath = os.path.join(incpath, filename) + #print('trying pub: ' + filepath) + incs = process_source(root, module, filepath, maxdepth=1) + incs = map(get_filename, incs) + incs = process_list(incs, lambda x: filter_ignore(x, module)) + incs = process_list(incs, filter_local.proc) + tree[filename] = incs + if recurse: + tree = explode(root, module, incs, tree, filter_local, recurse) + #print('{} => {}'.format(filepath, tree[filename])) + continue + except: + pass + + # Failed, but remember to avoid searching again. + tree[filename] = [] + + return tree + +def make_command_line(): + args = sys.argv[:] + # Remove command line flags and + # use internal flags. + for i in range(len(args)-1, 0, -1): + if args[i].startswith('--'): + args.pop(i) + + args.append('--cutoff=' + str(CUTOFF)) + if EXCLUDE_SYSTEM: + args.append('--exclude:system') + else: + args.append('--include:system') + if EXCLUDE_MODULE: + args.append('--exclude:module') + else: + args.append('--include:module') + if EXCLUDE_LOCAL: + args.append('--exclude:local') + else: + args.append('--include:local') + + return ' '.join(args) + +def generate_includes(includes): + """Generates the include lines of the pch. + """ + lines = [] + for osname, group in includes.items(): + if not len(group): + continue + + if len(osname): + not_eq = '' + if osname[0] == '!': + not_eq = '!' + osname = osname[1:] + lines.append('') + lines.append('#if {}defined({})'.format(not_eq, osname)) + + for i in group: + lines.append(i) + + if len(osname): + lines.append('#endif') + + return lines + +def generate(includes, libname, filename, module): + header = \ +"""/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* + This file has been autogenerated by update_pch.sh. It is possible to edit it + manually (such as when an include file has been moved/renamed/removed). All such + manual changes will be rewritten by the next run of update_pch.sh (which presumably + also fixes all possible problems, so it's usually better to use it). +""" + + footer = \ +""" +/* vim:set shiftwidth=4 softtabstop=4 expandtab: */ +""" + import datetime + + with open(filename, 'w') as f: + f.write(header) + f.write('\n Generated on {} using:\n {}\n'.format( + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + make_command_line())) + f.write('\n If after updating build fails, use the following command to locate conflicting headers:\n ./bin/update_pch_bisect {} "make {}.build" --find-conflicts\n*/\n'.format( + filename, module)) + + # sal needs this for rand_s() + if module == 'sal' and libname == 'sal': + sal_define = """ +#if defined(_WIN32) +#define _CRT_RAND_S +#endif +""" + f.write(sal_define) + + # Dump the headers. + f.write('\n') + for i in includes: + f.write(i + '\n') + + # Some libraries pull windows headers that aren't self contained. + if (module == 'connectivity' and libname == 'ado') or \ + (module == 'xmlsecurity' and libname == 'xsec_xmlsec'): + ado_define = """ +// Cleanup windows header macro pollution. +#if defined(_WIN32) && defined(WINAPI) +#include <postwin.h> +#undef RGB +#endif +""" + f.write(ado_define) + + f.write(footer) + +def remove_from_tree(filename, tree): + # Remove this file, if top-level. + incs = tree.pop(filename, []) + for i in incs: + tree = remove_from_tree(i, tree) + + # Also remove if included from another. + for (k, v) in tree.items(): + if filename in v: + v.remove(filename) + + return tree + +def tree_to_list(includes, filename, tree): + if filename in includes: + return includes + includes.append(filename) + #incs = tree.pop(filename, []) + incs = tree[filename] if filename in tree else [] + for i in incs: + tree_to_list(includes, i, tree) + + return includes + +def promote(includes): + """ Common library headers are heavily + referenced, even if they are included + from a few places. + Here we separate them to promote + their inclusion in the final pch. + """ + promo = [] + for inc in includes: + if inc.startswith('boost') or \ + inc.startswith('sal') or \ + inc.startswith('osl') or \ + inc.startswith('rtl'): + promo.append(inc) + return promo + +def make_pch_filename(root, module, libname): + """ PCH files are stored here: + <root>/<module>/inc/pch/precompiled_<libname>.hxx + """ + + path = os.path.join(root, module) + path = os.path.join(path, 'inc') + path = os.path.join(path, 'pch') + path = os.path.join(path, 'precompiled_' + libname + '.hxx') + return path + +def main(): + + global CUTOFF + global EXCLUDE_MODULE + global EXCLUDE_LOCAL + global EXCLUDE_SYSTEM + global SILENT + global WORKDIR + + if os.getenv('WORKDIR'): + WORKDIR = os.getenv('WORKDIR') + + root = '.' + module = sys.argv[1] + libname = sys.argv[2] + header = make_pch_filename(root, module, libname) + + if not os.path.exists(os.path.join(root, module)): + raise Exception('Error: module [{}] not found.'.format(module)) + + key = '{}.{}'.format(module, libname) + if key in DEFAULTS: + # Load the module-specific defaults. + CUTOFF = DEFAULTS[key][0] + EXCLUDE_SYSTEM = DEFAULTS[key][1] + EXCLUDE_MODULE = DEFAULTS[key][2] + EXCLUDE_LOCAL = DEFAULTS[key][3] + + force_update = False + for x in range(3, len(sys.argv)): + i = sys.argv[x] + if i.startswith('--cutoff='): + CUTOFF = int(i.split('=')[1]) + elif i.startswith('--exclude:'): + cat = i.split(':')[1] + if cat == 'module': + EXCLUDE_MODULE = True + elif cat == 'local': + EXCLUDE_LOCAL = True + elif cat == 'system': + EXCLUDE_SYSTEM = True + elif i.startswith('--include:'): + cat = i.split(':')[1] + if cat == 'module': + EXCLUDE_MODULE = False + elif cat == 'local': + EXCLUDE_LOCAL = False + elif cat == 'system': + EXCLUDE_SYSTEM = False + elif i == '--silent': + SILENT = True + elif i == '--force': + force_update = True + else: + sys.stderr.write('Unknown option [{}].'.format(i)) + return 1 + + filter_local = Filter_Local(root, module, \ + not EXCLUDE_MODULE, \ + not EXCLUDE_LOCAL) + + # Read input. + groups = process_makefile(root, module, libname) + + generic = [] + for osname, group in groups.items(): + if not len(group): + continue + + includes = [] + for filename in group: + includes += process_source(root, module, filename) + + # Save unique top-level includes. + unique = set(includes) + promoted = promote(unique) + + # Process includes. + includes = remove_rare(includes) + includes = process_list(includes, lambda x: filter_ignore(x, module)) + includes = process_list(includes, filter_local.proc) + + # Remove the already included ones. + for inc in includes: + unique.discard(inc) + + # Explode the excluded ones. + tree = {i:[] for i in includes} + tree = explode(root, module, unique, tree, filter_local, not EXCLUDE_MODULE) + + # Remove the already included ones from the tree. + for inc in includes: + filename = get_filename(inc) + tree = remove_from_tree(filename, tree) + + extra = [] + for (k, v) in tree.items(): + extra += tree_to_list([], k, tree) + + promoted += promote(extra) + promoted = process_list(promoted, lambda x: filter_ignore(x, module)) + promoted = process_list(promoted, filter_local.proc) + promoted = set(promoted) + # If a promoted header includes others, remove the rest. + for (k, v) in tree.items(): + if k in promoted: + for i in v: + promoted.discard(i) + includes += [x for x in promoted] + + extra = remove_rare(extra) + extra = process_list(extra, lambda x: filter_ignore(x, module)) + extra = process_list(extra, filter_local.proc) + includes += extra + + includes = [x for x in set(includes)] + fixes = fixup(includes, module) + fixes = map(lambda x: '#include <' + x + '>', fixes) + + includes = map(lambda x: '#include <' + x + '>', includes) + sorted = sort_by_category(includes, root, module, filter_local) + includes = list(fixes) + sorted + + if len(osname): + for i in generic: + if i in includes: + includes.remove(i) + + groups[osname] = includes + if not len(osname): + generic = includes + + # Open the old pch and compare its contents + # with new includes. + # Clobber only if they are different. + with open(header, 'r') as f: + old_pch_lines = [x.strip() for x in f.readlines()] + new_lines = generate_includes(groups) + # Find the first include in the old pch. + start = -1 + for i in range(len(old_pch_lines)): + if old_pch_lines[i].startswith('#include') or old_pch_lines[i].startswith('#if PCH_LEVEL'): + start = i + break + # Clobber if there is a mismatch. + if force_update or start < 0 or (len(old_pch_lines) - start < len(new_lines)): + generate(new_lines, libname, header, module) + return 0 + else: + for i in range(len(new_lines)): + if new_lines[i] != old_pch_lines[start + i]: + generate(new_lines, libname, header, module) + return 0 + else: + # Identical, but see if new pch removed anything. + for i in range(start + len(new_lines), len(old_pch_lines)): + if '#include' in old_pch_lines[i]: + generate(new_lines, libname, header, module) + return 0 + + # Didn't update. + # Use exit code 2 to distinguish it from exit code 1 used e.g. when an exception occurs. + return 2 + +if __name__ == '__main__': + """ Process all the includes in a Module + to make into a PCH file. + Run without arguments for unittests, + and to see usage. + """ + + if len(sys.argv) >= 3: + status = main() + sys.exit(status) + + print('Usage: {} <Module name> <Library name> [options]'.format(sys.argv[0])) + print(' Always run from the root of LO repository.\n') + print(' Options:') + print(' --cutoff=<count> - Threshold to excluding headers.') + print(' --exclude:<category> - Exclude category-specific headers.') + print(' --include:<category> - Include category-specific headers.') + print(' --force - Force updating the pch even when nothing changes.') + print(' Categories:') + print(' module - Headers in /inc directory of a module.') + print(' local - Headers local to a source file.') + print(' system - Platform-specific headers.') + print(' --silent - print only errors.') + print('\nRunning unit-tests...') + + +class TestMethods(unittest.TestCase): + + def test_sanitize(self): + self.assertEqual(sanitize('#include "blah/file.cxx"'), + '#include <blah/file.cxx>') + self.assertEqual(sanitize(' #include\t"blah/file.cxx" '), + '#include <blah/file.cxx>') + self.assertEqual(sanitize(' '), + '') + + def test_filter_ignore(self): + self.assertEqual(filter_ignore('blah/file.cxx', 'mod'), + '') + self.assertEqual(filter_ignore('vector', 'mod'), + 'vector') + self.assertEqual(filter_ignore('file.cxx', 'mod'), + '') + + def test_remove_rare(self): + self.assertEqual(remove_rare([]), + []) + +class TestMakefileParser(unittest.TestCase): + + def setUp(self): + global EXCLUDE_SYSTEM + EXCLUDE_SYSTEM = False + + def test_parse_singleline_eval(self): + source = "$(eval $(call gb_Library_Library,sal))" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + + def test_parse_multiline_eval(self): + source = """$(eval $(call gb_Library_set_include,sal,\\ + $$(INCLUDE) \\ + -I$(SRCDIR)/sal/inc \\ +)) +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + + def test_parse_multiline_eval_with_if(self): + source = """$(eval $(call gb_Library_add_defs,sal,\\ + $(if $(filter $(OS),iOS), \\ + -DNO_CHILD_PROCESSES \\ + ) \\ +)) +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + + def test_parse_multiline_add_with_if(self): + source = """$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/time \\ + $(if $(filter DESKTOP,$(BUILD_TYPE)), sal/osl/unx/salinit) \\ +)) +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 1) + self.assertEqual(groups[''][0], 'sal/osl/unx/time.cxx') + + def test_parse_if_else(self): + source = """ifeq ($(OS),MACOSX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/mac/mac \\ +)) +else +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/uunxapi \\ +)) +endif +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 3) + self.assertEqual(len(groups['']), 0) + self.assertEqual(len(groups['MACOSX']), 1) + self.assertEqual(len(groups['!MACOSX']), 1) + self.assertEqual(groups['MACOSX'][0], 'sal/osl/mac/mac.cxx') + self.assertEqual(groups['!MACOSX'][0], 'sal/osl/unx/uunxapi.cxx') + + def test_parse_nested_if(self): + source = """ifeq ($(OS),MACOSX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/mac/mac \\ +)) +else +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/uunxapi \\ +)) + +ifeq ($(OS),LINUX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/textenc/context \\ +)) +endif +endif +""" + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 4) + self.assertEqual(len(groups['']), 0) + self.assertEqual(len(groups['MACOSX']), 1) + self.assertEqual(len(groups['!MACOSX']), 1) + self.assertEqual(len(groups['LINUX']), 1) + self.assertEqual(groups['MACOSX'][0], 'sal/osl/mac/mac.cxx') + self.assertEqual(groups['!MACOSX'][0], 'sal/osl/unx/uunxapi.cxx') + self.assertEqual(groups['LINUX'][0], 'sal/textenc/context.cxx') + + def test_parse_exclude_system(self): + source = """ifeq ($(OS),MACOSX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/mac/mac \\ +)) +else +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/uunxapi \\ +)) + +ifeq ($(OS),LINUX) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/textenc/context \\ +)) +endif +endif +""" + global EXCLUDE_SYSTEM + EXCLUDE_SYSTEM = True + + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + + def test_parse_filter(self): + source = """ifneq ($(filter $(OS),MACOSX iOS),) +$(eval $(call gb_Library_add_exception_objects,sal,\\ + sal/osl/unx/osxlocale \\ +)) +endif +""" + # Filter is still unsupported. + lines = source.split('\n') + groups = {'':[]} + groups = parse_makefile(groups, lines, 0, None, 0) + self.assertEqual(len(groups), 1) + self.assertEqual(len(groups['']), 0) + +unittest.main() + +# vim: set et sw=4 ts=4 expandtab: diff --git a/bin/update_pch.sh b/bin/update_pch.sh new file mode 100755 index 000000000..4c17a0374 --- /dev/null +++ b/bin/update_pch.sh @@ -0,0 +1,70 @@ +#! /bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Usage: update_pch.sh [<module>/inc/pch/precompiled_xxx.hxx] +# Usage: update_pch.sh [<module>] +# Invoke: make cmd cmd="./bin/update_pch.sh [..]" + +if test -n "$SRC_DIR"; then + root="$SRC_DIR" +else + root=`dirname $0` + root=`cd $root/.. >/dev/null && pwd` +fi +root=`readlink -f $root` +cd $root + +if test -z "$1"; then + headers=`ls ./*/inc/pch/precompiled_*.hxx` +else + headers="$@" +fi + +# Split the headers into an array. +IFS=' ' read -a aheaders <<< $headers +hlen=${#aheaders[@]}; +if [ $hlen -gt 1 ]; then + if [ -z "$PARALLELISM" ]; then + PARALLELISM=0 # Let xargs decide + fi + echo $headers | xargs -n 1 -P $PARALLELISM $0 + exit $? +fi + +for x in $headers; do + if [ -d "$x" ]; then + # We got a directory, find pch files to update. + headers=`find $root/$x/ -type f -iname "precompiled_*.hxx"` + if test -n "$headers"; then + $0 "$headers" + fi + else + header=$x + update_msg=`echo $header | sed -e s%$root/%%` + module=`readlink -f $header | sed -e s%$root/%% -e s%/.*%%` + if [ "$module" = "pch" ]; then + continue # PCH's in pch/inc/pch/ are handled manually + fi + echo updating $update_msg + if [ "$module" = "external" ]; then + module=external/`readlink -f $header | sed -e s%$root/external/%% -e s%/.*%%` + fi + libname=`echo $header | sed -e s/.*precompiled_// -e s/\.hxx//` + + ./bin/update_pch "$module" "$libname" + exitcode=$? + if test $exitcode -ne 0 -a $exitcode -ne 2; then + echo Failed. + exit 1 + fi + fi +done + +#echo Done. +exit 0 diff --git a/bin/update_pch_autotune.sh b/bin/update_pch_autotune.sh new file mode 100755 index 000000000..806e1ad17 --- /dev/null +++ b/bin/update_pch_autotune.sh @@ -0,0 +1,229 @@ +#! /bin/bash +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +# Finds the optimal update_pch settings that results in, +# per module and library, the fastest build time and +# smallest intermediate files (.o/.obj) output. + +# Usage: update_pch_autotune.sh [<module1> <module2>] +# Invoke: /opt/lo/bin/make cmd cmd="./bin/update_pch_autotune.sh [..]" + +# The resulting values may be entered in update_pch +# to be use for generating PCH in the future. +# Run this script after major header changes. + +root=`dirname $0` +root=`cd $root/.. && pwd` +cd $root + +if test -z "$1"; then + modules=`ls ./*/inc/pch/precompiled_*.hxx | sed -e s%./%% -e s%/.*%% | uniq` +else + modules="$@" +fi + +if [[ "$OSTYPE" == "cygwin" ]]; then + MAKE=/opt/lo/bin/make +else + MAKE=make +fi + +function build() +{ + local START=$(date +%s.%N) + + $MAKE -sr "$module" > /dev/null + status=$? + if [ $status -ne 0 ]; + then + # Spurious failures happen. + $MAKE "$module.build" > /dev/null + status=$? + fi + + local END=$(date +%s.%N1) + build_time=$(printf %.1f $(echo "$END - $START" | bc)) + + size="FAILED" + score="FAILED" + if [ $status -eq 0 ]; + then + # The total size of the object files. + size="$(du -s workdir/CxxObject/$module/ | awk '{print $1}')" + # Add the pch file size. + filename_rel="workdir/PrecompiledHeader/nodebug/$(basename $header)*" + filename_dbg="workdir/PrecompiledHeader/debug/$(basename $header)*" + if [[ $filename_rel -nt $filename_dbg ]]; then + pch_size="$(du -s $filename_rel | awk '{print $1}' | paste -sd+ | bc)" + else + pch_size="$(du -s $filename_dbg | awk '{print $1}' | paste -sd+ | bc)" + fi + size="$(echo "$pch_size + $size" | bc)" + + # Compute a score based on the build time and size. + # The shorter the build time, and smaller disk usage, the higher the score. + score=$(printf %.2f $(echo "10000 / ($build_time * e($size/1048576))" | bc -l)) + fi +} + +function run() +{ + local msg="$module.$libname, ${@:3}, " + printf "$msg" + ./bin/update_pch "$module" "$libname" "${@:3}" --silent + status=$? + + if [ $status -eq 0 ]; + then + build + + summary="$build_time, $size, $score" + if [ $status -eq 0 ]; + then + new_best_for_cuttof=$(echo "$score > $best_score_for_cuttof" | bc -l) + if [ $new_best_for_cuttof -eq 1 ]; + then + best_score_for_cuttof=$score + fi + + new_best=$(echo "$score > $best_score" | bc -l) + if [ $new_best -eq 1 ]; + then + best_score=$score + best_args="${@:3}" + best_time=$build_time + best_cutoff=$cutoff + summary="$build_time, $size, $score,*" + fi + fi + else + # Skip if pch is not updated. + summary="0, 0, 0" + fi + + echo "$summary" +} + +function args_to_table() +{ + local sys="EXCLUDE" + local mod="EXCLUDE" + local loc="EXCLUDE" + local cutoff=0 + IFS=' ' read -r -a aargs <<< $best_args + for index in "${!aargs[@]}" + do + if [ "${aargs[index]}" = "--include:system" ]; + then + sys="INCLUDE" + elif [ "${aargs[index]}" = "--exclude:system" ]; + then + sys="EXCLUDE" + elif [ "${aargs[index]}" = "--include:module" ]; + then + mod="INCLUDE" + elif [ "${aargs[index]}" = "--exclude:module" ]; + then + mod="EXCLUDE" + elif [ "${aargs[index]}" = "--include:local" ]; + then + loc="INCLUDE" + elif [ "${aargs[index]}" = "--exclude:local" ]; + then + loc="EXCLUDE" + elif [[ "${aargs[index]}" == *"cutoff"* ]] + then + cutoff=$(echo "${aargs[index]}" | grep -Po '\-\-cutoff\=\K\d+') + fi + done + + local key=$(printf "'%s.%s'" $module $libname) + echo "$(printf " %-36s: (%2d, %s, %s, %s), # %5.1f" $key $cutoff $sys $mod $loc $best_time)" +} + +for module in $modules; do + + # Build without pch includes as sanity check. + #run "$root" "$module" --cutoff=999 + + # Build before updating pch. + $MAKE "$module.build" > /dev/null + if [ $? -ne 0 ]; + then + # Build with dependencies before updating pch. + echo "Failed to build $module, building known state with dependencies..." + ./bin/update_pch.sh "$module" > /dev/null + $MAKE "$module.clean" > /dev/null + $MAKE "$module.all" > /dev/null + if [ $? -ne 0 ]; + then + # Build all! + echo "Failed to build $module with dependencies, building all..." + $MAKE > /dev/null + if [ $? -ne 0 ]; + then + >&2 echo "Broken build. Please revert changes and try again." + exit 1 + fi + fi + fi + + # Find pch files in the module to update. + headers=`find $root/$module/ -type f -iname "precompiled_*.hxx"` + + # Each pch belongs to a library. + for header in $headers; do + libname=`echo $header | sed -e s/.*precompiled_// -e s/\.hxx//` + #TODO: Backup the header and restore when last tune fails. + + # Force update on first try below. + echo "Autotuning $module.$libname..." + ./bin/update_pch "$module" "$libname" --cutoff=999 --silent --force + + best_score=0 + best_args="" + best_time=0 + best_cutoff=0 + for i in {1..16}; do + cutoff=$i + best_score_for_cuttof=0 + #run "$root" "$module" "--cutoff=$i" --include:system --exclude:module --exclude:local + run "$root" "$module" "--cutoff=$i" --exclude:system --exclude:module --exclude:local + #run "$root" "$module" "--cutoff=$i" --include:system --include:module --exclude:local + run "$root" "$module" "--cutoff=$i" --exclude:system --include:module --exclude:local + #run "$root" "$module" "--cutoff=$i" --include:system --exclude:module --include:local + run "$root" "$module" "--cutoff=$i" --exclude:system --exclude:module --include:local + #run "$root" "$module" "--cutoff=$i" --include:system --include:module --include:local + run "$root" "$module" "--cutoff=$i" --exclude:system --include:module --include:local + + if [ $i -gt $((best_cutoff+2)) ]; + then + score_too_low=$(echo "$best_score_for_cuttof < $best_score / 1.10" | bc -l) + if [ $score_too_low -eq 1 ]; + then + echo "Score hit low of $best_score_for_cuttof, well below overall best of $best_score. Stopping." + break; + fi + fi + done + + ./bin/update_pch "$module" "$libname" $best_args --force --silent + echo "> $module.$libname, $best_args, $best_time, $size, $score" + echo + + table+=$'\n' + table+="$(args_to_table)" + done + +done + +echo "Update the relevant lines in ./bin/update_pch script:" +>&2 echo "$table" + +exit 0 diff --git a/bin/update_pch_bisect b/bin/update_pch_bisect new file mode 100755 index 000000000..271cbc88f --- /dev/null +++ b/bin/update_pch_bisect @@ -0,0 +1,354 @@ +#! /usr/bin/env python +# -*- Mode: python; tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +""" +This script is to fix precompiled headers. + +This script runs in two modes. +In one mode, it starts with a header +that doesn't compile. If finds the +minimum number of includes in the +header to remove to get a successful +run of the command (i.e. compile). + +In the second mode, it starts with a +header that compiles fine, however, +it contains one or more required +include without which it wouldn't +compile, which it identifies. + +Usage: ./bin/update_pch_bisect ./vcl/inc/pch/precompiled_vcl.hxx "make vcl.build" --find-required --verbose +""" + +import sys +import re +import os +import unittest +import subprocess + +SILENT = True +FIND_CONFLICTS = True + +IGNORE = 0 +GOOD = 1 +TEST_ON = 2 +TEST_OFF = 3 +BAD = 4 + +def run(command): + try: + cmd = command.split(' ', 1) + status = subprocess.call(cmd, stdout=open(os.devnull, 'w'), + stderr=subprocess.STDOUT, close_fds=True) + return True if status == 0 else False + except Exception as e: + sys.stderr.write('Error: {}\n'.format(e)) + return False + +def update_pch(filename, lines, marks): + with open(filename, 'w') as f: + for i in xrange(len(marks)): + mark = marks[i] + if mark <= TEST_ON: + f.write(lines[i]) + else: + f.write('//' + lines[i]) + +def log(*args, **kwargs): + global SILENT + if not SILENT: + print(*args, **kwargs) + +def bisect(lines, marks, min, max, update, command): + """ Disable half the includes and + calls the command. + Depending on the result, + recurse or return. + """ + global FIND_CONFLICTS + + log('Bisecting [{}, {}].'.format(min+1, max)) + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = TEST_ON if FIND_CONFLICTS else TEST_OFF + + assume_fail = False + if not FIND_CONFLICTS: + on_list = [x for x in marks if x in (TEST_ON, GOOD)] + assume_fail = (len(on_list) == 0) + + update(lines, marks) + if assume_fail or not command(): + # Failed + log('Failed [{}, {}].'.format(min+1, max)) + if min >= max - 1: + if not FIND_CONFLICTS: + # Try with this one alone. + marks[min] = TEST_ON + update(lines, marks) + if command(): + log(' Found @{}: {}'.format(min+1, lines[min].strip('\n'))) + marks[min] = GOOD + return marks + else: + log(' Found @{}: {}'.format(min+1, lines[min].strip('\n'))) + # Either way, this one is irrelevant. + marks[min] = BAD + return marks + + # Bisect + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = TEST_OFF if FIND_CONFLICTS else TEST_ON + + half = min + ((max - min) / 2) + marks = bisect(lines, marks, min, half, update, command) + marks = bisect(lines, marks, half, max, update, command) + else: + # Success + if FIND_CONFLICTS: + log(' Good [{}, {}].'.format(min+1, max)) + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = GOOD + + return marks + +def get_filename(line): + """ Strips the line from the + '#include' and angled brackets + and return the filename only. + """ + return re.sub(r'(.*#include\s*)<(.*)>(.*)', r'\2', line) + +def get_marks(lines): + marks = [] + min = -1 + max = -1 + for i in xrange(len(lines)): + line = lines[i] + if line.startswith('#include'): + marks.append(TEST_ON) + min = i if min < 0 else min + max = i + else: + marks.append(IGNORE) + + return (marks, min, max+1) + +def main(): + + global FIND_CONFLICTS + global SILENT + + filename = sys.argv[1] + command = sys.argv[2] + + for i in range(3, len(sys.argv)): + opt = sys.argv[i] + if opt == '--find-conflicts': + FIND_CONFLICTS = True + elif opt == '--find-required': + FIND_CONFLICTS = False + elif opt == '--verbose': + SILENT = False + else: + sys.stderr.write('Error: Unknown option [{}].\n'.format(opt)) + return 1 + + lines = [] + with open(filename) as f: + lines = f.readlines() + + (marks, min, max) = get_marks(lines) + + # Test preconditions. + log('Validating all-excluded state...') + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = TEST_OFF + update_pch(filename, lines, marks) + res = run(command) + + if FIND_CONFLICTS: + # Must build all excluded. + if not res: + sys.stderr.write("Error: broken state when all excluded, fix first and try again.") + return 1 + else: + # If builds all excluded, we can't bisect. + if res: + sys.stderr.write("Done: in good state when all excluded, nothing to do.") + return 1 + + # Must build all included. + log('Validating all-included state...') + for i in range(min, max): + if marks[i] != IGNORE: + marks[i] = TEST_ON + update_pch(filename, lines, marks) + if not run(command): + sys.stderr.write("Error: broken state without modifying, fix first and try again.") + return 1 + + marks = bisect(lines, marks, min, max+1, + lambda l, m: update_pch(filename, l, m), + lambda: run(command)) + if not FIND_CONFLICTS: + # Simplify further, as sometimes we can have + # false positives due to the benign nature + # of includes that are not absolutely required. + for i in xrange(len(marks)): + if marks[i] == GOOD: + marks[i] = TEST_OFF + update_pch(filename, lines, marks) + if not run(command): + # Revert. + marks[i] = GOOD + else: + marks[i] = BAD + elif marks[i] == TEST_OFF: + marks[i] = TEST_ON + + update_pch(filename, lines, marks) + + log('') + for i in xrange(len(marks)): + if marks[i] == (BAD if FIND_CONFLICTS else GOOD): + print("'{}',".format(get_filename(lines[i].strip('\n')))) + + return 0 + +if __name__ == '__main__': + + if len(sys.argv) in (3, 4, 5): + status = main() + sys.exit(status) + + print('Usage: {} <pch> <command> [--find-conflicts]|[--find-required] [--verbose]\n'.format(sys.argv[0])) + print(' --find-conflicts - Finds all conflicting includes. (Default)') + print(' Must compile without any includes.\n') + print(' --find-required - Finds all required includes.') + print(' Must compile with all includes.\n') + print(' --verbose - print noisy progress.') + print('Example: ./bin/update_pch_bisect ./vcl/inc/pch/precompiled_vcl.hxx "make vcl.build" --find-required --verbose') + print('\nRunning unit-tests...') + + +class TestBisectConflict(unittest.TestCase): + TEST = """ /* Test header. */ +#include <memory> +#include <set> +#include <algorithm> +#include <vector> +/* blah blah */ +""" + BAD_LINE = "#include <bad>" + + def setUp(self): + global FIND_CONFLICTS + FIND_CONFLICTS = True + + def _update_func(self, lines, marks): + self.lines = [] + for i in xrange(len(marks)): + mark = marks[i] + if mark <= TEST_ON: + self.lines.append(lines[i]) + else: + self.lines.append('//' + lines[i]) + + def _test_func(self): + """ Command function called by bisect. + Returns True on Success, False on failure. + """ + # If the bad line is still there, fail. + return self.BAD_LINE not in self.lines + + def test_success(self): + lines = self.TEST.split('\n') + (marks, min, max) = get_marks(lines) + marks = bisect(lines, marks, min, max, + lambda l, m: self._update_func(l, m), + lambda: self._test_func()) + self.assertTrue(BAD not in marks) + + def test_conflict(self): + lines = self.TEST.split('\n') + for pos in xrange(len(lines) + 1): + lines = self.TEST.split('\n') + lines.insert(pos, self.BAD_LINE) + (marks, min, max) = get_marks(lines) + + marks = bisect(lines, marks, min, max, + lambda l, m: self._update_func(l, m), + lambda: self._test_func()) + for i in xrange(len(marks)): + if i == pos: + self.assertEqual(BAD, marks[i]) + else: + self.assertNotEqual(BAD, marks[i]) + +class TestBisectRequired(unittest.TestCase): + TEST = """#include <algorithm> +#include <set> +#include <map> +#include <vector> +""" + REQ_LINE = "#include <req>" + + def setUp(self): + global FIND_CONFLICTS + FIND_CONFLICTS = False + + def _update_func(self, lines, marks): + self.lines = [] + for i in xrange(len(marks)): + mark = marks[i] + if mark <= TEST_ON: + self.lines.append(lines[i]) + else: + self.lines.append('//' + lines[i]) + + def _test_func(self): + """ Command function called by bisect. + Returns True on Success, False on failure. + """ + # If the required line is not there, fail. + found = self.REQ_LINE in self.lines + return found + + def test_success(self): + lines = self.TEST.split('\n') + (marks, min, max) = get_marks(lines) + marks = bisect(lines, marks, min, max, + lambda l, m: self._update_func(l, m), + lambda: self._test_func()) + self.assertTrue(GOOD not in marks) + + def test_required(self): + lines = self.TEST.split('\n') + for pos in xrange(len(lines) + 1): + lines = self.TEST.split('\n') + lines.insert(pos, self.REQ_LINE) + (marks, min, max) = get_marks(lines) + + marks = bisect(lines, marks, min, max, + lambda l, m: self._update_func(l, m), + lambda: self._test_func()) + for i in xrange(len(marks)): + if i == pos: + self.assertEqual(GOOD, marks[i]) + else: + self.assertNotEqual(GOOD, marks[i]) + +unittest.main() + +# vim: set et sw=4 ts=4 expandtab: |