Merging upstream version 1:2.45.1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

7 files changed, 215 insertions, 62 deletions

diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh
index 4f40753..b4e22de 100755
--- a/ci/install-dependencies.sh
+++ b/ci/install-dependencies.sh
@@ -37,15 +37,13 @@ macos-*)
 	test -z "$BREW_INSTALL_PACKAGES" ||
 	brew install $BREW_INSTALL_PACKAGES
 	brew link --force gettext
-	mkdir -p $HOME/bin
-	(
-		cd $HOME/bin
+
+	mkdir -p "$P4_PATH"
+	pushd "$P4_PATH"
 		wget -q "$P4WHENCE/bin.macosx1015x86_64/helix-core-server.tgz" &&
 		tar -xf helix-core-server.tgz &&
 		sudo xattr -d com.apple.quarantine p4 p4d 2>/dev/null || true
-	)
-	PATH="$PATH:${HOME}/bin"
-	export PATH
+	popd
 
 	if test -n "$CC_PACKAGE"
 	then
diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh
index 78b7e32..eb2c9e1 100755
--- a/ci/install-docker-dependencies.sh
+++ b/ci/install-docker-dependencies.sh
@@ -3,6 +3,10 @@
 # Install dependencies required to build and test Git inside container
 #
 
+. ${0%/*}/lib.sh
+
+begin_group "Install dependencies"
+
 case "$jobname" in
 linux32)
 	linux32 --32bit i386 sh -c '
@@ -12,11 +16,31 @@ linux32)
 	'
 	;;
 linux-musl)
-	apk add --update build-base curl-dev openssl-dev expat-dev gettext \
-		pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null
+	apk add --update shadow sudo build-base curl-dev openssl-dev expat-dev gettext \
+		pcre2-dev python3 musl-libintl perl-utils ncurses \
+		apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \
+		bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null
+	;;
+linux-*|StaticAnalysis)
+	# Required so that apt doesn't wait for user input on certain packages.
+	export DEBIAN_FRONTEND=noninteractive
+
+	apt update -q &&
+	apt install -q -y sudo git make language-pack-is libsvn-perl apache2 libssl-dev \
+		libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev \
+		perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl \
+		libdbd-sqlite3-perl libio-socket-ssl-perl libnet-smtp-ssl-perl ${CC_PACKAGE:-${CC:-gcc}} \
+		apache2 cvs cvsps gnupg libcgi-pm-perl subversion
+
+	if test "$jobname" = StaticAnalysis
+	then
+		apt install -q -y coccinelle
+	fi
 	;;
 pedantic)
 	dnf -yq update >/dev/null &&
 	dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
 	;;
 esac
+
+end_group "Install dependencies"
diff --git a/ci/lib.sh b/ci/lib.sh
index bc0b230..0a73fc7 100755
--- a/ci/lib.sh
+++ b/ci/lib.sh
@@ -1,16 +1,7 @@
 # Library of functions shared by all CI scripts
 
-if test true != "$GITHUB_ACTIONS"
+if test true = "$GITHUB_ACTIONS"
 then
-	begin_group () { :; }
-	end_group () { :; }
-
-	group () {
-		shift
-		"$@"
-	}
-	set -x
-else
 	begin_group () {
 		need_to_end_group=t
 		echo "::group::$1" >&2
@@ -23,27 +14,50 @@ else
 		need_to_end_group=
 		echo '::endgroup::' >&2
 	}
-	trap end_group EXIT
+elif test true = "$GITLAB_CI"
+then
+	begin_group () {
+		need_to_end_group=t
+		printf "\e[0Ksection_start:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K$1\n"
+		trap "end_group '$1'" EXIT
+		set -x
+	}
 
-	group () {
+	end_group () {
+		test -n "$need_to_end_group" || return 0
 		set +x
-		begin_group "$1"
-		shift
-		# work around `dash` not supporting `set -o pipefail`
-		(
-			"$@" 2>&1
-			echo $? >exit.status
-		) |
-		sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/'
-		res=$(cat exit.status)
-		rm exit.status
-		end_group
-		return $res
+		need_to_end_group=
+		printf "\e[0Ksection_end:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K\n"
+		trap - EXIT
 	}
+else
+	begin_group () { :; }
+	end_group () { :; }
 
-	begin_group "CI setup"
+	set -x
 fi
 
+group () {
+	group="$1"
+	shift
+	begin_group "$group"
+
+	# work around `dash` not supporting `set -o pipefail`
+	(
+		"$@" 2>&1
+		echo $? >exit.status
+	) |
+	sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/'
+	res=$(cat exit.status)
+	rm exit.status
+
+	end_group "$group"
+	return $res
+}
+
+begin_group "CI setup"
+trap "end_group 'CI setup'" EXIT
+
 # Set 'exit on error' for all CI scripts to let the caller know that
 # something went wrong.
 #
@@ -71,10 +85,32 @@ skip_branch_tip_with_tag () {
 	fi
 }
 
+# Check whether we can use the path passed via the first argument as Git
+# repository.
+is_usable_git_repository () {
+	# We require Git in our PATH, otherwise we cannot access repositories
+	# at all.
+	if ! command -v git >/dev/null
+	then
+		return 1
+	fi
+
+	# And the target directory needs to be a proper Git repository.
+	if ! git -C "$1" rev-parse 2>/dev/null
+	then
+		return 1
+	fi
+}
+
 # Save some info about the current commit's tree, so we can skip the build
 # job if we encounter the same tree again and can provide a useful info
 # message.
 save_good_tree () {
+	if ! is_usable_git_repository .
+	then
+		return
+	fi
+
 	echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file"
 	# limit the file size
 	tail -1000 "$good_trees_file" >"$good_trees_file".tmp
@@ -90,6 +126,11 @@ skip_good_tree () {
 		return
 	fi
 
+	if ! is_usable_git_repository .
+	then
+		return
+	fi
+
 	if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")"
 	then
 		# Haven't seen this tree yet, or no cached good trees file yet.
@@ -121,6 +162,11 @@ skip_good_tree () {
 }
 
 check_unignored_build_artifacts () {
+	if ! is_usable_git_repository .
+	then
+		return
+	fi
+
 	! git ls-files --other --exclude-standard --error-unmatch \
 		-- ':/*' 2>/dev/null ||
 	{
@@ -133,6 +179,26 @@ handle_failed_tests () {
 	return 1
 }
 
+create_failed_test_artifacts () {
+	mkdir -p t/failed-test-artifacts
+
+	for test_exit in t/test-results/*.exit
+	do
+		test 0 != "$(cat "$test_exit")" || continue
+
+		test_name="${test_exit%.exit}"
+		test_name="${test_name##*/}"
+		printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n"
+		echo "The full logs are in the 'print test failures' step below."
+		echo "See also the 'failed-tests-*' artifacts attached to this run."
+		cat "t/test-results/$test_name.markup"
+
+		trash_dir="t/trash directory.$test_name"
+		cp "t/test-results/$test_name.out" t/failed-test-artifacts/
+		tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
+	done
+}
+
 # GitHub Action doesn't set TERM, which is required by tput
 export TERM=${TERM:-dumb}
 
@@ -156,11 +222,8 @@ then
 	# among *all* phases)
 	cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME"
 
-	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
-	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
-	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test windows_nt != "$CI_OS_NAME" ||
-	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
+	GIT_TEST_OPTS="--write-junit-xml"
+	JOBS=10
 elif test true = "$GITHUB_ACTIONS"
 then
 	CI_TYPE=github-actions
@@ -173,40 +236,70 @@ then
 	CC="${CC_PACKAGE:-${CC:-gcc}}"
 	DONT_SKIP_TAGS=t
 	handle_failed_tests () {
-		mkdir -p t/failed-test-artifacts
 		echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
+		create_failed_test_artifacts
+		return 1
+	}
+
+	cache_dir="$HOME/none"
 
-		for test_exit in t/test-results/*.exit
-		do
-			test 0 != "$(cat "$test_exit")" || continue
-
-			test_name="${test_exit%.exit}"
-			test_name="${test_name##*/}"
-			printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n"
-			echo "The full logs are in the 'print test failures' step below."
-			echo "See also the 'failed-tests-*' artifacts attached to this run."
-			cat "t/test-results/$test_name.markup"
-
-			trash_dir="t/trash directory.$test_name"
-			cp "t/test-results/$test_name.out" t/failed-test-artifacts/
-			tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
-		done
+	GIT_TEST_OPTS="--github-workflow-markup"
+	JOBS=10
+elif test true = "$GITLAB_CI"
+then
+	CI_TYPE=gitlab-ci
+	CI_BRANCH="$CI_COMMIT_REF_NAME"
+	CI_COMMIT="$CI_COMMIT_SHA"
+	case "$CI_JOB_IMAGE" in
+	macos-*)
+		# GitLab CI has Python installed via multiple package managers,
+		# most notably via asdf and Homebrew. Ensure that our builds
+		# pick up the Homebrew one by prepending it to our PATH as the
+		# asdf one breaks tests.
+		export PATH="$(brew --prefix)/bin:$PATH"
+
+		CI_OS_NAME=osx
+		;;
+	alpine:*|fedora:*|ubuntu:*)
+		CI_OS_NAME=linux;;
+	*)
+		echo "Could not identify OS image" >&2
+		env >&2
+		exit 1
+		;;
+	esac
+	CI_REPO_SLUG="$CI_PROJECT_PATH"
+	CI_JOB_ID="$CI_JOB_ID"
+	CC="${CC_PACKAGE:-${CC:-gcc}}"
+	DONT_SKIP_TAGS=t
+	handle_failed_tests () {
+		create_failed_test_artifacts
 		return 1
 	}
 
 	cache_dir="$HOME/none"
 
-	export GIT_PROVE_OPTS="--timer --jobs 10"
-	export GIT_TEST_OPTS="--verbose-log -x --github-workflow-markup"
-	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test windows != "$CI_OS_NAME" ||
-	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
+	runs_on_pool=$(echo "$CI_JOB_IMAGE" | tr : -)
+	JOBS=$(nproc)
 else
 	echo "Could not identify CI type" >&2
 	env >&2
 	exit 1
 fi
 
+MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS"
+GIT_PROVE_OPTS="--timer --jobs $JOBS"
+
+GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x"
+case "$CI_OS_NAME" in
+windows|windows_nt)
+	GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers"
+	;;
+esac
+
+export GIT_TEST_OPTS
+export GIT_PROVE_OPTS
+
 good_trees_file="$cache_dir/good-trees"
 
 mkdir -p "$cache_dir"
@@ -258,6 +351,9 @@ macos-*)
 	then
 		MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes"
 	fi
+
+	P4_PATH="$HOME/custom/p4"
+	export PATH="$P4_PATH:$PATH"
 	;;
 esac
 
@@ -271,7 +367,7 @@ linux-musl)
 	MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes"
 	MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 	;;
-linux-leaks)
+linux-leaks|linux-reftable-leaks)
 	export SANITIZE=leak
 	export GIT_TEST_PASSING_SANITIZE_LEAK=true
 	export GIT_TEST_SANITIZE_LEAK_LOG=true
@@ -285,5 +381,5 @@ esac
 
 MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
 
-end_group
+end_group "CI setup"
 set -x
diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh
index 57277ee..b1f80ae 100755
--- a/ci/print-test-failures.sh
+++ b/ci/print-test-failures.sh
@@ -8,7 +8,7 @@
 # Tracing executed commands would produce too much noise in the loop below.
 set +x
 
-cd t/
+cd "${TEST_OUTPUT_DIRECTORY:-t/}"
 
 if ! ls test-results/*.exit >/dev/null 2>/dev/null
 then
@@ -51,6 +51,12 @@ do
 			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
 			continue
 			;;
+		gitlab-ci)
+			mkdir -p failed-test-artifacts
+			cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
+			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
+			continue
+			;;
 		*)
 			echo "Unhandled CI type: $CI_TYPE" >&2
 			exit 1
diff --git a/ci/run-build-and-minimal-fuzzers.sh b/ci/run-build-and-minimal-fuzzers.sh
new file mode 100755
index 0000000..a51076d
--- /dev/null
+++ b/ci/run-build-and-minimal-fuzzers.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# Build and test Git's fuzzers
+#
+
+. ${0%/*}/lib.sh
+
+group "Build fuzzers" make \
+	CC=clang \
+	CXX=clang++ \
+	CFLAGS="-fsanitize=fuzzer-no-link,address" \
+	LIB_FUZZING_ENGINE="-fsanitize=fuzzer,address" \
+	fuzz-all
+
+for fuzzer in commit-graph config date pack-headers pack-idx ; do
+	begin_group "fuzz-$fuzzer"
+	./oss-fuzz/fuzz-$fuzzer -verbosity=0 -runs=1 || exit 1
+	end_group "fuzz-$fuzzer"
+done
diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh
index 2528f25..c192bd6 100755
--- a/ci/run-build-and-tests.sh
+++ b/ci/run-build-and-tests.sh
@@ -37,6 +37,9 @@ linux-clang)
 linux-sha256)
 	export GIT_TEST_DEFAULT_HASH=sha256
 	;;
+linux-reftable|linux-reftable-leaks|osx-reftable)
+	export GIT_TEST_DEFAULT_REF_FORMAT=reftable
+	;;
 pedantic)
 	# Don't run the tests; we only care about whether Git can be
 	# built.
@@ -50,6 +53,8 @@ if test -n "$run_tests"
 then
 	group "Run tests" make test ||
 	handle_failed_tests
+	group "Run unit tests" \
+		make DEFAULT_UNIT_TEST_TARGET=unit-tests-prove unit-tests
 fi
 check_unignored_build_artifacts
 
diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh
index a3c6795..ae80943 100755
--- a/ci/run-test-slice.sh
+++ b/ci/run-test-slice.sh
@@ -15,4 +15,9 @@ group "Run tests" make --quiet -C t T="$(cd t &&
 	tr '\n' ' ')" ||
 handle_failed_tests
 
+# We only have one unit test at the moment, so run it in the first slice
+if [ "$1" == "0" ] ; then
+	group "Run unit tests" make --quiet -C t unit-tests-prove
+fi
+
 check_unignored_build_artifacts