Adding upstream version 1.1.0.upstream/1.1.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 353 insertions, 0 deletions

diff --git a/scripts/sync-kernel.sh b/scripts/sync-kernel.sh
new file mode 100755
index 0000000..eb09ef1
--- /dev/null
+++ b/scripts/sync-kernel.sh
@@ -0,0 +1,353 @@
+#!/bin/bash
+
+usage () {
+	echo "USAGE: ./sync-kernel.sh <libbpf-repo> <kernel-repo> <bpf-branch>"
+	echo ""
+	echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from <libbpf-repo>/CHECKPOINT-COMMIT."
+	echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from <libbpf-repo>/BPF-CHECKPOINT-COMMIT."
+	echo "Set MANUAL_MODE to 1 to manually control every cherry-picked commits."
+	exit 1
+}
+
+set -eu
+
+LIBBPF_REPO=${1-""}
+LINUX_REPO=${2-""}
+BPF_BRANCH=${3-""}
+BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)}
+BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat ${LIBBPF_REPO}/BPF-CHECKPOINT-COMMIT)}
+
+if [ -z "${LIBBPF_REPO}" ] || [ -z "${LINUX_REPO}" ]; then
+	echo "Error: libbpf or linux repos are not specified"
+	usage
+fi
+if [ -z "${BPF_BRANCH}" ]; then
+	echo "Error: linux's bpf tree branch is not specified"
+	usage
+fi
+if [ -z "${BASELINE_COMMIT}" ] || [ -z "${BPF_BASELINE_COMMIT}" ]; then
+	echo "Error: bpf or bpf-next baseline commits are not provided"
+	usage
+fi
+
+SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ)
+WORKDIR=$(pwd)
+TMP_DIR=$(mktemp -d)
+
+trap "cd ${WORKDIR}; exit" INT TERM EXIT
+
+declare -A PATH_MAP
+PATH_MAP=(									\
+	[tools/lib/bpf]=src							\
+	[tools/include/uapi/linux/bpf_common.h]=include/uapi/linux/bpf_common.h	\
+	[tools/include/uapi/linux/bpf.h]=include/uapi/linux/bpf.h		\
+	[tools/include/uapi/linux/btf.h]=include/uapi/linux/btf.h		\
+	[tools/include/uapi/linux/fcntl.h]=include/uapi/linux/fcntl.h		\
+	[tools/include/uapi/linux/if_link.h]=include/uapi/linux/if_link.h	\
+	[tools/include/uapi/linux/if_xdp.h]=include/uapi/linux/if_xdp.h		\
+	[tools/include/uapi/linux/netlink.h]=include/uapi/linux/netlink.h	\
+	[tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h	\
+	[tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h	\
+	[include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h	\
+	[Documentation/bpf/libbpf]=docs						\
+)
+
+LIBBPF_PATHS=("${!PATH_MAP[@]}" ":^tools/lib/bpf/Makefile" ":^tools/lib/bpf/Build" ":^tools/lib/bpf/.gitignore" ":^tools/include/tools/libc_compat.h")
+LIBBPF_VIEW_PATHS=("${PATH_MAP[@]}")
+LIBBPF_VIEW_EXCLUDE_REGEX='^src/(Makefile|Build|test_libbpf\.c|bpf_helper_defs\.h|\.gitignore)$|^docs/(\.gitignore|api\.rst|conf\.py)$|^docs/sphinx/.*'
+LINUX_VIEW_EXCLUDE_REGEX='^include/tools/libc_compat.h$'
+
+LIBBPF_TREE_FILTER="mkdir -p __libbpf/include/uapi/linux __libbpf/include/tools && "$'\\\n'
+for p in "${!PATH_MAP[@]}"; do
+	LIBBPF_TREE_FILTER+="git mv -kf ${p} __libbpf/${PATH_MAP[${p}]} && "$'\\\n'
+done
+LIBBPF_TREE_FILTER+="git rm --ignore-unmatch -f __libbpf/src/{Makefile,Build,test_libbpf.c,.gitignore} >/dev/null"
+
+cd_to()
+{
+	cd ${WORKDIR} && cd "$1"
+}
+
+# Output brief single-line commit description
+# $1 - commit ref
+commit_desc()
+{
+	git log -n1 --pretty='%h ("%s")' $1
+}
+
+# Create commit single-line signature, which consists of:
+# - full commit subject
+# - author date in ISO8601 format
+# - full commit body with newlines replaced with vertical bars (|)
+# - shortstat appended at the end
+# The idea is that this single-line signature is good enough to make final
+# decision about whether two commits are the same, across different repos.
+# $1 - commit ref
+# $2 - paths filter
+commit_signature()
+{
+	local ref=$1
+	shift
+	git show --pretty='("%s")|%aI|%b' --shortstat $ref -- "${@-.}" | tr '\n' '|'
+}
+
+# Cherry-pick commits touching libbpf-related files
+# $1 - baseline_tag
+# $2 - tip_tag
+cherry_pick_commits()
+{
+	local manual_mode=${MANUAL_MODE:-0}
+	local baseline_tag=$1
+	local tip_tag=$2
+	local new_commits
+	local signature
+	local should_skip
+	local synced_cnt
+	local manual_check
+	local libbpf_conflict_cnt
+	local desc
+
+	new_commits=$(git rev-list --no-merges --topo-order --reverse ${baseline_tag}..${tip_tag} -- "${LIBBPF_PATHS[@]}")
+	for new_commit in ${new_commits}; do
+		desc="$(commit_desc ${new_commit})"
+		signature="$(commit_signature ${new_commit} "${LIBBPF_PATHS[@]}")"
+		synced_cnt=$(grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | wc -l)
+		manual_check=0
+		if ((${synced_cnt} > 0)); then
+			# commit with the same subject is already in libbpf, but it's
+			# not 100% the same commit, so check with user
+			echo "Commit '${desc}' is synced into libbpf as:"
+			grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | \
+				cut -d'|' -f1 | sed -e 's/^/- /'
+			if ((${manual_mode} != 1 && ${synced_cnt} == 1)); then
+				echo "Skipping '${desc}' due to unique match..."
+				continue
+			fi
+			if ((${synced_cnt} > 1)); then
+				echo "'${desc} matches multiple commits, please, double-check!"
+				manual_check=1
+			fi
+		fi
+		if ((${manual_mode} == 1 || ${manual_check} == 1)); then
+			read -p "Do you want to skip '${desc}'? [y/N]: " should_skip
+			case "${should_skip}" in
+				"y" | "Y")
+					echo "Skipping '${desc}'..."
+					continue
+					;;
+			esac
+		fi
+		# commit hasn't been synced into libbpf yet
+		echo "Picking '${desc}'..."
+		if ! git cherry-pick ${new_commit} &>/dev/null; then
+			echo "Warning! Cherry-picking '${desc} failed, checking if it's non-libbpf files causing problems..."
+			libbpf_conflict_cnt=$(git diff --name-only --diff-filter=U -- "${LIBBPF_PATHS[@]}" | wc -l)
+			conflict_cnt=$(git diff --name-only | wc -l)
+			prompt_resolution=1
+
+			if ((${libbpf_conflict_cnt} == 0)); then
+				echo "Looks like only non-libbpf files have conflicts, ignoring..."
+				if ((${conflict_cnt} == 0)); then
+					echo "Empty cherry-pick, skipping it..."
+					git cherry-pick --abort
+					continue
+				fi
+
+				git add .
+				# GIT_EDITOR=true to avoid editor popping up to edit commit message
+				if ! GIT_EDITOR=true git cherry-pick --continue &>/dev/null; then
+					echo "Error! That still failed! Please resolve manually."
+				else
+					echo "Success! All cherry-pick conflicts were resolved for '${desc}'!"
+					prompt_resolution=0
+				fi
+			fi
+
+			if ((${prompt_resolution} == 1)); then
+				read -p "Error! Cherry-picking '${desc}' failed, please fix manually and press <return> to proceed..."
+			fi
+		fi
+		# Append signature of just cherry-picked commit to avoid
+		# potentially cherry-picking the same commit twice later when
+		# processing bpf tree commits. At this point we don't know yet
+		# the final commit sha in libbpf repo, so we record Linux SHA
+		# instead as LINUX_<sha>.
+		echo LINUX_$(git log --pretty='%h' -n1) "${signature}" >> ${TMP_DIR}/libbpf_commits.txt
+	done
+}
+
+cleanup()
+{
+	echo "Cleaning up..."
+	rm -r ${TMP_DIR}
+	cd_to ${LINUX_REPO}
+	git checkout ${TIP_SYM_REF}
+	git branch -D ${BASELINE_TAG} ${TIP_TAG} ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} \
+		      ${SQUASH_BASE_TAG} ${SQUASH_TIP_TAG} ${VIEW_TAG} || true
+
+	cd_to .
+	echo "DONE."
+}
+
+
+cd_to ${LIBBPF_REPO}
+GITHUB_ABS_DIR=$(pwd)
+echo "Dumping existing libbpf commit signatures..."
+for h in $(git log --pretty='%h' -n500); do
+	echo $h "$(commit_signature $h)" >> ${TMP_DIR}/libbpf_commits.txt
+done
+
+# Use current kernel repo HEAD as a source of patches
+cd_to ${LINUX_REPO}
+LINUX_ABS_DIR=$(pwd)
+TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD)
+TIP_COMMIT=$(git rev-parse HEAD)
+BPF_TIP_COMMIT=$(git rev-parse ${BPF_BRANCH})
+BASELINE_TAG=libbpf-baseline-${SUFFIX}
+TIP_TAG=libbpf-tip-${SUFFIX}
+BPF_BASELINE_TAG=libbpf-bpf-baseline-${SUFFIX}
+BPF_TIP_TAG=libbpf-bpf-tip-${SUFFIX}
+VIEW_TAG=libbpf-view-${SUFFIX}
+LIBBPF_SYNC_TAG=libbpf-sync-${SUFFIX}
+
+# Squash state of kernel repo at baseline into single commit
+SQUASH_BASE_TAG=libbpf-squash-base-${SUFFIX}
+SQUASH_TIP_TAG=libbpf-squash-tip-${SUFFIX}
+SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}")
+
+echo "WORKDIR:          ${WORKDIR}"
+echo "LINUX REPO:       ${LINUX_REPO}"
+echo "LIBBPF REPO:      ${LIBBPF_REPO}"
+echo "TEMP DIR:         ${TMP_DIR}"
+echo "SUFFIX:           ${SUFFIX}"
+echo "BASE COMMIT:      '$(commit_desc ${BASELINE_COMMIT})'"
+echo "TIP COMMIT:       '$(commit_desc ${TIP_COMMIT})'"
+echo "BPF BASE COMMIT:  '$(commit_desc ${BPF_BASELINE_COMMIT})'"
+echo "BPF TIP COMMIT:   '$(commit_desc ${BPF_TIP_COMMIT})'"
+echo "SQUASH COMMIT:    ${SQUASH_COMMIT}"
+echo "BASELINE TAG:     ${BASELINE_TAG}"
+echo "TIP TAG:          ${TIP_TAG}"
+echo "BPF BASELINE TAG: ${BPF_BASELINE_TAG}"
+echo "BPF TIP TAG:      ${BPF_TIP_TAG}"
+echo "SQUASH BASE TAG:  ${SQUASH_BASE_TAG}"
+echo "SQUASH TIP TAG:   ${SQUASH_TIP_TAG}"
+echo "VIEW TAG:         ${VIEW_TAG}"
+echo "LIBBPF SYNC TAG:  ${LIBBPF_SYNC_TAG}"
+echo "PATCHES:          ${TMP_DIR}/patches"
+
+git branch ${BASELINE_TAG} ${BASELINE_COMMIT}
+git branch ${TIP_TAG} ${TIP_COMMIT}
+git branch ${BPF_BASELINE_TAG} ${BPF_BASELINE_COMMIT}
+git branch ${BPF_TIP_TAG} ${BPF_TIP_COMMIT}
+git branch ${SQUASH_BASE_TAG} ${SQUASH_COMMIT}
+git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT}
+
+# Cherry-pick new commits onto squashed baseline commit
+cherry_pick_commits ${BASELINE_TAG} ${TIP_TAG}
+cherry_pick_commits ${BPF_BASELINE_TAG} ${BPF_TIP_TAG}
+
+# Move all libbpf files into __libbpf directory.
+FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${LIBBPF_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
+# Make __libbpf a new root directory
+FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __libbpf ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
+
+# If there are no new commits with  libbpf-related changes, bail out
+COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG})
+if ((${COMMIT_CNT} <= 0)); then
+    echo "No new changes to apply, we are done!"
+    cleanup
+    exit 2
+fi
+
+# Exclude baseline commit and generate nice cover letter with summary
+git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches
+
+# Now is time to re-apply libbpf-related linux patches to libbpf repo
+cd_to ${LIBBPF_REPO}
+git checkout -b ${LIBBPF_SYNC_TAG}
+
+for patch in $(ls -1 ${TMP_DIR}/patches | tail -n +2); do
+	if ! git am -3 --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then
+		if ! patch -p1 --merge < "${TMP_DIR}/patches/${patch}"; then
+			read -p "Applying ${TMP_DIR}/patches/${patch} failed, please resolve manually and press <return> to proceed..."
+		fi
+		git am --continue
+	fi
+done
+
+# Generate bpf_helper_defs.h and commit, if anything changed
+# restore Linux tip to use bpf_doc.py
+cd_to ${LINUX_REPO}
+git checkout ${TIP_TAG}
+# re-generate bpf_helper_defs.h
+cd_to ${LIBBPF_REPO}
+"${LINUX_ABS_DIR}/scripts/bpf_doc.py" --header					      \
+	--file include/uapi/linux/bpf.h > src/bpf_helper_defs.h
+# if anything changed, commit it
+helpers_changes=$(git status --porcelain src/bpf_helper_defs.h | wc -l)
+if ((${helpers_changes} == 1)); then
+	git add src/bpf_helper_defs.h
+	git commit -s -m "sync: auto-generate latest BPF helpers
+
+Latest changes to BPF helper definitions.
+" -- src/bpf_helper_defs.h
+fi
+
+# Use generated cover-letter as a template for "sync commit" with
+# baseline and checkpoint commits from kernel repo (and leave summary
+# from cover letter intact, of course)
+echo ${TIP_COMMIT} > CHECKPOINT-COMMIT &&					      \
+echo ${BPF_TIP_COMMIT} > BPF-CHECKPOINT-COMMIT &&				      \
+git add CHECKPOINT-COMMIT &&							      \
+git add BPF-CHECKPOINT-COMMIT &&						      \
+awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \
+sed "s/\*\*\* BLURB HERE \*\*\*/\
+sync: latest libbpf changes from kernel\n\
+\n\
+Syncing latest libbpf commits from kernel repository.\n\
+Baseline bpf-next commit:   ${BASELINE_COMMIT}\n\
+Checkpoint bpf-next commit: ${TIP_COMMIT}\n\
+Baseline bpf commit:        ${BPF_BASELINE_COMMIT}\n\
+Checkpoint bpf commit:      ${BPF_TIP_COMMIT}/" |				      \
+git commit -s --file=-
+
+echo "SUCCESS! ${COMMIT_CNT} commits synced."
+
+echo "Verifying Linux's and Github's libbpf state"
+
+cd_to ${LINUX_REPO}
+git checkout -b ${VIEW_TAG} ${TIP_COMMIT}
+FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${LIBBPF_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG}
+FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __libbpf ${VIEW_TAG}^..${VIEW_TAG}
+git ls-files -- "${LIBBPF_VIEW_PATHS[@]}" | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/linux-view.ls
+
+cd_to ${LIBBPF_REPO}
+git ls-files -- "${LIBBPF_VIEW_PATHS[@]}" | grep -v -E "${LIBBPF_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls
+
+echo "Comparing list of files..."
+diff -u ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls
+echo "Comparing file contents..."
+CONSISTENT=1
+for F in $(cat ${TMP_DIR}/linux-view.ls); do
+	if ! diff -u "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then
+		echo "${LINUX_ABS_DIR}/${F} and ${GITHUB_ABS_DIR}/${F} are different!"
+		CONSISTENT=0
+	fi
+done
+if ((${CONSISTENT} == 1)); then
+	echo "Great! Content is identical!"
+else
+	ignore_inconsistency=n
+	echo "Unfortunately, there are some inconsistencies, please double check."
+	read -p "Does everything look good? [y/N]: " ignore_inconsistency
+	case "${ignore_inconsistency}" in
+		"y" | "Y")
+			echo "Ok, proceeding..."
+			;;
+		*)
+			echo "Oops, exiting with error..."
+			exit 4
+	esac
+fi
+
+cleanup