Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 270 insertions, 0 deletions

diff --git a/mirroring/MIRRORS b/mirroring/MIRRORS
new file mode 100644
index 000000000..bd8f0dab5
--- /dev/null
+++ b/mirroring/MIRRORS
@@ -0,0 +1,8 @@
+download.ceph.com: Red Hat <ceph-users@lists.ceph.com>
+eu.ceph.com: Wido den Hollander <wido@42on.com>
+au.ceph.com: Matthew Taylor <matthew.taylor@digitalpacific.com.au>, Tim Dousset <tim.dousset@hostopia.com.au>
+de.ceph.com: Oliver Dzombic <info@ip-interactive.de>
+se.ceph.com: Josef Johansson <se-ceph-com@oderland.se>
+fr.ceph.com: Adrien Gillard <gillard.adrien@gmail.com>
+uk.ceph.com: Tim Bishop <T.D.Bishop@kent.ac.uk>
+ca.ceph.com: University of Waterloo Systems Committee <systems-committee@csclub.uwaterloo.ca>, School of Computer Science <cscf.csi.management@uwaterloo.ca>
diff --git a/mirroring/README.md b/mirroring/README.md
new file mode 100644
index 000000000..f540629ac
--- /dev/null
+++ b/mirroring/README.md
@@ -0,0 +1,66 @@
+# Mirroring Ceph
+Ceph is primarily distributed from download.ceph.com which is based in the US.
+
+However, globally there are multiple mirrors which offer the same content. Often
+faster than downloading from the primary source.
+
+Using the script found in this directory you can easily mirror Ceph to your local
+datacenter and serve packages from there to your servers.
+
+## Guidelines
+If you want to mirror Ceph please follow these guidelines:
+* Please use a mirror close to you
+* Do not sync in a shorter interval than 3 hours
+* Avoid syncing at minute 0 of the hour, use something between 0 and 59.
+
+## Mirror script
+The 'mirror-ceph.sh' script is written in Bash and will use rsync to mirror
+all the contents to a local directory.
+
+Usage is simple:
+
+<pre>
+./mirror-ceph.sh -q -s eu -t /srv/mirrors/ceph
+</pre>
+
+This example will mirror all contents from the source 'eu' which is *eu.ceph.com*.
+
+### Running with CRON
+The script can easily be run with CRON:
+
+<pre>
+13 1,5,9,13,17,21 * * * /home/ceph/mirror-ceph.sh -q -s eu -t /srv/mirrors/ceph
+</pre>
+
+This will sync from *eu.ceph.com* on 01:13, 05:13, 09:13, 13:13, 17:13 and 21:13.
+
+## Becoming a mirror source
+If you have spare hardware and resources available you can opt for becoming a mirror
+source for others.
+
+A few things which are required:
+* 1Gbit connection or more
+* Native IPv4 **and** IPv6
+* HTTP access
+* rsync access
+* 2TB of storage or more
+* Monitoring of the mirror/source
+
+You can then run the *mirror-ceph.sh* script and mirror all the contents.
+
+Mirror maintainers should sign up to the [Ceph-mirrors mailing list](https://lists.ceph.io/postorius/lists/ceph-mirrors.ceph.io/).
+
+### Logs
+The project wants to analyze the downloads of Ceph a few times a year. From mirrors
+we expect that they store HTTP access logs for at least 6 months so they can be
+used for analysis.
+
+### DNS
+Using a DNS CNAME record a XX.ceph.com entry can be forwarded to the server and
+added to the mirror script.
+
+You can request such a DNS entry on the ceph mailinglists.
+
+### Apache configuration
+A Apache 2.4 VirtualHost example configuration can be found the Git repository
+with the name *apache2.vhost.conf*
diff --git a/mirroring/apache2.vhost.conf b/mirroring/apache2.vhost.conf
new file mode 100644
index 000000000..b7fce7ebb
--- /dev/null
+++ b/mirroring/apache2.vhost.conf
@@ -0,0 +1,18 @@
+#
+# This is a example Apache 2 VirtualHost being used
+# on eu.ceph.com which runs on Ubuntu 14.04
+#
+
+<VirtualHost *:80>
+    ServerName eu.ceph.com
+    ServerAdmin webmaster@localhost
+    DocumentRoot /srv/mirror/ceph/download
+    <Directory /srv/mirror/ceph/download>
+        Options FollowSymLinks Indexes
+        AllowOverride none
+        Require all granted
+    </Directory>
+
+    ErrorLog ${APACHE_LOG_DIR}/error.log
+    CustomLog ${APACHE_LOG_DIR}/access.log combined
+</VirtualHost>
diff --git a/mirroring/mirror-ceph.sh b/mirroring/mirror-ceph.sh
new file mode 100755
index 000000000..93e5b6e34
--- /dev/null
+++ b/mirroring/mirror-ceph.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -e
+#
+# Script to mirror Ceph locally
+#
+# Please, choose a local source and do not sync in a shorter interval than
+# 3 hours.
+#
+SILENT=0
+
+# All available source mirrors
+declare -A SOURCES
+SOURCES[eu]="eu.ceph.com"
+SOURCES[de]="de.ceph.com"
+SOURCES[se]="se.ceph.com"
+SOURCES[au]="au.ceph.com"
+SOURCES[us]="download.ceph.com"
+SOURCES[fr]="fr.ceph.com"
+SOURCES[ca]="ca.ceph.com"
+SOURCES[us-west]="us-west.ceph.com"
+SOURCES[global]="download.ceph.com"
+
+function print_usage() {
+    echo "$0 [-q ] -s <source mirror> -t <target directory>"
+}
+
+while getopts ":qhs:t:" opt; do
+    case $opt in
+        q)
+            SILENT=1
+            ;;
+        s)
+            SOURCE=$OPTARG
+            ;;
+        t)
+            TARGET=$OPTARG
+            ;;
+        h)
+            HELP=1
+            ;;
+        \?)
+            print_usage
+            exit 1
+            ;;
+    esac
+done
+
+if [ ! -z "$HELP" ] || [ -z "$TARGET" ] || [ -z "$SOURCE" ]; then
+    print_usage
+    exit 1
+fi
+
+if [ ! -d "$TARGET" ]; then
+    echo "$TARGET is not a valid target directory"
+    exit 1
+fi
+
+for i in "${!SOURCES[@]}"; do
+    if [ "$i" == "$SOURCE" ]; then
+        SOURCE_HOST=${SOURCES[$i]}
+    fi
+done
+
+if [ -z "$SOURCE_HOST" ]; then
+    echo -n "Please select one of the following sources:"
+    for i in "${!SOURCES[@]}"; do
+        echo -n " $i"
+    done
+    echo ""
+    exit 1
+fi
+
+RSYNC_OPTS="--stats --progress"
+if [ $SILENT -eq 1 ]; then
+    RSYNC_OPTS="--quiet"
+fi
+
+# We start a two-stage sync here for DEB and RPM
+# Based on: https://www.debian.org/mirror/ftpmirror
+#
+# The idea is to prevent temporary situations where metadata points to files
+# which do not exist
+#
+
+# Exclude all metadata files
+rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
+                                         --hard-links \
+                                         --exclude Packages* \
+                                         --exclude Sources* \
+                                         --exclude Release* \
+                                         --exclude InRelease \
+                                         --exclude i18n/* \
+                                         --exclude ls-lR* \
+                                         --exclude repodata/* \
+                                         ${TARGET}
+
+# Now also transfer the metadata and delete afterwards
+rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
+                                         --hard-links --delete-after \
+                                         ${TARGET}
diff --git a/mirroring/rsyncd.conf b/mirroring/rsyncd.conf
new file mode 100644
index 000000000..fa4559573
--- /dev/null
+++ b/mirroring/rsyncd.conf
@@ -0,0 +1,9 @@
+uid = nobody
+gid = nogroup
+max connections = 15
+socket options = SO_KEEPALIVE
+
+[ceph]
+  path = /path/to/ceph/mirror/data
+  comment = Ceph mirror
+  read only = true
diff --git a/mirroring/test-mirrors.sh b/mirroring/test-mirrors.sh
new file mode 100755
index 000000000..751c31498
--- /dev/null
+++ b/mirroring/test-mirrors.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+# Simple script which performs a HTTP and rsync check on
+# all Ceph mirrors over IPv4 and IPv6 to see if they are online
+#
+# Requires IPv4, IPv6, rsync and curl
+#
+# Example usage:
+# - ./test-mirrors.sh eu.ceph.com,de.ceph.com,au.ceph.com
+# - cat MIRRORS |cut -d ':' -f 1|xargs -n 1 ./test-mirrors.sh
+#
+
+function print_usage {
+    echo "Usage: $0 mirror1,mirror2,mirror3,mirror4,etc"
+}
+
+function test_http {
+    HOST=$1
+
+    echo -n "$HOST HTTP IPv4: "
+    curl -s -I -4 -o /dev/null http://$HOST
+    if [ "$?" -ne 0 ]; then
+        echo "FAIL"
+    else
+        echo "OK"
+    fi
+
+    echo -n "$HOST HTTP IPv6: "
+    curl -s -I -6 -o /dev/null http://$HOST
+    if [ "$?" -ne 0 ]; then
+        echo "FAIL"
+    else
+        echo "OK"
+    fi
+}
+
+function test_rsync {
+    HOST=$1
+
+    echo -n "$HOST RSYNC IPv4: "
+    rsync -4 -avrqn ${HOST}::ceph /tmp 2>/dev/null
+    if [ "$?" -ne 0 ]; then
+        echo "FAIL"
+    else
+        echo "OK"
+    fi
+
+    echo -n "$HOST RSYNC IPv6: "
+    rsync -6 -avrqn ${HOST}::ceph /tmp 2>/dev/null
+    if [ "$?" -ne 0 ]; then
+        echo "FAIL"
+    else
+        echo "OK"
+    fi
+}
+
+MIRRORS=$1
+
+if [ -z "$MIRRORS" ]; then
+    print_usage
+    exit 1
+fi
+
+IFS=', ' read -r -a array <<< "$MIRRORS"
+
+for MIRROR in "${array[@]}"; do
+    test_http $MIRROR
+    test_rsync $MIRROR
+done