summaryrefslogtreecommitdiffstats
path: root/mirroring
diff options
context:
space:
mode:
Diffstat (limited to 'mirroring')
-rw-r--r--mirroring/MIRRORS9
-rw-r--r--mirroring/README.md64
-rw-r--r--mirroring/apache2.vhost.conf18
-rwxr-xr-xmirroring/mirror-ceph.sh101
-rw-r--r--mirroring/rsyncd.conf9
-rwxr-xr-xmirroring/test-mirrors.sh69
6 files changed, 270 insertions, 0 deletions
diff --git a/mirroring/MIRRORS b/mirroring/MIRRORS
new file mode 100644
index 00000000..51f02876
--- /dev/null
+++ b/mirroring/MIRRORS
@@ -0,0 +1,9 @@
+download.ceph.com: Red Hat <ceph-users@lists.ceph.com>
+eu.ceph.com: Wido den Hollander <wido@42on.com>
+au.ceph.com: Matthew Taylor <matthew.taylor@digitalpacific.com.au>
+de.ceph.com: Oliver Dzombic <info@ip-interactive.de>
+se.ceph.com: Josef Johansson <se-ceph-com@oderland.se>
+us-east.ceph.com: Tyler Bishop <tyler.bishop@beyondhosting.net>
+hk.ceph.com: Mart van Santen <mart@greenhost.nl>
+fr.ceph.com: Adrien Gillard <gillard.adrien@gmail.com>
+uk.ceph.com: Tim Bishop <T.D.Bishop@kent.ac.uk>
diff --git a/mirroring/README.md b/mirroring/README.md
new file mode 100644
index 00000000..8a295d86
--- /dev/null
+++ b/mirroring/README.md
@@ -0,0 +1,64 @@
+# Mirroring Ceph
+Ceph is primarily distributed from download.ceph.com which is based in the US.
+
+However, globally there are multiple mirrors which offer the same content. Often
+faster than downloading from the primary source.
+
+Using the script found in this directory you can easily mirror Ceph to your local
+datacenter and serve packages from there to your servers.
+
+## Guidelines
+If you want to mirror Ceph please follow these guidelines:
+* Please use a mirror close to you
+* Do not sync in a shorter interval than 3 hours
+* Avoid syncing at minute 0 of the hour, use something between 0 and 59.
+
+## Mirror script
+The 'mirror-ceph.sh' script is written in Bash and will use rsync to mirror
+all the contents to a local directory.
+
+Usage is simple:
+
+<pre>
+./mirror-ceph.sh -q -s eu -t /srv/mirrors/ceph
+</pre>
+
+This example will mirror all contents from the source 'eu' which is *eu.ceph.com*.
+
+### Running with CRON
+The script can easily be run with CRON:
+
+<pre>
+13 1,5,9,13,17,21 * * * /home/ceph/mirror-ceph.sh -q -s eu -t /srv/mirrors/ceph
+</pre>
+
+This will sync from *eu.ceph.com* on 01:13, 05:13, 09:13, 13:13, 17:13 and 21:13.
+
+## Becoming a mirror source
+If you have spare hardware and resources available you can opt for becoming a mirror
+source for others.
+
+A few things which are required:
+* 1Gbit connection or more
+* Native IPv4 **and** IPv6
+* HTTP access
+* rsync access
+* 2TB of storage or more
+* Monitoring of the mirror/source
+
+You can then run the *mirror-ceph.sh* script and mirror all the contents.
+
+### Logs
+The project wants to analyze the downloads of Ceph a few times a year. From mirrors
+we expect that they store HTTP access logs for at least 6 months so they can be
+used for analysis.
+
+### DNS
+Using a DNS CNAME record a XX.ceph.com entry can be forwarded to the server and
+added to the mirror script.
+
+You can request such a DNS entry on the ceph mailinglists.
+
+### Apache configuration
+A Apache 2.4 VirtualHost example configuration can be found the Git repository
+with the name *apache2.vhost.conf*
diff --git a/mirroring/apache2.vhost.conf b/mirroring/apache2.vhost.conf
new file mode 100644
index 00000000..b7fce7eb
--- /dev/null
+++ b/mirroring/apache2.vhost.conf
@@ -0,0 +1,18 @@
+#
+# This is a example Apache 2 VirtualHost being used
+# on eu.ceph.com which runs on Ubuntu 14.04
+#
+
+<VirtualHost *:80>
+ ServerName eu.ceph.com
+ ServerAdmin webmaster@localhost
+ DocumentRoot /srv/mirror/ceph/download
+ <Directory /srv/mirror/ceph/download>
+ Options FollowSymLinks Indexes
+ AllowOverride none
+ Require all granted
+ </Directory>
+
+ ErrorLog ${APACHE_LOG_DIR}/error.log
+ CustomLog ${APACHE_LOG_DIR}/access.log combined
+</VirtualHost>
diff --git a/mirroring/mirror-ceph.sh b/mirroring/mirror-ceph.sh
new file mode 100755
index 00000000..e5108b3a
--- /dev/null
+++ b/mirroring/mirror-ceph.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -e
+#
+# Script to mirror Ceph locally
+#
+# Please, choose a local source and do not sync in a shorter interval than
+# 3 hours.
+#
+SILENT=0
+
+# All available source mirrors
+declare -A SOURCES
+SOURCES[eu]="eu.ceph.com"
+SOURCES[de]="de.ceph.com"
+SOURCES[se]="se.ceph.com"
+SOURCES[au]="au.ceph.com"
+SOURCES[us]="download.ceph.com"
+SOURCES[hk]="hk.ceph.com"
+SOURCES[fr]="fr.ceph.com"
+SOURCES[us-east]="us-east.ceph.com"
+SOURCES[us-west]="us-west.ceph.com"
+SOURCES[global]="download.ceph.com"
+
+function print_usage() {
+ echo "$0 [-q ] -s <source mirror> -t <target directory>"
+}
+
+while getopts ":qhs:t:" opt; do
+ case $opt in
+ q)
+ SILENT=1
+ ;;
+ s)
+ SOURCE=$OPTARG
+ ;;
+ t)
+ TARGET=$OPTARG
+ ;;
+ h)
+ HELP=1
+ ;;
+ \?)
+ print_usage
+ exit 1
+ ;;
+ esac
+done
+
+if [ ! -z "$HELP" ] || [ -z "$TARGET" ] || [ -z "$SOURCE" ]; then
+ print_usage
+ exit 1
+fi
+
+if [ ! -d "$TARGET" ]; then
+ echo "$TARGET is not a valid target directory"
+ exit 1
+fi
+
+for i in "${!SOURCES[@]}"; do
+ if [ "$i" == "$SOURCE" ]; then
+ SOURCE_HOST=${SOURCES[$i]}
+ fi
+done
+
+if [ -z "$SOURCE_HOST" ]; then
+ echo -n "Please select one of the following sources:"
+ for i in "${!SOURCES[@]}"; do
+ echo -n " $i"
+ done
+ echo ""
+ exit 1
+fi
+
+RSYNC_OPTS="--stats --progress"
+if [ $SILENT -eq 1 ]; then
+ RSYNC_OPTS="--quiet"
+fi
+
+# We start a two-stage sync here for DEB and RPM
+# Based on: https://www.debian.org/mirror/ftpmirror
+#
+# The idea is to prevent temporary situations where metadata points to files
+# which do not exist
+#
+
+# Exclude all metadata files
+rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
+ --hard-links \
+ --exclude Packages* \
+ --exclude Sources* \
+ --exclude Release* \
+ --exclude InRelease \
+ --exclude i18n/* \
+ --exclude ls-lR* \
+ --exclude repodata/* \
+ ${TARGET}
+
+# Now also transfer the metadata and delete afterwards
+rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
+ --hard-links --delete-after \
+ ${TARGET}
diff --git a/mirroring/rsyncd.conf b/mirroring/rsyncd.conf
new file mode 100644
index 00000000..fa455957
--- /dev/null
+++ b/mirroring/rsyncd.conf
@@ -0,0 +1,9 @@
+uid = nobody
+gid = nogroup
+max connections = 15
+socket options = SO_KEEPALIVE
+
+[ceph]
+ path = /path/to/ceph/mirror/data
+ comment = Ceph mirror
+ read only = true
diff --git a/mirroring/test-mirrors.sh b/mirroring/test-mirrors.sh
new file mode 100755
index 00000000..280118bc
--- /dev/null
+++ b/mirroring/test-mirrors.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+# Simple script which performs a HTTP and rsync check on
+# all Ceph mirrors over IPv4 and IPv6 to see if they are online
+#
+# Requires IPv4, IPv6, rsync and curl
+#
+# Example usage:
+# - ./test-mirrors.sh eu.ceph.com,de.ceph.com,hk.ceph.com
+# - cat MIRRORS |cut -d ':' -f 1|xargs -n 1 ./test-mirrors.sh
+#
+
+function print_usage {
+ echo "Usage: $0 mirror1,mirror2,mirror3,mirror4,etc"
+}
+
+function test_http {
+ HOST=$1
+
+ echo -n "$HOST HTTP IPv4: "
+ curl -s -I -4 -o /dev/null http://$HOST
+ if [ "$?" -ne 0 ]; then
+ echo "FAIL"
+ else
+ echo "OK"
+ fi
+
+ echo -n "$HOST HTTP IPv6: "
+ curl -s -I -6 -o /dev/null http://$HOST
+ if [ "$?" -ne 0 ]; then
+ echo "FAIL"
+ else
+ echo "OK"
+ fi
+}
+
+function test_rsync {
+ HOST=$1
+
+ echo -n "$HOST RSYNC IPv4: "
+ rsync -4 -avrqn ${HOST}::ceph /tmp 2>/dev/null
+ if [ "$?" -ne 0 ]; then
+ echo "FAIL"
+ else
+ echo "OK"
+ fi
+
+ echo -n "$HOST RSYNC IPv6: "
+ rsync -6 -avrqn ${HOST}::ceph /tmp 2>/dev/null
+ if [ "$?" -ne 0 ]; then
+ echo "FAIL"
+ else
+ echo "OK"
+ fi
+}
+
+MIRRORS=$1
+
+if [ -z "$MIRRORS" ]; then
+ print_usage
+ exit 1
+fi
+
+IFS=', ' read -r -a array <<< "$MIRRORS"
+
+for MIRROR in "${array[@]}"; do
+ test_http $MIRROR
+ test_rsync $MIRROR
+done