Adding upstream version 2.7.6.upstream/2.7.6

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 222 insertions, 0 deletions

diff --git a/utils/expand_info.py b/utils/expand_info.py
new file mode 100755
index 0000000..787b1f0
--- /dev/null
+++ b/utils/expand_info.py
@@ -0,0 +1,11 @@
+#!/usr/bin/python3
+#
+# usage: ./utils/expand_info.py build/data/templates/Debian.info \
+#                               /usr/share/distro-info/debian.csv
+
+import sys
+
+import aptsources.distinfo
+
+for line in aptsources.distinfo._expand_template(sys.argv[1], sys.argv[2]):
+    print(line)
diff --git a/utils/get_debian_mirrors.py b/utils/get_debian_mirrors.py
new file mode 100755
index 0000000..408968c
--- /dev/null
+++ b/utils/get_debian_mirrors.py
@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+#  get_debian_mirrors.py - Parse Mirrors.masterlist and create a mirror list.
+#
+#  Copyright (c) 2010-2011 Julian Andres Klode <jak@debian.org>
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License as
+#  published by the Free Software Foundation; either version 2 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+#  USA
+import collections
+import sys
+import urllib.request
+
+from debian import deb822
+
+mirrors = collections.defaultdict(set)
+masterlist = urllib.request.urlopen(
+    "https://mirror-master.debian.org/" "status/Mirrors.masterlist"
+)
+
+for mirror in deb822.Deb822.iter_paragraphs(masterlist):
+    if "Country" not in mirror:
+        continue
+    country = mirror["Country"].split(None, 1)[0]
+    site = mirror["Site"]
+    for proto in "http", "ftp":
+        if "Archive-%s" % proto in mirror:
+            mirrors[country].add(
+                "{}://{}{}".format(proto, site, mirror["Archive-%s" % proto])
+            )
+
+if len(mirrors) == 0:
+    sys.stderr.write("E: Could not read the mirror list due to " "some unknown issue\n")
+    sys.exit(1)
+for country in sorted(mirrors):
+    print("#LOC:%s" % country)
+    print("\n".join(sorted(mirrors[country])))
diff --git a/utils/get_ubuntu_mirrors_from_lp.py b/utils/get_ubuntu_mirrors_from_lp.py
new file mode 100755
index 0000000..b3253de
--- /dev/null
+++ b/utils/get_ubuntu_mirrors_from_lp.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+#
+#  get_ubuntu_lp_mirrors.py
+#
+#  Download the latest list with available Ubuntu mirrors from Launchpad.net
+#  and extract the hosts from the raw page
+#
+#  Copyright (c) 2006 Free Software Foundation Europe
+#
+#  Author: Sebastian Heinlein <glatzor@ubuntu.com>
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License as
+#  published by the Free Software Foundation; either version 2 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+#  USA
+
+import sys
+
+import feedparser
+
+d = feedparser.parse("https://launchpad.net/ubuntu/+archivemirrors-rss")
+# d = feedparser.parse(open("+archivemirrors-rss"))
+
+countries = {}
+
+for entry in d.entries:
+    countrycode = entry.mirror_countrycode
+    if countrycode not in countries:
+        countries[countrycode] = set()
+    for link in entry.links:
+        countries[countrycode].add(link.href)
+
+
+keys = sorted(countries)
+
+if len(keys) == 0:
+    sys.stderr.write(
+        "E: Could not read the mirror list due to some issue"
+        " -- status code: %s\n" % d.status
+    )
+    sys.exit(1)
+
+print("mirror://mirrors.ubuntu.com/mirrors.txt")
+for country in keys:
+    print("#LOC:%s" % country)
+    print("\n".join(sorted(countries[country])))
diff --git a/utils/mirrortest b/utils/mirrortest
new file mode 100755
index 0000000..54dcfe7
--- /dev/null
+++ b/utils/mirrortest
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+import os
+import queue
+import random
+import re
+import socket
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import aptsources.distro
+import aptsources.sourceslist
+
+socket.setdefaulttimeout(2)
+
+
+class MirrorTest:
+    class PingWorker(threading.Thread):
+        def __init__(self, jobs, results, id):
+            self.id = id
+            self.jobs = jobs
+            self.results = results
+            self.match_result = re.compile(r"^rtt .* = [\.\d]+/([\.\d]+)/.*")
+            threading.Thread.__init__(self)
+
+        def run(self):
+            result = None
+            while MirrorTest.completed_pings < MirrorTest.todo:
+                try:
+                    mirror = self.jobs.get(True, 1)
+                    host = mirror.hostname
+                except:
+                    continue
+                print(
+                    "Pinging (Worker %s) %s (%s) ..."
+                    % (self.id, host, MirrorTest.completed_pings)
+                )
+                commando = os.popen("ping -q -c 4 -W 2 -i 0.3 %s" % host, "r")
+                while True:
+                    line = commando.readline()
+                    if not line:
+                        break
+                    result = re.findall(self.match_result, line)
+                MirrorTest.completed_pings_lock.acquire()
+                MirrorTest.completed_pings += 1
+                if result:
+                    self.results.append([float(result[0]), host, mirror])
+                MirrorTest.completed_pings_lock.release()
+
+    def speed_test(self, mirror):
+        url = f"{mirror.get_repo_urls()[0]}/{self.test_file}"
+        print("Downloading %s ..." % url)
+        start = time.time()
+        try:
+            urllib.request.urlopen(url).read(51200)
+            return 50 / (time.time() - start)
+        except:
+            return 0
+
+    def __init__(self, hosts, test_file):
+        self.test_file = test_file
+        jobs = queue.Queue()
+        results = []
+        for h in hosts:
+            jobs.put(h)
+        self.threads = []
+        MirrorTest.completed_pings = 0
+        MirrorTest.completed_pings_lock = threading.Lock()
+        MirrorTest.todo = len(hosts)
+
+        for i in range(10):
+            t = MirrorTest.PingWorker(jobs, results, i)
+            self.threads.append(t)
+            t.start()
+
+        for t in self.threads:
+            t.join()
+
+        results.sort()
+        print("\n\nTop ten RTTs:")
+        for r in results[0:10]:
+            print(f"{r[1]}: {r[0]}")
+        print("\n\n")
+
+        results.insert(0, [0, "rand", hosts[random.randint(1, len(hosts))]])
+        results.insert(0, [0, "rand", hosts[random.randint(1, len(hosts))]])
+
+        final = [(self.speed_test(r[2]), r[2]) for r in results[0:12]]
+        final.sort()
+        final.reverse()
+        print("\n\nBest mirrors:")
+        for f in final:
+            print(f"{f[1].hostname}: {int(f[0])} KByte/s")
+
+
+if __name__ == "__main__":
+    distro = aptsources.distro.get_distro()
+    distro.get_sources(aptsources.sourceslist.SourcesList())
+    pipe = os.popen("dpkg --print-architecture")
+    arch = pipe.read().strip()
+    test_file = "dists/{}/{}/binary-{}/Packages.gz".format(
+        distro.source_template.name,
+        distro.source_template.components[0].name,
+        arch,
+    )
+    app = MirrorTest(distro.source_template.mirror_set.values(), test_file)