2 files changed, 60 insertions, 42 deletions
diff --git a/debian/tests/unit-tests-flaky b/debian/tests/unit-tests-flaky
deleted file mode 100755
index b4cc64c70..000000000
--- a/debian/tests/unit-tests-flaky
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/sh
-
-# Don't let them fail
-# set -e
-
-BASEPATH=$(pwd)
-
-unit_test_dirs="
-ansible_collections/ngine_io/cloudstack          # missing python3-units
-ansible_collections/ansible/netcommon            # test broken
-ansible_collections/netapp/ontap                 # missing netapp_lib
-ansible_collections/netapp_eseries/santricity    # missing python3-units
-ansible_collections/community/aws                # import broken
-ansible_collections/community/vmware             # missing python3-units
-ansible_collections/community/hrobot             # import broken
-ansible_collections/community/general            # import broken
-ansible_collections/community/skydive            # STDERR: WARNING: All targets skipped.
-ansible_collections/junipernetworks/junos        # test broken
-ansible_collections/dellemc/os6                  # import broken
-ansible_collections/dellemc/os10                 # missing python3-units
-ansible_collections/dellemc/os9                  # import broken
-"
-
-for i in $unit_test_dirs; do
-
-  cd $i 2> /dev/null || continue
-
-  echo "\n\n"
-  echo "############################################################"
-  echo "############################################################"
-  echo "#### Running FLAKY tests in $i"
-  echo "############################################################"
-  echo "############################################################"
-
-  /usr/bin/ansible-test units \
-    --python-interpreter /usr/bin/python3 \
-    --local
-
-  cd $BASEPATH
-done
-
-exit 0
diff --git a/debian/tests/unit-tests-flaky.py b/debian/tests/unit-tests-flaky.py
new file mode 100755
index 000000000..db740ee5a
--- /dev/null
+++ b/debian/tests/unit-tests-flaky.py
@@ -0,0 +1,60 @@
+#!/usr/bin/python3
+
+import os
+import subprocess
+import sys
+
+sys.dont_write_bytecode = True
+from flaky_tests_list import flaky_test_dirs
+
+cwd = os.getcwd()
+
+overall_test_rc = 0
+succeeded_tests = []
+failed_tests = []
+
+for i in flaky_test_dirs:
+
+    os.chdir(i)
+
+    print ("\n\n", flush=True)
+    print ("############################################################", flush=True)
+    print ("############################################################", flush=True)
+    print ("#### Running FLAKY tests in", i, flush=True)
+    print ("############################################################", flush=True)
+    print ("############################################################", flush=True)
+
+    rc = subprocess.run([
+        '/usr/bin/ansible-test',
+        'units',
+        '--python-interpreter',
+        '/usr/bin/python3',
+        '--local'
+    ])
+
+    # don't fail on failed test, continue instead
+    print ("## return code is", rc.returncode)
+
+    if rc.returncode == 0:
+        succeeded_tests.append(i)
+    else:
+        failed_tests.append(i)
+        overall_test_rc = rc.returncode
+
+    os.chdir(cwd)
+
+print ("############################################################", flush=True)
+print ("############################################################", flush=True)
+print ("#### failed tests are:", flush=True)
+for i in failed_tests:
+    print ("####", i, flush=True)
+if len(succeeded_tests) > 0:
+    print ("#### succeeded tests are:", flush=True)
+    for i in succeeded_tests:
+        print ("####", i, flush=True)
+    print ("#### consider removing them from the flaky tests list.", flush=True)
+print ("############################################################", flush=True)
+print ("############################################################", flush=True)
+
+
+exit(overall_test_rc)