Adding upstream version 18.2.3.upstream/18.2.3

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 493 insertions, 41 deletions

diff --git a/src/pybind/mgr/rook/ci/Dockerfile b/src/pybind/mgr/rook/ci/Dockerfile
index 30ebea574..62ff0ab89 100644
--- a/src/pybind/mgr/rook/ci/Dockerfile
+++ b/src/pybind/mgr/rook/ci/Dockerfile
@@ -1,3 +1,4 @@
-FROM quay.io/ceph/daemon-base:latest-main
+FROM quay.io/ceph/daemon-base:latest-reef
 COPY ./tmp_build/orchestrator /usr/share/ceph/mgr/orchestrator
 COPY ./tmp_build/rook /usr/share/ceph/mgr/rook
+COPY ./tmp_build/ceph/ /usr/lib/python3.6/site-packages/ceph/
diff --git a/src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml b/src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml
new file mode 100644
index 000000000..2732286ab
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml
@@ -0,0 +1,198 @@
+#################################################################################################################
+# Define the settings for the rook-ceph cluster with settings for a minikube cluster with a single node
+
+# This example expects a single node minikube cluster with three extra disks: vdb, vdc and vdd. Please modify
+# it according to your environment. See the documentation for more details on storage settings available.
+
+# For example, to create the cluster:
+#   kubectl create -f crds.yaml -f common.yaml -f operator.yaml
+#   kubectl create -f cluster-on-pvc-minikube.yaml
+#################################################################################################################
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: local-storage
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: local0-0
+spec:
+  storageClassName: local-storage
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  # PV for mon must be a filesystem volume.
+  volumeMode: Filesystem
+  local:
+    # To use dm devices like logical volume, please replace `/dev/sdb` with their device names like `/dev/vg-name/lv-name`.
+    path: /dev/vdb
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - minikube
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: local0-1
+spec:
+  storageClassName: local-storage
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  # PV for mon must be a filesystem volume.
+  volumeMode: Block
+  local:
+    # To use dm devices like logical volume, please replace `/dev/sdb` with their device names like `/dev/vg-name/lv-name`.
+    path: /dev/vdc
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - minikube
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: local0-2
+spec:
+  storageClassName: local-storage
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  # PV for mon must be a filesystem volume.
+  volumeMode: Block
+  local:
+    # To use dm devices like logical volume, please replace `/dev/sdb` with their device names like `/dev/vg-name/lv-name`.
+    path: /dev/vdd
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - minikube
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: local0-3
+spec:
+  storageClassName: local-storage
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  # PV for mon must be a filesystem volume.
+  volumeMode: Block
+  local:
+    # To use dm devices like logical volume, please replace `/dev/sdb` with their device names like `/dev/vg-name/lv-name`.
+    path: /dev/vde
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - minikube
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: my-cluster
+  namespace: rook-ceph # namespace:cluster
+spec:
+  dataDirHostPath: /var/lib/rook
+  mon:
+    count: 1
+    allowMultiplePerNode: true
+    volumeClaimTemplate:
+      spec:
+        storageClassName: local-storage
+        resources:
+          requests:
+            storage: 10Gi
+  mgr:
+    count: 1
+    modules:
+      - name: pg_autoscaler
+        enabled: true
+  dashboard:
+    enabled: true
+    ssl: false
+  crashCollector:
+    disable: false
+  cephVersion:
+    image: quay.io/ceph/daemon-base:latest-main
+    allowUnsupported: true
+  skipUpgradeChecks: false
+  continueUpgradeAfterChecksEvenIfNotHealthy: false
+  storage:
+    storageClassDeviceSets:
+      - name: set1
+        count: 3
+        portable: false
+        tuneDeviceClass: true
+        tuneFastDeviceClass: false
+        encrypted: false
+        placement:
+        preparePlacement:
+        volumeClaimTemplates:
+          - metadata:
+              name: data
+              # if you are looking at giving your OSD a different CRUSH device class than the one detected by Ceph
+              # annotations:
+              #   crushDeviceClass: hybrid
+            spec:
+              resources:
+                requests:
+                  storage: 20Gi
+              # IMPORTANT: Change the storage class depending on your environment
+              storageClassName: local-storage
+              volumeMode: Block
+              accessModes:
+                - ReadWriteOnce
+    # when onlyApplyOSDPlacement is false, will merge both placement.All() and storageClassDeviceSets.Placement
+    onlyApplyOSDPlacement: false
+  priorityClassNames:
+    mon: system-node-critical
+    osd: system-node-critical
+    mgr: system-cluster-critical
+  disruptionManagement:
+    managePodBudgets: true
+    osdMaintenanceTimeout: 30
+    pgHealthCheckTimeout: 0
+  cephConfig:
+    global:
+      mon_warn_on_pool_no_redundancy: "false"
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: builtin-mgr
+  namespace: rook-ceph # namespace:cluster
+spec:
+  name: .mgr
+  failureDomain: osd
+  replicated:
+    size: 1
+    requireSafeReplicaSize: false
diff --git a/src/pybind/mgr/rook/ci/requirements.txt b/src/pybind/mgr/rook/ci/requirements.txt
new file mode 100644
index 000000000..9684f7742
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/requirements.txt
@@ -0,0 +1 @@
+behave
diff --git a/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh b/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
index a43e01a89..58d554757 100755
--- a/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
+++ b/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
@@ -2,8 +2,11 @@
 
 set -ex
 
+export PATH=$PATH:~/.local/bin # behave is installed on this directory
+
 # Execute tests
 : ${CEPH_DEV_FOLDER:=${PWD}}
 ${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
 cd ${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci/tests
+pip install --upgrade --force-reinstall -r ../requirements.txt
 behave
diff --git a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
index 4b97df6ba..24a6a5da2 100755
--- a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
+++ b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
@@ -3,7 +3,10 @@
 set -eEx
 
 : ${CEPH_DEV_FOLDER:=${PWD}}
+CLUSTER_SPEC=${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml
+DEFAULT_NS="rook-ceph"
 KUBECTL="minikube kubectl --"
+export ROOK_CLUSTER_NS="${ROOK_CLUSTER_NS:=$DEFAULT_NS}" ## CephCluster namespace
 
 # We build a local ceph image that contains the latest code
 # plus changes from the PR. This image will be used by the docker
@@ -15,14 +18,6 @@ on_error() {
     minikube delete
 }
 
-configure_libvirt(){
-    sudo usermod -aG libvirt $(id -un)
-    sudo su -l $USER  # Avoid having to log out and log in for group addition to take effect.
-    sudo systemctl enable --now libvirtd
-    sudo systemctl restart libvirtd
-    sleep 10 # wait some time for libvirtd service to restart
-}
-
 setup_minikube_env() {
 
     # Check if Minikube is running
@@ -35,20 +30,21 @@ setup_minikube_env() {
     fi
 
     rm -rf ~/.minikube
-    minikube start --memory="4096" --cpus="2" --disk-size=10g --extra-disks=1 --driver kvm2
+    minikube start --memory="6144" --disk-size=20g --extra-disks=4 --driver kvm2
     # point Docker env to use docker daemon running on minikube
     eval $(minikube docker-env -p minikube)
 }
 
 build_ceph_image() {
-    wget -q -O cluster-test.yaml https://raw.githubusercontent.com/rook/rook/master/deploy/examples/cluster-test.yaml
-    CURR_CEPH_IMG=$(grep -E '^\s*image:\s+' cluster-test.yaml | sed 's/.*image: *\([^ ]*\)/\1/')
+
+    CURR_CEPH_IMG=$(grep -E '^\s*image:\s+' $CLUSTER_SPEC | sed 's/.*image: *\([^ ]*\)/\1/')
 
     cd ${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci
     mkdir -p tmp_build/rook
     mkdir -p tmp_build/orchestrator
     cp ./../../orchestrator/*.py tmp_build/orchestrator
     cp ../*.py tmp_build/rook
+    cp -r ../../../../../src/python-common/ceph/ tmp_build/
 
     # we use the following tag to trick the Docker
     # running inside minikube so it uses this image instead
@@ -62,28 +58,39 @@ build_ceph_image() {
 }
 
 create_rook_cluster() {
-    wget -q -O cluster-test.yaml https://raw.githubusercontent.com/rook/rook/master/deploy/examples/cluster-test.yaml
     $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/crds.yaml
     $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/common.yaml
     $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/operator.yaml
-    $KUBECTL create -f cluster-test.yaml
-    $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/dashboard-external-http.yaml
+    $KUBECTL create -f $CLUSTER_SPEC
     $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/toolbox.yaml
 }
 
+is_operator_ready() {
+    local phase
+    phase=$($KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}')
+    echo "PHASE: $phase"
+    [[ "$phase" == "Ready" ]]
+}
+
 wait_for_rook_operator() {
     local max_attempts=10
     local sleep_interval=20
     local attempts=0
+
     $KUBECTL rollout status deployment rook-ceph-operator -n rook-ceph --timeout=180s
-    PHASE=$($KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}')
-    echo "PHASE: $PHASE"
-    while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}' | grep -q "Ready"; do
-	echo "Waiting for cluster to be ready..."
-	sleep $sleep_interval
-	attempts=$((attempts+1))
+
+    while ! is_operator_ready; do
+        echo "Waiting for rook operator to be ready..."
+        sleep $sleep_interval
+
+	# log current cluster state and pods info for debugging
+        PHASE=$($KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}')
+        $KUBECTL -n rook-ceph get pods
+
+        attempts=$((attempts + 1))
         if [ $attempts -ge $max_attempts ]; then
             echo "Maximum number of attempts ($max_attempts) reached. Exiting..."
+            $KUBECTL -n rook-ceph get pods | grep operator | awk '{print $1}' | xargs $KUBECTL -n rook-ceph logs
             return 1
         fi
     done
@@ -93,9 +100,9 @@ wait_for_ceph_cluster() {
     local max_attempts=10
     local sleep_interval=20
     local attempts=0
-    $KUBECTL rollout status deployment rook-ceph-tools -n rook-ceph --timeout=30s
+    $KUBECTL rollout status deployment rook-ceph-tools -n rook-ceph --timeout=90s
     while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.ceph.health}' | grep -q "HEALTH_OK"; do
-	echo "Waiting for Ceph cluster installed"
+	echo "Waiting for Ceph cluster to enter HEALTH_OK" state
 	sleep $sleep_interval
 	attempts=$((attempts+1))
         if [ $attempts -ge $max_attempts ]; then
@@ -104,18 +111,66 @@ wait_for_ceph_cluster() {
         fi
     done
     echo "Ceph cluster installed and running"
+
+    # add an additional wait to cover with any subttle change in the state
+    sleep 20
+}
+
+configure_libvirt(){
+    if sudo usermod -aG libvirt $(id -un); then
+	echo "User added to libvirt group successfully."
+	sudo systemctl enable --now libvirtd
+	sudo systemctl restart libvirtd
+	sleep 30 # wait some time for libvirtd service to restart
+	newgrp libvirt
+    else
+	echo "Error adding user to libvirt group."
+	return 1
+    fi
+}
+
+recreate_default_network(){
+
+    # destroy any existing kvm default network
+    if sudo virsh net-destroy default; then
+	sudo virsh net-undefine default
+    fi
+
+    # let's create a new kvm default network
+    sudo virsh net-define /usr/share/libvirt/networks/default.xml
+    if sudo virsh net-start default; then
+        echo "Network 'default' started successfully."
+    else
+        # Optionally, handle the error
+        echo "Failed to start network 'default', but continuing..."
+    fi
+
+    # restart libvirtd service and wait a little bit for the service
+    sudo systemctl restart libvirtd
+    sleep 30
+
+    # Just some debugging information
+    all_networks=$(virsh net-list --all)
+    groups=$(groups)
+}
+
+enable_rook_orchestrator() {
+    echo "Enabling rook orchestrator"
+    $KUBECTL rollout status deployment rook-ceph-tools -n "$ROOK_CLUSTER_NS" --timeout=90s
+    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph mgr module enable rook
+    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch set backend rook
+    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch status
 }
 
-show_info() {
-    DASHBOARD_PASSWORD=$($KUBECTL -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo)
-    IP_ADDR=$($KUBECTL get po --selector="app=rook-ceph-mgr" -n rook-ceph --output jsonpath='{.items[*].status.hostIP}')
-    PORT="$($KUBECTL -n rook-ceph -o=jsonpath='{.spec.ports[?(@.name == "dashboard")].nodePort}' get services rook-ceph-mgr-dashboard-external-http)"
-    BASE_URL="http://$IP_ADDR:$PORT"
-    echo "==========================="
-    echo "Ceph Dashboard:  "
-    echo "   IP_ADDRESS: $BASE_URL"
-    echo "   PASSWORD: $DASHBOARD_PASSWORD"
-    echo "==========================="
+enable_monitoring() {
+    echo "Enabling monitoring"
+    $KUBECTL apply -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.40.0/bundle.yaml
+    $KUBECTL wait --for=condition=ready pod -l app.kubernetes.io/name=prometheus-operator --timeout=90s
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/rbac.yaml
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/service-monitor.yaml
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/exporter-service-monitor.yaml
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/prometheus.yaml
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/prometheus-service.yaml
 }
 
 ####################################################################
@@ -124,12 +179,15 @@ show_info() {
 trap 'on_error $? $LINENO' ERR
 
 configure_libvirt
+recreate_default_network
 setup_minikube_env
 build_ceph_image
 create_rook_cluster
 wait_for_rook_operator
 wait_for_ceph_cluster
-show_info
+enable_rook_orchestrator
+enable_monitoring
+sleep 30 # wait for the metrics cache warmup
 
 ####################################################################
 ####################################################################
diff --git a/src/pybind/mgr/rook/ci/tests/features/cluster-prometheus-monitoring.feature b/src/pybind/mgr/rook/ci/tests/features/cluster-prometheus-monitoring.feature
new file mode 100644
index 000000000..5180c7293
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/tests/features/cluster-prometheus-monitoring.feature
@@ -0,0 +1,14 @@
+Feature: Testing Rook orchestrator commands
+    Ceph has been installed using the cluster CRD available in deploy/examples/cluster-test.yaml
+
+    Scenario: Verify Prometheus metrics endpoint is working properly
+      Given I can get prometheus server configuration
+      Given the prometheus server is serving metrics
+
+    Scenario: Verify some basic metrics are working properly
+      Given I can get prometheus server configuration
+      Given the prometheus server is serving metrics
+      Then the response contains the metric "ceph_osd_in" where "ceph_daemon" is "osd.0" and value equal to 1
+      Then the response contains the metric "ceph_osd_in" where "ceph_daemon" is "osd.1" and value equal to 1
+      Then the response contains the metric "ceph_osd_in" where "ceph_daemon" is "osd.2" and value equal to 1
+      Then the response contains the metric "ceph_mon_quorum_status" where "ceph_daemon" is "mon.a" and value equal to 1
diff --git a/src/pybind/mgr/rook/ci/tests/features/rook.feature b/src/pybind/mgr/rook/ci/tests/features/rook.feature
index ae0478f8b..acf733f55 100644
--- a/src/pybind/mgr/rook/ci/tests/features/rook.feature
+++ b/src/pybind/mgr/rook/ci/tests/features/rook.feature
@@ -1,8 +1,8 @@
 Feature: Testing Rook orchestrator commands
-    Ceph has been installed using the cluster CRD available in deploy/examples/cluster-test.yaml and
+    Ceph has been installed using the cluster CRD available in deploy/examples/cluster-test.yaml
 
     Scenario: Verify ceph cluster health
-      When I run
+      When I run ceph command
           """
           ceph health | grep HEALTH
           """
@@ -10,3 +10,58 @@ Feature: Testing Rook orchestrator commands
           """
           HEALTH_OK
           """
+
+    Scenario: Verify rook orchestrator has been enabled correctly
+      When I run ceph command
+          """
+          ceph mgr module ls | grep rook
+          """
+      Then I get something like
+          """
+          rook +on
+          """
+
+    Scenario: Verify rook orchestrator lists services correctly
+        When I run ceph command
+            """
+            ceph orch ls
+            """
+        Then I get something like
+            """
+            NAME +PORTS +RUNNING +REFRESHED +AGE +PLACEMENT
+            crash +1/1 .+
+            mgr +1/1 .+
+            mon +1/1 .+
+            osd +3 .+
+            """
+
+    Scenario: Verify rook orchestrator lists daemons correctly
+        When I run ceph command
+            """
+            ceph orch ps
+            """
+        Then I get something like
+            """
+            NAME +HOST +PORTS +STATUS +REFRESHED +AGE +MEM +USE +MEM +LIM +VERSION +IMAGE +ID
+            ceph-exporter.exporter +minikube +running .+
+            crashcollector.crash +minikube +running .+
+            mgr.a +minikube +running .+
+            mon.a +minikube +running .+
+            osd.0 +minikube +running .+
+            osd.1 +minikube +running .+
+            osd.2 +minikube +running .+
+            """
+
+    Scenario: Verify rook orchestrator lists devices correctly
+        When I run ceph command
+            """
+            ceph orch device ls
+            """
+        Then I get something like
+            """
+            HOST +PATH +TYPE +DEVICE +ID +SIZE +AVAILABLE +REFRESHED +REJECT +REASONS
+            minikube +/dev/vdb  +unknown +None +10.0G .+
+            minikube +/dev/vdc  +unknown +None +20.0G .+
+            minikube +/dev/vdd  +unknown +None +20.0G .+
+            minikube +/dev/vde  +unknown +None +20.0G .+
+            """
diff --git a/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py b/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
index adde61afd..59cb117c8 100644
--- a/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
+++ b/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
@@ -1,15 +1,35 @@
+import requests
+from behave import given, when, then
 from behave import *
 from utils import *
+import subprocess
 import re
 
-@when("I run")
+PROMETHEUS_SERVER_URL = None
+
+def get_prometheus_pod_host_ip():
+    try:
+        command = "minikube --profile minikube kubectl -- -n rook-ceph -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0"
+        result = subprocess.run(command, shell=True, capture_output=True, text=True, check=True)
+        host_ip = result.stdout.strip()
+        return host_ip
+    except subprocess.CalledProcessError as e:
+        print(f"Error running command: {e}")
+        return None
+
+@when("I run ceph command")
+def run_step(context):
+    context.output = run_ceph_commands(context.text)
+
+@when("I run k8s command")
 def run_step(context):
-    context.output = run_commands(context.text)
+    context.output = run_k8s_commands(context.text)
 
 @then("I get")
 def verify_result_step(context):
-    print(f"Output is:\n{context.output}\n--------------\n")
-    assert context.text == context.output
+    if (context.text != context.output):
+        display_side_by_side(context.text, context.output)
+    assert context.text == context.output, ""
 
 @then("I get something like")
 def verify_fuzzy_result_step(context):
@@ -18,4 +38,83 @@ def verify_fuzzy_result_step(context):
     num_lines = min(len(output_lines), len(expected_lines))
     for n in range(num_lines):
         if not re.match(expected_lines[n], output_lines[n]):
-            raise
+            display_side_by_side(expected_lines[n], output_lines[n])
+            assert False, ""
+
+@given('I can get prometheus server configuration')
+def step_get_prometheus_server_ip(context):
+    global PROMETHEUS_SERVER_URL
+    try:
+        PROMETHEUS_SERVER_URL = f"http://{get_prometheus_pod_host_ip()}:30900"
+    except requests.exceptions.RequestException as e:
+        print(f"Error connecting to Prometheus server: {e}")
+        assert False, f"Error connecting to Prometheus server: {e}"
+
+@given('the prometheus server is serving metrics')
+def step_given_server_running(context):
+    try:
+        params = {'match[]': '{__name__!=""}'}
+        response = requests.get(f"{PROMETHEUS_SERVER_URL}/federate", params)
+        # Check if the response status code is successful (2xx)
+        response.raise_for_status()
+        # Store the response object in the context for later use
+        context.response = response
+        print(f"Prometheus server is running. Status code: {response.status_code}")
+    except requests.exceptions.RequestException as e:
+        print(f"Error connecting to Prometheus server: {e}")
+        assert False, f"Error connecting to Prometheus server: {e}"
+
+@when('I query the Prometheus metrics endpoint')
+def step_when_query_metrics_endpoint(context):
+    params = {'match[]': '{__name__!=""}'}
+    context.response = requests.get(f"{PROMETHEUS_SERVER_URL}/federate", params)
+    context.response.raise_for_status()
+
+@then('the response contains the metric "{metric_name}"')
+def step_then_check_metric_value(context, metric_name):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+
+@then('the response contains the metric "{metric_name}" with value equal to {expected_value}')
+def step_then_check_metric_value(context, metric_name, expected_value):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value == float(expected_value), f"Metric '{metric_name}' value {metric_value} is not equal to {expected_value}"
+
+@then('the response contains the metric "{metric_name}" with value greater than {expected_value}')
+def step_then_check_metric_value(context, metric_name, expected_value):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value > float(expected_value), f"Metric '{metric_name}' value {metric_value} is not greater than {expected_value}"
+
+@then('the response contains the metric "{metric_name}" with value less than {expected_value}')
+def step_then_check_metric_value(context, metric_name, expected_value):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value < float(expected_value), f"Metric '{metric_name}' value {metric_value} is not less than {expected_value}"
+
+@then('the response contains the metric "{metric_name}" with value in the range {min_value}-{max_value}')
+def step_then_check_metric_value(context, metric_name, min_value, max_value):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value >= float(min_value) and metric_value <= float(max_value), f"Metric '{metric_name}' value {metric_value} is not in the range {min_value}-{max_value}"
+
+@then('the response contains the metric "{metric_name}" where "{filter_by_field}" is "{field_value}" and value equal to {expected_value}')
+def step_then_check_metric_value(context, metric_name, expected_value, filter_by_field, field_value):
+    metric_value = parse_metric_value(context.response.text, metric_name, filter_by_field, field_value)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value == float(expected_value), f"Metric '{metric_name}' value {metric_value} is not equal to {expected_value}"
+
+
+def parse_metric_value(metrics_text, metric_name, filter_by_field=None, field_value=None):
+    filter_condition = f'{filter_by_field}="{field_value}"' if filter_by_field and field_value else ''
+    pattern_str = rf'^{metric_name}\{{[^}}]*{filter_condition}[^}}]*\}} (\d+) (\d+)'
+    pattern = re.compile(pattern_str, re.MULTILINE)
+    match = pattern.search(metrics_text)
+    if match:
+        # Extract the values and timestamp from the matched groups
+        metric_value, _ = match.groups()
+        return float(metric_value)
+    else:
+        # Metric not found
+        return None
diff --git a/src/pybind/mgr/rook/ci/tests/features/steps/utils.py b/src/pybind/mgr/rook/ci/tests/features/steps/utils.py
index 41a71d0fb..f711ec3fe 100644
--- a/src/pybind/mgr/rook/ci/tests/features/steps/utils.py
+++ b/src/pybind/mgr/rook/ci/tests/features/steps/utils.py
@@ -1,4 +1,5 @@
 import subprocess
+from difflib import unified_diff
 
 ROOK_CEPH_COMMAND = "minikube kubectl -- -n rook-ceph exec -it deploy/rook-ceph-tools -- "
 CLUSTER_COMMAND = "minikube kubectl -- "
@@ -27,3 +28,25 @@ def run_commands(commands: str) -> str:
         output = execute_command(command)
 
     return output.strip("\n")
+
+def run_k8s_commands(commands: str) -> str:
+    commands_list = commands.split("\n")
+    output = ""
+    for cmd in commands_list:
+        command = CLUSTER_COMMAND + cmd
+        output = execute_command(command)
+
+    return output.strip("\n")
+
+def run_ceph_commands(commands: str) -> str:
+    commands_list = commands.split("\n")
+    output = ""
+    for cmd in commands_list:
+        command = ROOK_CEPH_COMMAND + cmd
+        output = execute_command(command)
+
+    return output.strip("\n")
+
+def display_side_by_side(expected, got):
+    diff = unified_diff(expected.splitlines(), got.splitlines(), lineterm='')
+    print('\n'.join(diff))