diff options
Diffstat (limited to 'src/spdk/ocf/tests/functional/pyocf')
22 files changed, 2679 insertions, 0 deletions
diff --git a/src/spdk/ocf/tests/functional/pyocf/__init__.py b/src/spdk/ocf/tests/functional/pyocf/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/__init__.py diff --git a/src/spdk/ocf/tests/functional/pyocf/ocf.py b/src/spdk/ocf/tests/functional/pyocf/ocf.py new file mode 100644 index 000000000..b24d8265f --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/ocf.py @@ -0,0 +1,30 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# +from ctypes import c_void_p, cdll +import inspect +import os + + +class OcfLib: + __lib__ = None + + @classmethod + def getInstance(cls): + if cls.__lib__ is None: + lib = cdll.LoadLibrary( + os.path.join( + os.path.dirname(inspect.getfile(inspect.currentframe())), + "libocf.so", + ) + ) + lib.ocf_volume_get_uuid.restype = c_void_p + lib.ocf_volume_get_uuid.argtypes = [c_void_p] + + lib.ocf_core_get_front_volume.restype = c_void_p + lib.ocf_core_get_front_volume.argtypes = [c_void_p] + + cls.__lib__ = lib + + return cls.__lib__ diff --git a/src/spdk/ocf/tests/functional/pyocf/types/__init__.py b/src/spdk/ocf/tests/functional/pyocf/types/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/__init__.py diff --git a/src/spdk/ocf/tests/functional/pyocf/types/cache.py b/src/spdk/ocf/tests/functional/pyocf/types/cache.py new file mode 100644 index 000000000..1a74a05f3 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/cache.py @@ -0,0 +1,593 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import ( + c_uint64, + c_uint32, + c_uint16, + c_int, + c_char, + c_char_p, + c_void_p, + c_bool, + c_uint8, + Structure, + byref, + cast, + create_string_buffer, +) +from enum import IntEnum +from datetime import timedelta + +from ..ocf import OcfLib +from .shared import ( + Uuid, + OcfError, + CacheLineSize, + CacheLines, + OcfCompletion, + SeqCutOffPolicy, +) +from ..utils import Size, struct_to_dict +from .core import Core +from .queue import Queue +from .stats.cache import CacheInfo +from .stats.shared import UsageStats, RequestsStats, BlocksStats, ErrorsStats + + +class Backfill(Structure): + _fields_ = [("_max_queue_size", c_uint32), ("_queue_unblock_size", c_uint32)] + + +class CacheConfig(Structure): + MAX_CACHE_NAME_SIZE = 32 + _fields_ = [ + ("_name", c_char * MAX_CACHE_NAME_SIZE), + ("_cache_mode", c_uint32), + ("_eviction_policy", c_uint32), + ("_promotion_policy", c_uint32), + ("_cache_line_size", c_uint64), + ("_metadata_layout", c_uint32), + ("_metadata_volatile", c_bool), + ("_backfill", Backfill), + ("_locked", c_bool), + ("_pt_unaligned_io", c_bool), + ("_use_submit_io_fast", c_bool), + ] + + +class CacheDeviceConfig(Structure): + _fields_ = [ + ("_uuid", Uuid), + ("_volume_type", c_uint8), + ("_cache_line_size", c_uint64), + ("_force", c_bool), + ("_min_free_ram", c_uint64), + ("_perform_test", c_bool), + ("_discard_on_start", c_bool), + ] + + +class ConfValidValues: + promotion_nhit_insertion_threshold_range = range(2, 1000) + promotion_nhit_trigger_threshold_range = range(0, 100) + + +class CacheMode(IntEnum): + WT = 0 + WB = 1 + WA = 2 + PT = 3 + WI = 4 + WO = 5 + DEFAULT = WT + + def lazy_write(self): + return self.value in [CacheMode.WB, CacheMode.WO] + + def write_insert(self): + return self.value not in [CacheMode.PT, CacheMode.WA, CacheMode.WI] + + def read_insert(self): + return self.value not in [CacheMode.PT, CacheMode.WO] + + +class EvictionPolicy(IntEnum): + LRU = 0 + DEFAULT = LRU + + +class PromotionPolicy(IntEnum): + ALWAYS = 0 + NHIT = 1 + DEFAULT = ALWAYS + + +class NhitParams(IntEnum): + INSERTION_THRESHOLD = 0 + TRIGGER_THRESHOLD = 1 + + +class CleaningPolicy(IntEnum): + NOP = 0 + ALRU = 1 + ACP = 2 + DEFAULT = ALRU + + +class AlruParams(IntEnum): + WAKE_UP_TIME = 0 + STALE_BUFFER_TIME = 1 + FLUSH_MAX_BUFFERS = 2 + ACTIVITY_THRESHOLD = 3 + + +class AcpParams(IntEnum): + WAKE_UP_TIME = 0 + FLUSH_MAX_BUFFERS = 1 + + +class MetadataLayout(IntEnum): + STRIPING = 0 + SEQUENTIAL = 1 + DEFAULT = STRIPING + + +class Cache: + DEFAULT_BACKFILL_QUEUE_SIZE = 65536 + DEFAULT_BACKFILL_UNBLOCK = 60000 + DEFAULT_PT_UNALIGNED_IO = False + DEFAULT_USE_SUBMIT_FAST = False + + def __init__( + self, + owner, + name: str = "cache", + cache_mode: CacheMode = CacheMode.DEFAULT, + eviction_policy: EvictionPolicy = EvictionPolicy.DEFAULT, + promotion_policy: PromotionPolicy = PromotionPolicy.DEFAULT, + cache_line_size: CacheLineSize = CacheLineSize.DEFAULT, + metadata_layout: MetadataLayout = MetadataLayout.DEFAULT, + metadata_volatile: bool = False, + max_queue_size: int = DEFAULT_BACKFILL_QUEUE_SIZE, + queue_unblock_size: int = DEFAULT_BACKFILL_UNBLOCK, + locked: bool = False, + pt_unaligned_io: bool = DEFAULT_PT_UNALIGNED_IO, + use_submit_fast: bool = DEFAULT_USE_SUBMIT_FAST, + ): + self.device = None + self.started = False + self.owner = owner + self.cache_line_size = cache_line_size + + self.cfg = CacheConfig( + _name=name.encode("ascii"), + _cache_mode=cache_mode, + _eviction_policy=eviction_policy, + _promotion_policy=promotion_policy, + _cache_line_size=cache_line_size, + _metadata_layout=metadata_layout, + _metadata_volatile=metadata_volatile, + _backfill=Backfill( + _max_queue_size=max_queue_size, _queue_unblock_size=queue_unblock_size + ), + _locked=locked, + _pt_unaligned_io=pt_unaligned_io, + _use_submit_fast=use_submit_fast, + ) + self.cache_handle = c_void_p() + self._as_parameter_ = self.cache_handle + self.io_queues = [] + self.cores = [] + + def start_cache(self, default_io_queue: Queue = None, mngt_queue: Queue = None): + status = self.owner.lib.ocf_mngt_cache_start( + self.owner.ctx_handle, byref(self.cache_handle), byref(self.cfg) + ) + if status: + raise OcfError("Creating cache instance failed", status) + self.owner.caches.append(self) + + self.mngt_queue = mngt_queue or Queue(self, "mgmt-{}".format(self.get_name())) + + if default_io_queue: + self.io_queues += [default_io_queue] + else: + self.io_queues += [Queue(self, "default-io-{}".format(self.get_name()))] + + status = self.owner.lib.ocf_mngt_cache_set_mngt_queue(self, self.mngt_queue) + if status: + raise OcfError("Error setting management queue", status) + + self.started = True + + def change_cache_mode(self, cache_mode: CacheMode): + self.write_lock() + status = self.owner.lib.ocf_mngt_cache_set_mode(self.cache_handle, cache_mode) + + self.write_unlock() + + if status: + raise OcfError("Error changing cache mode", status) + + def set_cleaning_policy(self, cleaning_policy: CleaningPolicy): + self.write_lock() + + status = self.owner.lib.ocf_mngt_cache_cleaning_set_policy( + self.cache_handle, cleaning_policy + ) + + self.write_unlock() + + if status: + raise OcfError("Error changing cleaning policy", status) + + def set_cleaning_policy_param( + self, cleaning_policy: CleaningPolicy, param_id, param_value + ): + self.write_lock() + + status = self.owner.lib.ocf_mngt_cache_cleaning_set_param( + self.cache_handle, cleaning_policy, param_id, param_value + ) + + self.write_unlock() + + if status: + raise OcfError("Error setting cleaning policy param", status) + + def set_promotion_policy(self, promotion_policy: PromotionPolicy): + self.write_lock() + + status = self.owner.lib.ocf_mngt_cache_promotion_set_policy( + self.cache_handle, promotion_policy + ) + + self.write_unlock() + if status: + raise OcfError("Error setting promotion policy", status) + + def get_promotion_policy_param(self, promotion_type, param_id): + self.read_lock() + + param_value = c_uint64() + + status = self.owner.lib.ocf_mngt_cache_promotion_get_param( + self.cache_handle, promotion_type, param_id, byref(param_value) + ) + + self.read_unlock() + if status: + raise OcfError("Error getting promotion policy parameter", status) + + return param_value + + def set_promotion_policy_param(self, promotion_type, param_id, param_value): + self.write_lock() + + status = self.owner.lib.ocf_mngt_cache_promotion_set_param( + self.cache_handle, promotion_type, param_id, param_value + ) + + self.write_unlock() + if status: + raise OcfError("Error setting promotion policy parameter", status) + + def set_seq_cut_off_policy(self, policy: SeqCutOffPolicy): + self.write_lock() + + status = self.owner.lib.ocf_mngt_core_set_seq_cutoff_policy_all( + self.cache_handle, policy + ) + + self.write_unlock() + + if status: + raise OcfError("Error setting cache seq cut off policy", status) + + def configure_device( + self, device, force=False, perform_test=True, cache_line_size=None + ): + self.device = device + self.device_name = device.uuid + self.dev_cfg = CacheDeviceConfig( + _uuid=Uuid( + _data=cast( + create_string_buffer(self.device_name.encode("ascii")), c_char_p + ), + _size=len(self.device_name) + 1, + ), + _volume_type=device.type_id, + _cache_line_size=cache_line_size + if cache_line_size + else self.cache_line_size, + _force=force, + _min_free_ram=0, + _perform_test=perform_test, + _discard_on_start=False, + ) + + def attach_device( + self, device, force=False, perform_test=False, cache_line_size=None + ): + self.configure_device(device, force, perform_test, cache_line_size) + self.write_lock() + + c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)]) + + device.owner.lib.ocf_mngt_cache_attach( + self.cache_handle, byref(self.dev_cfg), c, None + ) + + c.wait() + self.write_unlock() + + if c.results["error"]: + raise OcfError("Attaching cache device failed", c.results["error"]) + + def load_cache(self, device): + self.configure_device(device) + c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)]) + device.owner.lib.ocf_mngt_cache_load( + self.cache_handle, byref(self.dev_cfg), c, None + ) + + c.wait() + if c.results["error"]: + raise OcfError("Loading cache device failed", c.results["error"]) + + @classmethod + def load_from_device(cls, device, name="cache"): + c = cls(name=name, owner=device.owner) + + c.start_cache() + try: + c.load_cache(device) + except: # noqa E722 + c.stop() + raise + + return c + + @classmethod + def start_on_device(cls, device, **kwargs): + c = cls(owner=device.owner, **kwargs) + + c.start_cache() + try: + c.attach_device(device, force=True) + except: # noqa E722 + c.stop() + raise + + return c + + def put(self): + self.owner.lib.ocf_mngt_cache_put(self.cache_handle) + + def get(self): + status = self.owner.lib.ocf_mngt_cache_get(self.cache_handle) + if status: + raise OcfError("Couldn't get cache instance", status) + + def read_lock(self): + c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)]) + self.owner.lib.ocf_mngt_cache_read_lock(self.cache_handle, c, None) + c.wait() + if c.results["error"]: + raise OcfError("Couldn't lock cache instance", c.results["error"]) + + def write_lock(self): + c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)]) + self.owner.lib.ocf_mngt_cache_lock(self.cache_handle, c, None) + c.wait() + if c.results["error"]: + raise OcfError("Couldn't lock cache instance", c.results["error"]) + + def read_unlock(self): + self.owner.lib.ocf_mngt_cache_read_unlock(self.cache_handle) + + def write_unlock(self): + self.owner.lib.ocf_mngt_cache_unlock(self.cache_handle) + + def add_core(self, core: Core): + self.write_lock() + + c = OcfCompletion( + [ + ("cache", c_void_p), + ("core", c_void_p), + ("priv", c_void_p), + ("error", c_int), + ] + ) + + self.owner.lib.ocf_mngt_cache_add_core( + self.cache_handle, byref(core.get_cfg()), c, None + ) + + c.wait() + if c.results["error"]: + self.write_unlock() + raise OcfError("Failed adding core", c.results["error"]) + + core.cache = self + core.handle = c.results["core"] + self.cores.append(core) + + self.write_unlock() + + def remove_core(self, core: Core): + self.write_lock() + + c = OcfCompletion([("priv", c_void_p), ("error", c_int)]) + + self.owner.lib.ocf_mngt_cache_remove_core(core.handle, c, None) + + c.wait() + self.write_unlock() + + if c.results["error"]: + raise OcfError("Failed removing core", c.results["error"]) + + self.cores.remove(core) + + def get_stats(self): + cache_info = CacheInfo() + usage = UsageStats() + req = RequestsStats() + block = BlocksStats() + errors = ErrorsStats() + + self.read_lock() + + status = self.owner.lib.ocf_cache_get_info(self.cache_handle, byref(cache_info)) + if status: + self.read_unlock() + raise OcfError("Failed getting cache info", status) + + status = self.owner.lib.ocf_stats_collect_cache( + self.cache_handle, byref(usage), byref(req), byref(block), byref(errors) + ) + if status: + self.read_unlock() + raise OcfError("Failed getting stats", status) + + line_size = CacheLineSize(cache_info.cache_line_size) + cache_name = self.owner.lib.ocf_cache_get_name(self).decode("ascii") + + self.read_unlock() + return { + "conf": { + "attached": cache_info.attached, + "volume_type": self.owner.volume_types[cache_info.volume_type], + "size": CacheLines(cache_info.size, line_size), + "inactive": { + "occupancy": CacheLines( + cache_info.inactive.occupancy.value, line_size + ), + "dirty": CacheLines(cache_info.inactive.dirty.value, line_size), + "clean": CacheLines(cache_info.inactive.clean.value, line_size), + }, + "occupancy": CacheLines(cache_info.occupancy, line_size), + "dirty": CacheLines(cache_info.dirty, line_size), + "dirty_initial": CacheLines(cache_info.dirty_initial, line_size), + "dirty_for": timedelta(seconds=cache_info.dirty_for), + "cache_mode": CacheMode(cache_info.cache_mode), + "fallback_pt": { + "error_counter": cache_info.fallback_pt.error_counter, + "status": cache_info.fallback_pt.status, + }, + "state": cache_info.state, + "eviction_policy": EvictionPolicy(cache_info.eviction_policy), + "cleaning_policy": CleaningPolicy(cache_info.cleaning_policy), + "promotion_policy": PromotionPolicy(cache_info.promotion_policy), + "cache_line_size": line_size, + "flushed": CacheLines(cache_info.flushed, line_size), + "core_count": cache_info.core_count, + "metadata_footprint": Size(cache_info.metadata_footprint), + "metadata_end_offset": Size(cache_info.metadata_end_offset), + "cache_name": cache_name, + }, + "block": struct_to_dict(block), + "req": struct_to_dict(req), + "usage": struct_to_dict(usage), + "errors": struct_to_dict(errors), + } + + def reset_stats(self): + self.owner.lib.ocf_core_stats_initialize_all(self.cache_handle) + + def get_default_queue(self): + if not self.io_queues: + raise Exception("No queues added for cache") + + return self.io_queues[0] + + def save(self): + if not self.started: + raise Exception("Not started!") + + self.get_and_write_lock() + c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)]) + self.owner.lib.ocf_mngt_cache_save(self.cache_handle, c, None) + + c.wait() + self.put_and_write_unlock() + + if c.results["error"]: + raise OcfError("Failed saving cache", c.results["error"]) + + def stop(self): + if not self.started: + raise Exception("Already stopped!") + + self.write_lock() + + c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)]) + + self.owner.lib.ocf_mngt_cache_stop(self.cache_handle, c, None) + + c.wait() + if c.results["error"]: + self.write_unlock() + raise OcfError("Failed stopping cache", c.results["error"]) + + self.mngt_queue.put() + del self.io_queues[:] + self.started = False + + self.write_unlock() + + self.owner.caches.remove(self) + + def flush(self): + self.write_lock() + + c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)]) + self.owner.lib.ocf_mngt_cache_flush(self.cache_handle, c, None) + c.wait() + self.write_unlock() + + if c.results["error"]: + raise OcfError("Couldn't flush cache", c.results["error"]) + + def get_name(self): + self.read_lock() + + try: + return str(self.owner.lib.ocf_cache_get_name(self), encoding="ascii") + except: # noqa E722 + raise OcfError("Couldn't get cache name") + finally: + self.read_unlock() + + +lib = OcfLib.getInstance() +lib.ocf_mngt_cache_remove_core.argtypes = [c_void_p, c_void_p, c_void_p] +lib.ocf_mngt_cache_add_core.argtypes = [c_void_p, c_void_p, c_void_p, c_void_p] +lib.ocf_cache_get_name.argtypes = [c_void_p] +lib.ocf_cache_get_name.restype = c_char_p +lib.ocf_mngt_cache_cleaning_set_policy.argtypes = [c_void_p, c_uint32] +lib.ocf_mngt_cache_cleaning_set_policy.restype = c_int +lib.ocf_mngt_core_set_seq_cutoff_policy_all.argtypes = [c_void_p, c_uint32] +lib.ocf_mngt_core_set_seq_cutoff_policy_all.restype = c_int +lib.ocf_stats_collect_cache.argtypes = [ + c_void_p, + c_void_p, + c_void_p, + c_void_p, + c_void_p, +] +lib.ocf_stats_collect_cache.restype = c_int +lib.ocf_cache_get_info.argtypes = [c_void_p, c_void_p] +lib.ocf_cache_get_info.restype = c_int +lib.ocf_mngt_cache_cleaning_set_param.argtypes = [ + c_void_p, + c_uint32, + c_uint32, + c_uint32, +] +lib.ocf_mngt_cache_cleaning_set_param.restype = c_int diff --git a/src/spdk/ocf/tests/functional/pyocf/types/cleaner.py b/src/spdk/ocf/tests/functional/pyocf/types/cleaner.py new file mode 100644 index 000000000..df28290aa --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/cleaner.py @@ -0,0 +1,43 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import c_void_p, CFUNCTYPE, Structure, c_int +from .shared import SharedOcfObject + + +class CleanerOps(Structure): + INIT = CFUNCTYPE(c_int, c_void_p) + KICK = CFUNCTYPE(None, c_void_p) + STOP = CFUNCTYPE(None, c_void_p) + + _fields_ = [("init", INIT), ("kick", KICK), ("stop", STOP)] + + +class Cleaner(SharedOcfObject): + _instances_ = {} + _fields_ = [("cleaner", c_void_p)] + + def __init__(self): + self._as_parameter_ = self.cleaner + super().__init__() + + @classmethod + def get_ops(cls): + return CleanerOps(init=cls._init, kick=cls._kick, stop=cls._stop) + + @staticmethod + @CleanerOps.INIT + def _init(cleaner): + return 0 + + @staticmethod + @CleanerOps.KICK + def _kick(cleaner): + pass + + @staticmethod + @CleanerOps.STOP + def _stop(cleaner): + pass diff --git a/src/spdk/ocf/tests/functional/pyocf/types/core.py b/src/spdk/ocf/tests/functional/pyocf/types/core.py new file mode 100644 index 000000000..0003c0daf --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/core.py @@ -0,0 +1,227 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import logging +from ctypes import ( + c_size_t, + c_void_p, + Structure, + c_int, + c_uint8, + c_uint16, + c_uint32, + c_uint64, + c_char, + c_char_p, + c_bool, + cast, + byref, + create_string_buffer, +) +from datetime import timedelta + +from .data import Data +from .io import Io, IoDir +from .queue import Queue +from .shared import Uuid, OcfCompletion, OcfError, SeqCutOffPolicy +from .stats.core import CoreInfo +from .stats.shared import UsageStats, RequestsStats, BlocksStats, ErrorsStats +from .volume import Volume +from ..ocf import OcfLib +from ..utils import Size, struct_to_dict + + +class UserMetadata(Structure): + _fields_ = [("data", c_void_p), ("size", c_size_t)] + + +class CoreConfig(Structure): + MAX_CORE_NAME_SIZE = 32 + _fields_ = [ + ("_name", c_char * MAX_CORE_NAME_SIZE), + ("_uuid", Uuid), + ("_volume_type", c_uint8), + ("_try_add", c_bool), + ("_seq_cutoff_threshold", c_uint32), + ("_user_metadata", UserMetadata), + ] + + +class Core: + DEFAULT_ID = 4096 + DEFAULT_SEQ_CUTOFF_THRESHOLD = 1024 * 1024 + + def __init__( + self, + device: Volume, + try_add: bool, + name: str = "core", + seq_cutoff_threshold: int = DEFAULT_SEQ_CUTOFF_THRESHOLD, + ): + self.cache = None + self.device = device + self.device_name = device.uuid + self.handle = c_void_p() + self.cfg = CoreConfig( + _uuid=Uuid( + _data=cast( + create_string_buffer(self.device_name.encode("ascii")), + c_char_p, + ), + _size=len(self.device_name) + 1, + ), + _name=name.encode("ascii"), + _volume_type=self.device.type_id, + _try_add=try_add, + _seq_cutoff_threshold=seq_cutoff_threshold, + _user_metadata=UserMetadata(_data=None, _size=0), + ) + + @classmethod + def using_device(cls, device, **kwargs): + c = cls(device=device, try_add=False, **kwargs) + + return c + + def get_cfg(self): + return self.cfg + + def get_handle(self): + return self.handle + + def new_io( + self, queue: Queue, addr: int, length: int, direction: IoDir, + io_class: int, flags: int + ): + if not self.cache: + raise Exception("Core isn't attached to any cache") + + io = OcfLib.getInstance().ocf_core_new_io_wrapper( + self.handle, queue.handle, addr, length, direction, io_class, flags) + + if io is None: + raise Exception("Failed to create io!") + + return Io.from_pointer(io) + + def new_core_io( + self, queue: Queue, addr: int, length: int, direction: IoDir, + io_class: int, flags: int + ): + lib = OcfLib.getInstance() + volume = lib.ocf_core_get_volume(self.handle) + io = lib.ocf_volume_new_io( + volume, queue.handle, addr, length, direction, io_class, flags) + return Io.from_pointer(io) + + def get_stats(self): + core_info = CoreInfo() + usage = UsageStats() + req = RequestsStats() + blocks = BlocksStats() + errors = ErrorsStats() + + self.cache.read_lock() + status = self.cache.owner.lib.ocf_stats_collect_core( + self.handle, byref(usage), byref(req), byref(blocks), byref(errors) + ) + if status: + self.cache.read_unlock() + raise OcfError("Failed collecting core stats", status) + + status = self.cache.owner.lib.ocf_core_get_info( + self.handle, byref(core_info) + ) + if status: + self.cache.read_unlock() + raise OcfError("Failed getting core stats", status) + + self.cache.read_unlock() + return { + "size": Size(core_info.core_size_bytes), + "dirty_for": timedelta(seconds=core_info.dirty_for), + "seq_cutoff_policy": SeqCutOffPolicy(core_info.seq_cutoff_policy), + "seq_cutoff_threshold": core_info.seq_cutoff_threshold, + "usage": struct_to_dict(usage), + "req": struct_to_dict(req), + "blocks": struct_to_dict(blocks), + "errors": struct_to_dict(errors), + } + + def set_seq_cut_off_policy(self, policy: SeqCutOffPolicy): + self.cache.write_lock() + + status = self.cache.owner.lib.ocf_mngt_core_set_seq_cutoff_policy( + self.handle, policy + ) + if status: + self.cache.write_unlock() + raise OcfError("Error setting core seq cut off policy", status) + + self.cache.write_unlock() + + def reset_stats(self): + self.cache.owner.lib.ocf_core_stats_initialize(self.handle) + + def exp_obj_md5(self): + logging.getLogger("pyocf").warning( + "Reading whole exported object! This disturbs statistics values" + ) + + cache_line_size = int(self.cache.get_stats()['conf']['cache_line_size']) + read_buffer_all = Data(self.device.size) + + read_buffer = Data(cache_line_size) + + position = 0 + while position < read_buffer_all.size: + io = self.new_io(self.cache.get_default_queue(), position, + cache_line_size, IoDir.READ, 0, 0) + io.set_data(read_buffer) + + cmpl = OcfCompletion([("err", c_int)]) + io.callback = cmpl.callback + io.submit() + cmpl.wait() + + if cmpl.results["err"]: + raise Exception("Error reading whole exported object") + + read_buffer_all.copy(read_buffer, position, 0, cache_line_size) + position += cache_line_size + + return read_buffer_all.md5() + + +lib = OcfLib.getInstance() +lib.ocf_core_get_volume.restype = c_void_p +lib.ocf_volume_new_io.argtypes = [ + c_void_p, + c_void_p, + c_uint64, + c_uint32, + c_uint32, + c_uint32, + c_uint64, +] +lib.ocf_volume_new_io.restype = c_void_p +lib.ocf_core_get_volume.argtypes = [c_void_p] +lib.ocf_core_get_volume.restype = c_void_p +lib.ocf_mngt_core_set_seq_cutoff_policy.argtypes = [c_void_p, c_uint32] +lib.ocf_mngt_core_set_seq_cutoff_policy.restype = c_int +lib.ocf_stats_collect_core.argtypes = [c_void_p, c_void_p, c_void_p, c_void_p, c_void_p] +lib.ocf_stats_collect_core.restype = c_int +lib.ocf_core_get_info.argtypes = [c_void_p, c_void_p] +lib.ocf_core_get_info.restype = c_int +lib.ocf_core_new_io_wrapper.argtypes = [ + c_void_p, + c_void_p, + c_uint64, + c_uint32, + c_uint32, + c_uint32, + c_uint64, +] +lib.ocf_core_new_io_wrapper.restype = c_void_p diff --git a/src/spdk/ocf/tests/functional/pyocf/types/ctx.py b/src/spdk/ocf/tests/functional/pyocf/types/ctx.py new file mode 100644 index 000000000..14c4b5757 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/ctx.py @@ -0,0 +1,122 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import c_void_p, Structure, c_char_p, cast, pointer, byref, c_int + +from .logger import LoggerOps, Logger +from .data import DataOps, Data +from .cleaner import CleanerOps, Cleaner +from .metadata_updater import MetadataUpdaterOps, MetadataUpdater +from .shared import OcfError +from ..ocf import OcfLib +from .queue import Queue +from .volume import Volume + + +class OcfCtxOps(Structure): + _fields_ = [ + ("data", DataOps), + ("cleaner", CleanerOps), + ("metadata_updater", MetadataUpdaterOps), + ("logger", LoggerOps), + ] + + +class OcfCtxCfg(Structure): + _fields_ = [("name", c_char_p), ("ops", OcfCtxOps), ("logger_priv", c_void_p)] + + +class OcfCtx: + def __init__(self, lib, name, logger, data, mu, cleaner): + self.logger = logger + self.data = data + self.mu = mu + self.cleaner = cleaner + self.ctx_handle = c_void_p() + self.lib = lib + self.volume_types_count = 1 + self.volume_types = {} + self.caches = [] + + self.cfg = OcfCtxCfg( + name=name, + ops=OcfCtxOps( + data=self.data.get_ops(), + cleaner=self.cleaner.get_ops(), + metadata_updater=self.mu.get_ops(), + logger=logger.get_ops(), + ), + logger_priv=cast(pointer(logger.get_priv()), c_void_p), + ) + + result = self.lib.ocf_ctx_create(byref(self.ctx_handle), byref(self.cfg)) + if result != 0: + raise OcfError("Context initialization failed", result) + + def register_volume_type(self, volume_type): + self.volume_types[self.volume_types_count] = volume_type + volume_type.type_id = self.volume_types_count + volume_type.owner = self + + result = self.lib.ocf_ctx_register_volume_type( + self.ctx_handle, + self.volume_types_count, + byref(self.volume_types[self.volume_types_count].get_props()), + ) + if result != 0: + raise OcfError("Volume type registration failed", result) + + self.volume_types_count += 1 + + def unregister_volume_type(self, vol_type): + if not vol_type.type_id: + raise Exception("Already unregistered") + + self.lib.ocf_ctx_unregister_volume_type( + self.ctx_handle, vol_type.type_id + ) + + del self.volume_types[vol_type.type_id] + + def cleanup_volume_types(self): + for k, vol_type in list(self.volume_types.items()): + if vol_type: + self.unregister_volume_type(vol_type) + + def stop_caches(self): + for cache in self.caches[:]: + cache.stop() + + def exit(self): + self.stop_caches() + self.cleanup_volume_types() + + self.lib.ocf_ctx_put(self.ctx_handle) + + self.cfg = None + self.logger = None + self.data = None + self.mu = None + self.cleaner = None + Queue._instances_ = {} + Volume._instances_ = {} + Data._instances_ = {} + Logger._instances_ = {} + + +def get_default_ctx(logger): + return OcfCtx( + OcfLib.getInstance(), + b"PyOCF default ctx", + logger, + Data, + MetadataUpdater, + Cleaner, + ) + + +lib = OcfLib.getInstance() +lib.ocf_mngt_cache_get_by_name.argtypes = [c_void_p, c_void_p, c_void_p] +lib.ocf_mngt_cache_get_by_name.restype = c_int diff --git a/src/spdk/ocf/tests/functional/pyocf/types/data.py b/src/spdk/ocf/tests/functional/pyocf/types/data.py new file mode 100644 index 000000000..b032cf3ce --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/data.py @@ -0,0 +1,225 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import ( + c_void_p, + c_uint32, + CFUNCTYPE, + c_uint64, + create_string_buffer, + cast, + memset, + string_at, + Structure, + c_int, + memmove, + byref, +) +from enum import IntEnum +from hashlib import md5 +import weakref + +from ..utils import print_buffer, Size as S + + +class DataSeek(IntEnum): + BEGIN = 0 + CURRENT = 1 + + +class DataOps(Structure): + ALLOC = CFUNCTYPE(c_void_p, c_uint32) + FREE = CFUNCTYPE(None, c_void_p) + MLOCK = CFUNCTYPE(c_int, c_void_p) + MUNLOCK = CFUNCTYPE(None, c_void_p) + READ = CFUNCTYPE(c_uint32, c_void_p, c_void_p, c_uint32) + WRITE = CFUNCTYPE(c_uint32, c_void_p, c_void_p, c_uint32) + ZERO = CFUNCTYPE(c_uint32, c_void_p, c_uint32) + SEEK = CFUNCTYPE(c_uint32, c_void_p, c_uint32, c_uint32) + COPY = CFUNCTYPE(c_uint64, c_void_p, c_void_p, c_uint64, c_uint64, c_uint64) + SECURE_ERASE = CFUNCTYPE(None, c_void_p) + + _fields_ = [ + ("_alloc", ALLOC), + ("_free", FREE), + ("_mlock", MLOCK), + ("_munlock", MUNLOCK), + ("_read", READ), + ("_write", WRITE), + ("_zero", ZERO), + ("_seek", SEEK), + ("_copy", COPY), + ("_secure_erase", SECURE_ERASE), + ] + + +class Data: + DATA_POISON = 0xA5 + PAGE_SIZE = 4096 + + _instances_ = {} + _ocf_instances_ = [] + + def __init__(self, byte_count: int): + self.size = int(byte_count) + self.position = 0 + self.buffer = create_string_buffer(int(self.size)) + self.handle = cast(byref(self.buffer), c_void_p) + + memset(self.handle, self.DATA_POISON, self.size) + type(self)._instances_[self.handle.value] = weakref.ref(self) + self._as_parameter_ = self.handle + + @classmethod + def get_instance(cls, ref): + return cls._instances_[ref]() + + @classmethod + def get_ops(cls): + return DataOps( + _alloc=cls._alloc, + _free=cls._free, + _mlock=cls._mlock, + _munlock=cls._munlock, + _read=cls._read, + _write=cls._write, + _zero=cls._zero, + _seek=cls._seek, + _copy=cls._copy, + _secure_erase=cls._secure_erase, + ) + + @classmethod + def pages(cls, pages: int): + return cls(pages * Data.PAGE_SIZE) + + @classmethod + def from_bytes(cls, source: bytes, offset: int = 0, size: int = 0): + if size == 0: + size = len(source) - offset + d = cls(size) + + memmove(d.handle, cast(source, c_void_p).value + offset, size) + + return d + + @classmethod + def from_string(cls, source: str, encoding: str = "ascii"): + b = bytes(source, encoding) + # duplicate string to fill space up to sector boundary + padding_len = S.from_B(len(b), sector_aligned=True).B - len(b) + padding = b * (padding_len // len(b) + 1) + padding = padding[:padding_len] + b = b + padding + return cls.from_bytes(b) + + @staticmethod + @DataOps.ALLOC + def _alloc(pages): + data = Data.pages(pages) + Data._ocf_instances_.append(data) + + return data.handle.value + + @staticmethod + @DataOps.FREE + def _free(ref): + Data._ocf_instances_.remove(Data.get_instance(ref)) + + @staticmethod + @DataOps.MLOCK + def _mlock(ref): + return Data.get_instance(ref).mlock() + + @staticmethod + @DataOps.MUNLOCK + def _munlock(ref): + Data.get_instance(ref).munlock() + + @staticmethod + @DataOps.READ + def _read(dst, src, size): + return Data.get_instance(src).read(dst, size) + + @staticmethod + @DataOps.WRITE + def _write(dst, src, size): + return Data.get_instance(dst).write(src, size) + + @staticmethod + @DataOps.ZERO + def _zero(dst, size): + return Data.get_instance(dst).zero(size) + + @staticmethod + @DataOps.SEEK + def _seek(dst, seek, size): + return Data.get_instance(dst).seek(DataSeek(seek), size) + + @staticmethod + @DataOps.COPY + def _copy(dst, src, skip, seek, size): + return Data.get_instance(dst).copy( + Data.get_instance(src), skip, seek, size + ) + + @staticmethod + @DataOps.SECURE_ERASE + def _secure_erase(dst): + Data.get_instance(dst).secure_erase() + + def read(self, dst, size): + to_read = min(self.size - self.position, size) + memmove(dst, self.handle.value + self.position, to_read) + + self.position += to_read + return to_read + + def write(self, src, size): + to_write = min(self.size - self.position, size) + memmove(self.handle.value + self.position, src, to_write) + + self.position += to_write + return to_write + + def mlock(self): + return 0 + + def munlock(self): + pass + + def zero(self, size): + to_zero = min(self.size - self.position, size) + memset(self.handle.value + self.position, 0, to_zero) + + self.position += to_zero + return to_zero + + def seek(self, seek, size): + if seek == DataSeek.CURRENT: + to_move = min(self.size - self.position, size) + self.position += to_move + else: + to_move = min(self.size, size) + self.position = to_move + + return to_move + + def copy(self, src, skip, seek, size): + to_write = min(self.size - skip, size, src.size - seek) + + memmove(self.handle.value + skip, src.handle.value + seek, to_write) + return to_write + + def secure_erase(self): + pass + + def dump(self, ignore=DATA_POISON, **kwargs): + print_buffer(self.buffer, self.size, ignore=ignore, **kwargs) + + def md5(self): + m = md5() + m.update(string_at(self.handle, self.size)) + return m.hexdigest() diff --git a/src/spdk/ocf/tests/functional/pyocf/types/io.py b/src/spdk/ocf/tests/functional/pyocf/types/io.py new file mode 100644 index 000000000..7e3671c5b --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/io.py @@ -0,0 +1,118 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import ( + c_void_p, + c_int, + c_uint32, + c_uint64, + CFUNCTYPE, + Structure, + POINTER, + byref, + cast, +) +from enum import IntEnum + +from ..ocf import OcfLib +from .data import Data + + +class IoDir(IntEnum): + READ = 0 + WRITE = 1 + + +class IoOps(Structure): + pass + + +class Io(Structure): + START = CFUNCTYPE(None, c_void_p) + HANDLE = CFUNCTYPE(None, c_void_p, c_void_p) + END = CFUNCTYPE(None, c_void_p, c_int) + + _instances_ = {} + _fields_ = [ + ("_addr", c_uint64), + ("_flags", c_uint64), + ("_bytes", c_uint32), + ("_class", c_uint32), + ("_dir", c_uint32), + ("_io_queue", c_void_p), + ("_start", START), + ("_handle", HANDLE), + ("_end", END), + ("_priv1", c_void_p), + ("_priv2", c_void_p), + ] + + @classmethod + def from_pointer(cls, ref): + c = cls.from_address(ref) + cls._instances_[ref] = c + OcfLib.getInstance().ocf_io_set_cmpl_wrapper( + byref(c), None, None, c.c_end + ) + return c + + @classmethod + def get_instance(cls, ref): + return cls._instances_[cast(ref, c_void_p).value] + + def del_object(self): + del type(self)._instances_[cast(byref(self), c_void_p).value] + + def put(self): + OcfLib.getInstance().ocf_io_put(byref(self)) + + def get(self): + OcfLib.getInstance().ocf_io_get(byref(self)) + + @staticmethod + @END + def c_end(io, err): + Io.get_instance(io).end(err) + + @staticmethod + @START + def c_start(io): + Io.get_instance(io).start() + + @staticmethod + @HANDLE + def c_handle(io, opaque): + Io.get_instance(io).handle(opaque) + + def end(self, err): + try: + self.callback(err) + except: # noqa E722 + pass + + self.put() + self.del_object() + + def submit(self): + return OcfLib.getInstance().ocf_core_submit_io_wrapper(byref(self)) + + def set_data(self, data: Data, offset: int = 0): + self.data = data + OcfLib.getInstance().ocf_io_set_data(byref(self), data, offset) + + +IoOps.SET_DATA = CFUNCTYPE(c_int, POINTER(Io), c_void_p, c_uint32) +IoOps.GET_DATA = CFUNCTYPE(c_void_p, POINTER(Io)) + +IoOps._fields_ = [("_set_data", IoOps.SET_DATA), ("_get_data", IoOps.GET_DATA)] + +lib = OcfLib.getInstance() +lib.ocf_io_set_cmpl_wrapper.argtypes = [POINTER(Io), c_void_p, c_void_p, Io.END] + +lib.ocf_core_new_io_wrapper.argtypes = [c_void_p] +lib.ocf_core_new_io_wrapper.restype = c_void_p + +lib.ocf_io_set_data.argtypes = [POINTER(Io), c_void_p, c_uint32] +lib.ocf_io_set_data.restype = c_int diff --git a/src/spdk/ocf/tests/functional/pyocf/types/logger.py b/src/spdk/ocf/tests/functional/pyocf/types/logger.py new file mode 100644 index 000000000..3d85b3a7f --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/logger.py @@ -0,0 +1,182 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import ( + c_void_p, + Structure, + c_char_p, + c_uint, + c_int, + cast, + CFUNCTYPE, + pointer, +) +from enum import IntEnum +import logging +from io import StringIO +import weakref + +from ..ocf import OcfLib + +logger = logging.getLogger("pyocf") +logger.setLevel(logging.DEBUG) + + +class LogLevel(IntEnum): + EMERG = 0 + ALERT = 1 + CRIT = 2 + ERR = 3 + WARN = 4 + NOTICE = 5 + INFO = 6 + DEBUG = 7 + + +LevelMapping = { + LogLevel.EMERG: logging.CRITICAL, + LogLevel.ALERT: logging.CRITICAL, + LogLevel.CRIT: logging.CRITICAL, + LogLevel.ERR: logging.ERROR, + LogLevel.WARN: logging.WARNING, + LogLevel.NOTICE: logging.INFO, + LogLevel.INFO: logging.INFO, + LogLevel.DEBUG: logging.DEBUG, +} + + +class LoggerOps(Structure): + OPEN = CFUNCTYPE(c_int, c_void_p) + CLOSE = CFUNCTYPE(None, c_void_p) + # PRINTF ommited - we cannot make variadic function call in ctypes + LOG = CFUNCTYPE(c_int, c_void_p, c_uint, c_char_p) + PRINT_RL = CFUNCTYPE(c_int, c_void_p, c_char_p) + DUMP_STACK = CFUNCTYPE(c_int, c_void_p) + + _fields_ = [ + ("_open", OPEN), + ("_close", CLOSE), + ("_print", c_void_p), + ("_print_rl", PRINT_RL), + ("_dump_stack", DUMP_STACK), + ] + + +class LoggerPriv(Structure): + _fields_ = [("_log", LoggerOps.LOG)] + + +class Logger(Structure): + _instances_ = {} + + _fields_ = [("logger", c_void_p)] + + def __init__(self): + self.ops = LoggerOps( + _open=self._open, + _print=cast(OcfLib.getInstance().pyocf_printf_helper, c_void_p), + _close=self._close, + ) + self.priv = LoggerPriv(_log=self._log) + self._as_parameter_ = cast(pointer(self.priv), c_void_p).value + self._instances_[self._as_parameter_] = weakref.ref(self) + + def get_ops(self): + return self.ops + + def get_priv(self): + return self.priv + + @classmethod + def get_instance(cls, ctx: int): + priv = OcfLib.getInstance().ocf_logger_get_priv(ctx) + return cls._instances_[priv]() + + @staticmethod + @LoggerOps.LOG + def _log(ref, lvl, msg): + Logger.get_instance(ref).log(lvl, str(msg, "ascii").strip()) + return 0 + + @staticmethod + @LoggerOps.OPEN + def _open(ref): + if hasattr(Logger.get_instance(ref), "open"): + return Logger.get_instance(ref).open() + else: + return 0 + + @staticmethod + @LoggerOps.CLOSE + def _close(ref): + if hasattr(Logger.get_instance(ref), "close"): + return Logger.get_instance(ref).close() + else: + return 0 + + +class DefaultLogger(Logger): + def __init__(self, level: LogLevel = LogLevel.WARN): + super().__init__() + self.level = level + + ch = logging.StreamHandler() + fmt = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + ch.setFormatter(fmt) + ch.setLevel(LevelMapping[level]) + logger.addHandler(ch) + + def log(self, lvl: int, msg: str): + logger.log(LevelMapping[lvl], msg) + + def close(self): + logger.handlers = [] + + +class FileLogger(Logger): + def __init__(self, f, console_level=None): + super().__init__() + fmt = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + + fh = logging.FileHandler(f) + fh.setLevel(logging.DEBUG) + fh.setFormatter(fmt) + + logger.addHandler(fh) + + if console_level: + sh = logging.StreamHandler() + sh.setLevel(LevelMapping[console_level]) + sh.setFormatter(fmt) + logger.addHandler(sh) + + def log(self, lvl, msg): + logger.log(LevelMapping[lvl], msg) + + def close(self): + logger.handlers = [] + + +class BufferLogger(Logger): + def __init__(self, level: LogLevel): + super().__init__() + self.level = level + self.buffer = StringIO() + + def log(self, lvl, msg): + if lvl < self.level: + self.buffer.write(msg + "\n") + + def get_lines(self): + return self.buffer.getvalue().split("\n") + + +lib = OcfLib.getInstance() +lib.ocf_logger_get_priv.restype = c_void_p +lib.ocf_logger_get_priv.argtypes = [c_void_p] diff --git a/src/spdk/ocf/tests/functional/pyocf/types/metadata_updater.py b/src/spdk/ocf/tests/functional/pyocf/types/metadata_updater.py new file mode 100644 index 000000000..592d2a14c --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/metadata_updater.py @@ -0,0 +1,102 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import c_void_p, c_int, c_uint32, Structure, CFUNCTYPE +from threading import Thread, Event + +from ..ocf import OcfLib + + +class MetadataUpdaterOps(Structure): + INIT = CFUNCTYPE(c_int, c_void_p) + KICK = CFUNCTYPE(None, c_void_p) + STOP = CFUNCTYPE(None, c_void_p) + + _fields_ = [("_init", INIT), ("_kick", KICK), ("_stop", STOP)] + + +class MetadataUpdater: + pass + + +def mu_run(*, mu: MetadataUpdater, kick: Event, stop: Event): + while True: + kick.clear() + + if OcfLib.getInstance().ocf_metadata_updater_run(mu): + continue + + kick.wait() + if stop.is_set(): + break + + +class MetadataUpdater: + _instances_ = {} + ops = None + + def __init__(self, ref): + self._as_parameter_ = ref + MetadataUpdater._instances_[ref] = self + self.kick_event = Event() + self.stop_event = Event() + + lib = OcfLib.getInstance() + self.thread = Thread( + group=None, + target=mu_run, + name="mu-{}".format( + lib.ocf_cache_get_name(lib.ocf_metadata_updater_get_cache(self)) + ), + kwargs={"mu": self, "kick": self.kick_event, "stop": self.stop_event}, + ) + self.thread.start() + + @classmethod + def get_ops(cls): + if not cls.ops: + cls.ops = MetadataUpdaterOps( + _init=cls._init, _kick=cls._kick, _stop=cls._stop + ) + return cls.ops + + @classmethod + def get_instance(cls, ref): + return cls._instances_[ref] + + @classmethod + def del_instance(cls, ref): + del cls._instances_[ref] + + @staticmethod + @MetadataUpdaterOps.INIT + def _init(ref): + m = MetadataUpdater(ref) + return 0 + + @staticmethod + @MetadataUpdaterOps.KICK + def _kick(ref): + MetadataUpdater.get_instance(ref).kick() + + @staticmethod + @MetadataUpdaterOps.STOP + def _stop(ref): + MetadataUpdater.get_instance(ref).stop() + del MetadataUpdater._instances_[ref] + + def kick(self): + self.kick_event.set() + + def stop(self): + self.stop_event.set() + self.kick_event.set() + + +lib = OcfLib.getInstance() +lib.ocf_metadata_updater_get_cache.argtypes = [c_void_p] +lib.ocf_metadata_updater_get_cache.restype = c_void_p +lib.ocf_metadata_updater_run.argtypes = [c_void_p] +lib.ocf_metadata_updater_run.restype = c_uint32 diff --git a/src/spdk/ocf/tests/functional/pyocf/types/queue.py b/src/spdk/ocf/tests/functional/pyocf/types/queue.py new file mode 100644 index 000000000..da0963907 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/queue.py @@ -0,0 +1,105 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import c_void_p, CFUNCTYPE, Structure, byref +from threading import Thread, Condition, Event +import weakref + +from ..ocf import OcfLib +from .shared import OcfError + + +class QueueOps(Structure): + KICK = CFUNCTYPE(None, c_void_p) + KICK_SYNC = CFUNCTYPE(None, c_void_p) + STOP = CFUNCTYPE(None, c_void_p) + + _fields_ = [("kick", KICK), ("kick_sync", KICK_SYNC), ("stop", STOP)] + + +class Queue: + pass + + +def io_queue_run(*, queue: Queue, kick: Condition, stop: Event): + def wait_predicate(): + return stop.is_set() or OcfLib.getInstance().ocf_queue_pending_io(queue) + + while True: + with kick: + kick.wait_for(wait_predicate) + + OcfLib.getInstance().ocf_queue_run(queue) + + if stop.is_set() and not OcfLib.getInstance().ocf_queue_pending_io(queue): + break + + +class Queue: + _instances_ = {} + + def __init__(self, cache, name): + + self.ops = QueueOps(kick=type(self)._kick, stop=type(self)._stop) + + self.handle = c_void_p() + status = OcfLib.getInstance().ocf_queue_create( + cache.cache_handle, byref(self.handle), byref(self.ops) + ) + if status: + raise OcfError("Couldn't create queue object", status) + + Queue._instances_[self.handle.value] = weakref.ref(self) + self._as_parameter_ = self.handle + + self.stop_event = Event() + self.kick_condition = Condition() + self.thread = Thread( + group=None, + target=io_queue_run, + name=name, + kwargs={ + "queue": self, + "kick": self.kick_condition, + "stop": self.stop_event, + }, + ) + self.thread.start() + + @classmethod + def get_instance(cls, ref): + return cls._instances_[ref]() + + @staticmethod + @QueueOps.KICK_SYNC + def _kick_sync(ref): + Queue.get_instance(ref).kick_sync() + + @staticmethod + @QueueOps.KICK + def _kick(ref): + Queue.get_instance(ref).kick() + + @staticmethod + @QueueOps.STOP + def _stop(ref): + Queue.get_instance(ref).stop() + + def kick_sync(self): + OcfLib.getInstance().ocf_queue_run(self.handle) + + def kick(self): + with self.kick_condition: + self.kick_condition.notify_all() + + def put(self): + OcfLib.getInstance().ocf_queue_put(self) + + def stop(self): + with self.kick_condition: + self.stop_event.set() + self.kick_condition.notify_all() + + self.thread.join() diff --git a/src/spdk/ocf/tests/functional/pyocf/types/shared.py b/src/spdk/ocf/tests/functional/pyocf/types/shared.py new file mode 100644 index 000000000..5244b4d36 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/shared.py @@ -0,0 +1,160 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import logging +from ctypes import CFUNCTYPE, c_size_t, c_char_p, Structure, c_void_p +from enum import IntEnum, auto +from threading import Event + +from ..utils import Size as S + + +class OcfErrorCode(IntEnum): + OCF_ERR_INVAL = 1000000 + OCF_ERR_AGAIN = auto() + OCF_ERR_INTR = auto() + OCF_ERR_NOT_SUPP = auto() + OCF_ERR_NO_MEM = auto() + OCF_ERR_NO_LOCK = auto() + OCF_ERR_METADATA_VER = auto() + OCF_ERR_NO_METADATA = auto() + OCF_ERR_METADATA_FOUND = auto() + OCF_ERR_INVAL_VOLUME_TYPE = auto() + OCF_ERR_UNKNOWN = auto() + OCF_ERR_TOO_MANY_CACHES = auto() + OCF_ERR_NO_FREE_RAM = auto() + OCF_ERR_START_CACHE_FAIL = auto() + OCF_ERR_CACHE_NOT_EXIST = auto() + OCF_ERR_CORE_NOT_EXIST = auto() + OCF_ERR_CACHE_EXIST = auto() + OCF_ERR_CORE_EXIST = auto() + OCF_ERR_TOO_MANY_CORES = auto() + OCF_ERR_CORE_NOT_AVAIL = auto() + OCF_ERR_NOT_OPEN_EXC = auto() + OCF_ERR_CACHE_NOT_AVAIL = auto() + OCF_ERR_IO_CLASS_NOT_EXIST = auto() + OCF_ERR_IO = auto() + OCF_ERR_WRITE_CACHE = auto() + OCF_ERR_WRITE_CORE = auto() + OCF_ERR_DIRTY_SHUTDOWN = auto() + OCF_ERR_DIRTY_EXISTS = auto() + OCF_ERR_FLUSHING_INTERRUPTED = auto() + OCF_ERR_FLUSH_IN_PROGRESS = auto() + OCF_ERR_CANNOT_ADD_CORE_TO_POOL = auto() + OCF_ERR_CACHE_IN_INCOMPLETE_STATE = auto() + OCF_ERR_CORE_IN_INACTIVE_STATE = auto() + OCF_ERR_INVALID_CACHE_MODE = auto() + OCF_ERR_INVALID_CACHE_LINE_SIZE = auto() + OCF_ERR_CACHE_NAME_MISMATCH = auto() + OCF_ERR_INVAL_CACHE_DEV = auto() + + +class OcfCompletion: + """ + This class provides Completion mechanism for interacting with OCF async + management API. + """ + + class CompletionResult: + def __init__(self, completion_args): + self.completion_args = { + x[0]: i for i, x in enumerate(completion_args) + } + self.results = None + self.arg_types = [x[1] for x in completion_args] + + def __getitem__(self, key): + try: + position = self.completion_args[key] + return self.results[position] + except KeyError: + raise KeyError(f"No completion argument {key} specified") + + def __init__(self, completion_args: list): + """ + Provide ctypes arg list, and optionally index of status argument in + completion function which will be extracted (default - last argument). + + :param completion_args: list of tuples (parameter name, parameter type) + for OCF completion function + """ + self.e = Event() + self.results = OcfCompletion.CompletionResult(completion_args) + self._as_parameter_ = self.callback + + @property + def callback(self): + @CFUNCTYPE(c_void_p, *self.results.arg_types) + def complete(*args): + self.results.results = args + self.e.set() + + return complete + + def wait(self): + self.e.wait() + + +class OcfError(BaseException): + def __init__(self, msg, error_code): + super().__init__(self, msg) + self.error_code = OcfErrorCode(abs(error_code)) + self.msg = msg + + def __str__(self): + return "{} ({})".format(self.msg, repr(self.error_code)) + + +class SharedOcfObject(Structure): + _instances_ = {} + + def __init__(self): + super().__init__() + type(self)._instances_[self._as_parameter_] = self + + @classmethod + def get_instance(cls, ref: int): + try: + return cls._instances_[ref] + except: # noqa E722 + logging.getLogger("pyocf").error( + "OcfSharedObject corruption. wanted: {} instances: {}".format( + ref, cls._instances_ + ) + ) + return None + + @classmethod + def del_object(cls, ref: int): + del cls._instances_[ref] + + +class Uuid(Structure): + _fields_ = [("_size", c_size_t), ("_data", c_char_p)] + + +class CacheLineSize(IntEnum): + LINE_4KiB = S.from_KiB(4) + LINE_8KiB = S.from_KiB(8) + LINE_16KiB = S.from_KiB(16) + LINE_32KiB = S.from_KiB(32) + LINE_64KiB = S.from_KiB(64) + DEFAULT = LINE_4KiB + + +class SeqCutOffPolicy(IntEnum): + ALWAYS = 0 + FULL = 1 + NEVER = 2 + DEFAULT = FULL + + +class CacheLines(S): + def __init__(self, count: int, line_size: CacheLineSize): + self.bytes = count * line_size + self.line_size = line_size + + def __int__(self): + return int(self.bytes / self.line_size) diff --git a/src/spdk/ocf/tests/functional/pyocf/types/stats/__init__.py b/src/spdk/ocf/tests/functional/pyocf/types/stats/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/stats/__init__.py diff --git a/src/spdk/ocf/tests/functional/pyocf/types/stats/cache.py b/src/spdk/ocf/tests/functional/pyocf/types/stats/cache.py new file mode 100644 index 000000000..59a4bdfa0 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/stats/cache.py @@ -0,0 +1,39 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import c_uint8, c_uint32, c_uint64, c_bool, c_int, Structure +from pyocf.types.stats.shared import _Stat + + +class _Inactive(Structure): + _fields_ = [("occupancy", _Stat), ("clean", _Stat), ("dirty", _Stat)] + + +class _FallbackPt(Structure): + _fields_ = [("error_counter", c_int), ("status", c_bool)] + + +class CacheInfo(Structure): + _fields_ = [ + ("attached", c_bool), + ("volume_type", c_uint8), + ("size", c_uint32), + ("inactive", _Inactive), + ("occupancy", c_uint32), + ("dirty", c_uint32), + ("dirty_initial", c_uint32), + ("dirty_for", c_uint32), + ("cache_mode", c_uint32), + ("fallback_pt", _FallbackPt), + ("state", c_uint8), + ("eviction_policy", c_uint32), + ("cleaning_policy", c_uint32), + ("promotion_policy", c_uint32), + ("cache_line_size", c_uint64), + ("flushed", c_uint32), + ("core_count", c_uint32), + ("metadata_footprint", c_uint64), + ("metadata_end_offset", c_uint32), + ] diff --git a/src/spdk/ocf/tests/functional/pyocf/types/stats/core.py b/src/spdk/ocf/tests/functional/pyocf/types/stats/core.py new file mode 100644 index 000000000..dd2d06689 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/stats/core.py @@ -0,0 +1,21 @@ + +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import c_uint32, c_uint64, Structure + +from .shared import OcfStatsReq, OcfStatsBlock, OcfStatsDebug, OcfStatsError + + +class CoreInfo(Structure): + _fields_ = [ + ("core_size", c_uint64), + ("core_size_bytes", c_uint64), + ("dirty", c_uint32), + ("flushed", c_uint32), + ("dirty_for", c_uint32), + ("seq_cutoff_threshold", c_uint32), + ("seq_cutoff_policy", c_uint32), + ] diff --git a/src/spdk/ocf/tests/functional/pyocf/types/stats/shared.py b/src/spdk/ocf/tests/functional/pyocf/types/stats/shared.py new file mode 100644 index 000000000..e6719d985 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/stats/shared.py @@ -0,0 +1,88 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import c_uint64, c_uint32, Structure + + +class _Stat(Structure): + _fields_ = [("value", c_uint64), ("fraction", c_uint64)] + + +class OcfStatsReq(Structure): + _fields_ = [ + ("partial_miss", c_uint64), + ("full_miss", c_uint64), + ("total", c_uint64), + ("pass_through", c_uint64), + ] + + +class OcfStatsBlock(Structure): + _fields_ = [("read", c_uint64), ("write", c_uint64)] + + +class OcfStatsError(Structure): + _fields_ = [("read", c_uint32), ("write", c_uint32)] + + +class OcfStatsDebug(Structure): + _fields_ = [ + ("read_size", c_uint64 * 12), + ("write_size", c_uint64 * 12), + ("read_align", c_uint64 * 4), + ("write_align", c_uint64 * 4), + ] + + +class UsageStats(Structure): + _fields_ = [ + ("occupancy", _Stat), + ("free", _Stat), + ("clean", _Stat), + ("dirty", _Stat), + ] + + +class RequestsStats(Structure): + _fields_ = [ + ("rd_hits", _Stat), + ("rd_partial_misses", _Stat), + ("rd_full_misses", _Stat), + ("rd_total", _Stat), + ("wr_hits", _Stat), + ("wr_partial_misses", _Stat), + ("wr_full_misses", _Stat), + ("wr_total", _Stat), + ("rd_pt", _Stat), + ("wr_pt", _Stat), + ("serviced", _Stat), + ("total", _Stat), + ] + + +class BlocksStats(Structure): + _fields_ = [ + ("core_volume_rd", _Stat), + ("core_volume_wr", _Stat), + ("core_volume_total", _Stat), + ("cache_volume_rd", _Stat), + ("cache_volume_wr", _Stat), + ("cache_volume_total", _Stat), + ("volume_rd", _Stat), + ("volume_wr", _Stat), + ("volume_total", _Stat), + ] + + +class ErrorsStats(Structure): + _fields_ = [ + ("core_volume_rd", _Stat), + ("core_volume_wr", _Stat), + ("core_volume_total", _Stat), + ("cache_volume_rd", _Stat), + ("cache_volume_wr", _Stat), + ("cache_volume_total", _Stat), + ("total", _Stat), + ] diff --git a/src/spdk/ocf/tests/functional/pyocf/types/volume.py b/src/spdk/ocf/tests/functional/pyocf/types/volume.py new file mode 100644 index 000000000..4bca10bd1 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/types/volume.py @@ -0,0 +1,361 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import ( + POINTER, + c_void_p, + c_uint32, + c_char_p, + create_string_buffer, + memmove, + memset, + Structure, + CFUNCTYPE, + c_int, + c_uint, + c_uint64, + sizeof, + cast, + string_at, +) +from hashlib import md5 +import weakref + +from .io import Io, IoOps, IoDir +from .shared import OcfErrorCode, Uuid +from ..ocf import OcfLib +from ..utils import print_buffer, Size as S +from .data import Data + + +class VolumeCaps(Structure): + _fields_ = [("_atomic_writes", c_uint32, 1)] + + +class VolumeOps(Structure): + SUBMIT_IO = CFUNCTYPE(None, POINTER(Io)) + SUBMIT_FLUSH = CFUNCTYPE(None, c_void_p) + SUBMIT_METADATA = CFUNCTYPE(None, c_void_p) + SUBMIT_DISCARD = CFUNCTYPE(None, c_void_p) + SUBMIT_WRITE_ZEROES = CFUNCTYPE(None, c_void_p) + OPEN = CFUNCTYPE(c_int, c_void_p) + CLOSE = CFUNCTYPE(None, c_void_p) + GET_MAX_IO_SIZE = CFUNCTYPE(c_uint, c_void_p) + GET_LENGTH = CFUNCTYPE(c_uint64, c_void_p) + + _fields_ = [ + ("_submit_io", SUBMIT_IO), + ("_submit_flush", SUBMIT_FLUSH), + ("_submit_metadata", SUBMIT_METADATA), + ("_submit_discard", SUBMIT_DISCARD), + ("_submit_write_zeroes", SUBMIT_WRITE_ZEROES), + ("_open", OPEN), + ("_close", CLOSE), + ("_get_max_io_size", GET_MAX_IO_SIZE), + ("_get_length", GET_LENGTH), + ] + + +class VolumeProperties(Structure): + _fields_ = [ + ("_name", c_char_p), + ("_io_priv_size", c_uint32), + ("_volume_priv_size", c_uint32), + ("_caps", VolumeCaps), + ("_ops", VolumeOps), + ("_io_ops", IoOps), + ("_deinit", c_char_p), + ] + + +class VolumeIoPriv(Structure): + _fields_ = [("_data", c_void_p), ("_offset", c_uint64)] + + +class Volume(Structure): + VOLUME_POISON = 0x13 + + _fields_ = [("_storage", c_void_p)] + _instances_ = {} + _uuid_ = {} + + props = None + + def __init__(self, size: S, uuid=None): + super().__init__() + self.size = size + if uuid: + if uuid in type(self)._uuid_: + raise Exception( + "Volume with uuid {} already created".format(uuid) + ) + self.uuid = uuid + else: + self.uuid = str(id(self)) + + type(self)._uuid_[self.uuid] = weakref.ref(self) + + self.data = create_string_buffer(int(self.size)) + memset(self.data, self.VOLUME_POISON, self.size) + self._storage = cast(self.data, c_void_p) + + self.reset_stats() + self.opened = False + + def get_copy(self): + new_volume = Volume(self.size) + memmove(new_volume.data, self.data, self.size) + return new_volume + + @classmethod + def get_props(cls): + if not cls.props: + cls.props = VolumeProperties( + _name=str(cls.__name__).encode("ascii"), + _io_priv_size=sizeof(VolumeIoPriv), + _volume_priv_size=0, + _caps=VolumeCaps(_atomic_writes=0), + _ops=VolumeOps( + _submit_io=cls._submit_io, + _submit_flush=cls._submit_flush, + _submit_metadata=cls._submit_metadata, + _submit_discard=cls._submit_discard, + _submit_write_zeroes=cls._submit_write_zeroes, + _open=cls._open, + _close=cls._close, + _get_max_io_size=cls._get_max_io_size, + _get_length=cls._get_length, + ), + _io_ops=IoOps( + _set_data=cls._io_set_data, _get_data=cls._io_get_data + ), + _deinit=0, + ) + + return cls.props + + @classmethod + def get_instance(cls, ref): + instance = cls._instances_[ref]() + if instance is None: + print("tried to access {} but it's gone".format(ref)) + + return instance + + @classmethod + def get_by_uuid(cls, uuid): + return cls._uuid_[uuid]() + + @staticmethod + @VolumeOps.SUBMIT_IO + def _submit_io(io): + io_structure = cast(io, POINTER(Io)) + volume = Volume.get_instance( + OcfLib.getInstance().ocf_io_get_volume(io_structure) + ) + + volume.submit_io(io_structure) + + @staticmethod + @VolumeOps.SUBMIT_FLUSH + def _submit_flush(flush): + io_structure = cast(flush, POINTER(Io)) + volume = Volume.get_instance( + OcfLib.getInstance().ocf_io_get_volume(io_structure) + ) + + volume.submit_flush(io_structure) + + @staticmethod + @VolumeOps.SUBMIT_METADATA + def _submit_metadata(meta): + pass + + @staticmethod + @VolumeOps.SUBMIT_DISCARD + def _submit_discard(discard): + io_structure = cast(discard, POINTER(Io)) + volume = Volume.get_instance( + OcfLib.getInstance().ocf_io_get_volume(io_structure) + ) + + volume.submit_discard(io_structure) + + @staticmethod + @VolumeOps.SUBMIT_WRITE_ZEROES + def _submit_write_zeroes(write_zeroes): + pass + + @staticmethod + @CFUNCTYPE(c_int, c_void_p) + def _open(ref): + uuid_ptr = cast( + OcfLib.getInstance().ocf_volume_get_uuid(ref), POINTER(Uuid) + ) + uuid = str(uuid_ptr.contents._data, encoding="ascii") + try: + volume = Volume.get_by_uuid(uuid) + except: # noqa E722 TODO:Investigate whether this really should be so broad + print("Tried to access unallocated volume {}".format(uuid)) + print("{}".format(Volume._uuid_)) + return -1 + + if volume.opened: + return OcfErrorCode.OCF_ERR_NOT_OPEN_EXC + + Volume._instances_[ref] = weakref.ref(volume) + + return volume.open() + + @staticmethod + @VolumeOps.CLOSE + def _close(ref): + volume = Volume.get_instance(ref) + volume.close() + volume.opened = False + + @staticmethod + @VolumeOps.GET_MAX_IO_SIZE + def _get_max_io_size(ref): + return Volume.get_instance(ref).get_max_io_size() + + @staticmethod + @VolumeOps.GET_LENGTH + def _get_length(ref): + return Volume.get_instance(ref).get_length() + + @staticmethod + @IoOps.SET_DATA + def _io_set_data(io, data, offset): + io_priv = cast( + OcfLib.getInstance().ocf_io_get_priv(io), POINTER(VolumeIoPriv) + ) + data = Data.get_instance(data) + io_priv.contents._offset = offset + io_priv.contents._data = data.handle + + return 0 + + @staticmethod + @IoOps.GET_DATA + def _io_get_data(io): + io_priv = cast( + OcfLib.getInstance().ocf_io_get_priv(io), POINTER(VolumeIoPriv) + ) + return io_priv.contents._data + + def open(self): + self.opened = True + return 0 + + def close(self): + pass + + def get_length(self): + return self.size + + def get_max_io_size(self): + return S.from_KiB(128) + + def submit_flush(self, flush): + flush.contents._end(flush, 0) + + def submit_discard(self, discard): + try: + dst = self._storage + discard.contents._addr + memset(dst, 0, discard.contents._bytes) + + discard.contents._end(discard, 0) + except: # noqa E722 + discard.contents._end(discard, -5) + + def get_stats(self): + return self.stats + + def reset_stats(self): + self.stats = {IoDir.WRITE: 0, IoDir.READ: 0} + + def submit_io(self, io): + try: + self.stats[IoDir(io.contents._dir)] += 1 + + io_priv = cast( + OcfLib.getInstance().ocf_io_get_priv(io), POINTER(VolumeIoPriv)) + offset = io_priv.contents._offset + + if io.contents._dir == IoDir.WRITE: + src_ptr = cast(OcfLib.getInstance().ocf_io_get_data(io), c_void_p) + src = Data.get_instance(src_ptr.value).handle.value + offset + dst = self._storage + io.contents._addr + elif io.contents._dir == IoDir.READ: + dst_ptr = cast(OcfLib.getInstance().ocf_io_get_data(io), c_void_p) + dst = Data.get_instance(dst_ptr.value).handle.value + offset + src = self._storage + io.contents._addr + + memmove(dst, src, io.contents._bytes) + io_priv.contents._offset += io.contents._bytes + + io.contents._end(io, 0) + except: # noqa E722 + io.contents._end(io, -5) + + def dump(self, offset=0, size=0, ignore=VOLUME_POISON, **kwargs): + if size == 0: + size = int(self.size) - int(offset) + + print_buffer( + self._storage, + size, + ignore=ignore, + **kwargs + ) + + def md5(self): + m = md5() + m.update(string_at(self._storage, self.size)) + return m.hexdigest() + + +class ErrorDevice(Volume): + def __init__(self, size, error_sectors: set = None, uuid=None): + super().__init__(size, uuid) + self.error_sectors = error_sectors or set() + + def set_mapping(self, error_sectors: set): + self.error_sectors = error_sectors + + def submit_io(self, io): + if io.contents._addr in self.error_sectors: + io.contents._end(io, -5) + self.stats["errors"][io.contents._dir] += 1 + else: + super().submit_io(io) + + def reset_stats(self): + super().reset_stats() + self.stats["errors"] = {IoDir.WRITE: 0, IoDir.READ: 0} + + +class TraceDevice(Volume): + def __init__(self, size, trace_fcn=None, uuid=None): + super().__init__(size, uuid) + self.trace_fcn = trace_fcn + + def submit_io(self, io): + submit = True + + if self.trace_fcn: + submit = self.trace_fcn(self, io) + + if submit: + super().submit_io(io) + + +lib = OcfLib.getInstance() +lib.ocf_io_get_priv.restype = POINTER(VolumeIoPriv) +lib.ocf_io_get_volume.argtypes = [c_void_p] +lib.ocf_io_get_volume.restype = c_void_p +lib.ocf_io_get_data.argtypes = [c_void_p] +lib.ocf_io_get_data.restype = c_void_p diff --git a/src/spdk/ocf/tests/functional/pyocf/utils.py b/src/spdk/ocf/tests/functional/pyocf/utils.py new file mode 100644 index 000000000..d4ef42300 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/utils.py @@ -0,0 +1,173 @@ +# +# Copyright(c) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from ctypes import string_at + + +def print_buffer( + buf, + length, + offset=0, + width=16, + ignore=0, + stop_after_count_ignored=0, + print_fcn=print, +): + end = int(offset) + int(length) + offset = int(offset) + ignored_lines = 0 + buf = string_at(buf, length) + whole_buffer_ignored = True + stop_after_count_ignored = int(stop_after_count_ignored / width) + + for addr in range(offset, end, width): + cur_line = buf[addr : min(end, addr + width)] + byteline = "" + asciiline = "" + if not any(x != ignore for x in cur_line): + if ( + stop_after_count_ignored + and ignored_lines > stop_after_count_ignored + ): + print_fcn( + "<{} bytes of '0x{:02X}' encountered, stopping>".format( + stop_after_count_ignored * width, ignore + ) + ) + return + ignored_lines += 1 + continue + + if ignored_lines: + print_fcn( + "<{} of '0x{:02X}' bytes omitted>".format( + ignored_lines * width, ignore + ) + ) + ignored_lines = 0 + + for byte in cur_line: + byte = int(byte) + byteline += "{:02X} ".format(byte) + if 31 < byte < 126: + char = chr(byte) + else: + char = "." + asciiline += char + + print_fcn("0x{:08X}\t{}\t{}".format(addr, byteline, asciiline)) + whole_buffer_ignored = False + + if whole_buffer_ignored: + print_fcn("<whole buffer ignored>") + elif ignored_lines: + print_fcn("<'0x{:02X}' until end>".format(ignore)) + + +class Size: + _KiB = 1024 + _MiB = _KiB * 1024 + _GiB = _MiB * 1024 + _TiB = _GiB * 1024 + _SECTOR_SIZE = 512 + + def __init__(self, b: int, sector_aligned: bool = False): + if sector_aligned: + self.bytes = int( + ((b + self._SECTOR_SIZE - 1) // self._SECTOR_SIZE) + * self._SECTOR_SIZE + ) + else: + self.bytes = int(b) + + def __int__(self): + return self.bytes + + def __index__(self): + return self.bytes + + @classmethod + def from_B(cls, value, sector_aligned=False): + return cls(value, sector_aligned) + + @classmethod + def from_KiB(cls, value, sector_aligned=False): + return cls(value * cls._KiB, sector_aligned) + + @classmethod + def from_MiB(cls, value, sector_aligned=False): + return cls(value * cls._MiB, sector_aligned) + + @classmethod + def from_GiB(cls, value, sector_aligned=False): + return cls(value * cls._GiB, sector_aligned) + + @classmethod + def from_TiB(cls, value, sector_aligned=False): + return cls(value * cls._TiB, sector_aligned) + + @classmethod + def from_sector(cls, value): + return cls(value * cls._SECTOR_SIZE) + + @property + def B(self): + return self.bytes + + @property + def KiB(self): + return self.bytes / self._KiB + + @property + def MiB(self): + return self.bytes / self._MiB + + @property + def GiB(self): + return self.bytes / self._GiB + + @property + def TiB(self): + return self.bytes / self._TiB + + @property + def sectors(self): + return self.bytes // self._SECTOR_SIZE + + def __str__(self): + if self.bytes < self._KiB: + return "{} B".format(self.B) + elif self.bytes < self._MiB: + return "{} KiB".format(self.KiB) + elif self.bytes < self._GiB: + return "{} MiB".format(self.MiB) + elif self.bytes < self._TiB: + return "{} GiB".format(self.GiB) + else: + return "{} TiB".format(self.TiB) + + +def print_structure(struct, indent=0): + print(struct) + for field, field_type in struct._fields_: + value = getattr(struct, field) + if hasattr(value, "_fields_"): + print("{}{: <20} :".format(" " * indent, field)) + print_structure(value, indent=indent + 1) + continue + + print("{}{: <20} : {}".format(" " * indent, field, value)) + + +def struct_to_dict(struct): + d = {} + for field, field_type in struct._fields_: + value = getattr(struct, field) + if hasattr(value, "_fields_"): + d[field] = struct_to_dict(value) + continue + d[field] = value + + return d diff --git a/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_io_wrappers.c b/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_io_wrappers.c new file mode 100644 index 000000000..79b9331e0 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_io_wrappers.c @@ -0,0 +1,36 @@ +/* + * Copyright(c) 2012-2018 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#include "ocf/ocf_io.h" +#include "ocf/ocf_core.h" + +struct ocf_io *ocf_core_new_io_wrapper(ocf_core_t core, ocf_queue_t queue, + uint64_t addr, uint32_t bytes, uint32_t dir, + uint32_t io_class, uint64_t flags) +{ + return ocf_core_new_io(core, queue, addr, bytes, dir, io_class, flags); +} + +void ocf_io_set_cmpl_wrapper(struct ocf_io *io, void *context, + void *context2, ocf_end_io_t fn) +{ + ocf_io_set_cmpl(io, context, context2, fn); +} + +void ocf_io_set_start_wrapper(struct ocf_io *io, ocf_start_io_t fn) +{ + ocf_io_set_start(io, fn); +} + +void ocf_io_set_handle_wrapper(struct ocf_io *io, ocf_handle_io_t fn) +{ + ocf_io_set_handle(io, fn); +} + +void ocf_core_submit_io_wrapper(struct ocf_io *io) +{ + ocf_core_submit_io(io); +} + diff --git a/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_logger_wrappers.c b/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_logger_wrappers.c new file mode 100644 index 000000000..60ded8dad --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_logger_wrappers.c @@ -0,0 +1,42 @@ + +/* + * Copyright(c) 2012-2018 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#include <ocf/ocf_types.h> +#include <ocf/ocf_logger.h> +#include <stdarg.h> +#include "ocf_env.h" + +#define LOG_BUFFER_SIZE 4096 + +struct pyocf_logger_priv { + int (*pyocf_log)(void *pyocf_logger, ocf_logger_lvl_t lvl, char *msg); +}; + +int pyocf_printf_helper(ocf_logger_t logger, ocf_logger_lvl_t lvl, + const char *fmt, va_list args) +{ + char *buffer = env_zalloc(LOG_BUFFER_SIZE, ENV_MEM_NORMAL); + struct pyocf_logger_priv *priv = ocf_logger_get_priv(logger); + int ret; + + if (!buffer) { + ret = -ENOMEM; + goto out; + } + + ret = vsnprintf(buffer, LOG_BUFFER_SIZE, fmt, args); + if (ret < 0) { + env_free(buffer); + goto out; + } + + ret = priv->pyocf_log(logger, lvl, buffer); + + env_free(buffer); + +out: + return ret; +} diff --git a/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_volume_wrappers.c b/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_volume_wrappers.c new file mode 100644 index 000000000..cb3787761 --- /dev/null +++ b/src/spdk/ocf/tests/functional/pyocf/wrappers/ocf_volume_wrappers.c @@ -0,0 +1,12 @@ + +/* + * Copyright(c) 2012-2018 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#include "ocf/ocf_io.h" +#include "ocf/ocf_volume.h" + +const char *ocf_uuid_to_str_wrapper(const struct ocf_volume_uuid *uuid) { + return ocf_uuid_to_str(uuid); +} |