diff options
Diffstat (limited to 'src/pybind/rados')
| -rw-r--r-- | src/pybind/rados/CMakeLists.txt | 5 | ||||
| -rw-r--r-- | src/pybind/rados/MANIFEST.in | 2 | ||||
| -rw-r--r-- | src/pybind/rados/c_rados.pxd | 306 | ||||
| -rw-r--r-- | src/pybind/rados/ctime.pxd | 11 | ||||
| -rw-r--r-- | src/pybind/rados/mock_rados.pxi | 464 | ||||
| -rw-r--r-- | src/pybind/rados/rados.pxd | 45 | ||||
| -rw-r--r-- | src/pybind/rados/rados.pyx | 4338 | ||||
| -rwxr-xr-x | src/pybind/rados/setup.py | 207 |
8 files changed, 5378 insertions, 0 deletions
diff --git a/src/pybind/rados/CMakeLists.txt b/src/pybind/rados/CMakeLists.txt new file mode 100644 index 000000000..0cd19d031 --- /dev/null +++ b/src/pybind/rados/CMakeLists.txt @@ -0,0 +1,5 @@ +distutils_add_cython_module(cython_rados + rados + ${CMAKE_CURRENT_SOURCE_DIR}/rados.pyx) +add_dependencies(cython_rados rados) +distutils_install_cython_module(cython_rados) diff --git a/src/pybind/rados/MANIFEST.in b/src/pybind/rados/MANIFEST.in new file mode 100644 index 000000000..79f1d9ce8 --- /dev/null +++ b/src/pybind/rados/MANIFEST.in @@ -0,0 +1,2 @@ +include rados.pyx +include rados.pxd diff --git a/src/pybind/rados/c_rados.pxd b/src/pybind/rados/c_rados.pxd new file mode 100644 index 000000000..51564c03c --- /dev/null +++ b/src/pybind/rados/c_rados.pxd @@ -0,0 +1,306 @@ +# cython: embedsignature=True + +from libc.stdint cimport * +from ctime cimport time_t, timeval +from rados cimport rados_t, rados_config_t, rados_ioctx_t + +cdef extern from "err.h" nogil: + cdef int _MAX_ERRNO "MAX_ERRNO" + + +cdef extern from "rados/rados_types.h" nogil: + cdef char* _LIBRADOS_ALL_NSPACES "LIBRADOS_ALL_NSPACES" + cdef struct notify_ack_t: + unsigned long notifier_id + unsigned long cookie + char *payload + unsigned long payload_len + + cdef struct notify_timeout_t: + unsigned long notifier_id + unsigned long cookie + +cdef extern from "rados/librados.h" nogil: + enum: + _LIBRADOS_OP_FLAG_EXCL "LIBRADOS_OP_FLAG_EXCL" + _LIBRADOS_OP_FLAG_FAILOK "LIBRADOS_OP_FLAG_FAILOK" + _LIBRADOS_OP_FLAG_FADVISE_RANDOM "LIBRADOS_OP_FLAG_FADVISE_RANDOM" + _LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL "LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL" + _LIBRADOS_OP_FLAG_FADVISE_WILLNEED "LIBRADOS_OP_FLAG_FADVISE_WILLNEED" + _LIBRADOS_OP_FLAG_FADVISE_DONTNEED "LIBRADOS_OP_FLAG_FADVISE_DONTNEED" + _LIBRADOS_OP_FLAG_FADVISE_NOCACHE "LIBRADOS_OP_FLAG_FADVISE_NOCACHE" + + + enum: + _LIBRADOS_CMPXATTR_OP_EQ "LIBRADOS_CMPXATTR_OP_EQ" + _LIBRADOS_CMPXATTR_OP_NE "LIBRADOS_CMPXATTR_OP_NE" + _LIBRADOS_CMPXATTR_OP_GT "LIBRADOS_CMPXATTR_OP_GT" + _LIBRADOS_CMPXATTR_OP_GTE "LIBRADOS_CMPXATTR_OP_GTE" + _LIBRADOS_CMPXATTR_OP_LT "LIBRADOS_CMPXATTR_OP_LT" + _LIBRADOS_CMPXATTR_OP_LTE "LIBRADOS_CMPXATTR_OP_LTE" + + enum: + _LIBRADOS_OPERATION_NOFLAG "LIBRADOS_OPERATION_NOFLAG" + _LIBRADOS_OPERATION_BALANCE_READS "LIBRADOS_OPERATION_BALANCE_READS" + _LIBRADOS_OPERATION_LOCALIZE_READS "LIBRADOS_OPERATION_LOCALIZE_READS" + _LIBRADOS_OPERATION_ORDER_READS_WRITES "LIBRADOS_OPERATION_ORDER_READS_WRITES" + _LIBRADOS_OPERATION_IGNORE_CACHE "LIBRADOS_OPERATION_IGNORE_CACHE" + _LIBRADOS_OPERATION_SKIPRWLOCKS "LIBRADOS_OPERATION_SKIPRWLOCKS" + _LIBRADOS_OPERATION_IGNORE_OVERLAY "LIBRADOS_OPERATION_IGNORE_OVERLAY" + _LIBRADOS_CREATE_EXCLUSIVE "LIBRADOS_CREATE_EXCLUSIVE" + _LIBRADOS_CREATE_IDEMPOTENT "LIBRADOS_CREATE_IDEMPOTENT" + + cdef uint64_t _LIBRADOS_SNAP_HEAD "LIBRADOS_SNAP_HEAD" + + ctypedef void* rados_xattrs_iter_t + ctypedef void* rados_omap_iter_t + ctypedef void* rados_list_ctx_t + ctypedef uint64_t rados_snap_t + ctypedef void *rados_write_op_t + ctypedef void *rados_read_op_t + ctypedef void *rados_completion_t + ctypedef void (*rados_callback_t)(rados_completion_t cb, void *arg) + ctypedef void (*rados_log_callback_t)(void *arg, const char *line, const char *who, + uint64_t sec, uint64_t nsec, uint64_t seq, const char *level, const char *msg) + ctypedef void (*rados_log_callback2_t)(void *arg, const char *line, const char *channel, const char *who, const char *name, + uint64_t sec, uint64_t nsec, uint64_t seq, const char *level, const char *msg) + ctypedef void (*rados_watchcb2_t)(void *arg, int64_t notify_id, + uint64_t handle, uint64_t notifier_id, + void *data, size_t data_len) + ctypedef void (*rados_watcherrcb_t)(void *pre, uint64_t cookie, int err) + + + cdef struct rados_cluster_stat_t: + uint64_t kb + uint64_t kb_used + uint64_t kb_avail + uint64_t num_objects + + cdef struct rados_pool_stat_t: + uint64_t num_bytes + uint64_t num_kb + uint64_t num_objects + uint64_t num_object_clones + uint64_t num_object_copies + uint64_t num_objects_missing_on_primary + uint64_t num_objects_unfound + uint64_t num_objects_degraded + uint64_t num_rd + uint64_t num_rd_kb + uint64_t num_wr + uint64_t num_wr_kb + + void rados_buffer_free(char *buf) + + void rados_version(int *major, int *minor, int *extra) + int rados_create2(rados_t *pcluster, const char *const clustername, + const char * const name, uint64_t flags) + int rados_create_with_context(rados_t *cluster, rados_config_t cct) + int rados_connect(rados_t cluster) + void rados_shutdown(rados_t cluster) + uint64_t rados_get_instance_id(rados_t cluster) + int rados_conf_read_file(rados_t cluster, const char *path) + int rados_conf_parse_argv_remainder(rados_t cluster, int argc, const char **argv, const char **remargv) + int rados_conf_parse_env(rados_t cluster, const char *var) + int rados_conf_set(rados_t cluster, char *option, const char *value) + int rados_conf_get(rados_t cluster, char *option, char *buf, size_t len) + + rados_t rados_ioctx_get_cluster(rados_ioctx_t io) + int rados_ioctx_pool_stat(rados_ioctx_t io, rados_pool_stat_t *stats) + int64_t rados_pool_lookup(rados_t cluster, const char *pool_name) + int rados_pool_reverse_lookup(rados_t cluster, int64_t id, char *buf, size_t maxlen) + int rados_pool_create(rados_t cluster, const char *pool_name) + int rados_pool_create_with_crush_rule(rados_t cluster, const char *pool_name, uint8_t crush_rule_num) + int rados_pool_create_with_auid(rados_t cluster, const char *pool_name, uint64_t auid) + int rados_pool_create_with_all(rados_t cluster, const char *pool_name, uint64_t auid, uint8_t crush_rule_num) + int rados_pool_get_base_tier(rados_t cluster, int64_t pool, int64_t *base_tier) + int rados_pool_list(rados_t cluster, char *buf, size_t len) + int rados_pool_delete(rados_t cluster, const char *pool_name) + int rados_inconsistent_pg_list(rados_t cluster, int64_t pool, char *buf, size_t len) + + int rados_cluster_stat(rados_t cluster, rados_cluster_stat_t *result) + int rados_cluster_fsid(rados_t cluster, char *buf, size_t len) + int rados_blocklist_add(rados_t cluster, char *client_address, uint32_t expire_seconds) + int rados_getaddrs(rados_t cluster, char** addrs) + int rados_application_enable(rados_ioctx_t io, const char *app_name, + int force) + void rados_set_pool_full_try(rados_ioctx_t io) + void rados_unset_pool_full_try(rados_ioctx_t io) + int rados_application_list(rados_ioctx_t io, char *values, + size_t *values_len) + int rados_application_metadata_get(rados_ioctx_t io, const char *app_name, + const char *key, char *value, + size_t *value_len) + int rados_application_metadata_set(rados_ioctx_t io, const char *app_name, + const char *key, const char *value) + int rados_application_metadata_remove(rados_ioctx_t io, + const char *app_name, const char *key) + int rados_application_metadata_list(rados_ioctx_t io, + const char *app_name, char *keys, + size_t *key_len, char *values, + size_t *value_len) + int rados_ping_monitor(rados_t cluster, const char *mon_id, char **outstr, size_t *outstrlen) + int rados_mon_command(rados_t cluster, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen) + int rados_mgr_command(rados_t cluster, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen) + int rados_mgr_command_target(rados_t cluster, + const char *name, + const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen) + int rados_mon_command_target(rados_t cluster, const char *name, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen) + int rados_osd_command(rados_t cluster, int osdid, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen) + int rados_pg_command(rados_t cluster, const char *pgstr, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen) + int rados_monitor_log(rados_t cluster, const char *level, rados_log_callback_t cb, void *arg) + int rados_monitor_log2(rados_t cluster, const char *level, rados_log_callback2_t cb, void *arg) + + int rados_wait_for_latest_osdmap(rados_t cluster) + + int rados_service_register(rados_t cluster, const char *service, const char *daemon, const char *metadata_dict) + int rados_service_update_status(rados_t cluster, const char *status_dict) + + int rados_ioctx_create(rados_t cluster, const char *pool_name, rados_ioctx_t *ioctx) + int rados_ioctx_create2(rados_t cluster, int64_t pool_id, rados_ioctx_t *ioctx) + void rados_ioctx_destroy(rados_ioctx_t io) + void rados_ioctx_locator_set_key(rados_ioctx_t io, const char *key) + void rados_ioctx_set_namespace(rados_ioctx_t io, const char * nspace) + + uint64_t rados_get_last_version(rados_ioctx_t io) + int rados_stat(rados_ioctx_t io, const char *o, uint64_t *psize, time_t *pmtime) + int rados_write(rados_ioctx_t io, const char *oid, const char *buf, size_t len, uint64_t off) + int rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len) + int rados_writesame(rados_ioctx_t io, const char *oid, const char *buf, size_t data_len, size_t write_len, uint64_t off) + int rados_append(rados_ioctx_t io, const char *oid, const char *buf, size_t len) + int rados_read(rados_ioctx_t io, const char *oid, char *buf, size_t len, uint64_t off) + int rados_remove(rados_ioctx_t io, const char *oid) + int rados_trunc(rados_ioctx_t io, const char *oid, uint64_t size) + int rados_cmpext(rados_ioctx_t io, const char *o, const char *cmp_buf, size_t cmp_len, uint64_t off) + int rados_getxattr(rados_ioctx_t io, const char *o, const char *name, char *buf, size_t len) + int rados_setxattr(rados_ioctx_t io, const char *o, const char *name, const char *buf, size_t len) + int rados_rmxattr(rados_ioctx_t io, const char *o, const char *name) + int rados_getxattrs(rados_ioctx_t io, const char *oid, rados_xattrs_iter_t *iter) + int rados_getxattrs_next(rados_xattrs_iter_t iter, const char **name, const char **val, size_t *len) + void rados_getxattrs_end(rados_xattrs_iter_t iter) + + int rados_nobjects_list_open(rados_ioctx_t io, rados_list_ctx_t *ctx) + int rados_nobjects_list_next(rados_list_ctx_t ctx, const char **entry, const char **key, const char **nspace) + void rados_nobjects_list_close(rados_list_ctx_t ctx) + + int rados_ioctx_pool_requires_alignment2(rados_ioctx_t io, int * requires) + int rados_ioctx_pool_required_alignment2(rados_ioctx_t io, uint64_t * alignment) + + int rados_ioctx_snap_rollback(rados_ioctx_t io, const char * oid, const char * snapname) + int rados_ioctx_snap_create(rados_ioctx_t io, const char * snapname) + int rados_ioctx_snap_remove(rados_ioctx_t io, const char * snapname) + int rados_ioctx_snap_lookup(rados_ioctx_t io, const char * name, rados_snap_t * id) + int rados_ioctx_snap_get_name(rados_ioctx_t io, rados_snap_t id, char * name, int maxlen) + void rados_ioctx_snap_set_read(rados_ioctx_t io, rados_snap_t snap) + int rados_ioctx_snap_list(rados_ioctx_t io, rados_snap_t * snaps, int maxlen) + int rados_ioctx_snap_get_stamp(rados_ioctx_t io, rados_snap_t id, time_t * t) + int64_t rados_ioctx_get_id(rados_ioctx_t io) + int rados_ioctx_get_pool_name(rados_ioctx_t io, char *buf, unsigned maxlen) + + int rados_ioctx_selfmanaged_snap_create(rados_ioctx_t io, + rados_snap_t *snapid) + int rados_ioctx_selfmanaged_snap_remove(rados_ioctx_t io, + rados_snap_t snapid) + int rados_ioctx_selfmanaged_snap_set_write_ctx(rados_ioctx_t io, + rados_snap_t snap_seq, + rados_snap_t *snap, + int num_snaps) + int rados_ioctx_selfmanaged_snap_rollback(rados_ioctx_t io, const char *oid, + rados_snap_t snapid) + + int rados_lock_exclusive(rados_ioctx_t io, const char * oid, const char * name, + const char * cookie, const char * desc, + timeval * duration, uint8_t flags) + int rados_lock_shared(rados_ioctx_t io, const char * o, const char * name, + const char * cookie, const char * tag, const char * desc, + timeval * duration, uint8_t flags) + int rados_unlock(rados_ioctx_t io, const char * o, const char * name, const char * cookie) + + rados_write_op_t rados_create_write_op() + void rados_release_write_op(rados_write_op_t write_op) + + rados_read_op_t rados_create_read_op() + void rados_release_read_op(rados_read_op_t read_op) + + int rados_aio_create_completion2(void * cb_arg, rados_callback_t cb_complete, rados_completion_t * pc) + void rados_aio_release(rados_completion_t c) + int rados_aio_stat(rados_ioctx_t io, const char *oid, rados_completion_t completion, uint64_t *psize, time_t *pmtime) + int rados_aio_write(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len, uint64_t off) + int rados_aio_append(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len) + int rados_aio_write_full(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len) + int rados_aio_writesame(rados_ioctx_t io, const char *oid, rados_completion_t completion, const char *buf, size_t data_len, size_t write_len, uint64_t off) + int rados_aio_remove(rados_ioctx_t io, const char * oid, rados_completion_t completion) + int rados_aio_read(rados_ioctx_t io, const char * oid, rados_completion_t completion, char * buf, size_t len, uint64_t off) + int rados_aio_flush(rados_ioctx_t io) + int rados_aio_cmpext(rados_ioctx_t io, const char *o, rados_completion_t completion, const char *cmp_buf, size_t cmp_len, uint64_t off) + int rados_aio_rmxattr(rados_ioctx_t io, const char *o, rados_completion_t completion, const char *name) + + int rados_aio_get_return_value(rados_completion_t c) + int rados_aio_wait_for_complete_and_cb(rados_completion_t c) + int rados_aio_wait_for_complete(rados_completion_t c) + int rados_aio_is_complete(rados_completion_t c) + + int rados_exec(rados_ioctx_t io, const char * oid, const char * cls, const char * method, + const char * in_buf, size_t in_len, char * buf, size_t out_len) + int rados_aio_exec(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * cls, const char * method, + const char * in_buf, size_t in_len, char * buf, size_t out_len) + int rados_aio_setxattr(rados_ioctx_t io, const char *o, rados_completion_t completion, const char *name, const char *buf, size_t len) + + int rados_write_op_operate(rados_write_op_t write_op, rados_ioctx_t io, const char * oid, time_t * mtime, int flags) + int rados_aio_write_op_operate(rados_write_op_t write_op, rados_ioctx_t io, rados_completion_t completion, const char *oid, time_t *mtime, int flags) + void rados_write_op_cmpext(rados_write_op_t write_op, const char *cmp_buf, size_t cmp_len, uint64_t off, int *prval) + void rados_write_op_omap_cmp(rados_write_op_t write_op, const char *key, uint8_t comparison_operator, const char *val, size_t val_len, int *prval) + void rados_write_op_omap_set(rados_write_op_t write_op, const char * const* keys, const char * const* vals, const size_t * lens, size_t num) + void rados_write_op_omap_rm_keys(rados_write_op_t write_op, const char * const* keys, size_t keys_len) + void rados_write_op_omap_clear(rados_write_op_t write_op) + void rados_write_op_omap_rm_range2(rados_write_op_t write_op, const char *key_begin, size_t key_begin_len, const char *key_end, size_t key_end_len) + void rados_write_op_set_flags(rados_write_op_t write_op, int flags) + void rados_write_op_setxattr(rados_write_op_t write_op, const char *name, const char *value, size_t value_len) + void rados_write_op_rmxattr(rados_write_op_t write_op, const char *name) + + void rados_write_op_create(rados_write_op_t write_op, int exclusive, const char *category) + void rados_write_op_append(rados_write_op_t write_op, const char *buffer, size_t len) + void rados_write_op_write_full(rados_write_op_t write_op, const char *buffer, size_t len) + void rados_write_op_assert_version(rados_write_op_t write_op, uint64_t ver) + void rados_write_op_write(rados_write_op_t write_op, const char *buffer, size_t len, uint64_t offset) + void rados_write_op_remove(rados_write_op_t write_op) + void rados_write_op_truncate(rados_write_op_t write_op, uint64_t offset) + void rados_write_op_zero(rados_write_op_t write_op, uint64_t offset, uint64_t len) + void rados_write_op_exec(rados_write_op_t write_op, const char *cls, const char *method, const char *in_buf, size_t in_len, int *prval) + void rados_write_op_writesame(rados_write_op_t write_op, const char *buffer, size_t data_len, size_t write_len, uint64_t offset) + void rados_read_op_cmpext(rados_read_op_t read_op, const char *cmp_buf, size_t cmp_len, uint64_t off, int *prval) + void rados_read_op_omap_get_vals2(rados_read_op_t read_op, const char * start_after, const char * filter_prefix, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval) + void rados_read_op_omap_get_keys2(rados_read_op_t read_op, const char * start_after, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval) + void rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op, const char * const* keys, size_t keys_len, rados_omap_iter_t * iter, int * prval) + int rados_read_op_operate(rados_read_op_t read_op, rados_ioctx_t io, const char * oid, int flags) + int rados_aio_read_op_operate(rados_read_op_t read_op, rados_ioctx_t io, rados_completion_t completion, const char *oid, int flags) + void rados_read_op_set_flags(rados_read_op_t read_op, int flags) + int rados_omap_get_next(rados_omap_iter_t iter, const char * const* key, const char * const* val, size_t * len) + void rados_omap_get_end(rados_omap_iter_t iter) + int rados_notify2(rados_ioctx_t io, const char * o, const char *buf, int buf_len, uint64_t timeout_ms, char **reply_buffer, size_t *reply_buffer_len) + int rados_aio_notify(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, int len, uint64_t timeout_ms, char **reply_buffer, size_t *reply_buffer_len) + int rados_decode_notify_response(char *reply_buffer, size_t reply_buffer_len, notify_ack_t **acks, size_t *nr_acks, notify_timeout_t **timeouts, size_t *nr_timeouts) + void rados_free_notify_response(notify_ack_t *acks, size_t nr_acks, notify_timeout_t *timeouts) + int rados_notify_ack(rados_ioctx_t io, const char *o, uint64_t notify_id, uint64_t cookie, const char *buf, int buf_len) + int rados_watch3(rados_ioctx_t io, const char *o, uint64_t *cookie, rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, uint32_t timeout, void *arg) + int rados_watch_check(rados_ioctx_t io, uint64_t cookie) + int rados_unwatch2(rados_ioctx_t io, uint64_t cookie) + int rados_watch_flush(rados_t cluster) diff --git a/src/pybind/rados/ctime.pxd b/src/pybind/rados/ctime.pxd new file mode 100644 index 000000000..3e16ce4c2 --- /dev/null +++ b/src/pybind/rados/ctime.pxd @@ -0,0 +1,11 @@ +# cython: embedsignature=True + +cdef extern from "time.h": + ctypedef long int time_t + ctypedef long int suseconds_t + + +cdef extern from "sys/time.h": + cdef struct timeval: + time_t tv_sec + suseconds_t tv_usec diff --git a/src/pybind/rados/mock_rados.pxi b/src/pybind/rados/mock_rados.pxi new file mode 100644 index 000000000..4c1fe858b --- /dev/null +++ b/src/pybind/rados/mock_rados.pxi @@ -0,0 +1,464 @@ +# cython: embedsignature=True + +from libc.stdint cimport * +from ctime cimport time_t, timeval + +# mirrors the structure of c_rados, but instead *defines* the rados functions + +# err.h +cdef: + int _MAX_ERRNO "MAX_ERRNO" + +# rados/rados_types.h +cdef: + char* _LIBRADOS_ALL_NSPACES = "\001" + struct notify_ack_t: + unsigned long notifier_id + unsigned long cookie + char *payload + unsigned long payload_len + + struct notify_timeout_t: + unsigned long notifier_id + unsigned long cookie + +# rados/librados.h +cdef nogil: + enum: + _LIBRADOS_OP_FLAG_EXCL "LIBRADOS_OP_FLAG_EXCL" + _LIBRADOS_OP_FLAG_FAILOK "LIBRADOS_OP_FLAG_FAILOK" + _LIBRADOS_OP_FLAG_FADVISE_RANDOM "LIBRADOS_OP_FLAG_FADVISE_RANDOM" + _LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL "LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL" + _LIBRADOS_OP_FLAG_FADVISE_WILLNEED "LIBRADOS_OP_FLAG_FADVISE_WILLNEED" + _LIBRADOS_OP_FLAG_FADVISE_DONTNEED "LIBRADOS_OP_FLAG_FADVISE_DONTNEED" + _LIBRADOS_OP_FLAG_FADVISE_NOCACHE "LIBRADOS_OP_FLAG_FADVISE_NOCACHE" + + + enum: + _LIBRADOS_CMPXATTR_OP_EQ "LIBRADOS_CMPXATTR_OP_EQ" + _LIBRADOS_CMPXATTR_OP_NE "LIBRADOS_CMPXATTR_OP_NE" + _LIBRADOS_CMPXATTR_OP_GT "LIBRADOS_CMPXATTR_OP_GT" + _LIBRADOS_CMPXATTR_OP_GTE "LIBRADOS_CMPXATTR_OP_GTE" + _LIBRADOS_CMPXATTR_OP_LT "LIBRADOS_CMPXATTR_OP_LT" + _LIBRADOS_CMPXATTR_OP_LTE "LIBRADOS_CMPXATTR_OP_LTE" + + + enum: + _LIBRADOS_OPERATION_NOFLAG "LIBRADOS_OPERATION_NOFLAG" + _LIBRADOS_OPERATION_BALANCE_READS "LIBRADOS_OPERATION_BALANCE_READS" + _LIBRADOS_OPERATION_LOCALIZE_READS "LIBRADOS_OPERATION_LOCALIZE_READS" + _LIBRADOS_OPERATION_ORDER_READS_WRITES "LIBRADOS_OPERATION_ORDER_READS_WRITES" + _LIBRADOS_OPERATION_IGNORE_CACHE "LIBRADOS_OPERATION_IGNORE_CACHE" + _LIBRADOS_OPERATION_SKIPRWLOCKS "LIBRADOS_OPERATION_SKIPRWLOCKS" + _LIBRADOS_OPERATION_IGNORE_OVERLAY "LIBRADOS_OPERATION_IGNORE_OVERLAY" + _LIBRADOS_CREATE_EXCLUSIVE "LIBRADOS_CREATE_EXCLUSIVE" + _LIBRADOS_CREATE_IDEMPOTENT "LIBRADOS_CREATE_IDEMPOTENT" + + uint64_t _LIBRADOS_SNAP_HEAD "LIBRADOS_SNAP_HEAD" + + ctypedef void* rados_xattrs_iter_t + ctypedef void* rados_omap_iter_t + ctypedef void* rados_list_ctx_t + ctypedef uint64_t rados_snap_t + ctypedef void *rados_write_op_t + ctypedef void *rados_read_op_t + ctypedef void *rados_completion_t + ctypedef void (*rados_callback_t)(rados_completion_t cb, void *arg) + ctypedef void (*rados_log_callback_t)(void *arg, const char *line, const char *who, + uint64_t sec, uint64_t nsec, uint64_t seq, const char *level, const char *msg) + ctypedef void (*rados_log_callback2_t)(void *arg, const char *line, const char *channel, const char *who, const char *name, + uint64_t sec, uint64_t nsec, uint64_t seq, const char *level, const char *msg) + ctypedef void (*rados_watchcb2_t)(void *arg, int64_t notify_id, + uint64_t handle, uint64_t notifier_id, + void *data, size_t data_len) + ctypedef void (*rados_watcherrcb_t)(void *pre, uint64_t cookie, int err) + + + struct rados_cluster_stat_t: + uint64_t kb + uint64_t kb_used + uint64_t kb_avail + uint64_t num_objects + + struct rados_pool_stat_t: + uint64_t num_bytes + uint64_t num_kb + uint64_t num_objects + uint64_t num_object_clones + uint64_t num_object_copies + uint64_t num_objects_missing_on_primary + uint64_t num_objects_unfound + uint64_t num_objects_degraded + uint64_t num_rd + uint64_t num_rd_kb + uint64_t num_wr + uint64_t num_wr_kb + + void rados_buffer_free(char *buf): + pass + + void rados_version(int *major, int *minor, int *extra): + pass + + int rados_create2(rados_t *pcluster, const char *const clustername, + const char * const name, uint64_t flags): + pass + + int rados_create_with_context(rados_t *cluster, rados_config_t cct): + pass + int rados_connect(rados_t cluster): + pass + void rados_shutdown(rados_t cluster): + pass + cdef uint64_t rados_get_instance_id(rados_t cluster): + pass + int rados_conf_read_file(rados_t cluster, const char *path): + pass + int rados_conf_parse_argv_remainder(rados_t cluster, int argc, const char **argv, const char **remargv): + pass + int rados_conf_parse_env(rados_t cluster, const char *var): + pass + int rados_conf_set(rados_t cluster, char *option, const char *value): + pass + int rados_conf_get(rados_t cluster, char *option, char *buf, size_t len): + pass + + rados_t rados_ioctx_get_cluster(rados_ioctx_t io): + pass + int rados_ioctx_pool_stat(rados_ioctx_t io, rados_pool_stat_t *stats): + pass + int64_t rados_pool_lookup(rados_t cluster, const char *pool_name): + pass + int rados_pool_reverse_lookup(rados_t cluster, int64_t id, char *buf, size_t maxlen): + pass + int rados_pool_create(rados_t cluster, const char *pool_name): + pass + int rados_pool_create_with_crush_rule(rados_t cluster, const char *pool_name, uint8_t crush_rule_num): + pass + int rados_pool_create_with_auid(rados_t cluster, const char *pool_name, uint64_t auid): + pass + int rados_pool_create_with_all(rados_t cluster, const char *pool_name, uint64_t auid, uint8_t crush_rule_num): + pass + int rados_pool_get_base_tier(rados_t cluster, int64_t pool, int64_t *base_tier): + pass + int rados_pool_list(rados_t cluster, char *buf, size_t len): + pass + int rados_pool_delete(rados_t cluster, const char *pool_name): + pass + int rados_inconsistent_pg_list(rados_t cluster, int64_t pool, char *buf, size_t len): + pass + + int rados_cluster_stat(rados_t cluster, rados_cluster_stat_t *result): + pass + int rados_cluster_fsid(rados_t cluster, char *buf, size_t len): + pass + int rados_blocklist_add(rados_t cluster, char *client_address, uint32_t expire_seconds): + pass + int rados_getaddrs(rados_t cluster, char** addrs): + pass + int rados_application_enable(rados_ioctx_t io, const char *app_name, + int force): + pass + void rados_set_pool_full_try(rados_ioctx_t io): + pass + void rados_unset_pool_full_try(rados_ioctx_t io): + pass + int rados_application_list(rados_ioctx_t io, char *values, + size_t *values_len): + pass + int rados_application_metadata_get(rados_ioctx_t io, const char *app_name, + const char *key, char *value, + size_t *value_len): + pass + int rados_application_metadata_set(rados_ioctx_t io, const char *app_name, + const char *key, const char *value): + pass + int rados_application_metadata_remove(rados_ioctx_t io, + const char *app_name, const char *key): + pass + int rados_application_metadata_list(rados_ioctx_t io, + const char *app_name, char *keys, + size_t *key_len, char *values, + size_t *value_len): + pass + int rados_ping_monitor(rados_t cluster, const char *mon_id, char **outstr, size_t *outstrlen): + pass + int rados_mon_command(rados_t cluster, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen): + pass + int rados_mgr_command(rados_t cluster, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen): + pass + int rados_mgr_command_target(rados_t cluster, + const char *name, + const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen): + pass + int rados_mon_command_target(rados_t cluster, const char *name, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen): + pass + int rados_osd_command(rados_t cluster, int osdid, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen): + pass + int rados_pg_command(rados_t cluster, const char *pgstr, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, + char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen): + pass + int rados_monitor_log(rados_t cluster, const char *level, rados_log_callback_t cb, void *arg): + pass + int rados_monitor_log2(rados_t cluster, const char *level, rados_log_callback2_t cb, void *arg): + pass + + int rados_wait_for_latest_osdmap(rados_t cluster): + pass + + int rados_service_register(rados_t cluster, const char *service, const char *daemon, const char *metadata_dict): + pass + int rados_service_update_status(rados_t cluster, const char *status_dict): + pass + + int rados_ioctx_create(rados_t cluster, const char *pool_name, rados_ioctx_t *ioctx): + pass + int rados_ioctx_create2(rados_t cluster, int64_t pool_id, rados_ioctx_t *ioctx): + pass + void rados_ioctx_destroy(rados_ioctx_t io): + pass + void rados_ioctx_locator_set_key(rados_ioctx_t io, const char *key): + pass + void rados_ioctx_set_namespace(rados_ioctx_t io, const char * nspace): + pass + + cdef uint64_t rados_get_last_version(rados_ioctx_t io): + pass + int rados_stat(rados_ioctx_t io, const char *o, uint64_t *psize, time_t *pmtime): + pass + int rados_write(rados_ioctx_t io, const char *oid, const char *buf, size_t len, uint64_t off): + pass + int rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len): + pass + int rados_writesame(rados_ioctx_t io, const char *oid, const char *buf, size_t data_len, size_t write_len, uint64_t off): + pass + int rados_append(rados_ioctx_t io, const char *oid, const char *buf, size_t len): + pass + int rados_read(rados_ioctx_t io, const char *oid, char *buf, size_t len, uint64_t off): + pass + int rados_remove(rados_ioctx_t io, const char *oid): + pass + int rados_trunc(rados_ioctx_t io, const char *oid, uint64_t size): + pass + int rados_cmpext(rados_ioctx_t io, const char *o, const char *cmp_buf, size_t cmp_len, uint64_t off): + pass + int rados_getxattr(rados_ioctx_t io, const char *o, const char *name, char *buf, size_t len): + pass + int rados_setxattr(rados_ioctx_t io, const char *o, const char *name, const char *buf, size_t len): + pass + int rados_rmxattr(rados_ioctx_t io, const char *o, const char *name): + pass + int rados_getxattrs(rados_ioctx_t io, const char *oid, rados_xattrs_iter_t *iter): + pass + int rados_getxattrs_next(rados_xattrs_iter_t iter, const char **name, const char **val, size_t *len): + pass + void rados_getxattrs_end(rados_xattrs_iter_t iter): + pass + + int rados_nobjects_list_open(rados_ioctx_t io, rados_list_ctx_t *ctx): + pass + int rados_nobjects_list_next(rados_list_ctx_t ctx, const char **entry, const char **key, const char **nspace): + pass + void rados_nobjects_list_close(rados_list_ctx_t ctx): + pass + + int rados_ioctx_pool_requires_alignment2(rados_ioctx_t io, int * requires): + pass + int rados_ioctx_pool_required_alignment2(rados_ioctx_t io, uint64_t * alignment): + pass + + int rados_ioctx_snap_rollback(rados_ioctx_t io, const char * oid, const char * snapname): + pass + int rados_ioctx_snap_create(rados_ioctx_t io, const char * snapname): + pass + int rados_ioctx_snap_remove(rados_ioctx_t io, const char * snapname): + pass + int rados_ioctx_snap_lookup(rados_ioctx_t io, const char * name, rados_snap_t * id): + pass + int rados_ioctx_snap_get_name(rados_ioctx_t io, rados_snap_t id, char * name, int maxlen): + pass + void rados_ioctx_snap_set_read(rados_ioctx_t io, rados_snap_t snap): + pass + int rados_ioctx_snap_list(rados_ioctx_t io, rados_snap_t * snaps, int maxlen): + pass + int rados_ioctx_snap_get_stamp(rados_ioctx_t io, rados_snap_t id, time_t * t): + pass + int64_t rados_ioctx_get_id(rados_ioctx_t io): + pass + int rados_ioctx_get_pool_name(rados_ioctx_t io, char *buf, unsigned maxlen): + pass + + int rados_ioctx_selfmanaged_snap_create(rados_ioctx_t io, + rados_snap_t *snapid): + pass + int rados_ioctx_selfmanaged_snap_remove(rados_ioctx_t io, + rados_snap_t snapid): + pass + int rados_ioctx_selfmanaged_snap_set_write_ctx(rados_ioctx_t io, + rados_snap_t snap_seq, + rados_snap_t *snap, + int num_snaps): + pass + int rados_ioctx_selfmanaged_snap_rollback(rados_ioctx_t io, const char *oid, + rados_snap_t snapid): + pass + + int rados_lock_exclusive(rados_ioctx_t io, const char * oid, const char * name, + const char * cookie, const char * desc, + timeval * duration, uint8_t flags): + pass + int rados_lock_shared(rados_ioctx_t io, const char * o, const char * name, + const char * cookie, const char * tag, const char * desc, + timeval * duration, uint8_t flags): + pass + int rados_unlock(rados_ioctx_t io, const char * o, const char * name, const char * cookie): + pass + + rados_write_op_t rados_create_write_op(): + pass + void rados_release_write_op(rados_write_op_t write_op): + pass + + rados_read_op_t rados_create_read_op(): + pass + void rados_release_read_op(rados_read_op_t read_op): + pass + + int rados_aio_create_completion2(void * cb_arg, rados_callback_t cb_complete, rados_completion_t * pc): + pass + void rados_aio_release(rados_completion_t c): + pass + int rados_aio_stat(rados_ioctx_t io, const char *oid, rados_completion_t completion, uint64_t *psize, time_t *pmtime): + pass + int rados_aio_write(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len, uint64_t off): + pass + int rados_aio_append(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len): + pass + int rados_aio_write_full(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len): + pass + int rados_aio_writesame(rados_ioctx_t io, const char *oid, rados_completion_t completion, const char *buf, size_t data_len, size_t write_len, uint64_t off): + pass + int rados_aio_remove(rados_ioctx_t io, const char * oid, rados_completion_t completion): + pass + int rados_aio_read(rados_ioctx_t io, const char * oid, rados_completion_t completion, char * buf, size_t len, uint64_t off): + pass + int rados_aio_flush(rados_ioctx_t io): + pass + int rados_aio_cmpext(rados_ioctx_t io, const char *o, rados_completion_t completion, const char *cmp_buf, size_t cmp_len, uint64_t off): + pass + int rados_aio_rmxattr(rados_ioctx_t io, const char *o, rados_completion_t completion, const char *name): + pass + + int rados_aio_get_return_value(rados_completion_t c): + pass + int rados_aio_wait_for_complete_and_cb(rados_completion_t c): + pass + int rados_aio_wait_for_complete(rados_completion_t c): + pass + int rados_aio_is_complete(rados_completion_t c): + pass + + int rados_exec(rados_ioctx_t io, const char * oid, const char * cls, const char * method, + const char * in_buf, size_t in_len, char * buf, size_t out_len): + pass + int rados_aio_exec(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * cls, const char * method, + const char * in_buf, size_t in_len, char * buf, size_t out_len): + pass + int rados_aio_setxattr(rados_ioctx_t io, const char *o, rados_completion_t completion, const char *name, const char *buf, size_t len): + pass + int rados_write_op_operate(rados_write_op_t write_op, rados_ioctx_t io, const char * oid, time_t * mtime, int flags): + pass + int rados_aio_write_op_operate(rados_write_op_t write_op, rados_ioctx_t io, rados_completion_t completion, const char *oid, time_t *mtime, int flags): + pass + void rados_write_op_cmpext(rados_write_op_t write_op, const char *cmp_buf, size_t cmp_len, uint64_t off, int *prval): + pass + void rados_write_op_omap_cmp(rados_write_op_t write_op, const char *key, uint8_t comparison_operator, const char *val, size_t val_len, int *prval): + pass + void rados_write_op_omap_set(rados_write_op_t write_op, const char * const* keys, const char * const* vals, const size_t * lens, size_t num): + pass + void rados_write_op_omap_rm_keys(rados_write_op_t write_op, const char * const* keys, size_t keys_len): + pass + void rados_write_op_omap_clear(rados_write_op_t write_op): + pass + void rados_write_op_omap_rm_range2(rados_write_op_t write_op, const char *key_begin, size_t key_begin_len, const char *key_end, size_t key_end_len): + pass + void rados_write_op_set_flags(rados_write_op_t write_op, int flags): + pass + void rados_write_op_setxattr(rados_write_op_t write_op, const char *name, const char *value, size_t value_len): + pass + void rados_write_op_rmxattr(rados_write_op_t write_op, const char *name): + pass + + void rados_write_op_create(rados_write_op_t write_op, int exclusive, const char *category): + pass + void rados_write_op_append(rados_write_op_t write_op, const char *buffer, size_t len): + pass + void rados_write_op_write_full(rados_write_op_t write_op, const char *buffer, size_t len): + pass + void rados_write_op_assert_version(rados_write_op_t write_op, uint64_t ver): + pass + void rados_write_op_write(rados_write_op_t write_op, const char *buffer, size_t len, uint64_t offset): + pass + void rados_write_op_remove(rados_write_op_t write_op): + pass + void rados_write_op_truncate(rados_write_op_t write_op, uint64_t offset): + pass + void rados_write_op_zero(rados_write_op_t write_op, uint64_t offset, uint64_t len): + pass + void rados_write_op_exec(rados_write_op_t write_op, const char *cls, const char *method, const char *in_buf, size_t in_len, int *prval): + pass + void rados_write_op_writesame(rados_write_op_t write_op, const char *buffer, size_t data_len, size_t write_len, uint64_t offset): + pass + void rados_read_op_cmpext(rados_read_op_t read_op, const char *cmp_buf, size_t cmp_len, uint64_t off, int *prval): + pass + void rados_read_op_omap_get_vals2(rados_read_op_t read_op, const char * start_after, const char * filter_prefix, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval): + pass + void rados_read_op_omap_get_keys2(rados_read_op_t read_op, const char * start_after, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval): + pass + void rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op, const char * const* keys, size_t keys_len, rados_omap_iter_t * iter, int * prval): + pass + int rados_read_op_operate(rados_read_op_t read_op, rados_ioctx_t io, const char * oid, int flags): + pass + int rados_aio_read_op_operate(rados_read_op_t read_op, rados_ioctx_t io, rados_completion_t completion, const char *oid, int flags): + pass + void rados_read_op_set_flags(rados_read_op_t read_op, int flags): + pass + int rados_omap_get_next(rados_omap_iter_t iter, const char * const* key, const char * const* val, size_t * len): + pass + void rados_omap_get_end(rados_omap_iter_t iter): + pass + int rados_notify2(rados_ioctx_t io, const char * o, const char *buf, int buf_len, uint64_t timeout_ms, char **reply_buffer, size_t *reply_buffer_len): + pass + int rados_aio_notify(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, int len, uint64_t timeout_ms, char **reply_buffer, size_t *reply_buffer_len): + pass + int rados_decode_notify_response(char *reply_buffer, size_t reply_buffer_len, notify_ack_t **acks, size_t *nr_acks, notify_timeout_t **timeouts, size_t *nr_timeouts): + pass + void rados_free_notify_response(notify_ack_t *acks, size_t nr_acks, notify_timeout_t *timeouts): + pass + int rados_notify_ack(rados_ioctx_t io, const char *o, uint64_t notify_id, uint64_t cookie, const char *buf, int buf_len): + pass + int rados_watch3(rados_ioctx_t io, const char *o, uint64_t *cookie, rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, uint32_t timeout, void *arg): + pass + int rados_watch_check(rados_ioctx_t io, uint64_t cookie): + pass + int rados_unwatch2(rados_ioctx_t io, uint64_t cookie): + pass + int rados_watch_flush(rados_t cluster): + pass diff --git a/src/pybind/rados/rados.pxd b/src/pybind/rados/rados.pxd new file mode 100644 index 000000000..fec534e8c --- /dev/null +++ b/src/pybind/rados/rados.pxd @@ -0,0 +1,45 @@ +# cython: embedsignature=True +# +# Shared object for librbdpy +# +# Copyright 2016 Mehdi Abaakouk <sileht@redhat.com> + +IF BUILD_DOC: + cdef: + ctypedef void* rados_t + ctypedef void* rados_config_t + ctypedef void* rados_ioctx_t +ELSE: + cdef extern from "rados/librados.h" nogil: + ctypedef void* rados_t + ctypedef void* rados_config_t + ctypedef void* rados_ioctx_t + + +cdef class Rados(object): + cdef: + rados_t cluster + public object state + public object monitor_callback + public object monitor_callback2 + public object parsed_args + public object conf_defaults + public object conffile + public object rados_id + + +cdef class Ioctx(object): + cdef: + object rados + rados_ioctx_t io + public object name + public object state + public object locator_key + public object nspace + + # TODO(sileht): we need to track leaving completion objects + # I guess we can do that in a lighter ways, but keep code simple + # as before for now + public object safe_completions + public object complete_completions + public object lock diff --git a/src/pybind/rados/rados.pyx b/src/pybind/rados/rados.pyx new file mode 100644 index 000000000..67d8a1e2c --- /dev/null +++ b/src/pybind/rados/rados.pyx @@ -0,0 +1,4338 @@ +# cython: embedsignature=True, binding=True +""" +This module is a thin wrapper around librados. + +Error codes from librados are turned into exceptions that subclass +:class:`Error`. Almost all methods may raise :class:`Error(the base class of all rados exceptions), :class:`PermissionError` +(the base class of all rados exceptions), :class:`PermissionError` +and :class:`IOError`, in addition to those documented for the +method. +""" +# Copyright 2011 Josh Durgin +# Copyright 2011, Hannu Valtonen <hannu.valtonen@ormod.com> +# Copyright 2015 Hector Martin <marcan@marcan.st> +# Copyright 2016 Mehdi Abaakouk <sileht@redhat.com> + +from cpython cimport PyObject, ref +from cpython.pycapsule cimport * +from libc cimport errno +from libc.stdint cimport * +from libc.stdlib cimport malloc, realloc, free +IF BUILD_DOC: + include "mock_rados.pxi" +ELSE: + from c_rados cimport * + +import threading +import time + +from datetime import datetime, timedelta +from functools import partial, wraps +from itertools import chain +from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union + +cdef extern from "Python.h": + # These are in cpython/string.pxd, but use "object" types instead of + # PyObject*, which invokes assumptions in cpython that we need to + # legitimately break to implement zero-copy string buffers in Ioctx.read(). + # This is valid use of the Python API and documented as a special case. + PyObject *PyBytes_FromStringAndSize(char *v, Py_ssize_t len) except NULL + char* PyBytes_AsString(PyObject *string) except NULL + int _PyBytes_Resize(PyObject **string, Py_ssize_t newsize) except -1 + void PyEval_InitThreads() + +LIBRADOS_OP_FLAG_EXCL = _LIBRADOS_OP_FLAG_EXCL +LIBRADOS_OP_FLAG_FAILOK = _LIBRADOS_OP_FLAG_FAILOK +LIBRADOS_OP_FLAG_FADVISE_RANDOM = _LIBRADOS_OP_FLAG_FADVISE_RANDOM +LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL = _LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL +LIBRADOS_OP_FLAG_FADVISE_WILLNEED = _LIBRADOS_OP_FLAG_FADVISE_WILLNEED +LIBRADOS_OP_FLAG_FADVISE_DONTNEED = _LIBRADOS_OP_FLAG_FADVISE_DONTNEED +LIBRADOS_OP_FLAG_FADVISE_NOCACHE = _LIBRADOS_OP_FLAG_FADVISE_NOCACHE + +LIBRADOS_CMPXATTR_OP_EQ = _LIBRADOS_CMPXATTR_OP_EQ +LIBRADOS_CMPXATTR_OP_NE = _LIBRADOS_CMPXATTR_OP_NE +LIBRADOS_CMPXATTR_OP_GT = _LIBRADOS_CMPXATTR_OP_GT +LIBRADOS_CMPXATTR_OP_GTE = _LIBRADOS_CMPXATTR_OP_GTE +LIBRADOS_CMPXATTR_OP_LT = _LIBRADOS_CMPXATTR_OP_LT +LIBRADOS_CMPXATTR_OP_LTE = _LIBRADOS_CMPXATTR_OP_LTE + +LIBRADOS_SNAP_HEAD = _LIBRADOS_SNAP_HEAD + +LIBRADOS_OPERATION_NOFLAG = _LIBRADOS_OPERATION_NOFLAG +LIBRADOS_OPERATION_BALANCE_READS = _LIBRADOS_OPERATION_BALANCE_READS +LIBRADOS_OPERATION_LOCALIZE_READS = _LIBRADOS_OPERATION_LOCALIZE_READS +LIBRADOS_OPERATION_ORDER_READS_WRITES = _LIBRADOS_OPERATION_ORDER_READS_WRITES +LIBRADOS_OPERATION_IGNORE_CACHE = _LIBRADOS_OPERATION_IGNORE_CACHE +LIBRADOS_OPERATION_SKIPRWLOCKS = _LIBRADOS_OPERATION_SKIPRWLOCKS +LIBRADOS_OPERATION_IGNORE_OVERLAY = _LIBRADOS_OPERATION_IGNORE_OVERLAY + +LIBRADOS_ALL_NSPACES = _LIBRADOS_ALL_NSPACES.decode('utf-8') + +LIBRADOS_CREATE_EXCLUSIVE = _LIBRADOS_CREATE_EXCLUSIVE +LIBRADOS_CREATE_IDEMPOTENT = _LIBRADOS_CREATE_IDEMPOTENT + +MAX_ERRNO = _MAX_ERRNO + +ANONYMOUS_AUID = 0xffffffffffffffff +ADMIN_AUID = 0 + + +class Error(Exception): + """ `Error` class, derived from `Exception` """ + def __init__(self, message, errno=None): + super(Exception, self).__init__(message) + self.errno = errno + + def __str__(self): + msg = super(Exception, self).__str__() + if self.errno is None: + return msg + return '[errno {0}] {1}'.format(self.errno, msg) + + def __reduce__(self): + return (self.__class__, (self.message, self.errno)) + +class InvalidArgumentError(Error): + def __init__(self, message, errno=None): + super(InvalidArgumentError, self).__init__( + "RADOS invalid argument (%s)" % message, errno) + +class ExtendMismatch(Error): + def __init__(self, message, errno, offset): + super().__init__( + "object content does not match (%s)" % message, errno) + self.offset = offset + +class OSError(Error): + """ `OSError` class, derived from `Error` """ + pass + +class InterruptedOrTimeoutError(OSError): + """ `InterruptedOrTimeoutError` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(InterruptedOrTimeoutError, self).__init__( + "RADOS interrupted or timeout (%s)" % message, errno) + + +class PermissionError(OSError): + """ `PermissionError` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(PermissionError, self).__init__( + "RADOS permission error (%s)" % message, errno) + + +class PermissionDeniedError(OSError): + """ deal with EACCES related. """ + def __init__(self, message, errno=None): + super(PermissionDeniedError, self).__init__( + "RADOS permission denied (%s)" % message, errno) + + +class ObjectNotFound(OSError): + """ `ObjectNotFound` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(ObjectNotFound, self).__init__( + "RADOS object not found (%s)" % message, errno) + + +class NoData(OSError): + """ `NoData` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(NoData, self).__init__( + "RADOS no data (%s)" % message, errno) + + +class ObjectExists(OSError): + """ `ObjectExists` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(ObjectExists, self).__init__( + "RADOS object exists (%s)" % message, errno) + + +class ObjectBusy(OSError): + """ `ObjectBusy` class, derived from `IOError` """ + def __init__(self, message, errno=None): + super(ObjectBusy, self).__init__( + "RADOS object busy (%s)" % message, errno) + + +class IOError(OSError): + """ `ObjectBusy` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(IOError, self).__init__( + "RADOS I/O error (%s)" % message, errno) + + +class NoSpace(OSError): + """ `NoSpace` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(NoSpace, self).__init__( + "RADOS no space (%s)" % message, errno) + +class NotConnected(OSError): + """ `NotConnected` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(NotConnected, self).__init__( + "RADOS not connected (%s)" % message, errno) + +class RadosStateError(Error): + """ `RadosStateError` class, derived from `Error` """ + def __init__(self, message, errno=None): + super(RadosStateError, self).__init__( + "RADOS rados state (%s)" % message, errno) + + +class IoctxStateError(Error): + """ `IoctxStateError` class, derived from `Error` """ + def __init__(self, message, errno=None): + super(IoctxStateError, self).__init__( + "RADOS Ioctx state error (%s)" % message, errno) + + +class ObjectStateError(Error): + """ `ObjectStateError` class, derived from `Error` """ + def __init__(self, message, errno=None): + super(ObjectStateError, self).__init__( + "RADOS object state error (%s)" % message, errno) + + +class LogicError(Error): + """ `` class, derived from `Error` """ + def __init__(self, message, errno=None): + super(LogicError, self).__init__( + "RADOS logic error (%s)" % message, errno) + + +class TimedOut(OSError): + """ `TimedOut` class, derived from `OSError` """ + def __init__(self, message, errno=None): + super(TimedOut, self).__init__( + "RADOS timed out (%s)" % message, errno) + + +class InProgress(Error): + """ `InProgress` class, derived from `Error` """ + def __init__(self, message, errno=None): + super(InProgress, self).__init__( + "RADOS in progress error (%s)" % message, errno) + + +class IsConnected(Error): + """ `IsConnected` class, derived from `Error` """ + def __init__(self, message, errno=None): + super(IsConnected, self).__init__( + "RADOS is connected error (%s)" % message, errno) + + +IF UNAME_SYSNAME == "FreeBSD": + cdef errno_to_exception = { + errno.EPERM : PermissionError, + errno.ENOENT : ObjectNotFound, + errno.EIO : IOError, + errno.ENOSPC : NoSpace, + errno.EEXIST : ObjectExists, + errno.EBUSY : ObjectBusy, + errno.ENOATTR : NoData, + errno.EINTR : InterruptedOrTimeoutError, + errno.ETIMEDOUT : TimedOut, + errno.EACCES : PermissionDeniedError, + errno.EINPROGRESS : InProgress, + errno.EISCONN : IsConnected, + errno.EINVAL : InvalidArgumentError, + errno.ENOTCONN : NotConnected, + } +ELSE: + cdef errno_to_exception = { + errno.EPERM : PermissionError, + errno.ENOENT : ObjectNotFound, + errno.EIO : IOError, + errno.ENOSPC : NoSpace, + errno.EEXIST : ObjectExists, + errno.EBUSY : ObjectBusy, + errno.ENODATA : NoData, + errno.EINTR : InterruptedOrTimeoutError, + errno.ETIMEDOUT : TimedOut, + errno.EACCES : PermissionDeniedError, + errno.EINPROGRESS : InProgress, + errno.EISCONN : IsConnected, + errno.EINVAL : InvalidArgumentError, + errno.ENOTCONN : NotConnected, + } + + +cdef make_ex(ret: int, msg: str): + """ + Translate a librados return code into an exception. + + :param ret: the return code + :type ret: int + :param msg: the error message to use + :type msg: str + :returns: a subclass of :class:`Error` + """ + ret = abs(ret) + if ret in errno_to_exception: + return errno_to_exception[ret](msg, errno=ret) + elif ret > MAX_ERRNO: + offset = ret - MAX_ERRNO + return ExtendMismatch(msg, ret, offset) + else: + return OSError(msg, errno=ret) + + +def cstr(val, name, encoding="utf-8", opt=False): + """ + Create a byte string from a Python string + + :param basestring val: Python string + :param str name: Name of the string parameter, for exceptions + :param str encoding: Encoding to use + :param bool opt: If True, None is allowed + :rtype: bytes + :raises: :class:`InvalidArgument` + """ + if opt and val is None: + return None + if isinstance(val, bytes): + return val + elif isinstance(val, str): + return val.encode(encoding) + else: + raise TypeError('%s must be a string' % name) + + +def cstr_list(list_str, name, encoding="utf-8"): + return [cstr(s, name) for s in list_str] + + +def decode_cstr(val, encoding="utf-8"): + """ + Decode a byte string into a Python string. + + :param bytes val: byte string + :rtype: str or None + """ + if val is None: + return None + + return val.decode(encoding) + + +def flatten_dict(d, name): + items = chain.from_iterable(d.items()) + return cstr(''.join(i + '\0' for i in items), name) + + +cdef char* opt_str(s) except? NULL: + if s is None: + return NULL + return s + + +cdef void* realloc_chk(void* ptr, size_t size) except NULL: + cdef void *ret = realloc(ptr, size) + if ret == NULL: + raise MemoryError("realloc failed") + return ret + + +cdef size_t * to_csize_t_array(list_int): + cdef size_t *ret = <size_t *>malloc(len(list_int) * sizeof(size_t)) + if ret == NULL: + raise MemoryError("malloc failed") + for i in range(len(list_int)): + ret[i] = <size_t>list_int[i] + return ret + + +cdef char ** to_bytes_array(list_bytes): + cdef char **ret = <char **>malloc(len(list_bytes) * sizeof(char *)) + if ret == NULL: + raise MemoryError("malloc failed") + for i in range(len(list_bytes)): + ret[i] = <char *>list_bytes[i] + return ret + +cdef int __monitor_callback(void *arg, const char *line, const char *who, + uint64_t sec, uint64_t nsec, uint64_t seq, + const char *level, const char *msg) with gil: + cdef object cb_info = <object>arg + cb_info[0](cb_info[1], line, who, sec, nsec, seq, level, msg) + return 0 + +cdef int __monitor_callback2(void *arg, const char *line, const char *channel, + const char *who, + const char *name, + uint64_t sec, uint64_t nsec, uint64_t seq, + const char *level, const char *msg) with gil: + cdef object cb_info = <object>arg + cb_info[0](cb_info[1], line, channel, name, who, sec, nsec, seq, level, msg) + return 0 + + +class Version(object): + """ Version information """ + def __init__(self, major, minor, extra): + self.major = major + self.minor = minor + self.extra = extra + + def __str__(self): + return "%d.%d.%d" % (self.major, self.minor, self.extra) + + +cdef class Rados(object): + """This class wraps librados functions""" + # NOTE(sileht): attributes declared in .pyd + + def __init__(self, *args, **kwargs): + PyEval_InitThreads() + self.__setup(*args, **kwargs) + + NO_CONF_FILE = -1 + "special value that indicates no conffile should be read when creating a mount handle" + DEFAULT_CONF_FILES = -2 + "special value that indicates the default conffiles should be read when creating a mount handle" + + def __setup(self, + rados_id: Optional[str] = None, + name: Optional[str] = None, + clustername: Optional[str] = None, + conf_defaults: Optional[Dict[str, str]] = None, + conffile: Union[str, int, None] = NO_CONF_FILE, + conf: Optional[Dict[str, str]] = None, + flags: int = 0, + context: object = None): + self.monitor_callback = None + self.monitor_callback2 = None + self.parsed_args = [] + self.conf_defaults = conf_defaults + self.conffile = conffile + self.rados_id = rados_id + + if rados_id and name: + raise Error("Rados(): can't supply both rados_id and name") + elif rados_id: + name = 'client.' + rados_id + elif name is None: + name = 'client.admin' + if clustername is None: + clustername = '' + + name_raw = cstr(name, 'name') + clustername_raw = cstr(clustername, 'clustername') + cdef: + char *_name = name_raw + char *_clustername = clustername_raw + int _flags = flags + int ret + + if context: + # Unpack void* (aka rados_config_t) from capsule + rados_config = <rados_config_t> PyCapsule_GetPointer(context, NULL) + with nogil: + ret = rados_create_with_context(&self.cluster, rados_config) + else: + with nogil: + ret = rados_create2(&self.cluster, _clustername, _name, _flags) + if ret != 0: + raise Error("rados_initialize failed with error code: %d" % ret) + + self.state = "configuring" + # order is important: conf_defaults, then conffile, then conf + if conf_defaults: + for key, value in conf_defaults.items(): + self.conf_set(key, value) + if conffile in (self.NO_CONF_FILE, None): + pass + elif conffile in (self.DEFAULT_CONF_FILES, ''): + self.conf_read_file(None) + else: + self.conf_read_file(conffile) + if conf: + for key, value in conf.items(): + self.conf_set(key, value) + + def get_addrs(self): + """ + Get associated client addresses with this RADOS session. + """ + self.require_state("configuring", "connected") + + cdef: + char* addrs = NULL + + try: + + with nogil: + ret = rados_getaddrs(self.cluster, &addrs) + if ret: + raise make_ex(ret, "error calling getaddrs") + + return decode_cstr(addrs) + finally: + free(addrs) + + def require_state(self, *args): + """ + Checks if the Rados object is in a special state + + :raises: :class:`RadosStateError` + """ + if self.state in args: + return + raise RadosStateError("You cannot perform that operation on a \ +Rados object in state %s." % self.state) + + def shutdown(self): + """ + Disconnects from the cluster. Call this explicitly when a + Rados.connect()ed object is no longer used. + """ + if self.state != "shutdown": + with nogil: + rados_shutdown(self.cluster) + self.state = "shutdown" + + def __enter__(self): + self.connect() + return self + + def __exit__(self, type_, value, traceback): + self.shutdown() + return False + + def version(self) -> Version: + """ + Get the version number of the ``librados`` C library. + + :returns: a tuple of ``(major, minor, extra)`` components of the + librados version + """ + cdef int major = 0 + cdef int minor = 0 + cdef int extra = 0 + with nogil: + rados_version(&major, &minor, &extra) + return Version(major, minor, extra) + + def conf_read_file(self, path: Optional[str] = None): + """ + Configure the cluster handle using a Ceph config file. + + :param path: path to the config file + """ + self.require_state("configuring", "connected") + path_raw = cstr(path, 'path', opt=True) + cdef: + char *_path = opt_str(path_raw) + with nogil: + ret = rados_conf_read_file(self.cluster, _path) + if ret != 0: + raise make_ex(ret, "error calling conf_read_file") + + def conf_parse_argv(self, args: Sequence[str]): + """ + Parse known arguments from args, and remove; returned + args contain only those unknown to ceph + """ + self.require_state("configuring", "connected") + if not args: + return + + cargs = cstr_list(args, 'args') + cdef: + int _argc = len(args) + char **_argv = to_bytes_array(cargs) + char **_remargv = NULL + + try: + _remargv = <char **>malloc(_argc * sizeof(char *)) + with nogil: + ret = rados_conf_parse_argv_remainder(self.cluster, _argc, + <const char**>_argv, + <const char**>_remargv) + if ret: + raise make_ex(ret, "error calling conf_parse_argv_remainder") + + # _remargv was allocated with fixed argc; collapse return + # list to eliminate any missing args + retargs = [decode_cstr(a) for a in _remargv[:_argc] + if a != NULL] + self.parsed_args = args + return retargs + finally: + free(_argv) + free(_remargv) + + def conf_parse_env(self, var: Optional[str] = 'CEPH_ARGS'): + """ + Parse known arguments from an environment variable, normally + CEPH_ARGS. + """ + self.require_state("configuring", "connected") + if not var: + return + + var_raw = cstr(var, 'var') + cdef: + char *_var = var_raw + with nogil: + ret = rados_conf_parse_env(self.cluster, _var) + if ret != 0: + raise make_ex(ret, "error calling conf_parse_env") + + def conf_get(self, option: str) -> Optional[str]: + """ + Get the value of a configuration option + + :param option: which option to read + + :returns: str - value of the option or None + :raises: :class:`TypeError` + """ + self.require_state("configuring", "connected") + option_raw = cstr(option, 'option') + cdef: + char *_option = option_raw + size_t length = 20 + char *ret_buf = NULL + + try: + while True: + ret_buf = <char *>realloc_chk(ret_buf, length) + with nogil: + ret = rados_conf_get(self.cluster, _option, ret_buf, length) + if ret == 0: + return decode_cstr(ret_buf) + elif ret == -errno.ENAMETOOLONG: + length = length * 2 + elif ret == -errno.ENOENT: + return None + else: + raise make_ex(ret, "error calling conf_get") + finally: + free(ret_buf) + + def conf_set(self, option: str, val: str): + """ + Set the value of a configuration option + + :param option: which option to set + :param option: value of the option + + :raises: :class:`TypeError`, :class:`ObjectNotFound` + """ + self.require_state("configuring", "connected") + option_raw = cstr(option, 'option') + val_raw = cstr(val, 'val') + cdef: + char *_option = option_raw + char *_val = val_raw + + with nogil: + ret = rados_conf_set(self.cluster, _option, _val) + if ret != 0: + raise make_ex(ret, "error calling conf_set") + + def ping_monitor(self, mon_id: str): + """ + Ping a monitor to assess liveness + + May be used as a simply way to assess liveness, or to obtain + information about the monitor in a simple way even in the + absence of quorum. + + :param mon_id: the ID portion of the monitor's name (i.e., mon.<ID>) + :returns: the string reply from the monitor + """ + + self.require_state("configuring", "connected") + + mon_id_raw = cstr(mon_id, 'mon_id') + cdef: + char *_mon_id = mon_id_raw + size_t outstrlen = 0 + char *outstr + + with nogil: + ret = rados_ping_monitor(self.cluster, _mon_id, &outstr, &outstrlen) + + if ret != 0: + raise make_ex(ret, "error calling ping_monitor") + + if outstrlen: + my_outstr = outstr[:outstrlen] + rados_buffer_free(outstr) + return decode_cstr(my_outstr) + + def connect(self, timeout: int = 0): + """ + Connect to the cluster. Use shutdown() to release resources. + """ + self.require_state("configuring") + # NOTE(sileht): timeout was supported by old python API, + # but this is not something available in C API, so ignore + # for now and remove it later + with nogil: + ret = rados_connect(self.cluster) + if ret != 0: + raise make_ex(ret, "error connecting to the cluster") + self.state = "connected" + + def get_instance_id(self) -> int: + """ + Get a global id for current instance + """ + self.require_state("connected") + with nogil: + ret = rados_get_instance_id(self.cluster) + return ret; + + def get_cluster_stats(self) -> Dict[str, int]: + """ + Read usage info about the cluster + + This tells you total space, space used, space available, and number + of objects. These are not updated immediately when data is written, + they are eventually consistent. + + :returns: dict - contains the following keys: + + - ``kb`` (int) - total space + + - ``kb_used`` (int) - space used + + - ``kb_avail`` (int) - free space available + + - ``num_objects`` (int) - number of objects + + """ + cdef: + rados_cluster_stat_t stats + + with nogil: + ret = rados_cluster_stat(self.cluster, &stats) + + if ret < 0: + raise make_ex( + ret, "Rados.get_cluster_stats(%s): get_stats failed" % self.rados_id) + return {'kb': stats.kb, + 'kb_used': stats.kb_used, + 'kb_avail': stats.kb_avail, + 'num_objects': stats.num_objects} + + def pool_exists(self, pool_name: str) -> bool: + """ + Checks if a given pool exists. + + :param pool_name: name of the pool to check + + :raises: :class:`TypeError`, :class:`Error` + :returns: bool - whether the pool exists, false otherwise. + """ + self.require_state("connected") + + pool_name_raw = cstr(pool_name, 'pool_name') + cdef: + char *_pool_name = pool_name_raw + + with nogil: + ret = rados_pool_lookup(self.cluster, _pool_name) + if ret >= 0: + return True + elif ret == -errno.ENOENT: + return False + else: + raise make_ex(ret, "error looking up pool '%s'" % pool_name) + + def pool_lookup(self, pool_name: str) -> int: + """ + Returns a pool's ID based on its name. + + :param pool_name: name of the pool to look up + + :raises: :class:`TypeError`, :class:`Error` + :returns: int - pool ID, or None if it doesn't exist + """ + self.require_state("connected") + pool_name_raw = cstr(pool_name, 'pool_name') + cdef: + char *_pool_name = pool_name_raw + + with nogil: + ret = rados_pool_lookup(self.cluster, _pool_name) + if ret >= 0: + return int(ret) + elif ret == -errno.ENOENT: + return None + else: + raise make_ex(ret, "error looking up pool '%s'" % pool_name) + + def pool_reverse_lookup(self, pool_id: int): + """ + Returns a pool's name based on its ID. + + :param pool_id: ID of the pool to look up + + :raises: :class:`TypeError`, :class:`Error` + :returns: string - pool name, or None if it doesn't exist + """ + self.require_state("connected") + cdef: + int64_t _pool_id = pool_id + size_t size = 512 + char *name = NULL + + try: + while True: + name = <char *>realloc_chk(name, size) + with nogil: + ret = rados_pool_reverse_lookup(self.cluster, _pool_id, name, size) + if ret >= 0: + break + elif ret != -errno.ERANGE and size <= 4096: + size *= 2 + elif ret == -errno.ENOENT: + return None + elif ret < 0: + raise make_ex(ret, "error reverse looking up pool '%s'" % pool_id) + + return decode_cstr(name) + + finally: + free(name) + + def create_pool(self, pool_name: str, + crush_rule: Optional[int] = None, + auid: Optional[int] = None): + """ + Create a pool: + - with default settings: if crush_rule=None and auid=None + - with a specific CRUSH rule: crush_rule given + - with a specific auid: auid given + - with a specific CRUSH rule and auid: crush_rule and auid given + + :param pool_name: name of the pool to create + :param crush_rule: rule to use for placement in the new pool + :param auid: id of the owner of the new pool + + :raises: :class:`TypeError`, :class:`Error` + """ + self.require_state("connected") + + pool_name_raw = cstr(pool_name, 'pool_name') + cdef: + char *_pool_name = pool_name_raw + uint8_t _crush_rule + uint64_t _auid + + if crush_rule is None and auid is None: + with nogil: + ret = rados_pool_create(self.cluster, _pool_name) + elif crush_rule is not None and auid is None: + _crush_rule = crush_rule + with nogil: + ret = rados_pool_create_with_crush_rule(self.cluster, _pool_name, _crush_rule) + elif crush_rule is None and auid is not None: + _auid = auid + with nogil: + ret = rados_pool_create_with_auid(self.cluster, _pool_name, _auid) + else: + _crush_rule = crush_rule + _auid = auid + with nogil: + ret = rados_pool_create_with_all(self.cluster, _pool_name, _auid, _crush_rule) + if ret < 0: + raise make_ex(ret, "error creating pool '%s'" % pool_name) + + def get_pool_base_tier(self, pool_id: int) -> int: + """ + Get base pool + + :returns: base pool, or pool_id if tiering is not configured for the pool + """ + self.require_state("connected") + cdef: + int64_t base_tier = 0 + int64_t _pool_id = pool_id + + with nogil: + ret = rados_pool_get_base_tier(self.cluster, _pool_id, &base_tier) + if ret < 0: + raise make_ex(ret, "get_pool_base_tier(%d)" % pool_id) + return int(base_tier) + + def delete_pool(self, pool_name: str): + """ + Delete a pool and all data inside it. + + The pool is removed from the cluster immediately, + but the actual data is deleted in the background. + + :param pool_name: name of the pool to delete + + :raises: :class:`TypeError`, :class:`Error` + """ + self.require_state("connected") + + pool_name_raw = cstr(pool_name, 'pool_name') + cdef: + char *_pool_name = pool_name_raw + + with nogil: + ret = rados_pool_delete(self.cluster, _pool_name) + if ret < 0: + raise make_ex(ret, "error deleting pool '%s'" % pool_name) + + def get_inconsistent_pgs(self, pool_id: int) -> List[str]: + """ + List inconsistent placement groups in the given pool + + :param pool_id: ID of the pool in which PGs are listed + :returns: list - inconsistent placement groups + """ + self.require_state("connected") + cdef: + int64_t pool = pool_id + size_t size = 512 + char *pgs = NULL + + try: + while True: + pgs = <char *>realloc_chk(pgs, size); + with nogil: + ret = rados_inconsistent_pg_list(self.cluster, pool, + pgs, size) + if ret > <int>size: + size *= 2 + elif ret >= 0: + break + else: + raise make_ex(ret, "error calling inconsistent_pg_list") + return [pg for pg in decode_cstr(pgs[:ret]).split('\0') if pg] + finally: + free(pgs) + + def list_pools(self) -> List[str]: + """ + Gets a list of pool names. + + :returns: list - of pool names. + """ + self.require_state("connected") + cdef: + size_t size = 512 + char *c_names = NULL + + try: + while True: + c_names = <char *>realloc_chk(c_names, size) + with nogil: + ret = rados_pool_list(self.cluster, c_names, size) + if ret > <int>size: + size *= 2 + elif ret >= 0: + break + return [name for name in decode_cstr(c_names[:ret]).split('\0') + if name] + finally: + free(c_names) + + def get_fsid(self) -> str: + """ + Get the fsid of the cluster as a hexadecimal string. + + :raises: :class:`Error` + :returns: str - cluster fsid + """ + self.require_state("connected") + cdef: + char *ret_buf = NULL + size_t buf_len = 64 + + try: + while True: + ret_buf = <char *>realloc_chk(ret_buf, buf_len) + with nogil: + ret = rados_cluster_fsid(self.cluster, ret_buf, buf_len) + if ret == -errno.ERANGE: + buf_len = buf_len * 2 + elif ret < 0: + raise make_ex(ret, "error getting cluster fsid") + else: + break + return decode_cstr(ret_buf) + finally: + free(ret_buf) + + def open_ioctx(self, ioctx_name: str) -> Ioctx: + """ + Create an io context + + The io context allows you to perform operations within a particular + pool. + + :param ioctx_name: name of the pool + + :raises: :class:`TypeError`, :class:`Error` + :returns: Ioctx - Rados Ioctx object + """ + self.require_state("connected") + ioctx_name_raw = cstr(ioctx_name, 'ioctx_name') + cdef: + rados_ioctx_t ioctx + char *_ioctx_name = ioctx_name_raw + with nogil: + ret = rados_ioctx_create(self.cluster, _ioctx_name, &ioctx) + if ret < 0: + raise make_ex(ret, "error opening pool '%s'" % ioctx_name) + io = Ioctx(self, ioctx_name) + io.io = ioctx + return io + + def open_ioctx2(self, pool_id: int) -> Ioctx: + """ + Create an io context + + The io context allows you to perform operations within a particular + pool. + + :param pool_id: ID of the pool + + :raises: :class:`TypeError`, :class:`Error` + :returns: Ioctx - Rados Ioctx object + """ + self.require_state("connected") + cdef: + rados_ioctx_t ioctx + int64_t _pool_id = pool_id + with nogil: + ret = rados_ioctx_create2(self.cluster, _pool_id, &ioctx) + if ret < 0: + raise make_ex(ret, "error opening pool id '%s'" % pool_id) + io = Ioctx(self, str(pool_id)) + io.io = ioctx + return io + + def mon_command(self, + cmd: str, + inbuf: bytes, + timeout: int = 0, + target: Optional[Union[str, int]] = None) -> Tuple[int, bytes, str]: + """ + Send a command to the mon. + + mon_command[_target](cmd, inbuf, outbuf, outbuflen, outs, outslen) + + :param cmd: JSON formatted string. + :param inbuf: optional string. + :param timeout: This parameter is ignored. + :param target: name or rank of a specific mon. Optional + :return: (int ret, string outbuf, string outs) + + Example: + + >>> import json + >>> c = Rados(conffile='/etc/ceph/ceph.conf') + >>> c.connect() + >>> cmd = json.dumps({"prefix": "osd safe-to-destroy", "ids": ["2"], "format": "json"}) + >>> c.mon_command(cmd, b'') + """ + # NOTE(sileht): timeout is ignored because C API doesn't provide + # timeout argument, but we keep it for backward compat with old python binding + self.require_state("connected") + cmds = [cstr(cmd, 'cmd')] + + if isinstance(target, int): + # NOTE(sileht): looks weird but test_monmap_dump pass int + target = str(target) + + target = cstr(target, 'target', opt=True) + + cdef: + char *_target = opt_str(target) + char **_cmd = to_bytes_array(cmds) + size_t _cmdlen = len(cmds) + + char *_inbuf = inbuf + size_t _inbuf_len = len(inbuf) + + char *_outbuf + size_t _outbuf_len + char *_outs + size_t _outs_len + + try: + if target: + with nogil: + ret = rados_mon_command_target(self.cluster, _target, + <const char **>_cmd, _cmdlen, + <const char*>_inbuf, _inbuf_len, + &_outbuf, &_outbuf_len, + &_outs, &_outs_len) + else: + with nogil: + ret = rados_mon_command(self.cluster, + <const char **>_cmd, _cmdlen, + <const char*>_inbuf, _inbuf_len, + &_outbuf, &_outbuf_len, + &_outs, &_outs_len) + + my_outs = decode_cstr(_outs[:_outs_len]) + my_outbuf = _outbuf[:_outbuf_len] + if _outs_len: + rados_buffer_free(_outs) + if _outbuf_len: + rados_buffer_free(_outbuf) + return (ret, my_outbuf, my_outs) + finally: + free(_cmd) + + def osd_command(self, + osdid: int, + cmd: str, + inbuf: bytes, + timeout: int = 0) -> Tuple[int, bytes, str]: + """ + osd_command(osdid, cmd, inbuf, outbuf, outbuflen, outs, outslen) + + :return: (int ret, string outbuf, string outs) + """ + # NOTE(sileht): timeout is ignored because C API doesn't provide + # timeout argument, but we keep it for backward compat with old python binding + self.require_state("connected") + + cmds = [cstr(cmd, 'cmd')] + + cdef: + int _osdid = osdid + char **_cmd = to_bytes_array(cmds) + size_t _cmdlen = len(cmds) + + char *_inbuf = inbuf + size_t _inbuf_len = len(inbuf) + + char *_outbuf + size_t _outbuf_len + char *_outs + size_t _outs_len + + try: + with nogil: + ret = rados_osd_command(self.cluster, _osdid, + <const char **>_cmd, _cmdlen, + <const char*>_inbuf, _inbuf_len, + &_outbuf, &_outbuf_len, + &_outs, &_outs_len) + + my_outs = decode_cstr(_outs[:_outs_len]) + my_outbuf = _outbuf[:_outbuf_len] + if _outs_len: + rados_buffer_free(_outs) + if _outbuf_len: + rados_buffer_free(_outbuf) + return (ret, my_outbuf, my_outs) + finally: + free(_cmd) + + def mgr_command(self, + cmd: str, + inbuf: bytes, + timeout: int = 0, + target: Optional[str] = None) -> Tuple[int, str, bytes]: + """ + :return: (int ret, string outbuf, string outs) + """ + # NOTE(sileht): timeout is ignored because C API doesn't provide + # timeout argument, but we keep it for backward compat with old python binding + self.require_state("connected") + + cmds = [cstr(cmd, 'cmd')] + target = cstr(target, 'target', opt=True) + + cdef: + char *_target = opt_str(target) + + char **_cmd = to_bytes_array(cmds) + size_t _cmdlen = len(cmds) + + char *_inbuf = inbuf + size_t _inbuf_len = len(inbuf) + + char *_outbuf + size_t _outbuf_len + char *_outs + size_t _outs_len + + try: + if target is not None: + with nogil: + ret = rados_mgr_command_target(self.cluster, + <const char*>_target, + <const char **>_cmd, _cmdlen, + <const char*>_inbuf, _inbuf_len, + &_outbuf, &_outbuf_len, + &_outs, &_outs_len) + else: + with nogil: + ret = rados_mgr_command(self.cluster, + <const char **>_cmd, _cmdlen, + <const char*>_inbuf, _inbuf_len, + &_outbuf, &_outbuf_len, + &_outs, &_outs_len) + + my_outs = decode_cstr(_outs[:_outs_len]) + my_outbuf = _outbuf[:_outbuf_len] + if _outs_len: + rados_buffer_free(_outs) + if _outbuf_len: + rados_buffer_free(_outbuf) + return (ret, my_outbuf, my_outs) + finally: + free(_cmd) + + def pg_command(self, + pgid: str, + cmd: str, + inbuf: bytes, + timeout: int = 0) -> Tuple[int, bytes, str]: + """ + pg_command(pgid, cmd, inbuf, outbuf, outbuflen, outs, outslen) + + :return: (int ret, string outbuf, string outs) + """ + # NOTE(sileht): timeout is ignored because C API doesn't provide + # timeout argument, but we keep it for backward compat with old python binding + self.require_state("connected") + + pgid_raw = cstr(pgid, 'pgid') + cmds = [cstr(cmd, 'cmd')] + + cdef: + char *_pgid = pgid_raw + char **_cmd = to_bytes_array(cmds) + size_t _cmdlen = len(cmds) + + char *_inbuf = inbuf + size_t _inbuf_len = len(inbuf) + + char *_outbuf + size_t _outbuf_len + char *_outs + size_t _outs_len + + try: + with nogil: + ret = rados_pg_command(self.cluster, _pgid, + <const char **>_cmd, _cmdlen, + <const char *>_inbuf, _inbuf_len, + &_outbuf, &_outbuf_len, + &_outs, &_outs_len) + + my_outs = decode_cstr(_outs[:_outs_len]) + my_outbuf = _outbuf[:_outbuf_len] + if _outs_len: + rados_buffer_free(_outs) + if _outbuf_len: + rados_buffer_free(_outbuf) + return (ret, my_outbuf, my_outs) + finally: + free(_cmd) + + def wait_for_latest_osdmap(self) -> int: + self.require_state("connected") + with nogil: + ret = rados_wait_for_latest_osdmap(self.cluster) + return ret + + def blocklist_add(self, client_address: str, expire_seconds: int = 0): + """ + Blocklist a client from the OSDs + + :param client_address: client address + :param expire_seconds: number of seconds to blocklist + + :raises: :class:`Error` + """ + self.require_state("connected") + client_address_raw = cstr(client_address, 'client_address') + cdef: + uint32_t _expire_seconds = expire_seconds + char *_client_address = client_address_raw + + with nogil: + ret = rados_blocklist_add(self.cluster, _client_address, _expire_seconds) + if ret < 0: + raise make_ex(ret, "error blocklisting client '%s'" % client_address) + + def monitor_log(self, level: str, + callback: Optional[Callable[[object, str, str, str, int, int, int, str, str], None]] = None, + arg: Optional[object] = None): + if level not in MONITOR_LEVELS: + raise LogicError("invalid monitor level " + level) + if callback is not None and not callable(callback): + raise LogicError("callback must be a callable function or None") + + level_raw = cstr(level, 'level') + cdef char *_level = level_raw + + if callback is None: + with nogil: + r = rados_monitor_log(self.cluster, <const char*>_level, NULL, NULL) + self.monitor_callback = None + self.monitor_callback2 = None + return + + cb = (callback, arg) + cdef PyObject* _arg = <PyObject*>cb + with nogil: + r = rados_monitor_log(self.cluster, <const char*>_level, + <rados_log_callback_t>&__monitor_callback, _arg) + + if r: + raise make_ex(r, 'error calling rados_monitor_log') + # NOTE(sileht): Prevents the callback method from being garbage collected + self.monitor_callback = cb + self.monitor_callback2 = None + + def monitor_log2(self, level: str, + callback: Optional[Callable[[object, str, str, str, str, int, int, int, str, str], None]] = None, + arg: Optional[object] = None): + if level not in MONITOR_LEVELS: + raise LogicError("invalid monitor level " + level) + if callback is not None and not callable(callback): + raise LogicError("callback must be a callable function or None") + + level_raw = cstr(level, 'level') + cdef char *_level = level_raw + + if callback is None: + with nogil: + r = rados_monitor_log2(self.cluster, <const char*>_level, NULL, NULL) + self.monitor_callback = None + self.monitor_callback2 = None + return + + cb = (callback, arg) + cdef PyObject* _arg = <PyObject*>cb + with nogil: + r = rados_monitor_log2(self.cluster, <const char*>_level, + <rados_log_callback2_t>&__monitor_callback2, _arg) + + if r: + raise make_ex(r, 'error calling rados_monitor_log') + # NOTE(sileht): Prevents the callback method from being garbage collected + self.monitor_callback = None + self.monitor_callback2 = cb + + def service_daemon_register(self, service: str, daemon: str, metadata: Dict[str, str]): + """ + :param str service: service name (e.g. "rgw") + :param str daemon: daemon name (e.g. "gwfoo") + :param dict metadata: static metadata about the register daemon + (e.g., the version of Ceph, the kernel version.) + """ + service_raw = cstr(service, 'service') + daemon_raw = cstr(daemon, 'daemon') + metadata_dict = flatten_dict(metadata, 'metadata') + cdef: + char *_service = service_raw + char *_daemon = daemon_raw + char *_metadata = metadata_dict + + with nogil: + ret = rados_service_register(self.cluster, _service, _daemon, _metadata) + if ret != 0: + raise make_ex(ret, "error calling service_register()") + + def service_daemon_update(self, status: Dict[str, str]): + status_dict = flatten_dict(status, 'status') + cdef: + char *_status = status_dict + + with nogil: + ret = rados_service_update_status(self.cluster, _status) + if ret != 0: + raise make_ex(ret, "error calling service_daemon_update()") + + +cdef class OmapIterator(object): + """Omap iterator""" + + cdef public Ioctx ioctx + cdef rados_omap_iter_t ctx + + def __cinit__(self, Ioctx ioctx): + self.ioctx = ioctx + + def __iter__(self): + return self + + def __next__(self): + """ + Get the next key-value pair in the object + :returns: next rados.OmapItem + """ + cdef: + char *key_ = NULL + char *val_ = NULL + size_t len_ + + with nogil: + ret = rados_omap_get_next(self.ctx, &key_, &val_, &len_) + + if ret != 0: + raise make_ex(ret, "error iterating over the omap") + if key_ == NULL: + raise StopIteration() + key = decode_cstr(key_) + val = None + if val_ != NULL: + val = val_[:len_] + return (key, val) + + def __dealloc__(self): + with nogil: + rados_omap_get_end(self.ctx) + + +cdef class ObjectIterator(object): + """rados.Ioctx Object iterator""" + + cdef rados_list_ctx_t ctx + + cdef public object ioctx + + def __cinit__(self, Ioctx ioctx): + self.ioctx = ioctx + + with nogil: + ret = rados_nobjects_list_open(ioctx.io, &self.ctx) + if ret < 0: + raise make_ex(ret, "error iterating over the objects in ioctx '%s'" + % self.ioctx.name) + + def __iter__(self): + return self + + def __next__(self): + """ + Get the next object name and locator in the pool + + :raises: StopIteration + :returns: next rados.Ioctx Object + """ + cdef: + const char *key_ = NULL + const char *locator_ = NULL + const char *nspace_ = NULL + + with nogil: + ret = rados_nobjects_list_next(self.ctx, &key_, &locator_, &nspace_) + + if ret < 0: + raise StopIteration() + + key = decode_cstr(key_) + locator = decode_cstr(locator_) if locator_ != NULL else None + nspace = decode_cstr(nspace_) if nspace_ != NULL else None + return Object(self.ioctx, key, locator, nspace) + + def __dealloc__(self): + with nogil: + rados_nobjects_list_close(self.ctx) + + +cdef class XattrIterator(object): + """Extended attribute iterator""" + + cdef rados_xattrs_iter_t it + cdef char* _oid + + cdef public Ioctx ioctx + cdef public object oid + + def __cinit__(self, Ioctx ioctx, oid): + self.ioctx = ioctx + self.oid = cstr(oid, 'oid') + self._oid = self.oid + + with nogil: + ret = rados_getxattrs(ioctx.io, self._oid, &self.it) + if ret != 0: + raise make_ex(ret, "Failed to get rados xattrs for object %r" % oid) + + def __iter__(self): + return self + + def __next__(self): + """ + Get the next xattr on the object + + :raises: StopIteration + :returns: pair - of name and value of the next Xattr + """ + cdef: + const char *name_ = NULL + const char *val_ = NULL + size_t len_ = 0 + + with nogil: + ret = rados_getxattrs_next(self.it, &name_, &val_, &len_) + if ret != 0: + raise make_ex(ret, "error iterating over the extended attributes \ +in '%s'" % self.oid) + if name_ == NULL: + raise StopIteration() + name = decode_cstr(name_) + val = val_[:len_] + return (name, val) + + def __dealloc__(self): + with nogil: + rados_getxattrs_end(self.it) + + +cdef class SnapIterator(object): + """Snapshot iterator""" + + cdef public Ioctx ioctx + + cdef rados_snap_t *snaps + cdef int max_snap + cdef int cur_snap + + def __cinit__(self, Ioctx ioctx): + self.ioctx = ioctx + # We don't know how big a buffer we need until we've called the + # function. So use the exponential doubling strategy. + cdef int num_snaps = 10 + while True: + self.snaps = <rados_snap_t*>realloc_chk(self.snaps, + num_snaps * + sizeof(rados_snap_t)) + + with nogil: + ret = rados_ioctx_snap_list(ioctx.io, self.snaps, num_snaps) + if ret >= 0: + self.max_snap = ret + break + elif ret != -errno.ERANGE: + raise make_ex(ret, "error calling rados_snap_list for \ +ioctx '%s'" % self.ioctx.name) + num_snaps = num_snaps * 2 + self.cur_snap = 0 + + def __iter__(self): + return self + + def __next__(self): + """ + Get the next Snapshot + + :raises: :class:`Error`, StopIteration + :returns: Snap - next snapshot + """ + if self.cur_snap >= self.max_snap: + raise StopIteration + + cdef: + rados_snap_t snap_id = self.snaps[self.cur_snap] + int name_len = 10 + char *name = NULL + + try: + while True: + name = <char *>realloc_chk(name, name_len) + with nogil: + ret = rados_ioctx_snap_get_name(self.ioctx.io, snap_id, name, name_len) + if ret == 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, "rados_snap_get_name error") + else: + name_len = name_len * 2 + + snap = Snap(self.ioctx, decode_cstr(name[:name_len]).rstrip('\0'), snap_id) + self.cur_snap = self.cur_snap + 1 + return snap + finally: + free(name) + + +cdef class Snap(object): + """Snapshot object""" + cdef public Ioctx ioctx + cdef public object name + + # NOTE(sileht): old API was storing the ctypes object + # instead of the value .... + cdef public rados_snap_t snap_id + + def __cinit__(self, Ioctx ioctx, object name, rados_snap_t snap_id): + self.ioctx = ioctx + self.name = name + self.snap_id = snap_id + + def __str__(self): + return "rados.Snap(ioctx=%s,name=%s,snap_id=%d)" \ + % (str(self.ioctx), self.name, self.snap_id) + + def get_timestamp(self) -> float: + """ + Find when a snapshot in the current pool occurred + + :raises: :class:`Error` + :returns: datetime - the data and time the snapshot was created + """ + cdef time_t snap_time + + with nogil: + ret = rados_ioctx_snap_get_stamp(self.ioctx.io, self.snap_id, &snap_time) + if ret != 0: + raise make_ex(ret, "rados_ioctx_snap_get_stamp error") + return datetime.fromtimestamp(snap_time) + +# https://github.com/cython/cython/issues/1370 +unicode = str + +cdef class Completion(object): + """completion object""" + + cdef public: + Ioctx ioctx + object oncomplete + object onsafe + + cdef: + rados_callback_t complete_cb + rados_callback_t safe_cb + rados_completion_t rados_comp + PyObject* buf + + def __cinit__(self, Ioctx ioctx, object oncomplete, object onsafe): + self.oncomplete = oncomplete + self.onsafe = onsafe + self.ioctx = ioctx + + def is_safe(self) -> bool: + """ + Is an asynchronous operation safe? + + This does not imply that the safe callback has finished. + + :returns: True if the operation is safe + """ + return self.is_complete() + + def is_complete(self) -> bool: + """ + Has an asynchronous operation completed? + + This does not imply that the safe callback has finished. + + :returns: True if the operation is completed + """ + with nogil: + ret = rados_aio_is_complete(self.rados_comp) + return ret == 1 + + def wait_for_safe(self): + """ + Wait for an asynchronous operation to be marked safe + + wait_for_safe() is an alias of wait_for_complete() since Luminous + """ + self.wait_for_complete() + + def wait_for_complete(self): + """ + Wait for an asynchronous operation to complete + + This does not imply that the complete callback has finished. + """ + with nogil: + rados_aio_wait_for_complete(self.rados_comp) + + def wait_for_safe_and_cb(self): + """ + Wait for an asynchronous operation to be marked safe and for + the safe callback to have returned + """ + return self.wait_for_complete_and_cb() + + def wait_for_complete_and_cb(self): + """ + Wait for an asynchronous operation to complete and for the + complete callback to have returned + + :returns: whether the operation is completed + """ + with nogil: + ret = rados_aio_wait_for_complete_and_cb(self.rados_comp) + return ret + + def get_return_value(self) -> int: + """ + Get the return value of an asychronous operation + + The return value is set when the operation is complete or safe, + whichever comes first. + + :returns: int - return value of the operation + """ + with nogil: + ret = rados_aio_get_return_value(self.rados_comp) + return ret + + def __dealloc__(self): + """ + Release a completion + + Call this when you no longer need the completion. It may not be + freed immediately if the operation is not acked and committed. + """ + ref.Py_XDECREF(self.buf) + self.buf = NULL + if self.rados_comp != NULL: + with nogil: + rados_aio_release(self.rados_comp) + self.rados_comp = NULL + + def _complete(self): + self.oncomplete(self) + if self.onsafe: + self.onsafe(self) + self._cleanup() + + def _cleanup(self): + with self.ioctx.lock: + if self.oncomplete: + self.ioctx.complete_completions.remove(self) + if self.onsafe: + self.ioctx.safe_completions.remove(self) + + +class OpCtx(object): + def __enter__(self): + return self.create() + + def __exit__(self, type, msg, traceback): + self.release() + + +cdef class WriteOp(object): + cdef rados_write_op_t write_op + + def create(self): + with nogil: + self.write_op = rados_create_write_op() + return self + + def release(self): + with nogil: + rados_release_write_op(self.write_op) + + def new(self, exclusive: Optional[int] = None): + """ + Create the object. + """ + + cdef: + int _exclusive = exclusive + + with nogil: + rados_write_op_create(self.write_op, _exclusive, NULL) + + + def remove(self): + """ + Remove object. + """ + with nogil: + rados_write_op_remove(self.write_op) + + def set_flags(self, flags: int = LIBRADOS_OPERATION_NOFLAG): + """ + Set flags for the last operation added to this write_op. + :para flags: flags to apply to the last operation + """ + + cdef: + int _flags = flags + + with nogil: + rados_write_op_set_flags(self.write_op, _flags) + + def set_xattr(self, xattr_name: str, xattr_value: bytes): + """ + Set an extended attribute on an object. + :param xattr_name: name of the xattr + :param xattr_value: buffer to set xattr to + """ + xattr_name_raw = cstr(xattr_name, 'xattr_name') + cdef: + char *_xattr_name = xattr_name_raw + char *_xattr_value = xattr_value + size_t _xattr_value_len = len(xattr_value) + with nogil: + rados_write_op_setxattr(self.write_op, _xattr_name, _xattr_value, _xattr_value_len) + + def rm_xattr(self, xattr_name: str): + """ + Removes an extended attribute on from an object. + :param xattr_name: name of the xattr to remove + """ + xattr_name_raw = cstr(xattr_name, 'xattr_name') + cdef: + char *_xattr_name = xattr_name_raw + with nogil: + rados_write_op_rmxattr(self.write_op, _xattr_name) + + def append(self, to_write: bytes): + """ + Append data to an object synchronously + :param to_write: data to write + """ + + cdef: + char *_to_write = to_write + size_t length = len(to_write) + + with nogil: + rados_write_op_append(self.write_op, _to_write, length) + + def write_full(self, to_write: bytes): + """ + Write whole object, atomically replacing it. + :param to_write: data to write + """ + + cdef: + char *_to_write = to_write + size_t length = len(to_write) + + with nogil: + rados_write_op_write_full(self.write_op, _to_write, length) + + def write(self, to_write: bytes, offset: int = 0): + """ + Write to offset. + :param to_write: data to write + :param offset: byte offset in the object to begin writing at + """ + + cdef: + char *_to_write = to_write + size_t length = len(to_write) + uint64_t _offset = offset + + with nogil: + rados_write_op_write(self.write_op, _to_write, length, _offset) + + def assert_version(self, version: int): + """ + Check if object's version is the expected one. + :param version: expected version of the object + :param type: int + """ + cdef: + uint64_t _version = version + + with nogil: + rados_write_op_assert_version(self.write_op, _version) + + def zero(self, offset: int, length: int): + """ + Zero part of an object. + :param offset: byte offset in the object to begin writing at + :param offset: number of zero to write + """ + + cdef: + size_t _length = length + uint64_t _offset = offset + + with nogil: + rados_write_op_zero(self.write_op, _length, _offset) + + def truncate(self, offset: int): + """ + Truncate an object. + :param offset: byte offset in the object to begin truncating at + """ + + cdef: + uint64_t _offset = offset + + with nogil: + rados_write_op_truncate(self.write_op, _offset) + + def execute(self, cls: str, method: str, data: bytes): + """ + Execute an OSD class method on an object + + :param cls: name of the object class + :param method: name of the method + :param data: input data + """ + + cls_raw = cstr(cls, 'cls') + method_raw = cstr(method, 'method') + cdef: + char *_cls = cls_raw + char *_method = method_raw + char *_data = data + size_t _data_len = len(data) + + with nogil: + rados_write_op_exec(self.write_op, _cls, _method, _data, _data_len, NULL) + + def writesame(self, to_write: bytes, write_len: int, offset: int = 0): + """ + Write the same buffer multiple times + :param to_write: data to write + :param write_len: total number of bytes to write + :param offset: byte offset in the object to begin writing at + """ + cdef: + char *_to_write = to_write + size_t _data_len = len(to_write) + size_t _write_len = write_len + uint64_t _offset = offset + with nogil: + rados_write_op_writesame(self.write_op, _to_write, _data_len, _write_len, _offset) + + def cmpext(self, cmp_buf: bytes, offset: int = 0): + """ + Ensure that given object range (extent) satisfies comparison + :param cmp_buf: buffer containing bytes to be compared with object contents + :param offset: object byte offset at which to start the comparison + """ + cdef: + char *_cmp_buf = cmp_buf + size_t _cmp_buf_len = len(cmp_buf) + uint64_t _offset = offset + with nogil: + rados_write_op_cmpext(self.write_op, _cmp_buf, _cmp_buf_len, _offset, NULL) + + def omap_cmp(self, key: str, val: str, cmp_op: int = LIBRADOS_CMPXATTR_OP_EQ): + """ + Ensure that an omap key value satisfies comparison + :param key: omap key whose associated value is evaluated for comparison + :param val: value to compare with + :param cmp_op: comparison operator, one of LIBRADOS_CMPXATTR_OP_EQ (1), + LIBRADOS_CMPXATTR_OP_GT (3), or LIBRADOS_CMPXATTR_OP_LT (5). + """ + key_raw = cstr(key, 'key') + val_raw = cstr(val, 'val') + cdef: + char *_key = key_raw + char *_val = val_raw + size_t _val_len = len(val) + uint8_t _comparison_operator = cmp_op + with nogil: + rados_write_op_omap_cmp(self.write_op, _key, _comparison_operator, _val, _val_len, NULL) + +class WriteOpCtx(WriteOp, OpCtx): + """write operation context manager""" + + +cdef class ReadOp(object): + cdef rados_read_op_t read_op + + def create(self): + with nogil: + self.read_op = rados_create_read_op() + return self + + def release(self): + with nogil: + rados_release_read_op(self.read_op) + + def cmpext(self, cmp_buf: bytes, offset: int = 0): + """ + Ensure that given object range (extent) satisfies comparison + :param cmp_buf: buffer containing bytes to be compared with object contents + :param offset: object byte offset at which to start the comparison + """ + cdef: + char *_cmp_buf = cmp_buf + size_t _cmp_buf_len = len(cmp_buf) + uint64_t _offset = offset + with nogil: + rados_read_op_cmpext(self.read_op, _cmp_buf, _cmp_buf_len, _offset, NULL) + + def set_flags(self, flags: int = LIBRADOS_OPERATION_NOFLAG): + """ + Set flags for the last operation added to this read_op. + :para flags: flags to apply to the last operation + """ + + cdef: + int _flags = flags + + with nogil: + rados_read_op_set_flags(self.read_op, _flags) + + +class ReadOpCtx(ReadOp, OpCtx): + """read operation context manager""" + + +cdef void __watch_callback(void *_arg, int64_t _notify_id, uint64_t _cookie, + uint64_t _notifier_id, void *_data, + size_t _data_len) with gil: + """ + Watch callback + """ + cdef object watch = <object>_arg + data = None + if _data != NULL: + data = (<char *>_data)[:_data_len] + watch._callback(_notify_id, _notifier_id, _cookie, data) + +cdef void __watch_error_callback(void *_arg, uint64_t _cookie, + int _error) with gil: + """ + Watch error callback + """ + cdef object watch = <object>_arg + watch._error_callback(_cookie, _error) + + +cdef class Watch(object): + """watch object""" + + cdef: + object id + Ioctx ioctx + object oid + object callback + object error_callback + + def __cinit__(self, Ioctx ioctx, object oid, object callback, + object error_callback, object timeout): + self.id = 0 + self.ioctx = ioctx.dup() + self.oid = cstr(oid, 'oid') + self.callback = callback + self.error_callback = error_callback + + if timeout is None: + timeout = 0 + + cdef: + char *_oid = self.oid + uint64_t _cookie; + uint32_t _timeout = timeout; + void *_args = <PyObject*>self + + with nogil: + ret = rados_watch3(self.ioctx.io, _oid, &_cookie, + <rados_watchcb2_t>&__watch_callback, + <rados_watcherrcb_t>&__watch_error_callback, + _timeout, _args) + if ret < 0: + raise make_ex(ret, "watch error") + + self.id = int(_cookie); + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + self.close() + return False + + def __dealloc__(self): + self.ioctx.rados.require_state("connected") + self.close() + + def _callback(self, notify_id, notifier_id, watch_id, data): + replay = self.callback(notify_id, notifier_id, watch_id, data) + + cdef: + rados_ioctx_t _io = <rados_ioctx_t>self.ioctx.io + char *_obj = self.oid + int64_t _notify_id = notify_id + uint64_t _cookie = watch_id + char *_replay = NULL + int _replay_len = 0 + + if replay is not None: + replay = cstr(replay, 'replay') + _replay = replay + _replaylen = len(replay) + + with nogil: + rados_notify_ack(_io, _obj, _notify_id, _cookie, _replay, + _replaylen) + + def _error_callback(self, watch_id, error): + if self.error_callback is None: + return + self.error_callback(watch_id, error) + + def get_id(self) -> int: + return self.id + + def check(self): + """ + Check on watch validity. + + :raises: :class:`Error` + :returns: timedelta since last confirmed valid + """ + self.ioctx.require_ioctx_open() + + cdef: + uint64_t _cookie = self.id + + with nogil: + ret = rados_watch_check(self.ioctx.io, _cookie) + if ret < 0: + raise make_ex(ret, "check error") + + return timedelta(milliseconds=ret) + + def close(self): + """ + Unregister an interest in an object. + + :raises: :class:`Error` + """ + if self.id == 0: + return + + self.ioctx.require_ioctx_open() + + cdef: + uint64_t _cookie = self.id + + with nogil: + ret = rados_unwatch2(self.ioctx.io, _cookie) + if ret < 0 and ret != -errno.ENOENT: + raise make_ex(ret, "unwatch error") + self.id = 0 + + with nogil: + cluster = rados_ioctx_get_cluster(self.ioctx.io) + ret = rados_watch_flush(cluster); + if ret < 0: + raise make_ex(ret, "watch_flush error") + + self.ioctx.close() + + +cdef int __aio_complete_cb(rados_completion_t completion, void *args) with gil: + """ + Callback to oncomplete() for asynchronous operations + """ + cdef object cb = <object>args + cb._complete() + return 0 + +cdef class Ioctx(object): + """rados.Ioctx object""" + # NOTE(sileht): attributes declared in .pyd + + def __init__(self, rados, name): + self.rados = rados + self.name = name + self.state = "open" + + self.locator_key = "" + self.nspace = "" + self.lock = threading.Lock() + self.safe_completions = [] + self.complete_completions = [] + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + self.close() + return False + + def __dealloc__(self): + self.close() + + def __track_completion(self, completion_obj): + if completion_obj.oncomplete: + with self.lock: + self.complete_completions.append(completion_obj) + if completion_obj.onsafe: + with self.lock: + self.safe_completions.append(completion_obj) + + def __get_completion(self, + oncomplete: Callable[[Completion], None], + onsafe: Callable[[Completion], None]): + """ + Constructs a completion to use with asynchronous operations + + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + :param onsafe: what to do when the write is safe and complete on storage + on all replicas + + :raises: :class:`Error` + :returns: completion object + """ + + completion_obj = Completion(self, oncomplete, onsafe) + + cdef: + rados_callback_t complete_cb = NULL + rados_completion_t completion + PyObject* p_completion_obj= <PyObject*>completion_obj + + if oncomplete: + complete_cb = <rados_callback_t>&__aio_complete_cb + + with nogil: + ret = rados_aio_create_completion2(p_completion_obj, complete_cb, + &completion) + if ret < 0: + raise make_ex(ret, "error getting a completion") + + completion_obj.rados_comp = completion + return completion_obj + + def dup(self): + """ + Duplicate IoCtx + """ + + ioctx = self.rados.open_ioctx2(self.get_pool_id()) + ioctx.set_namespace(self.get_namespace()) + return ioctx + + def aio_stat(self, + object_name: str, + oncomplete: Callable[[Completion, Optional[int], Optional[time.struct_time]], None]) -> Completion: + """ + Asynchronously get object stats (size/mtime) + + oncomplete will be called with the returned size and mtime + as well as the completion: + + oncomplete(completion, size, mtime) + + :param object_name: the name of the object to get stats from + :param oncomplete: what to do when the stat is complete + + :raises: :class:`Error` + :returns: completion object + """ + + object_name_raw = cstr(object_name, 'object_name') + + cdef: + Completion completion + char *_object_name = object_name_raw + uint64_t psize + time_t pmtime + + def oncomplete_(completion_v): + cdef Completion _completion_v = completion_v + return_value = _completion_v.get_return_value() + if return_value >= 0: + return oncomplete(_completion_v, psize, time.localtime(pmtime)) + else: + return oncomplete(_completion_v, None, None) + + completion = self.__get_completion(oncomplete_, None) + self.__track_completion(completion) + with nogil: + ret = rados_aio_stat(self.io, _object_name, completion.rados_comp, + &psize, &pmtime) + + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error stating %s" % object_name) + return completion + + def aio_write(self, object_name: str, to_write: bytes, offset: int = 0, + oncomplete: Optional[Callable[[Completion], None]] = None, + onsafe: Optional[Callable[[Completion], None]] = None) -> Completion: + """ + Write data to an object asynchronously + + Queues the write and returns. + + :param object_name: name of the object + :param to_write: data to write + :param offset: byte offset in the object to begin writing at + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + :param onsafe: what to do when the write is safe and complete on storage + on all replicas + + :raises: :class:`Error` + :returns: completion object + """ + + object_name_raw = cstr(object_name, 'object_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + char* _to_write = to_write + size_t size = len(to_write) + uint64_t _offset = offset + + completion = self.__get_completion(oncomplete, onsafe) + self.__track_completion(completion) + with nogil: + ret = rados_aio_write(self.io, _object_name, completion.rados_comp, + _to_write, size, _offset) + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error writing object %s" % object_name) + return completion + + def aio_write_full(self, object_name: str, to_write: bytes, + oncomplete: Optional[Callable] = None, + onsafe: Optional[Callable] = None) -> Completion: + """ + Asynchronously write an entire object + + The object is filled with the provided data. If the object exists, + it is atomically truncated and then written. + Queues the write and returns. + + :param object_name: name of the object + :param to_write: data to write + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + :param onsafe: what to do when the write is safe and complete on storage + on all replicas + + :raises: :class:`Error` + :returns: completion object + """ + + object_name_raw = cstr(object_name, 'object_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + char* _to_write = to_write + size_t size = len(to_write) + + completion = self.__get_completion(oncomplete, onsafe) + self.__track_completion(completion) + with nogil: + ret = rados_aio_write_full(self.io, _object_name, + completion.rados_comp, + _to_write, size) + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error writing object %s" % object_name) + return completion + + def aio_writesame(self, object_name: str, to_write: bytes, + write_len: int, offset: int = 0, + oncomplete: Optional[Callable] = None) -> Completion: + """ + Asynchronously write the same buffer multiple times + + :param object_name: name of the object + :param to_write: data to write + :param write_len: total number of bytes to write + :param offset: byte offset in the object to begin writing at + :param oncomplete: what to do when the writesame is safe and + complete in memory on all replicas + :raises: :class:`Error` + :returns: completion object + """ + + object_name_raw = cstr(object_name, 'object_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + char* _to_write = to_write + size_t _data_len = len(to_write) + size_t _write_len = write_len + uint64_t _offset = offset + + completion = self.__get_completion(oncomplete, None) + self.__track_completion(completion) + with nogil: + ret = rados_aio_writesame(self.io, _object_name, completion.rados_comp, + _to_write, _data_len, _write_len, _offset) + + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error writing object %s" % object_name) + return completion + + def aio_append(self, object_name: str, to_append: bytes, + oncomplete: Optional[Callable] = None, + onsafe: Optional[Callable] = None) -> Completion: + """ + Asynchronously append data to an object + + Queues the write and returns. + + :param object_name: name of the object + :param to_append: data to append + :param offset: byte offset in the object to begin writing at + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + :param onsafe: what to do when the write is safe and complete on storage + on all replicas + + :raises: :class:`Error` + :returns: completion object + """ + object_name_raw = cstr(object_name, 'object_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + char* _to_append = to_append + size_t size = len(to_append) + + completion = self.__get_completion(oncomplete, onsafe) + self.__track_completion(completion) + with nogil: + ret = rados_aio_append(self.io, _object_name, + completion.rados_comp, + _to_append, size) + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error appending object %s" % object_name) + return completion + + def aio_flush(self): + """ + Block until all pending writes in an io context are safe + + :raises: :class:`Error` + """ + with nogil: + ret = rados_aio_flush(self.io) + if ret < 0: + raise make_ex(ret, "error flushing") + + def aio_cmpext(self, object_name: str, cmp_buf: bytes, offset: int = 0, + oncomplete: Optional[Callable] = None) -> Completion: + """ + Asynchronously compare an on-disk object range with a buffer + :param object_name: the name of the object + :param cmp_buf: buffer containing bytes to be compared with object contents + :param offset: object byte offset at which to start the comparison + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + + :raises: :class:`TypeError` + returns: 0 - on success, negative error code on failure, + (-MAX_ERRNO - mismatch_off) on mismatch + """ + object_name_raw = cstr(object_name, 'object_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + char* _cmp_buf = cmp_buf + size_t _cmp_buf_len = len(cmp_buf) + uint64_t _offset = offset + + completion = self.__get_completion(oncomplete, None) + self.__track_completion(completion) + + with nogil: + ret = rados_aio_cmpext(self.io, _object_name, completion.rados_comp, + _cmp_buf, _cmp_buf_len, _offset) + + if ret < 0: + completion._cleanup() + raise make_ex(ret, "failed to compare %s" % object_name) + return completion + + def aio_rmxattr(self, object_name: str, xattr_name: str, + oncomplete: Optional[Callable] = None) -> Completion: + """ + Asynchronously delete an extended attribute from an object + + :param object_name: the name of the object to remove xattr from + :param xattr_name: which extended attribute to remove + :param oncomplete: what to do when the rmxattr completes + + :raises: :class:`Error` + :returns: completion object + """ + object_name_raw = cstr(object_name, 'object_name') + xattr_name_raw = cstr(xattr_name , 'xattr_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + char* _xattr_name = xattr_name_raw + + completion = self.__get_completion(oncomplete, None) + self.__track_completion(completion) + with nogil: + ret = rados_aio_rmxattr(self.io, _object_name, + completion.rados_comp, _xattr_name) + + if ret < 0: + completion._cleanup() + raise make_ex(ret, "Failed to remove xattr %r" % xattr_name) + return completion + + def aio_read(self, object_name: str, length: int, offset: int, + oncomplete: Optional[Callable] = None) -> Completion: + """ + Asynchronously read data from an object + + oncomplete will be called with the returned read value as + well as the completion: + + oncomplete(completion, data_read) + + :param object_name: name of the object to read from + :param length: the number of bytes to read + :param offset: byte offset in the object to begin reading from + :param oncomplete: what to do when the read is complete + + :raises: :class:`Error` + :returns: completion object + """ + + object_name_raw = cstr(object_name, 'object_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + uint64_t _offset = offset + + char *ref_buf + size_t _length = length + + def oncomplete_(completion_v): + cdef Completion _completion_v = completion_v + return_value = _completion_v.get_return_value() + if return_value > 0 and return_value != length: + _PyBytes_Resize(&_completion_v.buf, return_value) + return oncomplete(_completion_v, <object>_completion_v.buf if return_value >= 0 else None) + + completion = self.__get_completion(oncomplete_, None) + completion.buf = PyBytes_FromStringAndSize(NULL, length) + ret_buf = PyBytes_AsString(completion.buf) + self.__track_completion(completion) + with nogil: + ret = rados_aio_read(self.io, _object_name, completion.rados_comp, + ret_buf, _length, _offset) + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error reading %s" % object_name) + return completion + + def aio_execute(self, object_name: str, cls: str, method: str, + data: bytes, length: int = 8192, + oncomplete: Optional[Callable[[Completion, bytes], None]] = None, + onsafe: Optional[Callable[[Completion, bytes], None]] = None) -> Completion: + """ + Asynchronously execute an OSD class method on an object. + + oncomplete and onsafe will be called with the data returned from + the plugin as well as the completion: + + oncomplete(completion, data) + onsafe(completion, data) + + :param object_name: name of the object + :param cls: name of the object class + :param method: name of the method + :param data: input data + :param length: size of output buffer in bytes (default=8192) + :param oncomplete: what to do when the execution is complete + :param onsafe: what to do when the execution is safe and complete + + :raises: :class:`Error` + :returns: completion object + """ + + object_name_raw = cstr(object_name, 'object_name') + cls_raw = cstr(cls, 'cls') + method_raw = cstr(method, 'method') + cdef: + Completion completion + char *_object_name = object_name_raw + char *_cls = cls_raw + char *_method = method_raw + char *_data = data + size_t _data_len = len(data) + + char *ref_buf + size_t _length = length + + def oncomplete_(completion_v): + cdef Completion _completion_v = completion_v + return_value = _completion_v.get_return_value() + if return_value > 0 and return_value != length: + _PyBytes_Resize(&_completion_v.buf, return_value) + return oncomplete(_completion_v, <object>_completion_v.buf if return_value >= 0 else None) + + def onsafe_(completion_v): + cdef Completion _completion_v = completion_v + return_value = _completion_v.get_return_value() + return onsafe(_completion_v, <object>_completion_v.buf if return_value >= 0 else None) + + completion = self.__get_completion(oncomplete_ if oncomplete else None, onsafe_ if onsafe else None) + completion.buf = PyBytes_FromStringAndSize(NULL, length) + ret_buf = PyBytes_AsString(completion.buf) + self.__track_completion(completion) + with nogil: + ret = rados_aio_exec(self.io, _object_name, completion.rados_comp, + _cls, _method, _data, _data_len, ret_buf, _length) + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error executing %s::%s on %s" % (cls, method, object_name)) + return completion + + def aio_setxattr(self, object_name: str, xattr_name: str, xattr_value: bytes, + oncomplete: Optional[Callable] = None) -> Completion: + """ + Asynchronously set an extended attribute on an object + + :param object_name: the name of the object to set xattr to + :param xattr_name: which extended attribute to set + :param xattr_value: the value of the extended attribute + :param oncomplete: what to do when the setxttr completes + + :raises: :class:`Error` + :returns: completion object + """ + object_name_raw = cstr(object_name, 'object_name') + xattr_name_raw = cstr(xattr_name , 'xattr_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + char* _xattr_name = xattr_name_raw + char* _xattr_value = xattr_value + size_t xattr_value_len = len(xattr_value) + + completion = self.__get_completion(oncomplete, None) + self.__track_completion(completion) + with nogil: + ret = rados_aio_setxattr(self.io, _object_name, + completion.rados_comp, + _xattr_name, _xattr_value, xattr_value_len) + + if ret < 0: + completion._cleanup() + raise make_ex(ret, "Failed to set xattr %r" % xattr_name) + return completion + + def aio_remove(self, object_name: str, + oncomplete: Optional[Callable] = None, + onsafe: Optional[Callable] = None) -> Completion: + """ + Asynchronously remove an object + + :param object_name: name of the object to remove + :param oncomplete: what to do when the remove is safe and complete in memory + on all replicas + :param onsafe: what to do when the remove is safe and complete on storage + on all replicas + + :raises: :class:`Error` + :returns: completion object + """ + object_name_raw = cstr(object_name, 'object_name') + + cdef: + Completion completion + char* _object_name = object_name_raw + + completion = self.__get_completion(oncomplete, onsafe) + self.__track_completion(completion) + with nogil: + ret = rados_aio_remove(self.io, _object_name, + completion.rados_comp) + if ret < 0: + completion._cleanup() + raise make_ex(ret, "error removing %s" % object_name) + return completion + + def require_ioctx_open(self): + """ + Checks if the rados.Ioctx object state is 'open' + + :raises: IoctxStateError + """ + if self.state != "open": + raise IoctxStateError("The pool is %s" % self.state) + + def set_locator_key(self, loc_key: str): + """ + Set the key for mapping objects to pgs within an io context. + + The key is used instead of the object name to determine which + placement groups an object is put in. This affects all subsequent + operations of the io context - until a different locator key is + set, all objects in this io context will be placed in the same pg. + + :param loc_key: the key to use as the object locator, or NULL to discard + any previously set key + + :raises: :class:`TypeError` + """ + self.require_ioctx_open() + cloc_key = cstr(loc_key, 'loc_key') + cdef char *_loc_key = cloc_key + with nogil: + rados_ioctx_locator_set_key(self.io, _loc_key) + self.locator_key = loc_key + + def get_locator_key(self) -> str: + """ + Get the locator_key of context + + :returns: locator_key + """ + return self.locator_key + + def set_read(self, snap_id: int): + """ + Set the snapshot for reading objects. + + To stop to read from snapshot, use set_read(LIBRADOS_SNAP_HEAD) + + :param snap_id: the snapshot Id + + :raises: :class:`TypeError` + """ + self.require_ioctx_open() + cdef rados_snap_t _snap_id = snap_id + with nogil: + rados_ioctx_snap_set_read(self.io, _snap_id) + + def set_namespace(self, nspace: str): + """ + Set the namespace for objects within an io context. + + The namespace in addition to the object name fully identifies + an object. This affects all subsequent operations of the io context + - until a different namespace is set, all objects in this io context + will be placed in the same namespace. + + :param nspace: the namespace to use, or None/"" for the default namespace + + :raises: :class:`TypeError` + """ + self.require_ioctx_open() + if nspace is None: + nspace = "" + cnspace = cstr(nspace, 'nspace') + cdef char *_nspace = cnspace + with nogil: + rados_ioctx_set_namespace(self.io, _nspace) + self.nspace = nspace + + def get_namespace(self) -> str: + """ + Get the namespace of context + + :returns: namespace + """ + return self.nspace + + def close(self): + """ + Close a rados.Ioctx object. + + This just tells librados that you no longer need to use the io context. + It may not be freed immediately if there are pending asynchronous + requests on it, but you should not use an io context again after + calling this function on it. + """ + if self.state == "open": + self.require_ioctx_open() + with nogil: + rados_ioctx_destroy(self.io) + self.state = "closed" + + + def write(self, key: str, data: bytes, offset: int = 0): + """ + Write data to an object synchronously + + :param key: name of the object + :param data: data to write + :param offset: byte offset in the object to begin writing at + + :raises: :class:`TypeError` + :raises: :class:`LogicError` + :returns: int - 0 on success + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + char *_data = data + size_t length = len(data) + uint64_t _offset = offset + + with nogil: + ret = rados_write(self.io, _key, _data, length, _offset) + if ret == 0: + return ret + elif ret < 0: + raise make_ex(ret, "Ioctx.write(%s): failed to write %s" + % (self.name, key)) + else: + raise LogicError("Ioctx.write(%s): rados_write \ +returned %d, but should return zero on success." % (self.name, ret)) + + def write_full(self, key: str, data: bytes): + """ + Write an entire object synchronously. + + The object is filled with the provided data. If the object exists, + it is atomically truncated and then written. + + :param key: name of the object + :param data: data to write + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: int - 0 on success + """ + self.require_ioctx_open() + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + char *_data = data + size_t length = len(data) + + with nogil: + ret = rados_write_full(self.io, _key, _data, length) + if ret == 0: + return ret + elif ret < 0: + raise make_ex(ret, "Ioctx.write_full(%s): failed to write %s" + % (self.name, key)) + else: + raise LogicError("Ioctx.write_full(%s): rados_write_full \ +returned %d, but should return zero on success." % (self.name, ret)) + + def writesame(self, key: str, data: bytes, write_len: int, offset: int = 0): + """ + Write the same buffer multiple times + :param key: name of the object + :param data: data to write + :param write_len: total number of bytes to write + :param offset: byte offset in the object to begin writing at + + :raises: :class:`TypeError` + :raises: :class:`LogicError` + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + char *_data = data + size_t _data_len = len(data) + size_t _write_len = write_len + uint64_t _offset = offset + + with nogil: + ret = rados_writesame(self.io, _key, _data, _data_len, _write_len, _offset) + if ret < 0: + raise make_ex(ret, "Ioctx.writesame(%s): failed to write %s" + % (self.name, key)) + assert(ret == 0) + + def append(self, key: str, data: bytes): + """ + Append data to an object synchronously + + :param key: name of the object + :param data: data to write + + :raises: :class:`TypeError` + :raises: :class:`LogicError` + :returns: int - 0 on success + """ + self.require_ioctx_open() + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + char *_data = data + size_t length = len(data) + + with nogil: + ret = rados_append(self.io, _key, _data, length) + if ret == 0: + return ret + elif ret < 0: + raise make_ex(ret, "Ioctx.append(%s): failed to append %s" + % (self.name, key)) + else: + raise LogicError("Ioctx.append(%s): rados_append \ +returned %d, but should return zero on success." % (self.name, ret)) + + def read(self, key: str, length: int = 8192, offset: int = 0) -> bytes: + """ + Read data from an object synchronously + + :param key: name of the object + :param length: the number of bytes to read (default=8192) + :param offset: byte offset in the object to begin reading at + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: str - data read from object + """ + self.require_ioctx_open() + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + char *ret_buf + uint64_t _offset = offset + size_t _length = length + PyObject* ret_s = NULL + + ret_s = PyBytes_FromStringAndSize(NULL, length) + try: + ret_buf = PyBytes_AsString(ret_s) + with nogil: + ret = rados_read(self.io, _key, ret_buf, _length, _offset) + if ret < 0: + raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key)) + + if ret != length: + _PyBytes_Resize(&ret_s, ret) + + return <object>ret_s + finally: + # We DECREF unconditionally: the cast to object above will have + # INCREFed if necessary. This also takes care of exceptions, + # including if _PyString_Resize fails (that will free the string + # itself and set ret_s to NULL, hence XDECREF). + ref.Py_XDECREF(ret_s) + + def execute(self, key: str, cls: str, method: str, data: bytes, length: int = 8192) -> Tuple[int, object]: + """ + Execute an OSD class method on an object. + + :param key: name of the object + :param cls: name of the object class + :param method: name of the method + :param data: input data + :param length: size of output buffer in bytes (default=8192) + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: (ret, method output) + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + cls_raw = cstr(cls, 'cls') + method_raw = cstr(method, 'method') + cdef: + char *_key = key_raw + char *_cls = cls_raw + char *_method = method_raw + char *_data = data + size_t _data_len = len(data) + + char *ref_buf + size_t _length = length + PyObject* ret_s = NULL + + ret_s = PyBytes_FromStringAndSize(NULL, length) + try: + ret_buf = PyBytes_AsString(ret_s) + with nogil: + ret = rados_exec(self.io, _key, _cls, _method, _data, + _data_len, ret_buf, _length) + if ret < 0: + raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key)) + + if ret != length: + _PyBytes_Resize(&ret_s, ret) + + return ret, <object>ret_s + finally: + # We DECREF unconditionally: the cast to object above will have + # INCREFed if necessary. This also takes care of exceptions, + # including if _PyString_Resize fails (that will free the string + # itself and set ret_s to NULL, hence XDECREF). + ref.Py_XDECREF(ret_s) + + def get_stats(self) -> Dict[str, int]: + """ + Get pool usage statistics + + :returns: dict - contains the following keys: + + - ``num_bytes`` (int) - size of pool in bytes + + - ``num_kb`` (int) - size of pool in kbytes + + - ``num_objects`` (int) - number of objects in the pool + + - ``num_object_clones`` (int) - number of object clones + + - ``num_object_copies`` (int) - number of object copies + + - ``num_objects_missing_on_primary`` (int) - number of objets + missing on primary + + - ``num_objects_unfound`` (int) - number of unfound objects + + - ``num_objects_degraded`` (int) - number of degraded objects + + - ``num_rd`` (int) - bytes read + + - ``num_rd_kb`` (int) - kbytes read + + - ``num_wr`` (int) - bytes written + + - ``num_wr_kb`` (int) - kbytes written + """ + self.require_ioctx_open() + cdef rados_pool_stat_t stats + with nogil: + ret = rados_ioctx_pool_stat(self.io, &stats) + if ret < 0: + raise make_ex(ret, "Ioctx.get_stats(%s): get_stats failed" % self.name) + return {'num_bytes': stats.num_bytes, + 'num_kb': stats.num_kb, + 'num_objects': stats.num_objects, + 'num_object_clones': stats.num_object_clones, + 'num_object_copies': stats.num_object_copies, + "num_objects_missing_on_primary": stats.num_objects_missing_on_primary, + "num_objects_unfound": stats.num_objects_unfound, + "num_objects_degraded": stats.num_objects_degraded, + "num_rd": stats.num_rd, + "num_rd_kb": stats.num_rd_kb, + "num_wr": stats.num_wr, + "num_wr_kb": stats.num_wr_kb} + + def remove_object(self, key: str) -> bool: + """ + Delete an object + + This does not delete any snapshots of the object. + + :param key: the name of the object to delete + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: bool - True on success + """ + self.require_ioctx_open() + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + + with nogil: + ret = rados_remove(self.io, _key) + if ret < 0: + raise make_ex(ret, "Failed to remove '%s'" % key) + return True + + def trunc(self, key: str, size: int): + """ + Resize an object + + If this enlarges the object, the new area is logically filled with + zeroes. If this shrinks the object, the excess data is removed. + + :param key: the name of the object to resize + :param size: the new size of the object in bytes + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: int - 0 on success, otherwise raises error + """ + + self.require_ioctx_open() + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + uint64_t _size = size + + with nogil: + ret = rados_trunc(self.io, _key, _size) + if ret < 0: + raise make_ex(ret, "Ioctx.trunc(%s): failed to truncate %s" % (self.name, key)) + return ret + + def cmpext(self, key: str, cmp_buf: bytes, offset: int = 0) -> int: + ''' + Compare an on-disk object range with a buffer + :param key: the name of the object + :param cmp_buf: buffer containing bytes to be compared with object contents + :param offset: object byte offset at which to start the comparison + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: 0 - on success, negative error code on failure, + (-MAX_ERRNO - mismatch_off) on mismatch + ''' + self.require_ioctx_open() + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + char *_cmp_buf = cmp_buf + size_t _cmp_buf_len = len(cmp_buf) + uint64_t _offset = offset + with nogil: + ret = rados_cmpext(self.io, _key, _cmp_buf, _cmp_buf_len, _offset) + assert ret < -MAX_ERRNO or ret == 0, "Ioctx.cmpext(%s): failed to compare %s" % (self.name, key) + return ret + + def stat(self, key: str) -> Tuple[int, time.struct_time]: + """ + Get object stats (size/mtime) + + :param key: the name of the object to get stats from + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: (size,timestamp) + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + cdef: + char *_key = key_raw + uint64_t psize + time_t pmtime + + with nogil: + ret = rados_stat(self.io, _key, &psize, &pmtime) + if ret < 0: + raise make_ex(ret, "Failed to stat %r" % key) + return psize, time.localtime(pmtime) + + def get_xattr(self, key: str, xattr_name: str) -> bytes: + """ + Get the value of an extended attribute on an object. + + :param key: the name of the object to get xattr from + :param xattr_name: which extended attribute to read + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: str - value of the xattr + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + xattr_name_raw = cstr(xattr_name, 'xattr_name') + cdef: + char *_key = key_raw + char *_xattr_name = xattr_name_raw + size_t ret_length = 4096 + char *ret_buf = NULL + + try: + while ret_length < 4096 * 1024 * 1024: + ret_buf = <char *>realloc_chk(ret_buf, ret_length) + with nogil: + ret = rados_getxattr(self.io, _key, _xattr_name, ret_buf, ret_length) + if ret == -errno.ERANGE: + ret_length *= 2 + elif ret < 0: + raise make_ex(ret, "Failed to get xattr %r" % xattr_name) + else: + break + return ret_buf[:ret] + finally: + free(ret_buf) + + def get_xattrs(self, oid: str) -> XattrIterator: + """ + Start iterating over xattrs on an object. + + :param oid: the name of the object to get xattrs from + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: XattrIterator + """ + self.require_ioctx_open() + return XattrIterator(self, oid) + + def set_xattr(self, key: str, xattr_name: str, xattr_value: bytes) -> bool: + """ + Set an extended attribute on an object. + + :param key: the name of the object to set xattr to + :param xattr_name: which extended attribute to set + :param xattr_value: the value of the extended attribute + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: bool - True on success, otherwise raise an error + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + xattr_name_raw = cstr(xattr_name, 'xattr_name') + cdef: + char *_key = key_raw + char *_xattr_name = xattr_name_raw + char *_xattr_value = xattr_value + size_t _xattr_value_len = len(xattr_value) + + with nogil: + ret = rados_setxattr(self.io, _key, _xattr_name, + _xattr_value, _xattr_value_len) + if ret < 0: + raise make_ex(ret, "Failed to set xattr %r" % xattr_name) + return True + + def rm_xattr(self, key: str, xattr_name: str) -> bool: + """ + Removes an extended attribute on from an object. + + :param key: the name of the object to remove xattr from + :param xattr_name: which extended attribute to remove + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: bool - True on success, otherwise raise an error + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + xattr_name_raw = cstr(xattr_name, 'xattr_name') + cdef: + char *_key = key_raw + char *_xattr_name = xattr_name_raw + + with nogil: + ret = rados_rmxattr(self.io, _key, _xattr_name) + if ret < 0: + raise make_ex(ret, "Failed to delete key %r xattr %r" % + (key, xattr_name)) + return True + + def notify(self, obj: str, msg: str = '', timeout_ms: int = 5000) -> bool: + """ + Send a rados notification to an object. + + :param obj: the name of the object to notify + :param msg: optional message to send in the notification + :param timeout_ms: notify timeout (in ms) + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: bool - True on success, otherwise raise an error + """ + self.require_ioctx_open() + + msglen = len(msg) + obj_raw = cstr(obj, 'obj') + msg_raw = cstr(msg, 'msg') + cdef: + char *_obj = obj_raw + char *_msg = msg_raw + int _msglen = msglen + uint64_t _timeout_ms = timeout_ms + + with nogil: + ret = rados_notify2(self.io, _obj, _msg, _msglen, _timeout_ms, + NULL, NULL) + if ret < 0: + raise make_ex(ret, "Failed to notify %r" % (obj)) + return True + + def aio_notify(self, obj: str, + oncomplete: Callable[[Completion, int, Optional[List], Optional[List]], None], + msg: str = '', timeout_ms: int = 5000) -> Completion: + """ + Asynchronously send a rados notification to an object + """ + self.require_ioctx_open() + + msglen = len(msg) + obj_raw = cstr(obj, 'obj') + msg_raw = cstr(msg, 'msg') + + cdef: + Completion completion + char *_obj = obj_raw + char *_msg = msg_raw + int _msglen = msglen + uint64_t _timeout_ms = timeout_ms + char *reply + size_t replylen = 0 + + def oncomplete_(completion_v): + cdef: + Completion _completion_v = completion_v + notify_ack_t *acks = NULL + notify_timeout_t *timeouts = NULL + size_t nr_acks + size_t nr_timeouts + return_value = _completion_v.get_return_value() + if return_value == 0: + return_value = rados_decode_notify_response(reply, replylen, &acks, &nr_acks, &timeouts, &nr_timeouts) + rados_buffer_free(reply) + if return_value == 0: + ack_list = [(ack.notifier_id, ack.cookie, '' if not ack.payload_len \ + else ack.payload[:ack.payload_len]) for ack in acks[:nr_acks]] + timeout_list = [(timeout.notifier_id, timeout.cookie) for timeout in timeouts[:nr_timeouts]] + rados_free_notify_response(acks, nr_acks, timeouts) + return oncomplete(_completion_v, 0, ack_list, timeout_list) + else: + return oncomplete(_completion_v, return_value, None, None) + + completion = self.__get_completion(oncomplete_, None) + self.__track_completion(completion) + with nogil: + ret = rados_aio_notify(self.io, _obj, completion.rados_comp, + _msg, _msglen, _timeout_ms, &reply, &replylen) + if ret < 0: + completion._cleanup() + raise make_ex(ret, "aio_notify error: %s" % obj) + return completion + + def watch(self, obj: str, + callback: Callable[[int, str, int, bytes], None], + error_callback: Optional[Callable[[int], None]] = None, + timeout: Optional[int] = None) -> Watch: + """ + Register an interest in an object. + + :param obj: the name of the object to notify + :param callback: what to do when a notify is received on this object + :param error_callback: what to do when the watch session encounters an error + :param timeout: how many seconds the connection will keep after disconnection + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: watch_id - internal id assigned to this watch + """ + self.require_ioctx_open() + + return Watch(self, obj, callback, error_callback, timeout) + + def list_objects(self) -> ObjectIterator: + """ + Get ObjectIterator on rados.Ioctx object. + + :returns: ObjectIterator + """ + self.require_ioctx_open() + return ObjectIterator(self) + + def list_snaps(self) -> SnapIterator: + """ + Get SnapIterator on rados.Ioctx object. + + :returns: SnapIterator + """ + self.require_ioctx_open() + return SnapIterator(self) + + def get_pool_id(self) -> int: + """ + Get pool id + + :returns: int - pool id + """ + with nogil: + ret = rados_ioctx_get_id(self.io) + return ret; + + def get_pool_name(self) -> str: + """ + Get pool name + + :returns: str - pool name + """ + cdef: + int name_len = 10 + char *name = NULL + + try: + while True: + name = <char *>realloc_chk(name, name_len) + with nogil: + ret = rados_ioctx_get_pool_name(self.io, name, name_len) + if ret > 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, "get pool name error") + else: + name_len = name_len * 2 + + return decode_cstr(name) + finally: + free(name) + + + def create_snap(self, snap_name: str): + """ + Create a pool-wide snapshot + + :param snap_name: the name of the snapshot + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + snap_name_raw = cstr(snap_name, 'snap_name') + cdef char *_snap_name = snap_name_raw + + with nogil: + ret = rados_ioctx_snap_create(self.io, _snap_name) + if ret != 0: + raise make_ex(ret, "Failed to create snap %s" % snap_name) + + def remove_snap(self, snap_name: str): + """ + Removes a pool-wide snapshot + + :param snap_name: the name of the snapshot + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + snap_name_raw = cstr(snap_name, 'snap_name') + cdef char *_snap_name = snap_name_raw + + with nogil: + ret = rados_ioctx_snap_remove(self.io, _snap_name) + if ret != 0: + raise make_ex(ret, "Failed to remove snap %s" % snap_name) + + def lookup_snap(self, snap_name: str) -> Snap: + """ + Get the id of a pool snapshot + + :param snap_name: the name of the snapshot to lookop + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: Snap - on success + """ + self.require_ioctx_open() + csnap_name = cstr(snap_name, 'snap_name') + cdef: + char *_snap_name = csnap_name + rados_snap_t snap_id + + with nogil: + ret = rados_ioctx_snap_lookup(self.io, _snap_name, &snap_id) + if ret != 0: + raise make_ex(ret, "Failed to lookup snap %s" % snap_name) + return Snap(self, snap_name, int(snap_id)) + + def snap_rollback(self, oid: str, snap_name: str): + """ + Rollback an object to a snapshot + + :param oid: the name of the object + :param snap_name: the name of the snapshot + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + oid_raw = cstr(oid, 'oid') + snap_name_raw = cstr(snap_name, 'snap_name') + cdef: + char *_oid = oid_raw + char *_snap_name = snap_name_raw + + with nogil: + ret = rados_ioctx_snap_rollback(self.io, _oid, _snap_name) + if ret != 0: + raise make_ex(ret, "Failed to rollback %s" % oid) + + def create_self_managed_snap(self): + """ + Creates a self-managed snapshot + + :returns: snap id on success + + :raises: :class:`Error` + """ + self.require_ioctx_open() + cdef: + rados_snap_t _snap_id + with nogil: + ret = rados_ioctx_selfmanaged_snap_create(self.io, &_snap_id) + if ret != 0: + raise make_ex(ret, "Failed to create self-managed snapshot") + return int(_snap_id) + + def remove_self_managed_snap(self, snap_id: int): + """ + Removes a self-managed snapshot + + :param snap_id: the name of the snapshot + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + cdef: + rados_snap_t _snap_id = snap_id + with nogil: + ret = rados_ioctx_selfmanaged_snap_remove(self.io, _snap_id) + if ret != 0: + raise make_ex(ret, "Failed to remove self-managed snapshot") + + def set_self_managed_snap_write(self, snaps: Sequence[Union[int, str]]): + """ + Updates the write context to the specified self-managed + snapshot ids. + + :param snaps: all associated self-managed snapshot ids + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + sorted_snaps = [] + snap_seq = 0 + if snaps: + sorted_snaps = sorted([int(x) for x in snaps], reverse=True) + snap_seq = sorted_snaps[0] + + cdef: + rados_snap_t _snap_seq = snap_seq + rados_snap_t *_snaps = NULL + int _num_snaps = len(sorted_snaps) + try: + _snaps = <rados_snap_t *>malloc(_num_snaps * sizeof(rados_snap_t)) + for i in range(len(sorted_snaps)): + _snaps[i] = sorted_snaps[i] + with nogil: + ret = rados_ioctx_selfmanaged_snap_set_write_ctx(self.io, + _snap_seq, + _snaps, + _num_snaps) + if ret != 0: + raise make_ex(ret, "Failed to update snapshot write context") + finally: + free(_snaps) + + def rollback_self_managed_snap(self, oid: str, snap_id: int): + """ + Rolls an specific object back to a self-managed snapshot revision + + :param oid: the name of the object + :param snap_id: the name of the snapshot + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + oid_raw = cstr(oid, 'oid') + cdef: + char *_oid = oid_raw + rados_snap_t _snap_id = snap_id + with nogil: + ret = rados_ioctx_selfmanaged_snap_rollback(self.io, _oid, _snap_id) + if ret != 0: + raise make_ex(ret, "Failed to rollback %s" % oid) + + def get_last_version(self) -> int: + """ + Return the version of the last object read or written to. + + This exposes the internal version number of the last object read or + written via this io context + + :returns: version of the last object used + """ + self.require_ioctx_open() + with nogil: + ret = rados_get_last_version(self.io) + return int(ret) + + def create_write_op(self) -> WriteOp: + """ + create write operation object. + need call release_write_op after use + """ + return WriteOp().create() + + def create_read_op(self) -> ReadOp: + """ + create read operation object. + need call release_read_op after use + """ + return ReadOp().create() + + def release_write_op(self, write_op): + """ + release memory alloc by create_write_op + """ + write_op.release() + + def release_read_op(self, read_op: ReadOp): + """ + release memory alloc by create_read_op + :para read_op: read_op object + """ + read_op.release() + + def set_omap(self, write_op: WriteOp, keys: Sequence[str], values: Sequence[bytes]): + """ + set keys values to write_op + :para write_op: write_operation object + :para keys: a tuple of keys + :para values: a tuple of values + """ + + if len(keys) != len(values): + raise Error("Rados(): keys and values must have the same number of items") + + keys = cstr_list(keys, 'keys') + values = cstr_list(values, 'values') + lens = [len(v) for v in values] + cdef: + WriteOp _write_op = write_op + size_t key_num = len(keys) + char **_keys = to_bytes_array(keys) + char **_values = to_bytes_array(values) + size_t *_lens = to_csize_t_array(lens) + + try: + with nogil: + rados_write_op_omap_set(_write_op.write_op, + <const char**>_keys, + <const char**>_values, + <const size_t*>_lens, key_num) + finally: + free(_keys) + free(_values) + free(_lens) + + def operate_write_op(self, + write_op: WriteOp, + oid: str, + mtime: int = 0, + flags: int = LIBRADOS_OPERATION_NOFLAG): + """ + execute the real write operation + :para write_op: write operation object + :para oid: object name + :para mtime: the time to set the mtime to, 0 for the current time + :para flags: flags to apply to the entire operation + """ + + oid_raw = cstr(oid, 'oid') + cdef: + WriteOp _write_op = write_op + char *_oid = oid_raw + time_t _mtime = mtime + int _flags = flags + + with nogil: + ret = rados_write_op_operate(_write_op.write_op, self.io, _oid, &_mtime, _flags) + if ret != 0: + raise make_ex(ret, "Failed to operate write op for oid %s" % oid) + + def operate_aio_write_op(self, write_op: WriteOp, oid: str, + oncomplete: Optional[Callable[[Completion], None]] = None, + onsafe: Optional[Callable[[Completion], None]] = None, + mtime: int = 0, + flags: int = LIBRADOS_OPERATION_NOFLAG) -> Completion: + """ + execute the real write operation asynchronously + :para write_op: write operation object + :para oid: object name + :param oncomplete: what to do when the remove is safe and complete in memory + on all replicas + :param onsafe: what to do when the remove is safe and complete on storage + on all replicas + :para mtime: the time to set the mtime to, 0 for the current time + :para flags: flags to apply to the entire operation + + :raises: :class:`Error` + :returns: completion object + """ + + oid_raw = cstr(oid, 'oid') + cdef: + WriteOp _write_op = write_op + char *_oid = oid_raw + Completion completion + time_t _mtime = mtime + int _flags = flags + + completion = self.__get_completion(oncomplete, onsafe) + self.__track_completion(completion) + + with nogil: + ret = rados_aio_write_op_operate(_write_op.write_op, self.io, completion.rados_comp, _oid, + &_mtime, _flags) + if ret != 0: + completion._cleanup() + raise make_ex(ret, "Failed to operate aio write op for oid %s" % oid) + return completion + + def operate_read_op(self, read_op: ReadOp, oid: str, flag: int = LIBRADOS_OPERATION_NOFLAG): + """ + execute the real read operation + :para read_op: read operation object + :para oid: object name + :para flag: flags to apply to the entire operation + """ + oid_raw = cstr(oid, 'oid') + cdef: + ReadOp _read_op = read_op + char *_oid = oid_raw + int _flag = flag + + with nogil: + ret = rados_read_op_operate(_read_op.read_op, self.io, _oid, _flag) + if ret != 0: + raise make_ex(ret, "Failed to operate read op for oid %s" % oid) + + def operate_aio_read_op(self, read_op: ReadOp, oid: str, + oncomplete: Optional[Callable[[Completion], None]] = None, + onsafe: Optional[Callable[[Completion], None]] = None, + flag: int = LIBRADOS_OPERATION_NOFLAG) -> Completion: + """ + execute the real read operation + :para read_op: read operation object + :para oid: object name + :param oncomplete: what to do when the remove is safe and complete in memory + on all replicas + :param onsafe: what to do when the remove is safe and complete on storage + on all replicas + :para flag: flags to apply to the entire operation + """ + oid_raw = cstr(oid, 'oid') + cdef: + ReadOp _read_op = read_op + char *_oid = oid_raw + Completion completion + int _flag = flag + + completion = self.__get_completion(oncomplete, onsafe) + self.__track_completion(completion) + + with nogil: + ret = rados_aio_read_op_operate(_read_op.read_op, self.io, completion.rados_comp, _oid, _flag) + if ret != 0: + completion._cleanup() + raise make_ex(ret, "Failed to operate aio read op for oid %s" % oid) + return completion + + def get_omap_vals(self, + read_op: ReadOp, + start_after: str, + filter_prefix: str, + max_return: int) -> Tuple[OmapIterator, int]: + """ + get the omap values + :para read_op: read operation object + :para start_after: list keys starting after start_after + :para filter_prefix: list only keys beginning with filter_prefix + :para max_return: list no more than max_return key/value pairs + :returns: an iterator over the requested omap values, return value from this action + """ + + start_after_raw = cstr(start_after, 'start_after') if start_after else None + filter_prefix_raw = cstr(filter_prefix, 'filter_prefix') if filter_prefix else None + cdef: + char *_start_after = opt_str(start_after_raw) + char *_filter_prefix = opt_str(filter_prefix_raw) + ReadOp _read_op = read_op + rados_omap_iter_t iter_addr = NULL + int _max_return = max_return + + with nogil: + rados_read_op_omap_get_vals2(_read_op.read_op, _start_after, _filter_prefix, + _max_return, &iter_addr, NULL, NULL) + it = OmapIterator(self) + it.ctx = iter_addr + return it, 0 # 0 is meaningless; there for backward-compat + + def get_omap_keys(self, read_op: ReadOp, start_after: str, max_return: int) -> Tuple[OmapIterator, int]: + """ + get the omap keys + :para read_op: read operation object + :para start_after: list keys starting after start_after + :para max_return: list no more than max_return key/value pairs + :returns: an iterator over the requested omap values, return value from this action + """ + start_after = cstr(start_after, 'start_after') if start_after else None + cdef: + char *_start_after = opt_str(start_after) + ReadOp _read_op = read_op + rados_omap_iter_t iter_addr = NULL + int _max_return = max_return + + with nogil: + rados_read_op_omap_get_keys2(_read_op.read_op, _start_after, + _max_return, &iter_addr, NULL, NULL) + it = OmapIterator(self) + it.ctx = iter_addr + return it, 0 # 0 is meaningless; there for backward-compat + + def get_omap_vals_by_keys(self, read_op: ReadOp, keys: Sequence[str]) -> Tuple[OmapIterator, int]: + """ + get the omap values by keys + :para read_op: read operation object + :para keys: input key tuple + :returns: an iterator over the requested omap values, return value from this action + """ + keys = cstr_list(keys, 'keys') + cdef: + ReadOp _read_op = read_op + rados_omap_iter_t iter_addr + char **_keys = to_bytes_array(keys) + size_t key_num = len(keys) + + try: + with nogil: + rados_read_op_omap_get_vals_by_keys(_read_op.read_op, + <const char**>_keys, + key_num, &iter_addr, NULL) + it = OmapIterator(self) + it.ctx = iter_addr + return it, 0 # 0 is meaningless; there for backward-compat + finally: + free(_keys) + + def remove_omap_keys(self, write_op: WriteOp, keys: Sequence[str]): + """ + remove omap keys specifiled + :para write_op: write operation object + :para keys: input key tuple + """ + + keys = cstr_list(keys, 'keys') + cdef: + WriteOp _write_op = write_op + size_t key_num = len(keys) + char **_keys = to_bytes_array(keys) + + try: + with nogil: + rados_write_op_omap_rm_keys(_write_op.write_op, <const char**>_keys, key_num) + finally: + free(_keys) + + def clear_omap(self, write_op: WriteOp): + """ + Remove all key/value pairs from an object + :para write_op: write operation object + """ + + cdef: + WriteOp _write_op = write_op + + with nogil: + rados_write_op_omap_clear(_write_op.write_op) + + def remove_omap_range2(self, write_op: WriteOp, key_begin: str, key_end: str): + """ + Remove key/value pairs from an object whose keys are in the range + [key_begin, key_end) + :param write_op: write operation object + :param key_begin: the lower bound of the key range to remove + :param key_end: the upper bound of the key range to remove + """ + key_begin_raw = cstr(key_begin, 'key_begin') + key_end_raw = cstr(key_end, 'key_end') + cdef: + WriteOp _write_op = write_op + char* _key_begin = key_begin_raw + size_t key_begin_len = len(key_begin) + char* _key_end = key_end_raw + size_t key_end_len = len(key_end) + with nogil: + rados_write_op_omap_rm_range2(_write_op.write_op, _key_begin, key_begin_len, + _key_end, key_end_len) + + def lock_exclusive(self, key: str, name: str, cookie: str, desc: str = "", + duration: Optional[int] = None, + flags: int = 0): + + """ + Take an exclusive lock on an object + + :param key: name of the object + :param name: name of the lock + :param cookie: cookie of the lock + :param desc: description of the lock + :param duration: duration of the lock in seconds + :param flags: flags + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + name_raw = cstr(name, 'name') + cookie_raw = cstr(cookie, 'cookie') + desc_raw = cstr(desc, 'desc') + + cdef: + char* _key = key_raw + char* _name = name_raw + char* _cookie = cookie_raw + char* _desc = desc_raw + uint8_t _flags = flags + timeval _duration + + if duration is None: + with nogil: + ret = rados_lock_exclusive(self.io, _key, _name, _cookie, _desc, + NULL, _flags) + else: + _duration.tv_sec = duration + with nogil: + ret = rados_lock_exclusive(self.io, _key, _name, _cookie, _desc, + &_duration, _flags) + + if ret < 0: + raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key)) + + def lock_shared(self, key: str, name: str, cookie: str, tag: str, desc: str = "", + duration: Optional[int] = None, + flags: int = 0): + + """ + Take a shared lock on an object + + :param key: name of the object + :param name: name of the lock + :param cookie: cookie of the lock + :param tag: tag of the lock + :param desc: description of the lock + :param duration: duration of the lock in seconds + :param flags: flags + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + tag_raw = cstr(tag, 'tag') + name_raw = cstr(name, 'name') + cookie_raw = cstr(cookie, 'cookie') + desc_raw = cstr(desc, 'desc') + + cdef: + char* _key = key_raw + char* _tag = tag_raw + char* _name = name_raw + char* _cookie = cookie_raw + char* _desc = desc_raw + uint8_t _flags = flags + timeval _duration + + if duration is None: + with nogil: + ret = rados_lock_shared(self.io, _key, _name, _cookie, _tag, _desc, + NULL, _flags) + else: + _duration.tv_sec = duration + with nogil: + ret = rados_lock_shared(self.io, _key, _name, _cookie, _tag, _desc, + &_duration, _flags) + if ret < 0: + raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key)) + + def unlock(self, key: str, name: str, cookie: str): + + """ + Release a shared or exclusive lock on an object + + :param key: name of the object + :param name: name of the lock + :param cookie: cookie of the lock + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + + key_raw = cstr(key, 'key') + name_raw = cstr(name, 'name') + cookie_raw = cstr(cookie, 'cookie') + + cdef: + char* _key = key_raw + char* _name = name_raw + char* _cookie = cookie_raw + + with nogil: + ret = rados_unlock(self.io, _key, _name, _cookie) + if ret < 0: + raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key)) + + def set_osdmap_full_try(self): + """ + Set global osdmap_full_try label to true + """ + with nogil: + rados_set_pool_full_try(self.io) + + def unset_osdmap_full_try(self): + """ + Unset + """ + with nogil: + rados_unset_pool_full_try(self.io) + + def application_enable(self, app_name: str, force: bool = False): + """ + Enable an application on an OSD pool + + :param app_name: application name + :type app_name: str + :param force: False if only a single app should exist per pool + :type expire_seconds: boool + + :raises: :class:`Error` + """ + app_name_raw = cstr(app_name, 'app_name') + cdef: + char *_app_name = app_name_raw + int _force = (1 if force else 0) + + with nogil: + ret = rados_application_enable(self.io, _app_name, _force) + if ret < 0: + raise make_ex(ret, "error enabling application") + + def application_list(self) -> List[str]: + """ + Returns a list of enabled applications + + :returns: list of app name string + """ + cdef: + size_t length = 128 + char *apps = NULL + + try: + while True: + apps = <char *>realloc_chk(apps, length) + with nogil: + ret = rados_application_list(self.io, apps, &length) + if ret == 0: + return [decode_cstr(app) for app in + apps[:length].split(b'\0') if app] + elif ret == -errno.ENOENT: + return None + elif ret == -errno.ERANGE: + pass + else: + raise make_ex(ret, "error listing applications") + finally: + free(apps) + + def application_metadata_get(self, app_name: str, key: str) -> str: + """ + Gets application metadata on an OSD pool for the given key + + :param app_name: application name + :type app_name: str + :param key: metadata key + :type key: str + :returns: str - metadata value + + :raises: :class:`Error` + """ + + app_name_raw = cstr(app_name, 'app_name') + key_raw = cstr(key, 'key') + cdef: + char *_app_name = app_name_raw + char *_key = key_raw + size_t size = 129 + char *value = NULL + int ret + try: + while True: + value = <char *>realloc_chk(value, size) + with nogil: + ret = rados_application_metadata_get(self.io, _app_name, + _key, value, &size) + if ret != -errno.ERANGE: + break + if ret == -errno.ENOENT: + raise KeyError('no metadata %s for application %s' % (key, _app_name)) + elif ret != 0: + raise make_ex(ret, 'error getting metadata %s for application %s' % + (key, _app_name)) + return decode_cstr(value) + finally: + free(value) + + def application_metadata_set(self, app_name: str, key: str, value: str): + """ + Sets application metadata on an OSD pool + + :param app_name: application name + :type app_name: str + :param key: metadata key + :type key: str + :param value: metadata value + :type value: str + + :raises: :class:`Error` + """ + app_name_raw = cstr(app_name, 'app_name') + key_raw = cstr(key, 'key') + value_raw = cstr(value, 'value') + cdef: + char *_app_name = app_name_raw + char *_key = key_raw + char *_value = value_raw + + with nogil: + ret = rados_application_metadata_set(self.io, _app_name, _key, + _value) + if ret < 0: + raise make_ex(ret, "error setting application metadata") + + def application_metadata_remove(self, app_name: str, key: str): + """ + Remove application metadata from an OSD pool + + :param app_name: application name + :type app_name: str + :param key: metadata key + :type key: str + + :raises: :class:`Error` + """ + app_name_raw = cstr(app_name, 'app_name') + key_raw = cstr(key, 'key') + cdef: + char *_app_name = app_name_raw + char *_key = key_raw + + with nogil: + ret = rados_application_metadata_remove(self.io, _app_name, _key) + if ret < 0: + raise make_ex(ret, "error removing application metadata") + + def application_metadata_list(self, app_name: str) -> List[Tuple[str, str]]: + """ + Returns a list of enabled applications + + :param app_name: application name + :type app_name: str + :returns: list of key/value tuples + """ + app_name_raw = cstr(app_name, 'app_name') + cdef: + char *_app_name = app_name_raw + size_t key_length = 128 + size_t val_length = 128 + char *c_keys = NULL + char *c_vals = NULL + + try: + while True: + c_keys = <char *>realloc_chk(c_keys, key_length) + c_vals = <char *>realloc_chk(c_vals, val_length) + with nogil: + ret = rados_application_metadata_list(self.io, _app_name, + c_keys, &key_length, + c_vals, &val_length) + if ret == 0: + keys = [decode_cstr(key) for key in + c_keys[:key_length].split(b'\0')] + vals = [decode_cstr(val) for val in + c_vals[:val_length].split(b'\0')] + return list(zip(keys, vals))[:-1] + elif ret == -errno.ERANGE: + pass + else: + raise make_ex(ret, "error listing application metadata") + finally: + free(c_keys) + free(c_vals) + + def alignment(self) -> int: + """ + Returns pool alignment + + :returns: + Number of alignment bytes required by the current pool, or None if + alignment is not required. + """ + cdef: + int requires = 0 + uint64_t _alignment + + with nogil: + ret = rados_ioctx_pool_requires_alignment2(self.io, &requires) + if ret != 0: + raise make_ex(ret, "error checking alignment") + + alignment = None + if requires: + with nogil: + ret = rados_ioctx_pool_required_alignment2(self.io, &_alignment) + if ret != 0: + raise make_ex(ret, "error querying alignment") + alignment = _alignment + return alignment + + +def set_object_locator(func): + def retfunc(self, *args, **kwargs): + if self.locator_key is not None: + old_locator = self.ioctx.get_locator_key() + self.ioctx.set_locator_key(self.locator_key) + retval = func(self, *args, **kwargs) + self.ioctx.set_locator_key(old_locator) + return retval + else: + return func(self, *args, **kwargs) + return retfunc + + +def set_object_namespace(func): + def retfunc(self, *args, **kwargs): + if self.nspace is None: + raise LogicError("Namespace not set properly in context") + old_nspace = self.ioctx.get_namespace() + self.ioctx.set_namespace(self.nspace) + retval = func(self, *args, **kwargs) + self.ioctx.set_namespace(old_nspace) + return retval + return retfunc + + +class Object(object): + """Rados object wrapper, makes the object look like a file""" + def __init__(self, ioctx, key, locator_key=None, nspace=None): + self.key = key + self.ioctx = ioctx + self.offset = 0 + self.state = "exists" + self.locator_key = locator_key + self.nspace = "" if nspace is None else nspace + + def __str__(self): + return "rados.Object(ioctx=%s,key=%s,nspace=%s,locator=%s)" % \ + (str(self.ioctx), self.key, "--default--" + if self.nspace is "" else self.nspace, self.locator_key) + + def require_object_exists(self): + if self.state != "exists": + raise ObjectStateError("The object is %s" % self.state) + + @set_object_locator + @set_object_namespace + def read(self, length=1024 * 1024): + self.require_object_exists() + ret = self.ioctx.read(self.key, length, self.offset) + self.offset += len(ret) + return ret + + @set_object_locator + @set_object_namespace + def write(self, string_to_write): + self.require_object_exists() + ret = self.ioctx.write(self.key, string_to_write, self.offset) + if ret == 0: + self.offset += len(string_to_write) + return ret + + @set_object_locator + @set_object_namespace + def remove(self): + self.require_object_exists() + self.ioctx.remove_object(self.key) + self.state = "removed" + + @set_object_locator + @set_object_namespace + def stat(self) -> Tuple[int, time.struct_time]: + self.require_object_exists() + return self.ioctx.stat(self.key) + + def seek(self, position: int): + self.require_object_exists() + self.offset = position + + @set_object_locator + @set_object_namespace + def get_xattr(self, xattr_name: str) -> bytes: + self.require_object_exists() + return self.ioctx.get_xattr(self.key, xattr_name) + + @set_object_locator + @set_object_namespace + def get_xattrs(self) -> XattrIterator: + self.require_object_exists() + return self.ioctx.get_xattrs(self.key) + + @set_object_locator + @set_object_namespace + def set_xattr(self, xattr_name: str, xattr_value: bytes) -> bool: + self.require_object_exists() + return self.ioctx.set_xattr(self.key, xattr_name, xattr_value) + + @set_object_locator + @set_object_namespace + def rm_xattr(self, xattr_name: str) -> bool: + self.require_object_exists() + return self.ioctx.rm_xattr(self.key, xattr_name) + +MONITOR_LEVELS = [ + "debug", + "info", + "warn", "warning", + "err", "error", + "sec", + ] + + +class MonitorLog(object): + # NOTE(sileht): Keep this class for backward compat + # method moved to Rados.monitor_log() + """ + For watching cluster log messages. Instantiate an object and keep + it around while callback is periodically called. Construct with + 'level' to monitor 'level' messages (one of MONITOR_LEVELS). + arg will be passed to the callback. + + callback will be called with: + arg (given to __init__) + line (the full line, including timestamp, who, level, msg) + who (which entity issued the log message) + timestamp_sec (sec of a struct timespec) + timestamp_nsec (sec of a struct timespec) + seq (sequence number) + level (string representing the level of the log message) + msg (the message itself) + callback's return value is ignored + """ + def __init__(self, cluster, level, callback, arg): + self.level = level + self.callback = callback + self.arg = arg + self.cluster = cluster + self.cluster.monitor_log(level, callback, arg) + diff --git a/src/pybind/rados/setup.py b/src/pybind/rados/setup.py new file mode 100755 index 000000000..0d1047825 --- /dev/null +++ b/src/pybind/rados/setup.py @@ -0,0 +1,207 @@ +from __future__ import print_function +import pkgutil +if not pkgutil.find_loader('setuptools'): + from distutils.core import setup + from distutils.extension import Extension +else: + from setuptools import setup + from setuptools.extension import Extension +import distutils.sysconfig +from distutils.errors import CompileError, LinkError +from distutils.ccompiler import new_compiler +from itertools import filterfalse, takewhile + +import os +import shutil +import sys +import tempfile +import textwrap + + +def filter_unsupported_flags(compiler, flags): + args = takewhile(lambda argv: not argv.startswith('-'), [compiler] + flags) + if any('clang' in arg for arg in args): + return list(filterfalse(lambda f: + f in ('-mcet', + '-fstack-clash-protection', + '-fno-var-tracking-assignments', + '-Wno-deprecated-register', + '-Wno-gnu-designator') or + f.startswith('-fcf-protection'), + flags)) + else: + return flags + + +def monkey_with_compiler(customize): + def patched(compiler): + customize(compiler) + if compiler.compiler_type != 'unix': + return + compiler.compiler[1:] = \ + filter_unsupported_flags(compiler.compiler[0], + compiler.compiler[1:]) + compiler.compiler_so[1:] = \ + filter_unsupported_flags(compiler.compiler_so[0], + compiler.compiler_so[1:]) + return patched + + +distutils.sysconfig.customize_compiler = \ + monkey_with_compiler(distutils.sysconfig.customize_compiler) + +# PEP 440 versioning of the Rados package on PyPI +# Bump this version, after every changeset +__version__ = '2.0.0' + + +def get_python_flags(libs): + py_libs = sum((libs.split() for libs in + distutils.sysconfig.get_config_vars('LIBS', 'SYSLIBS')), []) + ldflags = list(filterfalse(lambda lib: lib.startswith('-l'), py_libs)) + py_libs = [lib.replace('-l', '') for lib in + filter(lambda lib: lib.startswith('-l'), py_libs)] + compiler = new_compiler() + distutils.sysconfig.customize_compiler(compiler) + return dict( + include_dirs=[distutils.sysconfig.get_python_inc()], + library_dirs=distutils.sysconfig.get_config_vars('LIBDIR', 'LIBPL'), + libraries=libs + py_libs, + extra_compile_args=filter_unsupported_flags( + compiler.compiler[0], + compiler.compiler[1:] + distutils.sysconfig.get_config_var('CFLAGS').split()), + extra_link_args=(distutils.sysconfig.get_config_var('LDFLAGS').split() + + ldflags)) + + +def check_sanity(): + """ + Test if development headers and library for rados is available by compiling a dummy C program. + """ + CEPH_SRC_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + '..', + '..' + ) + + tmp_dir = tempfile.mkdtemp(dir=os.environ.get('TMPDIR', os.path.dirname(__file__))) + tmp_file = os.path.join(tmp_dir, 'rados_dummy.c') + + with open(tmp_file, 'w') as fp: + dummy_prog = textwrap.dedent(""" + #include <rados/librados.h> + + int main(void) { + rados_t cluster; + rados_create(&cluster, NULL); + return 0; + } + """) + fp.write(dummy_prog) + + compiler = new_compiler() + distutils.sysconfig.customize_compiler(compiler) + + if 'CEPH_LIBDIR' in os.environ: + # The setup.py has been invoked by a top-level Ceph make. + # Set the appropriate CFLAGS and LDFLAGS + compiler.set_include_dirs([os.path.join(CEPH_SRC_DIR, 'include')]) + compiler.set_library_dirs([os.environ.get('CEPH_LIBDIR')]) + + try: + link_objects = compiler.compile( + sources=[tmp_file], + output_dir=tmp_dir + ) + compiler.link_executable( + objects=link_objects, + output_progname=os.path.join(tmp_dir, 'rados_dummy'), + libraries=['rados'], + output_dir=tmp_dir, + ) + + except CompileError: + print('\nCompile Error: RADOS development headers not found', file=sys.stderr) + return False + except LinkError: + print('\nLink Error: RADOS library not found', file=sys.stderr) + return False + else: + return True + finally: + shutil.rmtree(tmp_dir) + + +if 'BUILD_DOC' in os.environ or 'READTHEDOCS' in os.environ: + ext_args = {} + cython_constants = dict(BUILD_DOC=True) +elif check_sanity(): + ext_args = get_python_flags(['rados']) + cython_constants = dict(BUILD_DOC=False) +else: + sys.exit(1) + +cmdclass = {} +try: + from Cython.Build import cythonize + from Cython.Distutils import build_ext + + cmdclass = {'build_ext': build_ext} +except ImportError: + print("WARNING: Cython is not installed.") + + if not os.path.isfile('rados.c'): + print('ERROR: Cannot find Cythonized file rados.c', file=sys.stderr) + sys.exit(1) + else: + def cythonize(x, **kwargs): + return x + + source = "rados.c" +else: + source = "rados.pyx" + +# Disable cythonification if we're not really building anything +if (len(sys.argv) >= 2 and + any(i in sys.argv[1:] for i in ('--help', 'clean', 'egg_info', '--version') + )): + def cythonize(x, **kwargs): + return x + +setup( + name='rados', + version=__version__, + description="Python bindings for the Ceph librados library", + long_description=( + "This package contains Python bindings for interacting with Ceph's " + "RADOS library. RADOS is a reliable, autonomic distributed object " + "storage cluster developed as part of the Ceph distributed storage " + "system. This is a shared library allowing applications to access " + "the distributed object store using a simple file-like interface." + ), + url='https://github.com/ceph/ceph/tree/master/src/pybind/rados', + license='LGPLv2+', + platforms='Linux', + ext_modules=cythonize( + [ + Extension( + "rados", + [source], + **ext_args + ) + ], + # use "3str" when Cython 3.0 is available + compiler_directives={'language_level': sys.version_info.major}, + compile_time_env=cython_constants, + build_dir=os.environ.get("CYTHON_BUILD_DIR", None), + ), + classifiers=[ + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Cython', + 'Programming Language :: Python :: 3' + ], + cmdclass=cmdclass, +) |