diff options
Diffstat (limited to '')
72 files changed, 19869 insertions, 0 deletions
diff --git a/src/spdk/doc/.gitignore b/src/spdk/doc/.gitignore new file mode 100644 index 000000000..e28511785 --- /dev/null +++ b/src/spdk/doc/.gitignore @@ -0,0 +1,3 @@ +# changelog.md is generated by Makefile +changelog.md +output/ diff --git a/src/spdk/doc/Doxyfile b/src/spdk/doc/Doxyfile new file mode 100644 index 000000000..3c33ab339 --- /dev/null +++ b/src/spdk/doc/Doxyfile @@ -0,0 +1,2487 @@ +# Doxyfile 1.8.13 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all text +# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv +# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv +# for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "SPDK" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = output + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = NO + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = YES + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 8 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines. + +ALIASES = + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding "class=itcl::class" +# will allow you to use the command class in the itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = YES + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, Javascript, +# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: +# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: +# Fortran. In the later case the parser tries to guess whether the code is fixed +# or free formatted code, this is the default for Fortran type files), VHDL. For +# instance to make doxygen treat .inc files as Fortran files (default is PHP), +# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 0. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 0 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = NO + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# (class|struct|union) declarations. If set to NO, these declarations will be +# included in the documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = NO + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if <section_label> ... \endif and \cond <section_label> +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = YES + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = NO + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = ../include/spdk \ + index.md \ + +# This list contains the top level pages listed in index.md. This list should +# remain in the same order as the contents of index.md. The order here also +# determines the order of these sections in the left-side navigation bar. +INPUT += \ + intro.md \ + concepts.md \ + user_guides.md \ + prog_guides.md \ + general.md \ + misc.md \ + driver_modules.md \ + tools.md \ + performance_reports.md \ + +# All remaining pages are listed here in alphabetical order by filename. +INPUT += \ + about.md \ + accel_fw.md \ + applications.md \ + bdev.md \ + bdevperf.md \ + bdev_module.md \ + bdev_pg.md \ + blob.md \ + blobfs.md \ + changelog.md \ + compression.md \ + concurrency.md \ + containers.md \ + event.md \ + ftl.md \ + gdb_macros.md \ + getting_started.md \ + idxd.md \ + ioat.md \ + iscsi.md \ + jsonrpc.md \ + jsonrpc_proxy.md \ + libraries.md \ + lvol.md \ + memory.md \ + notify.md \ + nvme.md \ + nvme-cli.md \ + nvme_spec.md \ + nvmf.md \ + nvmf_tgt_pg.md \ + nvmf_tracing.md \ + overview.md \ + peer_2_peer.md \ + porting.md \ + spdkcli.md \ + ssd_internals.md \ + system_configuration.md \ + userspace.md \ + vagrant.md \ + vhost.md \ + vhost_processing.md \ + virtio.md \ + vmd.md \ + vpp_integration.md + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = cap_lo_register \ + cap_hi_register \ + aqa_register \ + cc_register \ + nvme_registers + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = img + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# <filter> <input-file> +# +# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# function all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see http://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = header.html + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = footer.html + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = stylesheet.css + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = two.min.js + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: http://developer.apple.com/tools/xcode/), introduced with +# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the master .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# http://www.mathjax.org) which uses client side Javascript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from http://www.mathjax.org before deployment. +# The default value is: http://cdn.mathjax.org/mathjax/latest. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = http://www.mathjax.org/mathjax + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use <access key> + S +# (what the <access key> is depends on the OS and browser, but it is typically +# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down +# key> to jump into the search results window, the results can be navigated +# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel +# the search. The filter options can be selected when the cursor is inside the +# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys> +# to select a filter and <Enter> or <escape> to activate or cancel the filter +# option. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a web server instead of a web client using Javascript. There +# are two flavors of web server based searching depending on the EXTERNAL_SEARCH +# setting. When disabled, doxygen will generate a PHP script for searching and +# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing +# and searching needs to be provided by external tools. See the section +# "External Indexing and Searching" for details. +# The default value is: NO. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SERVER_BASED_SEARCH = NO + +# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP +# script for searching. Instead the search results are written to an XML file +# which needs to be processed by an external indexer. Doxygen will invoke an +# external search engine pointed to by the SEARCHENGINE_URL option to obtain the +# search results. +# +# Doxygen ships with an example indexer (doxyindexer) and search engine +# (doxysearch.cgi) which are based on the open source search engine library +# Xapian (see: http://xapian.org/). +# +# See the section "External Indexing and Searching" for details. +# The default value is: NO. +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTERNAL_SEARCH = NO + +# The SEARCHENGINE_URL should point to a search engine hosted by a web server +# which will return the search results when EXTERNAL_SEARCH is enabled. +# +# Doxygen ships with an example indexer (doxyindexer) and search engine +# (doxysearch.cgi) which are based on the open source search engine library +# Xapian (see: http://xapian.org/). See the section "External Indexing and +# Searching" for details. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SEARCHENGINE_URL = + +# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed +# search data is written to a file for indexing by an external tool. With the +# SEARCHDATA_FILE tag the name of this file can be specified. +# The default file is: searchdata.xml. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SEARCHDATA_FILE = searchdata.xml + +# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the +# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is +# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple +# projects and redirect the results back to the right project. +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTERNAL_SEARCH_ID = + +# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen +# projects other than the one defined by this configuration file, but that are +# all added to the same external search index. Each project needs to have a +# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of +# to a relative location where the documentation can be found. The format is: +# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ... +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTRA_SEARCH_MAPPINGS = + +#--------------------------------------------------------------------------- +# Configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = YES + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: latex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. +# +# Note that when enabling USE_PDFLATEX this option is only used for generating +# bitmaps for formulas in the HTML output, but not in the Makefile that is +# written to the output directory. +# The default file is: latex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate +# index for LaTeX. +# The default file is: makeindex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX +# documents. This may be useful for small projects and may help to save some +# trees in general. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used by the +# printer. +# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x +# 14 inches) and executive (7.25 x 10.5 inches). +# The default value is: a4. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +PAPER_TYPE = letter + +# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names +# that should be included in the LaTeX output. The package can be specified just +# by its name or with the correct syntax as to be used with the LaTeX +# \usepackage command. To get the times font for instance you can specify : +# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times} +# To use the option intlimits with the amsmath package you can specify: +# EXTRA_PACKAGES=[intlimits]{amsmath} +# If left blank no extra packages will be included. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the +# generated LaTeX document. The header should contain everything until the first +# chapter. If it is left blank doxygen will generate a standard header. See +# section "Doxygen usage" for information on how to let doxygen write the +# default header to a separate file. +# +# Note: Only use a user-defined header if you know what you are doing! The +# following commands have a special meaning inside the header: $title, +# $datetime, $date, $doxygenversion, $projectname, $projectnumber, +# $projectbrief, $projectlogo. Doxygen will replace $title with the empty +# string, for the replacement values of the other commands the user is referred +# to HTML_HEADER. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_HEADER = + +# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the +# generated LaTeX document. The footer should contain everything after the last +# chapter. If it is left blank doxygen will generate a standard footer. See +# LATEX_HEADER for more information on how to generate a default footer and what +# special commands can be used inside the footer. +# +# Note: Only use a user-defined footer if you know what you are doing! +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_FOOTER = + +# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# LaTeX style sheets that are included after the standard style sheets created +# by doxygen. Using this option one can overrule certain style aspects. Doxygen +# will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_EXTRA_STYLESHEET = + +# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the LATEX_OUTPUT output +# directory. Note that the files will be copied as-is; there are no commands or +# markers available. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_EXTRA_FILES = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is +# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will +# contain links (just like the HTML output) instead of page references. This +# makes the output suitable for online browsing using a PDF viewer. +# The default value is: YES. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate +# the PDF file directly from the LaTeX files. Set this option to YES, to get a +# higher quality PDF documentation. +# The default value is: YES. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode +# command to the generated LaTeX files. This will instruct LaTeX to keep running +# if errors occur, instead of asking the user for help. This option is also used +# when generating formulas in HTML. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_BATCHMODE = YES + +# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the +# index chapters (such as File Index, Compound Index, etc.) in the output. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_HIDE_INDICES = NO + +# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source +# code with syntax highlighting in the LaTeX output. +# +# Note that which sources are shown also depends on other settings such as +# SOURCE_BROWSER. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_SOURCE_CODE = NO + +# The LATEX_BIB_STYLE tag can be used to specify the style to use for the +# bibliography, e.g. plainnat, or ieeetr. See +# http://en.wikipedia.org/wiki/BibTeX and \cite for more info. +# The default value is: plain. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_BIB_STYLE = plain + +# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated +# page will contain the date and time when the page was generated. Setting this +# to NO can help when comparing the output of multiple runs. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_TIMESTAMP = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The +# RTF output is optimized for Word 97 and may not look too pretty with other RTF +# readers/editors. +# The default value is: NO. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: rtf. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF +# documents. This may be useful for small projects and may help to save some +# trees in general. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will +# contain hyperlink fields. The RTF file will contain links (just like the HTML +# output) instead of page references. This makes the output suitable for online +# browsing using Word or some other Word compatible readers that support those +# fields. +# +# Note: WordPad (write) and others do not support links. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's config +# file, i.e. a series of assignments. You only have to provide replacements, +# missing definitions are set to their default value. +# +# See also section "Doxygen usage" for information on how to generate the +# default style sheet that doxygen normally uses. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an RTF document. Syntax is +# similar to doxygen's config file. A template extensions file can be generated +# using doxygen -e rtf extensionFile. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_EXTENSIONS_FILE = + +# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code +# with syntax highlighting in the RTF output. +# +# Note that which sources are shown also depends on other settings such as +# SOURCE_BROWSER. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_SOURCE_CODE = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for +# classes and files. +# The default value is: NO. + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. A directory man3 will be created inside the directory specified by +# MAN_OUTPUT. +# The default directory is: man. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to the generated +# man pages. In case the manual section does not start with a number, the number +# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is +# optional. +# The default value is: .3. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_EXTENSION = .3 + +# The MAN_SUBDIR tag determines the name of the directory created within +# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by +# MAN_EXTENSION with the initial . removed. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_SUBDIR = + +# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it +# will generate one additional man file for each entity documented in the real +# man page(s). These additional files only source the real man page, but without +# them the man command would be unable to find the correct page. +# The default value is: NO. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that +# captures the structure of the code including all documentation. +# The default value is: NO. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: xml. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_OUTPUT = xml + +# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program +# listings (including syntax highlighting and cross-referencing information) to +# the XML output. Note that enabling this will significantly increase the size +# of the XML output. +# The default value is: YES. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the DOCBOOK output +#--------------------------------------------------------------------------- + +# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files +# that can be used to generate PDF. +# The default value is: NO. + +GENERATE_DOCBOOK = NO + +# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in +# front of it. +# The default directory is: docbook. +# This tag requires that the tag GENERATE_DOCBOOK is set to YES. + +DOCBOOK_OUTPUT = docbook + +# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the +# program listings (including syntax highlighting and cross-referencing +# information) to the DOCBOOK output. Note that enabling this will significantly +# increase the size of the DOCBOOK output. +# The default value is: NO. +# This tag requires that the tag GENERATE_DOCBOOK is set to YES. + +DOCBOOK_PROGRAMLISTING = NO + +#--------------------------------------------------------------------------- +# Configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an +# AutoGen Definitions (see http://autogen.sf.net) file that captures the +# structure of the code including all documentation. Note that this feature is +# still experimental and incomplete at the moment. +# The default value is: NO. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module +# file that captures the structure of the code including all documentation. +# +# Note that this feature is still experimental and incomplete at the moment. +# The default value is: NO. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary +# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI +# output from the Perl module output. +# The default value is: NO. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely +# formatted so it can be parsed by a human reader. This is useful if you want to +# understand what is going on. On the other hand, if this tag is set to NO, the +# size of the Perl module output will be much smaller and Perl will parse it +# just the same. +# The default value is: YES. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file are +# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful +# so different doxyrules.make files included by the same Makefile don't +# overwrite each other's variables. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all +# C-preprocessor directives found in the sources and include files. +# The default value is: YES. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + +# If the SEARCH_INCLUDES tag is set to YES, the include files in the +# INCLUDE_PATH will be searched if a #include is found. +# The default value is: YES. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will be +# used. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that are +# defined before the preprocessor is started (similar to the -D option of e.g. +# gcc). The argument of the tag is a list of macros of the form: name or +# name=definition (no spaces). If the definition and the "=" are omitted, "=1" +# is assumed. To prevent a macro definition from being undefined via #undef or +# recursively expanded use the := operator instead of the = operator. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +PREDEFINED = __attribute__(x)= + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will +# remove all references to function-like macros that are alone on a line, have +# an all uppercase name, and do not end with a semicolon. Such function macros +# are typically used for boiler-plate code, and will confuse the parser if not +# removed. +# The default value is: YES. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES tag can be used to specify one or more tag files. For each tag +# file the location of the external documentation should be added. The format of +# a tag file without this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where loc1 and loc2 can be relative or absolute paths or URLs. See the +# section "Linking to external documentation" for more information about the use +# of tag files. +# Note: Each tag file must have a unique name (where the name does NOT include +# the path). If a tag file is not located in the directory in which doxygen is +# run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create a +# tag file that is based on the input files it reads. See section "Linking to +# external documentation" for more information about the usage of tag files. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES, all external class will be listed in +# the class index. If set to NO, only the inherited external classes will be +# listed. +# The default value is: NO. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will be +# listed. +# The default value is: YES. + +EXTERNAL_GROUPS = YES + +# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in +# the related pages index. If set to NO, only the current project's pages will +# be listed. +# The default value is: YES. + +EXTERNAL_PAGES = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of 'which perl'). +# The default file (with absolute path) is: /usr/bin/perl. + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram +# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to +# NO turns the diagrams off. Note that this option also works with HAVE_DOT +# disabled, but it is recommended to install and use dot, since it yields more +# powerful graphs. +# The default value is: YES. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see: +# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. + +DIA_PATH = + +# If set to YES the inheritance and collaboration graphs will hide inheritance +# and usage relations if the target is undocumented or is not a class. +# The default value is: YES. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz (see: +# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# Bell Labs. The other options in this section have no effect if this option is +# set to NO +# The default value is: NO. + +HAVE_DOT = YES + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed +# to run in parallel. When set to 0 doxygen will base this on the number of +# processors available in the system. You can set it explicitly to a value +# larger than 0 to get control over the balance between CPU load and processing +# speed. +# Minimum value: 0, maximum value: 32, default value: 0. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_NUM_THREADS = 0 + +# When you want a differently looking font in the dot files that doxygen +# generates you can specify the font name using DOT_FONTNAME. You need to make +# sure dot is able to find the font, which can be done by putting it in a +# standard location or by setting the DOTFONTPATH environment variable or by +# setting DOT_FONTPATH to the directory containing the font. +# The default value is: Helvetica. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTNAME = Helvetica + +# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of +# dot graphs. +# Minimum value: 4, maximum value: 24, default value: 10. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the default font as specified with +# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set +# the path where dot can find it using this tag. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTPATH = + +# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for +# each documented class showing the direct and indirect inheritance relations. +# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a +# graph for each documented class showing the direct and indirect implementation +# dependencies (inheritance, containment, and class references variables) of the +# class with other documented classes. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for +# groups, showing the direct groups dependencies. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +UML_LOOK = NO + +# If the UML_LOOK tag is enabled, the fields and methods are shown inside the +# class node. If there are many fields or methods and many nodes the graph may +# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the +# number of items for each type to make the size more manageable. Set this to 0 +# for no limit. Note that the threshold may be exceeded by 50% before the limit +# is enforced. So when you set the threshold to 10, up to 15 fields may appear, +# but if the number exceeds 15, the total amount of fields shown is limited to +# 10. +# Minimum value: 0, maximum value: 100, default value: 10. +# This tag requires that the tag HAVE_DOT is set to YES. + +UML_LIMIT_NUM_FIELDS = 10 + +# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and +# collaboration graphs will show the relations between templates and their +# instances. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +TEMPLATE_RELATIONS = NO + +# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to +# YES then doxygen will generate a graph for each documented file showing the +# direct and indirect include dependencies of the file with other documented +# files. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +INCLUDE_GRAPH = NO + +# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are +# set to YES then doxygen will generate a graph for each documented file showing +# the direct and indirect include dependencies of the file with other documented +# files. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +INCLUDED_BY_GRAPH = NO + +# If the CALL_GRAPH tag is set to YES then doxygen will generate a call +# dependency graph for every global function or class method. +# +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. Disabling a call graph can be +# accomplished by means of the command \hidecallgraph. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller +# dependency graph for every global function or class method. +# +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. Disabling a caller graph can be +# accomplished by means of the command \hidecallergraph. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical +# hierarchy of all classes instead of a textual one. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the +# dependencies a directory has on other directories in a graphical way. The +# dependency relations are determined by the #include relations between the +# files in the directories. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +DIRECTORY_GRAPH = NO + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. For an explanation of the image formats see the section +# output formats in the documentation of the dot tool (Graphviz (see: +# http://www.graphviz.org/)). +# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order +# to make the SVG files visible in IE 9+ (other browsers do not have this +# requirement). +# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo, +# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and +# png:gdiplus:gdiplus. +# The default value is: png. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_IMAGE_FORMAT = png + +# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to +# enable generation of interactive SVG images that allow zooming and panning. +# +# Note that this requires a modern browser other than Internet Explorer. Tested +# and working are Firefox, Chrome, Safari, and Opera. +# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make +# the SVG files visible. Older versions of IE do not have SVG support. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +INTERACTIVE_SVG = NO + +# The DOT_PATH tag can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the \dotfile +# command). +# This tag requires that the tag HAVE_DOT is set to YES. + +DOTFILE_DIRS = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = + +# The DIAFILE_DIRS tag can be used to specify one or more directories that +# contain dia files that are included in the documentation (see the \diafile +# command). + +DIAFILE_DIRS = + +# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the +# path where java can find the plantuml.jar file. If left blank, it is assumed +# PlantUML is not used or called during a preprocessing step. Doxygen will +# generate a warning when it encounters a \startuml command in this case and +# will not generate output for the diagram. + +PLANTUML_JAR_PATH = + +# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a +# configuration file for plantuml. + +PLANTUML_CFG_FILE = + +# When using plantuml, the specified paths are searched for files specified by +# the !include statement in a plantuml block. + +PLANTUML_INCLUDE_PATH = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes +# that will be shown in the graph. If the number of nodes in a graph becomes +# larger than this value, doxygen will truncate the graph, which is visualized +# by representing a node as a red box. Note that doxygen if the number of direct +# children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that +# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. +# Minimum value: 0, maximum value: 10000, default value: 50. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs +# generated by dot. A depth value of 3 means that only nodes reachable from the +# root by following a path via at most 3 edges will be shown. Nodes that lay +# further from the root node will be omitted. Note that setting this option to 1 +# or 2 may greatly reduce the computation time needed for large code bases. Also +# note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. +# Minimum value: 0, maximum value: 1000, default value: 0. +# This tag requires that the tag HAVE_DOT is set to YES. + +MAX_DOT_GRAPH_DEPTH = 2 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not seem +# to support this out of the box. +# +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) support +# this, this feature is disabled by default. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_MULTI_TARGETS = YES + +# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page +# explaining the meaning of the various boxes and arrows in the dot generated +# graphs. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# files that are used to generate the various graphs. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_CLEANUP = YES diff --git a/src/spdk/doc/Makefile b/src/spdk/doc/Makefile new file mode 100644 index 000000000..ec3f396d9 --- /dev/null +++ b/src/spdk/doc/Makefile @@ -0,0 +1,23 @@ +SPDK_ROOT_DIR := $(abspath $(CURDIR)/..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +all: doc + @: + +.PHONY: all doc clean + +doc: output + +changelog.md: ../CHANGELOG.md + $(Q)sed -e 's/^# Changelog/# Changelog {#changelog}/' \ + -e 's/^##/#/' \ + -e 's/^# \(\(v..\...\):.*\)/# \1 {#changelog-\2}/' \ + -e '/# v..\...:/s/\./-/2' \ + < $< > $@ + +output: Doxyfile changelog.md $(wildcard *.md) $(wildcard ../include/spdk/*.h) + $(Q)rm -rf $@ + $(Q)doxygen Doxyfile + +clean: + $(Q)rm -rf output changelog.md diff --git a/src/spdk/doc/README.md b/src/spdk/doc/README.md new file mode 100644 index 000000000..c5c03b287 --- /dev/null +++ b/src/spdk/doc/README.md @@ -0,0 +1,14 @@ +SPDK Documentation +================== + +The current version of the SPDK documentation can be found online at +http://www.spdk.io/doc/ + +Building the Documentation +========================== + +To convert the documentation into HTML run `make` in the `doc` +directory. The output will be located in `doc/output/html`. Before +running `make` ensure all pre-requisites are installed. See +[Installing Prerequisites](http://www.spdk.io/doc/getting_started.html) +for more details. diff --git a/src/spdk/doc/about.md b/src/spdk/doc/about.md new file mode 100644 index 000000000..6e73cd551 --- /dev/null +++ b/src/spdk/doc/about.md @@ -0,0 +1,36 @@ +# What is SPDK {#about} + +The Storage Performance Development Kit (SPDK) provides a set of tools and +libraries for writing high performance, scalable, user-mode storage +applications. It achieves high performance through the use of a number of key +techniques: + +* Moving all of the necessary drivers into userspace, which avoids syscalls + and enables zero-copy access from the application. +* Polling hardware for completions instead of relying on interrupts, which + lowers both total latency and latency variance. +* Avoiding all locks in the I/O path, instead relying on message passing. + +The bedrock of SPDK is a user space, polled-mode, asynchronous, lockless +[NVMe](http://www.nvmexpress.org) driver. This provides zero-copy, highly +parallel access directly to an SSD from a user space application. The driver is +written as a C library with a single public header. See @ref nvme for more +details. + +SPDK further provides a full block stack as a user space library that performs +many of the same operations as a block stack in an operating system. This +includes unifying the interface between disparate storage devices, queueing to +handle conditions such as out of memory or I/O hangs, and logical volume +management. See @ref bdev for more information. + +Finally, SPDK provides +[NVMe-oF](http://www.nvmexpress.org/nvm-express-over-fabrics-specification-released), +[iSCSI](https://en.wikipedia.org/wiki/ISCSI), and +[vhost](http://blog.vmsplice.net/2011/09/qemu-internals-vhost-architecture.html) +servers built on top of these components that are capable of serving disks over +the network or to other processes. The standard Linux kernel initiators for +NVMe-oF and iSCSI interoperate with these targets, as well as QEMU with vhost. +These servers can be up to an order of magnitude more CPU efficient than other +implementations. These targets can be used as examples of how to implement a +high performance storage target, or used as the basis for production +deployments. diff --git a/src/spdk/doc/accel_fw.md b/src/spdk/doc/accel_fw.md new file mode 100644 index 000000000..ccedc3e84 --- /dev/null +++ b/src/spdk/doc/accel_fw.md @@ -0,0 +1,107 @@ +# Acceleration Framework {#accel_fw} + +SPDK provides a framework for abstracting general acceleration capabilities +that can be implemented through plug-in modules and low-level libraries. These +plug-in modules include support for hardware acceleration engines such as +the Intel(R) I/O Acceleration Technology (IOAT) engine and the Intel(R) Data +Streaming Accelerator (DSA) engine. Additionally, a software plug-in module +exists to enable use of the framework in environments without hardware +acceleration capabilities. ISA/L is used for optimized CRC32C calculation within +the software module. + +The framework includes an API for getting the current capabilities of the +selected module. See [`spdk_accel_get_capabilities`](https://spdk.io/doc/accel__engine_8h.html) for more details. For the software module, all capabilities will be reported as supported. For the hardware modules, only functions accelerated by hardware will be reported however any function can still be called, it will just be backed by software if it is not reported as a supported capability. + +# Acceleration Framework Functions {#accel_functions} + +Functions implemented via the framework can be found in the DoxyGen documentation of the +framework public header file here [accel_engine.h](https://spdk.io/doc/accel__engine_8h.html) + +# Acceleration Framework Design Considerations {#accel_dc} + +The general interface is defined by `/include/accel_engine.h` and implemented +in `/lib/accel`. These functions may be called by an SPDK application and in +most cases, except where otherwise documented, are asynchronous and follow the +standard SPDK model for callbacks with a callback argument. + +If the acceleration framework is started without initializing a hardware module, +optimized software implementations of the functions will back the public API. +Additionally, if any hardware module does not support a specific function and that +hardware module is initialized, the specific function will fallback to a software +optimized implementation. For example, IOAT does not support the dualcast function +in hardware but if the IOAT module has been initialized and the public dualcast API +is called, it will actually be done via software behind the scenes. + +# Acceleration Low Level Libraries {#accel_libs} + +Low level libraries provide only the most basic functions that are specific to +the hardware. Low level libraries are located in the '/lib' directory with the +exception of the software implementation which is implemented as part of the +framework itself. The software low level library does not expose a public API. +Applications may choose to interact directly with a low level library if there are +specific needs/considerations not met via accessing the library through the +framework/module. Note that when using the low level libraries directly, the +framework abstracted interface is bypassed as the application will call the public +functions exposed by the individual low level libraries. Thus, code written this +way needs to be certain that the underlying hardware exists everywhere that it runs. + +The low level library for IOAT is located in `/lib/ioat`. The low level library +for DSA is in `/liv/idxd` (IDXD stands for Intel(R) Data Acceleration Driver). + +# Acceleration Plug-In Modules {#accel_modules} + +Plug-in modules depend on low level libraries to interact with the hardware and +add additional functionality such as queueing during busy conditions or flow +control in some cases. The framework in turn depends on the modules to provide +the complete implementation of the acceleration component. A module must be +selected via startup RPC when the application is started. Otherwise, if no startup +RPC is provided, the framework is available and will use the software plug-in module. + +## IOAT Module {#accel_ioat} + +To use the IOAT engine, use the RPC [`ioat_scan_accel_engine`](https://spdk.io/doc/jsonrpc.html) before starting the application. + +## IDXD Module {#accel_idxd} + +To use the DSA engine, use the RPC [`idxd_scan_accel_engine`](https://spdk.io/doc/jsonrpc.html) with an optional parameter of `-c` and provide a configuration number of either 0 or 1. These pre-defined configurations determine how the DSA engine will be setup in terms +of work queues and engines. The DSA engine is very flexible allowing for various configurations of these elements to either account for different quality of service requirements or to isolate hardware paths where the back end media is of varying latency (i.e. persistent memory vs DRAM). The pre-defined configurations are as follows: + +0: Four separate work queues each backed with one DSA engine. This is a generic +configuration that provides 4 portals to submit operations to each with a +single engine behind it providing some level of isolation as operations are +submitted round-robin. + +1: Two separate work queues each backed with two DSA engines. This is another +generic configuration that provides 2 portals to submit operations to and +lets the DSA hardware decide which engine to select based on loading. + +There are several other configurations that are possible that include quality +of service parameters on the work queues that are not currently utilized by +the module. Specialized use of DSA may require different configurations that +can be added to the module as needed. + +## Software Module {#accel_sw} + +The software module is enabled by default. If no hardware engine is explicitly +enabled via startup RPC as discussed earlier, the software module will use ISA-L +if available for functions such as CRC32C. Otherwise, standard glibc calls are +used to back the framework API. + +## Batching {#batching} + +Batching is exposed by the acceleration framework and provides an interface to +batch sets of commands up and then submit them with a single command. The public +API is consistent with the implementation however each plug-in module behaves +differently depending on its capabilities. + +The DSA engine has complete support for batching all supported commands together +into one submission. This is advantageous as it reduces the overhead incurred in +the submission process to the hardware. + +The software engine supports batching only to be consistent with the framework API. +In software there is no savings by batching sets of commands versus submitting them +individually. + +The IOAT engine supports batching but it is only beneficial for `memmove` and `memfill` +as these are supported by the hardware. All other commands can be batched and the +framework will manage all other commands via software. diff --git a/src/spdk/doc/applications.md b/src/spdk/doc/applications.md new file mode 100644 index 000000000..f841b1c84 --- /dev/null +++ b/src/spdk/doc/applications.md @@ -0,0 +1,163 @@ + +# An Overview of SPDK Applications {#app_overview} + +SPDK is primarily a development kit that delivers libraries and header files for +use in other applications. However, SPDK also contains a number of applications. +These applications are primarily used to test the libraries, but many are full +featured and high quality. The major applications in SPDK are: + +- @ref iscsi +- @ref nvmf +- @ref vhost +- SPDK Target (a unified application combining the above three) + +There are also a number of tools and examples in the `examples` directory. + +The SPDK targets are all based on a common framework so they have much in +common. The framework defines a concept called a `subsystem` and all +functionality is implemented in various subsystems. Subsystems have a unified +initialization and teardown path. + +# Configuring SPDK Applications {#app_config} + +## Command Line Parameters {#app_cmd_line_args} + +The SPDK application framework defines a set of base command line flags for all +applications that use it. Specific applications may implement additional flags. + +Param | Long Param | Type | Default | Description +-------- | ---------------------- | -------- | ---------------------- | ----------- +-c | --config | string | | @ref cmd_arg_config_file +-d | --limit-coredump | flag | false | @ref cmd_arg_limit_coredump +-e | --tpoint-group-mask | integer | 0x0 | @ref cmd_arg_limit_tpoint_group_mask +-g | --single-file-segments | flag | | @ref cmd_arg_single_file_segments +-h | --help | flag | | show all available parameters and exit +-i | --shm-id | integer | | @ref cmd_arg_multi_process +-m | --cpumask | CPU mask | 0x1 | application @ref cpu_mask +-n | --mem-channels | integer | all channels | number of memory channels used for DPDK +-p | --master-core | integer | first core in CPU mask | master (primary) core for DPDK +-r | --rpc-socket | string | /var/tmp/spdk.sock | RPC listen address +-s | --mem-size | integer | all hugepage memory | @ref cmd_arg_memory_size +| | --silence-noticelog | flag | | disable notice level logging to `stderr` +-u | --no-pci | flag | | @ref cmd_arg_disable_pci_access. +| | --wait-for-rpc | flag | | @ref cmd_arg_deferred_initialization +-B | --pci-blacklist | B:D:F | | @ref cmd_arg_pci_blacklist_whitelist. +-W | --pci-whitelist | B:D:F | | @ref cmd_arg_pci_blacklist_whitelist. +-R | --huge-unlink | flag | | @ref cmd_arg_huge_unlink +| | --huge-dir | string | the first discovered | allocate hugepages from a specific mount +-L | --logflag | string | | @ref cmd_arg_debug_log_flags + +### Configuration file {#cmd_arg_config_file} + +Historically, the SPDK applications were configured using a configuration file. +This is still supported, but is considered deprecated in favor of JSON RPC +configuration. See @ref jsonrpc for details. + +Note that `--config` and `--wait-for-rpc` cannot be used at the same time. + +### Limit coredump {#cmd_arg_limit_coredump} + +By default, an SPDK application will set resource limits for core file sizes +to RLIM_INFINITY. Specifying `--limit-coredump` will not set the resource limits. + +### Tracepoint group mask {#cmd_arg_limit_tpoint_group_mask} + +SPDK has an experimental low overhead tracing framework. Tracepoints in this +framework are organized into tracepoint groups. By default, all tracepoint +groups are disabled. `--tpoint-group-mask` can be used to enable a specific +subset of tracepoint groups in the application. + +Note: Additional documentation on the tracepoint framework is in progress. + +### Deferred initialization {#cmd_arg_deferred_initialization} + +SPDK applications progress through a set of states beginning with `STARTUP` and +ending with `RUNTIME`. + +If the `--wait-for-rpc` parameter is provided SPDK will pause just before starting +framework initialization. This state is called `STARTUP`. The JSON RPC server is +ready but only a small subset of commands are available to set up initialization +parameters. Those parameters can't be changed after the SPDK application enters +`RUNTIME` state. When the client finishes configuring the SPDK subsystems it +needs to issue the @ref rpc_framework_start_init RPC command to begin the +initialization process. After `rpc_framework_start_init` returns `true` SPDK +will enter the `RUNTIME` state and the list of available commands becomes much +larger. + +To see which RPC methods are available in the current state, issue the +`rpc_get_methods` with the parameter `current` set to `true`. + +For more details see @ref jsonrpc documentation. + +### Create just one hugetlbfs file {#cmd_arg_single_file_segments} + +Instead of creating one hugetlbfs file per page, this option makes SPDK create +one file per hugepages per socket. This is needed for @ref virtio to be used +with more than 8 hugepages. See @ref virtio_2mb. + +### Multi process mode {#cmd_arg_multi_process} + +When `--shm-id` is specified, the application is started in multi-process mode. +Applications using the same shm-id share their memory and +[NVMe devices](@ref nvme_multi_process). The first app to start with a given id +becomes a primary process, with the rest, called secondary processes, only +attaching to it. When the primary process exits, the secondary ones continue to +operate, but no new processes can be attached at this point. All processes within +the same shm-id group must use the same +[--single-file-segments setting](@ref cmd_arg_single_file_segments). + +### Memory size {#cmd_arg_memory_size} + +Total size of the hugepage memory to reserve. If DPDK env layer is used, it will +reserve memory from all available hugetlbfs mounts, starting with the one with +the highest page size. This option accepts a number of bytes with a possible +binary prefix, e.g. 1024, 1024M, 1G. The default unit is megabyte. + +Starting with DPDK 18.05.1, it's possible to reserve hugepages at runtime, meaning +that SPDK application can be started with 0 pre-reserved memory. Unlike hugepages +pre-reserved at the application startup, the hugepages reserved at runtime will be +released to the system as soon as they're no longer used. + +### Disable PCI access {#cmd_arg_disable_pci_access} + +If SPDK is run with PCI access disabled it won't detect any PCI devices. This +includes primarily NVMe and IOAT devices. Also, the VFIO and UIO kernel modules +are not required in this mode. + +### PCI address blacklist and whitelist {#cmd_arg_pci_blacklist_whitelist} + +If blacklist is used, then all devices with the provided PCI address will be +ignored. If a whitelist is used, only whitelisted devices will be probed. +`-B` or `-W` can be used more than once, but cannot be mixed together. That is, +`-B` and `-W` cannot be used at the same time. + +### Unlink hugepage files after initialization {#cmd_arg_huge_unlink} + +By default, each DPDK-based application tries to remove any orphaned hugetlbfs +files during its initialization. This option removes hugetlbfs files of the current +process as soon as they're created, but is not compatible with `--shm-id`. + +### Debug log {#cmd_arg_debug_log_flags} + +Enable a specific debug log type. This option can be used more than once. A list of +all available types is provided in the `--help` output, with `--logflag all` +enabling all of them. Debug logs are only available in debug builds of SPDK. + +## CPU mask {#cpu_mask} + +Whenever the `CPU mask` is mentioned it is a string in one of the following formats: + +- Case insensitive hexadecimal string with or without "0x" prefix. +- Comma separated list of CPUs or list of CPU ranges. Use '-' to define range. + +### Example + +The following CPU masks are equal and correspond to CPUs 0, 1, 2, 8, 9, 10, 11 and 12: + +~~~ +0x1f07 +0x1F07 +1f07 +[0,1,2,8-12] +[0, 1, 2, 8, 9, 10, 11, 12] +~~~ diff --git a/src/spdk/doc/bdev.md b/src/spdk/doc/bdev.md new file mode 100644 index 000000000..9f842943f --- /dev/null +++ b/src/spdk/doc/bdev.md @@ -0,0 +1,602 @@ +# Block Device User Guide {#bdev} + +# Introduction {#bdev_ug_introduction} + +The SPDK block device layer, often simply called *bdev*, is a C library +intended to be equivalent to the operating system block storage layer that +often sits immediately above the device drivers in a traditional kernel +storage stack. Specifically, this library provides the following +functionality: + +* A pluggable module API for implementing block devices that interface with different types of block storage devices. +* Driver modules for NVMe, malloc (ramdisk), Linux AIO, virtio-scsi, Ceph RBD, Pmem and Vhost-SCSI Initiator and more. +* An application API for enumerating and claiming SPDK block devices and then performing operations (read, write, unmap, etc.) on those devices. +* Facilities to stack block devices to create complex I/O pipelines, including logical volume management (lvol) and partition support (GPT). +* Configuration of block devices via JSON-RPC. +* Request queueing, timeout, and reset handling. +* Multiple, lockless queues for sending I/O to block devices. + +Bdev module creates abstraction layer that provides common API for all devices. +User can use available bdev modules or create own module with any type of +device underneath (please refer to @ref bdev_module for details). SPDK +provides also vbdev modules which creates block devices on existing bdev. For +example @ref bdev_ug_logical_volumes or @ref bdev_ug_gpt + +# Prerequisites {#bdev_ug_prerequisites} + +This guide assumes that you can already build the standard SPDK distribution +on your platform. The block device layer is a C library with a single public +header file named bdev.h. All SPDK configuration described in following +chapters is done by using JSON-RPC commands. SPDK provides a python-based +command line tool for sending RPC commands located at `scripts/rpc.py`. User +can list available commands by running this script with `-h` or `--help` flag. +Additionally user can retrieve currently supported set of RPC commands +directly from SPDK application by running `scripts/rpc.py rpc_get_methods`. +Detailed help for each command can be displayed by adding `-h` flag as a +command parameter. + +# General Purpose RPCs {#bdev_ug_general_rpcs} + +## bdev_get_bdevs {#bdev_ug_get_bdevs} + +List of currently available block devices including detailed information about +them can be get by using `bdev_get_bdevs` RPC command. User can add optional +parameter `name` to get details about specified by that name bdev. + +Example response + +~~~ +{ + "num_blocks": 32768, + "assigned_rate_limits": { + "rw_ios_per_sec": 10000, + "rw_mbytes_per_sec": 20 + }, + "supported_io_types": { + "reset": true, + "nvme_admin": false, + "unmap": true, + "read": true, + "write_zeroes": true, + "write": true, + "flush": true, + "nvme_io": false + }, + "driver_specific": {}, + "claimed": false, + "block_size": 4096, + "product_name": "Malloc disk", + "name": "Malloc0" +} +~~~ + +## bdev_set_qos_limit {#bdev_set_qos_limit} + +Users can use the `bdev_set_qos_limit` RPC command to enable, adjust, and disable +rate limits on an existing bdev. Two types of rate limits are supported: +IOPS and bandwidth. The rate limits can be enabled, adjusted, and disabled at any +time for the specified bdev. The bdev name is a required parameter for this +RPC command and at least one of `rw_ios_per_sec` and `rw_mbytes_per_sec` must be +specified. When both rate limits are enabled, the first met limit will +take effect. The value 0 may be specified to disable the corresponding rate +limit. Users can run this command with `-h` or `--help` for more information. + +## Histograms {#rpc_bdev_histogram} + +The `bdev_enable_histogram` RPC command allows to enable or disable gathering +latency data for specified bdev. Histogram can be downloaded by the user by +calling `bdev_get_histogram` and parsed using scripts/histogram.py script. + +Example command + +`rpc.py bdev_enable_histogram Nvme0n1 --enable` + +The command will enable gathering data for histogram on Nvme0n1 device. + +`rpc.py bdev_get_histogram Nvme0n1 | histogram.py` + +The command will download gathered histogram data. The script will parse +the data and show table containing IO count for latency ranges. + +`rpc.py bdev_enable_histogram Nvme0n1 --disable` + +The command will disable histogram on Nvme0n1 device. + +# Ceph RBD {#bdev_config_rbd} + +The SPDK RBD bdev driver provides SPDK block layer access to Ceph RADOS block +devices (RBD). Ceph RBD devices are accessed via librbd and librados libraries +to access the RADOS block device exported by Ceph. To create Ceph bdev RPC +command `bdev_rbd_create` should be used. + +Example command + +`rpc.py bdev_rbd_create rbd foo 512` + +This command will create a bdev that represents the 'foo' image from a pool called 'rbd'. + +To remove a block device representation use the bdev_rbd_delete command. + +`rpc.py bdev_rbd_delete Rbd0` + +To resize a bdev use the bdev_rbd_resize command. + +`rpc.py bdev_rbd_resize Rbd0 4096` + +This command will resize the Rbd0 bdev to 4096 MiB. + +# Compression Virtual Bdev Module {#bdev_config_compress} + +The compression bdev module can be configured to provide compression/decompression +services for an underlying thinly provisioned logical volume. Although the underlying +module can be anything (i.e. NVME bdev) the overall compression benefits will not be realized +unless the data stored on disk is placed appropriately. The compression vbdev module +relies on an internal SPDK library called `reduce` to accomplish this, see @ref reduce +for detailed information. + +The vbdev module relies on the DPDK CompressDev Framework to provide all compression +functionality. The framework provides support for many different software only +compression modules as well as hardware assisted support for Intel QAT. At this +time the vbdev module supports the DPDK drivers for ISAL and QAT. + +Persistent memory is used to store metadata associated with the layout of the data on the +backing device. SPDK relies on [PMDK](http://pmem.io/pmdk/) to interface persistent memory so any hardware +supported by PMDK should work. If the directory for PMEM supplied upon vbdev creation does +not point to persistent memory (i.e. a regular filesystem) performance will be severely +impacted. The vbdev module and reduce libraries were designed to use persistent memory for +any production use. + +Example command + +`rpc.py bdev_compress_create -p /pmem_files -b myLvol` + +In this example, a compression vbdev is created using persistent memory that is mapped to +the directory `pmem_files` on top of the existing thinly provisioned logical volume `myLvol`. +The resulting compression bdev will be named `COMP_LVS/myLvol` where LVS is the name of the +logical volume store that `myLvol` resides on. + +The logical volume is referred to as the backing device and once the compression vbdev is +created it cannot be separated from the persistent memory file that will be created in +the specified directory. If the persistent memory file is not available, the compression +vbdev will also not be available. + +By default the vbdev module will choose the QAT driver if the hardware and drivers are +available and loaded. If not, it will revert to the software-only ISAL driver. By using +the following command, the driver may be specified however this is not persistent so it +must be done either upon creation or before the underlying logical volume is loaded to +be honored. In the example below, `0` is telling the vbdev module to use QAT if available +otherwise use ISAL, this is the default and if sufficient the command is not required. Passing +a value of 1 tells the driver to use QAT and if not available then the creation or loading +the vbdev should fail to create or load. A value of '2' as shown below tells the module +to use ISAL and if for some reason it is not available, the vbdev should fail to create or load. + +`rpc.py compress_set_pmd -p 2` + +To remove a compression vbdev, use the following command which will also delete the PMEM +file. If the logical volume is deleted the PMEM file will not be removed and the +compression vbdev will not be available. + +`rpc.py bdev_compress_delete COMP_LVS/myLvol` + +To list compression volumes that are only available for deletion because their PMEM file +was missing use the following. The name parameter is optional and if not included will list +all volumes, if used it will return the name or an error that the device does not exist. + +`rpc.py bdev_compress_get_orphans --name COMP_Nvme0n1` + +# Crypto Virtual Bdev Module {#bdev_config_crypto} + +The crypto virtual bdev module can be configured to provide at rest data encryption +for any underlying bdev. The module relies on the DPDK CryptoDev Framework to provide +all cryptographic functionality. The framework provides support for many different software +only cryptographic modules as well hardware assisted support for the Intel QAT board. The +framework also provides support for cipher, hash, authentication and AEAD functions. At this +time the SPDK virtual bdev module supports cipher only as follows: + +- AESN-NI Multi Buffer Crypto Poll Mode Driver: RTE_CRYPTO_CIPHER_AES128_CBC +- Intel(R) QuickAssist (QAT) Crypto Poll Mode Driver: RTE_CRYPTO_CIPHER_AES128_CBC + (Note: QAT is functional however is marked as experimental until the hardware has + been fully integrated with the SPDK CI system.) + +In order to support using the bdev block offset (LBA) as the initialization vector (IV), +the crypto module break up all I/O into crypto operations of a size equal to the block +size of the underlying bdev. For example, a 4K I/O to a bdev with a 512B block size, +would result in 8 cryptographic operations. + +For reads, the buffer provided to the crypto module will be used as the destination buffer +for unencrypted data. For writes, however, a temporary scratch buffer is used as the +destination buffer for encryption which is then passed on to the underlying bdev as the +write buffer. This is done to avoid encrypting the data in the original source buffer which +may cause problems in some use cases. + +Example command + +`rpc.py bdev_crypto_create NVMe1n1 CryNvmeA crypto_aesni_mb 0123456789123456` + +This command will create a crypto vbdev called 'CryNvmeA' on top of the NVMe bdev +'NVMe1n1' and will use the DPDK software driver 'crypto_aesni_mb' and the key +'0123456789123456'. + +To remove the vbdev use the bdev_crypto_delete command. + +`rpc.py bdev_crypto_delete CryNvmeA` + +# Delay Bdev Module {#bdev_config_delay} + +The delay vbdev module is intended to apply a predetermined additional latency on top of a lower +level bdev. This enables the simulation of the latency characteristics of a device during the functional +or scalability testing of an SPDK application. For example, to simulate the effect of drive latency when +processing I/Os, one could configure a NULL bdev with a delay bdev on top of it. + +The delay bdev module is not intended to provide a high fidelity replication of a specific NVMe drive's latency, +instead it's main purpose is to provide a "big picture" understanding of how a generic latency affects a given +application. + +A delay bdev is created using the `bdev_delay_create` RPC. This rpc takes 6 arguments, one for the name +of the delay bdev and one for the name of the base bdev. The remaining four arguments represent the following +latency values: average read latency, average write latency, p99 read latency, and p99 write latency. +Within the context of the delay bdev p99 latency means that one percent of the I/O will be delayed by at +least by the value of the p99 latency before being completed to the upper level protocol. All of the latency values +are measured in microseconds. + +Example command: + +`rpc.py bdev_delay_create -b Null0 -d delay0 -r 10 --nine-nine-read-latency 50 -w 30 --nine-nine-write-latency 90` + +This command will create a delay bdev with average read and write latencies of 10 and 30 microseconds and p99 read +and write latencies of 50 and 90 microseconds respectively. + +A delay bdev can be deleted using the `bdev_delay_delete` RPC + +Example command: + +`rpc.py bdev_delay_delete delay0` + +# GPT (GUID Partition Table) {#bdev_config_gpt} + +The GPT virtual bdev driver is enabled by default and does not require any configuration. +It will automatically detect @ref bdev_ug_gpt on any attached bdev and will create +possibly multiple virtual bdevs. + +## SPDK GPT partition table {#bdev_ug_gpt} + +The SPDK partition type GUID is `7c5222bd-8f5d-4087-9c00-bf9843c7b58c`. Existing SPDK bdevs +can be exposed as Linux block devices via NBD and then can be partitioned with +standard partitioning tools. After partitioning, the bdevs will need to be deleted and +attached again for the GPT bdev module to see any changes. NBD kernel module must be +loaded first. To create NBD bdev user should use `nbd_start_disk` RPC command. + +Example command + +`rpc.py nbd_start_disk Malloc0 /dev/nbd0` + +This will expose an SPDK bdev `Malloc0` under the `/dev/nbd0` block device. + +To remove NBD device user should use `nbd_stop_disk` RPC command. + +Example command + +`rpc.py nbd_stop_disk /dev/nbd0` + +To display full or specified nbd device list user should use `nbd_get_disks` RPC command. + +Example command + +`rpc.py nbd_stop_disk -n /dev/nbd0` + +## Creating a GPT partition table using NBD {#bdev_ug_gpt_create_part} + +~~~ +# Expose bdev Nvme0n1 as kernel block device /dev/nbd0 by JSON-RPC +rpc.py nbd_start_disk Nvme0n1 /dev/nbd0 + +# Create GPT partition table. +parted -s /dev/nbd0 mklabel gpt + +# Add a partition consuming 50% of the available space. +parted -s /dev/nbd0 mkpart MyPartition '0%' '50%' + +# Change the partition type to the SPDK GUID. +# sgdisk is part of the gdisk package. +sgdisk -t 1:7c5222bd-8f5d-4087-9c00-bf9843c7b58c /dev/nbd0 + +# Stop the NBD device (stop exporting /dev/nbd0). +rpc.py nbd_stop_disk /dev/nbd0 + +# Now Nvme0n1 is configured with a GPT partition table, and +# the first partition will be automatically exposed as +# Nvme0n1p1 in SPDK applications. +~~~ + +# iSCSI bdev {#bdev_config_iscsi} + +The SPDK iSCSI bdev driver depends on libiscsi and hence is not enabled by default. +In order to use it, build SPDK with an extra `--with-iscsi-initiator` configure option. + +The following command creates an `iSCSI0` bdev from a single LUN exposed at given iSCSI URL +with `iqn.2016-06.io.spdk:init` as the reported initiator IQN. + +`rpc.py bdev_iscsi_create -b iSCSI0 -i iqn.2016-06.io.spdk:init --url iscsi://127.0.0.1/iqn.2016-06.io.spdk:disk1/0` + +The URL is in the following format: +`iscsi://[<username>[%<password>]@]<host>[:<port>]/<target-iqn>/<lun>` + +# Linux AIO bdev {#bdev_config_aio} + +The SPDK AIO bdev driver provides SPDK block layer access to Linux kernel block +devices or a file on a Linux filesystem via Linux AIO. Note that O_DIRECT is +used and thus bypasses the Linux page cache. This mode is probably as close to +a typical kernel based target as a user space target can get without using a +user-space driver. To create AIO bdev RPC command `bdev_aio_create` should be +used. + +Example commands + +`rpc.py bdev_aio_create /dev/sda aio0` + +This command will create `aio0` device from /dev/sda. + +`rpc.py bdev_aio_create /tmp/file file 4096` + +This command will create `file` device with block size 4096 from /tmp/file. + +To delete an aio bdev use the bdev_aio_delete command. + +`rpc.py bdev_aio_delete aio0` + +# OCF Virtual bdev {#bdev_config_cas} + +OCF virtual bdev module is based on [Open CAS Framework](https://github.com/Open-CAS/ocf) - a +high performance block storage caching meta-library. +To enable the module, configure SPDK using `--with-ocf` flag. +OCF bdev can be used to enable caching for any underlying bdev. + +Below is an example command for creating OCF bdev: + +`rpc.py bdev_ocf_create Cache1 wt Malloc0 Nvme0n1` + +This command will create new OCF bdev `Cache1` having bdev `Malloc0` as caching-device +and `Nvme0n1` as core-device and initial cache mode `Write-Through`. +`Malloc0` will be used as cache for `Nvme0n1`, so data written to `Cache1` will be present +on `Nvme0n1` eventually. +By default, OCF will be configured with cache line size equal 4KiB +and non-volatile metadata will be disabled. + +To remove `Cache1`: + +`rpc.py bdev_ocf_delete Cache1` + +During removal OCF-cache will be stopped and all cached data will be written to the core device. + +Note that OCF has a per-device RAM requirement +of about 56000 + _cache device size_ * 58 / _cache line size_ (in bytes). +To get more information on OCF +please visit [OCF documentation](https://open-cas.github.io/). + +# Malloc bdev {#bdev_config_malloc} + +Malloc bdevs are ramdisks. Because of its nature they are volatile. They are created from hugepage memory given to SPDK +application. + +# Null {#bdev_config_null} + +The SPDK null bdev driver is a dummy block I/O target that discards all writes and returns undefined +data for reads. It is useful for benchmarking the rest of the bdev I/O stack with minimal block +device overhead and for testing configurations that can't easily be created with the Malloc bdev. +To create Null bdev RPC command `bdev_null_create` should be used. + +Example command + +`rpc.py bdev_null_create Null0 8589934592 4096` + +This command will create an 8 petabyte `Null0` device with block size 4096. + +To delete a null bdev use the bdev_null_delete command. + +`rpc.py bdev_null_delete Null0` + +# NVMe bdev {#bdev_config_nvme} + +There are two ways to create block device based on NVMe device in SPDK. First +way is to connect local PCIe drive and second one is to connect NVMe-oF device. +In both cases user should use `bdev_nvme_attach_controller` RPC command to achieve that. + +Example commands + +`rpc.py bdev_nvme_attach_controller -b NVMe1 -t PCIe -a 0000:01:00.0` + +This command will create NVMe bdev of physical device in the system. + +`rpc.py bdev_nvme_attach_controller -b Nvme0 -t RDMA -a 192.168.100.1 -f IPv4 -s 4420 -n nqn.2016-06.io.spdk:cnode1` + +This command will create NVMe bdev of NVMe-oF resource. + +To remove an NVMe controller use the bdev_nvme_detach_controller command. + +`rpc.py bdev_nvme_detach_controller Nvme0` + +This command will remove NVMe bdev named Nvme0. + +## NVMe bdev character device {#bdev_config_nvme_cuse} + +This feature is considered as experimental. + +Example commands + +`rpc.py bdev_nvme_cuse_register -n Nvme0 -p spdk/nvme0` + +This command will register /dev/spdk/nvme0 character device associated with Nvme0 +controller. If there are namespaces created on Nvme0 controller, for each namespace +device /dev/spdk/nvme0nX is created. + +Cuse devices are removed from system, when NVMe controller is detached or unregistered +with command: + +`rpc.py bdev_nvme_cuse_unregister -n Nvme0` + +# Logical volumes {#bdev_ug_logical_volumes} + +The Logical Volumes library is a flexible storage space management system. It allows +creating and managing virtual block devices with variable size on top of other bdevs. +The SPDK Logical Volume library is built on top of @ref blob. For detailed description +please refer to @ref lvol. + +## Logical volume store {#bdev_ug_lvol_store} + +Before creating any logical volumes (lvols), an lvol store has to be created first on +selected block device. Lvol store is lvols vessel responsible for managing underlying +bdev space assignment to lvol bdevs and storing metadata. To create lvol store user +should use using `bdev_lvol_create_lvstore` RPC command. + +Example command + +`rpc.py bdev_lvol_create_lvstore Malloc2 lvs -c 4096` + +This will create lvol store named `lvs` with cluster size 4096, build on top of +`Malloc2` bdev. In response user will be provided with uuid which is unique lvol store +identifier. + +User can get list of available lvol stores using `bdev_lvol_get_lvstores` RPC command (no +parameters available). + +Example response + +~~~ +{ + "uuid": "330a6ab2-f468-11e7-983e-001e67edf35d", + "base_bdev": "Malloc2", + "free_clusters": 8190, + "cluster_size": 8192, + "total_data_clusters": 8190, + "block_size": 4096, + "name": "lvs" +} +~~~ + +To delete lvol store user should use `bdev_lvol_delete_lvstore` RPC command. + +Example commands + +`rpc.py bdev_lvol_delete_lvstore -u 330a6ab2-f468-11e7-983e-001e67edf35d` + +`rpc.py bdev_lvol_delete_lvstore -l lvs` + +## Lvols {#bdev_ug_lvols} + +To create lvols on existing lvol store user should use `bdev_lvol_create` RPC command. +Each created lvol will be represented by new bdev. + +Example commands + +`rpc.py bdev_lvol_create lvol1 25 -l lvs` + +`rpc.py bdev_lvol_create lvol2 25 -u 330a6ab2-f468-11e7-983e-001e67edf35d` + +# RAID {#bdev_ug_raid} + +RAID virtual bdev module provides functionality to combine any SPDK bdevs into +one RAID bdev. Currently SPDK supports only RAID 0. RAID functionality does not +store on-disk metadata on the member disks, so user must recreate the RAID +volume when restarting application. User may specify member disks to create RAID +volume event if they do not exists yet - as the member disks are registered at +a later time, the RAID module will claim them and will surface the RAID volume +after all of the member disks are available. It is allowed to use disks of +different sizes - the smallest disk size will be the amount of space used on +each member disk. + +Example commands + +`rpc.py bdev_raid_create -n Raid0 -z 64 -r 0 -b "lvol0 lvol1 lvol2 lvol3"` + +`rpc.py bdev_raid_get_bdevs` + +`rpc.py bdev_raid_delete Raid0` + +# Passthru {#bdev_config_passthru} + +The SPDK Passthru virtual block device module serves as an example of how to write a +virtual block device module. It implements the required functionality of a vbdev module +and demonstrates some other basic features such as the use of per I/O context. + +Example commands + +`rpc.py bdev_passthru_create -b aio -p pt` + +`rpc.py bdev_passthru_delete pt` + +# Pmem {#bdev_config_pmem} + +The SPDK pmem bdev driver uses pmemblk pool as the target for block I/O operations. For +details on Pmem memory please refer to PMDK documentation on http://pmem.io website. +First, user needs to configure SPDK to include PMDK support: + +`configure --with-pmdk` + +To create pmemblk pool for use with SPDK user should use `bdev_pmem_create_pool` RPC command. + +Example command + +`rpc.py bdev_pmem_create_pool /path/to/pmem_pool 25 4096` + +To get information on created pmem pool file user can use `bdev_pmem_get_pool_info` RPC command. + +Example command + +`rpc.py bdev_pmem_get_pool_info /path/to/pmem_pool` + +To remove pmem pool file user can use `bdev_pmem_delete_pool` RPC command. + +Example command + +`rpc.py bdev_pmem_delete_pool /path/to/pmem_pool` + +To create bdev based on pmemblk pool file user should use `bdev_pmem_create ` RPC +command. + +Example command + +`rpc.py bdev_pmem_create /path/to/pmem_pool -n pmem` + +To remove a block device representation use the bdev_pmem_delete command. + +`rpc.py bdev_pmem_delete pmem` + +# Virtio Block {#bdev_config_virtio_blk} + +The Virtio-Block driver allows creating SPDK bdevs from Virtio-Block devices. + +The following command creates a Virtio-Block device named `VirtioBlk0` from a vhost-user +socket `/tmp/vhost.0` exposed directly by SPDK @ref vhost. Optional `vq-count` and +`vq-size` params specify number of request queues and queue depth to be used. + +`rpc.py bdev_virtio_attach_controller --dev-type blk --trtype user --traddr /tmp/vhost.0 --vq-count 2 --vq-size 512 VirtioBlk0` + +The driver can be also used inside QEMU-based VMs. The following command creates a Virtio +Block device named `VirtioBlk0` from a Virtio PCI device at address `0000:00:01.0`. +The entire configuration will be read automatically from PCI Configuration Space. It will +reflect all parameters passed to QEMU's vhost-user-scsi-pci device. + +`rpc.py bdev_virtio_attach_controller --dev-type blk --trtype pci --traddr 0000:01:00.0 VirtioBlk1` + +Virtio-Block devices can be removed with the following command + +`rpc.py bdev_virtio_detach_controller VirtioBlk0` + +# Virtio SCSI {#bdev_config_virtio_scsi} + +The Virtio-SCSI driver allows creating SPDK block devices from Virtio-SCSI LUNs. + +Virtio-SCSI bdevs are created the same way as Virtio-Block ones. + +`rpc.py bdev_virtio_attach_controller --dev-type scsi --trtype user --traddr /tmp/vhost.0 --vq-count 2 --vq-size 512 VirtioScsi0` + +`rpc.py bdev_virtio_attach_controller --dev-type scsi --trtype pci --traddr 0000:01:00.0 VirtioScsi0` + +Each Virtio-SCSI device may export up to 64 block devices named VirtioScsi0t0 ~ VirtioScsi0t63, +one LUN (LUN0) per SCSI device. The above 2 commands will output names of all exposed bdevs. + +Virtio-SCSI devices can be removed with the following command + +`rpc.py bdev_virtio_detach_controller VirtioScsi0` + +Removing a Virtio-SCSI device will destroy all its bdevs. diff --git a/src/spdk/doc/bdev_module.md b/src/spdk/doc/bdev_module.md new file mode 100644 index 000000000..f6c5392cc --- /dev/null +++ b/src/spdk/doc/bdev_module.md @@ -0,0 +1,154 @@ +# Writing a Custom Block Device Module {#bdev_module} + +## Target Audience + +This programming guide is intended for developers authoring their own block +device modules to integrate with SPDK's bdev layer. For a guide on how to use +the bdev layer, see @ref bdev_pg. + +## Introduction + +A block device module is SPDK's equivalent of a device driver in a traditional +operating system. The module provides a set of function pointers that are +called to service block device I/O requests. SPDK provides a number of block +device modules including NVMe, RAM-disk, and Ceph RBD. However, some users +will want to write their own to interact with either custom hardware or to an +existing storage software stack. This guide is intended to demonstrate exactly +how to write a module. + +## Creating A New Module + +Block device modules are located in subdirectories under lib/bdev today. It is not +currently possible to place the code for a bdev module elsewhere, but updates +to the build system could be made to enable this in the future. To create a +module, add a new directory with a single C file and a Makefile. A great +starting point is to copy the existing 'null' bdev module. + +The primary interface that bdev modules will interact with is in +include/spdk/bdev_module.h. In that header a macro is defined that registers +a new bdev module - SPDK_BDEV_MODULE_REGISTER. This macro take as argument a +pointer spdk_bdev_module structure that is used to register new bdev module. + +The spdk_bdev_module structure describes the module properties like +initialization (`module_init`) and teardown (`module_fini`) functions, +the function that returns context size (`get_ctx_size`) - scratch space that +will be allocated in each I/O request for use by this module, and a callback +that will be called each time a new bdev is registered by another module +(`examine_config` and `examine_disk`). Please check the documentation of +struct spdk_bdev_module for more details. + +## Creating Bdevs + +New bdevs are created within the module by calling spdk_bdev_register(). The +module must allocate a struct spdk_bdev, fill it out appropriately, and pass +it to the register call. The most important field to fill out is `fn_table`, +which points at this data structure: + +~~~{.c} +/* + * Function table for a block device backend. + * + * The backend block device function table provides a set of APIs to allow + * communication with a backend. The main commands are read/write API + * calls for I/O via submit_request. + */ +struct spdk_bdev_fn_table { + /* Destroy the backend block device object */ + int (*destruct)(void *ctx); + + /* Process the IO. */ + void (*submit_request)(struct spdk_io_channel *ch, struct spdk_bdev_io *); + + /* Check if the block device supports a specific I/O type. */ + bool (*io_type_supported)(void *ctx, enum spdk_bdev_io_type); + + /* Get an I/O channel for the specific bdev for the calling thread. */ + struct spdk_io_channel *(*get_io_channel)(void *ctx); + + /* + * Output driver-specific configuration to a JSON stream. Optional - may be NULL. + * + * The JSON write context will be initialized with an open object, so the bdev + * driver should write a name (based on the driver name) followed by a JSON value + * (most likely another nested object). + */ + int (*dump_config_json)(void *ctx, struct spdk_json_write_ctx *w); + + /* Get spin-time per I/O channel in microseconds. + * Optional - may be NULL. + */ + uint64_t (*get_spin_time)(struct spdk_io_channel *ch); +}; +~~~ + +The bdev module must implement these function callbacks. + +The `destruct` function is called to tear down the device when the system no +longer needs it. What `destruct` does is up to the module - it may just be +freeing memory or it may be shutting down a piece of hardware. + +The `io_type_supported` function returns whether a particular I/O type is +supported. The available I/O types are: + +~~~{.c} +/** bdev I/O type */ +enum spdk_bdev_io_type { + SPDK_BDEV_IO_TYPE_INVALID = 0, + SPDK_BDEV_IO_TYPE_READ, + SPDK_BDEV_IO_TYPE_WRITE, + SPDK_BDEV_IO_TYPE_UNMAP, + SPDK_BDEV_IO_TYPE_FLUSH, + SPDK_BDEV_IO_TYPE_RESET, + SPDK_BDEV_IO_TYPE_NVME_ADMIN, + SPDK_BDEV_IO_TYPE_NVME_IO, + SPDK_BDEV_IO_TYPE_NVME_IO_MD, + SPDK_BDEV_IO_TYPE_WRITE_ZEROES, +}; +~~~ + +For the simplest bdev modules, only `SPDK_BDEV_IO_TYPE_READ` and +`SPDK_BDEV_IO_TYPE_WRITE` are necessary. `SPDK_BDEV_IO_TYPE_UNMAP` is often +referred to as "trim" or "deallocate", and is a request to mark a set of +blocks as no longer containing valid data. `SPDK_BDEV_IO_TYPE_FLUSH` is a +request to make all previously completed writes durable. Many devices do not +require flushes. `SPDK_BDEV_IO_TYPE_WRITE_ZEROES` is just like a regular +write, but does not provide a data buffer (it would have just contained all +0's). If it isn't supported, the generic bdev code is capable of emulating it +by sending regular write requests. + +`SPDK_BDEV_IO_TYPE_RESET` is a request to abort all I/O and return the +underlying device to its initial state. Do not complete the reset request +until all I/O has been completed in some way. + +`SPDK_BDEV_IO_TYPE_NVME_ADMIN`, `SPDK_BDEV_IO_TYPE_NVME_IO`, and +`SPDK_BDEV_IO_TYPE_NVME_IO_MD` are all mechanisms for passing raw NVMe +commands through the SPDK bdev layer. They're strictly optional, and it +probably only makes sense to implement those if the backing storage device is +capable of handling NVMe commands. + +The `get_io_channel` function should return an I/O channel. For a detailed +explanation of I/O channels, see @ref concurrency. The generic bdev layer will +call `get_io_channel` one time per thread, cache the result, and pass that +result to `submit_request`. It will use the corresponding channel for the +thread it calls `submit_request` on. + +The `submit_request` function is called to actually submit I/O requests to the +block device. Once the I/O request is completed, the module must call +spdk_bdev_io_complete(). The I/O does not have to finish within the calling +context of `submit_request`. + +## Creating Virtual Bdevs + +Block devices are considered virtual if they handle I/O requests by routing +the I/O to other block devices. The canonical example would be a bdev module +that implements RAID. Virtual bdevs are created in the same way as regular +bdevs, but take one additional step. The module can look up the underlying +bdevs it wishes to route I/O to using spdk_bdev_get_by_name(), where the string +name is provided by the user in a configuration file or via an RPC. The module +then may proceed is normal by opening the bdev to obtain a descriptor, and +creating I/O channels for the bdev (probably in response to the +`get_io_channel` callback). The final step is to have the module use its open +descriptor to call spdk_bdev_module_claim_bdev(), indicating that it is +consuming the underlying bdev. This prevents other users from opening +descriptors with write permissions. This effectively 'promotes' the descriptor +to write-exclusive and is an operation only available to bdev modules. diff --git a/src/spdk/doc/bdev_pg.md b/src/spdk/doc/bdev_pg.md new file mode 100644 index 000000000..67261c2c6 --- /dev/null +++ b/src/spdk/doc/bdev_pg.md @@ -0,0 +1,147 @@ +# Block Device Layer Programming Guide {#bdev_pg} + +## Target Audience + +This programming guide is intended for developers authoring applications that +use the SPDK bdev library to access block devices. + +## Introduction + +A block device is a storage device that supports reading and writing data in +fixed-size blocks. These blocks are usually 512 or 4096 bytes. The +devices may be logical constructs in software or correspond to physical +devices like NVMe SSDs. + +The block device layer consists of a single generic library in `lib/bdev`, +plus a number of optional modules (as separate libraries) that implement +various types of block devices. The public header file for the generic library +is bdev.h, which is the entirety of the API needed to interact with any type +of block device. This guide will cover how to interact with bdevs using that +API. For a guide to implementing a bdev module, see @ref bdev_module. + +The bdev layer provides a number of useful features in addition to providing a +common abstraction for all block devices: + +- Automatic queueing of I/O requests in response to queue full or out-of-memory conditions +- Hot remove support, even while I/O traffic is occurring. +- I/O statistics such as bandwidth and latency +- Device reset support and I/O timeout tracking + +## Basic Primitives + +Users of the bdev API interact with a number of basic objects. + +struct spdk_bdev, which this guide will refer to as a *bdev*, represents a +generic block device. struct spdk_bdev_desc, heretofore called a *descriptor*, +represents a handle to a given block device. Descriptors are used to establish +and track permissions to use the underlying block device, much like a file +descriptor on UNIX systems. Requests to the block device are asynchronous and +represented by spdk_bdev_io objects. Requests must be submitted on an +associated I/O channel. The motivation and design of I/O channels is described +in @ref concurrency. + +Bdevs can be layered, such that some bdevs service I/O by routing requests to +other bdevs. This can be used to implement caching, RAID, logical volume +management, and more. Bdevs that route I/O to other bdevs are often referred +to as virtual bdevs, or *vbdevs* for short. + +## Initializing The Library + +The bdev layer depends on the generic message passing infrastructure +abstracted by the header file include/spdk/thread.h. See @ref concurrency for a +full description. Most importantly, calls into the bdev library may only be +made from threads that have been allocated with SPDK by calling +spdk_thread_create(). + +From an allocated thread, the bdev library may be initialized by calling +spdk_bdev_initialize(), which is an asynchronous operation. Until the completion +callback is called, no other bdev library functions may be invoked. Similarly, +to tear down the bdev library, call spdk_bdev_finish(). + +## Discovering Block Devices + +All block devices have a simple string name. At any time, a pointer to the +device object can be obtained by calling spdk_bdev_get_by_name(), or the entire +set of bdevs may be iterated using spdk_bdev_first() and spdk_bdev_next() and +their variants. + +Some block devices may also be given aliases, which are also string names. +Aliases behave like symlinks - they can be used interchangeably with the real +name to look up the block device. + +## Preparing To Use A Block Device + +In order to send I/O requests to a block device, it must first be opened by +calling spdk_bdev_open_ext(). This will return a descriptor. Multiple users may have +a bdev open at the same time, and coordination of reads and writes between +users must be handled by some higher level mechanism outside of the bdev +layer. Opening a bdev with write permission may fail if a virtual bdev module +has *claimed* the bdev. Virtual bdev modules implement logic like RAID or +logical volume management and forward their I/O to lower level bdevs, so they +mark these lower level bdevs as claimed to prevent outside users from issuing +writes. + +When a block device is opened, a callback and context must be provided that +will be called with appropriate spdk_bdev_event_type enum as an argument when +the bdev triggers asynchronous event such as bdev removal. For example, +the callback will be called on each open descriptor for a bdev backed by +a physical NVMe SSD when the NVMe SSD is hot-unplugged. In this case +the callback can be thought of as a request to close the open descriptor so +other memory may be freed. A bdev cannot be torn down while open descriptors +exist, so it is required that a callback is provided. + +When a user is done with a descriptor, they may release it by calling +spdk_bdev_close(). + +Descriptors may be passed to and used from multiple threads simultaneously. +However, for each thread a separate I/O channel must be obtained by calling +spdk_bdev_get_io_channel(). This will allocate the necessary per-thread +resources to submit I/O requests to the bdev without taking locks. To release +a channel, call spdk_put_io_channel(). A descriptor cannot be closed until +all associated channels have been destroyed. + +## Sending I/O + +Once a descriptor and a channel have been obtained, I/O may be sent by calling +the various I/O submission functions such as spdk_bdev_read(). These calls each +take a callback as an argument which will be called some time later with a +handle to an spdk_bdev_io object. In response to that completion, the user +must call spdk_bdev_free_io() to release the resources. Within this callback, +the user may also use the functions spdk_bdev_io_get_nvme_status() and +spdk_bdev_io_get_scsi_status() to obtain error information in the format of +their choosing. + +I/O submission is performed by calling functions such as spdk_bdev_read() or +spdk_bdev_write(). These functions take as an argument a pointer to a region of +memory or a scatter gather list describing memory that will be transferred to +the block device. This memory must be allocated through spdk_dma_malloc() or +its variants. For a full explanation of why the memory must come from a +special allocation pool, see @ref memory. Where possible, data in memory will +be *directly transferred to the block device* using +[Direct Memory Access](https://en.wikipedia.org/wiki/Direct_memory_access). +That means it is not copied. + +All I/O submission functions are asynchronous and non-blocking. They will not +block or stall the thread for any reason. However, the I/O submission +functions may fail in one of two ways. First, they may fail immediately and +return an error code. In that case, the provided callback will not be called. +Second, they may fail asynchronously. In that case, the associated +spdk_bdev_io will be passed to the callback and it will report error +information. + +Some I/O request types are optional and may not be supported by a given bdev. +To query a bdev for the I/O request types it supports, call +spdk_bdev_io_type_supported(). + +## Resetting A Block Device + +In order to handle unexpected failure conditions, the bdev library provides a +mechanism to perform a device reset by calling spdk_bdev_reset(). This will pass +a message to every other thread for which an I/O channel exists for the bdev, +pause it, then forward a reset request to the underlying bdev module and wait +for completion. Upon completion, the I/O channels will resume and the reset +will complete. The specific behavior inside the bdev module is +module-specific. For example, NVMe devices will delete all queue pairs, +perform an NVMe reset, then recreate the queue pairs and continue. Most +importantly, regardless of device type, *all I/O outstanding to the block +device will be completed prior to the reset completing.* diff --git a/src/spdk/doc/bdevperf.md b/src/spdk/doc/bdevperf.md new file mode 100644 index 000000000..8c5c5828c --- /dev/null +++ b/src/spdk/doc/bdevperf.md @@ -0,0 +1,86 @@ +# Using bdevperf application {#bdevperf} + +## Introduction + +bdevperf is an SPDK application that is used for performance testing +of block devices (bdevs) exposed by the SPDK bdev layer. It is an +alternative to the SPDK bdev fio plugin for benchmarking SPDK bdevs. +In some cases, bdevperf can provide much lower overhead than the fio +plugin, resulting in much better performance for tests using a limited +number of CPU cores. + +bdevperf exposes command line interface that allows to specify +SPDK framework options as well as testing options. +Since SPDK 20.07, bdevperf supports configuration file that is similar +to FIO. It allows user to create jobs parameterized by +filename, cpumask, blocksize, queuesize, etc. + +## Config file + +Bdevperf's config file is similar to FIO's config file format. + +Below is an example config file that uses all available parameters: + +~~~{.ini} +[global] +filename=Malloc0:Malloc1 +bs=1024 +iosize=256 +rw=randrw +rwmixread=90 + +[A] +cpumask=0xff + +[B] +cpumask=[0-128] +filename=Malloc1 + +[global] +filename=Malloc0 +rw=write + +[C] +bs=4096 +iosize=128 +offset=1000000 +length=1000000 +~~~ + +Jobs `[A]` `[B]` or `[C]`, inherit default values from `[global]` +section residing above them. So in the example, job `[A]` inherits +`filename` value and uses both `Malloc0` and `Malloc1` bdevs as targets, +job `[B]` overrides its `filename` value and uses `Malloc1` and +job `[C]` inherits value `Malloc0` for its `filename`. + +Interaction with CLI arguments is not the same as in FIO however. +If bdevperf receives CLI argument, it overrides values +of corresponding parameter for all `[global]` sections of config file. +So if example config is used, specifying `-q` argument +will make jobs `[A]` and `[B]` use its value. + +Below is a full list of supported parameters with descriptions. + +Param | Default | Description +--------- | ----------------- | ----------- +filename | | Bdevs to use, separated by ":" +cpumask | Maximum available | CPU mask. Format is defined at @ref cpu_mask +bs | | Block size (io size) +iodepth | | Queue depth +rwmixread | `50` | Percentage of a mixed workload that should be reads +offset | `0` | Start I/O at the provided offset on the bdev +length | 100% of bdev size | End I/O at `offset`+`length` on the bdev +rw | | Type of I/O pattern + +Available rw types: +- read +- randread +- write +- randwrite +- verify +- reset +- unmap +- write_zeroes +- flush +- rw +- randrw diff --git a/src/spdk/doc/blob.md b/src/spdk/doc/blob.md new file mode 100644 index 000000000..208ecc092 --- /dev/null +++ b/src/spdk/doc/blob.md @@ -0,0 +1,399 @@ +# Blobstore Programmer's Guide {#blob} + +# In this document {#blob_pg_toc} + +* @ref blob_pg_audience +* @ref blob_pg_intro +* @ref blob_pg_theory +* @ref blob_pg_design +* @ref blob_pg_examples +* @ref blob_pg_config +* @ref blob_pg_component + +## Target Audience {#blob_pg_audience} + +The programmer's guide is intended for developers authoring applications that utilize the SPDK Blobstore. It is +intended to supplement the source code in providing an overall understanding of how to integrate Blobstore into +an application as well as provide some high level insight into how Blobstore works behind the scenes. It is not +intended to serve as a design document or an API reference and in some cases source code snippets and high level +sequences will be discussed; for the latest source code reference refer to the [repo](https://github.com/spdk). + +## Introduction {#blob_pg_intro} + +Blobstore is a persistent, power-fail safe block allocator designed to be used as the local storage system +backing a higher level storage service, typically in lieu of a traditional filesystem. These higher level services +can be local databases or key/value stores (MySQL, RocksDB), they can be dedicated appliances (SAN, NAS), or +distributed storage systems (ex. Ceph, Cassandra). It is not designed to be a general purpose filesystem, however, +and it is intentionally not POSIX compliant. To avoid confusion, we avoid references to files or objects instead +using the term 'blob'. The Blobstore is designed to allow asynchronous, uncached, parallel reads and writes to +groups of blocks on a block device called 'blobs'. Blobs are typically large, measured in at least hundreds of +kilobytes, and are always a multiple of the underlying block size. + +The Blobstore is designed primarily to run on "next generation" media, which means the device supports fast random +reads and writes, with no required background garbage collection. However, in practice the design will run well on +NAND too. + +## Theory of Operation {#blob_pg_theory} + +### Abstractions + +The Blobstore defines a hierarchy of storage abstractions as follows. + +* **Logical Block**: Logical blocks are exposed by the disk itself, which are numbered from 0 to N, where N is the + number of blocks in the disk. A logical block is typically either 512B or 4KiB. +* **Page**: A page is defined to be a fixed number of logical blocks defined at Blobstore creation time. The logical + blocks that compose a page are always contiguous. Pages are also numbered from the beginning of the disk such + that the first page worth of blocks is page 0, the second page is page 1, etc. A page is typically 4KiB in size, + so this is either 8 or 1 logical blocks in practice. The SSD must be able to perform atomic reads and writes of + at least the page size. +* **Cluster**: A cluster is a fixed number of pages defined at Blobstore creation time. The pages that compose a cluster + are always contiguous. Clusters are also numbered from the beginning of the disk, where cluster 0 is the first cluster + worth of pages, cluster 1 is the second grouping of pages, etc. A cluster is typically 1MiB in size, or 256 pages. +* **Blob**: A blob is an ordered list of clusters. Blobs are manipulated (created, sized, deleted, etc.) by the application + and persist across power failures and reboots. Applications use a Blobstore provided identifier to access a particular blob. + Blobs are read and written in units of pages by specifying an offset from the start of the blob. Applications can also + store metadata in the form of key/value pairs with each blob which we'll refer to as xattrs (extended attributes). +* **Blobstore**: An SSD which has been initialized by a Blobstore-based application is referred to as "a Blobstore." A + Blobstore owns the entire underlying device which is made up of a private Blobstore metadata region and the collection of + blobs as managed by the application. + +@htmlonly + + <div id="blob_hierarchy"></div> + + <script> + let elem = document.getElementById('blob_hierarchy'); + + let canvasWidth = 800; + let canvasHeight = 200; + var two = new Two({ width: 800, height: 200 }).appendTo(elem); + + var blobRect = two.makeRectangle(canvasWidth / 2, canvasHeight / 2, canvasWidth, canvasWidth); + blobRect.fill = '#7ED3F7'; + + var blobText = two.makeText('Blob', canvasWidth / 2, 10, { alignment: 'center'}); + + for (var i = 0; i < 2; i++) { + let clusterWidth = 400; + let clusterHeight = canvasHeight; + var clusterRect = two.makeRectangle((clusterWidth / 2) + (i * clusterWidth), + clusterHeight / 2, + clusterWidth - 10, + clusterHeight - 50); + clusterRect.fill = '#00AEEF'; + + var clusterText = two.makeText('Cluster', + (clusterWidth / 2) + (i * clusterWidth), + 35, + { alignment: 'center', fill: 'white' }); + + for (var j = 0; j < 4; j++) { + let pageWidth = 100; + let pageHeight = canvasHeight; + var pageRect = two.makeRectangle((pageWidth / 2) + (j * pageWidth) + (i * clusterWidth), + pageHeight / 2, + pageWidth - 20, + pageHeight - 100); + pageRect.fill = '#003C71'; + + var pageText = two.makeText('Page', + (pageWidth / 2) + (j * pageWidth) + (i * clusterWidth), + pageHeight / 2, + { alignment: 'center', fill: 'white' }); + } + } + + two.update(); + </script> + +@endhtmlonly + +### Atomicity + +For all Blobstore operations regarding atomicity, there is a dependency on the underlying device to guarantee atomic +operations of at least one page size. Atomicity here can refer to multiple operations: + +* **Data Writes**: For the case of data writes, the unit of atomicity is one page. Therefore if a write operation of + greater than one page is underway and the system suffers a power failure, the data on media will be consistent at a page + size granularity (if a single page were in the middle of being updated when power was lost, the data at that page location + will be as it was prior to the start of the write operation following power restoration.) +* **Blob Metadata Updates**: Each blob has its own set of metadata (xattrs, size, etc). For performance reasons, a copy of + this metadata is kept in RAM and only synchronized with the on-disk version when the application makes an explicit call to + do so, or when the Blobstore is unloaded. Therefore, setting of an xattr, for example is not consistent until the call to + synchronize it (covered later) which is, however, performed atomically. +* **Blobstore Metadata Updates**: Blobstore itself has its own metadata which, like per blob metadata, has a copy in both + RAM and on-disk. Unlike the per blob metadata, however, the Blobstore metadata region is not made consistent via a blob + synchronization call, it is only synchronized when the Blobstore is properly unloaded via API. Therefore, if the Blobstore + metadata is updated (blob creation, deletion, resize, etc.) and not unloaded properly, it will need to perform some extra + steps the next time it is loaded which will take a bit more time than it would have if shutdown cleanly, but there will be + no inconsistencies. + +### Callbacks + +Blobstore is callback driven; in the event that any Blobstore API is unable to make forward progress it will +not block but instead return control at that point and make a call to the callback function provided in the API, along with +arguments, when the original call is completed. The callback will be made on the same thread that the call was made from, more on +threads later. Some API, however, offer no callback arguments; in these cases the calls are fully synchronous. Examples of +asynchronous calls that utilize callbacks include those that involve disk IO, for example, where some amount of polling +is required before the IO is completed. + +### Backend Support + +Blobstore requires a backing storage device that can be integrated using the `bdev` layer, or by directly integrating a +device driver to Blobstore. The blobstore performs operations on a backing block device by calling function pointers +supplied to it at initialization time. For convenience, an implementation of these function pointers that route I/O +to the bdev layer is available in `bdev_blob.c`. Alternatively, for example, the SPDK NVMe driver may be directly integrated +bypassing a small amount of `bdev` layer overhead. These options will be discussed further in the upcoming section on examples. + +### Metadata Operations + +Because Blobstore is designed to be lock-free, metadata operations need to be isolated to a single +thread to avoid taking locks on in memory data structures that maintain data on the layout of definitions of blobs (along +with other data). In Blobstore this is implemented as `the metadata thread` and is defined to be the thread on which the +application makes metadata related calls on. It is up to the application to setup a separate thread to make these calls on +and to assure that it does not mix relevant IO operations with metadata operations even if they are on separate threads. +This will be discussed further in the Design Considerations section. + +### Threads + +An application using Blobstore with the SPDK NVMe driver, for example, can support a variety of thread scenarios. +The simplest would be a single threaded application where the application, the Blobstore code and the NVMe driver share a +single core. In this case, the single thread would be used to submit both metadata operations as well as IO operations and +it would be up to the application to assure that only one metadata operation is issued at a time and not intermingled with +affected IO operations. + +### Channels + +Channels are an SPDK-wide abstraction and with Blobstore the best way to think about them is that they are +required in order to do IO. The application will perform IO to the channel and channels are best thought of as being +associated 1:1 with a thread. + +### Blob Identifiers + +When an application creates a blob, it does not provide a name as is the case with many other similar +storage systems, instead it is returned a unique identifier by the Blobstore that it needs to use on subsequent APIs to +perform operations on the Blobstore. + +## Design Considerations {#blob_pg_design} + +### Initialization Options + +When the Blobstore is initialized, there are multiple configuration options to consider. The +options and their defaults are: + +* **Cluster Size**: By default, this value is 1MB. The cluster size is required to be a multiple of page size and should be + selected based on the application’s usage model in terms of allocation. Recall that blobs are made up of clusters so when + a blob is allocated/deallocated or changes in size, disk LBAs will be manipulated in groups of cluster size. If the + application is expecting to deal with mainly very large (always multiple GB) blobs then it may make sense to change the + cluster size to 1GB for example. +* **Number of Metadata Pages**: By default, Blobstore will assume there can be as many clusters as there are metadata pages + which is the worst case scenario in terms of metadata usage and can be overridden here however the space efficiency is + not significant. +* **Maximum Simultaneous Metadata Operations**: Determines how many internally pre-allocated memory structures are set + aside for performing metadata operations. It is unlikely that changes to this value (default 32) would be desirable. +* **Maximum Simultaneous Operations Per Channel**: Determines how many internally pre-allocated memory structures are set + aside for channel operations. Changes to this value would be application dependent and best determined by both a knowledge + of the typical usage model, an understanding of the types of SSDs being used and empirical data. The default is 512. +* **Blobstore Type**: This field is a character array to be used by applications that need to identify whether the + Blobstore found here is appropriate to claim or not. The default is NULL and unless the application is being deployed in + an environment where multiple applications using the same disks are at risk of inadvertently using the wrong Blobstore, there + is no need to set this value. It can, however, be set to any valid set of characters. + +### Sub-page Sized Operations + +Blobstore is only capable of doing page sized read/write operations. If the application +requires finer granularity it will have to accommodate that itself. + +### Threads + +As mentioned earlier, Blobstore can share a single thread with an application or the application +can define any number of threads, within resource constraints, that makes sense. The basic considerations that must be +followed are: + +* Metadata operations (API with MD in the name) should be isolated from each other as there is no internal locking on the + memory structures affected by these API. +* Metadata operations should be isolated from conflicting IO operations (an example of a conflicting IO would be one that is + reading/writing to an area of a blob that a metadata operation is deallocating). +* Asynchronous callbacks will always take place on the calling thread. +* No assumptions about IO ordering can be made regardless of how many or which threads were involved in the issuing. + +### Data Buffer Memory + +As with all SPDK based applications, Blobstore requires memory used for data buffers to be allocated +with SPDK API. + +### Error Handling + +Asynchronous Blobstore callbacks all include an error number that should be checked; non-zero values +indicate and error. Synchronous calls will typically return an error value if applicable. + +### Asynchronous API + +Asynchronous callbacks will return control not immediately, but at the point in execution where no +more forward progress can be made without blocking. Therefore, no assumptions can be made about the progress of +an asynchronous call until the callback has completed. + +### Xattrs + +Setting and removing of xattrs in Blobstore is a metadata operation, xattrs are stored in per blob metadata. +Therefore, xattrs are not persisted until a blob synchronization call is made and completed. Having a step process for +persisting per blob metadata allows for applications to perform batches of xattr updates, for example, with only one +more expensive call to synchronize and persist the values. + +### Synchronizing Metadata + +As described earlier, there are two types of metadata in Blobstore, per blob and one global +metadata for the Blobstore itself. Only the per blob metadata can be explicitly synchronized via API. The global +metadata will be inconsistent during run-time and only synchronized on proper shutdown. The implication, however, of +an improper shutdown is only a performance penalty on the next startup as the global metadata will need to be rebuilt +based on a parsing of the per blob metadata. For consistent start times, it is important to always close down the Blobstore +properly via API. + +### Iterating Blobs + +Multiple examples of how to iterate through the blobs are included in the sample code and tools. +Worthy to note, however, if walking through the existing blobs via the iter API, if your application finds the blob its +looking for it will either need to explicitly close it (because was opened internally by the Blobstore) or complete walking +the full list. + +### The Super Blob + +The super blob is simply a single blob ID that can be stored as part of the global metadata to act +as sort of a "root" blob. The application may choose to use this blob to store any information that it needs or finds +relevant in understanding any kind of structure for what is on the Blobstore. + +## Examples {#blob_pg_examples} + +There are multiple examples of Blobstore usage in the [repo](https://github.com/spdk/spdk): + +* **Hello World**: Actually named `hello_blob.c` this is a very basic example of a single threaded application that + does nothing more than demonstrate the very basic API. Although Blobstore is optimized for NVMe, this example uses + a RAM disk (malloc) back-end so that it can be executed easily in any development environment. The malloc back-end + is a `bdev` module thus this example uses not only the SPDK Framework but the `bdev` layer as well. + +* **CLI**: The `blobcli.c` example is command line utility intended to not only serve as example code but as a test + and development tool for Blobstore itself. It is also a simple single threaded application that relies on both the + SPDK Framework and the `bdev` layer but offers multiple modes of operation to accomplish some real-world tasks. In + command mode, it accepts single-shot commands which can be a little time consuming if there are many commands to + get through as each one will take a few seconds waiting for DPDK initialization. It therefore has a shell mode that + allows the developer to get to a `blob>` prompt and then very quickly interact with Blobstore with simple commands + that include the ability to import/export blobs from/to regular files. Lastly there is a scripting mode to automate + a series of tasks, again, handy for development and/or test type activities. + +## Configuration {#blob_pg_config} + +Blobstore configuration options are described in the initialization options section under @ref blob_pg_design. + +## Component Detail {#blob_pg_component} + +The information in this section is not necessarily relevant to designing an application for use with Blobstore, but +understanding a little more about the internals may be interesting and is also included here for those wanting to +contribute to the Blobstore effort itself. + +### Media Format + +The Blobstore owns the entire storage device. The device is divided into clusters starting from the beginning, such +that cluster 0 begins at the first logical block. + + LBA 0 LBA N + +-----------+-----------+-----+-----------+ + | Cluster 0 | Cluster 1 | ... | Cluster N | + +-----------+-----------+-----+-----------+ + +Cluster 0 is special and has the following format, where page 0 is the first page of the cluster: + + +--------+-------------------+ + | Page 0 | Page 1 ... Page N | + +--------+-------------------+ + | Super | Metadata Region | + | Block | | + +--------+-------------------+ + +The super block is a single page located at the beginning of the partition. It contains basic information about +the Blobstore. The metadata region is the remainder of cluster 0 and may extend to additional clusters. Refer +to the latest source code for complete structural details of the super block and metadata region. + +Each blob is allocated a non-contiguous set of pages inside the metadata region for its metadata. These pages +form a linked list. The first page in the list will be written in place on update, while all other pages will +be written to fresh locations. This requires the backing device to support an atomic write size greater than +or equal to the page size to guarantee that the operation is atomic. See the section on atomicity for details. + +### Blob cluster layout {#blob_pg_cluster_layout} + +Each blob is an ordered list of clusters, where starting LBA of a cluster is called extent. A blob can be +thin provisioned, resulting in no extent for some of the clusters. When first write operation occurs +to the unallocated cluster - new extent is chosen. This information is stored in RAM and on-disk. + +There are two extent representations on-disk, dependent on `use_extent_table` (default:true) opts used +when creating a blob. + +* **use_extent_table=true**: EXTENT_PAGE descriptor is not part of linked list of pages. It contains extents + that are not run-length encoded. Each extent page is referenced by EXTENT_TABLE descriptor, which is serialized + as part of linked list of pages. Extent table is run-length encoding all unallocated extent pages. + Every new cluster allocation updates a single extent page, in case when extent page was previously allocated. + Otherwise additionally incurs serializing whole linked list of pages for the blob. + +* **use_extent_table=false**: EXTENT_RLE descriptor is serialized as part of linked list of pages. + Extents pointing to contiguous LBA are run-length encoded, including unallocated extents represented by 0. + Every new cluster allocation incurs serializing whole linked list of pages for the blob. + +### Sequences and Batches + +Internally Blobstore uses the concepts of sequences and batches to submit IO to the underlying device in either +a serial fashion or in parallel, respectively. Both are defined using the following structure: + +~~~{.sh} +struct spdk_bs_request_set; +~~~ + +These requests sets are basically bookkeeping mechanisms to help Blobstore efficiently deal with related groups +of IO. They are an internal construct only and are pre-allocated on a per channel basis (channels were discussed +earlier). They are removed from a channel associated linked list when the set (sequence or batch) is started and +then returned to the list when completed. + +### Key Internal Structures + +`blobstore.h` contains many of the key structures for the internal workings of Blobstore. Only a few notable ones +are reviewed here. Note that `blobstore.h` is an internal header file, the header file for Blobstore that defines +the public API is `blob.h`. + +~~~{.sh} +struct spdk_blob +~~~ +This is an in-memory data structure that contains key elements like the blob identifier, its current state and two +copies of the mutable metadata for the blob; one copy is the current metadata and the other is the last copy written +to disk. + +~~~{.sh} +struct spdk_blob_mut_data +~~~ +This is a per blob structure, included the `struct spdk_blob` struct that actually defines the blob itself. It has the +specific information on size and makeup of the blob (ie how many clusters are allocated for this blob and which ones.) + +~~~{.sh} +struct spdk_blob_store +~~~ +This is the main in-memory structure for the entire Blobstore. It defines the global on disk metadata region and maintains +information relevant to the entire system - initialization options such as cluster size, etc. + +~~~{.sh} +struct spdk_bs_super_block +~~~ +The super block is an on-disk structure that contains all of the relevant information that's in the in-memory Blobstore +structure just discussed along with other elements one would expect to see here such as signature, version, checksum, etc. + +### Code Layout and Common Conventions + +In general, `Blobstore.c` is laid out with groups of related functions blocked together with descriptive comments. For +example, + +~~~{.sh} +/* START spdk_bs_md_delete_blob */ +< relevant functions to accomplish the deletion of a blob > +/* END spdk_bs_md_delete_blob */ +~~~ + +And for the most part the following conventions are followed throughout: + +* functions beginning with an underscore are called internally only +* functions or variables with the letters `cpl` are related to set or callback completions diff --git a/src/spdk/doc/blobfs.md b/src/spdk/doc/blobfs.md new file mode 100644 index 000000000..221abed4d --- /dev/null +++ b/src/spdk/doc/blobfs.md @@ -0,0 +1,93 @@ +# BlobFS (Blobstore Filesystem) {#blobfs} + +# BlobFS Getting Started Guide {#blobfs_getting_started} + +# RocksDB Integration {#blobfs_rocksdb} + +Clone and build the SPDK repository as per https://github.com/spdk/spdk + +~~~{.sh} +git clone https://github.com/spdk/spdk.git +cd spdk +./configure +make +~~~ + +Clone the RocksDB repository from the SPDK GitHub fork into a separate directory. +Make sure you check out the `spdk-v5.14.3` branch. + +~~~{.sh} +cd .. +git clone -b spdk-v5.14.3 https://github.com/spdk/rocksdb.git +~~~ + +Build RocksDB. Only the `db_bench` benchmarking tool is integrated with BlobFS. + +~~~{.sh} +cd rocksdb +make db_bench SPDK_DIR=path/to/spdk +~~~ + +Or you can also add `DEBUG_LEVEL=0` for a release build (need to turn on `USE_RTTI`). + +~~~{.sh} +export USE_RTTI=1 && make db_bench DEBUG_LEVEL=0 SPDK_DIR=path/to/spdk +~~~ + +Create an NVMe section in the configuration file using SPDK's `gen_nvme.sh` script. + +~~~{.sh} +scripts/gen_nvme.sh > /usr/local/etc/spdk/rocksdb.conf +~~~ + +Verify the configuration file has specified the correct NVMe SSD. +If there are any NVMe SSDs you do not wish to use for RocksDB/SPDK testing, remove them from the configuration file. + +Make sure you have at least 5GB of memory allocated for huge pages. +By default, the SPDK `setup.sh` script only allocates 2GB. +The following will allocate 5GB of huge page memory (in addition to binding the NVMe devices to uio/vfio). + +~~~{.sh} +HUGEMEM=5120 scripts/setup.sh +~~~ + +Create an empty SPDK blobfs for testing. + +~~~{.sh} +test/blobfs/mkfs/mkfs /usr/local/etc/spdk/rocksdb.conf Nvme0n1 +~~~ + +At this point, RocksDB is ready for testing with SPDK. Three `db_bench` parameters are used to configure SPDK: + +1. `spdk` - Defines the name of the SPDK configuration file. If omitted, RocksDB will use the default PosixEnv implementation + instead of SpdkEnv. (Required) +2. `spdk_bdev` - Defines the name of the SPDK block device which contains the BlobFS to be used for testing. (Required) +3. `spdk_cache_size` - Defines the amount of userspace cache memory used by SPDK. Specified in terms of megabytes (MB). + Default is 4096 (4GB). (Optional) + +SPDK has a set of scripts which will run `db_bench` against a variety of workloads and capture performance and profiling +data. The primary script is `test/blobfs/rocksdb/rocksdb.sh`. + +# FUSE + +BlobFS provides a FUSE plug-in to mount an SPDK BlobFS as a kernel filesystem for inspection or debug purposes. +The FUSE plug-in requires fuse3 and will be built automatically when fuse3 is detected on the system. + +~~~{.sh} +test/blobfs/fuse/fuse /usr/local/etc/spdk/rocksdb.conf Nvme0n1 /mnt/fuse +~~~ + +Note that the FUSE plug-in has some limitations - see the list below. + +# Limitations + +* BlobFS has primarily been tested with RocksDB so far, so any use cases different from how RocksDB uses a filesystem + may run into issues. BlobFS will be tested in a broader range of use cases after this initial release. +* Only a synchronous API is currently supported. An asynchronous API has been developed but not thoroughly tested + yet so is not part of the public interface yet. This will be added in a future release. +* File renames are not atomic. This will be fixed in a future release. +* BlobFS currently supports only a flat namespace for files with no directory support. Filenames are currently stored + as xattrs in each blob. This means that filename lookup is an O(n) operation. An SPDK btree implementation is + underway which will be the underpinning for BlobFS directory support in a future release. +* Writes to a file must always append to the end of the file. Support for writes to any location within the file + will be added in a future release. diff --git a/src/spdk/doc/compression.md b/src/spdk/doc/compression.md new file mode 100644 index 000000000..32c076300 --- /dev/null +++ b/src/spdk/doc/compression.md @@ -0,0 +1,286 @@ +# SPDK "Reduce" Block Compression Algorithm {#reduce} + +## Overview + +The SPDK "reduce" block compression scheme is based on using SSDs for storing compressed blocks of +storage and persistent memory for metadata. This metadata includes mappings of logical blocks +requested by a user to the compressed blocks on SSD. The scheme described in this document +is generic and not tied to any specific block device framework such as the SPDK block device (bdev) +framework. This algorithm will be implemented in a library called "libreduce". Higher-level +software modules can built on top of this library to create and present block devices in a +specific block device framework. For SPDK, a bdev_reduce module will serve as a wrapper around +the libreduce library, to present the compressed block devices as an SPDK bdev. + +This scheme only describes how compressed blocks are stored on an SSD and the metadata for tracking +those compressed blocks. It relies on the higher-software module to perform the compression +algorithm itself. For SPDK, the bdev_reduce module will utilize the DPDK compressdev framework +to perform compression and decompression on behalf of the libreduce library. + +(Note that in some cases, blocks of storage may not be compressible, or cannot be compressed enough +to realize savings from the compression. In these cases, the data may be stored uncompressed on +disk. The phrase "compressed blocks of storage" includes these uncompressed blocks.) + +A compressed block device is a logical entity built on top of a similarly-sized backing storage +device. The backing storage device must be thin-provisioned to realize any savings from +compression for reasons described later in this document. This algorithm has no direct knowledge +of the implementation of the backing storage device, except that it will always use the +lowest-numbered blocks available on the backing storage device. This will ensure that when this +algorithm is used on a thin-provisioned backing storage device, blocks will not be allocated until +they are actually needed. + +The backing storage device must be sized for the worst case scenario, where no data can be +compressed. In this case, the size of the backing storage device would be the same as the +compressed block device. Since this algorithm ensures atomicity by never overwriting data +in place, some additional backing storage is required to temporarily store data for writes in +progress before the associated metadata is updated. + +Storage from the backing storage device will be allocated, read, and written to in 4KB units for +best NVMe performance. These 4KB units are called "backing IO units". They are indexed from 0 to N-1 +with the indices called "backing IO unit indices". At start, the full set of indices represent the +"free backing IO unit list". + +A compressed block device compresses and decompresses data in units of chunks, where a chunk is a +multiple of at least two 4KB backing IO units. The number of backing IO units per chunk determines +the chunk size and is specified when the compressed block device is created. A chunk +consumes a number of 4KB backing IO units between 1 and the number of 4KB units in the chunk. For +example, a 16KB chunk consumes 1, 2, 3 or 4 backing IO units. The number of backing IO units depends on how +much the chunk was able to be compressed. The blocks on disk associated with a chunk are stored in a +"chunk map" in persistent memory. Each chunk map consists of N 64-bit values, where N is the maximum +number of backing IO units in the chunk. Each 64-bit value corresponds to a backing IO unit index. A +special value (for example, 2^64-1) is used for backing IO units not needed due to compression. The +number of chunk maps allocated is equal to the size of the compressed block device divided by its chunk +size, plus some number of extra chunk maps. These extra chunk maps are used to ensure atomicity on +writes and will be explained later in this document. At start, all of the chunk maps represent the +"free chunk map list". + +Finally, the logical view of the compressed block device is represented by the "logical map". The +logical map is a mapping of chunk offsets into the compressed block device to the corresponding +chunk map. Each entry in the logical map is a 64-bit value, denoting the associated chunk map. +A special value (UINT64_MAX) is used if there is no associated chunk map. The mapping is +determined by dividing the byte offset by the chunk size to get an index, which is used as an +array index into the array of chunk map entries. At start, all entries in the logical map have no +associated chunk map. Note that while access to the backing storage device is in 4KB units, the +logical view may allow 4KB or 512B unit access and should perform similarly. + +## Example + +To illustrate this algorithm, we will use a real example at a very small scale. + +The size of the compressed block device is 64KB, with a chunk size of 16KB. This will +realize the following: + +* "Backing storage" will consist of an 80KB thin-provisioned logical volume. This + corresponds to the 64KB size of the compressed block device, plus an extra 16KB to handle + additional write operations under a worst-case compression scenario. +* "Free backing IO unit list" will consist of indices 0 through 19 (inclusive). These represent + the 20 4KB IO units in the backing storage. +* A "chunk map" will be 32 bytes in size. This corresponds to 4 backing IO units per chunk + (16KB / 4KB), and 8B (64b) per backing IO unit index. +* 5 chunk maps will be allocated in 160B of persistent memory. This corresponds to 4 chunk maps + for the 4 chunks in the compressed block device (64KB / 16KB), plus an extra chunk map for use + when overwriting an existing chunk. +* "Free chunk map list" will consist of indices 0 through 4 (inclusive). These represent the + 5 allocated chunk maps. +* The "logical map" will be allocated in 32B of persistent memory. This corresponds to + 4 entries for the 4 chunks in the compressed block device and 8B (64b) per entry. + +In these examples, the value "X" will represent the special value (2^64-1) described above. + +### Initial Creation + +``` + +--------------------+ + Backing Device | | + +--------------------+ + + Free Backing IO Unit List 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 + + +------------+------------+------------+------------+------------+ + Chunk Maps | | | | | | + +------------+------------+------------+------------+------------+ + + Free Chunk Map List 0, 1, 2, 3, 4 + + +---+---+---+---+ + Logical Map | X | X | X | X | + +---+---+---+---+ +``` + +### Write 16KB at Offset 32KB + +* Find the corresponding index into the logical map. Offset 32KB divided by the chunk size + (16KB) is 2. +* Entry 2 in the logical map is "X". This means no part of this 16KB has been written to yet. +* Allocate a 16KB buffer in memory +* Compress the incoming 16KB of data into this allocated buffer +* Assume this data compresses to 6KB. This requires 2 4KB backing IO units. +* Allocate 2 blocks (0 and 1) from the free backing IO unit list. Always use the lowest numbered + entries in the free backing IO unit list - this ensures that unnecessary backing storage + is not allocated in the thin-provisioned logical volume holding the backing storage. +* Write the 6KB of data to backing IO units 0 and 1. +* Allocate a chunk map (0) from the free chunk map list. +* Write (0, 1, X, X) to the chunk map. This represents that only 2 backing IO units were used to + store the 16KB of data. +* Write the chunk map index to entry 2 in the logical map. + +``` + +--------------------+ + Backing Device |01 | + +--------------------+ + + Free Backing IO Unit List 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 + + +------------+------------+------------+------------+------------+ + Chunk Maps | 0 1 X X | | | | | + +------------+------------+------------+------------+------------+ + + Free Chunk Map List 1, 2, 3, 4 + + +---+---+---+---+ + Logical Map | X | X | 0 | X | + +---+---+---+---+ +``` + +### Write 4KB at Offset 8KB + +* Find the corresponding index into the logical map. Offset 8KB divided by the chunk size is 0. +* Entry 0 in the logical map is "X". This means no part of this 16KB has been written to yet. +* The write is not for the entire 16KB chunk, so we must allocate a 16KB chunk-sized buffer for + source data. +* Copy the incoming 4KB data to offset 8KB of this 16KB buffer. Zero the rest of the 16KB buffer. +* Allocate a 16KB destination buffer. +* Compress the 16KB source data buffer into the 16KB destination buffer +* Assume this data compresses to 3KB. This requires 1 4KB backing IO unit. +* Allocate 1 block (2) from the free backing IO unit list. +* Write the 3KB of data to block 2. +* Allocate a chunk map (1) from the free chunk map list. +* Write (2, X, X, X) to the chunk map. +* Write the chunk map index to entry 0 in the logical map. + +``` + +--------------------+ + Backing Device |012 | + +--------------------+ + + Free Backing IO Unit List 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 + + +------------+------------+------------+------------+------------+ + Chunk Maps | 0 1 X X | 2 X X X | | | | + +------------+------------+------------+------------+------------+ + + Free Chunk Map List 2, 3, 4 + + +---+---+---+---+ + Logical Map | 1 | X | 0 | X | + +---+---+---+---+ +``` + +### Read 16KB at Offset 16KB + +* Offset 16KB maps to index 1 in the logical map. +* Entry 1 in the logical map is "X". This means no part of this 16KB has been written to yet. +* Since no data has been written to this chunk, return all 0's to satisfy the read I/O. + +### Write 4KB at Offset 4KB + +* Offset 4KB maps to index 0 in the logical map. +* Entry 0 in the logical map is "1". Since we are not overwriting the entire chunk, we must + do a read-modify-write. +* Chunk map 1 only specifies one backing IO unit (2). Allocate a 16KB buffer and read block + 2 into it. This will be called the compressed data buffer. Note that 16KB is allocated + instead of 4KB so that we can reuse this buffer to hold the compressed data that will + be written later back to disk. +* Allocate a 16KB buffer for the uncompressed data for this chunk. Decompress the data from + the compressed data buffer into this buffer. +* Copy the incoming 4KB of data to offset 4KB of the uncompressed data buffer. +* Compress the 16KB uncompressed data buffer into the compressed data buffer. +* Assume this data compresses to 5KB. This requires 2 4KB backing IO units. +* Allocate blocks 3 and 4 from the free backing IO unit list. +* Write the 5KB of data to blocks 3 and 4. +* Allocate chunk map 2 from the free chunk map list. +* Write (3, 4, X, X) to chunk map 2. Note that at this point, the chunk map is not referenced + by the logical map. If there was a power fail at this point, the previous data for this chunk + would still be fully valid. +* Write chunk map 2 to entry 0 in the logical map. +* Free chunk map 1 back to the free chunk map list. +* Free backing IO unit 2 back to the free backing IO unit list. + +``` + +--------------------+ + Backing Device |01 34 | + +--------------------+ + + Free Backing IO Unit List 2, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 + + +------------+------------+------------+------------+------------+ + Chunk Maps | 0 1 X X | | 3 4 X X | | | + +------------+------------+------------+------------+------------+ + + Free Chunk Map List 1, 3, 4 + + +---+---+---+---+ + Logical Map | 2 | X | 0 | X | + +---+---+---+---+ +``` + +### Operations that span across multiple chunks + +Operations that span a chunk boundary are logically split into multiple operations, each of +which is associated with a single chunk. + +Example: 20KB write at offset 4KB + +In this case, the write operation is split into a 12KB write at offset 4KB (affecting only +chunk 0 in the logical map) and a 8KB write at offset 16KB (affecting only chunk 1 in the +logical map). Each write is processed independently using the algorithm described above. +Completion of the 20KB write does not occur until both operations have completed. + +### Unmap Operations + +Unmap operations on an entire chunk are achieved by removing the chunk map entry (if any) from +the logical map. The chunk map is returned to the free chunk map list, and any backing IO units +associated with the chunk map are returned to the free backing IO unit list. + +Unmap operations that affect only part of a chunk can be treated as writing zeroes to that +region of the chunk. If the entire chunk is unmapped via several operations, it can be +detected via the uncompressed data equaling all zeroes. When this occurs, the chunk map entry +may be removed from the logical map. + +After an entire chunk has been unmapped, subsequent reads to the chunk will return all zeroes. +This is similar to the "Read 16KB at offset 16KB" example above. + +### Write Zeroes Operations + +Write zeroes operations are handled similarly to unmap operations. If a write zeroes +operation covers an entire chunk, we can remove the chunk's entry in the logical map +completely. Then subsequent reads to that chunk will return all zeroes. + +### Restart + +An application using libreduce will periodically exit and need to be restarted. When the +application restarts, it will reload compressed volumes so they can be used again from the +same state as when the application exited. + +When the compressed volume is reloaded, the free chunk map list and free backing IO unit list +are reconstructed by walking the logical map. The logical map will only point to valid +chunk maps, and the valid chunk maps will only point to valid backing IO units. Any chunk maps +and backing IO units not referenced go into their respective free lists. + +This ensures that if a system crashes in the middle of a write operation - i.e. during or +after a chunk map is updated, but before it is written to the logical map - that everything +related to that in-progress write will be ignored after the compressed volume is restarted. + +### Overlapping operations on same chunk + +Implementations must take care to handle overlapping operations on the same chunk. For example, +operation 1 writes some data to chunk A, and while this is in progress, operation 2 also writes +some data to chunk A. In this case, operation 2 should not start until operation 1 has +completed. Further optimizations are outside the scope of this document. + +### Thin provisioned backing storage + +Backing storage must be thin provisioned to realize any savings from compression. This algorithm +will always use (and reuse) backing IO units available closest to offset 0 on the backing device. +This ensures that even though backing storage device may have been sized similarly to the size of +the compressed volume, storage for the backing storage device will not actually be allocated +until the backing IO units are actually needed. diff --git a/src/spdk/doc/concepts.md b/src/spdk/doc/concepts.md new file mode 100644 index 000000000..9f0701637 --- /dev/null +++ b/src/spdk/doc/concepts.md @@ -0,0 +1,10 @@ +# Concepts {#concepts} + +- @subpage userspace +- @subpage memory +- @subpage concurrency +- @subpage ssd_internals +- @subpage nvme_spec +- @subpage vhost_processing +- @subpage overview +- @subpage porting diff --git a/src/spdk/doc/concurrency.md b/src/spdk/doc/concurrency.md new file mode 100644 index 000000000..47009e85d --- /dev/null +++ b/src/spdk/doc/concurrency.md @@ -0,0 +1,250 @@ +# Message Passing and Concurrency {#concurrency} + +# Theory + +One of the primary aims of SPDK is to scale linearly with the addition of +hardware. This can mean many things in practice. For instance, moving from one +SSD to two should double the number of I/O's per second. Or doubling the number +of CPU cores should double the amount of computation possible. Or even doubling +the number of NICs should double the network throughput. To achieve this, the +software's threads of execution must be independent from one another as much as +possible. In practice, that means avoiding software locks and even atomic +instructions. + +Traditionally, software achieves concurrency by placing some shared data onto +the heap, protecting it with a lock, and then having all threads of execution +acquire the lock only when accessing the data. This model has many great +properties: + +* It's easy to convert single-threaded programs to multi-threaded programs + because you don't have to change the data model from the single-threaded + version. You add a lock around the data. +* You can write your program as a synchronous, imperative list of statements + that you read from top to bottom. +* The scheduler can interrupt threads, allowing for efficient time-sharing + of CPU resources. + +Unfortunately, as the number of threads scales up, contention on the lock around +the shared data does too. More granular locking helps, but then also increases +the complexity of the program. Even then, beyond a certain number of contended +locks, threads will spend most of their time attempting to acquire the locks and +the program will not benefit from more CPU cores. + +SPDK takes a different approach altogether. Instead of placing shared data in a +global location that all threads access after acquiring a lock, SPDK will often +assign that data to a single thread. When other threads want to access the data, +they pass a message to the owning thread to perform the operation on their +behalf. This strategy, of course, is not at all new. For instance, it is one of +the core design principles of +[Erlang](http://erlang.org/download/armstrong_thesis_2003.pdf) and is the main +concurrency mechanism in [Go](https://tour.golang.org/concurrency/2). A message +in SPDK consists of a function pointer and a pointer to some context. Messages +are passed between threads using a +[lockless ring](http://dpdk.org/doc/guides/prog_guide/ring_lib.html). Message +passing is often much faster than most software developer's intuition leads them +to believe due to caching effects. If a single core is accessing the same data +(on behalf of all of the other cores), then that data is far more likely to be +in a cache closer to that core. It's often most efficient to have each core work +on a small set of data sitting in its local cache and then hand off a small +message to the next core when done. + +In more extreme cases where even message passing may be too costly, each thread +may make a local copy of the data. The thread will then only reference its local +copy. To mutate the data, threads will send a message to each other thread +telling them to perform the update on their local copy. This is great when the +data isn't mutated very often, but is read very frequently, and is often +employed in the I/O path. This of course trades memory size for computational +efficiency, so it is used in only the most critical code paths. + +# Message Passing Infrastructure + +SPDK provides several layers of message passing infrastructure. The most +fundamental libraries in SPDK, for instance, don't do any message passing on +their own and instead enumerate rules about when functions may be called in +their documentation (e.g. @ref nvme). Most libraries, however, depend on SPDK's +[thread](http://www.spdk.io/doc/thread_8h.html) +abstraction, located in `libspdk_thread.a`. The thread abstraction provides a +basic message passing framework and defines a few key primitives. + +First, `spdk_thread` is an abstraction for a lightweight, stackless thread of +execution. A lower level framework can execute an `spdk_thread` for a single +timeslice by calling `spdk_thread_poll()`. A lower level framework is allowed to +move an `spdk_thread` between system threads at any time, as long as there is +only a single system thread executing `spdk_thread_poll()` on that +`spdk_thread` at any given time. New lightweight threads may be created at any +time by calling `spdk_thread_create()` and destroyed by calling +`spdk_thread_destroy()`. The lightweight thread is the foundational abstraction for +threading in SPDK. + +There are then a few additional abstractions layered on top of the +`spdk_thread`. One is the `spdk_poller`, which is an abstraction for a +function that should be repeatedly called on the given thread. Another is an +`spdk_msg_fn`, which is a function pointer and a context pointer, that can +be sent to a thread for execution via `spdk_thread_send_msg()`. + +The library also defines two additional abstractions: `spdk_io_device` and +`spdk_io_channel`. In the course of implementing SPDK we noticed the same +pattern emerging in a number of different libraries. In order to implement a +message passing strategy, the code would describe some object with global state +and also some per-thread context associated with that object that was accessed +in the I/O path to avoid locking on the global state. The pattern was clearest +in the lowest layers where I/O was being submitted to block devices. These +devices often expose multiple queues that can be assigned to threads and then +accessed without a lock to submit I/O. To abstract that, we generalized the +device to `spdk_io_device` and the thread-specific queue to `spdk_io_channel`. +Over time, however, the pattern has appeared in a huge number of places that +don't fit quite so nicely with the names we originally chose. In today's code +`spdk_io_device` is any pointer, whose uniqueness is predicated only on its +memory address, and `spdk_io_channel` is the per-thread context associated with +a particular `spdk_io_device`. + +The threading abstraction provides functions to send a message to any other +thread, to send a message to all threads one by one, and to send a message to +all threads for which there is an io_channel for a given io_device. + +Most critically, the thread abstraction does not actually spawn any system level +threads of its own. Instead, it relies on the existence of some lower level +framework that spawns system threads and sets up event loops. Inside those event +loops, the threading abstraction simply requires the lower level framework to +repeatedly call `spdk_thread_poll()` on each `spdk_thread()` that exists. This +makes SPDK very portable to a wide variety of asynchronous, event-based +frameworks such as [Seastar](https://www.seastar.io) or [libuv](https://libuv.org/). + +# The event Framework + +The SPDK project didn't want to officially pick an asynchronous, event-based +framework for all of the example applications it shipped with, in the interest +of supporting the widest variety of frameworks possible. But the applications do +of course require something that implements an asynchronous event loop in order +to run, so enter the `event` framework located in `lib/event`. This framework +includes things like polling and scheduling the lightweight threads, installing +signal handlers to cleanly shutdown, and basic command line option parsing. +Only established applications should consider directly integrating the lower +level libraries. + +# Limitations of the C Language + +Message passing is efficient, but it results in asynchronous code. +Unfortunately, asynchronous code is a challenge in C. It's often implemented by +passing function pointers that are called when an operation completes. This +chops up the code so that it isn't easy to follow, especially through logic +branches. The best solution is to use a language with support for +[futures and promises](https://en.wikipedia.org/wiki/Futures_and_promises), +such as C++, Rust, Go, or almost any other higher level language. However, SPDK is a low +level library and requires very wide compatibility and portability, so we've +elected to stay with plain old C. + +We do have a few recommendations to share, though. For _simple_ callback chains, +it's easiest if you write the functions from bottom to top. By that we mean if +function `foo` performs some asynchronous operation and when that completes +function `bar` is called, then function `bar` performs some operation that +calls function `baz` on completion, a good way to write it is as such: + + void baz(void *ctx) { + ... + } + + void bar(void *ctx) { + async_op(baz, ctx); + } + + void foo(void *ctx) { + async_op(bar, ctx); + } + +Don't split these functions up - keep them as a nice unit that can be read from bottom to top. + +For more complex callback chains, especially ones that have logical branches +or loops, it's best to write out a state machine. It turns out that higher +level languages that support futures and promises are just generating state +machines at compile time, so even though we don't have the ability to generate +them in C we can still write them out by hand. As an example, here's a +callback chain that performs `foo` 5 times and then calls `bar` - effectively +an asynchronous for loop. + + enum states { + FOO_START = 0, + FOO_END, + BAR_START, + BAR_END + }; + + struct state_machine { + enum states state; + + int count; + }; + + static void + foo_complete(void *ctx) + { + struct state_machine *sm = ctx; + + sm->state = FOO_END; + run_state_machine(sm); + } + + static void + foo(struct state_machine *sm) + { + do_async_op(foo_complete, sm); + } + + static void + bar_complete(void *ctx) + { + struct state_machine *sm = ctx; + + sm->state = BAR_END; + run_state_machine(sm); + } + + static void + bar(struct state_machine *sm) + { + do_async_op(bar_complete, sm); + } + + static void + run_state_machine(struct state_machine *sm) + { + enum states prev_state; + + do { + prev_state = sm->state; + + switch (sm->state) { + case FOO_START: + foo(sm); + break; + case FOO_END: + /* This is the loop condition */ + if (sm->count++ < 5) { + sm->state = FOO_START; + } else { + sm->state = BAR_START; + } + break; + case BAR_START: + bar(sm); + break; + case BAR_END: + break; + } + } while (prev_state != sm->state); + } + + void do_async_for(void) + { + struct state_machine *sm; + + sm = malloc(sizeof(*sm)); + sm->state = FOO_START; + sm->count = 0; + + run_state_machine(sm); + } + +This is complex, of course, but the `run_state_machine` function can be read +from top to bottom to get a clear overview of what's happening in the code +without having to chase through each of the callbacks. diff --git a/src/spdk/doc/containers.md b/src/spdk/doc/containers.md new file mode 100644 index 000000000..f53cd68f0 --- /dev/null +++ b/src/spdk/doc/containers.md @@ -0,0 +1,91 @@ +# SPDK and Containers {#containers} + +This is a living document as there are many ways to use containers with +SPDK. As new usages are identified and tested, they will be documented +here. + +# In this document {#containers_toc} + +* @ref kata_containers_with_spdk_vhost +* @ref spdk_in_docker + +# Using SPDK vhost target to provide volume service to Kata Containers and Docker {#kata_containers_with_spdk_vhost} + +[Kata Containers](https://katacontainers.io) can build a secure container +runtime with lightweight virtual machines that feel and perform like +containers, but provide stronger workload isolation using hardware +virtualization technology as a second layer of defense. + +From Kata Containers [1.11.0](https://github.com/kata-containers/runtime/releases/tag/1.11.0), +vhost-user-blk support is enabled in `kata-containers/runtime`. That is to say +SPDK vhost target can be used to provide volume service to Kata Containers directly. +In addition, a container manager like Docker, can be configured easily to launch +a Kata container with an SPDK vhost-user block device. For operating details, visit +Kata containers use-case [Setup to run SPDK vhost-user devices with Kata Containers and Docker](https://github.com/kata-containers/documentation/blob/master/use-cases/using-SPDK-vhostuser-and-kata.md#host-setup-for-vhost-user-devices) + +# Containerizing an SPDK Application for Docker {#spdk_in_docker} + +There are no SPDK specific changes needed to run an SPDK based application in +a docker container, however this quick start guide should help you as you +containerize your SPDK based application. + +1. Make sure you have all of your app dependencies identified and included in your Dockerfile +2. Make sure you have compiled your application for the target arch +3. Make sure your host has hugepages enabled +4. Make sure your host has bound your nvme device to your userspace driver +5. Write your Dockerfile. The following is a simple Dockerfile to containerize the nvme `hello_world` + example: + +~~~{.sh} +# start with the latest Fedora +FROM fedora + +# if you are behind a proxy, set that up now +ADD dnf.conf /etc/dnf/dnf.conf + +# these are the min dependencies for the hello_world app +RUN dnf install libaio-devel -y +RUN dnf install numactl-devel -y + +# set our working dir +WORKDIR /app + +# add the hello_world binary +ADD hello_world hello_world + +# run the app +CMD ./hello_world +~~~ + +6. Create your image + +`sudo docker image build -t hello:1.0 .` + +7. You docker command line will need to include at least the following: +- the `--privileged` flag to enable sharing of hugepages +- use of the `-v` switch to map hugepages + +`sudo docker run --privileged -v /dev/hugepages:/dev/hugepages hello:1.0` + +or depending on the needs of your app you may need one or more of the following parameters: + +- If you are using the SPDK app framework: `-v /dev/shm:/dev/shm` +- If you need to use RPCs from outside of the container: `-v /var/tmp:/var/tmp` +- If you need to use the host network (i.e. NVMF target application): `--network host` + +Your output should look something like this: + +~~~{.sh} +$ sudo docker run --privileged -v //dev//hugepages://dev//hugepages hello:1.0 +Starting SPDK v20.01-pre git sha1 80da95481 // DPDK 19.11.0 initialization... +[ DPDK EAL parameters: hello_world -c 0x1 --log-level=lib.eal:6 --log-level=lib.cryptodev:5 --log-level=user1:6 --iova-mode=pa --base-virtaddr=0x200000000000 --match-allocations --file-prefix=spdk0 --proc-type=auto ] +EAL: No available hugepages reported in hugepages-1048576kB +Initializing NVMe Controllers +Attaching to 0000:06:00.0 +Attached to 0000:06:00.0 +Using controller INTEL SSDPEDMD400G4 (CVFT7203005M400LGN ) with 1 namespaces. + Namespace ID: 1 size: 400GB +Initialization complete. +INFO: using host memory buffer for IO +Hello world! +~~~ diff --git a/src/spdk/doc/driver_modules.md b/src/spdk/doc/driver_modules.md new file mode 100644 index 000000000..d1054fe35 --- /dev/null +++ b/src/spdk/doc/driver_modules.md @@ -0,0 +1,7 @@ +# Driver Modules {#driver_modules} + +- @subpage nvme +- @subpage ioat +- @subpage idxd +- @subpage virtio +- @subpage vmd diff --git a/src/spdk/doc/event.md b/src/spdk/doc/event.md new file mode 100644 index 000000000..657ca93e1 --- /dev/null +++ b/src/spdk/doc/event.md @@ -0,0 +1,75 @@ +# Event Framework {#event} + +SPDK provides a framework for writing asynchronous, polled-mode, +shared-nothing server applications. The event framework is intended to be +optional; most other SPDK components are designed to be integrated into an +application without specifically depending on the SPDK event library. The +framework defines several concepts - reactors, events, and pollers - that are +described in the following sections. The event framework spawns one thread per +core (reactor) and connects the threads with lockless queues. Messages +(events) can then be passed between the threads. On modern CPU architectures, +message passing is often much faster than traditional locking. For a +discussion of the theoretical underpinnings of this framework, see @ref +concurrency. + +The event framework public interface is defined in event.h. + +# Event Framework Design Considerations {#event_design} + +Simple server applications can be written in a single-threaded fashion. This +allows for straightforward code that can maintain state without any locking or +other synchronization. However, to scale up (for example, to allow more +simultaneous connections), the application may need to use multiple threads. +In the ideal case where each connection is independent from all other +connections, the application can be scaled by creating additional threads and +assigning connections to them without introducing cross-thread +synchronization. Unfortunately, in many real-world cases, the connections are +not entirely independent and cross-thread shared state is necessary. SPDK +provides an event framework to help solve this problem. + +# SPDK Event Framework Components {#event_components} + +## Events {#event_component_events} + +To accomplish cross-thread communication while minimizing synchronization +overhead, the framework provides message passing in the form of events. The +event framework runs one event loop thread per CPU core. These threads are +called reactors, and their main responsibility is to process incoming events +from a queue. Each event consists of a bundled function pointer and its +arguments, destined for a particular CPU core. Events are created using +spdk_event_allocate() and executed using spdk_event_call(). Unlike a +thread-per-connection server design, which achieves concurrency by depending +on the operating system to schedule many threads issuing blocking I/O onto a +limited number of cores, the event-driven model requires use of explicitly +asynchronous operations to achieve concurrency. Asynchronous I/O may be issued +with a non-blocking function call, and completion is typically signaled using +a callback function. + +## Reactors {#event_component_reactors} + +Each reactor has a lock-free queue for incoming events to that core, and +threads from any core may insert events into the queue of any other core. The +reactor loop running on each core checks for incoming events and executes them +in first-in, first-out order as they are received. Event functions should +never block and should preferably execute very quickly, since they are called +directly from the event loop on the destination core. + +## Pollers {#event_component_pollers} + +The framework also defines another type of function called a poller. Pollers +may be registered with the spdk_poller_register() function. Pollers, like +events, are functions with arguments that can be bundled and executed. +However, unlike events, pollers are executed repeatedly until unregistered and +are executed on the thread they are registered on. The reactor event loop +intersperses calls to the pollers with other event processing. Pollers are +intended to poll hardware as a replacement for interrupts. Normally, pollers +are executed on every iteration of the main event loop. Pollers may also be +scheduled to execute periodically on a timer if low latency is not required. + +## Application Framework {#event_component_app} + +The framework itself is bundled into a higher level abstraction called an "app". Once +spdk_app_start() is called, it will block the current thread until the application +terminates by calling spdk_app_stop() or an error condition occurs during the +initialization code within spdk_app_start(), itself, before invoking the caller's +supplied function. diff --git a/src/spdk/doc/footer.html b/src/spdk/doc/footer.html new file mode 100644 index 000000000..04f5b8449 --- /dev/null +++ b/src/spdk/doc/footer.html @@ -0,0 +1 @@ +</div> diff --git a/src/spdk/doc/ftl.md b/src/spdk/doc/ftl.md new file mode 100644 index 000000000..aa780d33b --- /dev/null +++ b/src/spdk/doc/ftl.md @@ -0,0 +1,289 @@ +# Flash Translation Layer {#ftl} + +The Flash Translation Layer library provides block device access on top of devices +implementing bdev_zone interface. +It handles the logical to physical address mapping, responds to the asynchronous +media management events, and manages the defragmentation process. + +# Terminology {#ftl_terminology} + +## Logical to physical address map + + * Shorthand: L2P + +Contains the mapping of the logical addresses (LBA) to their on-disk physical location. The LBAs +are contiguous and in range from 0 to the number of surfaced blocks (the number of spare blocks +are calculated during device formation and are subtracted from the available address space). The +spare blocks account for zones going offline throughout the lifespan of the device as well as +provide necessary buffer for data [defragmentation](#ftl_reloc). + +## Band {#ftl_band} + +A band describes a collection of zones, each belonging to a different parallel unit. All writes to +a band follow the same pattern - a batch of logical blocks is written to one zone, another batch +to the next one and so on. This ensures the parallelism of the write operations, as they can be +executed independently on different zones. Each band keeps track of the LBAs it consists of, as +well as their validity, as some of the data will be invalidated by subsequent writes to the same +logical address. The L2P mapping can be restored from the SSD by reading this information in order +from the oldest band to the youngest. + + +--------------+ +--------------+ +--------------+ + band 1 | zone 1 +--------+ zone 1 +---- --- --- --- --- ---+ zone 1 | + +--------------+ +--------------+ +--------------+ + band 2 | zone 2 +--------+ zone 2 +---- --- --- --- --- ---+ zone 2 | + +--------------+ +--------------+ +--------------+ + band 3 | zone 3 +--------+ zone 3 +---- --- --- --- --- ---+ zone 3 | + +--------------+ +--------------+ +--------------+ + | ... | | ... | | ... | + +--------------+ +--------------+ +--------------+ + band m | zone m +--------+ zone m +---- --- --- --- --- ---+ zone m | + +--------------+ +--------------+ +--------------+ + | ... | | ... | | ... | + +--------------+ +--------------+ +--------------+ + + parallel unit 1 pu 2 pu n + +The address map and valid map are, along with a several other things (e.g. UUID of the device it's +part of, number of surfaced LBAs, band's sequence number, etc.), parts of the band's metadata. The +metadata is split in two parts: + + head metadata band's data tail metadata + +-------------------+-------------------------------+------------------------+ + |zone 1 |...|zone n |...|...|zone 1 |...| | ... |zone m-1 |zone m| + |block 1| |block 1| | |block x| | | |block y |block y| + +-------------------+-------------+-----------------+------------------------+ + + * the head part, containing information already known when opening the band (device's UUID, band's + sequence number, etc.), located at the beginning blocks of the band, + * the tail part, containing the address map and the valid map, located at the end of the band. + +Bands are written sequentially (in a way that was described earlier). Before a band can be written +to, all of its zones need to be erased. During that time, the band is considered to be in a `PREP` +state. After that is done, the band transitions to the `OPENING` state, in which head metadata +is being written. Then the band moves to the `OPEN` state and actual user data can be written to the +band. Once the whole available space is filled, tail metadata is written and the band transitions to +`CLOSING` state. When that finishes the band becomes `CLOSED`. + +## Ring write buffer {#ftl_rwb} + + * Shorthand: RWB + +Because the smallest write size the SSD may support can be a multiple of block size, in order to +support writes to a single block, the data needs to be buffered. The write buffer is the solution to +this problem. It consists of a number of pre-allocated buffers called batches, each of size allowing +for a single transfer to the SSD. A single batch is divided into block-sized buffer entries. + + write buffer + +-----------------------------------+ + |batch 1 | + | +-----------------------------+ | + | |rwb |rwb | ... |rwb | | + | |entry 1|entry 2| |entry n| | + | +-----------------------------+ | + +-----------------------------------+ + | ... | + +-----------------------------------+ + |batch m | + | +-----------------------------+ | + | |rwb |rwb | ... |rwb | | + | |entry 1|entry 2| |entry n| | + | +-----------------------------+ | + +-----------------------------------+ + +When a write is scheduled, it needs to acquire an entry for each of its blocks and copy the data +onto this buffer. Once all blocks are copied, the write can be signalled as completed to the user. +In the meantime, the `rwb` is polled for filled batches and, if one is found, it's sent to the SSD. +After that operation is completed the whole batch can be freed. For the whole time the data is in +the `rwb`, the L2P points at the buffer entry instead of a location on the SSD. This allows for +servicing read requests from the buffer. + +## Defragmentation and relocation {#ftl_reloc} + + * Shorthand: defrag, reloc + +Since a write to the same LBA invalidates its previous physical location, some of the blocks on a +band might contain old data that basically wastes space. As there is no way to overwrite an already +written block, this data will stay there until the whole zone is reset. This might create a +situation in which all of the bands contain some valid data and no band can be erased, so no writes +can be executed anymore. Therefore a mechanism is needed to move valid data and invalidate whole +bands, so that they can be reused. + + band band + +-----------------------------------+ +-----------------------------------+ + | ** * * *** * *** * * | | | + |** * * * * * * *| +----> | | + |* *** * * * | | | + +-----------------------------------+ +-----------------------------------+ + +Valid blocks are marked with an asterisk '\*'. + +Another reason for data relocation might be an event from the SSD telling us that the data might +become corrupt if it's not relocated. This might happen due to its old age (if it was written a +long time ago) or due to read disturb (media characteristic, that causes corruption of neighbouring +blocks during a read operation). + +Module responsible for data relocation is called `reloc`. When a band is chosen for defragmentation +or a media management event is received, the appropriate blocks are marked as +required to be moved. The `reloc` module takes a band that has some of such blocks marked, checks +their validity and, if they're still valid, copies them. + +Choosing a band for defragmentation depends on several factors: its valid ratio (1) (proportion of +valid blocks to all user blocks), its age (2) (when was it written) and its write count / wear level +index of its zones (3) (how many times the band was written to). The lower the ratio (1), the +higher its age (2) and the lower its write count (3), the higher the chance the band will be chosen +for defrag. + +# Usage {#ftl_usage} + +## Prerequisites {#ftl_prereq} + +In order to use the FTL module, a device capable of zoned interface is required e.g. `zone_block` +bdev or OCSSD `nvme` bdev. + +## FTL bdev creation {#ftl_create} + +Similar to other bdevs, the FTL bdevs can be created either based on JSON config files or via RPC. +Both interfaces require the same arguments which are described by the `--help` option of the +`bdev_ftl_create` RPC call, which are: + + - bdev's name + - base bdev's name (base bdev must implement bdev_zone API) + - UUID of the FTL device (if the FTL is to be restored from the SSD) + +## FTL usage with OCSSD nvme bdev {#ftl_ocssd} + +This option requires an Open Channel SSD, which can be emulated using QEMU. + +The QEMU with the patches providing Open Channel support can be found on the SPDK's QEMU fork +on [spdk-3.0.0](https://github.com/spdk/qemu/tree/spdk-3.0.0) branch. + +## Configuring QEMU {#ftl_qemu_config} + +To emulate an Open Channel device, QEMU expects parameters describing the characteristics and +geometry of the SSD: + + - `serial` - serial number, + - `lver` - version of the OCSSD standard (0 - disabled, 1 - "1.2", 2 - "2.0"), libftl only supports + 2.0, + - `lba_index` - default LBA format. Possible values can be found in the table below (libftl only supports lba_index >= 3): + - `lnum_ch` - number of groups, + - `lnum_lun` - number of parallel units + - `lnum_pln` - number of planes (logical blocks from all planes constitute a chunk) + - `lpgs_per_blk` - number of pages (smallest programmable unit) per chunk + - `lsecs_per_pg` - number of sectors in a page + - `lblks_per_pln` - number of chunks in a parallel unit + - `laer_thread_sleep` - timeout in ms between asynchronous events requesting the host to relocate + the data based on media feedback + - `lmetadata` - metadata file + + |lba_index| data| metadata| + |---------|-----|---------| + | 0 | 512B| 0B | + | 1 | 512B| 8B | + | 2 | 512B| 16B | + | 3 |4096B| 0B | + | 4 |4096B| 64B | + | 5 |4096B| 128B | + | 6 |4096B| 16B | + +For more detailed description of the available options, consult the `hw/block/nvme.c` file in +the QEMU repository. + +Example: + +``` +$ /path/to/qemu [OTHER PARAMETERS] -drive format=raw,file=/path/to/data/file,if=none,id=myocssd0 + -device nvme,drive=myocssd0,serial=deadbeef,lver=2,lba_index=3,lnum_ch=1,lnum_lun=8,lnum_pln=4, + lpgs_per_blk=1536,lsecs_per_pg=4,lblks_per_pln=512,lmetadata=/path/to/md/file +``` + +In the above example, a device is created with 1 channel, 8 parallel units, 512 chunks per parallel +unit, 24576 (`lnum_pln` * `lpgs_per_blk` * `lsecs_per_pg`) logical blocks in each chunk with logical +block being 4096B. Therefore the data file needs to be at least 384G (8 * 512 * 24576 * 4096B) of +size and can be created with the following command: + +``` +fallocate -l 384G /path/to/data/file +``` + +## Configuring SPDK {#ftl_spdk_config} + +To verify that the drive is emulated correctly, one can check the output of the NVMe identify app +(assuming that `scripts/setup.sh` was called before and the driver has been changed for that +device): + +``` +$ build/examples/identify +===================================================== +NVMe Controller at 0000:00:0a.0 [1d1d:1f1f] +===================================================== +Controller Capabilities/Features +================================ +Vendor ID: 1d1d +Subsystem Vendor ID: 1af4 +Serial Number: deadbeef +Model Number: QEMU NVMe Ctrl + +... other info ... + +Namespace OCSSD Geometry +======================= +OC version: maj:2 min:0 + +... other info ... + +Groups (channels): 1 +PUs (LUNs) per group: 8 +Chunks per LUN: 512 +Logical blks per chunk: 24576 + +... other info ... + +``` + +In order to create FTL on top Open Channel SSD, the following steps are required: + +1) Attach OCSSD NVMe controller +2) Create OCSSD bdev on the controller attached in step 1 (user could specify parallel unit range +and create multiple OCSSD bdevs on single OCSSD NVMe controller) +3) Create FTL bdev on top of bdev created in step 2 + +Example: +``` +$ scripts/rpc.py bdev_nvme_attach_controller -b nvme0 -a 00:0a.0 -t pcie + +$ scripts/rpc.py bdev_ocssd_create -c nvme0 -b nvme0n1 + nvme0n1 + +$ scripts/rpc.py bdev_ftl_create -b ftl0 -d nvme0n1 +{ + "name": "ftl0", + "uuid": "3b469565-1fa5-4bfb-8341-747ec9fca9b9" +} +``` + +## FTL usage with zone block bdev {#ftl_zone_block} + +Zone block bdev is a bdev adapter between regular `bdev` and `bdev_zone`. It emulates a zoned +interface on top of a regular block device. + +In order to create FTL on top of a regular bdev: +1) Create regular bdev e.g. `bdev_nvme`, `bdev_null`, `bdev_malloc` +2) Create zone block bdev on top of a regular bdev created in step 1 (user could specify zone capacity +and optimal number of open zones) +3) Create FTL bdev on top of bdev created in step 2 + +Example: +``` +$ scripts/rpc.py bdev_nvme_attach_controller -b nvme0 -a 00:05.0 -t pcie + nvme0n1 + +$ scripts/rpc.py bdev_zone_block_create -b zone1 -n nvme0n1 -z 4096 -o 32 + zone1 + +$ scripts/rpc.py bdev_ftl_create -b ftl0 -d zone1 +{ + "name": "ftl0", + "uuid": "3b469565-1fa5-4bfb-8341-747ec9f3a9b9" +} +``` diff --git a/src/spdk/doc/gdb_macros.md b/src/spdk/doc/gdb_macros.md new file mode 100644 index 000000000..6fc65aed7 --- /dev/null +++ b/src/spdk/doc/gdb_macros.md @@ -0,0 +1,221 @@ +# GDB Macros User Guide {#gdb_macros} + +# Introduction + +When debugging an spdk application using gdb we may need to view data structures +in lists, e.g. information about bdevs or threads. + +If, for example I have several bdevs, and I wish to get information on bdev by +the name 'test_vols3', I will need to manually iterate over the list as follows: + +~~~{.sh} +(gdb) p g_bdev_mgr->bdevs->tqh_first->name +$5 = 0x7f7dcc0b21b0 "test_vols1" +(gdb) p g_bdev_mgr->bdevs->tqh_first->internal->link->tqe_next->name +$6 = 0x7f7dcc0b1a70 "test_vols2" +(gdb) p +g_bdev_mgr->bdevs->tqh_first->internal->link->tqe_next->internal->link->tqe_next->name +$7 = 0x7f7dcc215a00 "test_vols3" +(gdb) p +g_bdev_mgr->bdevs->tqh_first->internal->link->tqe_next->internal->link->tqe_next +$8 = (struct spdk_bdev *) 0x7f7dcc2c7c08 +~~~ + +At this stage, we can start looking at the relevant fields of our bdev which now +we know is in address 0x7f7dcc2c7c08. + +This can be somewhat troublesome if there are 100 bdevs, and the one we need is +56th in the list... + +Instead, we can use a gdb macro in order to get information about all the +devices. + +Examples: + +Printing bdevs: + +~~~{.sh} +(gdb) spdk_print_bdevs + +SPDK object of type struct spdk_bdev at 0x7f7dcc1642a8 +((struct spdk_bdev*) 0x7f7dcc1642a8) +name 0x7f7dcc0b21b0 "test_vols1" + +--------------- + +SPDK object of type struct spdk_bdev at 0x7f7dcc216008 +((struct spdk_bdev*) 0x7f7dcc216008) +name 0x7f7dcc0b1a70 "test_vols2" + +--------------- + +SPDK object of type struct spdk_bdev at 0x7f7dcc2c7c08 +((struct spdk_bdev*) 0x7f7dcc2c7c08) +name 0x7f7dcc215a00 "test_vols3" + +--------------- +~~~ + +Finding a bdev by name: + +~~~{.sh} +(gdb) spdk_find_bdev test_vols1 +test_vols1 + +SPDK object of type struct spdk_bdev at 0x7f7dcc1642a8 +((struct spdk_bdev*) 0x7f7dcc1642a8) +name 0x7f7dcc0b21b0 "test_vols1" +~~~ + +Printing spdk threads: + +~~~{.sh} +(gdb) spdk_print_threads + +SPDK object of type struct spdk_thread at 0x7fffd0008b50 +((struct spdk_thread*) 0x7fffd0008b50) +name 0x7fffd00008e0 "reactor_1" +IO Channels: + SPDK object of type struct spdk_io_channel at 0x7fffd0052610 + ((struct spdk_io_channel*) 0x7fffd0052610) + name + ref 1 + device 0x7fffd0008c80 (0x7fffd0008ce0 "nvmf_tgt") + --------------- + + SPDK object of type struct spdk_io_channel at 0x7fffd0056cd0 + ((struct spdk_io_channel*) 0x7fffd0056cd0) + name + ref 2 + device 0x7fffd0056bf0 (0x7fffd0008e70 "test_vol1") + --------------- + + SPDK object of type struct spdk_io_channel at 0x7fffd00582e0 + ((struct spdk_io_channel*) 0x7fffd00582e0) + name + ref 1 + device 0x7fffd0056c50 (0x7fffd0056cb0 "bdev_test_vol1") + --------------- + + SPDK object of type struct spdk_io_channel at 0x7fffd00583b0 + ((struct spdk_io_channel*) 0x7fffd00583b0) + name + ref 1 + device 0x7fffd0005630 (0x7fffd0005690 "bdev_mgr") + --------------- +~~~ + +Printing nvmf subsystems: + +~~~{.sh} +(gdb) spdk_print_nvmf_subsystems + +SPDK object of type struct spdk_nvmf_subsystem at 0x7fffd0008d00 +((struct spdk_nvmf_subsystem*) 0x7fffd0008d00) +name "nqn.2014-08.org.nvmexpress.discovery", '\000' <repeats 187 times> +nqn "nqn.2014-08.org.nvmexpress.discovery", '\000' <repeats 187 times> +ID 0 + +--------------- + +SPDK object of type struct spdk_nvmf_subsystem at 0x7fffd0055760 +((struct spdk_nvmf_subsystem*) 0x7fffd0055760) +name "nqn.2016-06.io.spdk.umgmt:cnode1", '\000' <repeats 191 times> +nqn "nqn.2016-06.io.spdk.umgmt:cnode1", '\000' <repeats 191 times> +ID 1 +~~~ + +# Loading The gdb Macros + +Copy the gdb macros to the host where you are about to debug. +It is best to copy the file either to somewhere within the PYTHONPATH, or to add +the destination directory to the PYTHONPATH. This is not mandatory, and can be +worked around, but can save a few steps when loading the module to gdb. + +From gdb, with the application core open, invoke python and load the modules. + +In the example below, I copied the macros to the /tmp directory which is not in +the PYTHONPATH, so I had to manually add the directory to the path. + +~~~{.sh} +(gdb) python +>import sys +>sys.path.append('/tmp') +>import gdb_macros +>end +(gdb) spdk_load_macros +~~~ + +# Using the gdb Data Directory + +On most systems, the data directory is /usr/share/gdb. The python script should +be copied into the python/gdb/function (or python/gdb/command) directory under +the data directory, e.g. /usr/share/gdb/python/gdb/function. + +If the python script is in there, then the only thing you need to do when +starting gdb is type "spdk_load_macros". + +# Using .gdbinit To Load The Macros + +.gdbinit can also be used in order to run automatically run the manual steps +above prior to starting gdb. + +Exmaple .gdbinit: + +~~~{.sh} +source /opt/km/install/tools/gdb_macros/gdb_macros.py +~~~ + +When starting gdb you still have to call spdk_load_macros. + +# Why Do We Need to Explicitly Call spdk_load_macros + +The reason is that the macros need to use globals provided by spdk in order to +iterate the spdk lists and build iterable representations of the list objects. +This will result in errors if these are not available which is very possible if +gdb is used for reasons other than debugging spdk core dumps. + +In the example bellow, I attempted to load the macros when the globals are not +available causing gdb to fail loading the gdb_macros: + +~~~{.sh} +(gdb) spdk_load_macros +Traceback (most recent call last): + File "/opt/km/install/tools/gdb_macros/gdb_macros.py", line 257, in invoke + spdk_print_threads() + File "/opt/km/install/tools/gdb_macros/gdb_macros.py", line 241, in __init__ + threads = SpdkThreads() + File "/opt/km/install/tools/gdb_macros/gdb_macros.py", line 234, in __init__ + super(SpdkThreads, self).__init__('g_threads', SpdkThread) + File "/opt/km/install/tools/gdb_macros/gdb_macros.py", line 25, in __init__ + ['tailq']) + File "/opt/km/install/tools/gdb_macros/gdb_macros.py", line 10, in __init__ + self.list = gdb.parse_and_eval(self.list_pointer) +RuntimeError: No symbol table is loaded. Use the "file" command. +Error occurred in Python command: No symbol table is loaded. Use the "file" +command. +~~~ + +# Macros available + +- spdk_load_macros: load the macros (use --reload in order to reload them) +- spdk_print_bdevs: information about bdevs +- spdk_find_bdev: find a bdev (substring search) +- spdk_print_io_devices: information about io devices +- spdk_print_nvmf_subsystems: information about nvmf subsystems +- spdk_print_threads: information about threads + +# Adding New Macros + +The list iteration macros are usually built from 3 layers: + +- SpdkPrintCommand: inherits from gdb.Command and invokes the list iteration +- SpdkTailqList: Performs the iteration of a tailq list according to the tailq + member implementation +- SpdkObject: Provides the __str__ function so that the list iteration can print + the object + +Other useful objects: + +- SpdkNormalTailqList: represents a list which has 'tailq' as the tailq object +- SpdkArr: Iteration over an array (instead of a linked list) diff --git a/src/spdk/doc/general.md b/src/spdk/doc/general.md new file mode 100644 index 000000000..168f89fd7 --- /dev/null +++ b/src/spdk/doc/general.md @@ -0,0 +1,6 @@ +# General Information {#general} + +- @subpage event +- @subpage logical_volumes +- @subpage vpp_integration +- @subpage accel_fw diff --git a/src/spdk/doc/getting_started.md b/src/spdk/doc/getting_started.md new file mode 100644 index 000000000..7dadaae20 --- /dev/null +++ b/src/spdk/doc/getting_started.md @@ -0,0 +1,117 @@ +# Getting Started {#getting_started} + +# Getting the Source Code {#getting_started_source} + +~~~{.sh} +git clone https://github.com/spdk/spdk +cd spdk +git submodule update --init +~~~ + +# Installing Prerequisites {#getting_started_prerequisites} + +The `scripts/pkgdep.sh` script will automatically install the bare minimum +dependencies required to build SPDK. +Use `--help` to see information on installing dependencies for optional components. + +~~~{.sh} +sudo scripts/pkgdep.sh +~~~ + +Option --all will install all dependencies needed by SPDK features. + +~~~{.sh} +sudo scripts/pkgdep.sh --all +~~~ + +# Building {#getting_started_building} + +Linux: + +~~~{.sh} +./configure +make +~~~ + +FreeBSD: +Note: Make sure you have the matching kernel source in /usr/src/ + +~~~{.sh} +./configure +gmake +~~~ + +There are a number of options available for the configure script, which can +be viewed by running + +~~~{.sh} +./configure --help +~~~ + +Note that not all features are enabled by default. For example, RDMA +support (and hence NVMe over Fabrics) is not enabled by default. You +can enable it by doing the following: + +~~~{.sh} +./configure --with-rdma +make +~~~ + +# Running the Unit Tests {#getting_started_unittests} + +It's always a good idea to confirm your build worked by running the +unit tests. + +~~~{.sh} +./test/unit/unittest.sh +~~~ + +You will see several error messages when running the unit tests, but they are +part of the test suite. The final message at the end of the script indicates +success or failure. + +# Running the Example Applications {#getting_started_examples} + +Before running an SPDK application, some hugepages must be allocated and +any NVMe and I/OAT devices must be unbound from the native kernel drivers. +SPDK includes a script to automate this process on both Linux and FreeBSD. +This script should be run as root. It only needs to be run once on the +system. + +~~~{.sh} +sudo scripts/setup.sh +~~~ + +To rebind devices back to the kernel, you can run + +~~~{.sh} +sudo scripts/setup.sh reset +~~~ + +By default, the script allocates 2048MB of hugepages. To change this number, +specify HUGEMEM (in MB) as follows: + +~~~{.sh} +sudo HUGEMEM=4096 scripts/setup.sh +~~~ + +On Linux machines HUGEMEM will be rounded up to system-default huge page +size boundary. + +All available params can be viewed by running + +~~~{.sh} +scripts/setup.sh help +~~~ + +Example code is located in the examples directory. The examples are compiled +automatically as part of the build process. Simply call any of the examples +with no arguments to see the help output. If your system has its IOMMU +enabled you can run the examples as your regular user. If it doesn't, you'll +need to run as a privileged user (root). + +A good example to start with is `build/examples/identify`, which prints +out information about all of the NVMe devices on your system. + +Larger, more fully functional applications are available in the `app` +directory. This includes the iSCSI and NVMe-oF target. diff --git a/src/spdk/doc/header.html b/src/spdk/doc/header.html new file mode 100644 index 000000000..6299578b7 --- /dev/null +++ b/src/spdk/doc/header.html @@ -0,0 +1,25 @@ +<!DOCTYPE html> +<html> +<head> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta http-equiv="Content-Type" content="text/xhtml; charset=utf-8"> + <meta name="generator" content="Doxygen $doxygenversion"> + + <title>$projectname: $title</title> + + <script type="text/javascript" src="$relpath^jquery.js"></script> + <script type="text/javascript" src="$relpath^dynsections.js"></script> + <script type="text/javascript" src="$relpath^two.min.js"></script> + + $treeview + $search + $mathjax + + <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:400,900" type="text/css"> + <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous"> + <link rel="stylesheet" href="$relpath^tabs.css" type="text/css"> + $extrastylesheet +</head> +<body> +<div class="container-fluid"> + <div id="top"> <!-- do not remove this div, it is closed by doxygen! --> diff --git a/src/spdk/doc/idxd.md b/src/spdk/doc/idxd.md new file mode 100644 index 000000000..8c33770ef --- /dev/null +++ b/src/spdk/doc/idxd.md @@ -0,0 +1,28 @@ +# IDXD Driver {#idxd} + +# Public Interface {#idxd_interface} + +- spdk/idxd.h + +# Key Functions {#idxd_key_functions} + +Function | Description +--------------------------------------- | ----------- +spdk_idxd_probe() | @copybrief spdk_idxd_probe() +spdk_idxd_batch_get_max() | @copybrief spdk_idxd_batch_get_max() +spdk_idxd_batch_create() | @copybrief spdk_idxd_batch_create() +spdk_idxd_batch_prep_copy() | @copybrief spdk_idxd_batch_prep_copy() +spdk_idxd_batch_submit() | @copybrief spdk_idxd_batch_submit() +spdk_idxd_submit_copy() | @copybrief spdk_idxd_submit_copy() +spdk_idxd_submit_compare() | @copybrief spdk_idxd_submit_compare() +spdk_idxd_submit_crc32c() | @copybrief spdk_idxd_submit_crc32c() +spdk_idxd_submit_dualcast | @copybrief spdk_idxd_submit_dualcast() +spdk_idxd_submit_fill() | @copybrief spdk_idxd_submit_fill() + +# Pre-defined configurations {#idxd_configs} + +The RPC `idxd_scan_accel_engine` is used to both enable IDXD and set it's +configuration to one of two pre-defined configs: + +Config #0: 4 groups, 1 work queue per group, 1 engine per group. +Config #1: 2 groups, 2 work queues per group, 2 engines per group. diff --git a/src/spdk/doc/img/iscsi.svg b/src/spdk/doc/img/iscsi.svg new file mode 100644 index 000000000..2ba4b9634 --- /dev/null +++ b/src/spdk/doc/img/iscsi.svg @@ -0,0 +1,827 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="174.625mm" + height="82.020836mm" + version="1.1" + viewBox="0 0 174.625 82.020833" + id="svg136" + sodipodi:docname="iscsi.svg" + inkscape:version="0.92.3 (2405546, 2018-03-11)"> + <sodipodi:namedview + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1" + objecttolerance="10" + gridtolerance="10" + guidetolerance="10" + inkscape:pageopacity="0" + inkscape:pageshadow="2" + inkscape:window-width="1387" + inkscape:window-height="888" + id="namedview138" + showgrid="true" + inkscape:zoom="0.9096286" + inkscape:cx="242.15534" + inkscape:cy="182.31015" + inkscape:window-x="1974" + inkscape:window-y="112" + inkscape:window-maximized="0" + inkscape:current-layer="svg136" + fit-margin-top="0" + fit-margin-left="0" + fit-margin-right="0" + fit-margin-bottom="0"> + <inkscape:grid + type="xygrid" + id="grid2224" + originx="38.364584" + originy="-17.197913" /> + </sodipodi:namedview> + <title + id="title2">Thin Provisioning Write</title> + <defs + id="defs22"> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker5538" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path5536" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker5348" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mstart" + inkscape:collect="always"> + <path + inkscape:connector-curvature="0" + transform="matrix(0.4,0,0,0.4,4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path5346" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker5152" + style="overflow:visible" + inkscape:isstock="true" + inkscape:collect="always"> + <path + id="path5150" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker4974" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mstart" + inkscape:collect="always"> + <path + inkscape:connector-curvature="0" + transform="matrix(0.4,0,0,0.4,4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path4972" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker4802" + style="overflow:visible" + inkscape:isstock="true" + inkscape:collect="always"> + <path + id="path4800" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker4636" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mstart" + inkscape:collect="always"> + <path + inkscape:connector-curvature="0" + transform="matrix(0.4,0,0,0.4,4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path4634" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker4476" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mstart"> + <path + inkscape:connector-curvature="0" + transform="matrix(0.4,0,0,0.4,4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path4474" /> + </marker> + <marker + inkscape:stockid="Arrow1Mend" + orient="auto" + refY="0" + refX="0" + id="marker2468" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2466" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff2a2a;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2464" + style="overflow:visible" + inkscape:isstock="true" + inkscape:collect="always"> + <path + id="path2462" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff2a2a;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="Arrow1Mstart" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2198" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff2a2a;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mend" + orient="auto" + refY="0" + refX="0" + id="Arrow1Mend" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2201" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff2a2a;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-5" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-9" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-9" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-6" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-5-2" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-9-3" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-9-4" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-6-9" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-5-27" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-9-4" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-5-27-9" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-9-4-4" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2683-6" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2681-3" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mend" + orient="auto" + refY="0" + refX="0" + id="marker2679-9" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2677-8" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + inkscape:connector-curvature="0" /> + </marker> + </defs> + <metadata + id="metadata24"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title>Thin Provisioning Write</dc:title> + </cc:Work> + </rdf:RDF> + </metadata> + <rect + style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.26458332;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + id="rect7030" + width="174.625" + height="82.020836" + x="0" + y="1.4210855e-014" + ry="0" /> + <rect + style="fill:none;fill-opacity:1;stroke:#999999;stroke-width:0.5;stroke-opacity:1" + id="rect132-6" + ry="1.3229001" + height="50.270832" + width="75.406242" + y="-91.281242" + x="2.6458344" + transform="rotate(90)" /> + <rect + x="50.270416" + y="19.84375" + width="22.49" + height="6.6146002" + id="rect104" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <rect + style="fill:none;stroke:#999999;stroke-width:0.26458332;stroke-miterlimit:4;stroke-dasharray:none" + id="rect132" + ry="1.3229001" + height="30.427082" + width="33.072914" + y="-76.729164" + x="11.906253" + transform="rotate(90)" /> + <text + x="56.69899" + y="24.392132" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90">LUN0</text> + <rect + style="fill:none;fill-opacity:1;stroke:#999999;stroke-width:0.5;stroke-opacity:1" + id="rect132-6-8" + ry="1.3229001" + height="33.072914" + width="64.822906" + y="-35.718758" + x="10.583331" + transform="rotate(90)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2" + d="m 30.427087,23.812498 19.843748,3e-6" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26511249;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker4476);marker-end:url(#marker1826-2-4-7-1-7)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path11761-9-7" + d="m 105.83333,33.072917 38.36458,2e-6" + style="fill:#ff0000;stroke:#ff2a2a;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker2464);marker-end:url(#marker2468)" /> + <rect + x="50.270416" + y="27.781233" + width="22.49" + height="6.6146002" + id="rect104-6" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <rect + x="50.270836" + y="35.718746" + width="22.49" + height="6.6146002" + id="rect104-5" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="49.004951" + y="16.552654" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5">Target1</text> + <text + x="56.810654" + y="32.229481" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-59">LUN1</text> + <text + x="56.853249" + y="40.350986" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-0">LUN2</text> + <text + x="43.28257" + y="6.9284844" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5">iSCSI Target server</text> + <rect + x="50.270416" + y="55.562496" + width="22.49" + height="6.6146002" + id="rect104-0" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <rect + style="fill:none;stroke:#999999;stroke-width:0.26458332;stroke-miterlimit:4;stroke-dasharray:none" + id="rect132-3" + ry="1.3229001" + height="30.427078" + width="25.135414" + y="-76.729164" + x="47.624996" + transform="rotate(90)" /> + <text + x="56.69899" + y="60.110878" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-05">LUN0</text> + <rect + x="50.270416" + y="63.499977" + width="22.49" + height="6.6146002" + id="rect104-6-8" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="49.004944" + y="52.2714" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-2">Target2</text> + <text + x="56.810646" + y="67.948235" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-59-4">LUN1</text> + <rect + x="7.937088" + y="19.84375" + width="22.49" + height="6.6146002" + id="rect104-64" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="14.365662" + y="24.392132" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-56">bdev0</text> + <rect + x="7.937088" + y="27.781233" + width="22.49" + height="6.6146002" + id="rect104-6-9" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <rect + x="7.9375038" + y="35.718746" + width="22.49" + height="6.6146002" + id="rect104-5-4" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="14.477322" + y="32.229481" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-59-2">bdev1</text> + <text + x="14.51992" + y="40.350986" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-0-5">bdev2</text> + <rect + x="7.937088" + y="55.562496" + width="22.49" + height="6.6146002" + id="rect104-0-8" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="14.365662" + y="60.110878" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-05-7">bdev3</text> + <rect + x="7.937088" + y="63.499977" + width="22.49" + height="6.6146002" + id="rect104-6-8-2" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="14.477322" + y="67.948235" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-59-4-0">bdev4</text> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2-6" + d="m 30.427087,31.749998 19.843748,3e-6" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker4636);marker-end:url(#marker1826-2-4-7-1-7-5)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2-4" + d="m 30.427087,39.687498 19.843748,2e-6" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker4802);marker-end:url(#marker1826-2-4-7-1-7-9)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2-6-5" + d="m 30.427087,59.531248 19.843748,2e-6" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker4974);marker-end:url(#marker1826-2-4-7-1-7-5-2)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2-4-5" + d="m 30.427087,67.468748 19.843748,10e-7" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker5152);marker-end:url(#marker1826-2-4-7-1-7-9-4)" /> + <rect + x="83.343323" + y="29.104166" + width="22.49" + height="6.6146002" + id="rect104-63" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="84.467346" + y="33.405464" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-1">portal grp 0</text> + <rect + x="83.343323" + y="54.239578" + width="22.49" + height="6.6146002" + id="rect104-63-1" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="84.673019" + y="58.540874" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-1-7">portal grp 1</text> + <text + x="4.7052402" + y="14.717848" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5-8">SPDK bdevs</text> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2-6-4" + d="m 76.729167,33.072917 h 6.614587" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker5348);marker-end:url(#marker1826-2-4-7-1-7-5-27)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2-6-4-2" + d="m 76.729167,58.208333 h 6.614587" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker5538);marker-end:url(#marker1826-2-4-7-1-7-5-27-9)" /> + <rect + x="144.19748" + y="29.104151" + width="22.49" + height="6.6146002" + id="rect104-63-9" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="147.16313" + y="33.713963" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-1-8">initiator 0</text> + <rect + x="144.19748" + y="54.239567" + width="22.49" + height="6.6146002" + id="rect104-63-1-5" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="147.23584" + y="58.922092" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-1-7-0">initiator 1</text> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path11761-9-7-9" + d="m 105.83333,58.208333 38.36458,2e-6" + style="fill:#ff0000;stroke:#ff2a2a;stroke-width:0.26511249;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow1Mstart);marker-end:url(#Arrow1Mend)" /> + <rect + style="fill:none;stroke:#999999;stroke-width:0.5" + id="rect132-6-1" + ry="1.3229001" + height="33.072926" + width="38.364586" + y="-171.97916" + x="2.6458333" + transform="rotate(90)" /> + <rect + style="fill:none;stroke:#999999;stroke-width:0.5" + id="rect132-6-1-3" + ry="1.3229001" + height="33.072914" + width="35.71875" + y="-171.97916" + x="43.65625" + transform="rotate(90)" /> + <text + x="141.38495" + y="7.1341634" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5-7">iSCSI client 0</text> + <text + x="141.15009" + y="48.275509" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5-7-5">iSCSI client 1</text> + <path + style="display:inline;fill:none;stroke:#999999;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + d="M 105.83333,87.312502 124.35416,1.3229172" + id="path2638" + inkscape:connector-curvature="0" + sodipodi:nodetypes="cc" /> + <path + style="display:inline;fill:none;stroke:#999999;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + d="M 107.15625,88.635419 125.67708,2.6458333" + id="path2640" + inkscape:connector-curvature="0" + sodipodi:nodetypes="cc" /> + <text + x="105.28584" + y="13.99068" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5-9">TCP Network</text> + <path + style="display:inline;fill:none;stroke:#000000;stroke-width:0.52916664;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker2683-6);marker-end:url(#marker2679-9)" + d="m 107.15625,17.197917 h 18.52083" + id="path2669" + inkscape:connector-curvature="0" + sodipodi:nodetypes="cc" /> + <g + id="g4350-40" + transform="matrix(1,0,0,0.61904764,50.020836,28.004467)"> + <ellipse + ry="2.6458333" + rx="6.614583" + cy="-11.045678" + cx="104.76043" + id="path4344-1" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path4346-6" + d="m 98.145835,-11.045677 v 6.4110574 c 10e-6,3.968751 13.229165,3.968751 13.229165,0 v -6.4110574 c 0,4.2740384 -13.229155,3.9687504 -13.229165,0 z" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" /> + <ellipse + ry="2.645833" + rx="6.6145835" + cy="-17.456738" + cx="104.76044" + id="path4344-1-7" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path4346-6-3" + d="m 98.145841,-17.456734 v 6.411057 c 10e-6,3.968751 13.229159,3.968751 13.229159,0 v -6.411057 c 0,4.274038 -13.229149,3.96875 -13.229159,0 z" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" /> + <ellipse + ry="2.645833" + rx="6.6145835" + cy="-23.867794" + cx="104.76044" + id="path4344-1-9" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path4346-6-2" + d="m 98.145841,-23.867792 v 6.411058 c 10e-6,3.968751 13.229159,3.968751 13.229159,0 v -6.411058 c 0,4.274039 -13.229149,3.968751 -13.229159,0 z" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" /> + <ellipse + ry="2.645833" + rx="6.6145835" + cy="72.298073" + cx="106.08334" + id="path4344-1-5" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" /> + <ellipse + ry="2.645833" + rx="6.6145835" + cy="65.887009" + cx="106.08335" + id="path4344-1-7-3" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path4346-6-3-4" + d="m 99.468754,65.887013 v 6.411057 c 10e-6,3.968751 13.229156,3.968751 13.229156,0 v -6.411057 c 0,4.274038 -13.229146,3.96875 -13.229156,0 z" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" /> + <ellipse + ry="2.645833" + rx="6.6145835" + cy="59.475952" + cx="106.08335" + id="path4344-1-9-1" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path4346-6-2-9" + d="m 99.468754,59.475955 v 6.411058 c 10e-6,3.968751 13.229156,3.968751 13.229156,0 v -6.411058 c 0,4.274039 -13.229146,3.968751 -13.229156,0 z" + style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.52916664;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" /> + </g> +</svg> diff --git a/src/spdk/doc/img/iscsi_example.svg b/src/spdk/doc/img/iscsi_example.svg new file mode 100644 index 000000000..b5f7ea09c --- /dev/null +++ b/src/spdk/doc/img/iscsi_example.svg @@ -0,0 +1,540 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="169.33331mm" + height="53.006062mm" + version="1.1" + viewBox="0 0 169.33331 53.00606" + id="svg136" + sodipodi:docname="iscsi_example.svg" + inkscape:version="0.92.3 (2405546, 2018-03-11)"> + <sodipodi:namedview + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1" + objecttolerance="10" + gridtolerance="10" + guidetolerance="10" + inkscape:pageopacity="0" + inkscape:pageshadow="2" + inkscape:window-width="1742" + inkscape:window-height="910" + id="namedview138" + showgrid="true" + inkscape:zoom="1.2864091" + inkscape:cx="231.4415" + inkscape:cy="205.83148" + inkscape:window-x="1676" + inkscape:window-y="113" + inkscape:window-maximized="0" + inkscape:current-layer="layer1" + fit-margin-top="0" + fit-margin-left="0" + fit-margin-right="0" + fit-margin-bottom="0"> + <inkscape:grid + type="xygrid" + id="grid2224" + originx="33.072915" + originy="-46.257384" /> + </sodipodi:namedview> + <title + id="title2">Thin Provisioning Write</title> + <defs + id="defs22"> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2683-6" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2681-3" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mend" + orient="auto" + refY="0" + refX="0" + id="marker2679-9" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2677-8" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2464-2-6-1" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2462-7-8-2" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#999999;fill-opacity:1;fill-rule:evenodd;stroke:#999999;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mend" + orient="auto" + refY="0" + refX="0" + id="marker2468-8-9-5" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2466-1-3-2" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#999999;fill-opacity:1;fill-rule:evenodd;stroke:#999999;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2464-2-0" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2462-7-6" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#999999;fill-opacity:1;fill-rule:evenodd;stroke:#999999;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mend" + orient="auto" + refY="0" + refX="0" + id="marker2468-8-8" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2466-1-5" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#999999;fill-opacity:1;fill-rule:evenodd;stroke:#999999;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2659-1" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2657-7" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-5-27-1" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-9-4-0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2667-4" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2665-0" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-5-9" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-9-9" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2464-3" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2462-5" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff2a2a;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mend" + orient="auto" + refY="0" + refX="0" + id="marker2468-5" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2466-4" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff2a2a;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:stockid="Arrow1Mstart" + orient="auto" + refY="0" + refX="0" + id="marker2663-8" + style="overflow:visible" + inkscape:isstock="true"> + <path + id="path2661-0" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + transform="matrix(0.4,0,0,0.4,4,0)" + inkscape:connector-curvature="0" /> + </marker> + <marker + inkscape:isstock="true" + style="overflow:visible" + id="marker1826-2-4-7-1-7-97" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Mend"> + <path + inkscape:connector-curvature="0" + transform="matrix(-0.4,0,0,-0.4,-4,0)" + style="fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#0000ff;stroke-width:1.00000003pt;stroke-opacity:1" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path1824-9-4-2-5-2-93" /> + </marker> + </defs> + <metadata + id="metadata24"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title>Thin Provisioning Write</dc:title> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:groupmode="layer" + id="layer1" + inkscape:label="Layer 1" + style="display:inline" + transform="translate(-20.09375,9.9883163e-4)"> + <rect + style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.52916664;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + id="rect2890" + width="169.33331" + height="52.916664" + x="20.09375" + y="0.043701001" /> + <rect + x="70.364159" + y="19.887449" + width="22.49" + height="6.6146002" + id="rect104" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <rect + style="fill:none;stroke:#999999;stroke-width:0.26458332;stroke-miterlimit:4;stroke-dasharray:none" + id="rect132" + ry="1.3229001" + height="30.427082" + width="33.072914" + y="-96.822914" + x="11.949952" + transform="rotate(90)" /> + <text + x="76.792732" + y="24.435831" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90">LUN0</text> + <rect + x="70.364159" + y="27.824934" + width="22.49" + height="6.6146002" + id="rect104-6" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="69.098686" + y="16.596354" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5">Target: disk1</text> + <text + x="76.904396" + y="32.273182" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-59">LUN1</text> + <text + x="63.376305" + y="6.9721842" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5">iSCSI Target server</text> + <rect + x="28.030828" + y="19.887449" + width="22.49" + height="6.6146002" + id="rect104-64" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="33.225346" + y="24.641508" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-56">Malloc0</text> + <rect + x="28.03083" + y="27.824945" + width="22.49" + height="6.6146002" + id="rect104-6-9" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="33.337006" + y="32.273182" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-59-2">Malloc1</text> + <rect + style="fill:none;stroke:#999999;stroke-width:0.5" + id="rect132-6" + ry="1.3229001" + height="50.270836" + width="47.624996" + y="-111.375" + x="2.6895342" + transform="rotate(90)" /> + <rect + style="fill:none;stroke:#999999;stroke-width:0.5" + id="rect132-6-8" + ry="1.3229001" + height="33.072918" + width="27.781242" + y="-55.812492" + x="11.949948" + transform="rotate(90)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2-6" + d="m 50.520827,31.793698 19.843748,3e-6" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker2667-4);marker-end:url(#marker1826-2-4-7-1-7-5-9)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2" + d="m 50.520827,23.856198 19.843748,2e-6" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker2663-8);marker-end:url(#marker1826-2-4-7-1-7-97)" /> + <rect + x="103.4371" + y="37.085365" + width="18.521248" + height="6.6145835" + id="rect104-63" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="105.57915" + y="41.386662" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-1">portal 1</text> + <text + x="25.394737" + y="15.738133" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5-8">SPDK bdevs</text> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path1192-8-7-7-4-2-6-4" + d="M 96.822918,41.054113 H 103.4375" + style="fill:#0000ff;stroke:#0000ff;stroke-width:0.26511249;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker2659-1);marker-end:url(#marker1826-2-4-7-1-7-5-27-1)" /> + <rect + x="158.99957" + y="37.08535" + width="22.49" + height="6.6146002" + id="rect104-63-9" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="161.96524" + y="41.69516" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-1-8">initiator 2</text> + <rect + style="fill:none;stroke:#999999;stroke-width:0.5" + id="rect132-6-1" + ry="1.3229001" + height="33.072933" + width="38.364578" + y="-186.78125" + x="11.949951" + transform="rotate(90)" /> + <text + x="156.03279" + y="15.81625" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5-7">iSCSI client 0</text> + <text + x="101.36903" + y="47.613781" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-7">10.0.0.1:3260</text> + <rect + x="161.64542" + y="19.887432" + width="19.844177" + height="6.6146011" + id="rect104-9" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="168.07399" + y="24.435814" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-7">sdd</text> + <rect + x="161.64542" + y="27.824913" + width="19.844177" + height="6.6146178" + id="rect104-6-8" + style="fill:#fff6d5;fill-opacity:1;stroke:#000000;stroke-width:0.26458001" /> + <text + x="168.18565" + y="32.273163" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-59-1">sde</text> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path11761-9-7-0" + d="m 92.854164,23.8562 68.791666,-1e-6" + style="fill:#999999;fill-opacity:1;stroke:#999999;stroke-width:0.26511249;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.06044998, 1.06044998;stroke-dashoffset:0;stroke-opacity:1;marker-start:url(#marker2464-2-0);marker-end:url(#marker2468-8-8)" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path11761-9-7-0-0" + d="m 92.854164,31.7937 68.791666,-2e-6" + style="fill:#999999;fill-opacity:1;stroke:#999999;stroke-width:0.26511249;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.06044998, 1.06044998;stroke-dashoffset:0;stroke-opacity:1;marker-start:url(#marker2464-2-6-1);marker-end:url(#marker2468-8-9-5)" /> + <text + x="160.41017" + y="47.490952" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-7-2">10.0.0.2/32</text> + <path + style="fill:none;stroke:#999999;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + d="M 125.92708,51.63745 144.44792,0.04369787" + id="path2638" + inkscape:connector-curvature="0" + sodipodi:nodetypes="cc" /> + <path + style="fill:none;stroke:#999999;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + d="M 127.25,52.960366 145.77084,1.3666139" + id="path2640" + inkscape:connector-curvature="0" + sodipodi:nodetypes="cc" /> + <path + sodipodi:nodetypes="cc" + inkscape:connector-curvature="0" + id="path11761-9-7" + d="M 121.95833,41.054117 159,41.054115" + style="fill:#ff0000;stroke:#ff2a2a;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker2464-3);marker-end:url(#marker2468-5)" /> + <text + x="122.73377" + y="8.7427139" + font-size="3.5278px" + style="font-size:3.52780008px;line-height:1.25;font-family:sans-serif;font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;letter-spacing:0px;word-spacing:0px;fill:#000000;stroke-width:0.26458001" + xml:space="preserve" + id="text90-5-5-9">TCP Network</text> + <path + style="fill:none;stroke:#000000;stroke-width:0.52916664;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#marker2683-6);marker-end:url(#marker2679-9)" + d="M 124.60417,11.949951 H 143.125" + id="path2669" + inkscape:connector-curvature="0" /> + </g> +</svg> diff --git a/src/spdk/doc/img/lvol_clone_snapshot_read.svg b/src/spdk/doc/img/lvol_clone_snapshot_read.svg new file mode 100644 index 000000000..0f91d417b --- /dev/null +++ b/src/spdk/doc/img/lvol_clone_snapshot_read.svg @@ -0,0 +1,3 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg width="173.3mm" height="87.312mm" version="1.1" viewBox="0 0 173.3 87.312" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><defs><marker id="a" overflow="visible" orient="auto"><path transform="scale(.4) rotate(180) translate(10)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="Arrow1Mend" overflow="visible" orient="auto"><path transform="scale(.4) rotate(180) translate(10)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="g" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="f" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="h" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="d" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="b" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="c" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="e" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker></defs><metadata><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title/></cc:Work></rdf:RDF></metadata><g transform="translate(2.6458 2.5135)"><rect x="-2.6458" y="-2.5135" width="173.3" height="87.312" ry="0" fill="#fff"/><path d="m5.2917 53.049v-9.2604" fill="none" marker-end="url(#a)" stroke="#000" stroke-width=".26458px"/><path d="m5.2917 38.497v-9.2604" fill="none" marker-end="url(#Arrow1Mend)" stroke="#000" stroke-width=".26458px"/></g><g transform="translate(-14.552 -4.6354)"><g stroke="#000" stroke-width=".26458"><rect x="47.625" y="29.771" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect x="70.115" y="29.771" width="22.49" height="6.6146" fill="#d7d7f4"/><rect x="92.604" y="29.771" width="22.49" height="6.6146" fill="#d7d7f4"/><rect x="115.09" y="29.771" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect x="137.58" y="29.771" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect x="160.07" y="29.771" width="22.49" height="6.6146" fill="#d7d7f4"/></g><g fill="#d7f4d7" stroke="#000" stroke-width=".26458"><rect x="92.604" y="44.323" width="22.49" height="6.6146"/><rect x="115.09" y="44.323" width="22.49" height="6.6146"/><rect x="137.58" y="44.323" width="22.49" height="6.6146"/><rect x="160.07" y="44.323" width="22.49" height="6.6146"/></g><text x="18.309231" y="49.435097" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="18.309231" y="49.435097" stroke-width=".26458">snapshot</tspan></text><text x="18.362614" y="11.230336" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="18.362614" y="11.230336" stroke-width=".26458">read</tspan></text><g stroke="#000"><path d="m17.198 21.833 168.01-1e-6" fill="none" stroke-dasharray="1.59, 1.59" stroke-width=".265"/><g stroke-width=".26458"><rect x="47.625" y="7.2812" width="22.49" height="6.6146" fill="#f4d7d7" stroke-dasharray="0.52916663, 0.26458332"/><rect x="115.09" y="7.2813" width="22.49" height="6.6146" fill="#d7f4d7"/><rect x="137.58" y="7.2812" width="22.49" height="6.6146" fill="#d7f4d7"/></g></g><g fill="#d7d7f4" stroke="#000" stroke-width=".26458"><rect x="70.115" y="7.2813" width="22.49" height="6.6146"/><rect x="92.604" y="7.2813" width="22.49" height="6.6146"/><rect x="160.07" y="7.2813" width="22.49" height="6.6146"/></g><text x="18.280743" y="35.463409" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="18.280743" y="35.463409" stroke-width=".26458">clone</tspan></text><rect x="47.625" y="58.875" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="59.058754" y="63.505211" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="59.058754" y="63.505211" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="70.115" y="58.875" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="81.54834" y="63.505211" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="81.54834" y="63.505211" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="92.604" y="58.875" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="104.03793" y="63.505211" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="104.03793" y="63.505211" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="115.09" y="58.875" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="126.52751" y="63.505211" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="126.52751" y="63.505211" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="137.58" y="58.875" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="149.01709" y="63.505211" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="149.01709" y="63.505211" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="160.07" y="58.875" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="171.50665" y="63.505211" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="171.50665" y="63.505211" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><text x="18.309223" y="63.987183" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="18.309223" y="63.987183" stroke-width=".26458">zeroes_dev</tspan></text><g fill="none"><rect x="47.625" y="44.323" width="22.49" height="6.6146" stroke="#000" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"/><rect x="70.115" y="44.323" width="22.49" height="6.6146" stroke="#000" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"/><rect x="39.688" y="27.125" width="145.52" height="11.906" ry="1.3229" stroke="#808080" stroke-width=".265"/><rect x="39.688" y="41.677" width="145.52" height="11.906" ry="1.3229" stroke="#808080" stroke-width=".265"/></g><g fill="#00f" stroke="#00f" stroke-width=".265"><path d="m80.698 13.896v15.875" marker-end="url(#c)"/><path d="m58.208 13.896v44.979" marker-end="url(#b)"/><path d="m103.19 13.896v15.875" marker-end="url(#d)"/><path d="m171.98 13.896v15.875" marker-end="url(#h)"/><path d="m127 13.896v30.427" marker-end="url(#f)"/><path d="m149.49 13.896v30.427" marker-end="url(#g)"/><path d="m124.35 74.75 10.583 3e-6" marker-end="url(#e)"/></g><g stroke-width=".26458"><text x="137.58331" y="76.072906" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="137.58331" y="76.072906" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">read cluster</tspan></text><rect x="124.35" y="78.719" width="10.583" height="2.6458" fill="none" stroke="#000" stroke-dasharray="0.52916663, 0.52916663"/><text x="137.58331" y="81.364571" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="137.58331" y="81.364571" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">unallocated cluster</tspan></text></g><rect x="124.35" y="84.01" width="10.583" height="2.6458" fill="none" stroke="#000" stroke-width=".265"/><text x="137.58331" y="86.656242" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="137.58331" y="86.656242" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">allocated cluster</tspan></text></g></svg> diff --git a/src/spdk/doc/img/lvol_clone_snapshot_write.svg b/src/spdk/doc/img/lvol_clone_snapshot_write.svg new file mode 100644 index 000000000..d5da58132 --- /dev/null +++ b/src/spdk/doc/img/lvol_clone_snapshot_write.svg @@ -0,0 +1,3 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg width="173.3mm" height="91.281mm" version="1.1" viewBox="0 0 173.3 91.281" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><defs><marker id="h" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#f00" fill-rule="evenodd" stroke="#ff2a2a" stroke-width="1pt"/></marker><marker id="e" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="d" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="g" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#f00" fill-rule="evenodd" stroke="#ff2a2a" stroke-width="1pt"/></marker><marker id="b" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="c" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="f" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#f00" fill-rule="evenodd" stroke="#ff2a2a" stroke-width="1pt"/></marker><marker id="a" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="Arrow1Mend" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker></defs><metadata><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title/></cc:Work></rdf:RDF></metadata><g transform="translate(3.9688 -99.351)"><rect x="-3.9688" y="99.351" width="173.3" height="91.281" ry="0" fill="#fff"/></g><g transform="translate(1.3229 -101.86)"><g stroke="#000" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"><rect x="31.75" y="127" width="22.49" height="6.6146" fill="none"/><rect x="99.219" y="127" width="22.49" height="6.6146" fill="#d7f4d7"/><rect x="121.71" y="127" width="22.49" height="6.6146" fill="none"/></g><g fill="#d7d7f4" stroke="#000" stroke-width=".26458"><rect x="76.729" y="141.55" width="22.49" height="6.6146"/><rect x="99.219" y="141.55" width="22.49" height="6.6146"/><rect x="121.71" y="141.55" width="22.49" height="6.6146"/><rect x="144.2" y="141.55" width="22.49" height="6.6146"/></g><text x="2.4342299" y="146.66425" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="2.4342299" y="146.66425" stroke-width=".26458">snapshot</tspan></text><text x="2.4876127" y="108.4595" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="2.4876127" y="108.4595" stroke-width=".26458">write</tspan></text><path d="m1.3229 119.06 168.01 1e-5" fill="none" stroke="#000" stroke-dasharray="1.59000004, 1.59000004" stroke-width=".265"/><rect x="99.219" y="104.51" width="22.49" height="6.6146" fill="#fff6d5" stroke="#000" stroke-width=".26458"/><text x="2.4057417" y="132.69257" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="2.4057417" y="132.69257" stroke-width=".26458">clone</tspan></text><rect x="31.75" y="156.1" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="43.183758" y="160.73438" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="43.183758" y="160.73438" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="54.24" y="156.1" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="65.673347" y="160.73438" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="65.673347" y="160.73438" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="76.729" y="156.1" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="88.162926" y="160.73438" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="88.162926" y="160.73438" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="99.219" y="156.1" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="110.65253" y="160.73438" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="110.65253" y="160.73438" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="121.71" y="156.1" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="133.14209" y="160.73438" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="133.14209" y="160.73438" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="144.2" y="156.1" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="155.63165" y="160.73438" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="155.63165" y="160.73438" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><text x="2.4342222" y="161.21635" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="2.4342222" y="161.21635" stroke-width=".26458">zeroes_dev</tspan></text><g fill="none"><rect x="31.75" y="141.55" width="22.49" height="6.6146" stroke="#000" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"/><rect x="54.24" y="141.55" width="22.49" height="6.6146" stroke="#000" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"/><rect x="23.813" y="124.35" width="145.52" height="11.906" ry="1.3229" stroke="#808080" stroke-width=".265"/><rect x="23.813" y="138.91" width="145.52" height="11.906" ry="1.3229" stroke="#808080" stroke-width=".265"/></g><path d="m115.09 111.12v15.875" fill="#00f" marker-end="url(#e)" stroke="#00f" stroke-width=".265"/><path d="m105.83 141.55v-7.9375" fill="#f00" marker-end="url(#h)" stroke="#ff2a2a" stroke-width=".265"/><g stroke="#000" stroke-width=".26458"><rect x="54.24" y="127" width="22.49" height="6.6146" fill="#ffe3be" stroke-dasharray="0.52916663, 0.52916663"/><rect x="144.2" y="127" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect x="54.24" y="104.51" width="22.49" height="6.6146" fill="#fff6d5"/></g><g><path d="m70.115 111.12v15.875" fill="#00f" marker-end="url(#d)" stroke="#00f" stroke-width=".265"/><path d="m60.854 156.1v-22.49" fill="#f00" marker-end="url(#g)" stroke="#ff2a2a" stroke-width=".265"/><rect x="76.729" y="104.51" width="22.49" height="6.6146" fill="#fff6d5" stroke="#000" stroke-width=".26458"/></g><g stroke-width=".265"><path d="m87.313 111.12v15.875" fill="#00f" marker-end="url(#b)" stroke="#00f"/><path d="m109.8 170.66 10.583 1e-5" fill="#00f" marker-end="url(#c)" stroke="#00f"/><path d="m109.8 175.95h10.583" fill="#f00" marker-end="url(#f)" stroke="#ff2a2a"/></g><g stroke-width=".26458"><text x="123.03123" y="171.97917" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="123.03123" y="171.97917" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">write</tspan></text><text x="123.03123" y="177.27084" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="123.03123" y="177.27084" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">allocate and copy cluster</tspan></text><rect x="109.8" y="179.92" width="10.583" height="2.6458" fill="none" stroke="#000" stroke-dasharray="0.52916663, 0.52916663"/><text x="123.03123" y="182.5625" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="123.03123" y="182.5625" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">unallocated cluster before write</tspan></text></g><rect x="109.8" y="185.21" width="10.583" height="2.6458" fill="none" stroke="#000" stroke-width=".265"/><text x="123.03123" y="187.85417" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="123.03123" y="187.85417" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">allocated cluster</tspan></text><g stroke="#000"><rect x="76.729" y="127" width="22.49" height="6.6146" fill="#fff6d5" stroke-width=".26458"/><path d="m6.6146 157.43v-9.2604" fill="none" marker-end="url(#a)" stroke-width=".26458px"/><path d="m6.6146 142.88v-9.2604" fill="none" marker-end="url(#Arrow1Mend)" stroke-width=".26458px"/></g></g></svg> diff --git a/src/spdk/doc/img/lvol_inflate_clone_snapshot.svg b/src/spdk/doc/img/lvol_inflate_clone_snapshot.svg new file mode 100644 index 000000000..85c85b4eb --- /dev/null +++ b/src/spdk/doc/img/lvol_inflate_clone_snapshot.svg @@ -0,0 +1,3 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg width="169.33mm" height="89.958mm" version="1.1" viewBox="0 0 169.33 89.958" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><defs><marker id="h" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="i" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="Arrow1Mend" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="e" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="g" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="Arrow1Mstart" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="j" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="f" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="a" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="d" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="b" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="c" overflow="visible" orient="auto"><path transform="matrix(.4 0 0 .4 4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker></defs><metadata><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title/></cc:Work></rdf:RDF></metadata><g transform="translate(3.9687 2.5133)"><rect x="-3.9687" y="-2.5133" width="169.33" height="89.958" fill="#fff"/></g><g transform="translate(-37.042 -17.865)"><g stroke="#000" stroke-width=".26458"><rect transform="scale(1,-1)" x="66.146" y="-56.229" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect transform="scale(1,-1)" x="88.635" y="-56.229" width="22.49" height="6.6146" fill="#d7d7f4"/><rect transform="scale(1,-1)" x="111.13" y="-56.229" width="22.49" height="6.6146" fill="#d7d7f4"/><rect transform="scale(1,-1)" x="133.61" y="-56.229" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect transform="scale(1,-1)" x="156.1" y="-56.229" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect transform="scale(1,-1)" x="178.59" y="-56.229" width="22.49" height="6.6146" fill="#d7d7f4"/></g><g fill="#d7f4d7" stroke="#000" stroke-width=".26458"><rect transform="scale(1,-1)" x="111.13" y="-41.677" width="22.49" height="6.6146"/><rect transform="scale(1,-1)" x="133.61" y="-41.677" width="22.49" height="6.6146"/><rect transform="scale(1,-1)" x="156.1" y="-41.677" width="22.49" height="6.6146"/><rect transform="scale(1,-1)" x="178.59" y="-41.677" width="22.49" height="6.6146"/></g><text x="39.850315" y="38.377769" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="39.850315" y="38.377769" stroke-width=".26458">snapshot</tspan></text><text x="39.348198" y="97.549011" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="39.348198" y="97.549011">Inflated</tspan><tspan x="39.348198" y="101.95873">clone</tspan></text><g stroke="#000"><path d="m39.688 64.167 161.4 9e-6" fill="none" stroke-dasharray="1.59000006, 1.59000006" stroke-width=".265"/><g stroke-width=".26458"><rect transform="scale(1,-1)" x="66.146" y="-102.53" width="22.49" height="6.6146" fill="#f4d7d7"/><rect transform="scale(1,-1)" x="133.61" y="-102.53" width="22.49" height="6.6146" fill="#d7f4d7"/><rect transform="scale(1,-1)" x="156.1" y="-102.53" width="22.49" height="6.6146" fill="#d7f4d7"/></g></g><g fill="#d7d7f4" stroke="#000" stroke-width=".26458"><rect transform="scale(1,-1)" x="88.635" y="-102.53" width="22.49" height="6.6146"/><rect transform="scale(1,-1)" x="111.13" y="-102.53" width="22.49" height="6.6146"/><rect transform="scale(1,-1)" x="178.59" y="-102.53" width="22.49" height="6.6146"/></g><text x="39.821827" y="53.033295" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="39.821827" y="53.033295" stroke-width=".26458">clone</tspan></text><rect transform="scale(1,-1)" x="66.146" y="-27.125" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="77.57962" y="25.063118" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="77.57962" y="25.063118" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect transform="scale(1,-1)" x="88.635" y="-27.125" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="100.06921" y="25.063118" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="100.06921" y="25.063118" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect transform="scale(1,-1)" x="111.13" y="-27.125" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="122.55879" y="25.063118" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="122.55879" y="25.063118" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect transform="scale(1,-1)" x="133.61" y="-27.125" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="145.04839" y="25.063118" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="145.04839" y="25.063118" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect transform="scale(1,-1)" x="156.1" y="-27.125" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="167.53796" y="25.063118" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="167.53796" y="25.063118" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect transform="scale(1,-1)" x="178.59" y="-27.125" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.53, 0.265" stroke-width=".265"/><text x="190.02751" y="25.063118" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="190.02751" y="25.063118" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><text x="39.850307" y="23.727493" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="39.850307" y="23.727493" stroke-width=".26458">zeroes_dev</tspan></text><g fill="none" stroke="#000"><rect transform="scale(1,-1)" x="66.146" y="-41.677" width="22.49" height="6.6146" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"/><rect transform="scale(1,-1)" x="88.635" y="-41.677" width="22.49" height="6.6146" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"/><rect x="66.146" y="72.104" width="134.94" height="7.9375" ry="2.6458" stroke-width=".5"/></g><text x="113.74464" y="77.262192" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="113.74464" y="77.262192" stroke-width=".26458">Allocate and copy</tspan></text><g fill="none" stroke-width=".265"><path d="m39.688 87.979 161.4 4e-6" stroke="#000" stroke-dasharray="1.59000008, 1.59000008"/><g stroke="#808080"><rect x="58.208" y="32.417" width="145.52" height="11.906" ry="1.3229"/><rect x="58.208" y="46.969" width="145.52" height="11.906" ry="1.3229"/><rect x="58.208" y="93.271" width="145.52" height="11.906" ry="1.3229"/></g></g><g fill="#00f" stroke="#00f" stroke-width=".265"><path d="m76.729 72.104v-44.979" marker-start="url(#e)"/><path d="m145.52 72.104v-30.427" marker-start="url(#g)"/><path d="m168.01 72.104v-30.427" marker-start="url(#Arrow1Mstart)"/><path d="m76.729 95.917 1.4e-5 -15.875" marker-start="url(#a)"/><path d="m145.52 95.917v-15.875" marker-start="url(#f)"/><path d="m168.01 95.917 1e-5 -15.875" marker-start="url(#j)"/></g><path d="m58.208 53.583c-7.4527 13.662-6.6489 28.799 0 44.979" fill="none" marker-end="url(#Arrow1Mend)" stroke="#000" stroke-width=".5"/><g fill="#00f" stroke="#00f" stroke-width=".265"><path d="m100.54 95.917v-39.688" marker-start="url(#d)" stroke-dasharray="1.58999992, 1.58999992"/><path d="m123.03 95.917v-39.688" marker-start="url(#b)" stroke-dasharray="1.58999994, 1.58999994"/><path d="m190.5 95.917v-39.688" marker-start="url(#c)" stroke-dasharray="1.58999994, 1.58999994"/></g><text x="39.498512" y="77.301338" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="39.498512" y="77.301338" stroke-width=".26458">Inflate</tspan></text><path d="m44.979 24.479v10.583" fill="none" marker-end="url(#h)" stroke="#000" stroke-width=".26458px"/><path d="m44.979 40.354v7.9375" fill="none" marker-end="url(#i)" stroke="#000" stroke-width=".26458px"/></g></svg> diff --git a/src/spdk/doc/img/lvol_thin_provisioning.svg b/src/spdk/doc/img/lvol_thin_provisioning.svg new file mode 100644 index 000000000..1d95d1b08 --- /dev/null +++ b/src/spdk/doc/img/lvol_thin_provisioning.svg @@ -0,0 +1,3 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg width="181.24mm" height="79.375mm" version="1.1" viewBox="0 0 181.24 79.375" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><title>Thin Provisioning</title><defs><marker id="marker2036" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="marker1960" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="marker1890" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="marker1826" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="marker1816" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="Arrow1Mend" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="marker11771-4-9" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#f00" fill-rule="evenodd" stroke="#ff2a2a" stroke-width="1pt"/></marker><marker id="marker1826-2-4-7-1-7" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker></defs><metadata><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title>Thin Provisioning</dc:title></cc:Work></rdf:RDF></metadata><g transform="translate(2.6458 2.3956)"><rect x="-2.6458" y="-2.3956" width="181.24" height="79.375" fill="#fffffe" stroke-width=".26458"/></g><g transform="translate(-3.9688 -4.6356)"><g stroke="#000"><g stroke-width=".26458"><rect x="44.979" y="32.417" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect x="67.469" y="32.417" width="22.49" height="6.6146" fill="#d7d7f4"/><rect x="89.958" y="32.417" width="22.49" height="6.6146" fill="#d7d7f4"/><rect x="112.45" y="32.417" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect x="134.94" y="32.417" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect x="157.43" y="32.417" width="22.49" height="6.6146" fill="#d7d7f4"/></g><rect x="44.979" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/></g><text x="56.412949" y="51.598957" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="56.412949" y="51.598957" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="67.469" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="78.902527" y="51.598961" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="78.902527" y="51.598961" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="89.958" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="101.39211" y="51.598961" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="101.39211" y="51.598961" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="112.45" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="123.88169" y="51.598961" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="123.88169" y="51.598961" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="134.94" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="146.37128" y="51.598957" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="146.37128" y="51.598957" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="157.43" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><g font-family="sans-serif" letter-spacing="0px" stroke-width=".26458" word-spacing="0px"><text x="168.86086" y="51.598961" font-size="10.583px" style="line-height:1.25" xml:space="preserve"><tspan x="168.86086" y="51.598961" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><text x="6.6430736" y="51.680019" font-size="3.5278px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="6.6430736" y="51.680019" stroke-width=".26458">zeroes_dev</tspan></text><text x="6.6296382" y="12.539818" font-size="3.5278px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="6.6296382" y="12.539818">Thin-provisioned</tspan><tspan x="6.6296382" y="16.949539">volume</tspan></text></g><g stroke="#000"><path d="m6.6146 24.479 173.3 1e-6" fill="none" stroke-dasharray="1.59, 1.59" stroke-width=".265"/><g fill="#f4d7d7" stroke-dasharray="0.52916663, 0.26458332" stroke-width=".26458"><rect x="44.979" y="9.9271" width="22.49" height="6.6146"/><rect x="112.45" y="9.9271" width="22.49" height="6.6146"/><rect x="134.94" y="9.9271" width="22.49" height="6.6146"/></g><g fill="#d7d7f4" stroke-width=".26458"><rect x="67.469" y="9.9271" width="22.49" height="6.6146"/><rect x="89.958" y="9.9271" width="22.49" height="6.6146"/><rect x="157.43" y="9.9271" width="22.49" height="6.6146"/></g></g><text x="6.614583" y="37.708332" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="6.614583" y="37.708332" stroke-width=".26458">active clusters</tspan></text><rect x="37.042" y="7.2812" width="145.52" height="11.906" ry="1.3229" fill="none" stroke="#999" stroke-width=".5"/><rect x="37.042" y="29.771" width="145.52" height="26.458" ry="1.3229" fill="none" stroke="#999" stroke-width=".5"/><g fill="#00f" stroke="#00f"><g stroke-width=".26458"><path d="m78.052 16.542v15.875" marker-end="url(#marker1960)"/><path d="m55.562 16.542v30.427" marker-end="url(#marker2036)"/><path d="m100.54 16.542v15.875" marker-end="url(#marker1890)"/><path d="m169.33 16.542v15.875" marker-end="url(#Arrow1Mend)"/><path d="m124.35 16.542v30.427" marker-end="url(#marker1826)"/><path d="m146.84 16.542v30.427" marker-end="url(#marker1816)"/></g><path d="m132.29 61.521 10.583 1e-5" marker-end="url(#marker1826-2-4-7-1-7)" stroke-width=".265"/></g><path d="m132.29 66.813h10.583" fill="#f00" marker-end="url(#marker11771-4-9)" stroke="#ff2a2a" stroke-width=".265"/><g stroke-width=".26458"><text x="145.52083" y="62.843975" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="145.52083" y="62.843975" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">read</tspan></text><text x="145.52083" y="68.135651" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="145.52083" y="68.135651" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">allocate and copy cluster</tspan></text><rect x="132.29" y="70.781" width="10.583" height="2.6458" fill="none" stroke="#000" stroke-dasharray="0.52916664, 0.52916664"/><text x="145.52083" y="73.427307" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="145.52083" y="73.427307" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">unallocated cluster</tspan></text></g><rect x="132.29" y="76.073" width="10.583" height="2.6458" fill="none" stroke="#000" stroke-width=".265"/><text x="145.52083" y="78.718971" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="145.52083" y="78.718971" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">allocated cluster</tspan></text></g></svg> diff --git a/src/spdk/doc/img/lvol_thin_provisioning_write.svg b/src/spdk/doc/img/lvol_thin_provisioning_write.svg new file mode 100644 index 000000000..37cf6af93 --- /dev/null +++ b/src/spdk/doc/img/lvol_thin_provisioning_write.svg @@ -0,0 +1,3 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg width="181.24mm" height="79.375mm" version="1.1" viewBox="0 0 181.24 79.375" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><title>Thin Provisioning Write</title><defs><marker id="marker11771-4-9" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#f00" fill-rule="evenodd" stroke="#ff2a2a" stroke-width="1pt"/></marker><marker id="marker1826-2-4-7-1-7" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="marker11771-4-9-6" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#f00" fill-rule="evenodd" stroke="#ff2a2a" stroke-width="1pt"/></marker><marker id="marker1826-2-4-7-1-7-4" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker><marker id="marker1826-2-4-7-1-7-4-1" overflow="visible" orient="auto"><path transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill="#00f" fill-rule="evenodd" stroke="#00f" stroke-width="1pt"/></marker></defs><metadata><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title>Thin Provisioning Write</dc:title></cc:Work></rdf:RDF></metadata><g transform="translate(2.6458 2.3956)"><rect x="-2.6458" y="-2.3956" width="181.24" height="79.375" fill="#fffffe" stroke-width=".26458"/></g><g transform="translate(-3.9688 -4.6356)"><g stroke="#000"><rect x="44.979" y="32.417" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"/><rect x="67.469" y="32.417" width="22.49" height="6.6146" fill="#ffe3be" stroke-dasharray="0.52916663, 0.52916663" stroke-width=".26458"/><g stroke-width=".26458"><rect x="112.45" y="32.417" width="22.49" height="6.6146" fill="#fff6d5"/><rect x="89.959" y="32.417" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/><rect x="134.94" y="32.417" width="22.49" height="6.6146" fill="none" stroke-dasharray="0.52916663, 0.52916663"/></g><rect x="44.979" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/></g><text x="56.412949" y="51.598957" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="56.412949" y="51.598957" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="67.469" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="78.902527" y="51.598961" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="78.902527" y="51.598961" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="89.958" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="101.39211" y="51.598961" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="101.39211" y="51.598961" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="112.45" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="123.88169" y="51.598961" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="123.88169" y="51.598961" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="134.94" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><text x="146.37128" y="51.598957" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="146.37128" y="51.598957" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><rect x="157.43" y="46.969" width="22.49" height="6.6146" fill="#f4d7d7" stroke="#000" stroke-dasharray="0.52999997, 0.26499999" stroke-width=".265"/><g font-family="sans-serif" letter-spacing="0px" stroke-width=".26458" word-spacing="0px"><text x="168.86086" y="51.598961" font-size="10.583px" style="line-height:1.25" xml:space="preserve"><tspan x="168.86086" y="51.598961" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" text-align="center" text-anchor="middle" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">00000...</tspan></text><text x="6.6430736" y="51.680019" font-size="3.5278px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="6.6430736" y="51.680019" stroke-width=".26458">zeroes_dev</tspan></text><text x="6.6296382" y="12.539818" font-size="3.5278px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve">write</text></g><g stroke="#000"><path d="m6.6146 24.479 173.3 1e-6" fill="none" stroke-dasharray="1.59, 1.59" stroke-width=".265"/><rect x="67.469" y="9.9271" width="22.49" height="6.6146" fill="#fff6d5" stroke-width=".26458"/><rect x="112.45" y="9.927" width="22.49" height="6.6146" fill="#fff6d5" stroke-width=".26458"/></g><text x="6.614583" y="37.708332" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="6.614583" y="37.708332" stroke-width=".26458">active clusters</tspan></text><rect x="37.042" y="29.771" width="145.52" height="26.458" ry="1.3229" fill="none" stroke="#999" stroke-width=".5"/><path d="m121.71 61.521 10.583 1e-5" fill="#00f" marker-end="url(#marker1826-2-4-7-1-7)" stroke="#00f" stroke-width=".265"/><path d="m121.71 66.813h10.583" fill="#f00" marker-end="url(#marker11771-4-9)" stroke="#ff2a2a" stroke-width=".265"/><g stroke-width=".26458"><text x="134.93752" y="62.843967" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="134.93752" y="62.843967" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">write</tspan></text><text x="134.93752" y="68.135635" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="134.93752" y="68.135635" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">allocate and copy cluster</tspan></text><rect x="121.71" y="70.781" width="10.583" height="2.6458" fill="none" stroke="#000" stroke-dasharray="0.52916663, 0.52916663"/><text x="134.93752" y="73.427292" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="134.93752" y="73.427292" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">unallocated cluster before write</tspan></text></g><rect x="121.71" y="76.073" width="10.583" height="2.6458" fill="none" stroke="#000" stroke-width=".265"/><text x="134.93752" y="78.718956" fill="#000000" font-family="sans-serif" font-size="3.5278px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="134.93752" y="78.718956" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">allocated cluster</tspan></text><rect x="157.43" y="32.417" width="22.49" height="6.6146" fill="none" stroke="#000" stroke-dasharray="0.52916664, 0.52916664" stroke-width=".26458"/><path d="m72.76 46.969 7e-6 -7.9375" fill="#f00" marker-end="url(#marker11771-4-9-6)" stroke="#ff2a2a" stroke-width=".265"/></g><path d="m79.375 11.906v15.875" fill="#00f" marker-end="url(#marker1826-2-4-7-1-7-4)" stroke="#00f" stroke-width=".265"/><path d="m119.06 11.906v15.875" fill="#00f" marker-end="url(#marker1826-2-4-7-1-7-4-1)" stroke="#00f" stroke-width=".265"/></svg> diff --git a/src/spdk/doc/img/nvme_cuse.svg b/src/spdk/doc/img/nvme_cuse.svg new file mode 100644 index 000000000..bed843402 --- /dev/null +++ b/src/spdk/doc/img/nvme_cuse.svg @@ -0,0 +1,124 @@ +<?xml version="1.0" encoding="UTF-8"?> +<svg width="193.94mm" height="139.71mm" version="1.1" viewBox="0 0 193.94 139.71" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <title>NVMe CUSE</title> + <defs> + <marker id="marker9353" overflow="visible" orient="auto"> + <path transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker7156" overflow="visible" orient="auto"> + <path transform="matrix(.8 0 0 .8 10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker4572" overflow="visible" orient="auto"> + <path transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker4436" overflow="visible" orient="auto"> + <path transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker4324" overflow="visible" orient="auto"> + <path transform="matrix(.8 0 0 .8 10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker2300" overflow="visible" orient="auto"> + <path transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker2110" overflow="visible" orient="auto"> + <path transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker2028" overflow="visible" orient="auto"> + <path transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker1219" overflow="visible" orient="auto"> + <path transform="matrix(.8 0 0 .8 10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="Arrow1Lstart" overflow="visible" orient="auto"> + <path transform="matrix(.8 0 0 .8 10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="marker1127" overflow="visible" orient="auto"> + <path transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + <marker id="Arrow1Lend" overflow="visible" orient="auto"> + <path transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/> + </marker> + </defs> + <metadata> + <rdf:RDF> + <cc:Work rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/> + <dc:title>NVMe CUSE</dc:title> + </cc:Work> + </rdf:RDF> + </metadata> + <g transform="translate(-2.1066 -22.189)"> + <rect x="11.906" y="134.85" width="72.004" height="20.6" ry="3.7798" fill="none" stroke="#000" stroke-width=".5"/> + <text x="14.363094" y="149.02231" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="14.363094" y="149.02231" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">/dev/spdk/nvme0</tspan></text> + <text x="47.625" y="149.02231" fill="#000000" font-family="sans-serif" font-size="10.583px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px" style="line-height:1.25" xml:space="preserve"><tspan x="47.625" y="149.02231" font-family="sans-serif" font-size="3.5278px" stroke-width=".26458" writing-mode="lr" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">/dev/spdk/nvme0n1</tspan></text> + <g stroke="#000"> + <rect x="12.095" y="35.818" width="71.249" height="88.446" ry="4.3467" fill="none" stroke-width=".5"/> + <rect x="133.43" y="33.929" width="62.366" height="76.351" ry="4.7247" fill="none" stroke-width=".5"/> + <g fill="#fff" stroke-width=".26458"> + <rect x="14.174" y="91.57" width="64.256" height="24.568"/> + <g fill-opacity=".9798"> + <rect x="46.302" y="100.64" width="26.62" height="11.061"/> + </g> + </g> + <g transform="translate(-.53932 -.16291)"> + <path d="m63.878 111.98v32.884" fill="none" marker-end="url(#marker1127)" marker-start="url(#Arrow1Lstart)" stroke-width=".26458px"/> + <g stroke-width=".265"> + <path d="m34.585 115.57v28.726" fill="none" marker-end="url(#Arrow1Lend)" marker-start="url(#marker1219)"/> + <rect x="136.26" y="39.031" width="54.996" height="58.586" fill="#fff"/> + <rect x="153.84" y="52.26" width="34.018" height="11.906" ry="5.8544" fill="none"/> + </g> + <path d="m112.45 24.479v137.58" fill="none" stroke-dasharray="1.5874999, 1.5874999" stroke-width=".26458"/> + </g> + <g fill="#fff" stroke-width=".265"> + <rect x="89.58" y="54.339" width="38.365" height="8.8824"/> + </g> + </g> + <g font-family="sans-serif" font-size="4.2333px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px"> + <text x="93.54911" y="59.800339" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="93.54911" y="59.800339" stroke-width=".26458">io_msg queue</tspan></text> + <text x="11.906249" y="27.31399" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="11.906249" y="27.31399" stroke-width=".26458">CUSE threads</tspan></text> + <text x="165.36458" y="27.502975" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="165.36458" y="27.502975" stroke-width=".26458">SPDK threads</tspan></text> + </g> + <g stroke="#000"> + <rect x="17.009" y="47.914" width="29.482" height="13.04" ry="6.5201" fill="#fff" stroke-width=".265"/> + <rect x="49.921" y="68.161" width="28.915" height="13.04" ry="6.5201" fill="#fff" stroke-width=".265"/> + <g fill="none"> + <path d="m32.506 61.143v30.427" marker-start="url(#marker7156)" stroke-width=".26458px"/> + <path d="m63.689 81.176 0.18899 19.277" marker-start="url(#marker4324)" stroke-width=".265"/> + <g stroke-width=".26458px"> + <path d="m46.113 54.339h43.467" marker-end="url(#marker2028)"/> + <path d="m64.284 67.972c0.02768-6.3997-1.3229-5.2917 25.135-5.2917" marker-end="url(#marker2110)"/> + <path d="m127.78 56.066h25.135" marker-end="url(#marker2300)"/> + </g> + </g> + </g> + <g stroke-width=".26458"> + <g transform="translate(-.25341)" font-family="sans-serif" font-size="4.2333px" letter-spacing="0px" word-spacing="0px"> + <text x="138.90625" y="44.889877" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="138.90625" y="44.889877" stroke-width=".26458">NVMe</tspan></text> + <text x="16.063986" y="97.050598" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="16.063986" y="97.050598" stroke-width=".26458">CUSE ctrlr</tspan></text> + <text x="48.380947" y="106.12202" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="48.380947" y="106.12202" stroke-width=".26458">CUSE ns</tspan></text> + <text x="51.420551" y="75.799461" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="51.420551" y="75.799461" stroke-width=".26458">ioctl pthread</tspan></text> + <text x="18.906757" y="55.833015" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="18.906757" y="55.833015" stroke-width=".26458">ioctl pthread</tspan></text> + </g> + <path d="m160.86 85.17c0.38097 13.154-7.1538 11.542-82.052 10.936" fill="none" marker-end="url(#marker4572)" stroke="#000" stroke-dasharray="0.79374995, 0.79374995"/> + <path d="m179.38 85.17c0.37797 22.25-6.5765 20.83-106.08 20.641" fill="none" marker-end="url(#marker4436)" stroke="#000" stroke-dasharray="0.79374995, 0.79374995"/> + </g> + <g font-family="sans-serif" font-size="4.2333px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px"> + <text x="13.229166" y="139.7619" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="13.229166" y="139.7619" stroke-width=".26458">Kernel</tspan></text> + <text x="14.552083" y="41.488094" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="14.552083" y="41.488094" stroke-width=".26458">CUSE</tspan></text> + <text x="161.73709" y="59.415913" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="161.73709" y="59.415913" stroke-width=".26458">io poller</tspan></text> + </g> + <g fill="none" stroke="#000"> + <path d="m111.91 127.5h-109.8" stroke-dasharray="1.58749992, 1.58749992" stroke-width=".26458"/> + <rect x="153.3" y="71.941" width="34.018" height="13.229" ry="6.6146" stroke-width=".265"/> + <path d="m170.12 64.003v7.9375" marker-end="url(#marker9353)" stroke-width=".265"/> + </g> + <g font-family="sans-serif" font-size="4.2333px" letter-spacing="0px" stroke-width=".26458" word-spacing="0px"> + <text x="159.72221" y="79.76664" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="159.72221" y="79.76664" stroke-width=".26458">io execute</tspan></text> + <text x="172.34003" y="68.59539" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="172.34003" y="68.59539" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">fn(arg)</tspan></text> + <text x="53.046707" y="52.192699" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="53.046707" y="52.192699" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">nvme_io_msg send()</tspan></text> + <text x="53.102341" y="60.250244" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="53.102341" y="60.250244" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">nvme_io_msg send()</tspan></text> + <text x="120.79763" y="50.70586" font-size="12px" stroke-width="1" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;line-height:1.25" xml:space="preserve"><tspan x="120.79763" y="50.70586" font-family="sans-serif" font-size="2.8222px" stroke-width=".26458" style="font-feature-settings:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">spdk_nvme_io_msg process()</tspan></text> + </g> + </g> +</svg> diff --git a/src/spdk/doc/img/qemu_vhost_data_flow.svg b/src/spdk/doc/img/qemu_vhost_data_flow.svg new file mode 100644 index 000000000..96b4673e1 --- /dev/null +++ b/src/spdk/doc/img/qemu_vhost_data_flow.svg @@ -0,0 +1,2 @@ +<?xml version="1.0" encoding="UTF-8"?> +<svg version="1.1" viewBox="0 0 187.85 104.34" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><defs><marker id="a" style="overflow:visible" orient="auto"><path transform="matrix(1.1,0,0,1.1,1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="b" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="j" style="overflow:visible" orient="auto"><path transform="matrix(1.1,0,0,1.1,1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="l" style="overflow:visible" orient="auto"><path transform="matrix(1.1,0,0,1.1,1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="c" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="d" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="e" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="f" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="g" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="h" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="i" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker><marker id="k" style="overflow:visible" orient="auto"><path transform="matrix(-1.1,0,0,-1.1,-1.1,0)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" style="fill-rule:evenodd;fill:#000000;stroke-linejoin:round;stroke-width:.625;stroke:#000000"/></marker></defs><metadata><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title/><dc:creator><cc:Agent><dc:title>Tomasz Kulasek</dc:title></cc:Agent></dc:creator></cc:Work></rdf:RDF></metadata><g transform="translate(2.5133 -10.794)"><rect x="-2.5133" y="10.794" width="187.85" height="104.34" style="fill:#fffffe"/></g><g transform="translate(-2.6458 -133.04)"><rect x="10.583" y="135.6" width="79.375" height="74.083" ry="2.6458" style="fill:#d5e5ff;stroke-width:.5;stroke:#000000"/><rect x="63.5" y="199.1" width="21.167" height="5.2917" style="fill:none;stroke-dasharray:1.05833327, 1.05833327;stroke-width:.26458;stroke:#000000"/><rect x="21.167" y="152.8" width="58.208" height="30.427" ry="2.6458" style="fill:none;stroke-width:.26458;stroke:#000000"/><path d="m26.458 191.17v-7.9375" style="fill:none;marker-end:url(#h);stroke-dasharray:1.05999995, 1.05999995;stroke-width:.265;stroke:#000000"/><path d="m34.396 183.23v7.9375" style="fill:none;marker-end:url(#i);stroke-dasharray:1.05999995, 1.05999995;stroke-width:.265;stroke:#000000"/><rect x="63.5" y="218.95" width="15.875" height="15.875" ry="2.6458" style="fill:#ffd5e5;stroke-width:.5;stroke:#000000"/><path d="m71.438 204.4v14.552" style="fill:none;marker-end:url(#g);stroke-dasharray:1.5874999, 1.5874999;stroke-width:.26458;stroke:#000000"/><rect x="15.875" y="143.54" width="68.792" height="39.687" ry="2.6458" style="fill:none;stroke-width:.26458;stroke:#000000"/><path d="m63.5 226.89h-58.208l-1e-7-62.177 10.583-1e-5" style="fill:none;marker-end:url(#f);stroke-dasharray:1.58999992, 1.58999992;stroke-width:.265;stroke:#000000"/><g transform="translate(-.13251 -.1325)"><rect x="90.091" y="135.74" width="97.896" height="52.917" ry="2.6458" style="fill:#fff6d5;stroke-width:.5;stroke:#000000"/><g><g transform="translate(-10.451 .1325)"><rect x="132.29" y="152.8" width="60.854" height="31.75" ry="2.6458" style="fill:none;stroke-width:.265;stroke:#000000"/><text x="150.91241" y="158.04379" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;text-align:center;text-anchor:middle;word-spacing:0px" xml:space="preserve"><tspan x="150.91241" y="158.04379">Vhost-SCSI device</tspan></text></g><g transform="translate(-10.451 -2.5133)"><rect x="137.58" y="164.71" width="50.271" height="15.875" ry="2.6458" style="fill:none;stroke-opacity:.99034;stroke-width:.265;stroke:#000000"/><text x="139.99663" y="171.41422" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="139.99663" y="171.41422" style="font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">SPDK bdev(s)</tspan><tspan x="139.99663" y="175.82394" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal">(NVMe, NVMf, Ceph RBD, PMEM)</tspan></text></g></g></g><text x="92.37162" y="140.93028" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="92.37162" y="140.93028" style="font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">SPDK vhost</tspan></text><text x="13.482382" y="140.44109" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="13.482382" y="140.44109" style="font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">QEMU</tspan></text><g transform="translate(2.5609e-6 -5.2917)"><rect x="15.875" y="196.46" width="29.104" height="13.229" style="fill:none;stroke-width:.26458;stroke:#000000"/><text x="16.636366" y="201.47922" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="16.636366" y="201.47922" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">Virtio-SCSI device</tspan></text></g><g transform="translate(-5.0685e-6 -5.2917)"><rect x="44.979" y="196.46" width="39.688" height="13.229" ry="0" style="fill:none;stroke-width:.26458;stroke:#000000"/><text x="46.195286" y="201.31316" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="46.195286" y="201.31316" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">Vhost-user driver (master)</tspan></text></g><g transform="translate(0 6.6146)"><g><rect x="54.24" y="154.12" width="55.562" height="17.198" ry="2.6458" style="fill:#e3f4d7;stroke-dasharray:0.52999997, 0.52999997;stroke-width:.265;stroke:#000000"/><text x="64.25103" y="157.5067" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="64.25103" y="157.5067" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">Shared hugepage memory</tspan></text></g><g><g><rect x="58.208" y="159.42" width="47.625" height="5.2917" ry="5.0849e-6" style="fill:#c3e8aa;stroke-width:.265;stroke:#000000"/><text x="76.179329" y="162.79837" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="76.179329" y="162.79837" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">Virtqueues</tspan></text></g><g><rect x="58.208" y="164.71" width="47.625" height="5.2917" ry="1.5259e-5" style="fill:#c3e8aa;stroke-width:.265;stroke:#000000"/><text x="76.452179" y="168.17548" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="76.452179" y="168.17548" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">I/O buffers</tspan></text></g></g></g><path d="m127 173.97-21.167 1e-5" style="fill:none;marker-end:url(#c);marker-start:url(#l);stroke-dasharray:1.05833327, 1.05833327;stroke-width:.26458;stroke:#000000"/><path d="m106.28 168.61 15.433.0673" style="fill:none;marker-end:url(#d);stroke-dasharray:1.05833327, 1.05833327;stroke-width:.26458;stroke:#000000"/><text x="110.86868" y="166.85257" style="fill:#000000;font-family:sans-serif;font-size:10.583px;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="110.86868" y="166.85257" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">polling</tspan></text><text x="112.17093" y="173.11574" style="fill:#000000;font-family:sans-serif;font-size:10.583px;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="112.17093" y="173.11574" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">DMA</tspan></text><text x="67.980347" y="228.13426" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="67.980347" y="228.13426" style="stroke-width:.26458">KVM</tspan></text><text x="69.351151" y="202.76837" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="69.351151" y="202.76837" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">callfd</tspan></text><text x="106.9275" y="200.63379" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="106.9275" y="200.63379" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">eventfd interrupt</tspan></text><path d="m166.69 184.55v17.198c-21.167 0-60.854 1e-5-82.021 1e-5" style="fill:none;marker-end:url(#e);stroke-dasharray:1.58999992, 1.58999992;stroke-width:.265;stroke:#000000"/><text x="72.494453" y="214.47757" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="72.494453" y="214.47757" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">irqfd</tspan></text><text x="23.784939" y="158.1282" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="23.784939" y="158.1282">Virtio-SCSI PCI</tspan><tspan x="23.784939" y="162.53793">driver</tspan></text><path d="m54.24 171.32h-10.583" style="fill:none;marker-end:url(#k);marker-start:url(#j);stroke-dasharray:0.79499996, 0.79499996;stroke-width:.265;stroke:#000000"/><text x="18.493273" y="148.83333" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="18.493273" y="148.83333" style="font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">VM</tspan></text><rect x="127" y="179.26" width="30.427" height="5.2917" style="fill:none;stroke-dasharray:1.05833328, 1.05833328;stroke-width:.26458;stroke:#000000"/><text x="128.16721" y="182.72755" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="128.16721" y="182.72755" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">Unix domain socket</tspan></text><text x="104.54487" y="194.10464" style="fill:#000000;font-family:sans-serif;font-feature-settings:normal;font-size:3.5278px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;letter-spacing:0px;line-height:1.25;stroke-width:.26458;word-spacing:0px" xml:space="preserve"><tspan x="104.54487" y="194.10464" style="font-family:sans-serif;font-feature-settings:normal;font-size:2.8222px;font-variant-caps:normal;font-variant-ligatures:normal;font-variant-numeric:normal;stroke-width:.26458">Vhost-user messages</tspan></text><path d="m142.88 184.55v10.583h-58.208" style="fill:none;marker-end:url(#b);marker-start:url(#a);stroke-dasharray:1.58999992, 1.58999992;stroke-width:.265;stroke:#000000"/></g></svg> diff --git a/src/spdk/doc/index.md b/src/spdk/doc/index.md new file mode 100644 index 000000000..23ab87870 --- /dev/null +++ b/src/spdk/doc/index.md @@ -0,0 +1,37 @@ +# Storage Performance Development Kit {#index} + +# Introduction + +@copydoc intro + +# Concepts + +@copydoc concepts + +# User Guides + +@copydoc user_guides + +# Programmer Guides + +@copydoc prog_guides + +# General Information + +@copydoc general + +# Miscellaneous + +@copydoc misc + +# Driver Modules + +@copydoc driver_modules + +# Tools + +@copydoc tools + +# Performance Reports + +@copydoc performance_reports diff --git a/src/spdk/doc/intro.md b/src/spdk/doc/intro.md new file mode 100644 index 000000000..ebae8d454 --- /dev/null +++ b/src/spdk/doc/intro.md @@ -0,0 +1,7 @@ +# Introduction {#intro} + +- @subpage about +- @subpage getting_started +- @subpage vagrant +- @subpage changelog +- [Source Code (GitHub)](https://github.com/spdk/spdk) diff --git a/src/spdk/doc/ioat.md b/src/spdk/doc/ioat.md new file mode 100644 index 000000000..1eb4a27f7 --- /dev/null +++ b/src/spdk/doc/ioat.md @@ -0,0 +1,14 @@ +# I/OAT Driver {#ioat} + +# Public Interface {#ioat_interface} + +- spdk/ioat.h + +# Key Functions {#ioat_key_functions} + +Function | Description +--------------------------------------- | ----------- +spdk_ioat_probe() | @copybrief spdk_ioat_probe() +spdk_ioat_get_dma_capabilities() | @copybrief spdk_ioat_get_dma_capabilities() +spdk_ioat_submit_copy() | @copybrief spdk_ioat_submit_copy() +spdk_ioat_submit_fill() | @copybrief spdk_ioat_submit_fill() diff --git a/src/spdk/doc/iscsi.md b/src/spdk/doc/iscsi.md new file mode 100644 index 000000000..7d81623d4 --- /dev/null +++ b/src/spdk/doc/iscsi.md @@ -0,0 +1,334 @@ +# iSCSI Target {#iscsi} + +# iSCSI Target Getting Started Guide {#iscsi_getting_started} + +The Storage Performance Development Kit iSCSI target application is named `iscsi_tgt`. +This following section describes how to run iscsi from your cloned package. + +## Prerequisites {#iscsi_prereqs} + +This guide starts by assuming that you can already build the standard SPDK distribution on your +platform. + +Once built, the binary will be in `build/bin`. + +If you want to kill the application by using signal, make sure use the SIGTERM, then the application +will release all the shared memory resource before exit, the SIGKILL will make the shared memory +resource have no chance to be released by applications, you may need to release the resource manually. + +## Introduction + +The following diagram shows relations between different parts of iSCSI structure described in this +document. + + + +## Configuring iSCSI Target via config file {#iscsi_config} + +A `iscsi_tgt` specific configuration file is used to configure the iSCSI target. A fully documented +example configuration file is located at `etc/spdk/iscsi.conf.in`. + +The configuration file is used to configure the SPDK iSCSI target. This file defines the following: +TCP ports to use as iSCSI portals; general iSCSI parameters; initiator names and addresses to allow +access to iSCSI target nodes; number and types of storage backends to export over iSCSI LUNs; iSCSI +target node mappings between portal groups, initiator groups, and LUNs. + +You should make a copy of the example configuration file, modify it to suit your environment, and +then run the iscsi_tgt application and pass it the configuration file using the -c option. Right now, +the target requires elevated privileges (root) to run. + +~~~ +build/bin/iscsi_tgt -c /path/to/iscsi.conf +~~~ + +### Assigning CPU Cores to the iSCSI Target {#iscsi_config_lcore} + +SPDK uses the [DPDK Environment Abstraction Layer](http://dpdk.org/doc/guides/prog_guide/env_abstraction_layer.html) +to gain access to hardware resources such as huge memory pages and CPU core(s). DPDK EAL provides +functions to assign threads to specific cores. +To ensure the SPDK iSCSI target has the best performance, place the NICs and the NVMe devices on the +same NUMA node and configure the target to run on CPU cores associated with that node. The following +command line option is used to configure the SPDK iSCSI target: + +~~~ +-m 0xF000000 +~~~ + +This is a hexadecimal bit mask of the CPU cores where the iSCSI target will start polling threads. +In this example, CPU cores 24, 25, 26 and 27 would be used. + +### Configuring a LUN in the iSCSI Target {#iscsi_lun} + +Each LUN in an iSCSI target node is associated with an SPDK block device. See @ref bdev +for details on configuring SPDK block devices. The block device to LUN mappings are specified in the +configuration file as: + +~~~~ +[TargetNodeX] + LUN0 Malloc0 + LUN1 Nvme0n1 +~~~~ + +This exports a malloc'd target. The disk is a RAM disk that is a chunk of memory allocated by iscsi in +user space. It will use offload engine to do the copy job instead of memcpy if the system has enough DMA +channels. + +## Configuring iSCSI Target via RPC method {#iscsi_rpc} + +In addition to the configuration file, the iSCSI target may also be configured via JSON-RPC calls. See +@ref jsonrpc for details. + +### Portal groups + + - iscsi_create_portal_group -- Add a portal group. + - iscsi_delete_portal_group -- Delete an existing portal group. + - iscsi_target_node_add_pg_ig_maps -- Add initiator group to portal group mappings to an existing iSCSI target node. + - iscsi_target_node_remove_pg_ig_maps -- Delete initiator group to portal group mappings from an existing iSCSI target node. + - iscsi_get_portal_groups -- Show information about all available portal groups. + +~~~ +/path/to/spdk/scripts/rpc.py iscsi_create_portal_group 1 10.0.0.1:3260 +~~~ + +### Initiator groups + + - iscsi_create_initiator_group -- Add an initiator group. + - iscsi_delete_initiator_group -- Delete an existing initiator group. + - iscsi_initiator_group_add_initiators -- Add initiators to an existing initiator group. + - iscsi_get_initiator_groups -- Show information about all available initiator groups. + +~~~ +/path/to/spdk/scripts/rpc.py iscsi_create_initiator_group 2 ANY 10.0.0.2/32 +~~~ + +### Target nodes + + - iscsi_create_target_node -- Add an iSCSI target node. + - iscsi_delete_target_node -- Delete an iSCSI target node. + - iscsi_target_node_add_lun -- Add a LUN to an existing iSCSI target node. + - iscsi_get_target_nodes -- Show information about all available iSCSI target nodes. + +~~~ +/path/to/spdk/scripts/rpc.py iscsi_create_target_node Target3 Target3_alias MyBdev:0 1:2 64 -d +~~~ + +## Configuring iSCSI Initiator {#iscsi_initiator} + +The Linux initiator is open-iscsi. + +Installing open-iscsi package +Fedora: +~~~ +yum install -y iscsi-initiator-utils +~~~ + +Ubuntu: +~~~ +apt-get install -y open-iscsi +~~~ + +### Setup + +Edit /etc/iscsi/iscsid.conf +~~~ +node.session.cmds_max = 4096 +node.session.queue_depth = 128 +~~~ + +iscsid must be restarted or receive SIGHUP for changes to take effect. To send SIGHUP, run: +~~~ +killall -HUP iscsid +~~~ + +Recommended changes to /etc/sysctl.conf +~~~ +net.ipv4.tcp_timestamps = 1 +net.ipv4.tcp_sack = 0 + +net.ipv4.tcp_rmem = 10000000 10000000 10000000 +net.ipv4.tcp_wmem = 10000000 10000000 10000000 +net.ipv4.tcp_mem = 10000000 10000000 10000000 +net.core.rmem_default = 524287 +net.core.wmem_default = 524287 +net.core.rmem_max = 524287 +net.core.wmem_max = 524287 +net.core.optmem_max = 524287 +net.core.netdev_max_backlog = 300000 +~~~ + +### Discovery + +Assume target is at 10.0.0.1 +~~~ +iscsiadm -m discovery -t sendtargets -p 10.0.0.1 +~~~ + +### Connect to target + +~~~ +iscsiadm -m node --login +~~~ + +At this point the iSCSI target should show up as SCSI disks. Check dmesg to see what +they came up as. + +### Disconnect from target + +~~~ +iscsiadm -m node --logout +~~~ + +### Deleting target node cache + +~~~ +iscsiadm -m node -o delete +~~~ + +This will cause the initiator to forget all previously discovered iSCSI target nodes. + +### Finding /dev/sdX nodes for iSCSI LUNs + +~~~ +iscsiadm -m session -P 3 | grep "Attached scsi disk" | awk '{print $4}' +~~~ + +This will show the /dev node name for each SCSI LUN in all logged in iSCSI sessions. + +### Tuning + +After the targets are connected, they can be tuned. For example if /dev/sdc is +an iSCSI disk then the following can be done: +Set noop to scheduler + +~~~ +echo noop > /sys/block/sdc/queue/scheduler +~~~ + +Disable merging/coalescing (can be useful for precise workload measurements) + +~~~ +echo "2" > /sys/block/sdc/queue/nomerges +~~~ + +Increase requests for block queue + +~~~ +echo "1024" > /sys/block/sdc/queue/nr_requests +~~~ + +### Example: Configure simple iSCSI Target with one portal and two LUNs + +Assuming we have one iSCSI Target server with portal at 10.0.0.1:3200, two LUNs (Malloc0 and Malloc1), + and accepting initiators on 10.0.0.2/32, like on diagram below: + + + +#### Configure iSCSI Target + +Start iscsi_tgt application: +``` +./build/bin/iscsi_tgt +``` + +Construct two 64MB Malloc block devices with 512B sector size "Malloc0" and "Malloc1": + +``` +./scripts/rpc.py bdev_malloc_create -b Malloc0 64 512 +./scripts/rpc.py bdev_malloc_create -b Malloc1 64 512 +``` + +Create new portal group with id 1, and address 10.0.0.1:3260: + +``` +./scripts/rpc.py iscsi_create_portal_group 1 10.0.0.1:3260 +``` + +Create one initiator group with id 2 to accept any connection from 10.0.0.2/32: + +``` +./scripts/rpc.py iscsi_create_initiator_group 2 ANY 10.0.0.2/32 +``` + +Finally construct one target using previously created bdevs as LUN0 (Malloc0) and LUN1 (Malloc1) +with a name "disk1" and alias "Data Disk1" using portal group 1 and initiator group 2. + +``` +./scripts/rpc.py iscsi_create_target_node disk1 "Data Disk1" "Malloc0:0 Malloc1:1" 1:2 64 -d +``` + +#### Configure initiator + +Discover target + +~~~ +$ iscsiadm -m discovery -t sendtargets -p 10.0.0.1 +10.0.0.1:3260,1 iqn.2016-06.io.spdk:disk1 +~~~ + +Connect to the target + +~~~ +iscsiadm -m node --login +~~~ + +At this point the iSCSI target should show up as SCSI disks. + +Check dmesg to see what they came up as. In this example it can look like below: + +~~~ +... +[630111.860078] scsi host68: iSCSI Initiator over TCP/IP +[630112.124743] scsi 68:0:0:0: Direct-Access INTEL Malloc disk 0001 PQ: 0 ANSI: 5 +[630112.125445] sd 68:0:0:0: [sdd] 131072 512-byte logical blocks: (67.1 MB/64.0 MiB) +[630112.125468] sd 68:0:0:0: Attached scsi generic sg3 type 0 +[630112.125926] sd 68:0:0:0: [sdd] Write Protect is off +[630112.125934] sd 68:0:0:0: [sdd] Mode Sense: 83 00 00 08 +[630112.126049] sd 68:0:0:0: [sdd] Write cache: enabled, read cache: disabled, doesn't support DPO or FUA +[630112.126483] scsi 68:0:0:1: Direct-Access INTEL Malloc disk 0001 PQ: 0 ANSI: 5 +[630112.127096] sd 68:0:0:1: Attached scsi generic sg4 type 0 +[630112.127143] sd 68:0:0:1: [sde] 131072 512-byte logical blocks: (67.1 MB/64.0 MiB) +[630112.127566] sd 68:0:0:1: [sde] Write Protect is off +[630112.127573] sd 68:0:0:1: [sde] Mode Sense: 83 00 00 08 +[630112.127728] sd 68:0:0:1: [sde] Write cache: enabled, read cache: disabled, doesn't support DPO or FUA +[630112.128246] sd 68:0:0:0: [sdd] Attached SCSI disk +[630112.129789] sd 68:0:0:1: [sde] Attached SCSI disk +... +~~~ + +You may also use simple bash command to find /dev/sdX nodes for each iSCSI LUN +in all logged iSCSI sessions: + +~~~ +$ iscsiadm -m session -P 3 | grep "Attached scsi disk" | awk '{print $4}' +sdd +sde +~~~ + +# iSCSI Hotplug {#iscsi_hotplug} + +At the iSCSI level, we provide the following support for Hotplug: + +1. bdev/nvme: + At the bdev/nvme level, we start one hotplug monitor which will call + spdk_nvme_probe() periodically to get the hotplug events. We provide the + private attach_cb and remove_cb for spdk_nvme_probe(). For the attach_cb, + we will create the block device base on the NVMe device attached, and for the + remove_cb, we will unregister the block device, which will also notify the + upper level stack (for iSCSI target, the upper level stack is scsi/lun) to + handle the hot-remove event. + +2. scsi/lun: + When the LUN receive the hot-remove notification from block device layer, + the LUN will be marked as removed, and all the IOs after this point will + return with check condition status. Then the LUN starts one poller which will + wait for all the commands which have already been submitted to block device to + return back; after all the commands return back, the LUN will be deleted. + +## Known bugs and limitations {#iscsi_hotplug_bugs} + +For write command, if you want to test hotplug with write command which will +cause r2t, for example 1M size IO, it will crash the iscsi tgt. +For read command, if you want to test hotplug with large read IO, for example 1M +size IO, it will probably crash the iscsi tgt. + +@sa spdk_nvme_probe diff --git a/src/spdk/doc/jsonrpc.md b/src/spdk/doc/jsonrpc.md new file mode 100644 index 000000000..055c3b11a --- /dev/null +++ b/src/spdk/doc/jsonrpc.md @@ -0,0 +1,6743 @@ +# JSON-RPC Methods {#jsonrpc} + +# Overview {#jsonrpc_overview} + +SPDK implements a [JSON-RPC 2.0](http://www.jsonrpc.org/specification) server +to allow external management tools to dynamically configure SPDK components. + +## Parameters + +Most of the commands can take parameters. If present, parameter is validated against its domain. If this check fail +whole command will fail with response error message [Invalid params](@ref jsonrpc_error_message). + +### Required parameters + +These parameters are mandatory. If any required parameter is missing RPC command will fail with proper error response. + +### Optional parameters + +Those parameters might be omitted. If an optional parameter is present it must be valid otherwise command will fail +proper error response. + +## Error response message {#jsonrpc_error_message} + +Each error response will contain proper message. As much as possible the message should indicate what went wrong during +command processing. + +There is ongoing effort to customize this messages but some RPC methods just return "Invalid parameters" as message body +for any kind of error. + +Code | Description +------ | ----------- +-1 | Invalid state - given method exists but it is not callable in [current runtime state](@ref rpc_framework_start_init) +-32600 | Invalid request - not compliant with JSON-RPC 2.0 Specification +-32601 | Method not found +-32602 | @ref jsonrpc_invalid_params +-32603 | Internal error for e.g.: errors like out of memory +-32700 | @ref jsonrpc_parser_error + +### Parser error {#jsonrpc_parser_error} + +Encountered some error during parsing request like: + +- the JSON object is malformed +- parameter is too long +- request is too long + +### Invalid params {#jsonrpc_invalid_params} + +This type of error is most common one. It mean that there is an error while processing the request like: + +- Parameters decoding in RPC method handler failed because required parameter is missing. +- Unknown parameter present encountered. +- Parameter type doesn't match expected type e.g.: given number when expected a string. +- Parameter domain check failed. +- Request is valid but some other error occurred during processing request. If possible message explains the error reason nature. + +## Adding external RPC methods + +SPDK includes both in-tree modules as well as the ability to use external modules. The in-tree modules include some python +scripts to ease the process of sending RPCs to in-tree modules. External modules can utilize this same framework to add new RPC +methods as follows: + +If PYTHONPATH doesn't include the location of the external RPC python script, it should be updated: + +~~~ +export PYTHONPATH=/home/user/plugin_example/ +~~~ + +In provided location, create python module file (e.g. rpc_plugin.py) with new RPC methods. The file should contain +spdk_rpc_plugin_initialize() method that will be called when the plugin is loaded to define new parsers for provided subparsers +argument that adds new RPC calls (subparsers.add_parser()). The new parsers should use the client.call() method to call RPC +functions registered within the external module using the SPDK_RPC_REGISTER() macro. Example: + +~~~ +from rpc.client import print_json + + +def example_create(client, num_blocks, block_size, name=None, uuid=None): + """Construct an example block device. + + Args: + num_blocks: size of block device in blocks + block_size: block size of device; must be a power of 2 and at least 512 + name: name of block device (optional) + uuid: UUID of block device (optional) + + Returns: + Name of created block device. + """ + params = {'num_blocks': num_blocks, 'block_size': block_size} + if name: + params['name'] = name + if uuid: + params['uuid'] = uuid + return client.call('bdev_example_create', params) + + +def example_delete(client, name): + """Delete example block device. + + Args: + bdev_name: name of bdev to delete + """ + params = {'name': name} + return client.call('bdev_example_delete', params) + + +def spdk_rpc_plugin_initialize(subparsers): + def bdev_example_create(args): + num_blocks = (args.total_size * 1024 * 1024) // args.block_size + print_json(example_create(args.client, + num_blocks=int(num_blocks), + block_size=args.block_size, + name=args.name, + uuid=args.uuid)) + + p = subparsers.add_parser('bdev_example_create', + help='Create an example bdev') + p.add_argument('-b', '--name', help="Name of the bdev") + p.add_argument('-u', '--uuid', help="UUID of the bdev") + p.add_argument( + 'total_size', help='Size of bdev in MB (float > 0)', type=float) + p.add_argument('block_size', help='Block size for this bdev', type=int) + p.set_defaults(func=bdev_example_create) + + def bdev_example_delete(args): + example_delete(args.client, + name=args.name) + + p = subparsers.add_parser('bdev_example_delete', + help='Delete an example disk') + p.add_argument('name', help='example bdev name') + p.set_defaults(func=bdev_example_delete) +~~~ + +Finally, call the rpc.py script with '--plugin' parameter to provide above python module name: + +~~~ +./scripts/rpc.py --plugin rpc_plugin bdev_example_create 10 4096 +~~~ + +# App Framework {#jsonrpc_components_app} + +## spdk_kill_instance {#rpc_spdk_kill_instance} + +Send a signal to the application. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +sig_name | Required | string | Signal to send (SIGINT, SIGTERM, SIGQUIT, SIGHUP, or SIGKILL) + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "spdk_kill_instance", + "params": { + "sig_name": "SIGINT" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## framework_monitor_context_switch {#rpc_framework_monitor_context_switch} + +Query, enable, or disable the context switch monitoring functionality. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +enabled | Optional | boolean | Enable (`true`) or disable (`false`) monitoring (omit this parameter to query the current state) + +### Response + +Name | Type | Description +----------------------- | ----------- | ----------- +enabled | boolean | The current state of context switch monitoring + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_monitor_context_switch", + "params": { + "enabled": false + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "enabled": false + } +} +~~~ + +## framework_start_init {#rpc_framework_start_init} + +Start initialization of SPDK subsystems when it is deferred by starting SPDK application with option -w. +During its deferral some RPCs can be used to set global parameters for SPDK subsystems. +This RPC can be called only once. + +### Parameters + +This method has no parameters. + +### Response + +Completion status of SPDK subsystem initialization is returned as a boolean. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_start_init" +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## framework_wait_init {#rpc_framework_wait_init} + +Do not return until all subsystems have been initialized and the RPC system state is running. +If the application is already running, this call will return immediately. This RPC can be called at any time. + +### Parameters + +This method has no parameters. + +### Response + +Returns True when subsystems have been initialized. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_wait_init" +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## rpc_get_methods {#rpc_rpc_get_methods} + +Get an array of supported RPC methods. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +current | Optional | boolean | Get an array of RPC methods only callable in the current state. + +### Response + +The response is an array of supported RPC methods. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "rpc_get_methods" +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + "framework_start_init", + "rpc_get_methods", + "scsi_get_devices", + "net_get_interfaces", + "delete_ip_address", + "net_interface_add_ip_address", + "nbd_get_disks", + "nbd_stop_disk", + "nbd_start_disk", + "log_get_flags", + "log_clear_flag", + "log_set_flag", + "log_get_level", + "log_set_level", + "log_get_print_level", + "log_set_print_level", + "iscsi_get_options", + "iscsi_target_node_add_lun", + "iscsi_get_connections", + "iscsi_delete_portal_group", + "iscsi_create_portal_group", + "iscsi_get_portal_groups", + "iscsi_delete_target_node", + "iscsi_target_node_remove_pg_ig_maps", + "iscsi_target_node_add_pg_ig_maps", + "iscsi_create_target_node", + "iscsi_get_target_nodes", + "iscsi_delete_initiator_group", + "iscsi_initiator_group_remove_initiators", + "iscsi_initiator_group_add_initiators", + "iscsi_create_initiator_group", + "iscsi_get_initiator_groups", + "iscsi_set_options", + "bdev_set_options", + "bdev_set_qos_limit", + "bdev_get_bdevs", + "bdev_get_iostat", + "framework_get_config", + "framework_get_subsystems", + "framework_monitor_context_switch", + "spdk_kill_instance", + "ioat_scan_accel_engine", + "idxd_scan_accel_engine", + "bdev_virtio_attach_controller", + "bdev_virtio_scsi_get_devices", + "bdev_virtio_detach_controller", + "bdev_aio_delete", + "bdev_aio_create", + "bdev_split_delete", + "bdev_split_create", + "bdev_error_inject_error", + "bdev_error_delete", + "bdev_error_create", + "bdev_passthru_create", + "bdev_passthru_delete" + "bdev_nvme_apply_firmware", + "bdev_nvme_detach_controller", + "bdev_nvme_attach_controller", + "bdev_null_create", + "bdev_malloc_delete", + "bdev_malloc_create", + "bdev_ftl_delete", + "bdev_ftl_create", + "bdev_lvol_get_lvstores", + "bdev_lvol_delete", + "bdev_lvol_resize", + "bdev_lvol_set_read_only", + "bdev_lvol_decouple_parent", + "bdev_lvol_inflate", + "bdev_lvol_rename", + "bdev_lvol_clone", + "bdev_lvol_snapshot", + "bdev_lvol_create", + "bdev_lvol_delete_lvstore", + "bdev_lvol_rename_lvstore", + "bdev_lvol_create_lvstore" + ] +} +~~~ + +## framework_get_subsystems {#rpc_framework_get_subsystems} + +Get an array of name and dependency relationship of SPDK subsystems in initialization order. + +### Parameters + +None + +### Response + +The response is an array of name and dependency relationship of SPDK subsystems in initialization order. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_get_subsystems" +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "subsystem": "accel", + "depends_on": [] + }, + { + "subsystem": "interface", + "depends_on": [] + }, + { + "subsystem": "net_framework", + "depends_on": [ + "interface" + ] + }, + { + "subsystem": "bdev", + "depends_on": [ + "accel" + ] + }, + { + "subsystem": "nbd", + "depends_on": [ + "bdev" + ] + }, + { + "subsystem": "nvmf", + "depends_on": [ + "bdev" + ] + }, + { + "subsystem": "scsi", + "depends_on": [ + "bdev" + ] + }, + { + "subsystem": "vhost", + "depends_on": [ + "scsi" + ] + }, + { + "subsystem": "iscsi", + "depends_on": [ + "scsi" + ] + } + ] +} +~~~ + +## framework_get_config {#rpc_framework_get_config} + +Get current configuration of the specified SPDK framework + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | SPDK subsystem name + +### Response + +The response is current configuration of the specified SPDK subsystem. +Null is returned if it is not retrievable by the framework_get_config method and empty array is returned if it is empty. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_get_config", + "params": { + "name": "bdev" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "params": { + "base_bdev": "Malloc2", + "split_size_mb": 0, + "split_count": 2 + }, + "method": "bdev_split_create" + }, + { + "params": { + "trtype": "PCIe", + "name": "Nvme1", + "traddr": "0000:01:00.0" + }, + "method": "bdev_nvme_attach_controller" + }, + { + "params": { + "trtype": "PCIe", + "name": "Nvme2", + "traddr": "0000:03:00.0" + }, + "method": "bdev_nvme_attach_controller" + }, + { + "params": { + "block_size": 512, + "num_blocks": 131072, + "name": "Malloc0", + "uuid": "913fc008-79a7-447f-b2c4-c73543638c31" + }, + "method": "bdev_malloc_create" + }, + { + "params": { + "block_size": 512, + "num_blocks": 131072, + "name": "Malloc1", + "uuid": "dd5b8f6e-b67a-4506-b606-7fff5a859920" + }, + "method": "bdev_malloc_create" + } + ] +} +~~~ + +## framework_get_reactors {#rpc_framework_get_reactors} + +Retrieve an array of all reactors. + +### Parameters + +This method has no parameters. + +### Response + +The response is an array of all reactors. + +### Example + +Example request: +~~~ +{ + "jsonrpc": "2.0", + "method": "framework_get_reactors", + "id": 1 +} +~~~ + +Example response: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tick_rate": 2400000000, + "reactors": [ + { + "lcore": 0, + "busy": 41289723495, + "idle": 3624832946, + "lw_threads": [ + { + "name": "app_thread", + "id", 1, + "cpumask": "1", + "elapsed": 44910853363 + } + ] + } + ] + } +} +~~~ + +## thread_get_stats {#rpc_thread_get_stats} + +Retrieve current statistics of all the threads. + +### Parameters + +This method has no parameters. + +### Response + +The response is an array of objects containing threads statistics. + +### Example + +Example request: +~~~ +{ + "jsonrpc": "2.0", + "method": "thread_get_stats", + "id": 1 +} +~~~ + +Example response: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tick_rate": 2400000000, + "threads": [ + { + "name": "app_thread", + "id": 1, + "cpumask": "1", + "busy": 139223208, + "idle": 8641080608, + "active_pollers_count": 1, + "timed_pollers_count": 2, + "paused_pollers_count": 0 + } + ] + } +} +~~~ + +## thread_set_cpumask {#rpc_thread_set_cpumask} + +Set the cpumask of the thread to the specified value. The thread may be migrated +to one of the specified CPUs. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +id | Required | string | Thread ID +cpumask | Required | string | Cpumask for this thread + +### Response + +Completion status of the operation is returned as a boolean. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "thread_set_cpumask", + "id": 1, + "params": { + "id": "1", + "cpumask": "1" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## thread_get_pollers {#rpc_thread_get_pollers} + +Retrieve current pollers of all the threads. + +### Parameters + +This method has no parameters. + +### Response + +The response is an array of objects containing pollers of all the threads. + +### Example + +Example request: +~~~ +{ + "jsonrpc": "2.0", + "method": "thread_get_pollers", + "id": 1 +} +~~~ + +Example response: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tick_rate": 2500000000, + "threads": [ + { + "name": "app_thread", + "id": 1, + "active_pollers": [], + "timed_pollers": [ + { + "name": "spdk_rpc_subsystem_poll", + "state": "waiting", + "run_count": 12345, + "busy_count": 10000, + "period_ticks": 10000000 + } + ], + "paused_pollers": [] + } + ] + } +} +~~~ + +## thread_get_io_channels {#rpc_thread_get_io_channels} + +Retrieve current IO channels of all the threads. + +### Parameters + +This method has no parameters. + +### Response + +The response is an array of objects containing IO channels of all the threads. + +### Example + +Example request: +~~~ +{ + "jsonrpc": "2.0", + "method": "thread_get_io_channels", + "id": 1 +} +~~~ + +Example response: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tick_rate": 2500000000, + "threads": [ + { + "name": "app_thread", + "io_channels": [ + { + "name": "nvmf_tgt", + "ref": 1 + } + ] + } + ] + } +} +~~~ +# Block Device Abstraction Layer {#jsonrpc_components_bdev} + +## bdev_set_options {#rpc_bdev_set_options} + +Set global parameters for the block device (bdev) subsystem. This RPC may only be called +before SPDK subsystems have been initialized. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_io_pool_size | Optional | number | Number of spdk_bdev_io structures in shared buffer pool +bdev_io_cache_size | Optional | number | Maximum number of spdk_bdev_io structures cached per thread +bdev_auto_examine | Optional | boolean | If set to false, the bdev layer will not examine every disks automatically + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_set_options", + "params": { + "bdev_io_pool_size": 65536, + "bdev_io_cache_size": 256 + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_get_bdevs {#rpc_bdev_get_bdevs} + +Get information about block devices (bdevs). + +### Parameters + +The user may specify no parameters in order to list all block devices, or a block device may be +specified by name. + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | Block device name + +### Response + +The response is an array of objects containing information about the requested block devices. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_get_bdevs", + "params": { + "name": "Malloc0" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "name": "Malloc0", + "product_name": "Malloc disk", + "block_size": 512, + "num_blocks": 20480, + "claimed": false, + "zoned": false, + "supported_io_types": { + "read": true, + "write": true, + "unmap": true, + "write_zeroes": true, + "flush": true, + "reset": true, + "nvme_admin": false, + "nvme_io": false + }, + "driver_specific": {} + } + ] +} +~~~ + +## bdev_get_iostat {#rpc_bdev_get_iostat} + +Get I/O statistics of block devices (bdevs). + +### Parameters + +The user may specify no parameters in order to list all block devices, or a block device may be +specified by name. + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | Block device name + +### Response + +The response is an array of objects containing I/O statistics of the requested block devices. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_get_iostat", + "params": { + "name": "Nvme0n1" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tick_rate": 2200000000, + "bdevs" : [ + { + "name": "Nvme0n1", + "bytes_read": 36864, + "num_read_ops": 2, + "bytes_written": 0, + "num_write_ops": 0, + "bytes_unmapped": 0, + "num_unmap_ops": 0, + "read_latency_ticks": 178904, + "write_latency_ticks": 0, + "unmap_latency_ticks": 0, + "queue_depth_polling_period": 2, + "queue_depth": 0, + "io_time": 0, + "weighted_io_time": 0 + } + ] + } +} +~~~ + +## bdev_enable_histogram {#rpc_bdev_enable_histogram} + +Control whether collecting data for histogram is enabled for specified bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Block device name +enable | Required | boolean | Enable or disable histogram on specified device + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_enable_histogram", + "params": { + "name": "Nvme0n1" + "enable": true + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_get_histogram {#rpc_bdev_get_histogram} + +Get latency histogram for specified bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Block device name + +### Result + +Name | Description +------------------------| ----------- +histogram | Base64 encoded histogram +bucket_shift | Granularity of the histogram buckets +tsc_rate | Ticks per second + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_get_histogram", + "params": { + "name": "Nvme0n1" + } +} +~~~ + +Example response: +Note that histogram field is trimmed, actual encoded histogram length is ~80kb. + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "histogram": "AAAAAAAAAAAAAA...AAAAAAAAA==", + "tsc_rate": 2300000000, + "bucket_shift": 7 + } +} +~~~ + +## bdev_set_qos_limit {#rpc_bdev_set_qos_limit} + +Set the quality of service rate limit on a bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Block device name +rw_ios_per_sec | Optional | number | Number of R/W I/Os per second to allow. 0 means unlimited. +rw_mbytes_per_sec | Optional | number | Number of R/W megabytes per second to allow. 0 means unlimited. +r_mbytes_per_sec | Optional | number | Number of Read megabytes per second to allow. 0 means unlimited. +w_mbytes_per_sec | Optional | number | Number of Write megabytes per second to allow. 0 means unlimited. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_set_qos_limit", + "params": { + "name": "Malloc0" + "rw_ios_per_sec": 20000 + "rw_mbytes_per_sec": 100 + "r_mbytes_per_sec": 50 + "w_mbytes_per_sec": 50 + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_ocf_create {#rpc_bdev_ocf_create} + +Construct new OCF bdev. +Command accepts cache mode that is going to be used. +Currently, we support Write-Through, Pass-Through and Write-Back OCF cache modes. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name to use +mode | Required | string | OCF cache mode ('wb' or 'wt' or 'pt') +cache_bdev_name | Required | string | Name of underlying cache bdev +core_bdev_name | Required | string | Name of underlying core bdev + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "ocf0", + "mode": "wt", + "cache_bdev_name": "Nvme0n1" + "core_bdev_name": "aio0" + }, + "jsonrpc": "2.0", + "method": "bdev_ocf_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "ocf0" +} +~~~ + +## bdev_ocf_delete {#rpc_bdev_ocf_delete} + +Delete the OCF bdev + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "ocf0" + }, + "jsonrpc": "2.0", + "method": "bdev_ocf_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_ocf_get_stats {#rpc_bdev_ocf_get_stats} + +Get statistics of chosen OCF block device. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Block device name + +### Response + +Statistics as json object. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_ocf_get_stats", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + "usage": { + "clean": { + "count": 76033, + "units": "4KiB blocks", + "percentage": "100.0" + }, + "free": { + "count": 767, + "units": "4KiB blocks", + "percentage": "0.9" + }, + "occupancy": { + "count": 76033, + "units": "4KiB blocks", + "percentage": "99.0" + }, + "dirty": { + "count": 0, + "units": "4KiB blocks", + "percentage": "0.0" + } + }, + "requests": { + "rd_total": { + "count": 2, + "units": "Requests", + "percentage": "0.0" + }, + "wr_full_misses": { + "count": 76280, + "units": "Requests", + "percentage": "35.6" + }, + "rd_full_misses": { + "count": 1, + "units": "Requests", + "percentage": "0.0" + }, + "rd_partial_misses": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "wr_total": { + "count": 212416, + "units": "Requests", + "percentage": "99.2" + }, + "wr_pt": { + "count": 1535, + "units": "Requests", + "percentage": "0.7" + }, + "wr_partial_misses": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "serviced": { + "count": 212418, + "units": "Requests", + "percentage": "99.2" + }, + "rd_pt": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "total": { + "count": 213953, + "units": "Requests", + "percentage": "100.0" + }, + "rd_hits": { + "count": 1, + "units": "Requests", + "percentage": "0.0" + }, + "wr_hits": { + "count": 136136, + "units": "Requests", + "percentage": "63.6" + } + }, + "errors": { + "total": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "cache_obj_total": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "core_obj_total": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "cache_obj_rd": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "core_obj_wr": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "core_obj_rd": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + }, + "cache_obj_wr": { + "count": 0, + "units": "Requests", + "percentage": "0.0" + } + }, + "blocks": { + "volume_rd": { + "count": 9, + "units": "4KiB blocks", + "percentage": "0.0" + }, + "volume_wr": { + "count": 213951, + "units": "4KiB blocks", + "percentage": "99.9" + }, + "cache_obj_total": { + "count": 212425, + "units": "4KiB blocks", + "percentage": "100.0" + }, + "core_obj_total": { + "count": 213959, + "units": "4KiB blocks", + "percentage": "100.0" + }, + "cache_obj_rd": { + "count": 1, + "units": "4KiB blocks", + "percentage": "0.0" + }, + "core_obj_wr": { + "count": 213951, + "units": "4KiB blocks", + "percentage": "99.9" + }, + "volume_total": { + "count": 213960, + "units": "4KiB blocks", + "percentage": "100.0" + }, + "core_obj_rd": { + "count": 8, + "units": "4KiB blocks", + "percentage": "0.0" + }, + "cache_obj_wr": { + "count": 212424, + "units": "4KiB blocks", + "percentage": "99.9" + } + ] +} +~~~ + +## bdev_ocf_get_bdevs {#rpc_bdev_ocf_get_bdevs} + +Get list of OCF devices including unregistered ones. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | Name of OCF vbdev or name of cache device or name of core device + +### Response + +Array of OCF devices with their current status, along with core and cache bdevs. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_ocf_get_bdevs", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "name": "PartCache", + "started": false, + "cache": { + "name": "Malloc0", + "attached": true + }, + "core": { + "name": "Malloc1", + "attached": false + } + } + ] +} +~~~ + +## bdev_malloc_create {#rpc_bdev_malloc_create} + +Construct @ref bdev_config_malloc + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | Bdev name to use +block_size | Required | number | Block size in bytes -must be multiple of 512 +num_blocks | Required | number | Number of blocks +uuid | Optional | string | UUID of new bdev + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "block_size": 4096, + "num_blocks": 16384, + "name": "Malloc0", + "uuid": "2b6601ba-eada-44fb-9a83-a20eb9eb9e90" + }, + "jsonrpc": "2.0", + "method": "bdev_malloc_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "Malloc0" +} +~~~ + +## bdev_malloc_delete {#rpc_bdev_malloc_delete} + +Delete @ref bdev_config_malloc + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Malloc0" + }, + "jsonrpc": "2.0", + "method": "bdev_malloc_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_null_create {#rpc_bdev_null_create} + +Construct @ref bdev_config_null + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | Bdev name to use +block_size | Required | number | Block size in bytes +num_blocks | Required | number | Number of blocks +uuid | Optional | string | UUID of new bdev +md_size | Optional | number | Metadata size in bytes +dif_type | Optional | number | Protection information type (0, 1, 2 or 3). Default: 0 - no protection. +dif_is_head_of_md | Optional | boolean | Protection information is in the first 8 bytes of MD. Default: in the last 8 bytes. + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "block_size": 4104, + "num_blocks": 16384, + "name": "Null0", + "uuid": "2b6601ba-eada-44fb-9a83-a20eb9eb9e90", + "md_size": 8, + "dif_type": 1, + "dif_is_head_of_md": true + }, + "jsonrpc": "2.0", + "method": "bdev_null_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "Null0" +} +~~~ + +## bdev_null_delete {#rpc_bdev_null_delete} + +Delete @ref bdev_config_null. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Null0" + }, + "jsonrpc": "2.0", + "method": "bdev_null_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_aio_create {#rpc_bdev_aio_create} + +Construct @ref bdev_config_aio. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name to use +filename | Required | number | Path to device or file +block_size | Optional | number | Block size in bytes + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "block_size": 4096, + "name": "Aio0", + "filename": "/tmp/aio_bdev_file" + }, + "jsonrpc": "2.0", + "method": "bdev_aio_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "Aio0" +} +~~~ + +## bdev_aio_delete {#rpc_bdev_aio_delete} + +Delete @ref bdev_config_aio. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Aio0" + }, + "jsonrpc": "2.0", + "method": "bdev_aio_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_nvme_set_options {#rpc_bdev_nvme_set_options} + +Set global parameters for all bdev NVMe. This RPC may only be called before SPDK subsystems have been initialized or any bdev NVMe has been created. + +### Parameters + +Name | Optional | Type | Description +-------------------------- | -------- | ----------- | ----------- +action_on_timeout | Optional | string | Action to take on command time out: none, reset or abort +timeout_us | Optional | number | Timeout for each command, in microseconds. If 0, don't track timeouts +retry_count | Optional | number | The number of attempts per I/O before an I/O fails +arbitration_burst | Optional | number | The value is expressed as a power of two, a value of 111b indicates no limit +low_priority_weight | Optional | number | The maximum number of commands that the controller may launch at one time from a low priority queue +medium_priority_weight | Optional | number | The maximum number of commands that the controller may launch at one time from a medium priority queue +high_priority_weight | Optional | number | The maximum number of commands that the controller may launch at one time from a high priority queue +nvme_adminq_poll_period_us | Optional | number | How often the admin queue is polled for asynchronous events in microseconds +nvme_ioq_poll_period_us | Optional | number | How often I/O queues are polled for completions, in microseconds. Default: 0 (as fast as possible). +io_queue_requests | Optional | number | The number of requests allocated for each NVMe I/O queue. Default: 512. +delay_cmd_submit | Optional | boolean | Enable delaying NVMe command submission to allow batching of multiple commands. Default: `true`. + +### Example + +Example request: + +~~~ +request: +{ + "params": { + "retry_count": 5, + "arbitration_burst": 3, + "low_priority_weight": 8, + "medium_priority_weight":8, + "high_priority_weight": 8, + "nvme_adminq_poll_period_us": 2000, + "timeout_us": 10000000, + "action_on_timeout": "reset", + "io_queue_requests" : 2048, + "delay_cmd_submit": true + }, + "jsonrpc": "2.0", + "method": "bdev_nvme_set_options", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_nvme_set_hotplug {#rpc_bdev_nvme_set_hotplug} + +Change settings of the NVMe hotplug feature. If enabled, PCIe NVMe bdevs will be automatically discovered on insertion +and deleted on removal. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +enabled | Required | string | True to enable, false to disable +period_us | Optional | number | How often to poll for hot-insert and hot-remove events. Values: 0 - reset/use default or 1 to 10000000. + +### Example + +Example request: + +~~~ +request: +{ + "params": { + "enabled": true, + "period_us": 2000 + }, + "jsonrpc": "2.0", + "method": "bdev_nvme_set_hotplug", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_nvme_attach_controller {#rpc_bdev_nvme_attach_controller} + +Construct @ref bdev_config_nvme + +### Result + +Array of names of newly created bdevs. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Name of the NVMe controller, prefix for each bdev name +trtype | Required | string | NVMe-oF target trtype: rdma or pcie +traddr | Required | string | NVMe-oF target address: ip or BDF +adrfam | Optional | string | NVMe-oF target adrfam: ipv4, ipv6, ib, fc, intra_host +trsvcid | Optional | string | NVMe-oF target trsvcid: port number +subnqn | Optional | string | NVMe-oF target subnqn +hostnqn | Optional | string | NVMe-oF target hostnqn +hostaddr | Optional | string | NVMe-oF host address: ip address +hostsvcid | Optional | string | NVMe-oF host trsvcid: port number +prchk_reftag | Optional | bool | Enable checking of PI reference tag for I/O processing +prchk_guard | Optional | bool | Enable checking of PI guard for I/O processing + +### Example + +Example request: + +~~~ +{ + "params": { + "trtype": "pcie", + "name": "Nvme0", + "traddr": "0000:0a:00.0" + }, + "jsonrpc": "2.0", + "method": "bdev_nvme_attach_controller", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + "Nvme0n1" + ] +} +~~~ + +## bdev_nvme_get_controllers {#rpc_bdev_nvme_get_controllers} + +Get information about NVMe controllers. + +### Parameters + +The user may specify no parameters in order to list all NVMe controllers, or one NVMe controller may be +specified by name. + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | NVMe controller name + +### Response + +The response is an array of objects containing information about the requested NVMe controllers. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_nvme_get_controllers", + "params": { + "name": "Nvme0" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "name": "Nvme0", + "trid": { + "trtype": "PCIe", + "traddr": "0000:05:00.0" + } + } + ] +} +~~~ + +## bdev_nvme_detach_controller {#rpc_bdev_nvme_detach_controller} + +Detach NVMe controller and delete any associated bdevs. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Controller name + +### Example + +Example requests: + +~~~ +{ + "params": { + "name": "Nvme0" + }, + "jsonrpc": "2.0", + "method": "bdev_nvme_detach_controller", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register} + +Register CUSE device on NVMe controller. +This feature is considered as experimental. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Name of the NVMe controller +dev_path | Required | string | Path to the CUSE controller device, e.g. spdk/nvme0 + +### Example + +Example request: + +~~~ +{ + "params": { + "dev_path": "spdk/nvme0", + "name": "Nvme0" + }, + "jsonrpc": "2.0", + "method": "bdev_nvme_cuse_register", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_nvme_cuse_unregister {#rpc_bdev_nvme_cuse_unregister} + +Unregister CUSE device on NVMe controller. +This feature is considered as experimental. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Name of the NVMe controller + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Nvme0" + }, + "jsonrpc": "2.0", + "method": "bdev_nvme_cuse_unregister", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_rbd_create {#rpc_bdev_rbd_create} + +Create @ref bdev_config_rbd bdev + +This method is available only if SPDK was build with Ceph RBD support. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | Bdev name +user_id | Optional | string | Ceph ID (i.e. admin, not client.admin) +pool_name | Required | string | Pool name +rbd_name | Required | string | Image name +block_size | Required | number | Block size +config | Optional | string map | Explicit librados configuration + +If no config is specified, Ceph configuration files must exist with +all relevant settings for accessing the pool. If a config map is +passed, the configuration files are ignored and instead all key/value +pairs are passed to rados_conf_set to configure cluster access. In +practice, "mon_host" (= list of monitor address+port) and "key" (= the +secret key stored in Ceph keyrings) are enough. + +When accessing the image as some user other than "admin" (the +default), the "user_id" has to be set. + +### Result + +Name of newly created bdev. + +### Example + +Example request with `key` from `/etc/ceph/ceph.client.admin.keyring`: + +~~~ +{ + "params": { + "pool_name": "rbd", + "rbd_name": "foo", + "config": { + "mon_host": "192.168.7.1:6789,192.168.7.2:6789", + "key": "AQDwf8db7zR1GRAA5k7NKXjS5S5V4mntwUDnGQ==", + } + "block_size": 4096 + }, + "jsonrpc": "2.0", + "method": "bdev_rbd_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +response: +{ + "jsonrpc": "2.0", + "id": 1, + "result": "Ceph0" +} +~~~ + +## bdev_rbd_delete {#rpc_bdev_rbd_delete} + +Delete @ref bdev_config_rbd bdev + +This method is available only if SPDK was build with Ceph RBD support. + +### Result + +`true` if bdev with provided name was deleted or `false` otherwise. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Rbd0" + }, + "jsonrpc": "2.0", + "method": "bdev_rbd_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_rbd_resize {#rpc_bdev_rbd_resize} + +Resize @ref bdev_config_rbd bdev + +This method is available only if SPDK was build with Ceph RBD support. + +### Result + +`true` if bdev with provided name was resized or `false` otherwise. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name +new_size | Required | int | New bdev size for resize operation in MiB + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Rbd0" + "new_size": "4096" + }, + "jsonrpc": "2.0", + "method": "bdev_rbd_resize", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_delay_create {#rpc_bdev_delay_create} + +Create delay bdev. This bdev type redirects all IO to it's base bdev and inserts a delay on the completion +path to create an artificial drive latency. All latency values supplied to this bdev should be in microseconds. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name +base_bdev_name | Required | string | Base bdev name +avg_read_latency | Required | number | average read latency (us) +p99_read_latency | Required | number | p99 read latency (us) +avg_write_latency | Required | number | average write latency (us) +p99_write_latency | Required | number | p99 write latency (us) + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "base_bdev_name": "Null0", + "name": "Delay0", + "avg_read_latency": "15", + "p99_read_latency": "50", + "avg_write_latency": "40", + "p99_write_latency": "110", + }, + "jsonrpc": "2.0", + "method": "bdev_delay_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "Delay0" +} +~~~ + +## bdev_delay_delete {#rpc_bdev_delay_delete} + +Delete delay bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Delay0" + }, + "jsonrpc": "2.0", + "method": "bdev_delay_delete", + "id": 1 +} + +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_delay_update_latency {#rpc_bdev_delay_update_latency} + +Update a target latency value associated with a given delay bdev. Any currently +outstanding I/O will be completed with the old latency. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +delay_bdev_name | Required | string | Name of the delay bdev +latency_type | Required | string | One of: avg_read, avg_write, p99_read, p99_write +latency_us | Required | number | The new latency value in microseconds + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "delay_bdev_name": "Delay0", + "latency_type": "avg_read", + "latency_us": "100", + }, + "jsonrpc": "2.0", + "method": "bdev_delay_update_latency", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "result": "true" +} +~~~ + +## bdev_error_create {#rpc_bdev_error_create} + +Construct error bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +base_name | Required | string | Base bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "base_name": "Malloc0" + }, + "jsonrpc": "2.0", + "method": "bdev_error_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_error_delete {#rpc_bdev_error_delete} + +Delete error bdev + +### Result + +`true` if bdev with provided name was deleted or `false` otherwise. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Error bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "EE_Malloc0" + }, + "jsonrpc": "2.0", + "method": "bdev_error_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_iscsi_create {#rpc_bdev_iscsi_create} + +Connect to iSCSI target and create bdev backed by this connection. + +This method is available only if SPDK was build with iSCSI initiator support. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name +initiator_iqn | Required | string | IQN name used during connection +url | Required | string | iSCSI resource URI + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "url": "iscsi://127.0.0.1/iqn.2016-06.io.spdk:disk1/0", + "initiator_iqn": "iqn.2016-06.io.spdk:init", + "name": "iSCSI0" + }, + "jsonrpc": "2.0", + "method": "bdev_iscsi_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "iSCSI0" +} +~~~ + +## bdev_iscsi_delete {#rpc_bdev_iscsi_delete} + +Delete iSCSI bdev and terminate connection to target. + +This method is available only if SPDK was built with iSCSI initiator support. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "iSCSI0" + }, + "jsonrpc": "2.0", + "method": "bdev_iscsi_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_ftl_create {#rpc_bdev_ftl_create} + +Create FTL bdev. + +This RPC is subject to change. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name +trtype | Required | string | Transport type +traddr | Required | string | NVMe target address +punits | Required | string | Parallel unit range in the form of start-end e.g 4-8 +uuid | Optional | string | UUID of restored bdev (not applicable when creating new instance) +cache | Optional | string | Name of the bdev to be used as a write buffer cache + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "nvme0" + "trtype" "pcie" + "traddr": "0000:00:04.0" + "punits": "0-3" + "uuid": "4a7481ce-786f-41a0-9b86-8f7465c8f4d3" + }, + "jsonrpc": "2.0", + "method": "bdev_ftl_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "name" : "nvme0" + "uuid" : "4a7481ce-786f-41a0-9b86-8f7465c8f4d3" + } +} +~~~ + +## bdev_ftl_delete {#rpc_bdev_ftl_delete} + +Delete FTL bdev. + +This RPC is subject to change. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "nvme0" + }, + "jsonrpc": "2.0", + "method": "bdev_ftl_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_pmem_create_pool {#rpc_bdev_pmem_create_pool} + +Create a @ref bdev_config_pmem blk pool file. It is equivalent of following `pmempool create` command: + +~~~ +pmempool create -s $((num_blocks * block_size)) blk $block_size $pmem_file +~~~ + +This method is available only if SPDK was built with PMDK support. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +pmem_file | Required | string | Path to new pmem file +num_blocks | Required | number | Number of blocks +block_size | Required | number | Size of each block in bytes + +### Example + +Example request: + +~~~ +{ + "params": { + "block_size": 512, + "num_blocks": 131072, + "pmem_file": "/tmp/pmem_file" + }, + "jsonrpc": "2.0", + "method": "bdev_pmem_create_pool", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_pmem_get_pool_info {#rpc_bdev_pmem_get_pool_info} + +Retrieve basic information about PMDK memory pool. + +This method is available only if SPDK was built with PMDK support. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +pmem_file | Required | string | Path to existing pmem file + +### Result + +Array of objects describing memory pool: + +Name | Type | Description +----------------------- | ----------- | ----------- +num_blocks | number | Number of blocks +block_size | number | Size of each block in bytes + +### Example + +Example request: + +~~~ +request: +{ + "params": { + "pmem_file": "/tmp/pmem_file" + }, + "jsonrpc": "2.0", + "method": "bdev_pmem_get_pool_info", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "block_size": 512, + "num_blocks": 129728 + } + ] +} +~~~ + +## bdev_pmem_delete_pool {#rpc_bdev_pmem_delete_pool} + +Delete pmem pool by removing file `pmem_file`. This method will fail if `pmem_file` is not a +valid pmem pool file. + +This method is available only if SPDK was built with PMDK support. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +pmem_file | Required | string | Path to new pmem file + +### Example + +Example request: + +~~~ +{ + "params": { + "pmem_file": "/tmp/pmem_file" + }, + "jsonrpc": "2.0", + "method": "bdev_pmem_delete_pool", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_pmem_create {#rpc_bdev_pmem_create} + +Construct @ref bdev_config_pmem bdev. + +This method is available only if SPDK was built with PMDK support. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name +pmem_file | Required | string | Path to existing pmem blk pool file + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "pmem_file": "/tmp/pmem_file", + "name": "Pmem0" + }, + "jsonrpc": "2.0", + "method": "bdev_pmem_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "Pmem0" +} +~~~ + +## bdev_pmem_delete {#rpc_bdev_pmem_delete} + +Delete @ref bdev_config_pmem bdev. This call will not remove backing pool files. + +This method is available only if SPDK was built with PMDK support. + +### Result + +`true` if bdev with provided name was deleted or `false` otherwise. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Pmem0" + }, + "jsonrpc": "2.0", + "method": "bdev_pmem_delete", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_passthru_create {#rpc_bdev_passthru_create} + +Create passthru bdev. This bdev type redirects all IO to it's base bdev. It has no other purpose than being an example +and a starting point in development of new bdev type. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name +base_bdev_name | Required | string | Base bdev name + +### Result + +Name of newly created bdev. + +### Example + +Example request: + +~~~ +{ + "params": { + "base_bdev_name": "Malloc0", + "name": "Passsthru0" + }, + "jsonrpc": "2.0", + "method": "bdev_passthru_create", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "Passsthru0" +} +~~~ + +## bdev_passthru_delete {#rpc_bdev_passthru_delete} + +Delete passthru bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "Passsthru0" + }, + "jsonrpc": "2.0", + "method": "bdev_passthru_delete", + "id": 1 +} + +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_virtio_attach_controller {#rpc_bdev_virtio_attach_controller} + +Create new initiator @ref bdev_config_virtio_scsi or @ref bdev_config_virtio_blk and expose all found bdevs. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Virtio SCSI base bdev name or Virtio Blk bdev name +trtype | Required | string | Virtio target trtype: pci or user +traddr | Required | string | target address: BDF or UNIX socket file path +dev_type | Required | string | Virtio device type: blk or scsi +vq_count | Optional | number | Number of queues this controller will utilize (default: 1) +vq_size | Optional | number | Size of each queue. Must be power of 2. (default: 512) + +In case of Virtio SCSI the `name` parameter will be base name for new created bdevs. For Virtio Blk `name` will be the +name of created bdev. + +`vq_count` and `vq_size` parameters are valid only if `trtype` is `user`. + +### Result + +Array of names of newly created bdevs. + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "VirtioScsi0", + "trtype": "user", + "vq_size": 128, + "dev_type": "scsi", + "traddr": "/tmp/VhostScsi0", + "vq_count": 4 + }, + "jsonrpc": "2.0", + "method": "bdev_virtio_attach_controller", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": ["VirtioScsi0t2", "VirtioScsi0t4"] +} +~~~ + +## bdev_virtio_scsi_get_devices {#rpc_bdev_virtio_scsi_get_devices} + +Show information about all available Virtio SCSI devices. + +### Parameters + +This method has no parameters. + +### Result + +Array of Virtio SCSI information objects. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_virtio_scsi_get_devices", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "name": "VirtioScsi0", + "virtio": { + "vq_size": 128, + "vq_count": 4, + "type": "user", + "socket": "/tmp/VhostScsi0" + } + } + ] +} +~~~ + +## bdev_virtio_detach_controller {#rpc_bdev_virtio_detach_controller} + +Remove a Virtio device. This command can be used to remove any type of virtio device. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Virtio name + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "VirtioUser0" + }, + "jsonrpc": "2.0", + "method": "bdev_virtio_detach_controller", + "id": 1 +} + +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +# iSCSI Target {#jsonrpc_components_iscsi_tgt} + +## iscsi_set_options method {#rpc_iscsi_set_options} + +Set global parameters for iSCSI targets. + +This RPC may only be called before SPDK subsystems have been initialized. This RPC can be called only once. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | ------- | ----------- +auth_file | Optional | string | Path to CHAP shared secret file (default: "") +node_base | Optional | string | Prefix of the name of iSCSI target node (default: "iqn.2016-06.io.spdk") +nop_timeout | Optional | number | Timeout in seconds to nop-in request to the initiator (default: 60) +nop_in_interval | Optional | number | Time interval in secs between nop-in requests by the target (default: 30) +disable_chap | Optional | boolean | CHAP for discovery session should be disabled (default: `false`) +require_chap | Optional | boolean | CHAP for discovery session should be required (default: `false`) +mutual_chap | Optional | boolean | CHAP for discovery session should be unidirectional (`false`) or bidirectional (`true`) (default: `false`) +chap_group | Optional | number | CHAP group ID for discovery session (default: 0) +max_sessions | Optional | number | Maximum number of sessions in the host (default: 128) +max_queue_depth | Optional | number | Maximum number of outstanding I/Os per queue (default: 64) +max_connections_per_session | Optional | number | Session specific parameter, MaxConnections (default: 2) +default_time2wait | Optional | number | Session specific parameter, DefaultTime2Wait (default: 2) +default_time2retain | Optional | number | Session specific parameter, DefaultTime2Retain (default: 20) +first_burst_length | Optional | number | Session specific parameter, FirstBurstLength (default: 8192) +immediate_data | Optional | boolean | Session specific parameter, ImmediateData (default: `true`) +error_recovery_level | Optional | number | Session specific parameter, ErrorRecoveryLevel (default: 0) +allow_duplicated_isid | Optional | boolean | Allow duplicated initiator session ID (default: `false`) + +To load CHAP shared secret file, its path is required to specify explicitly in the parameter `auth_file`. + +Parameters `disable_chap` and `require_chap` are mutually exclusive. Parameters `no_discovery_auth`, `req_discovery_auth`, `req_discovery_auth_mutual`, and `discovery_auth_group` are still available instead of `disable_chap`, `require_chap`, `mutual_chap`, and `chap_group`, respectivey but will be removed in future releases. + +### Example + +Example request: + +~~~ +{ + "params": { + "allow_duplicated_isid": true, + "default_time2retain": 60, + "first_burst_length": 8192, + "immediate_data": true, + "node_base": "iqn.2016-06.io.spdk", + "max_sessions": 128, + "nop_timeout": 30, + "nop_in_interval": 30, + "auth_file": "/usr/local/etc/spdk/auth.conf", + "disable_chap": true, + "default_time2wait": 2 + }, + "jsonrpc": "2.0", + "method": "iscsi_set_options", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_get_options method {#rpc_iscsi_get_options} + +Show global parameters of iSCSI targets. + +### Parameters + +This method has no parameters. + +### Example + +Example request: + +~~~ +request: +{ + "jsonrpc": "2.0", + "method": "iscsi_get_options", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "allow_duplicated_isid": true, + "default_time2retain": 60, + "first_burst_length": 8192, + "immediate_data": true, + "node_base": "iqn.2016-06.io.spdk", + "mutual_chap": false, + "nop_in_interval": 30, + "chap_group": 0, + "max_connections_per_session": 2, + "max_queue_depth": 64, + "nop_timeout": 30, + "max_sessions": 128, + "error_recovery_level": 0, + "auth_file": "/usr/local/etc/spdk/auth.conf", + "disable_chap": true, + "default_time2wait": 2, + "require_chap": false + } +} +~~~ +## iscsi_set_discovery_auth method {#rpc_iscsi_set_discovery_auth} + +Set CHAP authentication for sessions dynamically. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +disable_chap | Optional | boolean | CHAP for discovery session should be disabled (default: `false`) +require_chap | Optional | boolean | CHAP for discovery session should be required (default: `false`) +mutual_chap | Optional | boolean | CHAP for discovery session should be unidirectional (`false`) or bidirectional (`true`) (default: `false`) +chap_group | Optional | number | CHAP group ID for discovery session (default: 0) + +Parameters `disable_chap` and `require_chap` are mutually exclusive. + +### Example + +Example request: + +~~~ +request: +{ + "params": { + "chap_group": 1, + "require_chap": true, + "mutual_chap": true + }, + "jsonrpc": "2.0", + "method": "iscsi_set_discovery_auth", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_create_auth_group method {#rpc_iscsi_create_auth_group} + +Create an authentication group for CHAP authentication. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Authentication group tag (unique, integer > 0) +secrets | Optional | array | Array of @ref rpc_iscsi_create_auth_group_secret objects + +### secret {#rpc_iscsi_create_auth_group_secret} + +Name | Optional | Type | Description +--------------------------- | ---------| --------| ----------- +user | Required | string | Unidirectional CHAP name +secret | Required | string | Unidirectional CHAP secret +muser | Optional | string | Bidirectional CHAP name +msecret | Optional | string | Bidirectional CHAP secret + +### Example + +Example request: + +~~~ +{ + "params": { + "secrets": [ + { + "muser": "mu1", + "secret": "s1", + "user": "u1", + "msecret": "ms1" + } + ], + "tag": 2 + }, + "jsonrpc": "2.0", + "method": "iscsi_create_auth_group", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_delete_auth_group method {#rpc_iscsi_delete_auth_group} + +Delete an existing authentication group for CHAP authentication. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Authentication group tag (unique, integer > 0) + +### Example + +Example request: + +~~~ +{ + "params": { + "tag": 2 + }, + "jsonrpc": "2.0", + "method": "iscsi_delete_auth_group", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_get_auth_groups {#rpc_iscsi_get_auth_groups} + +Show information about all existing authentication group for CHAP authentication. + +### Parameters + +This method has no parameters. + +### Result + +Array of objects describing authentication group. + +Name | Type | Description +--------------------------- | --------| ----------- +tag | number | Authentication group tag +secrets | array | Array of @ref rpc_iscsi_create_auth_group_secret objects + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "iscsi_get_auth_groups", + "id": 1 +} +~~~ +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "secrets": [ + { + "muser": "mu1", + "secret": "s1", + "user": "u1", + "msecret": "ms1" + } + ], + "tag": 1 + }, + { + "secrets": [ + { + "secret": "s2", + "user": "u2" + } + ], + "tag": 2 + } + ] +} +~~~ + +## iscsi_auth_group_add_secret {#rpc_iscsi_auth_group_add_secret} + +Add a secret to an existing authentication group for CHAP authentication. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Authentication group tag (unique, integer > 0) +user | Required | string | Unidirectional CHAP name +secret | Required | string | Unidirectional CHAP secret +muser | Optional | string | Bidirectional CHAP name +msecret | Optional | string | Bidirectional CHAP secret + +### Example + +Example request: + +~~~ +{ + "params": { + "muser": "mu3", + "secret": "s3", + "tag": 2, + "user": "u3", + "msecret": "ms3" + }, + "jsonrpc": "2.0", + "method": "iscsi_auth_group_add_secret", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_auth_group_remove_secret {#rpc_iscsi_auth_group_remove_secret} + +Remove a secret from an existing authentication group for CHAP authentication. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Authentication group tag (unique, integer > 0) +user | Required | string | Unidirectional CHAP name + +### Example + +Example request: + +~~~ +{ + "params": { + "tag": 2, + "user": "u3" + }, + "jsonrpc": "2.0", + "method": "iscsi_auth_group_remove_secret", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_get_initiator_groups method {#rpc_iscsi_get_initiator_groups} + +Show information about all available initiator groups. + +### Parameters + +This method has no parameters. + +### Result + +Array of objects describing initiator groups. + +Name | Type | Description +--------------------------- | --------| ----------- +tag | number | Initiator group tag +initiators | array | Array of initiator hostnames or IP addresses +netmasks | array | Array of initiator netmasks + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "iscsi_get_initiator_groups", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "initiators": [ + "iqn.2016-06.io.spdk:host1", + "iqn.2016-06.io.spdk:host2" + ], + "tag": 1, + "netmasks": [ + "192.168.1.0/24" + ] + } + ] +} +~~~ + +## iscsi_create_initiator_group method {#rpc_iscsi_create_initiator_group} + +Add an initiator group. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Initiator group tag (unique, integer > 0) +initiators | Required | array | Not empty array of initiator hostnames or IP addresses +netmasks | Required | array | Not empty array of initiator netmasks + +### Example + +Example request: + +~~~ +{ + "params": { + "initiators": [ + "iqn.2016-06.io.spdk:host1", + "iqn.2016-06.io.spdk:host2" + ], + "tag": 1, + "netmasks": [ + "192.168.1.0/24" + ] + }, + "jsonrpc": "2.0", + "method": "iscsi_create_initiator_group", + "id": 1 +} +~~~ + +Example response: + +~~~ +response: +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_delete_initiator_group method {#rpc_iscsi_delete_initiator_group} + +Delete an existing initiator group. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Initiator group tag (unique, integer > 0) + +### Example + +Example request: + +~~~ +{ + "params": { + "tag": 1 + }, + "jsonrpc": "2.0", + "method": "iscsi_delete_initiator_group", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_initiator_group_add_initiators method {#rpc_iscsi_initiator_group_add_initiators} + +Add initiators to an existing initiator group. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Existing initiator group tag. +initiators | Optional | array | Array of initiator hostnames or IP addresses +netmasks | Optional | array | Array of initiator netmasks + +### Example + +Example request: + +~~~ +request: +{ + "params": { + "initiators": [ + "iqn.2016-06.io.spdk:host3" + ], + "tag": 1, + "netmasks": [ + "255.255.255.1" + ] + }, + "jsonrpc": "2.0", + "method": "iscsi_initiator_group_add_initiators", + "id": 1 +} +~~~ + +Example response: + +~~~ +response: +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_get_target_nodes method {#rpc_iscsi_get_target_nodes} + +Show information about all available iSCSI target nodes. + +### Parameters + +This method has no parameters. + +### Result + +Array of objects describing target node. + +Name | Type | Description +--------------------------- | --------| ----------- +name | string | Target node name (ASCII) +alias_name | string | Target node alias name (ASCII) +pg_ig_maps | array | Array of Portal_Group_Tag:Initiator_Group_Tag mappings +luns | array | Array of Bdev names to LUN ID mappings +queue_depth | number | Target queue depth +disable_chap | boolean | CHAP authentication should be disabled for this target +require_chap | boolean | CHAP authentication should be required for this target +mutual_chap | boolean | CHAP authentication should be bidirectional (`true`) or unidirectional (`false`) +chap_group | number | Authentication group ID for this target node +header_digest | boolean | Header Digest should be required for this target node +data_digest | boolean | Data Digest should be required for this target node + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "iscsi_get_target_nodes", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "luns": [ + { + "lun_id": 0, + "bdev_name": "Nvme0n1" + } + ], + "mutual_chap": false, + "name": "iqn.2016-06.io.spdk:target1", + "alias_name": "iscsi-target1-alias", + "require_chap": false, + "chap_group": 0, + "pg_ig_maps": [ + { + "ig_tag": 1, + "pg_tag": 1 + } + ], + "data_digest": false, + "disable_chap": false, + "header_digest": false, + "queue_depth": 64 + } + ] +} +~~~ + +## iscsi_create_target_node method {#rpc_iscsi_create_target_node} + +Add an iSCSI target node. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +name | Required | string | Target node name (ASCII) +alias_name | Required | string | Target node alias name (ASCII) +pg_ig_maps | Required | array | Array of (Portal_Group_Tag:Initiator_Group_Tag) mappings +luns | Required | array | Array of Bdev names to LUN ID mappings +queue_depth | Required | number | Target queue depth +disable_chap | Optional | boolean | CHAP authentication should be disabled for this target +require_chap | Optional | boolean | CHAP authentication should be required for this target +mutual_chap | Optional | boolean | CHAP authentication should be bidirectional (`true`) or unidirectional (`false`) +chap_group | Optional | number | Authentication group ID for this target node +header_digest | Optional | boolean | Header Digest should be required for this target node +data_digest | Optional | boolean | Data Digest should be required for this target node + +Parameters `disable_chap` and `require_chap` are mutually exclusive. + +### Example + +Example request: + +~~~ +{ + "params": { + "luns": [ + { + "lun_id": 0, + "bdev_name": "Nvme0n1" + } + ], + "mutual_chap": true, + "name": "target2", + "alias_name": "iscsi-target2-alias", + "pg_ig_maps": [ + { + "ig_tag": 1, + "pg_tag": 1 + }, + { + "ig_tag": 2, + "pg_tag": 2 + } + ], + "data_digest": true, + "disable_chap": true, + "header_digest": true, + "queue_depth": 24 + }, + "jsonrpc": "2.0", + "method": "iscsi_create_target_node", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_target_node_set_auth method {#rpc_iscsi_target_node_set_auth} + +Set CHAP authentication to an existing iSCSI target node. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +name | Required | string | Target node name (ASCII) +disable_chap | Optional | boolean | CHAP authentication should be disabled for this target +require_chap | Optional | boolean | CHAP authentication should be required for this target +mutual_chap | Optional | boolean | CHAP authentication should be bidirectional (`true`) or unidirectional (`false`) +chap_group | Optional | number | Authentication group ID for this target node + +Parameters `disable_chap` and `require_chap` are mutually exclusive. + +### Example + +Example request: + +~~~ +{ + "params": { + "chap_group": 1, + "require_chap": true, + "name": "iqn.2016-06.io.spdk:target1", + "mutual_chap": true + }, + "jsonrpc": "2.0", + "method": "iscsi_target_node_set_auth", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_target_node_add_pg_ig_maps method {#rpc_iscsi_target_node_add_pg_ig_maps} + +Add initiator group to portal group mappings to an existing iSCSI target node. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +name | Required | string | Target node name (ASCII) +pg_ig_maps | Required | array | Not empty array of initiator to portal group mappings objects + +Portal to Initiator group mappings object: + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +ig_tag | Required | number | Existing initiator group tag +pg_tag | Required | number | Existing portal group tag + +### Example + +Example request: + +~~~ +{ + "params": { + "pg_ig_maps": [ + { + "ig_tag": 1, + "pg_tag": 1 + }, + { + "ig_tag": 2, + "pg_tag": 2 + }, + { + "ig_tag": 3, + "pg_tag": 3 + } + ], + "name": "iqn.2016-06.io.spdk:target3" + }, + "jsonrpc": "2.0", + "method": "iscsi_target_node_add_pg_ig_maps", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_target_node_remove_pg_ig_maps method {#rpc_iscsi_target_node_remove_pg_ig_maps} + +Delete initiator group to portal group mappings from an existing iSCSI target node. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +name | Required | string | Target node name (ASCII) +pg_ig_maps | Required | array | Not empty array of Portal to Initiator group mappings objects + +Portal to Initiator group mappings object: + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +ig_tag | Required | number | Existing initiator group tag +pg_tag | Required | number | Existing portal group tag + +### Example + +Example request: + +~~~ +{ + "params": { + "pg_ig_maps": [ + { + "ig_tag": 1, + "pg_tag": 1 + }, + { + "ig_tag": 2, + "pg_tag": 2 + }, + { + "ig_tag": 3, + "pg_tag": 3 + } + ], + "name": "iqn.2016-06.io.spdk:target3" + }, + "jsonrpc": "2.0", + "method": "iscsi_target_node_remove_pg_ig_maps", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_delete_target_node method {#rpc_iscsi_delete_target_node} + +Delete an iSCSI target node. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +name | Required | string | Target node name (ASCII) + +### Example + +Example request: + +~~~ +{ + "params": { + "name": "iqn.2016-06.io.spdk:target1" + }, + "jsonrpc": "2.0", + "method": "iscsi_delete_target_node", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_get_portal_groups method {#rpc_iscsi_get_portal_groups} + +Show information about all available portal groups. + +### Parameters + +This method has no parameters. + +### Example + +Example request: + +~~~ +request: +{ + "jsonrpc": "2.0", + "method": "iscsi_get_portal_groups", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "portals": [ + { + "host": "127.0.0.1", + "port": "3260" + } + ], + "tag": 1 + } + ] +} +~~~ + +## iscsi_create_portal_group method {#rpc_iscsi_create_portal_group} + +Add a portal group. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Portal group tag +portals | Required | array | Not empty array of portals + +Portal object + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +host | Required | string | Hostname or IP address +port | Required | string | Port number + +### Example + +Example request: + +~~~ +{ + "params": { + "portals": [ + { + "host": "127.0.0.1", + "port": "3260" + } + ], + "tag": 1 + }, + "jsonrpc": "2.0", + "method": "iscsi_create_portal_group", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_delete_portal_group method {#rpc_iscsi_delete_portal_group} + +Delete an existing portal group. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +tag | Required | number | Existing portal group tag + +### Example + +Example request: + +~~~ +{ + "params": { + "tag": 1 + }, + "jsonrpc": "2.0", + "method": "iscsi_delete_portal_group", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +### iscsi_portal_group_set_auth method {#rpc_iscsi_portal_group_set_auth} + +Set CHAP authentication for discovery sessions specific for the existing iSCSI portal group. +This RPC overwrites the setting by the global parameters for the iSCSI portal group. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +disable_chap | Optional | boolean | CHAP for discovery session should be disabled (default: `false`) +require_chap | Optional | boolean | CHAP for discovery session should be required (default: `false`) +mutual_chap | Optional | boolean | CHAP for discovery session should be unidirectional (`false`) or bidirectional (`true`) (default: `false`) +chap_group | Optional | number | CHAP group ID for discovery session (default: 0) + +Parameters `disable_chap` and `require_chap` are mutually exclusive. + +### Example + +Example request: + +~~~ +request: +{ + "params": { + "tag": 1, + "chap_group": 1, + "require_chap": true, + "mutual_chap": true + }, + "jsonrpc": "2.0", + "method": "iscsi_portal_group_set_auth", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## iscsi_get_connections method {#rpc_iscsi_get_connections} + +Show information about all active connections. + +### Parameters + +This method has no parameters. + +### Results + +Array of objects describing iSCSI connection. + +Name | Type | Description +--------------------------- | --------| ----------- +id | number | Index (used for TTT - Target Transfer Tag) +cid | number | CID (Connection ID) +tsih | number | TSIH (Target Session Identifying Handle) +lcore_id | number | Core number on which the iSCSI connection runs +initiator_addr | string | Initiator address +target_addr | string | Target address +target_node_name | string | Target node name (ASCII) without prefix + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "iscsi_get_connections", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "tsih": 4, + "cid": 0, + "target_node_name": "target1", + "lcore_id": 0, + "initiator_addr": "10.0.0.2", + "target_addr": "10.0.0.1", + "id": 0 + } + ] +} +~~~ + +## iscsi_target_node_add_lun method {#rpc_iscsi_target_node_add_lun} + +Add an LUN to an existing iSCSI target node. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +name | Required | string | Target node name (ASCII) +bdev_name | Required | string | bdev name to be added as a LUN +lun_id | Optional | number | LUN ID (default: first free ID) + +### Example + +Example request: + +~~~ +{ + "params": { + "lun_id": 2, + "name": "iqn.2016-06.io.spdk:target1", + "bdev_name": "Malloc0" + }, + "jsonrpc": "2.0", + "method": "iscsi_target_node_add_lun", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +# NVMe-oF Target {#jsonrpc_components_nvmf_tgt} + +## nvmf_create_transport method {#rpc_nvmf_create_transport} + +Initialize an NVMe-oF transport with the given options. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | --------| ----------- +trtype | Required | string | Transport type (ex. RDMA) +tgt_name | Optional | string | Parent NVMe-oF target name. +max_queue_depth | Optional | number | Max number of outstanding I/O per queue +max_qpairs_per_ctrlr | Optional | number | Max number of SQ and CQ per controller (deprecated, use max_io_qpairs_per_ctrlr) +max_io_qpairs_per_ctrlr | Optional | number | Max number of IO qpairs per controller +in_capsule_data_size | Optional | number | Max number of in-capsule data size +max_io_size | Optional | number | Max I/O size (bytes) +io_unit_size | Optional | number | I/O unit size (bytes) +max_aq_depth | Optional | number | Max number of admin cmds per AQ +num_shared_buffers | Optional | number | The number of pooled data buffers available to the transport +buf_cache_size | Optional | number | The number of shared buffers to reserve for each poll group +max_srq_depth | Optional | number | The number of elements in a per-thread shared receive queue (RDMA only) +no_srq | Optional | boolean | Disable shared receive queue even for devices that support it. (RDMA only) +c2h_success | Optional | boolean | Disable C2H success optimization (TCP only) +dif_insert_or_strip | Optional | boolean | Enable DIF insert for write I/O and DIF strip for read I/O DIF +sock_priority | Optional | number | The socket priority of the connection owned by this transport (TCP only) +acceptor_backlog | Optional | number | The number of pending connections allowed in backlog before failing new connection attempts (RDMA only) +abort_timeout_sec | Optional | number | Abort execution timeout value, in seconds + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "nvmf_create_transport", + "id": 1, + "params": { + "trtype": "RDMA", + "max_queue_depth": 32 + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_get_subsystems method {#rpc_nvmf_get_subsystems} + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | ------------| ----------- +tgt_name | Optional | string | Parent NVMe-oF target name. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_get_subsystems" +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "nqn": "nqn.2014-08.org.nvmexpress.discovery", + "subtype": "Discovery" + "listen_addresses": [], + "hosts": [], + "allow_any_host": true + }, + { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "subtype": "NVMe", + "listen_addresses": [ + { + "trtype": "RDMA", + "adrfam": "IPv4", + "traddr": "192.168.0.123", + "trsvcid": "4420" + } + ], + "hosts": [ + {"nqn": "nqn.2016-06.io.spdk:host1"} + ], + "allow_any_host": false, + "serial_number": "abcdef", + "model_number": "ghijklmnop", + "namespaces": [ + {"nsid": 1, "name": "Malloc2"}, + {"nsid": 2, "name": "Nvme0n1"} + ] + } + ] +} +~~~ + +## nvmf_create_subsystem method {#rpc_nvmf_create_subsystem} + +Construct an NVMe over Fabrics target subsystem. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nqn | Required | string | Subsystem NQN +tgt_name | Optional | string | Parent NVMe-oF target name. +serial_number | Optional | string | Serial number of virtual controller +model_number | Optional | string | Model number of virtual controller +max_namespaces | Optional | number | Maximum number of namespaces that can be attached to the subsystem. Default: 0 (Unlimited) +allow_any_host | Optional | boolean | Allow any host (`true`) or enforce allowed host whitelist (`false`). Default: `false`. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_create_subsystem", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "allow_any_host": false, + "serial_number": "abcdef", + "model_number": "ghijklmnop" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_delete_subsystem method {#rpc_nvmf_delete_subsystem} + +Delete an existing NVMe-oF subsystem. + +### Parameters + +Parameter | Optional | Type | Description +---------------------- | -------- | ----------- | ----------- +nqn | Required | string | Subsystem NQN to delete. +tgt_name | Optional | string | Parent NVMe-oF target name. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_delete_subsystem", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_subsystem_add_listener method {#rpc_nvmf_subsystem_add_listener} + +Add a new listen address to an NVMe-oF subsystem. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nqn | Required | string | Subsystem NQN +tgt_name | Optional | string | Parent NVMe-oF target name. +listen_address | Required | object | @ref rpc_nvmf_listen_address object + +### listen_address {#rpc_nvmf_listen_address} + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +trtype | Required | string | Transport type ("RDMA") +adrfam | Required | string | Address family ("IPv4", "IPv6", "IB", or "FC") +traddr | Required | string | Transport address +trsvcid | Required | string | Transport service ID + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_subsystem_add_listener", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "listen_address": { + "trtype": "RDMA", + "adrfam": "IPv4", + "traddr": "192.168.0.123", + "trsvcid": "4420" + } + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_subsystem_add_ns method {#rpc_nvmf_subsystem_add_ns} + +Add a namespace to a subsystem. The namespace ID is returned as the result. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nqn | Required | string | Subsystem NQN +namespace | Required | object | @ref rpc_nvmf_namespace object +tgt_name | Optional | string | Parent NVMe-oF target name. + +### namespace {#rpc_nvmf_namespace} + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nsid | Optional | number | Namespace ID between 1 and 4294967294, inclusive. Default: Automatically assign NSID. +bdev_name | Required | string | Name of bdev to expose as a namespace. +nguid | Optional | string | 16-byte namespace globally unique identifier in hexadecimal (e.g. "ABCDEF0123456789ABCDEF0123456789") +eui64 | Optional | string | 8-byte namespace EUI-64 in hexadecimal (e.g. "ABCDEF0123456789") +uuid | Optional | string | RFC 4122 UUID (e.g. "ceccf520-691e-4b46-9546-34af789907c5") +ptpl_file | Optional | string | File path to save/restore persistent reservation information + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_subsystem_add_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "namespace": { + "nsid": 3, + "bdev_name": "Nvme0n1", + "ptpl_file": "/opt/Nvme0n1PR.json" + } + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": 3 +} +~~~ + +## nvmf_subsystem_remove_ns method {#rpc_nvmf_subsystem_remove_ns} + +Remove a namespace from a subsystem. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nqn | Required | string | Subsystem NQN +nsid | Required | number | Namespace ID +tgt_name | Optional | string | Parent NVMe-oF target name. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_subsystem_remove_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "nsid": 1 + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_subsystem_add_host method {#rpc_nvmf_subsystem_add_host} + +Add a host NQN to the whitelist of allowed hosts. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nqn | Required | string | Subsystem NQN +host | Required | string | Host NQN to add to the list of allowed host NQNs +tgt_name | Optional | string | Parent NVMe-oF target name. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_subsystem_add_host", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "host": "nqn.2016-06.io.spdk:host1" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_subsystem_remove_host method {#rpc_nvmf_subsystem_remove_host} + +Remove a host NQN from the whitelist of allowed hosts. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nqn | Required | string | Subsystem NQN +host | Required | string | Host NQN to remove from the list of allowed host NQNs +tgt_name | Optional | string | Parent NVMe-oF target name. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_subsystem_remove_host", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "host": "nqn.2016-06.io.spdk:host1" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_subsystem_allow_any_host method {#rpc_nvmf_subsystem_allow_any_host} + +Configure a subsystem to allow any host to connect or to enforce the host NQN whitelist. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nqn | Required | string | Subsystem NQN +allow_any_host | Required | boolean | Allow any host (`true`) or enforce allowed host whitelist (`false`). +tgt_name | Optional | string | Parent NVMe-oF target name. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_subsystem_allow_any_host", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "allow_any_host": true + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_set_max_subsystems {#rpc_nvmf_set_max_subsystems} + +Set the maximum allowed subsystems for the NVMe-oF target. This RPC may only be called +before SPDK subsystems have been initialized. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +max_subsystems | Required | number | Maximum number of NVMe-oF subsystems + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_set_max_subsystems", + "params": { + "max_subsystems": 1024 + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_set_config {#rpc_nvmf_set_config} + +Set global configuration of NVMe-oF target. This RPC may only be called before SPDK subsystems +have been initialized. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +acceptor_poll_rate | Optional | number | Polling interval of the acceptor for incoming connections (microseconds) +admin_cmd_passthru | Optional | object | Admin command passthru configuration + +### admin_cmd_passthru {#spdk_nvmf_admin_passthru_conf} + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +identify_ctrlr | Required | bool | If true, enables custom identify handler that reports some identify attributes from the underlying NVMe drive + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_set_config", + "params": { + "acceptor_poll_rate": 10000 + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## nvmf_get_transports method {#rpc_nvmf_get_transports} + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | ------------| ----------- +tgt_name | Optional | string | Parent NVMe-oF target name. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "nvmf_get_transports" +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "type": "RDMA". + "max_queue_depth": 128, + "max_io_qpairs_per_ctrlr": 64, + "in_capsule_data_size": 4096, + "max_io_size": 131072, + "io_unit_size": 131072, + "abort_timeout_sec": 1 + } + ] +} +~~~ + +## nvmf_get_stats method {#rpc_nvmf_get_stats} + +Retrieve current statistics of the NVMf subsystem. + +### Parameters + +Name | Optional | Type | Description +--------------------------- | -------- | ------------| ----------- +tgt_name | Optional | string | Parent NVMe-oF target name. + +### Response + +The response is an object containing NVMf subsystem statistics. + +### Example + +Example request: +~~~ +{ + "jsonrpc": "2.0", + "method": "nvmf_get_stats", + "id": 1 +} +~~~ + +Example response: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tick_rate": 2400000000, + "poll_groups": [ + { + "name": "app_thread", + "admin_qpairs": 1, + "io_qpairs": 4, + "pending_bdev_io": 1721, + "transports": [ + { + "trtype": "RDMA", + "pending_data_buffer": 12131888, + "devices": [ + { + "name": "mlx5_1", + "polls": 72284105, + "completions": 0, + "requests": 0, + "request_latency": 0, + "pending_free_request": 0, + "pending_rdma_read": 0, + "pending_rdma_write": 0 + }, + { + "name": "mlx5_0", + "polls": 72284105, + "completions": 15165875, + "requests": 7582935, + "request_latency": 1249323766184, + "pending_free_request": 0, + "pending_rdma_read": 337602, + "pending_rdma_write": 0 + } + ] + } + ] + } + ] + } +} +~~~ + +# Vhost Target {#jsonrpc_components_vhost_tgt} + +The following common preconditions need to be met in all target types. + +Controller name will be used to create UNIX domain socket. This implies that name concatenated with vhost socket +directory path needs to be valid UNIX socket name. + +@ref cpu_mask parameter is used to choose CPU on which pollers will be launched when new initiator is connecting. +It must be a subset of application CPU mask. Default value is CPU mask of the application. + +## vhost_controller_set_coalescing {#rpc_vhost_controller_set_coalescing} + +Controls interrupt coalescing for specific target. Because `delay_base_us` is used to calculate delay in CPU ticks +there is no hardcoded limit for this parameter. Only limitation is that final delay in CPU ticks might not overflow +32 bit unsigned integer (which is more than 1s @ 4GHz CPU). In real scenarios `delay_base_us` should be much lower +than 150us. To disable coalescing set `delay_base_us` to 0. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +ctrlr | Required | string | Controller name +delay_base_us | Required | number | Base (minimum) coalescing time in microseconds +iops_threshold | Required | number | Coalescing activation level greater than 0 in IO per second + +### Example + +Example request: + +~~~ +{ + "params": { + "iops_threshold": 100000, + "ctrlr": "VhostScsi0", + "delay_base_us": 80 + }, + "jsonrpc": "2.0", + "method": "vhost_controller_set_coalescing", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## vhost_create_scsi_controller {#rpc_vhost_create_scsi_controller} + +Construct vhost SCSI target. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +ctrlr | Required | string | Controller name +cpumask | Optional | string | @ref cpu_mask for this controller + +### Example + +Example request: + +~~~ +{ + "params": { + "cpumask": "0x2", + "ctrlr": "VhostScsi0" + }, + "jsonrpc": "2.0", + "method": "vhost_create_scsi_controller", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## vhost_scsi_controller_add_target {#rpc_vhost_scsi_controller_add_target} + +In vhost target `ctrlr` create SCSI target with ID `scsi_target_num` and add `bdev_name` as LUN 0. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +ctrlr | Required | string | Controller name +scsi_target_num | Required | number | SCSI target ID between 0 and 7 or -1 to use first free ID. +bdev_name | Required | string | Name of bdev to expose as a LUN 0 + +### Response + +SCSI target ID. + +### Example + +Example request: + +~~~ +{ + "params": { + "scsi_target_num": 1, + "bdev_name": "Malloc0", + "ctrlr": "VhostScsi0" + }, + "jsonrpc": "2.0", + "method": "vhost_scsi_controller_add_target", + "id": 1 +} +~~~ + +Example response: + +~~~ +response: +{ + "jsonrpc": "2.0", + "id": 1, + "result": 1 +} +~~~ + +## vhost_scsi_controller_remove_target {#rpc_vhost_scsi_controller_remove_target} + +Remove SCSI target ID `scsi_target_num` from vhost target `scsi_target_num`. + +This method will fail if initiator is connected, but doesn't support hot-remove (the `VIRTIO_SCSI_F_HOTPLUG` is not negotiated). + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +ctrlr | Required | string | Controller name +scsi_target_num | Required | number | SCSI target ID between 0 and 7 + +### Example + +Example request: + +~~~ +request: +{ + "params": { + "scsi_target_num": 1, + "ctrlr": "VhostScsi0" + }, + "jsonrpc": "2.0", + "method": "vhost_scsi_controller_remove_target", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## vhost_create_nvme_controller {#rpc_vhost_create_nvme_controller} + +Construct empty vhost NVMe controller. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +ctrlr | Required | string | Controller name +io_queues | Required | number | Number between 1 and 31 of IO queues for the controller +cpumask | Optional | string | @ref cpu_mask for this controller + +### Example + +Example request: + +~~~ +{ + "params": { + "cpumask": "0x2", + "io_queues": 4, + "ctrlr": "VhostNvme0" + }, + "jsonrpc": "2.0", + "method": "vhost_create_nvme_controller", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## vhost_nvme_controller_add_ns {#rpc_vhost_nvme_controller_add_ns} + +Add namespace backed by `bdev_name` + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +ctrlr | Required | string | Controller name +bdev_name | Required | string | Name of bdev to expose as a namespace +cpumask | Optional | string | @ref cpu_mask for this controller + +### Example + +Example request: + +~~~ +{ + "params": { + "bdev_name": "Malloc0", + "ctrlr": "VhostNvme0" + }, + "jsonrpc": "2.0", + "method": "vhost_nvme_controller_add_ns", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## vhost_create_blk_controller {#rpc_vhost_create_blk_controller} + +Create vhost block controller + +If `readonly` is `true` then vhost block target will be created as read only and fail any write requests. +The `VIRTIO_BLK_F_RO` feature flag will be offered to the initiator. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +ctrlr | Required | string | Controller name +bdev_name | Required | string | Name of bdev to expose block device +readonly | Optional | boolean | If true, this target will be read only (default: false) +cpumask | Optional | string | @ref cpu_mask for this controller + +### Example + +Example request: + +~~~ +{ + "params": { + "dev_name": "Malloc0", + "ctrlr": "VhostBlk0" + }, + "jsonrpc": "2.0", + "method": "vhost_create_blk_controller", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## vhost_get_controllers {#rpc_vhost_get_controllers} + +Display information about all or specific vhost controller(s). + +### Parameters + +The user may specify no parameters in order to list all controllers, or a controller may be +specified by name. + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | Vhost controller name + +### Response {#rpc_vhost_get_controllers_response} + +Response is an array of objects describing requested controller(s). Common fields are: + +Name | Type | Description +----------------------- | ----------- | ----------- +ctrlr | string | Controller name +cpumask | string | @ref cpu_mask of this controller +delay_base_us | number | Base (minimum) coalescing time in microseconds (0 if disabled) +iops_threshold | number | Coalescing activation level +backend_specific | object | Backend specific informations + +### Vhost block {#rpc_vhost_get_controllers_blk} + +`backend_specific` contains one `block` object of type: + +Name | Type | Description +----------------------- | ----------- | ----------- +bdev | string | Backing bdev name or Null if bdev is hot-removed +readonly | boolean | True if controllers is readonly, false otherwise + +### Vhost SCSI {#rpc_vhost_get_controllers_scsi} + +`backend_specific` contains `scsi` array of following objects: + +Name | Type | Description +----------------------- | ----------- | ----------- +target_name | string | Name of this SCSI target +id | number | Unique SPDK global SCSI target ID +scsi_dev_num | number | SCSI target ID initiator will see when scanning this controller +luns | array | array of objects describing @ref rpc_vhost_get_controllers_scsi_luns + +### Vhost SCSI LUN {#rpc_vhost_get_controllers_scsi_luns} + +Object of type: + +Name | Type | Description +----------------------- | ----------- | ----------- +id | number | SCSI LUN ID +bdev_name | string | Backing bdev name + +### Vhost NVMe {#rpc_vhost_get_controllers_nvme} + +`backend_specific` contains `namespaces` array of following objects: + +Name | Type | Description +----------------------- | ----------- | ----------- +nsid | number | Namespace ID +bdev | string | Backing bdev name + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "vhost_get_controllers", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "cpumask": "0x2", + "backend_specific": { + "block": { + "readonly": false, + "bdev": "Malloc0" + } + }, + "iops_threshold": 60000, + "ctrlr": "VhostBlk0", + "delay_base_us": 100 + }, + { + "cpumask": "0x2", + "backend_specific": { + "scsi": [ + { + "target_name": "Target 2", + "luns": [ + { + "id": 0, + "bdev_name": "Malloc1" + } + ], + "id": 0, + "scsi_dev_num": 2 + }, + { + "target_name": "Target 5", + "luns": [ + { + "id": 0, + "bdev_name": "Malloc2" + } + ], + "id": 1, + "scsi_dev_num": 5 + } + ] + }, + "iops_threshold": 60000, + "ctrlr": "VhostScsi0", + "delay_base_us": 0 + }, + { + "cpumask": "0x2", + "backend_specific": { + "namespaces": [ + { + "bdev": "Malloc3", + "nsid": 1 + }, + { + "bdev": "Malloc4", + "nsid": 2 + } + ] + }, + "iops_threshold": 60000, + "ctrlr": "VhostNvme0", + "delay_base_us": 0 + } + ] +} +~~~ + +## vhost_delete_controller {#rpc_vhost_delete_controller} + +Remove vhost target. + +This call will fail if there is an initiator connected or there is at least one SCSI target configured in case of +vhost SCSI target. In the later case please remove all SCSI targets first using @ref rpc_vhost_scsi_controller_remove_target. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +ctrlr | Required | string | Controller name + +### Example + +Example request: + +~~~ +{ + "params": { + "ctrlr": "VhostNvme0" + }, + "jsonrpc": "2.0", + "method": "vhost_delete_controller", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +# Logical Volume {#jsonrpc_components_lvol} + +Identification of logical volume store and logical volume is explained first. + +A logical volume store has a UUID and a name for its identification. +The UUID is generated on creation and it can be used as a unique identifier. +The name is specified on creation and can be renamed. +Either UUID or name is used to access logical volume store in RPCs. + +A logical volume has a UUID and a name for its identification. +The UUID of the logical volume is generated on creation and it can be unique identifier. +The alias of the logical volume takes the format _lvs_name/lvol_name_ where: + +* _lvs_name_ is the name of the logical volume store. +* _lvol_name_ is specified on creation and can be renamed. + +## bdev_lvol_create_lvstore {#rpc_bdev_lvol_create_lvstore} + +Construct a logical volume store. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | Bdev on which to construct logical volume store +lvs_name | Required | string | Name of the logical volume store to create +cluster_sz | Optional | number | Cluster size of the logical volume store in bytes +clear_method | Optional | string | Change clear method for data region. Available: none, unmap (default), write_zeroes + +### Response + +UUID of the created logical volume store is returned. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_lvol_create_lvstore", + "params": { + "lvs_name": "LVS0", + "bdev_name": "Malloc0" + "clear_method": "write_zeroes" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "a9959197-b5e2-4f2d-8095-251ffb6985a5" +} +~~~ + +## bdev_lvol_delete_lvstore {#rpc_bdev_lvol_delete_lvstore} + +Destroy a logical volume store. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +uuid | Optional | string | UUID of the logical volume store to destroy +lvs_name | Optional | string | Name of the logical volume store to destroy + +Either uuid or lvs_name must be specified, but not both. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_delete_lvstore", + "id": 1 + "params": { + "uuid": "a9959197-b5e2-4f2d-8095-251ffb6985a5" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_lvol_get_lvstores {#rpc_bdev_lvol_get_lvstores} + +Get a list of logical volume stores. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +uuid | Optional | string | UUID of the logical volume store to retrieve information about +lvs_name | Optional | string | Name of the logical volume store to retrieve information about + +Either uuid or lvs_name may be specified, but not both. +If both uuid and lvs_name are omitted, information about all logical volume stores is returned. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_get_lvstores", + "id": 1, + "params": { + "lvs_name": "LVS0" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "uuid": "a9959197-b5e2-4f2d-8095-251ffb6985a5", + "base_bdev": "Malloc0", + "free_clusters": 31, + "cluster_size": 4194304, + "total_data_clusters": 31, + "block_size": 4096, + "name": "LVS0" + } + ] +} +~~~ + +## bdev_lvol_rename_lvstore {#rpc_bdev_lvol_rename_lvstore} + +Rename a logical volume store. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +old_name | Required | string | Existing logical volume store name +new_name | Required | string | New logical volume store name + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_rename_lvstore", + "id": 1, + "params": { + "old_name": "LVS0", + "new_name": "LVS2" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_lvol_create {#rpc_bdev_lvol_create} + +Create a logical volume on a logical volume store. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +lvol_name | Required | string | Name of logical volume to create +size | Required | number | Desired size of logical volume in bytes +thin_provision | Optional | boolean | True to enable thin provisioning +uuid | Optional | string | UUID of logical volume store to create logical volume on +lvs_name | Optional | string | Name of logical volume store to create logical volume on +clear_method | Optional | string | Change default data clusters clear method. Available: none, unmap, write_zeroes + +Size will be rounded up to a multiple of cluster size. Either uuid or lvs_name must be specified, but not both. +lvol_name will be used in the alias of the created logical volume. + +### Response + +UUID of the created logical volume is returned. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_create", + "id": 1, + "params": { + "lvol_name": "LVOL0", + "size": 1048576, + "lvs_name": "LVS0", + "clear_method": "unmap", + "thin_provision": true + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "1b38702c-7f0c-411e-a962-92c6a5a8a602" +} +~~~ + +## bdev_lvol_snapshot {#rpc_bdev_lvol_snapshot} + +Capture a snapshot of the current state of a logical volume. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +lvol_name | Required | string | UUID or alias of the logical volume to create a snapshot from +snapshot_name | Required | string | Name for the newly created snapshot + +### Response + +UUID of the created logical volume snapshot is returned. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_snapshot", + "id": 1, + "params": { + "lvol_name": "1b38702c-7f0c-411e-a962-92c6a5a8a602", + "snapshot_name": "SNAP1" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "cc8d7fdf-7865-4d1f-9fc6-35da8e368670" +} +~~~ + +## bdev_lvol_clone {#rpc_bdev_lvol_clone} + +Create a logical volume based on a snapshot. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +snapshot_name | Required | string | UUID or alias of the snapshot to clone +clone_name | Required | string | Name for the logical volume to create + +### Response + +UUID of the created logical volume clone is returned. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0" + "method": "bdev_lvol_clone", + "id": 1, + "params": { + "snapshot_name": "cc8d7fdf-7865-4d1f-9fc6-35da8e368670", + "clone_name": "CLONE1" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "8d87fccc-c278-49f0-9d4c-6237951aca09" +} +~~~ + +## bdev_lvol_rename {#rpc_bdev_lvol_rename} + +Rename a logical volume. New name will rename only the alias of the logical volume. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +old_name | Required | string | UUID or alias of the existing logical volume +new_name | Required | string | New logical volume name + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_rename", + "id": 1, + "params": { + "old_name": "067df606-6dbc-4143-a499-0d05855cb3b8", + "new_name": "LVOL2" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_lvol_resize {#rpc_bdev_lvol_resize} + +Resize a logical volume. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | UUID or alias of the logical volume to resize +size | Required | number | Desired size of the logical volume in bytes + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_resize", + "id": 1, + "params": { + "name": "51638754-ca16-43a7-9f8f-294a0805ab0a", + "size": 2097152 + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_lvol_set_read_only{#rpc_bdev_lvol_set_read_only} + +Mark logical volume as read only. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | UUID or alias of the logical volume to set as read only + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_set_read_only", + "id": 1, + "params": { + "name": "51638754-ca16-43a7-9f8f-294a0805ab0a", + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_lvol_delete {#rpc_bdev_lvol_delete} + +Destroy a logical volume. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | UUID or alias of the logical volume to destroy + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_delete", + "id": 1, + "params": { + "name": "51638754-ca16-43a7-9f8f-294a0805ab0a" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_lvol_inflate {#rpc_bdev_lvol_inflate} + +Inflate a logical volume. All unallocated clusters are allocated and copied from the parent or zero filled if not allocated in the parent. Then all dependencies on the parent are removed. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | UUID or alias of the logical volume to inflate + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_inflate", + "id": 1, + "params": { + "name": "8d87fccc-c278-49f0-9d4c-6237951aca09" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_lvol_decouple_parent {#rpc_bdev_lvol_decouple_parent} + +Decouple the parent of a logical volume. For unallocated clusters which is allocated in the parent, they are allocated and copied from the parent, but for unallocated clusters which is thin provisioned in the parent, they are kept thin provisioned. Then all dependencies on the parent are removed. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | UUID or alias of the logical volume to decouple the parent of it + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_lvol_decouple_parent", + "id": 1. + "params": { + "name": "8d87fccc-c278-49f0-9d4c-6237951aca09" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +# RAID + +## bdev_raid_get_bdevs {#rpc_bdev_raid_get_bdevs} + +This is used to list all the raid bdev names based on the input category requested. Category should be one +of 'all', 'online', 'configuring' or 'offline'. 'all' means all the raid bdevs whether they are online or +configuring or offline. 'online' is the raid bdev which is registered with bdev layer. 'configuring' is +the raid bdev which does not have full configuration discovered yet. 'offline' is the raid bdev which is +not registered with bdev as of now and it has encountered any error or user has requested to offline +the raid bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +category | Required | string | all or online or configuring or offline + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_raid_get_bdevs", + "id": 1, + "params": { + "category": "all" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + "Raid0" + ] +} +~~~ + +## bdev_raid_create {#rpc_bdev_raid_create} + +Constructs new RAID bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | RAID bdev name +strip_size_kb | Required | number | Strip size in KB +raid_level | Required | number | RAID level +base_bdevs | Required | string | Base bdevs name, whitespace separated list in quotes + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_raid_create", + "id": 1, + "params": { + "name": "Raid0", + "raid_level": 0, + "base_bdevs": [ + "Malloc0", + "Malloc1", + "Malloc2", + "Malloc3" + ], + "strip_size": 4096 + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_raid_delete {#rpc_bdev_raid_delete} + +Removes RAID bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | RAID bdev name + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_raid_delete", + "id": 1, + "params": { + "name": "Raid0" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +# OPAL + +## bdev_nvme_opal_init {#rpc_bdev_nvme_opal_init} + +This is used to initialize OPAL of a given NVMe ctrlr, including taking ownership and activating. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nvme_ctrlr_name | Required | string | name of nvme ctrlr +password | Required | string | admin password of OPAL + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_nvme_opal_init", + "id": 1, + "params": { + "nvme_ctrlr_name": "nvme0", + "password": "*****" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_nvme_opal_revert {#rpc_bdev_nvme_opal_revert} + +This is used to revert OPAL to its factory settings. Erase all user configuration and data. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nvme_ctrlr_name | Required | string | name of nvme ctrlr +password | Required | string | admin password of OPAL + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_nvme_opal_revert", + "id": 1, + "params": { + "nvme_ctrlr_name": "nvme0", + "password": "*****" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_opal_create {#rpc_bdev_opal_create} + +This is used to create an OPAL virtual bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nvme_ctrlr_name | Required | string | name of nvme ctrlr that supports OPAL +nsid | Required | number | namespace ID +locking_range_id | Required | number | OPAL locking range ID +range_start | Required | number | locking range start LBA +range_length | Required | number | locking range length +password | Required | string | admin password of OPAL + +### Response + +The response is the name of created OPAL virtual bdev. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_opal_create", + "id": 1, + "params": { + "nvme_ctrlr_name": "nvme0", + "nsid": 1, + "locking_range_id": 1, + "range_start": 0, + "range_length": 4096, + "password": "*****" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "nvme0n1r1" +} +~~~ + +## bdev_opal_get_info {#rpc_bdev_opal_get_info} + +This is used to get information of a given OPAL bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | name of OPAL vbdev +password | Required | string | admin password + +### Response + +The response is the locking info of OPAL virtual bdev. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_opal_get_info", + "id": 1, + "params": { + "bdev_name": "nvme0n1r1", + "password": "*****" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "name": "nvme0n1r1", + "range_start": 0, + "range_length": 4096, + "read_lock_enabled": true, + "write_lock_enabled": true, + "read_locked": false, + "write_locked": false + } +} +~~~ + +## bdev_opal_delete {#rpc_bdev_opal_delete} + +This is used to delete OPAL vbdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | name of OPAL vbdev +password | Required | string | admin password + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_opal_delete", + "id": 1, + "params": { + "bdev_name": "nvme0n1r1", + "password": "*****" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_opal_new_user {#rpc_bdev_opal_new_user} + +This enables a new user to the specified opal bdev so that the user can lock/unlock the bdev. +Recalling this for the same opal bdev, only the newest user will have the privilege. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | name of OPAL vbdev +admin_password | Required | string | admin password +user_id | Required | number | user ID +user_password | Required | string | user password + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_opal_new_user", + "id": 1, + "params": { + "bdev_name": "nvme0n1r1", + "admin_password": "*****", + "user_id": "1", + "user_password": "********" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +## bdev_opal_set_lock_state {#rpc_bdev_opal_set_lock_state} + +This is used to lock/unlock specific opal bdev providing user ID and password. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | name of OPAL vbdev +user_id | Required | number | user ID +password | Required | string | user password +lock_state | Required | string | lock state + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_opal_set_lock_state", + "id": 1, + "params": { + "bdev_name": "nvme0n1r1", + "user_id": "1", + "user_password": "********", + "lock_state": "rwlock" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +# Notifications + +## notify_get_types {#rpc_notify_get_types} + +Return list of all supported notification types. + +### Parameters + +None + +### Response + +The response is an array of strings - supported RPC notification types. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "notify_get_types", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "id": 1, + "result": [ + "bdev_register", + "bdev_unregister" + ], + "jsonrpc": "2.0" +} +~~~ + +## notify_get_notifications {#notify_get_notifications} + +Request notifications. Returns array of notifications that happend since the specified id (or first that is available). + +Notice: Notifications are kept in circular buffer with limited size. Older notifications might be inaccesible due to being overwritten by new ones. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +id | Optional | number | First Event ID to fetch (default: first available). +max | Optional | number | Maximum number of event to return (default: no limit). + +### Response + +Response is an array of event objects. + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +id | Optional | number | Event ID. +type | Optional | number | Type of the event. +ctx | Optional | string | Event context. + +### Example + +Example request: + +~~~ +{ + "id": 1, + "jsonrpc": "2.0", + "method": "notify_get_notifications", + "params": { + "id": 1, + "max": 10 + } +} + +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "ctx": "Malloc0", + "type": "bdev_register", + "id": 1 + }, + { + "ctx": "Malloc2", + "type": "bdev_register", + "id": 2 + } + ] +} +~~~ + +# Linux Network Block Device (NBD) {#jsonrpc_components_nbd} + +SPDK supports exporting bdevs through Linux nbd. These devices then appear as standard Linux kernel block devices and can be accessed using standard utilities like fdisk. + +In order to export a device over nbd, first make sure the Linux kernel nbd driver is loaded by running 'modprobe nbd'. + +## nbd_start_disk {#rpc_nbd_start_disk} + +Start to export one SPDK bdev as NBD disk + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | Bdev name to export +nbd_device | Optional | string | NBD device name to assign + +### Response + +Path of exported NBD disk + +### Example + +Example request: + +~~~ +{ + "params": { + "nbd_device": "/dev/nbd1", + "bdev_name": "Malloc1" + }, + "jsonrpc": "2.0", + "method": "nbd_start_disk", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "/dev/nbd1" +} +~~~ + +## nbd_stop_disk {#rpc_nbd_stop_disk} + +Stop one NBD disk which is based on SPDK bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nbd_device | Required | string | NBD device name to stop + +### Example + +Example request: + +~~~ +{ + "params": { + "nbd_device": "/dev/nbd1", + }, + "jsonrpc": "2.0", + "method": "nbd_stop_disk", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "true" +} +~~~ + +## nbd_get_disks {#rpc_nbd_get_disks} + +Display all or specified NBD device list + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +nbd_device | Optional | string | NBD device name to display + +### Response + +The response is an array of exported NBD devices and their corresponding SPDK bdev. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "nbd_get_disks", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "bdev_name": "Malloc0", + "nbd_device": "/dev/nbd0" + }, + { + "bdev_name": "Malloc1", + "nbd_device": "/dev/nbd1" + } + ] +} +~~~ + +# Blobfs {#jsonrpc_components_blobfs} + +## blobfs_detect {#rpc_blobfs_detect} + +Detect whether a blobfs exists on bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | Block device name to detect blobfs + +### Response + +True if a blobfs exists on the bdev; False otherwise. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "blobfs_detect", + "params": { + "bdev_name": "Malloc0" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "true" +} +~~~ + +## blobfs_create {#rpc_blobfs_create} + +Build blobfs on bdev. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | Block device name to create blobfs +cluster_sz | Optional | number | Size of cluster in bytes. Must be multiple of 4KiB page size, default and minimal value is 1M. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "blobfs_create", + "params": { + "bdev_name": "Malloc0", + "cluster_sz": 1M + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "true" +} +~~~ + +## blobfs_mount {#rpc_blobfs_mount} + +Mount a blobfs on bdev to one host path through FUSE + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +bdev_name | Required | string | Block device name where the blobfs is +mountpoint | Required | string | Mountpoint path in host to mount blobfs + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": ""blobfs_mount"", + "params": { + "bdev_name": "Malloc0", + "mountpoint": "/mnt/" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": "true" +} +~~ + +## blobfs_set_cache_size {#rpc_blobfs_set_cache_size} + +Set cache pool size for blobfs filesystems. This RPC is only permitted when the cache pool is not already initialized. + +The cache pool is initialized when the first blobfs filesystem is initialized or loaded. It is freed when the all initialized or loaded filesystems are unloaded. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +size_in_mb | Required | number | Cache size in megabytes + +### Response + +True if cache size is set successfully; False if failed to set. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "blobfs_set_cache_size", + "params": { + "size_in_mb": 512, + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +# Socket layer {#jsonrpc_components_sock} + +## sock_impl_get_options {#rpc_sock_impl_get_options} + +Get parameters for the socket layer implementation. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +impl_name | Required | string | Name of socket implementation, e.g. posix + +### Response + +Response is an object with current socket layer options for requested implementation. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "sock_impl_get_options", + "id": 1, + "params": { + "impl_name": "posix" + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "recv_buf_size": 2097152, + "send_buf_size": 2097152, + "enable_recv_pipe": true + "enable_zerocopy_send": true + } +} +~~~ + +## sock_impl_set_options {#rpc_sock_impl_set_options} + +Set parameters for the socket layer implementation. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +impl_name | Required | string | Name of socket implementation, e.g. posix +recv_buf_size | Optional | number | Size of socket receive buffer in bytes +send_buf_size | Optional | number | Size of socket send buffer in bytes +enable_recv_pipe | Optional | boolean | Enable or disable receive pipe +enable_zerocopy_send | Optional | boolean | Enable or disable zero copy on send + +### Response + +True if socket layer options were set successfully. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "sock_impl_set_options", + "id": 1, + "params": { + "impl_name": "posix", + "recv_buf_size": 2097152, + "send_buf_size": 2097152, + "enable_recv_pipe": false + "enable_zerocopy_send": true + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +# Miscellaneous RPC commands + +## bdev_nvme_send_cmd {#rpc_bdev_nvme_send_cmd} + +Send NVMe command directly to NVMe controller or namespace. Parameters and responses encoded by base64 urlsafe need further processing. + +Notice: bdev_nvme_send_cmd requires user to guarentee the correctness of NVMe command itself, and also optional parameters. Illegal command contents or mismatching buffer size may result in unpredictable behavior. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Name of the operating NVMe controller +cmd_type | Required | string | Type of nvme cmd. Valid values are: admin, io +data_direction | Required | string | Direction of data transfer. Valid values are: c2h, h2c +cmdbuf | Required | string | NVMe command encoded by base64 urlsafe +data | Optional | string | Data transferring to controller from host, encoded by base64 urlsafe +metadata | Optional | string | Metadata transferring to controller from host, encoded by base64 urlsafe +data_len | Optional | number | Data length required to transfer from controller to host +metadata_len | Optional | number | Metadata length required to transfer from controller to host +timeout_ms | Optional | number | Command execution timeout value, in milliseconds + +### Response + +Name | Type | Description +----------------------- | ----------- | ----------- +cpl | string | NVMe completion queue entry, encoded by base64 urlsafe +data | string | Data transferred from controller to host, encoded by base64 urlsafe +metadata | string | Metadata transferred from controller to host, encoded by base64 urlsafe + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "method": "bdev_nvme_send_cmd", + "id": 1, + "params": { + "name": "Nvme0", + "cmd_type": "admin" + "data_direction": "c2h", + "cmdbuf": "BgAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAsGUs9P5_AAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "data_len": 60, + } +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "cpl": "AAAAAAAAAAARAAAAWrmwABAA==", + "data": "sIjg6AAAAACwiODoAAAAALCI4OgAAAAAAAYAAREAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + +} +~~~ + +## spdk_get_version {#rpc_spdk_get_version} + +Get the version info of the running SPDK application. + +### Parameters + +This method has no parameters. + +### Response + +The response is the version number including major version number, minor version number, patch level number and suffix string. + +### Example + +Example request: +~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "spdk_get_version" +} +~~ + +Example response: +~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "version": "19.04-pre", + "fields" : { + "major": 19, + "minor": 4, + "patch": 0, + "suffix": "-pre" + } + } +} +~~ diff --git a/src/spdk/doc/jsonrpc_proxy.md b/src/spdk/doc/jsonrpc_proxy.md new file mode 100644 index 000000000..8c8247219 --- /dev/null +++ b/src/spdk/doc/jsonrpc_proxy.md @@ -0,0 +1,51 @@ +# JSON-RPC Remote access {#jsonrpc_proxy} + +SPDK provides a sample python script `rpc_http_proxy.py`, that provides http server which listens for JSON objects from users. It uses HTTP POST method to receive JSON objects including methods and parameters described in this chapter. + +## Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +server IP | Required | string | IP address that JSON objects shall be received on +server port | Required | number | Port number that JSON objects shall be received on +user name | Required | string | User name that will be used for authentication +password | Required | string | Password that will be used for authentication +RPC listen address | Optional | string | Path to SPDK JSON RPC socket. Default: /var/tmp/spdk.sock + +## Example usage + +`spdk/scripts/rpc_http_proxy.py 192.168.0.2 8000 user password` + +## Returns + +Error 401 - missing or incorrect user and/or password. + +Error 400 - wrong JSON syntax or incorrect JSON method + +Status 200 with resultant JSON object included on success. + +## Client side + +Below is a sample python script acting as a client side. It sends `bdev_get_bdevs` method with optional `name` parameter and prints JSON object returned from remote_rpc script. + +~~~ +import json +import requests + +if __name__ == '__main__': + payload = {'id':1, 'method': 'bdev_get_bdevs', 'params': {'name': 'Malloc0'}} + url = 'http://192.168.0.2:8000/' + req = requests.post(url, + data=json.dumps(payload), + auth=('user', 'password'), + verify=False, + timeout=30) + print (req.json()) +~~~ + +Output: + +~~~ +python client.py +[{u'num_blocks': 2621440, u'name': u'Malloc0', u'uuid': u'fb57e59c-599d-42f1-8b89-3e46dbe12641', u'claimed': True, u'driver_specific': {}, u'supported_io_types': {u'reset': True, u'nvme_admin': False, u'unmap': True, u'read': True, u'nvme_io': False, u'write': True, u'flush': True, u'write_zeroes': True}, u'qos_ios_per_sec': 0, u'block_size': 4096, u'product_name': u'Malloc disk', u'aliases': []}] +~~~ diff --git a/src/spdk/doc/libraries.md b/src/spdk/doc/libraries.md new file mode 100644 index 000000000..6bdc0106b --- /dev/null +++ b/src/spdk/doc/libraries.md @@ -0,0 +1,193 @@ +# SPDK Libraries {#libraries} + +The SPDK repository is, first and foremost, a collection of high-performance +storage-centric software libraries. With this in mind, much care has been taken +to ensure that these libraries have consistent and robust naming and versioning +conventions. The libraries themselves are also divided across two directories +(`lib` and `module`) inside of the SPDK repository in a deliberate way to prevent +mixing of SPDK event framework dependent code and lower level libraries. This document +is aimed at explaining the structure, naming conventions, versioning scheme, and use cases +of the libraries contained in these two directories. + +# Directory Structure {#structure} + +The SPDK libraries are divided into two directories. The `lib` directory contains the base libraries that +compose SPDK. Some of these base libraries define plug-in systems. Instances of those plug-ins are called +modules and are located in the `module` directory. For example, the `spdk_sock` library is contained in the +`lib` directory while the implementations of socket abstractions, `sock_posix`, `sock_uring`, and `sock_vpp` +are contained in the `module` directory. + +## lib {#lib} + +The libraries in the `lib` directory can be readily divided into four categories: + +- Utility Libraries: These libraries contain basic, commonly used functions that make more complex +libraries easier to implement. For example, `spdk_log` contains macro definitions that provide a +consistent logging paradigm and `spdk_json` is a general purpose JSON parsing library. +- Protocol Libraries: These libraries contain the building blocks for a specific service. For example, +`spdk_nvmf` and `spdk_vhost` each define the storage protocols after which they are named. +- Storage Service Libraries: These libraries provide a specific abstraction that can be mapped to somewhere +between the physical drive and the filesystem level of your typical storage stack. For example `spdk_bdev` +provides a general block device abstraction layer, `spdk_lvol` provides a logical volume abstraction, +`spdk_blobfs` provides a filesystem abstraction, and `spdk_ftl` provides a flash translation layer +abstraction. +- System Libraries: These libraries provide system level services such as a JSON based RPC service +(see `spdk_jsonrpc`) and thread abstractions (see `spdk_thread`). The most notable library in this category +is the `spdk_env_dpdk` library which provides a shim for the underlying Data Plane Development Kit (DPDK) +environment and provides services like memory management. + +The one library in the `lib` directory that doesn't fit into the above classification is the `spdk_event` library. +This library defines a framework used by the applications contained in the `app` and `example` directories. Much +care has been taken to keep the SPDK libraries independent from this framework. The libraries in `lib` are engineered +to allow plugging directly into independent application frameworks such as Seastar or libuv with minimal effort. + +Currently there are two exceptions in the `lib` directory which still rely on `spdk_event`, `spdk_vhost` and `spdk_iscsi`. +There are efforts underway to remove all remaining dependencies these libraries have on the `spdk_event` library. + +Much like the `spdk_event` library, the `spdk_env_dpdk` library has been architected in such a way that it +can be readily replaced by an alternate environment shim. More information on replacing the `spdk_env_dpdk` +module and the underlying `dpdk` environment can be found in the [environment](#env_replacement) section. + +## module {#module} + +The component libraries in the `module` directory represent specific implementations of the base libraries in +the `lib` directory. As with the `lib` directory, much care has been taken to avoid dependencies on the +`spdk_event` framework except for those libraries which directly implement the `spdk_event` module plugin system. + +There are seven sub-directories in the `module` directory which each hold a different class of libraries. These +sub-directories can be divided into two types. + +- plug-in libraries: These libraries are explicitly tied to one of the libraries in the `lib` directory and +are registered with that library at runtime by way of a specific constructor function. The parent library in +the `lib` directory then manages the module directly. These types of libraries each implement a function table +defined by their parent library. The following table shows these directories and their corresponding parent +libraries: + +<center> +| module directory | parent library | dependent on event library | +|------------------|----------------|----------------------------| +| module/accel | spdk_accel | no | +| module/bdev | spdk_bdev | no | +| module/event | spdk_event | yes | +| module/sock | spdk_sock | no | +</center> + +- Free libraries: These libraries are highly dependent upon a library in the `lib` directory but are not +explicitly registered to that library via a constructor. The libraries in the `blob`, `blobfs`, and `env_dpdk` +directories fall into this category. None of the libraries in this category depend explicitly on the +`spdk_event` library. + +# Library Conventions {#conventions} + +The SPDK libraries follow strict conventions for naming functions, logging, versioning, and header files. + +## Headers {#headers} + +All public SPDK header files exist in the `include` directory of the SPDK repository. These headers +are divided into two sub-directories. + +`include/spdk` contains headers intended to be used by consumers of the SPDK libraries. All of the +functions, variables, and types in these functions are intended for public consumption. Multiple headers +in this directory may depend upon the same underlying library and work together to expose different facets +of the library. The `spdk_bdev` library, for example, is exposed in three different headers. `bdev_module.h` +defines the interfaces a bdev module library would need to implement, `bdev.h` contains general block device +functions that would be used by an application consuming block devices exposed by SPDK, and `bdev_zone.h` +exposes zoned bdev specific functions. Many of the other libraries exhibit a similar behavior of splitting +headers between consumers of the library and those wishing to register a module with that library. + +`include/spdk_internal`, as its name suggests contains header files intended to be consumed only by other +libraries inside of the SPDK repository. These headers are typically used for sharing lower level functions +between two libraries that both require similar functions. For example `spdk_internal/nvme_tcp.h` contains +low level tcp functions used by both the `spdk_nvme` and `spdk_nvmf` libraries. These headers are *NOT* +intended for general consumption. + +Other header files contained directly in the `lib` and `module` directories are intended to be consumed *only* +by source files of their corresponding library. Any symbols intended to be used across libraries need to be +included in a header in the `include/spdk_internal` directory. + +## Naming Conventions {#naming} + +All public types and functions in SPDK libraries begin with the prefix `spdk_`. They are also typically +further namespaced using the spdk library name. The rest of the function or type name describes its purpose. + +There are no internal library functions that begin with the `spdk_` prefix. This naming convention is +enforced by the SPDK continuous Integration testing. Functions not intended for use outside of their home +library should be namespaced with the name of the library only. + +## Map Files {#map} + +SPDK libraries can be built as both static and shared object files. To facilitate building libraries as shared +objects, each one has a corresponding map file (e.g. `spdk_nvmf` relies on `spdk_nvmf.map`). SPDK libraries +not exporting any symbols rely on a blank map file located at `mk/spdk_blank.map`. + +# SPDK Shared Objects {#shared_objects} + +## Shared Object Versioning {#versioning} + +SPDK shared objects follow a semantic versioning pattern with a major and minor version. Any changes which +break backwards compatibility (symbol removal or change) will cause a shared object major increment and +backwards compatible changes will cause a minor version increment; i.e. an application that relies on +`libspdk_nvmf.so.3.0` will be compatible with `libspdk_nvmf.so.3.1` but not with `libspdk_nvmf.so.4.0`. + +Shared object versions are incremented only once between each release cycle. This means that at most, the +major version of each SPDK shared library will increment only once between each SPDK release. + +There are currently no guarantees in SPDK of ABI compatibility between two major SPDK releases. + +The point releases of an LTS release will be ABI compatible with the corresponding LTS major release. + +Shared objects are versioned independently of one another. This means that `libspdk_nvme.so.3.0` and +`libspdk_bdev.so.3.0` do not necessarily belong to the same release. This also means that shared objects +with the same suffix are not necessarily compatible with each other. It is important to source all of your +SPDK libraries from the same repository and version to ensure inter-library compatibility. + +## Linking to Shared Objects {#so_linking} + +Shared objects in SPDK are created on a per-library basis. There is a top level `libspdk.so` object +which is a linker script. It simply contains references to all of the other spdk shared objects. + +There are essentially two ways of linking to SPDK libraries. + +1. An application can link to the top level shared object library as follows: +~~~{.sh} + gcc -o my_app ./my_app.c -lspdk -lspdk_env_dpdk -ldpdk +~~~ + +2. An application can link to only a subset of libraries by linking directly to the ones it relies on: +~~~{.sh} + gcc -o my_app ./my_app.c -lpassthru_external -lspdk_event_bdev -lspdk_bdev -lspdk_bdev_malloc + -lspdk_log -lspdk_thread -lspdk_util -lspdk_event -lspdk_env_dpdk -ldpdk +~~~ + +In the second instance, please note that applications need only link to the libraries upon which they +directly depend. All SPDK libraries have their dependencies specified at object compile time. This means +that when linking to `spdk_net`, one does not also have to specify `spdk_log`, `spdk_util`, `spdk_json`, +`spdk_jsonrpc`, and `spdk_rpc`. However, this dependency inclusion does not extend to the application +itself; i.e. if an application directly uses symbols from both `spdk_bdev` and `spdk_log`, both libraries +will need to be supplied to the linker when linking the application even though `spdk_log` is a dependency +of `spdk_bdev`. + +Please also note that when linking to SPDK libraries, both the spdk_env shim library and the env library +itself need to be supplied to the linker. In the examples above, these are `spdk_env_dpdk` and `dpdk` +respectively. This was intentional and allows one to easily swap out both the environment and the +environment shim. + +## Replacing the env abstraction {#env_replacement} + +SPDK depends on an environment abstraction that provides crucial pinned memory management and PCIe +bus management operations. The interface for this environment abstraction is defined in the +`include/env.h` header file. The default implementation of this environment is located in `spdk_env_dpdk`. +This abstraction in turn relies upon the DPDK libraries. This two part implementation was deliberate +and allows for easily swapping out the dpdk version upon which the spdk libraries rely without making +modifications to the spdk source directly. + +Any environment can replace the `spdk_env_dpdk` environment by implementing the `include/env.h` header +file. The environment can either be implemented wholesale in a single library or as a two-part +shim/implementation library system. +~~~{.sh} + # single library + gcc -o my_app ./my_app.c -lspdk -lcustom_env_implementation + + # two libraries + gcc -o my_app ./my_app.c -lspdk -lcustom_env_shim -lcustom_env_implementation +~~~ diff --git a/src/spdk/doc/lvol.md b/src/spdk/doc/lvol.md new file mode 100644 index 000000000..a587cf48c --- /dev/null +++ b/src/spdk/doc/lvol.md @@ -0,0 +1,160 @@ +# Logical Volumes {#logical_volumes} + +The Logical Volumes library is a flexible storage space management system. It provides creating and managing virtual block devices with variable size. The SPDK Logical Volume library is built on top of @ref blob. + +# Terminology {#lvol_terminology} + +## Logical volume store {#lvs} + +* Shorthand: lvolstore, lvs +* Type name: struct spdk_lvol_store + +A logical volume store uses the super blob feature of blobstore to hold uuid (and in future other metadata). Blobstore types are implemented in blobstore itself, and saved on disk. An lvolstore will generate a UUID on creation, so that it can be uniquely identified from other lvolstores. +By default when creating lvol store data region is unmapped. Optional --clear-method parameter can be passed on creation to change that behavior to writing zeroes or performing no operation. + +## Logical volume {#lvol} + +* Shorthand: lvol +* Type name: struct spdk_lvol + +A logical volume is implemented as an SPDK blob created from an lvolstore. An lvol is uniquely identified by its UUID. Lvol additional can have alias name. + +## Logical volume block device {#lvol_bdev} + +* Shorthand: lvol_bdev +* Type name: struct spdk_lvol_bdev + +Representation of an SPDK block device (spdk_bdev) with an lvol implementation. +A logical volume block device translates generic SPDK block device I/O (spdk_bdev_io) operations into the equivalent SPDK blob operations. Combination of lvol name and lvolstore name gives lvol_bdev alias name in a form "lvs_name/lvol_name". block_size of the created bdev is always 4096, due to blobstore page size. Cluster_size is configurable by parameter. +Size of the new bdev will be rounded up to nearest multiple of cluster_size. +By default lvol bdevs claim part of lvol store equal to their set size. When thin provision option is enabled, no space is taken from lvol store until data is written to lvol bdev. +By default when deleting lvol bdev or resizing down, allocated clusters are unmapped. Optional --clear-method parameter can be passed on creation to change that behavior to writing zeroes or performing no operation. + +## Thin provisioning {#lvol_thin_provisioning} + +Thin provisioned lvols rely on dynamic cluster allocation (e.g. when the first write operation on a cluster is performed), only space required to store data is used and unallocated clusters are obtained from underlying device (e.g. zeroes_dev). + +Sample write operations of thin provisioned blob are shown on the diagram below: + + + +Sample read operations and the structure of thin provisioned blob are shown on the diagram below: + + + +## Snapshots and clone {#lvol_snapshots} + +Logical volumes support snapshots and clones functionality. User may at any given time create snapshot of existing logical volume to save a backup of current volume state. +When creating snapshot original volume becomes thin provisioned and saves only incremental differences from its underlying snapshot. This means that every read from unallocated cluster is actually a read from the snapshot and +every write to unallocated cluster triggers new cluster allocation and data copy from corresponding cluster in snapshot to the new cluster in logical volume before the actual write occurs. + +The read operation is performed as shown in the diagram below: + + +The write operation is performed as shown in the diagram below: + + +User may also create clone of existing snapshot that will be thin provisioned and it will behave in the same way as logical volume from which snapshot is created. +There is no limit of clones and snapshots that may be created as long as there is enough space on logical volume store. Snapshots are read only. Clones may be created only from snapshots or read only logical volumes. + +A snapshot can be removed only if there is a single clone on top of it. The relation chain will be updated accordingly. The cluster map of clone and snapshot will be merged and entries for unallocated clusters in the clone +will be updated with addresses from the snapshot cluster map. The entire operation modifies metadata only - no data is copied during this process. + +## Inflation {#lvol_inflation} + +Blobs can be inflated to copy data from backing devices (e.g. snapshots) and allocate all remaining clusters. As a result of this operation all dependencies for the blob are removed. + + + +## Decoupling {#lvol_decoupling} + +Blobs can be decoupled from their parent blob by copying data from backing devices (e.g. snapshots) for all allocated clusters. Remaining unallocated clusters are kept thin provisioned. +Note: When decouple is performed, only single dependency is removed. To remove all dependencies in a chain of blobs depending on each other, multiple calls need to be issued. + +# Configuring Logical Volumes + +There is no static configuration available for logical volumes. All configuration is done trough RPC. Information about logical volumes is kept on block devices. + +# RPC overview {#lvol_rpc} + +RPC regarding lvolstore: + +``` +bdev_lvol_create_lvstore [-h] [-c CLUSTER_SZ] bdev_name lvs_name + Constructs lvolstore on specified bdev with specified name. During + construction bdev is unmapped at initialization and all data is + erased. Then original bdev is claimed by + SPDK, but no additional spdk bdevs are created. + Returns uuid of created lvolstore. + Optional parameters: + -h show help + -c CLUSTER_SZ Specifies the size of cluster. By default its 4MiB. + --clear-method specify data region clear method "none", "unmap" (default), "write_zeroes" +bdev_lvol_delete_lvstore [-h] [-u UUID] [-l LVS_NAME] + Destroy lvolstore on specified bdev. Removes lvolstore along with lvols on + it. User can identify lvol store by UUID or its name. Note that destroying + lvolstore requires using this call, while deleting single lvol requires + using bdev_lvol_delete rpc call. + optional arguments: + -h, --help show help +bdev_lvol_get_lvstores [-h] [-u UUID] [-l LVS_NAME] + Display current logical volume store list + optional arguments: + -h, --help show help + -u UUID, --uuid UUID show details of specified lvol store + -l LVS_NAME, --lvs_name LVS_NAME show details of specified lvol store +bdev_lvol_rename_lvstore [-h] old_name new_name + Change logical volume store name + optional arguments: + -h, --help show this help message and exit +``` + +RPC regarding lvol and spdk bdev: + +``` +bdev_lvol_create [-h] [-u UUID] [-l LVS_NAME] [-t] [-c CLEAR_METHOD] lvol_name size + Creates lvol with specified size and name on lvolstore specified by its uuid + or name. Then constructs spdk bdev on top of that lvol and presents it as spdk bdev. + User may use -t switch to create thin provisioned lvol. + Returns the name of new spdk bdev + optional arguments: + -h, --help show help + -c, --clear-method specify data clusters clear method "none", "unmap" (default), "write_zeroes" +bdev_get_bdevs [-h] [-b NAME] + User can view created bdevs using this call including those created on top of lvols. + optional arguments: + -h, --help show help + -b NAME, --name NAME Name of the block device. Example: Nvme0n1 +bdev_lvol_delete [-h] bdev_name + Deletes a logical volume previously created by bdev_lvol_create. + optional arguments: + -h, --help show help +bdev_lvol_snapshot [-h] lvol_name snapshot_name + Create a snapshot with snapshot_name of a given lvol bdev. + optional arguments: + -h, --help show help +bdev_lvol_clone [-h] snapshot_name clone_name + Create a clone with clone_name of a given lvol snapshot. + optional arguments: + -h, --help show help +bdev_lvol_rename [-h] old_name new_name + Change lvol bdev name + optional arguments: + -h, --help show help +bdev_lvol_resize [-h] name size + Resize existing lvol bdev + optional arguments: + -h, --help show help +bdev_lvol_set_read_only [-h] name + Mark lvol bdev as read only + optional arguments: + -h, --help show help +bdev_lvol_inflate [-h] name + Inflate lvol bdev + optional arguments: + -h, --help show help +bdev_lvol_decouple_parent [-h] name + Decouple parent of a logical volume + optional arguments: + -h, --help show help +``` diff --git a/src/spdk/doc/memory.md b/src/spdk/doc/memory.md new file mode 100644 index 000000000..f18af4717 --- /dev/null +++ b/src/spdk/doc/memory.md @@ -0,0 +1,115 @@ +# Direct Memory Access (DMA) From User Space {#memory} + +The following is an attempt to explain why all data buffers passed to SPDK must +be allocated using spdk_dma_malloc() or its siblings, and why SPDK relies on +DPDK's proven base functionality to implement memory management. + +Computing platforms generally carve physical memory up into 4KiB segments +called pages. They number the pages from 0 to N starting from the beginning of +addressable memory. Operating systems then overlay 4KiB virtual memory pages on +top of these physical pages using arbitrarily complex mappings. See +[Virtual Memory](https://en.wikipedia.org/wiki/Virtual_memory) for an overview. + +Physical memory is attached on channels, where each memory channel provides +some fixed amount of bandwidth. To optimize total memory bandwidth, the +physical addressing is often set up to automatically interleave between +channels. For instance, page 0 may be located on channel 0, page 1 on channel +1, page 2 on channel 2, etc. This is so that writing to memory sequentially +automatically utilizes all available channels. In practice, interleaving is +done at a much more granular level than a full page. + +Modern computing platforms support hardware acceleration for virtual to +physical translation inside of their Memory Management Unit (MMU). The MMU +often supports multiple different page sizes. On recent x86_64 systems, 4KiB, +2MiB, and 1GiB pages are supported. Typically, operating systems use 4KiB pages +by default. + +NVMe devices transfer data to and from system memory using Direct Memory Access +(DMA). Specifically, they send messages across the PCI bus requesting data +transfers. In the absence of an IOMMU, these messages contain *physical* memory +addresses. These data transfers happen without involving the CPU, and the MMU +is responsible for making access to memory coherent. + +NVMe devices also may place additional requirements on the physical layout of +memory for these transfers. The NVMe 1.0 specification requires all physical +memory to be describable by what is called a *PRP list*. To be described by a +PRP list, memory must have the following properties: + +* The memory is broken into physical 4KiB pages, which we'll call device pages. +* The first device page can be a partial page starting at any 4-byte aligned + address. It may extend up to the end of the current physical page, but not + beyond. +* If there is more than one device page, the first device page must end on a + physical 4KiB page boundary. +* The last device page begins on a physical 4KiB page boundary, but is not + required to end on a physical 4KiB page boundary. + +The specification allows for device pages to be other sizes than 4KiB, but all +known devices as of this writing use 4KiB. + +The NVMe 1.1 specification added support for fully flexible scatter gather lists, +but the feature is optional and most devices available today do not support it. + +User space drivers run in the context of a regular process and so have access +to virtual memory. In order to correctly program the device with physical +addresses, some method for address translation must be implemented. + +The simplest way to do this on Linux is to inspect `/proc/self/pagemap` from +within a process. This file contains the virtual address to physical address +mappings. As of Linux 4.0, accessing these mappings requires root privileges. +However, operating systems make absolutely no guarantee that the mapping of +virtual to physical pages is static. The operating system has no visibility +into whether a PCI device is directly transferring data to a set of physical +addresses, so great care must be taken to coordinate DMA requests with page +movement. When an operating system flags a page such that the virtual to +physical address mapping cannot be modified, this is called **pinning** the +page. + +There are several reasons why the virtual to physical mappings may change, too. +By far the most common reason is due to page swapping to disk. However, the +operating system also moves pages during a process called compaction, which +collapses identical virtual pages onto the same physical page to save memory. +Some operating systems are also capable of doing transparent memory +compression. It is also increasingly possible to hot-add additional memory, +which may trigger a physical address rebalance to optimize interleaving. + +POSIX provides the `mlock` call that forces a virtual page of memory to always +be backed by a physical page. In effect, this is disabling swapping. This does +*not* guarantee, however, that the virtual to physical address mapping is +static. The `mlock` call should not be confused with a **pin** call, and it +turns out that POSIX does not define an API for pinning memory. Therefore, the +mechanism to allocate pinned memory is operating system specific. + +SPDK relies on DPDK to allocate pinned memory. On Linux, DPDK does this by +allocating `hugepages` (by default, 2MiB). The Linux kernel treats hugepages +differently than regular 4KiB pages. Specifically, the operating system will +never change their physical location. This is not by intent, and so things +could change in future versions, but it is true today and has been for a number +of years (see the later section on the IOMMU for a future-proof solution). + +With this explanation, hopefully it is now clear why all data buffers passed to +SPDK must be allocated using spdk_dma_malloc() or its siblings. The buffers +must be allocated specifically so that they are pinned and so that physical +addresses are known. + +# IOMMU Support + +Many platforms contain an extra piece of hardware called an I/O Memory +Management Unit (IOMMU). An IOMMU is much like a regular MMU, except it +provides virtualized address spaces to peripheral devices (i.e. on the PCI +bus). The MMU knows about virtual to physical mappings per process on the +system, so the IOMMU associates a particular device with one of these mappings +and then allows the user to assign arbitrary *bus addresses* to virtual +addresses in their process. All DMA operations between the PCI device and +system memory are then translated through the IOMMU by converting the bus +address to a virtual address and then the virtual address to the physical +address. This allows the operating system to freely modify the virtual to +physical address mapping without breaking ongoing DMA operations. Linux +provides a device driver, `vfio-pci`, that allows a user to configure the IOMMU +with their current process. + +This is a future-proof, hardware-accelerated solution for performing DMA +operations into and out of a user space process and forms the long-term +foundation for SPDK and DPDK's memory management strategy. We highly recommend +that applications are deployed using vfio and the IOMMU enabled, which is fully +supported today. diff --git a/src/spdk/doc/misc.md b/src/spdk/doc/misc.md new file mode 100644 index 000000000..a290afe94 --- /dev/null +++ b/src/spdk/doc/misc.md @@ -0,0 +1,4 @@ +# Miscellaneous {#misc} + +- @subpage peer_2_peer +- @subpage containers diff --git a/src/spdk/doc/notify.md b/src/spdk/doc/notify.md new file mode 100644 index 000000000..7c52244e3 --- /dev/null +++ b/src/spdk/doc/notify.md @@ -0,0 +1,40 @@ +# Notify library {#notify} + +The notify library implements an event bus, allowing users to register, generate, +and listen for events. For example, the bdev library may register a new event type +for bdev creation. Any time a bdev is created, it "sends" the event. Consumers of +that event may periodically poll for new events to retrieve them. +The event bus is implemented as a circular ring of fixed size. If event consumers +do not poll frequently enough, events may be lost. All events are identified by a +monotonically increasing integer, so missing events may be detected, although +not recovered. + +# Register event types {#notify_register} + +During initialization the sender library should register its own event types using +`spdk_notify_type_register(const char *type)`. Parameter 'type' is the name of +notification type. + +# Get info about events {#notify_get_info} + +A consumer can get information about the available event types during runtime using +`spdk_notify_foreach_type`, which iterates over registered notification types and +calls a callback on each of them, so that user can produce detailed information +about notification. + +# Get new events {#notify_listen} + +A consumer can get events by calling function `spdk_notify_foreach_event`. +The caller should specify last received event and the maximum number of invocations. +There might be multiple consumers of each event. The event bus is implemented as a +circular buffer, so older events may be overwritten by newer ones. + +# Send events {#notify_send} + +When an event occurs, a library can invoke `spdk_notify_send` with two strings. +One containing the type of the event, like "spdk_bdev_register", second with context, +for example "Nvme0n1" + +# RPC Calls {#rpc_calls} + +See [JSON-RPC documentation](jsonrpc.md/#rpc_notify_get_types) diff --git a/src/spdk/doc/nvme-cli.md b/src/spdk/doc/nvme-cli.md new file mode 100644 index 000000000..dc477bf16 --- /dev/null +++ b/src/spdk/doc/nvme-cli.md @@ -0,0 +1,96 @@ +# nvme-cli {#nvme-cli} + +# nvme-cli with SPDK Getting Started Guide + +Now nvme-cli can support both kernel driver and SPDK user mode driver for most of its available commands and +Intel specific commands. + +1. Clone the nvme-cli repository from the SPDK GitHub fork. Make sure you check out the spdk-1.6 branch. + + ~~~{.sh} + git clone -b spdk-1.6 https://github.com/spdk/nvme-cli.git + ~~~ + +2. Clone the SPDK repository from https://github.com/spdk/spdk under the nvme-cli folder. + +3. Refer to the "README.md" under SPDK folder to properly build SPDK. + +4. Refer to the "README.md" under nvme-cli folder to properly build nvme-cli. + +5. Execute "<spdk_folder>/scripts/setup.sh" with the "root" account. + +6. Update the "spdk.conf" file under nvme-cli folder to properly configure the SPDK. Notes as following: + + ~~~{.sh} + spdk=1 + Indicates whether or not to use spdk. Can be 0 (off) or 1 (on). + Defaults to 1 which assumes that you have run "<spdk_folder>/scripts/setup.sh", unbinding your drives from the kernel. + + core_mask=0x1 + A bitmask representing which core(s) to use for nvme-cli operations. + Defaults to core 0. + + mem_size=512 + The amount of reserved hugepage memory to use for nvme-cli (in MB). + Defaults to 512MB. + + shm_id=0 + Indicates the shared memory ID for the spdk application with which your NVMe drives are associated, + and should be adjusted accordingly. + Defaults to 0. +~~~ + +7. Run the "./nvme list" command to get the domain:bus:device.function for each found NVMe SSD. + +8. Run the other nvme commands with domain:bus:device.function instead of "/dev/nvmeX" for the specified device. + + ~~~{.sh} + Example: ./nvme smart-log 0000:01:00.0 + ~~~ + +9. Run the "./nvme intel" commands for Intel specific commands against Intel NVMe SSD. + + ~~~{.sh} + Example: ./nvme intel internal-log 0000:08:00.0 + ~~~ + +10. Execute "<spdk_folder>/scripts/setup.sh reset" with the "root" account and update "spdk=0" in spdk.conf to + use the kernel driver if wanted. + +## Use scenarios + +### Run as the only SPDK application on the system + +1. Modify the spdk to 1 in spdk.conf. If the system has fewer cores or less memory, update the spdk.conf accordingly. + +### Run together with other running SPDK applications on shared NVMe SSDs + +1. For the other running SPDK application, start with the parameter like "-i 1" to have the same "shm_id". + +2. Use the default spdk.conf setting where "shm_id=1" to start the nvme-cli. + +3. If other SPDK applications run with different shm_id parameter, update the "spdk.conf" accordingly. + +### Run with other running SPDK applications on non-shared NVMe SSDs + +1. Properly configure the other running SPDK applications. + + ~~~{.sh} + a. Only access the NVMe SSDs it wants. + b. Allocate a fixed number of memory instead of all available memory. + ~~~ + +2. Properly configure the spdk.conf setting for nvme-cli. + + ~~~{.sh} + a. Not access the NVMe SSDs from other SPDK applications. + b. Change the mem_size to a proper size. + ~~~ + +## Note + +1. To run the newly built nvme-cli, either explicitly run as "./nvme" or added it into the $PATH to avoid + invoke other already installed version. + +2. To run the newly built nvme-cli with SPDK support in arbitrary directory, copy "spdk.conf" to that + directory from the nvme cli folder and update the configuration as suggested. diff --git a/src/spdk/doc/nvme.md b/src/spdk/doc/nvme.md new file mode 100644 index 000000000..ca1ca3cd8 --- /dev/null +++ b/src/spdk/doc/nvme.md @@ -0,0 +1,358 @@ +# NVMe Driver {#nvme} + +# In this document {#nvme_toc} + +* @ref nvme_intro +* @ref nvme_examples +* @ref nvme_interface +* @ref nvme_design +* @ref nvme_fabrics_host +* @ref nvme_multi_process +* @ref nvme_hotplug +* @ref nvme_cuse + +# Introduction {#nvme_intro} + +The NVMe driver is a C library that may be linked directly into an application +that provides direct, zero-copy data transfer to and from +[NVMe SSDs](http://nvmexpress.org/). It is entirely passive, meaning that it spawns +no threads and only performs actions in response to function calls from the +application itself. The library controls NVMe devices by directly mapping the +[PCI BAR](https://en.wikipedia.org/wiki/PCI_configuration_space) into the local +process and performing [MMIO](https://en.wikipedia.org/wiki/Memory-mapped_I/O). +I/O is submitted asynchronously via queue pairs and the general flow isn't +entirely dissimilar from Linux's +[libaio](http://man7.org/linux/man-pages/man2/io_submit.2.html). + +More recently, the library has been improved to also connect to remote NVMe +devices via NVMe over Fabrics. Users may now call spdk_nvme_probe() on both +local PCI busses and on remote NVMe over Fabrics discovery services. The API is +otherwise unchanged. + +# Examples {#nvme_examples} + +## Getting Start with Hello World {#nvme_helloworld} + +There are a number of examples provided that demonstrate how to use the NVMe +library. They are all in the [examples/nvme](https://github.com/spdk/spdk/tree/master/examples/nvme) +directory in the repository. The best place to start is +[hello_world](https://github.com/spdk/spdk/blob/master/examples/nvme/hello_world/hello_world.c). + +## Running Benchmarks with Fio Plugin {#nvme_fioplugin} + +SPDK provides a plugin to the very popular [fio](https://github.com/axboe/fio) +tool for running some basic benchmarks. See the fio start up +[guide](https://github.com/spdk/spdk/blob/master/examples/nvme/fio_plugin/) +for more details. + +## Running Benchmarks with Perf Tool {#nvme_perf} + +NVMe perf utility in the [examples/nvme/perf](https://github.com/spdk/spdk/tree/master/examples/nvme/perf) +is one of the examples which also can be used for performance tests. The fio +tool is widely used because it is very flexible. However, that flexibility adds +overhead and reduces the efficiency of SPDK. Therefore, SPDK provides a perf +benchmarking tool which has minimal overhead during benchmarking. We have +measured up to 2.6 times more IOPS/core when using perf vs. fio with the +4K 100% Random Read workload. The perf benchmarking tool provides several +run time options to support the most common workload. The following examples +demonstrate how to use perf. + +Example: Using perf for 4K 100% Random Read workload to a local NVMe SSD for 300 seconds +~~~{.sh} +perf -q 128 -o 4096 -w randread -r 'trtype:PCIe traddr:0000:04:00.0' -t 300 +~~~ + +Example: Using perf for 4K 100% Random Read workload to a remote NVMe SSD exported over the network via NVMe-oF +~~~{.sh} +perf -q 128 -o 4096 -w randread -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420' -t 300 +~~~ + +Example: Using perf for 4K 70/30 Random Read/Write mix workload to all local NVMe SSDs for 300 seconds +~~~{.sh} +perf -q 128 -o 4096 -w randrw -M 70 -t 300 +~~~ + +Example: Using perf for extended LBA format CRC guard test to a local NVMe SSD, +users must write to the SSD before reading the LBA from SSD +~~~{.sh} +perf -q 1 -o 4096 -w write -r 'trtype:PCIe traddr:0000:04:00.0' -t 300 -e 'PRACT=0,PRCKH=GUARD' +perf -q 1 -o 4096 -w read -r 'trtype:PCIe traddr:0000:04:00.0' -t 200 -e 'PRACT=0,PRCKH=GUARD' +~~~ + +# Public Interface {#nvme_interface} + +- spdk/nvme.h + +Key Functions | Description +------------------------------------------- | ----------- +spdk_nvme_probe() | @copybrief spdk_nvme_probe() +spdk_nvme_ctrlr_alloc_io_qpair() | @copybrief spdk_nvme_ctrlr_alloc_io_qpair() +spdk_nvme_ctrlr_get_ns() | @copybrief spdk_nvme_ctrlr_get_ns() +spdk_nvme_ns_cmd_read() | @copybrief spdk_nvme_ns_cmd_read() +spdk_nvme_ns_cmd_readv() | @copybrief spdk_nvme_ns_cmd_readv() +spdk_nvme_ns_cmd_read_with_md() | @copybrief spdk_nvme_ns_cmd_read_with_md() +spdk_nvme_ns_cmd_write() | @copybrief spdk_nvme_ns_cmd_write() +spdk_nvme_ns_cmd_writev() | @copybrief spdk_nvme_ns_cmd_writev() +spdk_nvme_ns_cmd_write_with_md() | @copybrief spdk_nvme_ns_cmd_write_with_md() +spdk_nvme_ns_cmd_write_zeroes() | @copybrief spdk_nvme_ns_cmd_write_zeroes() +spdk_nvme_ns_cmd_dataset_management() | @copybrief spdk_nvme_ns_cmd_dataset_management() +spdk_nvme_ns_cmd_flush() | @copybrief spdk_nvme_ns_cmd_flush() +spdk_nvme_qpair_process_completions() | @copybrief spdk_nvme_qpair_process_completions() +spdk_nvme_ctrlr_cmd_admin_raw() | @copybrief spdk_nvme_ctrlr_cmd_admin_raw() +spdk_nvme_ctrlr_process_admin_completions() | @copybrief spdk_nvme_ctrlr_process_admin_completions() +spdk_nvme_ctrlr_cmd_io_raw() | @copybrief spdk_nvme_ctrlr_cmd_io_raw() +spdk_nvme_ctrlr_cmd_io_raw_with_md() | @copybrief spdk_nvme_ctrlr_cmd_io_raw_with_md() + +# NVMe Driver Design {#nvme_design} + +## NVMe I/O Submission {#nvme_io_submission} + +I/O is submitted to an NVMe namespace using nvme_ns_cmd_xxx functions. The NVMe +driver submits the I/O request as an NVMe submission queue entry on the queue +pair specified in the command. The function returns immediately, prior to the +completion of the command. The application must poll for I/O completion on each +queue pair with outstanding I/O to receive completion callbacks by calling +spdk_nvme_qpair_process_completions(). + +@sa spdk_nvme_ns_cmd_read, spdk_nvme_ns_cmd_write, spdk_nvme_ns_cmd_dataset_management, +spdk_nvme_ns_cmd_flush, spdk_nvme_qpair_process_completions + +### Fused operations {#nvme_fuses} + +To "fuse" two commands, the first command should have the SPDK_NVME_IO_FLAGS_FUSE_FIRST +io flag set, and the next one should have the SPDK_NVME_IO_FLAGS_FUSE_SECOND. + +In addition, the following rules must be met to execute two commands as an atomic unit: + + - The commands shall be inserted next to each other in the same submission queue. + - The LBA range, should be the same for the two commands. + +E.g. To send fused compare and write operation user must call spdk_nvme_ns_cmd_compare +followed with spdk_nvme_ns_cmd_write and make sure no other operations are submitted +in between on the same queue, like in example below: + +~~~ + rc = spdk_nvme_ns_cmd_compare(ns, qpair, cmp_buf, 0, 1, nvme_fused_first_cpl_cb, + NULL, SPDK_NVME_CMD_FUSE_FIRST); + if (rc != 0) { + ... + } + + rc = spdk_nvme_ns_cmd_write(ns, qpair, write_buf, 0, 1, nvme_fused_second_cpl_cb, + NULL, SPDK_NVME_CMD_FUSE_SECOND); + if (rc != 0) { + ... + } +~~~ + +The NVMe specification currently defines compare-and-write as a fused operation. +Support for compare-and-write is reported by the controller flag +SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED. + +### Scaling Performance {#nvme_scaling} + +NVMe queue pairs (struct spdk_nvme_qpair) provide parallel submission paths for +I/O. I/O may be submitted on multiple queue pairs simultaneously from different +threads. Queue pairs contain no locks or atomics, however, so a given queue +pair may only be used by a single thread at a time. This requirement is not +enforced by the NVMe driver (doing so would require a lock), and violating this +requirement results in undefined behavior. + +The number of queue pairs allowed is dictated by the NVMe SSD itself. The +specification allows for thousands, but most devices support between 32 +and 128. The specification makes no guarantees about the performance available from +each queue pair, but in practice the full performance of a device is almost +always achievable using just one queue pair. For example, if a device claims to +be capable of 450,000 I/O per second at queue depth 128, in practice it does +not matter if the driver is using 4 queue pairs each with queue depth 32, or a +single queue pair with queue depth 128. + +Given the above, the easiest threading model for an application using SPDK is +to spawn a fixed number of threads in a pool and dedicate a single NVMe queue +pair to each thread. A further improvement would be to pin each thread to a +separate CPU core, and often the SPDK documentation will use "CPU core" and +"thread" interchangeably because we have this threading model in mind. + +The NVMe driver takes no locks in the I/O path, so it scales linearly in terms +of performance per thread as long as a queue pair and a CPU core are dedicated +to each new thread. In order to take full advantage of this scaling, +applications should consider organizing their internal data structures such +that data is assigned exclusively to a single thread. All operations that +require that data should be done by sending a request to the owning thread. +This results in a message passing architecture, as opposed to a locking +architecture, and will result in superior scaling across CPU cores. + +## NVMe Driver Internal Memory Usage {#nvme_memory_usage} + +The SPDK NVMe driver provides a zero-copy data transfer path, which means that +there are no data buffers for I/O commands. However, some Admin commands have +data copies depending on the API used by the user. + +Each queue pair has a number of trackers used to track commands submitted by the +caller. The number trackers for I/O queues depend on the users' input for queue +size and the value read from controller capabilities register field Maximum Queue +Entries Supported(MQES, 0 based value). Each tracker has a fixed size 4096 Bytes, +so the maximum memory used for each I/O queue is: (MQES + 1) * 4 KiB. + +I/O queue pairs can be allocated in host memory, this is used for most NVMe controllers, +some NVMe controllers which can support Controller Memory Buffer may put I/O queue +pairs at controllers' PCI BAR space, SPDK NVMe driver can put I/O submission queue +into controller memory buffer, it depends on users' input and controller capabilities. +Each submission queue entry (SQE) and completion queue entry (CQE) consumes 64 bytes +and 16 bytes respectively. Therefore, the maximum memory used for each I/O queue +pair is (MQES + 1) * (64 + 16) Bytes. + +# NVMe over Fabrics Host Support {#nvme_fabrics_host} + +The NVMe driver supports connecting to remote NVMe-oF targets and +interacting with them in the same manner as local NVMe SSDs. + +## Specifying Remote NVMe over Fabrics Targets {#nvme_fabrics_trid} + +The method for connecting to a remote NVMe-oF target is very similar +to the normal enumeration process for local PCIe-attached NVMe devices. +To connect to a remote NVMe over Fabrics subsystem, the user may call +spdk_nvme_probe() with the `trid` parameter specifying the address of +the NVMe-oF target. + +The caller may fill out the spdk_nvme_transport_id structure manually +or use the spdk_nvme_transport_id_parse() function to convert a +human-readable string representation into the required structure. + +The spdk_nvme_transport_id may contain the address of a discovery service +or a single NVM subsystem. If a discovery service address is specified, +the NVMe library will call the spdk_nvme_probe() `probe_cb` for each +discovered NVM subsystem, which allows the user to select the desired +subsystems to be attached. Alternatively, if the address specifies a +single NVM subsystem directly, the NVMe library will call `probe_cb` +for just that subsystem; this allows the user to skip the discovery step +and connect directly to a subsystem with a known address. + +## RDMA Limitations + +Please refer to NVMe-oF target's @ref nvmf_rdma_limitations + +# NVMe Multi Process {#nvme_multi_process} + +This capability enables the SPDK NVMe driver to support multiple processes accessing the +same NVMe device. The NVMe driver allocates critical structures from shared memory, so +that each process can map that memory and create its own queue pairs or share the admin +queue. There is a limited number of I/O queue pairs per NVMe controller. + +The primary motivation for this feature is to support management tools that can attach +to long running applications, perform some maintenance work or gather information, and +then detach. + +## Configuration {#nvme_multi_process_configuration} + +DPDK EAL allows different types of processes to be spawned, each with different permissions +on the hugepage memory used by the applications. + +There are two types of processes: + +1. a primary process which initializes the shared memory and has full privileges and +2. a secondary process which can attach to the primary process by mapping its shared memory + regions and perform NVMe operations including creating queue pairs. + +This feature is enabled by default and is controlled by selecting a value for the shared +memory group ID. This ID is a positive integer and two applications with the same shared +memory group ID will share memory. The first application with a given shared memory group +ID will be considered the primary and all others secondary. + +Example: identical shm_id and non-overlapping core masks +~~~{.sh} +./perf options [AIO device(s)]... + [-c core mask for I/O submission/completion] + [-i shared memory group ID] + +./perf -q 1 -o 4096 -w randread -c 0x1 -t 60 -i 1 +./perf -q 8 -o 131072 -w write -c 0x10 -t 60 -i 1 +~~~ + +## Limitations {#nvme_multi_process_limitations} + +1. Two processes sharing memory may not share any cores in their core mask. +2. If a primary process exits while secondary processes are still running, those processes + will continue to run. However, a new primary process cannot be created. +3. Applications are responsible for coordinating access to logical blocks. +4. If a process exits unexpectedly, the allocated memory will be released when the last + process exits. + +@sa spdk_nvme_probe, spdk_nvme_ctrlr_process_admin_completions + +# NVMe Hotplug {#nvme_hotplug} + +At the NVMe driver level, we provide the following support for Hotplug: + +1. Hotplug events detection: + The user of the NVMe library can call spdk_nvme_probe() periodically to detect + hotplug events. The probe_cb, followed by the attach_cb, will be called for each + new device detected. The user may optionally also provide a remove_cb that will be + called if a previously attached NVMe device is no longer present on the system. + All subsequent I/O to the removed device will return an error. + +2. Hot remove NVMe with IO loads: + When a device is hot removed while I/O is occurring, all access to the PCI BAR will + result in a SIGBUS error. The NVMe driver automatically handles this case by installing + a SIGBUS handler and remapping the PCI BAR to a new, placeholder memory location. + This means I/O in flight during a hot remove will complete with an appropriate error + code and will not crash the application. + +@sa spdk_nvme_probe + +# NVMe Character Devices {#nvme_cuse} + +This feature is considered as experimental. + + + +For each controller as well as namespace, character devices are created in the +locations: +~~~{.sh} + /dev/spdk/nvmeX + /dev/spdk/nvmeXnY + ... +~~~ +Where X is unique SPDK NVMe controller index and Y is namespace id. + +Requests from CUSE are handled by pthreads when controller and namespaces are created. +Those pass the I/O or admin commands via a ring to a thread that processes them using +nvme_io_msg_process(). + +Ioctls that request information attained when attaching NVMe controller receive an +immediate response, without passing them through the ring. + +This interface reserves one qpair for sending down the I/O for each controller. + +## Enabling cuse support for NVMe + +Cuse support is disabled by default. To enable support for NVMe devices SPDK +must be compiled with "./configure --with-nvme-cuse". + +## Limitations + +NVMe namespaces are created as character devices and their use may be limited for +tools expecting block devices. + +Sysfs is not updated by SPDK. + +SPDK NVMe CUSE creates nodes in "/dev/spdk/" directory to explicitly differentiate +from other devices. Tools that only search in the "/dev" directory might not work +with SPDK NVMe CUSE. + +SCSI to NVMe Translation Layer is not implemented. Tools that are using this layer to +identify, manage or operate device might not work properly or their use may be limited. + +### Examples of using smartctl + +smartctl tool recognizes device type based on the device path. If none of expected +patterns match, SCSI translation layer is used to identify device. + +To use smartctl '-d nvme' parameter must be used in addition to full path to +the NVMe device. + +~~~{.sh} + smartctl -d nvme -i /dev/spdk/nvme0 + smartctl -d nvme -H /dev/spdk/nvme1 + ... +~~~ diff --git a/src/spdk/doc/nvme_spec.md b/src/spdk/doc/nvme_spec.md new file mode 100644 index 000000000..123e631e4 --- /dev/null +++ b/src/spdk/doc/nvme_spec.md @@ -0,0 +1,123 @@ +# Submitting I/O to an NVMe Device {#nvme_spec} + +## The NVMe Specification + +The NVMe specification describes a hardware interface for interacting with +storage devices. The specification includes network transport definitions for +remote storage as well as a hardware register layout for local PCIe devices. +What follows here is an overview of how an I/O is submitted to a local PCIe +device through SPDK. + +NVMe devices allow host software (in our case, the SPDK NVMe driver) to allocate +queue pairs in host memory. The term "host" is used a lot, so to clarify that's +the system that the NVMe SSD is plugged into. A queue pair consists of two +queues - a submission queue and a completion queue. These queues are more +accurately described as circular rings of fixed size entries. The submission +queue is an array of 64 byte command structures, plus 2 integers (head and tail +indices). The completion queue is similarly an array of 16 byte completion +structures, plus 2 integers (head and tail indices). There are also two 32-bit +registers involved that are called doorbells. + +An I/O is submitted to an NVMe device by constructing a 64 byte command, placing +it into the submission queue at the current location of the submission queue +head index, and then writing the new index of the submission queue head to the +submission queue head doorbell register. It's actually valid to copy a whole set +of commands into open slots in the ring and then write the doorbell just one +time to submit the whole batch. + +There is a very detailed description of the command submission and completion +process in the NVMe specification, which is conveniently available from the main +page over at [NVM Express](https://nvmexpress.org). + +Most importantly, the command itself describes the operation and also, if +necessary, a location in host memory containing a descriptor for host memory +associated with the command. This host memory is the data to be written on a +write command, or the location to place the data on a read command. Data is +transferred to or from this location using a DMA engine on the NVMe device. + +The completion queue works similarly, but the device is instead the one writing +entries into the ring. Each entry contains a "phase" bit that toggles between 0 +and 1 on each loop through the entire ring. When a queue pair is set up to +generate interrupts, the interrupt contains the index of the completion queue +head. However, SPDK doesn't enable interrupts and instead polls on the phase +bit to detect completions. Interrupts are very heavy operations, so polling this +phase bit is often far more efficient. + +## The SPDK NVMe Driver I/O Path + +Now that we know how the ring structures work, let's cover how the SPDK NVMe +driver uses them. The user is going to construct a queue pair at some early time +in the life cycle of the program, so that's not part of the "hot" path. Then, +they'll call functions like spdk_nvme_ns_cmd_read() to perform an I/O operation. +The user supplies a data buffer, the target LBA, and the length, as well as +other information like which NVMe namespace the command is targeted at and which +NVMe queue pair to use. Finally, the user provides a callback function and +context pointer that will be called when a completion for the resulting command +is discovered during a later call to spdk_nvme_qpair_process_completions(). + +The first stage in the driver is allocating a request object to track the operation. The +operations are asynchronous, so it can't simply track the state of the request +on the call stack. Allocating a new request object on the heap would be far too +slow, so SPDK keeps a pre-allocated set of request objects inside of the NVMe +queue pair object - `struct spdk_nvme_qpair`. The number of requests allocated to +the queue pair is larger than the actual queue depth of the NVMe submission +queue because SPDK supports a couple of key convenience features. The first is +software queueing - SPDK will allow the user to submit more requests than the +hardware queue can actually hold and SPDK will automatically queue in software. +The second is splitting. SPDK will split a request for many reasons, some of +which are outlined next. The number of request objects is configurable at queue +pair creation time and if not specified, SPDK will pick a sensible number based +on the hardware queue depth. + +The second stage is building the 64 byte NVMe command itself. The command is +built into memory embedded into the request object - not directly into an NVMe +submission queue slot. Once the command has been constructed, SPDK attempts to +obtain an open slot in the NVMe submission queue. For each element in the +submission queue an object called a tracker is allocated. The trackers are +allocated in an array, so they can be quickly looked up by an index. The tracker +itself contains a pointer to the request currently occupying that slot. When a +particular tracker is obtained, the command's CID value is updated with the +index of the tracker. The NVMe specification provides that CID value in the +completion, so the request can be recovered by looking up the tracker via the +CID value and then following the pointer. + +Once a tracker (slot) is obtained, the data buffer associated with it is +processed to build a PRP list. That's essentially an NVMe scatter gather list, +although it is a bit more restricted. The user provides SPDK with the virtual +address of the buffer, so SPDK has to go do a page table look up to find the +physical address (pa) or I/O virtual addresses (iova) backing that virtual +memory. A virtually contiguous memory region may not be physically contiguous, +so this may result in a PRP list with multiple elements. Sometimes this may +result in a set of physical addresses that can't actually be expressed as a +single PRP list, so SPDK will automatically split the user operation into two +separate requests transparently. For more information on how memory is managed, +see @ref memory. + +The reason the PRP list is not built until a tracker is obtained is because the +PRP list description must be allocated in DMA-able memory and can be quite +large. Since SPDK typically allocates a large number of requests, we didn't want +to allocate enough space to pre-build the worst case scenario PRP list, +especially given that the common case does not require a separate PRP list at +all. + +Each NVMe command has two PRP list elements embedded into it, so a separate PRP +list isn't required if the request is 4KiB (or if it is 8KiB and aligned +perfectly). Profiling shows that this section of the code is not a major +contributor to the overall CPU use. + +With a tracker filled out, SPDK copies the 64 byte command into the actual NVMe +submission queue slot and then rings the submission queue tail doorbell to tell +the device to go process it. SPDK then returns back to the user, without waiting +for a completion. + +The user can periodically call `spdk_nvme_qpair_process_completions()` to tell +SPDK to examine the completion queue. Specifically, it reads the phase bit of +the next expected completion slot and when it flips, looks at the CID value to +find the tracker, which points at the request object. The request object +contains a function pointer that the user provided initially, which is then +called to complete the command. + +The `spdk_nvme_qpair_process_completions()` function will keep advancing to the +next completion slot until it runs out of completions, at which point it will +write the completion queue head doorbell to let the device know that it can use +the completion queue slots for new completions and return. diff --git a/src/spdk/doc/nvmf.md b/src/spdk/doc/nvmf.md new file mode 100644 index 000000000..34a69b66f --- /dev/null +++ b/src/spdk/doc/nvmf.md @@ -0,0 +1,271 @@ +# NVMe over Fabrics Target {#nvmf} + +@sa @ref nvme_fabrics_host +@sa @ref nvmf_tgt_tracepoints + +# NVMe-oF Target Getting Started Guide {#nvmf_getting_started} + +The SPDK NVMe over Fabrics target is a user space application that presents block devices over a fabrics +such as Ethernet, Infiniband or Fibre Channel. SPDK currently supports RDMA and TCP transports. + +The NVMe over Fabrics specification defines subsystems that can be exported over different transports. +SPDK has chosen to call the software that exports these subsystems a "target", which is the term used +for iSCSI. The specification refers to the "client" that connects to the target as a "host". Many +people will also refer to the host as an "initiator", which is the equivalent thing in iSCSI +parlance. SPDK will try to stick to the terms "target" and "host" to match the specification. + +The Linux kernel also implements an NVMe-oF target and host, and SPDK is tested for +interoperability with the Linux kernel implementations. + +If you want to kill the application using signal, make sure use the SIGTERM, then the application +will release all the share memory resource before exit, the SIGKILL will make the share memory +resource have no chance to be released by application, you may need to release the resource manually. + +## RDMA transport support {#nvmf_rdma_transport} + +It requires an RDMA-capable NIC with its corresponding OFED (OpenFabrics Enterprise Distribution) +software package installed to run. Maybe OS distributions provide packages, but OFED is also +available [here](https://downloads.openfabrics.org/OFED/). + +### Prerequisites {#nvmf_prereqs} + +To build nvmf_tgt with the RDMA transport, there are some additional dependencies, +which can be install using pkgdep.sh script. + +~~~{.sh} +sudo scripts/pkgdep.sh --rdma +~~~ + +Then build SPDK with RDMA enabled: + +~~~{.sh} +./configure --with-rdma <other config parameters> +make +~~~ + +Once built, the binary will be in `build/bin`. + +### Prerequisites for InfiniBand/RDMA Verbs {#nvmf_prereqs_verbs} + +Before starting our NVMe-oF target with the RDMA transport we must load the InfiniBand and RDMA modules +that allow userspace processes to use InfiniBand/RDMA verbs directly. + +~~~{.sh} +modprobe ib_cm +modprobe ib_core +# Please note that ib_ucm does not exist in newer versions of the kernel and is not required. +modprobe ib_ucm || true +modprobe ib_umad +modprobe ib_uverbs +modprobe iw_cm +modprobe rdma_cm +modprobe rdma_ucm +~~~ + +### Prerequisites for RDMA NICs {#nvmf_prereqs_rdma_nics} + +Before starting our NVMe-oF target we must detect RDMA NICs and assign them IP addresses. + +### Finding RDMA NICs and associated network interfaces + +~~~{.sh} +ls /sys/class/infiniband/*/device/net +~~~ + +#### Mellanox ConnectX-3 RDMA NICs + +~~~{.sh} +modprobe mlx4_core +modprobe mlx4_ib +modprobe mlx4_en +~~~ + +#### Mellanox ConnectX-4 RDMA NICs + +~~~{.sh} +modprobe mlx5_core +modprobe mlx5_ib +~~~ + +#### Assigning IP addresses to RDMA NICs + +~~~{.sh} +ifconfig eth1 192.168.100.8 netmask 255.255.255.0 up +ifconfig eth2 192.168.100.9 netmask 255.255.255.0 up +~~~ + +### RDMA Limitations {#nvmf_rdma_limitations} + +As RDMA NICs put a limitation on the number of memory regions registered, the SPDK NVMe-oF +target application may eventually start failing to allocate more DMA-able memory. This is +an imperfection of the DPDK dynamic memory management and is most likely to occur with too +many 2MB hugepages reserved at runtime. One type of memory bottleneck is the number of NIC memory +regions, e.g., some NICs report as many as 2048 for the maximum number of memory regions. This +gives us a 4GB memory limit with 2MB hugepages for the total memory regions. It can be overcome by +using 1GB hugepages or by pre-reserving memory at application startup with `--mem-size` or `-s` +option. All pre-reserved memory will be registered as a single region, but won't be returned to the +system until the SPDK application is terminated. + +## TCP transport support {#nvmf_tcp_transport} + +The transport is built into the nvmf_tgt by default, and it does not need any special libraries. + +## Configuring the SPDK NVMe over Fabrics Target {#nvmf_config} + +An NVMe over Fabrics target can be configured using JSON RPCs. +The basic RPCs needed to configure the NVMe-oF subsystem are detailed below. More information about +working with NVMe over Fabrics specific RPCs can be found on the @ref jsonrpc_components_nvmf_tgt RPC page. + +Using .ini style configuration files for configuration of the NVMe-oF target is deprecated and should +be replaced with JSON based RPCs. .ini style configuration files can be converted to json format by way +of the new script `scripts/config_converter.py`. + +## FC transport support {#nvmf_fc_transport} + +To build nvmf_tgt with the FC transport, there is an additional FC LLD (Low Level Driver) code dependency. +Please contact your FC vendor for instructions to obtain FC driver module. + +### Broadcom FC LLD code + +FC LLD driver for Broadcom FC NVMe capable adapters can be obtained from, +https://github.com/ecdufcdrvr/bcmufctdrvr. + +### Fetch FC LLD module and then build SPDK with FC enabled + +After cloning SPDK repo and initialize submodules, FC LLD library is built which then can be linked with +the fc transport. + +~~~{.sh} +git clone https://github.com/spdk/spdk spdk +git clone https://github.com/ecdufcdrvr/bcmufctdrvr fc +cd spdk +git submodule update --init +cd ../fc +make DPDK_DIR=../spdk/dpdk/build SPDK_DIR=../spdk +cd ../spdk +./configure --with-fc=../fc/build +make +~~~ + +### Using RPCs {#nvmf_config_rpc} + +Start the nvmf_tgt application with elevated privileges. Once the target is started, +the nvmf_create_transport rpc can be used to initialize a given transport. Below is an +example where the target is started and configured with two different transports. +The RDMA transport is configured with an I/O unit size of 8192 bytes, 4 max qpairs per controller, +and an in capsule data size of 0 bytes. The TCP transport is configured with an I/O unit size of +16384 bytes, 8 max qpairs per controller, and an in capsule data size of 8192 bytes. + +~~~{.sh} +build/bin/nvmf_tgt +scripts/rpc.py nvmf_create_transport -t RDMA -u 8192 -p 4 -c 0 +scripts/rpc.py nvmf_create_transport -t TCP -u 16384 -p 8 -c 8192 +~~~ + +Below is an example of creating a malloc bdev and assigning it to a subsystem. Adjust the bdevs, +NQN, serial number, and IP address with RDMA transport to your own circumstances. If you replace +"rdma" with "TCP", then the subsystem will add a listener with TCP transport. + +~~~{.sh} +scripts/rpc.py bdev_malloc_create -b Malloc0 512 512 +scripts/rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -d SPDK_Controller1 +scripts/rpc.py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 +scripts/rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t rdma -a 192.168.100.8 -s 4420 +~~~ + +### NQN Formal Definition + +NVMe qualified names or NQNs are defined in section 7.9 of the +[NVMe specification](http://nvmexpress.org/wp-content/uploads/NVM_Express_Revision_1.3.pdf). SPDK has attempted to +formalize that definition using [Extended Backus-Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form). +SPDK modules use this formal definition (provided below) when validating NQNs. + +~~~{.sh} + +Basic Types +year = 4 * digit ; +month = '01' | '02' | '03' | '04' | '05' | '06' | '07' | '08' | '09' | '10' | '11' | '12' ; +digit = '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ; +hex digit = 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ; + +NQN Definition +NVMe Qualified Name = ( NVMe-oF Discovery NQN | NVMe UUID NQN | NVMe Domain NQN ), '\0' ; +NVMe-oF Discovery NQN = "nqn.2014-08.org.nvmexpress.discovery" ; +NVMe UUID NQN = "nqn.2014-08.org.nvmexpress:uuid:", string UUID ; +string UUID = 8 * hex digit, '-', 3 * (4 * hex digit, '-'), 12 * hex digit ; +NVMe Domain NQN = "nqn.", year, '-', month, '.', reverse domain, ':', utf-8 string ; + +~~~ + +Please note that the following types from the definition above are defined elsewhere: + +1. utf-8 string: Defined in [rfc 3629](https://tools.ietf.org/html/rfc3629). +2. reverse domain: Equivalent to domain name as defined in [rfc 1034](https://tools.ietf.org/html/rfc1034). + +While not stated in the formal definition, SPDK enforces the requirement from the spec that the +"maximum name is 223 bytes in length". SPDK does not include the null terminating character when +defining the length of an nqn, and will accept an nqn containing up to 223 valid bytes with an +additional null terminator. To be precise, SPDK follows the same conventions as the c standard +library function [strlen()](http://man7.org/linux/man-pages/man3/strlen.3.html). + +#### NQN Comparisons + +SPDK compares NQNs byte for byte without case matching or unicode normalization. This has specific implications for +uuid based NQNs. The following pair of NQNs, for example, would not match when compared in the SPDK NVMe-oF Target: + +nqn.2014-08.org.nvmexpress:uuid:11111111-aaaa-bbdd-ffee-123456789abc +nqn.2014-08.org.nvmexpress:uuid:11111111-AAAA-BBDD-FFEE-123456789ABC + +In order to ensure the consistency of uuid based NQNs while using SPDK, users should use lowercase when representing +alphabetic hex digits in their NQNs. + +### Assigning CPU Cores to the NVMe over Fabrics Target {#nvmf_config_lcore} + +SPDK uses the [DPDK Environment Abstraction Layer](http://dpdk.org/doc/guides/prog_guide/env_abstraction_layer.html) +to gain access to hardware resources such as huge memory pages and CPU core(s). DPDK EAL provides +functions to assign threads to specific cores. +To ensure the SPDK NVMe-oF target has the best performance, configure the NICs and NVMe devices to +be located on the same NUMA node. + +The `-m` core mask option specifies a bit mask of the CPU cores that +SPDK is allowed to execute work items on. +For example, to allow SPDK to use cores 24, 25, 26 and 27: +~~~{.sh} +build/bin/nvmf_tgt -m 0xF000000 +~~~ + +## Configuring the Linux NVMe over Fabrics Host {#nvmf_host} + +Both the Linux kernel and SPDK implement an NVMe over Fabrics host. +The Linux kernel NVMe-oF RDMA host support is provided by the `nvme-rdma` driver +(to support RDMA transport) and `nvme-tcp` (to support TCP transport). And the +following shows two different commands for loading the driver. + +~~~{.sh} +modprobe nvme-rdma +modprobe nvme-tcp +~~~ + +The nvme-cli tool may be used to interface with the Linux kernel NVMe over Fabrics host. +See below for examples of the discover, connect and disconnect commands. In all three instances, the +transport can be changed to TCP by interchanging 'rdma' for 'tcp'. + +Discovery: +~~~{.sh} +nvme discover -t rdma -a 192.168.100.8 -s 4420 +~~~ + +Connect: +~~~{.sh} +nvme connect -t rdma -n "nqn.2016-06.io.spdk:cnode1" -a 192.168.100.8 -s 4420 +~~~ + +Disconnect: +~~~{.sh} +nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" +~~~ + +## Enabling NVMe-oF target tracepoints for offline analysis and debug {#nvmf_trace} + +SPDK has a tracing framework for capturing low-level event information at runtime. +@ref nvmf_tgt_tracepoints enable analysis of both performance and application crashes. diff --git a/src/spdk/doc/nvmf_tgt_pg.md b/src/spdk/doc/nvmf_tgt_pg.md new file mode 100644 index 000000000..fe1ca4222 --- /dev/null +++ b/src/spdk/doc/nvmf_tgt_pg.md @@ -0,0 +1,204 @@ +# NVMe over Fabrics Target Programming Guide {#nvmf_tgt_pg} + +## Target Audience + +This programming guide is intended for developers authoring applications that +use the SPDK NVMe-oF target library (`lib/nvmf`). It is intended to provide +background context, architectural insight, and design recommendations. This +guide will not cover how to use the SPDK NVMe-oF target application. For a +guide on how to use the existing application as-is, see @ref nvmf. + +## Introduction + +The SPDK NVMe-oF target library is located in `lib/nvmf`. The library +implements all logic required to create an NVMe-oF target application. It is +used in the implementation of the example NVMe-oF target application in +`app/nvmf_tgt`, but is intended to be consumed independently. + +This guide is written assuming that the reader is familiar with both NVMe and +NVMe over Fabrics. The best way to become familiar with those is to read their +[specifications](http://nvmexpress.org/resources/specifications/). + +## Primitives + +The library exposes a number of primitives - basic objects that the user +creates and interacts with. They are: + +`struct spdk_nvmf_tgt`: An NVMe-oF target. This concept, surprisingly, does +not appear in the NVMe-oF specification. SPDK defines this to mean the +collection of subsystems with the associated namespaces, plus the set of +transports and their associated network connections. This will be referred to +throughout this guide as a **target**. + +`struct spdk_nvmf_subsystem`: An NVMe-oF subsystem, as defined by the NVMe-oF +specification. Subsystems contain namespaces and controllers and perform +access control. This will be referred to throughout this guide as a +**subsystem**. + +`struct spdk_nvmf_ns`: An NVMe-oF namespace, as defined by the NVMe-oF +specification. Namespaces are **bdevs**. See @ref bdev for an explanation of +the SPDK bdev layer. This will be referred to throughout this guide as a +**namespace**. + +`struct spdk_nvmf_qpair`: An NVMe-oF queue pair, as defined by the NVMe-oF +specification. These map 1:1 to network connections. This will be referred to +throughout this guide as a **qpair**. + +`struct spdk_nvmf_transport`: An abstraction for a network fabric, as defined +by the NVMe-oF specification. The specification is designed to allow for many +different network fabrics, so the code mirrors that and implements a plugin +system. Currently, only the RDMA transport is available. This will be referred +to throughout this guide as a **transport**. + +`struct spdk_nvmf_poll_group`: An abstraction for a collection of network +connections that can be polled as a unit. This is an SPDK-defined concept that +does not appear in the NVMe-oF specification. Often, network transports have +facilities to check for incoming data on groups of connections more +efficiently than checking each one individually (e.g. epoll), so poll groups +provide a generic abstraction for that. This will be referred to throughout +this guide as a **poll group**. + +`struct spdk_nvmf_listener`: A network address at which the target will accept +new connections. + +`struct spdk_nvmf_host`: An NVMe-oF NQN representing a host (initiator) +system. This is used for access control. + +## The Basics + +A user of the NVMe-oF target library begins by creating a target using +spdk_nvmf_tgt_create(), setting up a set of addresses on which to accept +connections by calling spdk_nvmf_tgt_listen(), then creating a subsystem +using spdk_nvmf_subsystem_create(). + +Subsystems begin in an inactive state and must be activated by calling +spdk_nvmf_subsystem_start(). Subsystems may be modified at run time, but only +when in the paused or inactive state. A running subsystem may be paused by +calling spdk_nvmf_subsystem_pause() and resumed by calling +spdk_nvmf_subsystem_resume(). + +Namespaces may be added to the subsystem by calling +spdk_nvmf_subsystem_add_ns() when the subsystem is inactive or paused. +Namespaces are bdevs. See @ref bdev for more information about the SPDK bdev +layer. A bdev may be obtained by calling spdk_bdev_get_by_name(). + +Once a subsystem exists and the target is listening on an address, new +connections may be accepted by polling spdk_nvmf_tgt_accept(). + +All I/O to a subsystem is driven by a poll group, which polls for incoming +network I/O. Poll groups may be created by calling +spdk_nvmf_poll_group_create(). They automatically request to begin polling +upon creation on the thread from which they were created. Most importantly, *a +poll group may only be accessed from the thread on which it was created.* + +When spdk_nvmf_tgt_accept() detects a new connection, it will construct a new +struct spdk_nvmf_qpair object and call the user provided `new_qpair_fn` +callback for each new qpair. In response to this callback, the user must +assign the qpair to a poll group by calling spdk_nvmf_poll_group_add(). +Remember, a poll group may only be accessed from the thread on which it was created, +so making a call to spdk_nvmf_poll_group_add() may require passing a message +to the appropriate thread. + +## Access Control + +Access control is performed at the subsystem level by adding allowed listen +addresses and hosts to a subsystem (see spdk_nvmf_subsystem_add_listener() and +spdk_nvmf_subsystem_add_host()). By default, a subsystem will not accept +connections from any host or over any established listen address. Listeners +and hosts may only be added to inactive or paused subsystems. + +## Discovery Subsystems + +A discovery subsystem, as defined by the NVMe-oF specification, is +automatically created for each NVMe-oF target constructed. Connections to the +discovery subsystem are handled in the same way as any other subsystem - new +qpairs are created in response to spdk_nvmf_tgt_accept() and they must be +assigned to a poll group. + +## Transports + +The NVMe-oF specification defines multiple network transports (the "Fabrics" +in NVMe over Fabrics) and has an extensible system for adding new fabrics +in the future. The SPDK NVMe-oF target library implements a plugin system for +network transports to mirror the specification. The API a new transport must +implement is located in lib/nvmf/transport.h. As of this writing, only an RDMA +transport has been implemented. + +The SPDK NVMe-oF target is designed to be able to process I/O from multiple +fabrics simultaneously. + +## Choosing a Threading Model + +The SPDK NVMe-oF target library does not strictly dictate threading model, but +poll groups do all of their polling and I/O processing on the thread they are +created on. Given that, it almost always makes sense to create one poll group +per thread used in the application. New qpairs created in response to +spdk_nvmf_tgt_accept() can be handed out round-robin to the poll groups. This +is how the SPDK NVMe-oF target application currently functions. + +More advanced algorithms for distributing qpairs to poll groups are possible. +For instance, a NUMA-aware algorithm would be an improvement over basic +round-robin, where NUMA-aware means assigning qpairs to poll groups running on +CPU cores that are on the same NUMA node as the network adapter and storage +device. Load-aware algorithms also may have benefits. + +## Scaling Across CPU Cores + +Incoming I/O requests are picked up by the poll group polling their assigned +qpair. For regular NVMe commands such as READ and WRITE, the I/O request is +processed on the initial thread from start to the point where it is submitted +to the backing storage device, without interruption. Completions are +discovered by polling the backing storage device and also processed to +completion on the polling thread. **Regular NVMe commands (READ, WRITE, etc.) +do not require any cross-thread coordination, and therefore take no locks.** + +NVMe ADMIN commands, which are used for managing the NVMe device itself, may +modify global state in the subsystem. For instance, an NVMe ADMIN command may +perform namespace management, such as shrinking a namespace. For these +commands, the subsystem will temporarily enter a paused state by sending a +message to each thread in the system. All new incoming I/O on any thread +targeting the subsystem will be queued during this time. Once the subsystem is +fully paused, the state change will occur, and messages will be sent to each +thread to release queued I/O and resume. Management commands are rare, so this +style of coordination is preferable to forcing all commands to take locks in +the I/O path. + +## Zero Copy Support + +For the RDMA transport, data is transferred from the RDMA NIC to host memory +and then host memory to the SSD (or vice versa), without any intermediate +copies. Data is never moved from one location in host memory to another. Other +transports in the future may require data copies. + +## RDMA + +The SPDK NVMe-oF RDMA transport is implemented on top of the libibverbs and +rdmacm libraries, which are packaged and available on most Linux +distributions. It does not use a user-space RDMA driver stack through DPDK. + +In order to scale to large numbers of connections, the SPDK NVMe-oF RDMA +transport allocates a single RDMA completion queue per poll group. All new +qpairs assigned to the poll group are given their own RDMA send and receive +queues, but share this common completion queue. This allows the poll group to +poll a single queue for incoming messages instead of iterating through each +one. + +Each RDMA request is handled by a state machine that walks the request through +a number of states. This keeps the code organized and makes all of the corner +cases much more obvious. + +RDMA SEND, READ, and WRITE operations are ordered with respect to one another, +but RDMA RECVs are not necessarily ordered with SEND acknowledgements. For +instance, it is possible to detect an incoming RDMA RECV message containing a +new NVMe-oF capsule prior to detecting the acknowledgement of a previous SEND +containing an NVMe completion. This is problematic at full queue depth because +there may not yet be a free request structure. To handle this, the RDMA +request structure is broken into two parts - an rdma_recv and an rdma_request. +New RDMA RECVs will always grab a free rdma_recv, but may need to wait in a +queue for a SEND acknowledgement before they can acquire a full rdma_request +object. + +Further, RDMA NICs expose different queue depths for READ/WRITE operations +than they do for SEND/RECV operations. The RDMA transport reports available +queue depth based on SEND/RECV operation limits and will queue in software as +necessary to accommodate (usually lower) limits on READ/WRITE operations. diff --git a/src/spdk/doc/nvmf_tracing.md b/src/spdk/doc/nvmf_tracing.md new file mode 100644 index 000000000..a5a5d86cd --- /dev/null +++ b/src/spdk/doc/nvmf_tracing.md @@ -0,0 +1,205 @@ +# NVMe-oF Target Tracepoints {#nvmf_tgt_tracepoints} + +# Introduction {#tracepoints_intro} + +SPDK has a tracing framework for capturing low-level event information at runtime. +Tracepoints provide a high-performance tracing mechanism that is accessible at runtime. +They are implemented as a circular buffer in shared memory that is accessible from other +processes. The NVMe-oF target is instrumented with tracepoints to enable analysis of +both performance and application crashes. (Note: the SPDK tracing framework should still +be considered experimental. Work to formalize and document the framework is in progress.) + +# Enabling Tracepoints {#enable_tracepoints} + +Tracepoints are placed in groups. They are enabled and disabled as a group. To enable +the instrumentation of all the tracepoints group in an SPDK target application, start the +target with -e parameter set to 0xFFFF: + +~~~ +build/bin/nvmf_tgt -e 0xFFFF +~~~ + +To enable the instrumentation of just the NVMe-oF RDMA tracepoints in an SPDK target +application, start the target with the -e parameter set to 0x10: + +~~~ +build/bin/nvmf_tgt -e 0x10 +~~~ + +When the target starts, a message is logged with the information you need to view +the tracepoints in a human-readable format using the spdk_trace application. The target +will also log information about the shared memory file. + +~~~{.sh} +app.c: 527:spdk_app_setup_trace: *NOTICE*: Tracepoint Group Mask 0xFFFF specified. +app.c: 531:spdk_app_setup_trace: *NOTICE*: Use 'spdk_trace -s nvmf -p 24147' to capture a snapshot of events at runtime. +app.c: 533:spdk_app_setup_trace: *NOTICE*: Or copy /dev/shm/nvmf_trace.pid24147 for offline analysis/debug. +~~~ + +Note that when tracepoints are enabled, the shared memory files are not deleted when the application +exits. This ensures the file can be used for analysis after the application exits. On Linux, the +shared memory files are in /dev/shm, and can be deleted manually to free shm space if needed. A system +reboot will also free all of the /dev/shm files. + +# Capturing a snapshot of events {#capture_tracepoints} + +Send I/Os to the SPDK target application to generate events. The following is +an example usage of perf to send I/Os to the NVMe-oF target over an RDMA network +interface for 10 minutes. + +~~~ +./perf -q 128 -s 4096 -w randread -t 600 -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.2 trsvcid:4420' +~~~ + +The spdk_trace program can be found in the app/trace directory. To analyze the tracepoints on the same +system running the NVMe-oF target, simply execute the command line shown in the log: + +~~~{.sh} +build/bin/spdk_trace -s nvmf -p 24147 +~~~ + +To analyze the tracepoints on a different system, first prepare the tracepoint file for transfer. The +tracepoint file can be large, but usually compresses very well. This step can also be used to prepare +a tracepoint file to attach to a GitHub issue for debugging NVMe-oF application crashes. + +~~~{.sh} +bzip2 -c /dev/shm/nvmf_trace.pid24147 > /tmp/trace.bz2 +~~~ + +After transferring the /tmp/trace.bz2 tracepoint file to a different system: + +~~~{.sh} +bunzip2 /tmp/trace.bz2 +build/bin/spdk_trace -f /tmp/trace +~~~ + +The following is sample trace capture showing the cumulative time that each +I/O spends at each RDMA state. All the trace captures with the same id are for +the same I/O. + +~~~ +28: 6026.658 ( 12656064) RDMA_REQ_NEED_BUFFER id: r3622 time: 0.019 +28: 6026.694 ( 12656140) RDMA_REQ_RDY_TO_EXECUTE id: r3622 time: 0.055 +28: 6026.820 ( 12656406) RDMA_REQ_EXECUTING id: r3622 time: 0.182 +28: 6026.992 ( 12656766) RDMA_REQ_EXECUTED id: r3477 time: 228.510 +28: 6027.010 ( 12656804) RDMA_REQ_TX_PENDING_C_TO_H id: r3477 time: 228.528 +28: 6027.022 ( 12656828) RDMA_REQ_RDY_TO_COMPLETE id: r3477 time: 228.539 +28: 6027.115 ( 12657024) RDMA_REQ_COMPLETING id: r3477 time: 228.633 +28: 6027.471 ( 12657770) RDMA_REQ_COMPLETED id: r3518 time: 171.577 +28: 6028.027 ( 12658940) RDMA_REQ_NEW id: r3623 +28: 6028.057 ( 12659002) RDMA_REQ_NEED_BUFFER id: r3623 time: 0.030 +28: 6028.095 ( 12659082) RDMA_REQ_RDY_TO_EXECUTE id: r3623 time: 0.068 +28: 6028.216 ( 12659336) RDMA_REQ_EXECUTING id: r3623 time: 0.189 +28: 6028.408 ( 12659740) RDMA_REQ_EXECUTED id: r3505 time: 190.509 +28: 6028.441 ( 12659808) RDMA_REQ_TX_PENDING_C_TO_H id: r3505 time: 190.542 +28: 6028.452 ( 12659832) RDMA_REQ_RDY_TO_COMPLETE id: r3505 time: 190.553 +28: 6028.536 ( 12660008) RDMA_REQ_COMPLETING id: r3505 time: 190.637 +28: 6028.854 ( 12660676) RDMA_REQ_COMPLETED id: r3465 time: 247.000 +28: 6029.433 ( 12661892) RDMA_REQ_NEW id: r3624 +28: 6029.452 ( 12661932) RDMA_REQ_NEED_BUFFER id: r3624 time: 0.019 +28: 6029.482 ( 12661996) RDMA_REQ_RDY_TO_EXECUTE id: r3624 time: 0.050 +28: 6029.591 ( 12662224) RDMA_REQ_EXECUTING id: r3624 time: 0.158 +28: 6029.782 ( 12662624) RDMA_REQ_EXECUTED id: r3564 time: 96.937 +28: 6029.798 ( 12662658) RDMA_REQ_TX_PENDING_C_TO_H id: r3564 time: 96.953 +28: 6029.812 ( 12662688) RDMA_REQ_RDY_TO_COMPLETE id: r3564 time: 96.967 +28: 6029.899 ( 12662870) RDMA_REQ_COMPLETING id: r3564 time: 97.054 +28: 6030.262 ( 12663634) RDMA_REQ_COMPLETED id: r3477 time: 231.780 +28: 6030.786 ( 12664734) RDMA_REQ_NEW id: r3625 +28: 6030.804 ( 12664772) RDMA_REQ_NEED_BUFFER id: r3625 time: 0.018 +28: 6030.841 ( 12664848) RDMA_REQ_RDY_TO_EXECUTE id: r3625 time: 0.054 +28: 6030.963 ( 12665104) RDMA_REQ_EXECUTING id: r3625 time: 0.176 +28: 6031.139 ( 12665474) RDMA_REQ_EXECUTED id: r3552 time: 114.906 +28: 6031.196 ( 12665594) RDMA_REQ_TX_PENDING_C_TO_H id: r3552 time: 114.963 +28: 6031.210 ( 12665624) RDMA_REQ_RDY_TO_COMPLETE id: r3552 time: 114.977 +28: 6031.293 ( 12665798) RDMA_REQ_COMPLETING id: r3552 time: 115.060 +28: 6031.633 ( 12666512) RDMA_REQ_COMPLETED id: r3505 time: 193.734 +28: 6032.230 ( 12667766) RDMA_REQ_NEW id: r3626 +28: 6032.248 ( 12667804) RDMA_REQ_NEED_BUFFER id: r3626 time: 0.018 +28: 6032.288 ( 12667888) RDMA_REQ_RDY_TO_EXECUTE id: r3626 time: 0.058 +28: 6032.396 ( 12668114) RDMA_REQ_EXECUTING id: r3626 time: 0.166 +28: 6032.593 ( 12668528) RDMA_REQ_EXECUTED id: r3570 time: 90.443 +28: 6032.611 ( 12668564) RDMA_REQ_TX_PENDING_C_TO_H id: r3570 time: 90.460 +28: 6032.623 ( 12668590) RDMA_REQ_RDY_TO_COMPLETE id: r3570 time: 90.473 +28: 6032.707 ( 12668766) RDMA_REQ_COMPLETING id: r3570 time: 90.557 +28: 6033.056 ( 12669500) RDMA_REQ_COMPLETED id: r3564 time: 100.211 +~~~ + +# Capturing sufficient trace events {#capture_trace_events} + +Since the tracepoint file generated directly by SPDK application is a circular buffer in shared memory, +the trace events captured by it may be insufficient for further analysis. +The spdk_trace_record program can be found in the app/trace_record directory. +spdk_trace_record is used to poll the spdk tracepoint shared memory, record new entries from it, +and store all entries into specified output file at its shutdown on SIGINT or SIGTERM. +After SPDK nvmf target is launched, simply execute the command line shown in the log: + +~~~{.sh} +build/bin/spdk_trace_record -q -s nvmf -p 24147 -f /tmp/spdk_nvmf_record.trace +~~~ + +Also send I/Os to the SPDK target application to generate events by previous perf example for 10 minutes. + +~~~{.sh} +./perf -q 128 -s 4096 -w randread -t 600 -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.2 trsvcid:4420' +~~~ + +After the completion of perf example, shut down spdk_trace_record by signal SIGINT (Ctrl + C). +To analyze the tracepoints output file from spdk_trace_record, simply run spdk_trace program by: + +~~~{.sh} +build/bin/spdk_trace -f /tmp/spdk_nvmf_record.trace +~~~ + +# Adding New Tracepoints {#add_tracepoints} + +SPDK applications and libraries provide several trace points. You can add new +tracepoints to the existing trace groups. For example, to add a new tracepoints +to the SPDK RDMA library (lib/nvmf/rdma.c) trace group TRACE_GROUP_NVMF_RDMA, +define the tracepoints and assigning them a unique ID using the SPDK_TPOINT_ID macro: + +~~~ +#define TRACE_GROUP_NVMF_RDMA 0x4 +#define TRACE_RDMA_REQUEST_STATE_NEW SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x0) +... +#define NEW_TRACE_POINT_NAME SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, UNIQUE_ID) +~~~ + +You also need to register the new trace points in the SPDK_TRACE_REGISTER_FN macro call +within the application/library using the spdk_trace_register_description function +as shown below: + +~~~ +SPDK_TRACE_REGISTER_FN(nvmf_trace) +{ + spdk_trace_register_object(OBJECT_NVMF_RDMA_IO, 'r'); + spdk_trace_register_description("RDMA_REQ_NEW", "", + TRACE_RDMA_REQUEST_STATE_NEW, + OWNER_NONE, OBJECT_NVMF_RDMA_IO, 1, 1, "cmid: "); + ... + spdk_trace_register_description("NEW_RDMA_REQ_NAME", "", + NEW_TRACE_POINT_NAME, + OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid: "); +} +~~~ + +Finally, use the spdk_trace_record function at the appropriate point in the +application/library to record the current trace state for the new trace points. +The following example shows the usage of the spdk_trace_record function to +record the current trace state of several tracepoints. + +~~~ + case RDMA_REQUEST_STATE_NEW: + spdk_trace_record(TRACE_RDMA_REQUEST_STATE_NEW, 0, 0, (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); + ... + break; + case RDMA_REQUEST_STATE_NEED_BUFFER: + spdk_trace_record(TRACE_RDMA_REQUEST_STATE_NEED_BUFFER, 0, 0, (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); + ... + break; + case RDMA_REQUEST_STATE_TRANSFER_PENDING_HOST_TO_CONTROLLER: + spdk_trace_record(TRACE_RDMA_REQUEST_STATE_TRANSFER_PENDING_HOST_TO_CONTROLLER, 0, 0, + (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id); + ... +~~~ + +All the tracing functions are documented in the [Tracepoint library documentation](https://www.spdk.io/doc/trace_8h.html) diff --git a/src/spdk/doc/overview.md b/src/spdk/doc/overview.md new file mode 100644 index 000000000..7407a2026 --- /dev/null +++ b/src/spdk/doc/overview.md @@ -0,0 +1,102 @@ +# SPDK Structural Overview {#overview} + +# Overview {#dir_overview} + +SPDK is composed of a set of C libraries residing in `lib` with public interface +header files in `include/spdk`, plus a set of applications built out of those +libraries in `app`. Users can use the C libraries in their software or deploy +the full SPDK applications. + +SPDK is designed around message passing instead of locking, and most of the SPDK +libraries make several assumptions about the underlying threading model of the +application they are embedded into. However, SPDK goes to great lengths to remain +agnostic to the specific message passing, event, co-routine, or light-weight +threading framework actually in use. To accomplish this, all SPDK libraries +interact with an abstraction library in `lib/thread` (public interface at +`include/spdk/thread.h`). Any framework can initialize the threading abstraction +and provide callbacks to implement the functionality that the SPDK libraries +need. For more information on this abstraction, see @ref concurrency. + +SPDK is built on top of POSIX for most operations. To make porting to non-POSIX +environments easier, all POSIX headers are isolated into +`include/spdk/stdinc.h`. However, SPDK requires a number of operations that +POSIX does not provide, such as enumerating the PCI devices on the system or +allocating memory that is safe for DMA. These additional operations are all +abstracted in a library called `env` whose public header is at +`include/spdk/env.h`. By default, SPDK implements the `env` interface using a +library based on DPDK. However, that implementation can be swapped out. See @ref +porting for additional information. + +## Applications {#dir_app} + +The `app` top-level directory contains full-fledged applications, built out of the SPDK +components. For a full overview, see @ref app_overview. + +SPDK applications can typically be started with a small number of configuration +options. Full configuration of the applications is then performed using +JSON-RPC. See @ref jsonrpc for additional information. + +## Libraries {#dir_lib} + +The `lib` directory contains the real heart of SPDK. Each component is a C library with +its own directory under `lib`. Some of the key libraries are: + +- @ref bdev +- @ref nvme + +## Documentation {#dir_doc} + +The `doc` top-level directory contains all of SPDK's documentation. API Documentation +is created using Doxygen directly from the code, but more general articles and longer +explanations reside in this directory, as well as the Doxygen config file. + +To build the documentation, just type `make` within the doc directory. + +## Examples {#dir_examples} + +The `examples` top-level directory contains a set of examples intended to be used +for reference. These are different than the applications, which are doing a "real" +task that could reasonably be deployed. The examples are instead either heavily +contrived to demonstrate some facet of SPDK, or aren't considered complete enough +to warrant tagging them as a full blown SPDK application. + +This is a great place to learn about how SPDK works. In particular, check out +`examples/nvme/hello_world`. + +## Include {#dir_include} + +The `include` directory is where all of the header files are located. The public API +is all placed in the `spdk` subdirectory of `include` and we highly +recommend that applications set their include path to the top level `include` +directory and include the headers by prefixing `spdk/` like this: + +~~~{.c} +#include "spdk/nvme.h" +~~~ + +Most of the headers here correspond with a library in the `lib` directory. There +are a few headers that stand alone, however. They are: + + - `assert.h` + - `barrier.h` + - `endian.h` + - `fd.h` + - `mmio.h` + - `queue.h` and `queue_extras.h` + - `string.h` + +There is also an `spdk_internal` directory that contains header files widely included +by libraries within SPDK, but that are not part of the public API and would not be +installed on a user's system. + +## Scripts {#dir_scripts} + +The `scripts` directory contains convenient scripts for a number of operations. The two most +important are `check_format.sh`, which will use astyle and pep8 to check C, C++, and Python +coding style against our defined conventions, and `setup.sh` which binds and unbinds devices +from kernel drivers. + +## Tests {#dir_tests} + +The `test` directory contains all of the tests for SPDK's components and the subdirectories mirror +the structure of the entire repository. The tests are a mixture of unit tests and functional tests. diff --git a/src/spdk/doc/peer_2_peer.md b/src/spdk/doc/peer_2_peer.md new file mode 100644 index 000000000..11fdc77e6 --- /dev/null +++ b/src/spdk/doc/peer_2_peer.md @@ -0,0 +1,72 @@ +# Peer-2-Peer DMAs {#peer_2_peer} + +Please note that the functionality discussed in this document is +currently tagged as experimental. + +# In this document {#p2p_toc} + +* @ref p2p_overview +* @ref p2p_nvme_api +* @ref p2p_cmb_copy +* @ref p2p_issues + +# Overview {#p2p_overview} + +Peer-2-Peer (P2P) is the concept of DMAing data directly from one PCI +End Point (EP) to another without using a system memory buffer. The +most obvious example of this from an SPDK perspective is using a NVMe +Controller Memory Buffer (CMB) to enable direct copies of data between +two NVMe SSDs. + +In this section of documentation we outline how to perform P2P +operations in SPDK and outline some of the issues that can occur when +performing P2P operations. + +# The P2P API for NVMe {#p2p_nvme_api} + +The functions that provide access to the NVMe CMBs for P2P +capabilities are given in the table below. + +Key Functions | Description +------------------------------------------- | ----------- +spdk_nvme_ctrlr_map_cmb() | @copybrief spdk_nvme_ctrlr_map_cmb() +spdk_nvme_ctrlr_unmap_cmb() | @copybrief spdk_nvme_ctrlr_unmap_cmb() +spdk_nvme_ctrlr_get_regs_cmbsz() | @copybrief spdk_nvme_ctrlr_get_regs_cmbsz() + +# Determining device support {#p2p_support} + +SPDK's identify example application displays whether a device has a controller +memory buffer and which operations it supports. Run it as follows: + +~~~{.sh} +./build/examples/identify -r traddr:<pci id of ssd> +~~~ + +# cmb_copy: An example P2P Application {#p2p_cmb_copy} + +Run the cmb_copy example application. + +~~~{.sh} +./build/examples/cmb_copy -r <pci id of write ssd>-1-0-1 -w <pci id of write ssd>-1-0-1 -c <pci id of the ssd with cmb> +~~~ +This should copy a single LBA (LBA 0) from namespace 1 on the read +NVMe SSD to LBA 0 on namespace 1 on the write SSD using the CMB as the +DMA buffer. + +# Issues with P2P {#p2p_issues} + +* In some systems when performing peer-2-peer DMAs between PCIe EPs + that are directly connected to the Root Complex (RC) the DMA may + fail or the performance may not be great. Basically your milage may + vary. It is recommended that you use a PCIe switch (such as those + provided by Broadcom or Microsemi) as that is know to provide good + performance. +* Even with a PCIe switch there may be occasions where peer-2-peer + DMAs fail to work. This is probably due to PCIe Access Control + Services (ACS) being enabled by the BIOS and/or OS. You can disable + ACS using setpci or via out of tree kernel patches that can be found + on the internet. +* In more complex topologies involving several switches it may be + possible to construct multiple paths between EPs. This could lead to + TLP ordering problems. If you are working in these environments be + careful! diff --git a/src/spdk/doc/performance_reports.md b/src/spdk/doc/performance_reports.md new file mode 100644 index 000000000..745b7892d --- /dev/null +++ b/src/spdk/doc/performance_reports.md @@ -0,0 +1,41 @@ +# Performance Reports {#performance_reports} + +## Release 20.04 + +- [SPDK 20.04 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2004.pdf) +- [SPDK 20.04 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2004.pdf) +- [SPDK 20.04 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2004.pdf) + +## Release 20.01 + +- [SPDK 20.01 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2001.pdf) +- [SPDK 20.01 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2001.pdf) +- [SPDK 20.01 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2001.pdf) + +## Release 19.10 + +- [SPDK 19.10 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_1910.pdf) +- [SPDK 19.10 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvmeof_tcp_perf_report_1910.pdf) +- [SPDK 19.10 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvmeof_rdma_perf_report_1910.pdf) + +## Release 19.07 + +- [SPDK 19.07 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_19.07.pdf) +- [SPDK 19.07 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvmeof_tcp_perf_report_19.07.pdf) + +## Release 19.04 + +- [SPDK 19.04 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_19.04_NVMeOF_RDMA_benchmark_report.pdf) + +## Release 19.01 + +- [SPDK 19.01.1 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvmeof_perf_report_19.01.1.pdf) + +## Release 18.04 + +- [SPDK 18.04 NVMe BDEV Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_18.04.pdf) +- [SPDK 18.04 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvmeof_perf_report_18.04.pdf) + +## Release 17.07 + +- [SPDK 17.07 vhost-scsi Performance Report](https://ci.spdk.io/download/performance-reports/SPDK17_07_vhost_scsi_performance_report.pdf) diff --git a/src/spdk/doc/porting.md b/src/spdk/doc/porting.md new file mode 100644 index 000000000..b6872bef1 --- /dev/null +++ b/src/spdk/doc/porting.md @@ -0,0 +1,21 @@ +# SPDK Porting Guide {#porting} + +SPDK is ported to new environments by implementing the *env* +library interface. The *env* interface provides APIs for drivers +to allocate physically contiguous and pinned memory, perform PCI +operations (config cycles and mapping BARs), virtual to physical +address translation and managing memory pools. The *env* API is +defined in include/spdk/env.h. + +SPDK includes a default implementation of the *env* library based +on the Data Plane Development Kit ([DPDK](http://dpdk.org/)). +This DPDK implementation can be found in `lib/env_dpdk`. + +DPDK is currently supported on Linux and FreeBSD only. +Users who want to use SPDK on other operating systems, or in +userspace driver frameworks other than DPDK, will need to implement +a new version of the *env* library. The new implementation can be +integrated into the SPDK build by updating the following line +in CONFIG: + + CONFIG_ENV?=$(SPDK_ROOT_DIR)/lib/env_dpdk diff --git a/src/spdk/doc/prog_guides.md b/src/spdk/doc/prog_guides.md new file mode 100644 index 000000000..554b81860 --- /dev/null +++ b/src/spdk/doc/prog_guides.md @@ -0,0 +1,11 @@ +# Programmer Guides {#prog_guides} + +- [Public API header files](files.html) +- @subpage blob +- @subpage bdev_pg +- @subpage bdev_module +- @subpage nvmf_tgt_pg +- @subpage ftl +- @subpage gdb_macros +- @subpage reduce +- @subpage notify diff --git a/src/spdk/doc/spdkcli.md b/src/spdk/doc/spdkcli.md new file mode 100644 index 000000000..74994fc4a --- /dev/null +++ b/src/spdk/doc/spdkcli.md @@ -0,0 +1,62 @@ +# SPDK CLI {#spdkcli} + +Spdkcli is a command-line management application for SPDK. +Spdkcli has support for a limited number of applications and bdev modules, +and should be considered experimental for the v18.04 release. +This experimental version was added for v18.04 to get early feedback +that can be incorporated as spdkcli becomes more fully-featured +for the next SPDK release. + +### Install needed dependencies + +All dependencies should be handled by scripts/pkgdep.sh script. +Package dependencies at the moment include: + + - configshell + +### Run SPDK application instance + +~~~{.sh} +./scripts/setup.sh +./build/bin/vhost -c vhost.conf +~~~ + +### Run SPDK CLI + +Spdkcli should be run with the same privileges as SPDK application. +In order to use SPDK CLI in interactive mode please use: +~~~{.sh} +scripts/spdkcli.py +~~~ +Use "help" command to get a list of available commands for each tree node. + +It is also possible to use SPDK CLI to run just a single command, +just use the command as an argument to the application. +For example, to view current configuration and immediately exit: + ~~~{.sh} +scripts/spdkcli.py ls +~~~ + +### Optional - create Python virtual environment + +You can use Python virtual environment if you don't want to litter your +system Python installation. + +First create the virtual environment: +~~~{.sh} +cd spdk +mkdir venv +virtualenv-3 ./venv +source ./venv/bin/activate +~~~ + +Then install the dependencies using pip. That way dependencies will be +installed only inside the virtual environment. +~~~{.sh} +(venv) pip install configshell-fb +~~~ + +Tip: if you are using "sudo" instead of root account, it is suggested to do +"sudo -s" before activating the environment. This is because venv might not work +correctly when calling spdkcli with sudo, like "sudo python spdkcli.py" - +some environment variables might not be passed and you will experience errors. diff --git a/src/spdk/doc/ssd_internals.md b/src/spdk/doc/ssd_internals.md new file mode 100644 index 000000000..f11ad9802 --- /dev/null +++ b/src/spdk/doc/ssd_internals.md @@ -0,0 +1,96 @@ +# NAND Flash SSD Internals {#ssd_internals} + +Solid State Devices (SSD) are complex devices and their performance depends on +how they're used. The following description is intended to help software +developers understand what is occurring inside the SSD, so that they can come +up with better software designs. It should not be thought of as a strictly +accurate guide to how SSD hardware really works. + + As of this writing, SSDs are generally implemented on top of + [NAND Flash](https://en.wikipedia.org/wiki/Flash_memory) memory. At a + very high level, this media has a few important properties: + +* The media is grouped onto chips called NAND dies and each die can + operate in parallel. +* Flipping a bit is a highly asymmetric process. Flipping it one way is + easy, but flipping it back is quite hard. + +NAND Flash media is grouped into large units often referred to as **erase +blocks**. The size of an erase block is highly implementation specific, but +can be thought of as somewhere between 1MiB and 8MiB. For each erase block, +each bit may be written to (i.e. have its bit flipped from 0 to 1) with +bit-granularity once. In order to write to the erase block a second time, the +entire block must be erased (i.e. all bits in the block are flipped back to +0). This is the asymmetry part from above. Erasing a block causes a measurable +amount of wear and each block may only be erased a limited number of times. + +SSDs expose an interface to the host system that makes it appear as if the +drive is composed of a set of fixed size **logical blocks** which are usually +512B or 4KiB in size. These blocks are entirely logical constructs of the +device firmware and they do not statically map to a location on the backing +media. Instead, upon each write to a logical block, a new location on the NAND +Flash is selected and written and the mapping of the logical block to its +physical location is updated. The algorithm for choosing this location is a +key part of overall SSD performance and is often called the **flash +translation layer** or FTL. This algorithm must correctly distribute the +blocks to account for wear (called **wear-leveling**) and spread them across +NAND dies to improve total available performance. The simplest model is to +group all of the physical media on each die together using an algorithm +similar to RAID and then write to that set sequentially. Real SSDs are far +more complicated, but this is an excellent simple model for software +developers - imagine they are simply logging to a RAID volume and updating an +in-memory hash-table. + +One consequence of the flash translation layer is that logical blocks do not +necessarily correspond to physical locations on the NAND at all times. In +fact, there is a command that clears the translation for a block. In NVMe, +this command is called deallocate, in SCSI it is called unmap, and in SATA it +is called trim. When a user attempts to read a block that doesn't have a +mapping to a physical location, drives will do one of two things: + +1. Immediately complete the read request successfully, without performing any + data transfer. This is acceptable because the data the drive would return + is no more valid than the data already in the user's data buffer. +2. Return all 0's as the data. + +Choice #1 is much more common and performing reads to a fully deallocated +device will often show performance far beyond what the drive claims to be +capable of precisely because it is not actually transferring any data. Write +to all blocks prior to reading them when benchmarking! + +As SSDs are written to, the internal log will eventually consume all of the +available erase blocks. In order to continue writing, the SSD must free some +of them. This process is often called **garbage collection**. All SSDs reserve +some number of erase blocks so that they can guarantee there are free erase +blocks available for garbage collection. Garbage collection generally proceeds +by: + +1. Selecting a target erase block (a good mental model is that it picks the least recently used erase block) +2. Walking through each entry in the erase block and determining if it is still a valid logical block. +3. Moving valid logical blocks by reading them and writing them to a different erase block (i.e. the current head of the log) +4. Erasing the entire erase block and marking it available for use. + +Garbage collection is clearly far more efficient when step #3 can be skipped +because the erase block is already empty. There are two ways to make it much +more likely that step #3 can be skipped. The first is that SSDs reserve +additional erase blocks beyond their reported capacity (called +**over-provisioning**), so that statistically its much more likely that an +erase block will not contain valid data. The second is software can write to +the blocks on the device in sequential order in a circular pattern, throwing +away old data when it is no longer needed. In this case, the software +guarantees that the least recently used erase blocks will not contain any +valid data that must be moved. + +The amount of over-provisioning a device has can dramatically impact the +performance on random read and write workloads, if the workload is filling up +the entire device. However, the same effect can typically be obtained by +simply reserving a given amount of space on the device in software. This +understanding is critical to producing consistent benchmarks. In particular, +if background garbage collection cannot keep up and the drive must switch to +on-demand garbage collection, the latency of writes will increase +dramatically. Therefore the internal state of the device must be forced into +some known state prior to running benchmarks for consistency. This is usually +accomplished by writing to the device sequentially two times, from start to +finish. For a highly detailed description of exactly how to force an SSD into +a known state for benchmarking see this +[SNIA Article](http://www.snia.org/sites/default/files/SSS_PTS_Enterprise_v1.1.pdf). diff --git a/src/spdk/doc/stylesheet.css b/src/spdk/doc/stylesheet.css new file mode 100644 index 000000000..06b2b40b1 --- /dev/null +++ b/src/spdk/doc/stylesheet.css @@ -0,0 +1,1619 @@ +/* SPDK specific */ + +body, +html { + width: 100%; + height: 100%; +} + +body, +h1, +h2, +h3, +h4, +h5, +h6 { + font-family: 'Roboto', sans-serif; + font-weight: 900; +} + +.row.no-gutters > [class^="col-"], +.row.no-gutters > [class*=" col-"] { + padding-right: 0; + padding-left: 0; +} + +.container-fluid { + padding: 0; +} + +#side-nav * { + box-sizing: content-box; +} + +/* Original Stylesheet */ + +h1.groupheader { + font-size: 150%; +} + +h2.groupheader { + border-bottom: 1px solid #879ECB; + color: #354C7B; + font-size: 150%; + font-weight: normal; + margin-top: 1.75em; + padding-top: 8px; + padding-bottom: 4px; + width: 100%; +} + +h3.groupheader { + font-size: 100%; +} + +dt { + font-weight: bold; +} + +div.multicol { + -moz-column-gap: 1em; + -webkit-column-gap: 1em; + -moz-column-count: 3; + -webkit-column-count: 3; +} + +p.startli, p.startdd { + margin-top: 2px; +} + +p.starttd { + margin-top: 0px; +} + +p.endli { + margin-bottom: 0px; +} + +p.enddd { + margin-bottom: 4px; +} + +p.endtd { + margin-bottom: 2px; +} + +/* @end */ + +caption { + font-weight: bold; +} + +span.legend { + font-size: 70%; + text-align: center; +} + +h3.version { + font-size: 90%; + text-align: center; +} + +div.qindex, div.navtab{ + background-color: #EBEFF6; + border: 1px solid #A3B4D7; + text-align: center; +} + +div.qindex, div.navpath { + width: 100%; + line-height: 140%; +} + +div.navtab { + margin-right: 15px; +} + +dl.el { + margin-left: -1cm; +} + +pre.fragment { + border: 1px solid #C4CFE5; + background-color: #FBFCFD; + padding: 4px 6px; + margin: 4px 8px 4px 2px; + overflow: auto; + word-wrap: break-word; + font-size: 9pt; + line-height: 125%; + font-family: monospace, fixed; + font-size: 105%; +} + +div.fragment { + padding: 4px 6px; + margin: 4px 8px 4px 2px; + background-color: #FBFCFD; + border: 1px solid #C4CFE5; +} + +div.line { + font-family: monospace, fixed; + font-size: 13px; + min-height: 13px; + line-height: 1.0; + text-wrap: unrestricted; + white-space: -moz-pre-wrap; /* Moz */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + white-space: pre-wrap; /* CSS3 */ + word-wrap: break-word; /* IE 5.5+ */ + text-indent: -53px; + padding-left: 53px; + padding-bottom: 0px; + margin: 0px; + -webkit-transition-property: background-color, box-shadow; + -webkit-transition-duration: 0.5s; + -moz-transition-property: background-color, box-shadow; + -moz-transition-duration: 0.5s; + -ms-transition-property: background-color, box-shadow; + -ms-transition-duration: 0.5s; + -o-transition-property: background-color, box-shadow; + -o-transition-duration: 0.5s; + transition-property: background-color, box-shadow; + transition-duration: 0.5s; +} + +div.line.glow { + background-color: cyan; + box-shadow: 0 0 10px cyan; +} + + +span.lineno { + padding-right: 4px; + text-align: right; + border-right: 2px solid #0F0; + background-color: #E8E8E8; + white-space: pre; +} +span.lineno a { + background-color: #D8D8D8; +} + +span.lineno a:hover { + background-color: #C8C8C8; +} + +div.ah, span.ah { + background-color: black; + font-weight: bold; + color: #ffffff; + margin-bottom: 3px; + margin-top: 3px; + padding: 0.2em; + border: solid thin #333; + border-radius: 0.5em; + -webkit-border-radius: .5em; + -moz-border-radius: .5em; + box-shadow: 2px 2px 3px #999; + -webkit-box-shadow: 2px 2px 3px #999; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + background-image: -webkit-gradient(linear, left top, left bottom, from(#eee), to(#000),color-stop(0.3, #444)); + background-image: -moz-linear-gradient(center top, #eee 0%, #444 40%, #000); +} + +div.classindex ul { + list-style: none; + padding-left: 0; +} + +div.classindex span.ai { + display: inline-block; +} + +div.groupHeader { + margin-left: 16px; + margin-top: 12px; + font-weight: bold; +} + +div.groupText { + margin-left: 16px; + font-style: italic; +} + +div.contents { + margin-right: auto; + margin-left: auto; + padding-left: 15px; + padding-right: 15px; + margin-bottom: 100px; + font-size: 18px; + font-weight: 400; +} + +div.header { + margin-right: auto; + margin-left: auto; + padding-left: 15px; + padding-right: 15px; +} + +div.headertitle { + font-size: 36px; + margin-left: 0; + margin-right: 0; + margin-top: 20px; + margin-bottom: 10px; +} + +@media (min-width: 768px) { + div.contents, div.header { + width: 750px; + padding-left: 0; + padding-right: 0; + } +} + +@media (min-width: 992px) { + div.contents, div.header { + width:970px; + padding-left: 0; + padding-right: 0; + } +} + +@media (min-width: 1200px) { + div.contents, div.header { + width: 1170px; + padding-left: 0; + padding-right: 0; + } +} + +td.indexkey { + background-color: #EBEFF6; + font-weight: bold; + border: 1px solid #C4CFE5; + margin: 2px 0px 2px 0; + padding: 2px 10px; + white-space: nowrap; + vertical-align: top; +} + +td.indexvalue { + background-color: #EBEFF6; + border: 1px solid #C4CFE5; + padding: 2px 10px; + margin: 2px 0px; +} + +tr.memlist { + background-color: #EEF1F7; +} + +p.formulaDsp { + text-align: center; +} + +img.formulaDsp { + +} + +img.formulaInl { + vertical-align: middle; +} + +div.center { + text-align: center; + margin-top: 0px; + margin-bottom: 0px; + padding: 0px; +} + +div.center img { + border: 0px; +} + +address.footer { + text-align: right; + padding-right: 12px; +} + +img.footer { + border: 0px; + vertical-align: middle; +} + +/* @group Code Colorization */ + +span.keyword { + color: #008000 +} + +span.keywordtype { + color: #604020 +} + +span.keywordflow { + color: #e08000 +} + +span.comment { + color: #800000 +} + +span.preprocessor { + color: #806020 +} + +span.stringliteral { + color: #002080 +} + +span.charliteral { + color: #008080 +} + +span.vhdldigit { + color: #ff00ff +} + +span.vhdlchar { + color: #000000 +} + +span.vhdlkeyword { + color: #700070 +} + +span.vhdllogic { + color: #ff0000 +} + +blockquote { + background-color: #F7F8FB; + border-left: 2px solid #9CAFD4; + margin: 0 24px 0 4px; + padding: 0 12px 0 16px; +} + +/* @end */ + +/* +.search { + color: #003399; + font-weight: bold; +} + +form.search { + margin-bottom: 0px; + margin-top: 0px; +} + +input.search { + font-size: 75%; + color: #000080; + font-weight: normal; + background-color: #e8eef2; +} +*/ + +td.tiny { + font-size: 75%; +} + +.dirtab { + padding: 4px; + border-collapse: collapse; + border: 1px solid #A3B4D7; +} + +th.dirtab { + background: #EBEFF6; + font-weight: bold; +} + +hr { + height: 0px; + border: none; + border-top: 1px solid #4A6AAA; +} + +hr.footer { + height: 1px; +} + +/* @group Member Descriptions */ + +table.memberdecls { + border-spacing: 0px; + padding: 0px; +} + +.memberdecls td, .fieldtable tr { + -webkit-transition-property: background-color, box-shadow; + -webkit-transition-duration: 0.5s; + -moz-transition-property: background-color, box-shadow; + -moz-transition-duration: 0.5s; + -ms-transition-property: background-color, box-shadow; + -ms-transition-duration: 0.5s; + -o-transition-property: background-color, box-shadow; + -o-transition-duration: 0.5s; + transition-property: background-color, box-shadow; + transition-duration: 0.5s; +} + +.memberdecls td.glow, .fieldtable tr.glow { + background-color: cyan; + box-shadow: 0 0 15px cyan; +} + +.mdescLeft, .mdescRight, +.memItemLeft, .memItemRight, +.memTemplItemLeft, .memTemplItemRight, .memTemplParams { + background-color: #F9FAFC; + border: none; + margin: 4px; + padding: 1px 0 0 8px; +} + +.mdescLeft, .mdescRight { + padding: 0px 8px 4px 8px; + color: #555; +} + +.memSeparator { + border-bottom: 1px solid #DEE4F0; + line-height: 1px; + margin: 0px; + padding: 0px; +} + +.memItemLeft, .memTemplItemLeft { + white-space: nowrap; +} + +.memItemRight { + width: 100%; +} + +.memTemplParams { + color: #4665A2; + white-space: nowrap; + font-size: 80%; +} + +/* @end */ + +/* @group Member Details */ + +/* Styles for detailed member documentation */ + +.memtemplate { + font-size: 80%; + color: #4665A2; + font-weight: normal; + margin-left: 9px; +} + +.memnav { + background-color: #EBEFF6; + border: 1px solid #A3B4D7; + text-align: center; + margin: 2px; + margin-right: 15px; + padding: 2px; +} + +.mempage { + width: 100%; +} + +.memitem { + padding: 0; + margin-bottom: 10px; + margin-right: 5px; + -webkit-transition: box-shadow 0.5s linear; + -moz-transition: box-shadow 0.5s linear; + -ms-transition: box-shadow 0.5s linear; + -o-transition: box-shadow 0.5s linear; + transition: box-shadow 0.5s linear; + display: table !important; + width: 100%; +} + +.memitem.glow { + box-shadow: 0 0 15px cyan; +} + +.memname { + font-weight: bold; + margin-left: 6px; +} + +.memname td { + vertical-align: bottom; +} + +.memproto, dl.reflist dt { + border-top: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + padding: 6px 0px 6px 0px; + color: #253555; + font-weight: bold; + text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); + background-image:url('nav_f.png'); + background-repeat:repeat-x; + background-color: #E2E8F2; + /* opera specific markup */ + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + border-top-right-radius: 4px; + border-top-left-radius: 4px; + /* firefox specific markup */ + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + -moz-border-radius-topright: 4px; + -moz-border-radius-topleft: 4px; + /* webkit specific markup */ + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + -webkit-border-top-right-radius: 4px; + -webkit-border-top-left-radius: 4px; + +} + +.memdoc, dl.reflist dd { + border-bottom: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + padding: 6px 10px 2px 10px; + background-color: #FBFCFD; + border-top-width: 0; + background-image:url('nav_g.png'); + background-repeat:repeat-x; + background-color: #FFFFFF; + /* opera specific markup */ + border-bottom-left-radius: 4px; + border-bottom-right-radius: 4px; + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + /* firefox specific markup */ + -moz-border-radius-bottomleft: 4px; + -moz-border-radius-bottomright: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + /* webkit specific markup */ + -webkit-border-bottom-left-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); +} + +dl.reflist dt { + padding: 5px; +} + +dl.reflist dd { + margin: 0px 0px 10px 0px; + padding: 5px; +} + +.paramkey { + text-align: right; +} + +.paramtype { + white-space: nowrap; +} + +.paramname { + color: #602020; + white-space: nowrap; +} +.paramname em { + font-style: normal; +} +.paramname code { + line-height: 14px; +} + +.params, .retval, .exception, .tparams { + margin-left: 0px; + padding-left: 0px; +} + +.params .paramname, .retval .paramname { + font-weight: bold; + vertical-align: top; +} + +.params .paramtype { + font-style: italic; + vertical-align: top; +} + +.params .paramdir { + font-family: "courier new",courier,monospace; + vertical-align: top; +} + +table.mlabels { + border-spacing: 0px; +} + +td.mlabels-left { + width: 100%; + padding: 0px; +} + +td.mlabels-right { + vertical-align: bottom; + padding: 0px; + white-space: nowrap; +} + +span.mlabels { + margin-left: 8px; +} + +span.mlabel { + background-color: #728DC1; + border-top:1px solid #5373B4; + border-left:1px solid #5373B4; + border-right:1px solid #C4CFE5; + border-bottom:1px solid #C4CFE5; + text-shadow: none; + color: white; + margin-right: 4px; + padding: 2px 3px; + border-radius: 3px; + font-size: 7pt; + white-space: nowrap; + vertical-align: middle; +} + + + +/* @end */ + +/* these are for tree view inside a (index) page */ + +div.directory { + margin: 10px 0px; + border-top: 1px solid #9CAFD4; + border-bottom: 1px solid #9CAFD4; + width: 100%; +} + +.directory table { + border-collapse:collapse; +} + +.directory td { + margin: 0px; + padding: 0px; + vertical-align: top; +} + +.directory td.entry { + white-space: nowrap; + padding-right: 6px; + padding-top: 3px; +} + +.directory td.entry a { + outline:none; +} + +.directory td.entry a img { + border: none; +} + +.directory td.desc { + width: 100%; + padding-left: 6px; + padding-right: 6px; + padding-top: 3px; + border-left: 1px solid rgba(0,0,0,0.05); +} + +.directory tr.even { + padding-left: 6px; + background-color: #F7F8FB; +} + +.directory img { + vertical-align: -30%; +} + +.directory .levels { + white-space: nowrap; + width: 100%; + text-align: right; + font-size: 9pt; +} + +.directory .levels span { + cursor: pointer; + padding-left: 2px; + padding-right: 2px; + color: #3D578C; +} + +.arrow { + color: #9CAFD4; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + cursor: pointer; + font-size: 80%; + display: inline-block; + width: 16px; + height: 22px; +} + +.icon { + font-family: Arial, Helvetica; + font-weight: bold; + font-size: 12px; + height: 14px; + width: 16px; + display: inline-block; + background-color: #728DC1; + color: white; + text-align: center; + border-radius: 4px; + margin-left: 2px; + margin-right: 2px; +} + +.icona { + width: 24px; + height: 22px; + display: inline-block; +} + +.iconfopen { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('folderopen.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +.iconfclosed { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('folderclosed.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +.icondoc { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('doc.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +table.directory { + font: 400 14px Roboto,sans-serif; +} + +/* @end */ + +div.dynheader { + margin-top: 8px; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +address { + font-style: normal; + color: #2A3D61; +} + +table.doxtable, table.markdownTable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.doxtable td, table.doxtable th, table.markdownTable td, table.markdownTable th { + border: 1px solid #2D4068; + padding: 3px 7px 2px; +} + +table.doxtable th, table.markdownTable th { + background-color: #374F7F; + color: #FFFFFF; + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +table.fieldtable { + /*width: 100%;*/ + margin-bottom: 10px; + border: 1px solid #A8B8D9; + border-spacing: 0px; + -moz-border-radius: 4px; + -webkit-border-radius: 4px; + border-radius: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + -webkit-box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); + box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); +} + +.fieldtable td, .fieldtable th { + padding: 3px 7px 2px; +} + +.fieldtable td.fieldtype, .fieldtable td.fieldname { + white-space: nowrap; + border-right: 1px solid #A8B8D9; + border-bottom: 1px solid #A8B8D9; + vertical-align: top; +} + +.fieldtable td.fieldname { + padding-top: 3px; +} + +.fieldtable td.fielddoc { + border-bottom: 1px solid #A8B8D9; + /*width: 100%;*/ +} + +.fieldtable td.fielddoc p:first-child { + margin-top: 0px; +} + +.fieldtable td.fielddoc p:last-child { + margin-bottom: 2px; +} + +.fieldtable tr:last-child td { + border-bottom: none; +} + +.fieldtable th { + background-image:url('nav_f.png'); + background-repeat:repeat-x; + background-color: #E2E8F2; + font-size: 90%; + color: #253555; + padding-bottom: 4px; + padding-top: 5px; + text-align:left; + -moz-border-radius-topleft: 4px; + -moz-border-radius-topright: 4px; + -webkit-border-top-left-radius: 4px; + -webkit-border-top-right-radius: 4px; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom: 1px solid #A8B8D9; +} + + +.tabsearch { + top: 0px; + left: 10px; + height: 36px; + background-image: url('tab_b.png'); + z-index: 101; + overflow: hidden; + font-size: 13px; +} + +.navpath ul +{ + font-size: 11px; + background-image:url('tab_b.png'); + background-repeat:repeat-x; + background-position: 0 -5px; + height:30px; + line-height:30px; + color:#8AA0CC; + border:solid 1px #C2CDE4; + overflow:hidden; + margin:0px; + padding:0px; +} + +.navpath li +{ + list-style-type:none; + float:left; + padding-left:10px; + padding-right:15px; + background-image:url('bc_s.png'); + background-repeat:no-repeat; + background-position:right; + color:#364D7C; +} + +.navpath li.navelem a +{ + height:32px; + display:block; + text-decoration: none; + outline: none; + color: #283A5D; + font-family: 'Lucida Grande',Geneva,Helvetica,Arial,sans-serif; + text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); + text-decoration: none; +} + +.navpath li.navelem a:hover +{ + color:#6884BD; +} + +.navpath li.footer +{ + list-style-type:none; + float:right; + padding-left:10px; + padding-right:15px; + background-image:none; + background-repeat:no-repeat; + background-position:right; + color:#364D7C; + font-size: 8pt; +} + + +div.summary +{ + float: right; + font-size: 8pt; + padding-right: 5px; + width: 50%; + text-align: right; +} + +div.summary a +{ + white-space: nowrap; +} + +div.ingroups +{ + font-size: 8pt; + width: 50%; + text-align: left; +} + +div.ingroups a +{ + white-space: nowrap; +} + +dl +{ + padding: 0 0 0 10px; +} + +/* dl.note, dl.warning, dl.attention, dl.pre, dl.post, dl.invariant, dl.deprecated, dl.todo, dl.test, dl.bug */ +dl.section +{ + margin-left: 0px; + padding-left: 0px; +} + +dl.note +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #D0C000; +} + +dl.warning, dl.attention +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #FF0000; +} + +dl.pre, dl.post, dl.invariant +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #00D000; +} + +dl.deprecated +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #505050; +} + +dl.todo +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #00C0E0; +} + +dl.test +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #3030E0; +} + +dl.bug +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #C08050; +} + +dl.section dd { + margin-bottom: 6px; +} + +.image +{ + text-align: center; +} + +.dotgraph +{ + text-align: center; +} + +.mscgraph +{ + text-align: center; +} + +.diagraph +{ + text-align: center; +} + +.caption +{ + font-weight: bold; +} + +div.zoom +{ + border: 1px solid #90A5CE; +} + +dl.citelist { + margin-bottom:50px; +} + +dl.citelist dt { + color:#334975; + float:left; + font-weight:bold; + margin-right:10px; + padding:5px; +} + +dl.citelist dd { + margin:2px 0; + padding:5px 0; +} + +div.toc { + padding: 14px 25px; + background-color: #F4F6FA; + border: 1px solid #D8DFEE; + border-radius: 7px 7px 7px 7px; + float: right; + height: auto; + margin: 0 20px 10px 10px; + width: 200px; +} + +div.toc li { + background: url("bdwn.png") no-repeat scroll 0 5px transparent; + font: 10px/1.2 Verdana,DejaVu Sans,Geneva,sans-serif; + margin-top: 5px; + padding-left: 10px; + padding-top: 2px; +} + +div.toc h3 { + font: bold 12px/1.2 Arial,FreeSans,sans-serif; + color: #4665A2; + border-bottom: 0 none; + margin: 0; +} + +div.toc ul { + list-style: none outside none; + border: medium none; + padding: 0px; +} + +div.toc li.level1 { + margin-left: 0px; +} + +div.toc li.level2 { + margin-left: 15px; +} + +div.toc li.level3 { + margin-left: 30px; +} + +div.toc li.level4 { + margin-left: 45px; +} + +.inherit_header { + font-weight: bold; + color: gray; + cursor: pointer; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.inherit_header td { + padding: 6px 0px 2px 5px; +} + +.inherit { + display: none; +} + +tr.heading h2 { + margin-top: 12px; + margin-bottom: 4px; +} + +/* tooltip related style info */ + +.ttc { + position: absolute; + display: none; +} + +#powerTip { + cursor: default; + white-space: nowrap; + background-color: white; + border: 1px solid gray; + border-radius: 4px 4px 4px 4px; + box-shadow: 1px 1px 7px gray; + display: none; + font-size: smaller; + max-width: 80%; + opacity: 0.9; + padding: 1ex 1em 1em; + position: absolute; + z-index: 2147483647; +} + +#powerTip div.ttdoc { + color: grey; + font-style: italic; +} + +#powerTip div.ttname a { + font-weight: bold; +} + +#powerTip div.ttname { + font-weight: bold; +} + +#powerTip div.ttdeci { + color: #006318; +} + +#powerTip div { + margin: 0px; + padding: 0px; + font: 12px/16px Roboto,sans-serif; +} + +#powerTip:before, #powerTip:after { + content: ""; + position: absolute; + margin: 0px; +} + +#powerTip.n:after, #powerTip.n:before, +#powerTip.s:after, #powerTip.s:before, +#powerTip.w:after, #powerTip.w:before, +#powerTip.e:after, #powerTip.e:before, +#powerTip.ne:after, #powerTip.ne:before, +#powerTip.se:after, #powerTip.se:before, +#powerTip.nw:after, #powerTip.nw:before, +#powerTip.sw:after, #powerTip.sw:before { + border: solid transparent; + content: " "; + height: 0; + width: 0; + position: absolute; +} + +#powerTip.n:after, #powerTip.s:after, +#powerTip.w:after, #powerTip.e:after, +#powerTip.nw:after, #powerTip.ne:after, +#powerTip.sw:after, #powerTip.se:after { + border-color: rgba(255, 255, 255, 0); +} + +#powerTip.n:before, #powerTip.s:before, +#powerTip.w:before, #powerTip.e:before, +#powerTip.nw:before, #powerTip.ne:before, +#powerTip.sw:before, #powerTip.se:before { + border-color: rgba(128, 128, 128, 0); +} + +#powerTip.n:after, #powerTip.n:before, +#powerTip.ne:after, #powerTip.ne:before, +#powerTip.nw:after, #powerTip.nw:before { + top: 100%; +} + +#powerTip.n:after, #powerTip.ne:after, #powerTip.nw:after { + border-top-color: #ffffff; + border-width: 10px; + margin: 0px -10px; +} +#powerTip.n:before { + border-top-color: #808080; + border-width: 11px; + margin: 0px -11px; +} +#powerTip.n:after, #powerTip.n:before { + left: 50%; +} + +#powerTip.nw:after, #powerTip.nw:before { + right: 14px; +} + +#powerTip.ne:after, #powerTip.ne:before { + left: 14px; +} + +#powerTip.s:after, #powerTip.s:before, +#powerTip.se:after, #powerTip.se:before, +#powerTip.sw:after, #powerTip.sw:before { + bottom: 100%; +} + +#powerTip.s:after, #powerTip.se:after, #powerTip.sw:after { + border-bottom-color: #ffffff; + border-width: 10px; + margin: 0px -10px; +} + +#powerTip.s:before, #powerTip.se:before, #powerTip.sw:before { + border-bottom-color: #808080; + border-width: 11px; + margin: 0px -11px; +} + +#powerTip.s:after, #powerTip.s:before { + left: 50%; +} + +#powerTip.sw:after, #powerTip.sw:before { + right: 14px; +} + +#powerTip.se:after, #powerTip.se:before { + left: 14px; +} + +#powerTip.e:after, #powerTip.e:before { + left: 100%; +} +#powerTip.e:after { + border-left-color: #ffffff; + border-width: 10px; + top: 50%; + margin-top: -10px; +} +#powerTip.e:before { + border-left-color: #808080; + border-width: 11px; + top: 50%; + margin-top: -11px; +} + +#powerTip.w:after, #powerTip.w:before { + right: 100%; +} +#powerTip.w:after { + border-right-color: #ffffff; + border-width: 10px; + top: 50%; + margin-top: -10px; +} +#powerTip.w:before { + border-right-color: #808080; + border-width: 11px; + top: 50%; + margin-top: -11px; +} + +@media print +{ + #top { display: none; } + #side-nav { display: none; } + #nav-path { display: none; } + body { overflow:visible; } + h1, h2, h3, h4, h5, h6 { page-break-after: avoid; } + .summary { display: none; } + .memitem { page-break-inside: avoid; } + #doc-content + { + margin-left:0 !important; + height:auto !important; + width:auto !important; + overflow:inherit; + display:inline; + } +} + +/* Non-responsive overrides + * + * Utilize the following CSS to disable the responsive-ness of the container, + * grid system, and navbar. + */ + +/* Reset the container */ +.container { + width: 970px; + max-width: none !important; +} + +/* Demonstrate the grids */ +.col-xs-4 { + padding-top: 15px; + padding-bottom: 15px; + background-color: #eee; + background-color: rgba(86,61,124,.15); + border: 1px solid #ddd; + border: 1px solid rgba(86,61,124,.2); +} + +.container .navbar-header, +.container .navbar-collapse { + margin-right: 0; + margin-left: 0; +} + +/* Always float the navbar header */ +.navbar-header { + float: left; +} + +/* Undo the collapsing navbar */ +.navbar-collapse { + display: block !important; + height: auto !important; + padding-bottom: 0; + overflow: visible !important; + visibility: visible !important; +} + +.navbar-toggle { + display: none; +} +.navbar-collapse { + border-top: 0; +} + +.navbar-brand { + margin-left: -15px; +} + +/* Always apply the floated nav */ +.navbar-nav { + float: left; + margin: 0; +} +.navbar-nav > li { + float: left; +} +.navbar-nav > li > a { + padding: 15px; +} + +/* Redeclare since we override the float above */ +.navbar-nav.navbar-right { + float: right; +} + +/* Undo custom dropdowns */ +.navbar .navbar-nav .open .dropdown-menu { + position: absolute; + float: left; + background-color: #fff; + border: 1px solid #ccc; + border: 1px solid rgba(0, 0, 0, .15); + border-width: 0 1px 1px; + border-radius: 0 0 4px 4px; + -webkit-box-shadow: 0 6px 12px rgba(0, 0, 0, .175); + box-shadow: 0 6px 12px rgba(0, 0, 0, .175); +} +.navbar-default .navbar-nav .open .dropdown-menu > li > a { + color: #333; +} +.navbar .navbar-nav .open .dropdown-menu > li > a:hover, +.navbar .navbar-nav .open .dropdown-menu > li > a:focus, +.navbar .navbar-nav .open .dropdown-menu > .active > a, +.navbar .navbar-nav .open .dropdown-menu > .active > a:hover, +.navbar .navbar-nav .open .dropdown-menu > .active > a:focus { + color: #fff !important; + background-color: #428bca !important; +} +.navbar .navbar-nav .open .dropdown-menu > .disabled > a, +.navbar .navbar-nav .open .dropdown-menu > .disabled > a:hover, +.navbar .navbar-nav .open .dropdown-menu > .disabled > a:focus { + color: #999 !important; + background-color: transparent !important; +} + +/* Undo form expansion */ +.navbar-form { + float: left; + width: auto; + padding-top: 0; + padding-bottom: 0; + margin-right: 0; + margin-left: 0; + border: 0; + -webkit-box-shadow: none; + box-shadow: none; +} + +/* Copy-pasted from forms.less since we mixin the .form-inline styles. */ +.navbar-form .form-group { + display: inline-block; + margin-bottom: 0; + vertical-align: middle; +} + +.navbar-form .form-control { + display: inline-block; + width: auto; + vertical-align: middle; +} + +.navbar-form .form-control-static { + display: inline-block; +} + +.navbar-form .input-group { + display: inline-table; + vertical-align: middle; +} + +.navbar-form .input-group .input-group-addon, +.navbar-form .input-group .input-group-btn, +.navbar-form .input-group .form-control { + width: auto; +} + +.navbar-form .input-group > .form-control { + width: 100%; +} + +.navbar-form .control-label { + margin-bottom: 0; + vertical-align: middle; +} + +.navbar-form .radio, +.navbar-form .checkbox { + display: inline-block; + margin-top: 0; + margin-bottom: 0; + vertical-align: middle; +} + +.navbar-form .radio label, +.navbar-form .checkbox label { + padding-left: 0; +} + +.navbar-form .radio input[type="radio"], +.navbar-form .checkbox input[type="checkbox"] { + position: relative; + margin-left: 0; +} + +.navbar-form .has-feedback .form-control-feedback { + top: 0; +} + +/* Undo inline form compaction on small screens */ +.form-inline .form-group { + display: inline-block; + margin-bottom: 0; + vertical-align: middle; +} + +.form-inline .form-control { + display: inline-block; + width: auto; + vertical-align: middle; +} + +.form-inline .form-control-static { + display: inline-block; +} + +.form-inline .input-group { + display: inline-table; + vertical-align: middle; +} +.form-inline .input-group .input-group-addon, +.form-inline .input-group .input-group-btn, +.form-inline .input-group .form-control { + width: auto; +} + +.form-inline .input-group > .form-control { + width: 100%; +} + +.form-inline .control-label { + margin-bottom: 0; + vertical-align: middle; +} + +.form-inline .radio, +.form-inline .checkbox { + display: inline-block; + margin-top: 0; + margin-bottom: 0; + vertical-align: middle; +} +.form-inline .radio label, +.form-inline .checkbox label { + padding-left: 0; +} + +.form-inline .radio input[type="radio"], +.form-inline .checkbox input[type="checkbox"] { + position: relative; + margin-left: 0; +} + +.form-inline .has-feedback .form-control-feedback { + top: 0; +} diff --git a/src/spdk/doc/system_configuration.md b/src/spdk/doc/system_configuration.md new file mode 100644 index 000000000..bbdebdf01 --- /dev/null +++ b/src/spdk/doc/system_configuration.md @@ -0,0 +1,24 @@ +# System Configuration User Guide {#system_configuration} + +This system configuration guide describes how to configure a system for use with SPDK. + +# IOMMU configuration {#iommu_config} + +An IOMMU may be present and enabled on many platforms. When an IOMMU is present and enabled, it is +recommended that SPDK applications are deployed with the `vfio-pci` kernel driver. SPDK's +`scripts/setup.sh` script will automatically select `vfio-pci` in this case. + +However, some devices do not function correctly when bound to `vfio-pci` and instead must be +attached to the `uio_pci_generic` kernel driver. In that case, users should take care to disable +the IOMMU or to set it into passthrough mode prior to running `scripts/setup.sh`. + +To disable the IOMMU or place it into passthrough mode, add `intel_iommu=off` +or `amd_iommu=off` or `intel_iommu=on iommu=pt` to the GRUB command line on +x86_64 system, or add `iommu.passthrough=1` on arm64 systems. + +There are also some instances where a user may not want to use `uio_pci_generic` or the kernel +version they are using has a bug where `uio_pci_generic` [fails to bind to NVMe drives](https://github.com/spdk/spdk/issues/399). +In these cases, users building with the DPDK submodule can build the `igb_uio` kernel module by +supplying `--with-igb-uio-driver` to `./configure`. Upon a successful make, the file will be +located at `dpdk/build/build/kmod/igb_uio.ko`. To ensure that the driver is properly bound, users +should specify `DRIVER_OVERRIDE=/path/to/igb_uio.ko`. diff --git a/src/spdk/doc/template_pg.md b/src/spdk/doc/template_pg.md new file mode 100644 index 000000000..535a980c3 --- /dev/null +++ b/src/spdk/doc/template_pg.md @@ -0,0 +1,80 @@ +# ComponentName Programmer's Guide {#componentname_pg} + +# In this document {#componentname_pg_toc} + +@ref componentname_pg_audience +@ref componentname_pg_intro +@ref componentname_pg_theory +@ref componentname_pg_design +@ref componentname_pg_examples +@ref componentname_pg_config +@ref componentname_pg_component +@ref componentname_pg_sequences + +## Target Audience {#componentname_pg_audience} + +This programmer's guide is intended for developers authoring applications that utilize the SPDK <COMPONENT NAME>. It is +intended to supplement the source code to provide an overall understanding of how to integrate <COMPONENT NAME> into +an application as well as provide some high level insight into how <COMPONENT NAME> works behind the scenes. It is not +intended to serve as a design document or an API reference but in some cases source code snippets and high level +sequences will be discussed. For the latest source code reference refer to the [repo](https://github.com/spdk). + +## Introduction {#componentname_pg_intro} + +Provide some high level description of what this component is, what it does and maybe why it exists. This shouldn't be +a lengthy tutorial or commentary on storage in general or the goodness of SPDK but provide enough information to +set the stage for someone about to write an application to integrate with this component. They won't be totally +starting from scratch if they're at this point, they are by definition a storage application developer if they are +reading this guide. + +## Theory of Operation {#componentname_pg_theory} + +Create subsections here to drill down into the "how" this component works. This isn't a design section however so +avoid getting into too many details, just hit the high level concepts that would leave the developer with a +50K foot overview of the major elements/assumptions/concepts that should have some baseline knowledge about before +they start writing code. + +Some questions to consider when authoring this section: + +* What are the basic primitives that this component exposes? +* How are these primitives related to one another? +* What are the threading rules when using these primitives? +* What are the theoretical performance implications for different scaling vectors? +* Are there any other documents or specifications that the user should be familiar with? +* What are the intended use cases? + +## Design Considerations {#componentname_pg_design} + +Here is where you want to highlight things they need to think about in *their* design. If you have written test code +for this module think about the things that you needed to go learn about to properly interact with this module. Think +about how they need to consider initialization options, threading, limitations, any sort of quirky or non-obious +interactions or module behaviors that might save them some time and effort by thinking about before they start their +design. + +## Examples {#componentname_pg_examples} + +List all of the relevant examples we have in the repo that use this module and describe a little about what they do. + +## Configuration {#componentname_pg_config} + +This section should describe the mechanisms for configuring the component at a high level (i.e. you can configure it +using a config file, or you can configure it using RPC calls over a unix domain socket). It should also talk about +when you can configure it - i.e. at run time or only up front. For specifics about how the RPCs work or the config +file format, link to the appropriate user guide instead of putting that information here. + +## Component Detail {#componentname_pg_component} + +This is where we can provide some design level detail if it makes sense for this module. We don't want to have +design docs as part of SPDK, the overhead and maintenance is too much for open source. We do, however, want +to provide some level of insight into the codebase to promote getting more people involved and understanding +of what the design is all about. The PG is meant to help a developer write their own application but we +can use this section, per module, to test out a way to build out some internal design info as well. I see +this as including an overview of key structures, concepts, etc., of the module itself. So, interesting info +not required to write an application using the module but maybe just enough to provide the next level of +detail into what's behind the scenes to get someone more interested in becoming a community contributor. + +## Sequences {#componentname_pg_sequences} + +If sequence diagrams makes sense for this module, use mscgen to create simple UML-style (they don't need to be 100% +UML compliant) diagrams for the API that an application needs to interact with. Details internal to the component +should not be included. diff --git a/src/spdk/doc/tools.md b/src/spdk/doc/tools.md new file mode 100644 index 000000000..234017ba7 --- /dev/null +++ b/src/spdk/doc/tools.md @@ -0,0 +1,5 @@ +# Tools {#tools} + +- @subpage spdkcli +- @subpage nvme-cli +- @subpage bdevperf diff --git a/src/spdk/doc/two.min.js b/src/spdk/doc/two.min.js new file mode 100644 index 000000000..dc2c0db9e --- /dev/null +++ b/src/spdk/doc/two.min.js @@ -0,0 +1,274 @@ +/* +MIT License + +Copyright (c) 2012 - 2017 jonobr1 / http://jonobr1.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +var $jscomp = $jscomp || {}; $jscomp.scope = {}; $jscomp.ASSUME_ES5 = !1; $jscomp.ASSUME_NO_NATIVE_MAP = !1; $jscomp.ASSUME_NO_NATIVE_SET = !1; $jscomp.defineProperty = $jscomp.ASSUME_ES5 || "function" == typeof Object.defineProperties ? Object.defineProperty : function (c, k, m) { c != Array.prototype && c != Object.prototype && (c[k] = m.value) }; $jscomp.getGlobal = function (c) { return "undefined" != typeof window && window === c ? c : "undefined" != typeof global && null != global ? global : c }; $jscomp.global = $jscomp.getGlobal(this); $jscomp.SYMBOL_PREFIX = "jscomp_symbol_"; +$jscomp.initSymbol=function(){$jscomp.initSymbol=function(){};$jscomp.global.Symbol||($jscomp.global.Symbol=$jscomp.Symbol)};$jscomp.symbolCounter_=0;$jscomp.Symbol=function(c){return $jscomp.SYMBOL_PREFIX+(c||"")+$jscomp.symbolCounter_++}; +$jscomp.initSymbolIterator=function(){$jscomp.initSymbol();var c=$jscomp.global.Symbol.iterator;c||(c=$jscomp.global.Symbol.iterator=$jscomp.global.Symbol("iterator"));"function"!=typeof Array.prototype[c]&&$jscomp.defineProperty(Array.prototype,c,{configurable:!0,writable:!0,value:function(){return $jscomp.arrayIterator(this)}});$jscomp.initSymbolIterator=function(){}};$jscomp.arrayIterator=function(c){var k=0;return $jscomp.iteratorPrototype(function(){return k<c.length?{done:!1,value:c[k++]}:{done:!0}})}; +$jscomp.iteratorPrototype=function(c){$jscomp.initSymbolIterator();c={next:c};c[$jscomp.global.Symbol.iterator]=function(){return this};return c};$jscomp.iteratorFromArray=function(c,k){$jscomp.initSymbolIterator();c instanceof String&&(c+="");var m=0,l={next:function(){if(m<c.length){var h=m++;return{value:k(h,c[h]),done:!1}}l.next=function(){return{done:!0,value:void 0}};return l.next()}};l[Symbol.iterator]=function(){return l};return l}; +$jscomp.polyfill=function(c,k,m,l){if(k){m=$jscomp.global;c=c.split(".");for(l=0;l<c.length-1;l++){var h=c[l];h in m||(m[h]={});m=m[h]}c=c[c.length-1];l=m[c];k=k(l);k!=l&&null!=k&&$jscomp.defineProperty(m,c,{configurable:!0,writable:!0,value:k})}};$jscomp.polyfill("Array.prototype.keys",function(c){return c?c:function(){return $jscomp.iteratorFromArray(this,function(c){return c})}},"es6-impl","es3"); +$jscomp.polyfill("Array.prototype.values",function(c){return c?c:function(){return $jscomp.iteratorFromArray(this,function(c,m){return m})}},"es6","es3");$jscomp.polyfill("Array.prototype.fill",function(c){return c?c:function(c,m,l){var h=this.length||0;0>m&&(m=Math.max(0,h+m));if(null==l||l>h)l=h;l=Number(l);0>l&&(l=Math.max(0,h+l));for(m=Number(m||0);m<l;m++)this[m]=c;return this}},"es6-impl","es3"); +this.Two=function(c){function k(){var a=document.body.getBoundingClientRect(),c=this.width=a.width,a=this.height=a.height;this.renderer.setSize(c,a,this.ratio);this.trigger(p.Events.resize,c,a)}function m(){L(m);for(var a=0;a<p.Instances.length;a++){var c=p.Instances[a];c.playing&&c.update()}}var l="undefined"!=typeof window?window:"undefined"!=typeof global?global:null,h=Object.prototype.toString,d={_indexAmount:0,natural:{slice:Array.prototype.slice,indexOf:Array.prototype.indexOf,keys:Object.keys, +bind:Function.prototype.bind,create:Object.create},identity:function(a){return a},isArguments:function(a){return"[object Arguments]"===h.call(a)},isFunction:function(a){return"[object Function]"===h.call(a)},isString:function(a){return"[object String]"===h.call(a)},isNumber:function(a){return"[object Number]"===h.call(a)},isDate:function(a){return"[object Date]"===h.call(a)},isRegExp:function(a){return"[object RegExp]"===h.call(a)},isError:function(a){return"[object Error]"===h.call(a)},isFinite:function(a){return isFinite(a)&& +!isNaN(parseFloat(a))},isNaN:function(a){return d.isNumber(a)&&a!==+a},isBoolean:function(a){return!0===a||!1===a||"[object Boolean]"===h.call(a)},isNull:function(a){return null===a},isUndefined:function(a){return void 0===a},isEmpty:function(a){return null==a?!0:q&&(d.isArray(a)||d.isString(a)||d.isArguments(a))?0===a.length:0===d.keys(a).length},isElement:function(a){return!(!a||1!==a.nodeType)},isArray:Array.isArray||function(a){return"[object Array]"===h.call(a)},isObject:function(a){var c=typeof a; +return"function"===c||"object"===c&&!!a},toArray:function(a){return a?d.isArray(a)?x.call(a):q(a)?d.map(a,d.identity):d.values(a):[]},range:function(a,c,f){null==c&&(c=a||0,a=0);f=f||1;c=Math.max(Math.ceil((c-a)/f),0);for(var e=Array(c),d=0;d<c;d++,a+=f)e[d]=a;return e},indexOf:function(a,c){if(d.natural.indexOf)return d.natural.indexOf.call(a,c);for(var f=0;f<a.length;f++)if(a[f]===c)return f;return-1},has:function(a,c){return null!=a&&hasOwnProperty.call(a,c)},bind:function(a,c){var f=d.natural.bind; +if(f&&a.bind===f)return f.apply(a,x.call(arguments,1));var e=x.call(arguments,2);return function(){a.apply(c,e)}},extend:function(a){for(var c=x.call(arguments,1),f=0;f<c.length;f++){var e=c[f],d;for(d in e)a[d]=e[d]}return a},defaults:function(a){for(var c=x.call(arguments,1),f=0;f<c.length;f++){var e=c[f],d;for(d in e)void 0===a[d]&&(a[d]=e[d])}return a},keys:function(a){if(!d.isObject(a))return[];if(d.natural.keys)return d.natural.keys(a);var c=[],f;for(f in a)d.has(a,f)&&c.push(f);return c},values:function(a){for(var c= +d.keys(a),f=[],e=0;e<c.length;e++)f.push(a[c[e]]);return f},each:function(a,c,f){f=f||this;for(var e=!q(a)&&d.keys(a),g=(e||a).length,y=0;y<g;y++){var n=e?e[y]:y;c.call(f,a[n],n,a)}return a},map:function(a,c,f){f=f||this;for(var e=!q(a)&&d.keys(a),g=(e||a).length,n=[],y=0;y<g;y++){var t=e?e[y]:y;n[y]=c.call(f,a[t],t,a)}return n},once:function(a){var c=!1;return function(){if(c)return a;c=!0;return a.apply(this,arguments)}},after:function(a,c){return function(){for(;1>--a;)return c.apply(this,arguments)}}, +uniqueId:function(a){var c=++d._indexAmount+"";return a?a+c:c}},e=Math.sin,a=Math.cos,g=Math.atan2,n=Math.sqrt,f=Math.PI,t=f/2,v=Math.pow,B=Math.min,z=Math.max,A=0,x=d.natural.slice,u=l.performance&&l.performance.now?l.performance:Date,r=Math.pow(2,53)-1,q=function(a){a=null==a?void 0:a.length;return"number"==typeof a&&0<=a&&a<=r},w={temp:l.document?l.document.createElement("div"):{},hasEventListeners:d.isFunction(l.addEventListener),bind:function(a,c,f,e){this.hasEventListeners?a.addEventListener(c, +f,!!e):a.attachEvent("on"+c,f);return w},unbind:function(a,c,f,e){w.hasEventListeners?a.removeEventListeners(c,f,!!e):a.detachEvent("on"+c,f);return w},getRequestAnimationFrame:function(){var a=0,c=["ms","moz","webkit","o"],f=l.requestAnimationFrame;if(!f){for(var e=0;e<c.length;e++)f=l[c[e]+"RequestAnimationFrame"]||f;f=f||function(c,f){var e=(new Date).getTime(),d=Math.max(0,16-(e-a));f=l.setTimeout(function(){c(e+d)},d);a=e+d;return f}}f.init=d.once(m);return f}},p=l.Two=function(a){a=d.defaults(a|| +{},{fullscreen:!1,width:640,height:480,type:p.Types.svg,autostart:!1});d.each(a,function(a,c){"fullscreen"!==c&&"autostart"!==c&&(this[c]=a)},this);if(d.isElement(a.domElement)){var c=a.domElement.tagName.toLowerCase();/^(CanvasRenderer-canvas|WebGLRenderer-canvas|SVGRenderer-svg)$/.test(this.type+"-"+c)||(this.type=p.Types[c])}this.renderer=new p[this.type](this);p.Utils.setPlaying.call(this,a.autostart);this.frameCount=0;a.fullscreen?(a=d.bind(k,this),d.extend(document.body.style,{overflow:"hidden", +margin:0,padding:0,top:0,left:0,right:0,bottom:0,position:"fixed"}),d.extend(this.renderer.domElement.style,{display:"block",top:0,left:0,right:0,bottom:0,position:"fixed"}),w.bind(l,"resize",a),a()):d.isElement(a.domElement)||(this.renderer.setSize(a.width,a.height,this.ratio),this.width=a.width,this.height=a.height);this.scene=this.renderer.scene;p.Instances.push(this);L.init()};d.extend(p,{root:l,Array:l.Float32Array||Array,Types:{webgl:"WebGLRenderer",svg:"SVGRenderer",canvas:"CanvasRenderer"}, +Version:"v0.7.0",Identifier:"two_",Properties:{hierarchy:"hierarchy",demotion:"demotion"},Events:{play:"play",pause:"pause",update:"update",render:"render",resize:"resize",change:"change",remove:"remove",insert:"insert",order:"order",load:"load"},Commands:{move:"M",line:"L",curve:"C",close:"Z"},Resolution:8,Instances:[],noConflict:function(){l.Two=c;return this},uniqueId:function(){var a=A;A++;return a},Utils:d.extend(d,{performance:u,defineProperty:function(a){var c="_"+a,f="_flag"+a.charAt(0).toUpperCase()+ +a.slice(1);Object.defineProperty(this,a,{enumerable:!0,get:function(){return this[c]},set:function(a){this[c]=a;this[f]=!0}})},release:function(a){d.isObject(a)&&(d.isFunction(a.unbind)&&a.unbind(),a.vertices&&(d.isFunction(a.vertices.unbind)&&a.vertices.unbind(),d.each(a.vertices,function(a){d.isFunction(a.unbind)&&a.unbind()})),a.children&&d.each(a.children,function(a){p.Utils.release(a)}))},xhr:function(a,c){var f=new XMLHttpRequest;f.open("GET",a);f.onreadystatechange=function(){4===f.readyState&& +200===f.status&&c(f.responseText)};f.send();return f},Curve:{CollinearityEpsilon:v(10,-30),RecursionLimit:16,CuspLimit:0,Tolerance:{distance:.25,angle:0,epsilon:.01},abscissas:[[.5773502691896257],[0,.7745966692414834],[.33998104358485626,.8611363115940526],[0,.5384693101056831,.906179845938664],[.2386191860831969,.6612093864662645,.932469514203152],[0,.4058451513773972,.7415311855993945,.9491079123427585],[.1834346424956498,.525532409916329,.7966664774136267,.9602898564975363],[0,.3242534234038089, +.6133714327005904,.8360311073266358,.9681602395076261],[.14887433898163122,.4333953941292472,.6794095682990244,.8650633666889845,.9739065285171717],[0,.26954315595234496,.5190961292068118,.7301520055740494,.8870625997680953,.978228658146057],[.1252334085114689,.3678314989981802,.5873179542866175,.7699026741943047,.9041172563704749,.9815606342467192],[0,.2304583159551348,.44849275103644687,.6423493394403402,.8015780907333099,.9175983992229779,.9841830547185881],[.10805494870734367,.31911236892788974, +.5152486363581541,.6872929048116855,.827201315069765,.9284348836635735,.9862838086968123],[0,.20119409399743451,.3941513470775634,.5709721726085388,.7244177313601701,.8482065834104272,.937273392400706,.9879925180204854],[.09501250983763744,.2816035507792589,.45801677765722737,.6178762444026438,.755404408355003,.8656312023878318,.9445750230732326,.9894009349916499]],weights:[[1],[.8888888888888888,.5555555555555556],[.6521451548625461,.34785484513745385],[.5688888888888889,.47862867049936647,.23692688505618908], +[.46791393457269104,.3607615730481386,.17132449237917036],[.4179591836734694,.3818300505051189,.27970539148927664,.1294849661688697],[.362683783378362,.31370664587788727,.22238103445337448,.10122853629037626],[.3302393550012598,.31234707704000286,.26061069640293544,.1806481606948574,.08127438836157441],[.29552422471475287,.26926671930999635,.21908636251598204,.1494513491505806,.06667134430868814],[.2729250867779006,.26280454451024665,.23319376459199048,.18629021092773426,.1255803694649046,.05566856711617366], +[.24914704581340277,.2334925365383548,.20316742672306592,.16007832854334622,.10693932599531843,.04717533638651183],[.2325515532308739,.22628318026289723,.2078160475368885,.17814598076194574,.13887351021978725,.09212149983772845,.04048400476531588],[.2152638534631578,.2051984637212956,.18553839747793782,.15720316715819355,.12151857068790319,.08015808715976021,.03511946033175186],[.2025782419255613,.19843148532711158,.1861610000155622,.16626920581699392,.13957067792615432,.10715922046717194,.07036604748810812, +.03075324199611727],[.1894506104550685,.18260341504492358,.16915651939500254,.14959598881657674,.12462897125553388,.09515851168249279,.062253523938647894,.027152459411754096]]},devicePixelRatio:l.devicePixelRatio||1,getBackingStoreRatio:function(a){return a.webkitBackingStorePixelRatio||a.mozBackingStorePixelRatio||a.msBackingStorePixelRatio||a.oBackingStorePixelRatio||a.backingStorePixelRatio||1},getRatio:function(a){return p.Utils.devicePixelRatio/O(a)},setPlaying:function(a){this.playing=!!a;return this}, +getComputedMatrix:function(a,c){c=c&&c.identity()||new p.Matrix;for(var f=[];a&&a._matrix;)f.push(a._matrix),a=a.parent;f.reverse();d.each(f,function(a){a=a.elements;c.multiply(a[0],a[1],a[2],a[3],a[4],a[5],a[6],a[7],a[8],a[9])});return c},deltaTransformPoint:function(a,c,f){return new p.Vector(c*a.a+f*a.c+0,c*a.b+f*a.d+0)},decomposeMatrix:function(a){var c=p.Utils.deltaTransformPoint(a,0,1),f=p.Utils.deltaTransformPoint(a,1,0),c=180/Math.PI*Math.atan2(c.y,c.x)-90;return{translateX:a.e,translateY:a.f, +scaleX:Math.sqrt(a.a*a.a+a.b*a.b),scaleY:Math.sqrt(a.c*a.c+a.d*a.d),skewX:c,skewY:180/Math.PI*Math.atan2(f.y,f.x),rotation:c}},applySvgAttributes:function(a,c){var f={},e={},g;if(getComputedStyle){var n=getComputedStyle(a);for(g=n.length;g--;){var t=n[g];var y=n[t];void 0!==y&&(e[t]=y)}}for(g=a.attributes.length;g--;)y=a.attributes[g],f[y.nodeName]=y.value;d.isUndefined(e.opacity)||(e["stroke-opacity"]=e.opacity,e["fill-opacity"]=e.opacity);d.extend(e,f);e.visible=!(d.isUndefined(e.display)&&"none"=== +e.display)||d.isUndefined(e.visibility)&&"hidden"===e.visibility;for(t in e)switch(y=e[t],t){case "transform":if("none"===y)break;if(null===(a.getCTM?a.getCTM():null))break;f=p.Utils.decomposeMatrix(a.getCTM());c.translation.set(f.translateX,f.translateY);c.rotation=f.rotation;c.scale=f.scaleX;f=parseFloat((e.x+"").replace("px"));g=parseFloat((e.y+"").replace("px"));f&&(c.translation.x=f);g&&(c.translation.y=g);break;case "visible":c.visible=y;break;case "stroke-linecap":c.cap=y;break;case "stroke-linejoin":c.join= +y;break;case "stroke-miterlimit":c.miter=y;break;case "stroke-width":c.linewidth=parseFloat(y);break;case "stroke-opacity":case "fill-opacity":case "opacity":c.opacity=parseFloat(y);break;case "fill":case "stroke":/url\(\#.*\)/i.test(y)?c[t]=this.getById(y.replace(/url\(\#(.*)\)/i,"$1")):c[t]="none"===y?"transparent":y;break;case "id":c.id=y;break;case "class":c.classList=y.split(" ")}return c},read:{svg:function(){return p.Utils.read.g.apply(this,arguments)},g:function(a){var c=new p.Group;p.Utils.applySvgAttributes.call(this, +a,c);for(var f=0,e=a.childNodes.length;f<e;f++){var d=a.childNodes[f],g=d.nodeName;if(!g)return;g=g.replace(/svg\:/ig,"").toLowerCase();g in p.Utils.read&&(d=p.Utils.read[g].call(c,d),c.add(d))}return c},polygon:function(a,c){var f=[];a.getAttribute("points").replace(/(-?[\d\.?]+)[,|\s](-?[\d\.?]+)/g,function(a,c,e){f.push(new p.Anchor(parseFloat(c),parseFloat(e)))});c=(new p.Path(f,!c)).noStroke();c.fill="black";return p.Utils.applySvgAttributes.call(this,a,c)},polyline:function(a){return p.Utils.read.polygon.call(this, +a,!0)},path:function(a){var c=a.getAttribute("d"),f=new p.Anchor,e,g,n=!1,t=!1,y=c.match(/[a-df-z][^a-df-z]*/ig),h=y.length-1;d.each(y.slice(0),function(a,c){var f=a[0],e=f.toLowerCase(),g=a.slice(1).trim().split(/[\s,]+|(?=\s?[+\-])/),d=[],n;0>=c&&(y=[]);switch(e){case "h":case "v":1<g.length&&(n=1);break;case "m":case "l":case "t":2<g.length&&(n=2);break;case "s":case "q":4<g.length&&(n=4);break;case "c":6<g.length&&(n=6)}if(n){a=0;c=g.length;for(e=0;a<c;a+=n){var t=f;if(0<e)switch(f){case "m":t= +"l";break;case "M":t="L"}d.push([t].concat(g.slice(a,a+n)).join(" "));e++}y=Array.prototype.concat.apply(y,d)}else y.push(a)});var m=[];d.each(y,function(a,c){var y=a[0],D=y.toLowerCase();g=a.slice(1).trim();g=g.replace(/(-?\d+(?:\.\d*)?)[eE]([+\-]?\d+)/g,function(a,c,f){return parseFloat(c)*v(10,f)});g=g.split(/[\s,]+|(?=\s?[+\-])/);t=y===D;switch(D){case "z":if(c>=h)n=!0;else{a=f.x;c=f.y;var k=new p.Anchor(a,c,void 0,void 0,void 0,void 0,p.Commands.close)}break;case "m":case "l":a=parseFloat(g[0]); +c=parseFloat(g[1]);k=new p.Anchor(a,c,void 0,void 0,void 0,void 0,"m"===D?p.Commands.move:p.Commands.line);t&&k.addSelf(f);f=k;break;case "h":case "v":c="h"===D?"x":"y";D="x"===c?"y":"x";k=new p.Anchor(void 0,void 0,void 0,void 0,void 0,void 0,p.Commands.line);k[c]=parseFloat(g[0]);k[D]=f[D];t&&(k[c]+=f[c]);f=k;break;case "c":case "s":k=f.x;c=f.y;e||(e=new p.Vector);if("c"===D){y=parseFloat(g[0]);var B=parseFloat(g[1]);var l=parseFloat(g[2]);var z=parseFloat(g[3]);D=parseFloat(g[4]);a=parseFloat(g[5])}else D= +M(f,e,t),y=D.x,B=D.y,l=parseFloat(g[0]),z=parseFloat(g[1]),D=parseFloat(g[2]),a=parseFloat(g[3]);t&&(y+=k,B+=c,l+=k,z+=c,D+=k,a+=c);d.isObject(f.controls)||p.Anchor.AppendCurveProperties(f);f.controls.right.set(y-f.x,B-f.y);f=k=new p.Anchor(D,a,l-D,z-a,void 0,void 0,p.Commands.curve);e=k.controls.left;break;case "t":case "q":k=f.x;c=f.y;e||(e=new p.Vector);e.isZero()?(y=k,B=c):(y=e.x,c=e.y);"q"===D?(l=parseFloat(g[0]),z=parseFloat(g[1]),D=parseFloat(g[1]),a=parseFloat(g[2])):(D=M(f,e,t),l=D.x,z=D.y, +D=parseFloat(g[0]),a=parseFloat(g[1]));t&&(y+=k,B+=c,l+=k,z+=c,D+=k,a+=c);d.isObject(f.controls)||p.Anchor.AppendCurveProperties(f);f.controls.right.set(y-f.x,B-f.y);f=k=new p.Anchor(D,a,l-D,z-a,void 0,void 0,p.Commands.curve);e=k.controls.left;break;case "a":k=f.x;c=f.y;var J=parseFloat(g[0]),x=parseFloat(g[1]);B=parseFloat(g[2])*Math.PI/180;y=parseFloat(g[3]);l=parseFloat(g[4]);D=parseFloat(g[5]);a=parseFloat(g[6]);t&&(D+=k,a+=c);var u=(D-k)/2,A=(a-c)/2;z=u*Math.cos(B)+A*Math.sin(B);var u=-u*Math.sin(B)+ +A*Math.cos(B),A=J*J,q=x*x,r=z*z,K=u*u,Q=r/A+K/q;1<Q&&(J*=Math.sqrt(Q),x*=Math.sqrt(Q));q=Math.sqrt((A*q-A*K-q*r)/(A*K+q*r));d.isNaN(q)?q=0:y!=l&&0<q&&(q*=-1);A=q*J*u/x;q=-q*x*z/J;k=A*Math.cos(B)-q*Math.sin(B)+(k+D)/2;var r=A*Math.sin(B)+q*Math.cos(B)+(c+a)/2,w=function(a,c){return(a[0]*c[0]+a[1]*c[1])/(Math.sqrt(Math.pow(a[0],2)+Math.pow(a[1],2))*Math.sqrt(Math.pow(c[0],2)+Math.pow(c[1],2)))};c=function(a,c){return(a[0]*c[1]<a[1]*c[0]?-1:1)*Math.acos(w(a,c))};var S=c([1,0],[(z-A)/J,(u-q)/x]),K=[(z- +A)/J,(u-q)/x];z=[(-z-A)/J,(-u-q)/x];var C=c(K,z);-1>=w(K,z)&&(C=Math.PI);1<=w(K,z)&&(C=0);y&&(C=I(C,2*Math.PI));l&&0<C&&(C-=2*Math.PI);var R=p.Resolution,T=(new p.Matrix).translate(k,r).rotate(B);k=d.map(d.range(R),function(a){a=(1-a/(R-1))*C+S;a=T.multiply(J*Math.cos(a),x*Math.sin(a),1);return new p.Anchor(a.x,a.y,!1,!1,!1,!1,p.Commands.line)});k.push(new p.Anchor(D,a,!1,!1,!1,!1,p.Commands.line));f=k[k.length-1];e=f.controls.left}k&&(d.isArray(k)?m=m.concat(k):m.push(k))});if(!(1>=m.length)){c= +(new p.Path(m,n,void 0,!0)).noStroke();c.fill="black";var k=c.getBoundingClientRect(!0);k.centroid={x:k.left+k.width/2,y:k.top+k.height/2};d.each(c.vertices,function(a){a.subSelf(k.centroid)});c.translation.addSelf(k.centroid);return p.Utils.applySvgAttributes.call(this,a,c)}},circle:function(a){var c=parseFloat(a.getAttribute("cx")),f=parseFloat(a.getAttribute("cy")),e=parseFloat(a.getAttribute("r")),c=(new p.Circle(c,f,e)).noStroke();c.fill="black";return p.Utils.applySvgAttributes.call(this,a, +c)},ellipse:function(a){var c=parseFloat(a.getAttribute("cx")),f=parseFloat(a.getAttribute("cy")),e=parseFloat(a.getAttribute("rx")),g=parseFloat(a.getAttribute("ry")),c=(new p.Ellipse(c,f,e,g)).noStroke();c.fill="black";return p.Utils.applySvgAttributes.call(this,a,c)},rect:function(a){var c=parseFloat(a.getAttribute("x"))||0,f=parseFloat(a.getAttribute("y"))||0,e=parseFloat(a.getAttribute("width")),g=parseFloat(a.getAttribute("height")),c=(new p.Rectangle(c+e/2,f+g/2,e,g)).noStroke();c.fill="black"; +return p.Utils.applySvgAttributes.call(this,a,c)},line:function(a){var c=parseFloat(a.getAttribute("x1")),f=parseFloat(a.getAttribute("y1")),e=parseFloat(a.getAttribute("x2")),g=parseFloat(a.getAttribute("y2")),c=(new p.Line(c,f,e,g)).noFill();return p.Utils.applySvgAttributes.call(this,a,c)},lineargradient:function(a){for(var c,f=parseFloat(a.getAttribute("x1")),e=parseFloat(a.getAttribute("y1")),g=parseFloat(a.getAttribute("x2")),n=parseFloat(a.getAttribute("y2")),t=(g+f)/2,h=(n+e)/2,y=[],v=0;v< +a.children.length;v++){c=a.children[v];var k=parseFloat(c.getAttribute("offset")),m=c.getAttribute("stop-color"),B=c.getAttribute("stop-opacity"),l=c.getAttribute("style");d.isNull(m)&&(m=(c=l?l.match(/stop\-color\:\s?([\#a-fA-F0-9]*)/):!1)&&1<c.length?c[1]:void 0);d.isNull(B)&&(B=(c=l?l.match(/stop\-opacity\:\s?([0-9\.\-]*)/):!1)&&1<c.length?parseFloat(c[1]):1);y.push(new p.Gradient.Stop(k,m,B))}f=new p.LinearGradient(f-t,e-h,g-t,n-h,y);return p.Utils.applySvgAttributes.call(this,a,f)},radialgradient:function(a){var c= +parseFloat(a.getAttribute("cx"))||0,f=parseFloat(a.getAttribute("cy"))||0,e=parseFloat(a.getAttribute("r")),g=parseFloat(a.getAttribute("fx")),n=parseFloat(a.getAttribute("fy"));d.isNaN(g)&&(g=c);d.isNaN(n)&&(n=f);for(var t=Math.abs(c+g)/2,h=Math.abs(f+n)/2,v=[],y=0;y<a.children.length;y++){var k=a.children[y];var m=parseFloat(k.getAttribute("offset")),B=k.getAttribute("stop-color"),l=k.getAttribute("stop-opacity"),z=k.getAttribute("style");d.isNull(B)&&(B=(k=z?z.match(/stop\-color\:\s?([\#a-fA-F0-9]*)/): +!1)&&1<k.length?k[1]:void 0);d.isNull(l)&&(l=(k=z?z.match(/stop\-opacity\:\s?([0-9\.\-]*)/):!1)&&1<k.length?parseFloat(k[1]):1);v.push(new p.Gradient.Stop(m,B,l))}c=new p.RadialGradient(c-t,f-h,e,v,g-t,n-h);return p.Utils.applySvgAttributes.call(this,a,c)}},subdivide:function(a,c,f,e,g,n,t,h,v){v=v||p.Utils.Curve.RecursionLimit;var y=v+1;return a===t&&c===h?[new p.Anchor(t,h)]:d.map(d.range(0,y),function(d){var v=d/y;d=N(v,a,f,g,t);v=N(v,c,e,n,h);return new p.Anchor(d,v)})},getPointOnCubicBezier:function(a, +c,f,e,g){var d=1-a;return d*d*d*c+3*d*d*a*f+3*d*a*a*e+a*a*a*g},getCurveLength:function(a,c,f,e,g,d,t,v,h){if(a===f&&c===e&&g===t&&d===v)return a=t-a,c=v-c,n(a*a+c*c);var y=9*(f-g)+3*(t-a),k=6*(a+g)-12*f,B=3*(f-a),m=9*(e-d)+3*(v-c),l=6*(c+d)-12*e,D=3*(e-c);return P(function(a){var c=(y*a+k)*a+B;a=(m*a+l)*a+D;return n(c*c+a*a)},0,1,h||p.Utils.Curve.RecursionLimit)},integrate:function(a,c,f,e){var g=p.Utils.Curve.abscissas[e-2],d=p.Utils.Curve.weights[e-2];f=.5*(f-c);c=f+c;var n=0,t=e+1>>1;for(e=e&1? +d[n++]*a(c):0;n<t;){var v=f*g[n];e+=d[n++]*(a(c+v)+a(c-v))}return f*e},getCurveFromPoints:function(a,c){for(var f=a.length,e=f-1,g=0;g<f;g++){var n=a[g];d.isObject(n.controls)||p.Anchor.AppendCurveProperties(n);var t=c?I(g-1,f):z(g-1,0),v=c?I(g+1,f):B(g+1,e);F(a[t],n,a[v]);n._command=0===g?p.Commands.move:p.Commands.curve;n.controls.left.x=d.isNumber(n.controls.left.x)?n.controls.left.x:n.x;n.controls.left.y=d.isNumber(n.controls.left.y)?n.controls.left.y:n.y;n.controls.right.x=d.isNumber(n.controls.right.x)? +n.controls.right.x:n.x;n.controls.right.y=d.isNumber(n.controls.right.y)?n.controls.right.y:n.y}},getControlPoints:function(c,g,n){var v=G(c,g),h=G(n,g);c=E(c,g);n=E(n,g);var k=(v+h)/2;g.u=d.isObject(g.controls.left)?g.controls.left:new p.Vector(0,0);g.v=d.isObject(g.controls.right)?g.controls.right:new p.Vector(0,0);if(.0001>c||.0001>n)return g._relative||(g.controls.left.copy(g),g.controls.right.copy(g)),g;c*=.33;n*=.33;k=h<v?k+t:k-t;g.controls.left.x=a(k)*c;g.controls.left.y=e(k)*c;k-=f;g.controls.right.x= +a(k)*n;g.controls.right.y=e(k)*n;g._relative||(g.controls.left.x+=g.x,g.controls.left.y+=g.y,g.controls.right.x+=g.x,g.controls.right.y+=g.y);return g},getReflection:function(a,c,f){return new p.Vector(2*a.x-(c.x+a.x)-(f?a.x:0),2*a.y-(c.y+a.y)-(f?a.y:0))},getAnchorsFromArcData:function(a,c,f,e,g,n,t){(new p.Matrix).translate(a.x,a.y).rotate(c);var v=p.Resolution;return d.map(d.range(v),function(a){a=(a+1)/v;t&&(a=1-a);a=a*n+g;a=new p.Anchor(f*Math.cos(a),e*Math.sin(a));p.Anchor.AppendCurveProperties(a); +a.command=p.Commands.line;return a})},ratioBetween:function(a,c){return(a.x*c.x+a.y*c.y)/(a.length()*c.length())},angleBetween:function(a,c){if(4<=arguments.length){var f=arguments[0]-arguments[2];var e=arguments[1]-arguments[3];return g(e,f)}f=a.x-c.x;e=a.y-c.y;return g(e,f)},distanceBetweenSquared:function(a,c){var f=a.x-c.x;a=a.y-c.y;return f*f+a*a},distanceBetween:function(a,c){return n(H(a,c))},lerp:function(a,c,f){return f*(c-a)+a},toFixed:function(a){return Math.floor(1E3*a)/1E3},mod:function(a, +c){for(;0>a;)a+=c;return a%c},Collection:function(){Array.call(this);1<arguments.length?Array.prototype.push.apply(this,arguments):arguments[0]&&Array.isArray(arguments[0])&&Array.prototype.push.apply(this,arguments[0])},Error:function(a){this.name="two.js";this.message=a},Events:{on:function(a,c){this._events||(this._events={});(this._events[a]||(this._events[a]=[])).push(c);return this},off:function(a,c){if(!this._events)return this;if(!a&&!c)return this._events={},this;for(var f=a?[a]:d.keys(this._events), +e=0,g=f.length;e<g;e++){a=f[e];var n=this._events[a];if(n){var t=[];if(c)for(var v=0,h=n.length;v<h;v++){var k=n[v],k=k.callback?k.callback:k;c&&c!==k&&t.push(k)}this._events[a]=t}}return this},trigger:function(a){if(!this._events)return this;var c=x.call(arguments,1),f=this._events[a];f&&C(this,f,c);return this},listen:function(a,c,f){var e=this;if(a){var g=function(){f.apply(e,arguments)};g.obj=a;g.name=c;g.callback=f;a.on(c,g)}return this},ignore:function(a,c,f){a.off(c,f);return this}}})});p.Utils.Events.bind= +p.Utils.Events.on;p.Utils.Events.unbind=p.Utils.Events.off;var C=function(a,c,f){switch(f.length){case 0:var e=function(e){c[e].call(a,f[0])};break;case 1:e=function(e){c[e].call(a,f[0],f[1])};break;case 2:e=function(e){c[e].call(a,f[0],f[1],f[2])};break;case 3:e=function(e){c[e].call(a,f[0],f[1],f[2],f[3])};break;default:e=function(e){c[e].apply(a,f)}}for(var g=0;g<c.length;g++)e(g)};p.Utils.Error.prototype=Error();p.Utils.Error.prototype.constructor=p.Utils.Error;p.Utils.Collection.prototype=[]; +p.Utils.Collection.prototype.constructor=p.Utils.Collection;d.extend(p.Utils.Collection.prototype,p.Utils.Events,{pop:function(){var a=Array.prototype.pop.apply(this,arguments);this.trigger(p.Events.remove,[a]);return a},shift:function(){var a=Array.prototype.shift.apply(this,arguments);this.trigger(p.Events.remove,[a]);return a},push:function(){var a=Array.prototype.push.apply(this,arguments);this.trigger(p.Events.insert,arguments);return a},unshift:function(){var a=Array.prototype.unshift.apply(this, +arguments);this.trigger(p.Events.insert,arguments);return a},splice:function(){var a=Array.prototype.splice.apply(this,arguments);this.trigger(p.Events.remove,a);if(2<arguments.length){var c=this.slice(arguments[0],arguments[0]+arguments.length-2);this.trigger(p.Events.insert,c);this.trigger(p.Events.order)}return a},sort:function(){Array.prototype.sort.apply(this,arguments);this.trigger(p.Events.order);return this},reverse:function(){Array.prototype.reverse.apply(this,arguments);this.trigger(p.Events.order); +return this}});var E=p.Utils.distanceBetween,H=p.Utils.distanceBetweenSquared,G=p.Utils.angleBetween,F=p.Utils.getControlPoints,I=p.Utils.mod,O=p.Utils.getBackingStoreRatio,N=p.Utils.getPointOnCubicBezier,P=p.Utils.integrate,M=p.Utils.getReflection;d.extend(p.prototype,p.Utils.Events,{appendTo:function(a){a.appendChild(this.renderer.domElement);return this},play:function(){p.Utils.setPlaying.call(this,!0);return this.trigger(p.Events.play)},pause:function(){this.playing=!1;return this.trigger(p.Events.pause)}, +update:function(){var a=!!this._lastFrame,c=u.now();this.frameCount++;a&&(this.timeDelta=parseFloat((c-this._lastFrame).toFixed(3)));this._lastFrame=c;var a=this.width,c=this.height,f=this.renderer;a===f.width&&c===f.height||f.setSize(a,c,this.ratio);this.trigger(p.Events.update,this.frameCount,this.timeDelta);return this.render()},render:function(){this.renderer.render();return this.trigger(p.Events.render,this.frameCount)},add:function(a){var c=a;c instanceof Array||(c=d.toArray(arguments));this.scene.add(c); +return this},remove:function(a){var c=a;c instanceof Array||(c=d.toArray(arguments));this.scene.remove(c);return this},clear:function(){this.scene.remove(d.toArray(this.scene.children));return this},makeLine:function(a,c,f,e){a=new p.Line(a,c,f,e);this.scene.add(a);return a},makeRectangle:function(a,c,f,e){a=new p.Rectangle(a,c,f,e);this.scene.add(a);return a},makeRoundedRectangle:function(a,c,f,e,g){a=new p.RoundedRectangle(a,c,f,e,g);this.scene.add(a);return a},makeCircle:function(a,c,f){a=new p.Circle(a, +c,f);this.scene.add(a);return a},makeEllipse:function(a,c,f,e){a=new p.Ellipse(a,c,f,e);this.scene.add(a);return a},makeStar:function(a,c,f,e,g){a=new p.Star(a,c,f,e,g);this.scene.add(a);return a},makeCurve:function(a){var c=arguments.length,f=a;if(!d.isArray(a))for(var f=[],e=0;e<c;e+=2){var g=arguments[e];if(!d.isNumber(g))break;f.push(new p.Anchor(g,arguments[e+1]))}c=arguments[c-1];f=new p.Path(f,!(d.isBoolean(c)&&c),!0);c=f.getBoundingClientRect();f.center().translation.set(c.left+c.width/2, +c.top+c.height/2);this.scene.add(f);return f},makePolygon:function(a,c,f,e){a=new p.Polygon(a,c,f,e);this.scene.add(a);return a},makeArcSegment:function(a,c,f,e,g,d,n){a=new p.ArcSegment(a,c,f,e,g,d,n);this.scene.add(a);return a},makePath:function(a){var c=arguments.length,f=a;if(!d.isArray(a))for(var f=[],e=0;e<c;e+=2){var g=arguments[e];if(!d.isNumber(g))break;f.push(new p.Anchor(g,arguments[e+1]))}c=arguments[c-1];f=new p.Path(f,!(d.isBoolean(c)&&c));c=f.getBoundingClientRect();f.center().translation.set(c.left+ +c.width/2,c.top+c.height/2);this.scene.add(f);return f},makeText:function(a,c,f,e){a=new p.Text(a,c,f,e);this.add(a);return a},makeLinearGradient:function(a,c,f,e){var g=x.call(arguments,4),g=new p.LinearGradient(a,c,f,e,g);this.add(g);return g},makeRadialGradient:function(a,c,f){var e=x.call(arguments,3),e=new p.RadialGradient(a,c,f,e);this.add(e);return e},makeSprite:function(a,c,f,e,g,d,n){a=new p.Sprite(a,c,f,e,g,d);n&&a.play();this.add(a);return a},makeImageSequence:function(a,c,f,e,g){a=new p.ImageSequence(a, +c,f,e);g&&a.play();this.add(a);return a},makeTexture:function(a,c){return new p.Texture(a,c)},makeGroup:function(a){var c=a;c instanceof Array||(c=d.toArray(arguments));var f=new p.Group;this.scene.add(f);f.add(c);return f},interpret:function(a,c){var f=a.tagName.toLowerCase();if(!(f in p.Utils.read))return null;a=p.Utils.read[f].call(this,a);c&&a instanceof p.Group?this.add(a.children):this.add(a);return a},load:function(a,c){var f=[],e;if(/.*\.svg/ig.test(a))return p.Utils.xhr(a,d.bind(function(a){w.temp.innerHTML= +a;for(e=0;e<w.temp.children.length;e++)g=w.temp.children[e],f.push(this.interpret(g));c(1>=f.length?f[0]:f,1>=w.temp.children.length?w.temp.children[0]:w.temp.children)},this)),this;w.temp.innerHTML=a;for(e=0;e<w.temp.children.length;e++){var g=w.temp.children[e];f.push(this.interpret(g))}c(1>=f.length?f[0]:f,1>=w.temp.children.length?w.temp.children[0]:w.temp.children);return this}});var L=w.getRequestAnimationFrame();"function"===typeof define&&define.amd?define("two",[],function(){return p}):"undefined"!= +typeof module&&module.exports&&(module.exports=p);return p}(("undefined"!==typeof global?global:this).Two);(function(c){var k=c.Utils;c=c.Registry=function(){this.map={}};k.extend(c,{});k.extend(c.prototype,{add:function(c,k){this.map[c]=k;return this},remove:function(c){delete this.map[c];return this},get:function(c){return this.map[c]},contains:function(c){return c in this.map}})})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils,m=c.Vector=function(c,a){this.x=c||0;this.y=a||0};k.extend(m,{zero:new c.Vector});k.extend(m.prototype,c.Utils.Events,{set:function(c,a){this.x=c;this.y=a;return this},copy:function(c){this.x=c.x;this.y=c.y;return this},clear:function(){this.y=this.x=0;return this},clone:function(){return new m(this.x,this.y)},add:function(c,a){this.x=c.x+a.x;this.y=c.y+a.y;return this},addSelf:function(c){this.x+=c.x;this.y+=c.y;return this},sub:function(c,a){this.x=c.x-a.x;this.y=c.y- +a.y;return this},subSelf:function(c){this.x-=c.x;this.y-=c.y;return this},multiplySelf:function(c){this.x*=c.x;this.y*=c.y;return this},multiplyScalar:function(c){this.x*=c;this.y*=c;return this},divideScalar:function(c){c?(this.x/=c,this.y/=c):this.set(0,0);return this},negate:function(){return this.multiplyScalar(-1)},dot:function(c){return this.x*c.x+this.y*c.y},lengthSquared:function(){return this.x*this.x+this.y*this.y},length:function(){return Math.sqrt(this.lengthSquared())},normalize:function(){return this.divideScalar(this.length())}, +distanceTo:function(c){return Math.sqrt(this.distanceToSquared(c))},distanceToSquared:function(c){var a=this.x-c.x;c=this.y-c.y;return a*a+c*c},setLength:function(c){return this.normalize().multiplyScalar(c)},equals:function(c,a){a="undefined"===typeof a?.0001:a;return this.distanceTo(c)<a},lerp:function(c,a){return this.set((c.x-this.x)*a+this.x,(c.y-this.y)*a+this.y)},isZero:function(c){c="undefined"===typeof c?.0001:c;return this.length()<c},toString:function(){return this.x+", "+this.y},toObject:function(){return{x:this.x, +y:this.y}},rotate:function(c){var a=Math.cos(c);c=Math.sin(c);this.x=this.x*a-this.y*c;this.y=this.x*c+this.y*a;return this}});var l={set:function(e,a){this._x=e;this._y=a;return this.trigger(c.Events.change)},copy:function(e){this._x=e.x;this._y=e.y;return this.trigger(c.Events.change)},clear:function(){this._y=this._x=0;return this.trigger(c.Events.change)},clone:function(){return new m(this._x,this._y)},add:function(e,a){this._x=e.x+a.x;this._y=e.y+a.y;return this.trigger(c.Events.change)},addSelf:function(e){this._x+= +e.x;this._y+=e.y;return this.trigger(c.Events.change)},sub:function(e,a){this._x=e.x-a.x;this._y=e.y-a.y;return this.trigger(c.Events.change)},subSelf:function(e){this._x-=e.x;this._y-=e.y;return this.trigger(c.Events.change)},multiplySelf:function(e){this._x*=e.x;this._y*=e.y;return this.trigger(c.Events.change)},multiplyScalar:function(e){this._x*=e;this._y*=e;return this.trigger(c.Events.change)},divideScalar:function(e){return e?(this._x/=e,this._y/=e,this.trigger(c.Events.change)):this.clear()}, +negate:function(){return this.multiplyScalar(-1)},dot:function(c){return this._x*c.x+this._y*c.y},lengthSquared:function(){return this._x*this._x+this._y*this._y},length:function(){return Math.sqrt(this.lengthSquared())},normalize:function(){return this.divideScalar(this.length())},distanceTo:function(c){return Math.sqrt(this.distanceToSquared(c))},distanceToSquared:function(c){var a=this._x-c.x;c=this._y-c.y;return a*a+c*c},setLength:function(c){return this.normalize().multiplyScalar(c)},equals:function(c, +a){a="undefined"===typeof a?.0001:a;return this.distanceTo(c)<a},lerp:function(c,a){return this.set((c.x-this._x)*a+this._x,(c.y-this._y)*a+this._y)},isZero:function(c){c="undefined"===typeof c?.0001:c;return this.length()<c},toString:function(){return this._x+", "+this._y},toObject:function(){return{x:this._x,y:this._y}},rotate:function(c){var a=Math.cos(c);c=Math.sin(c);this._x=this._x*a-this._y*c;this._y=this._x*c+this._y*a;return this}},h={enumerable:!0,get:function(){return this._x},set:function(e){this._x= +e;this.trigger(c.Events.change,"x")}},d={enumerable:!0,get:function(){return this._y},set:function(e){this._y=e;this.trigger(c.Events.change,"y")}};c.Vector.prototype.bind=c.Vector.prototype.on=function(){this._bound||(this._x=this.x,this._y=this.y,Object.defineProperty(this,"x",h),Object.defineProperty(this,"y",d),k.extend(this,l),this._bound=!0);c.Utils.Events.bind.apply(this,arguments);return this}})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Commands,m=c.Utils,l=c.Anchor=function(d,e,a,g,n,f,t){c.Vector.call(this,d,e);this._broadcast=m.bind(function(){this.trigger(c.Events.change)},this);this._command=t||k.move;this._relative=!0;if(!t)return this;l.AppendCurveProperties(this);m.isNumber(a)&&(this.controls.left.x=a);m.isNumber(g)&&(this.controls.left.y=g);m.isNumber(n)&&(this.controls.right.x=n);m.isNumber(f)&&(this.controls.right.y=f)};m.extend(l,{AppendCurveProperties:function(d){d.controls={left:new c.Vector(0, +0),right:new c.Vector(0,0)}}});var h={listen:function(){m.isObject(this.controls)||l.AppendCurveProperties(this);this.controls.left.bind(c.Events.change,this._broadcast);this.controls.right.bind(c.Events.change,this._broadcast);return this},ignore:function(){this.controls.left.unbind(c.Events.change,this._broadcast);this.controls.right.unbind(c.Events.change,this._broadcast);return this},clone:function(){var d=this.controls,d=new c.Anchor(this.x,this.y,d&&d.left.x,d&&d.left.y,d&&d.right.x,d&&d.right.y, +this.command);d.relative=this._relative;return d},toObject:function(){var c={x:this.x,y:this.y};this._command&&(c.command=this._command);this._relative&&(c.relative=this._relative);this.controls&&(c.controls={left:this.controls.left.toObject(),right:this.controls.right.toObject()});return c},toString:function(){return this.controls?[this._x,this._y,this.controls.left.x,this.controls.left.y,this.controls.right.x,this.controls.right.y].join(", "):[this._x,this._y].join(", ")}};Object.defineProperty(l.prototype, +"command",{enumerable:!0,get:function(){return this._command},set:function(d){this._command=d;this._command!==k.curve||m.isObject(this.controls)||l.AppendCurveProperties(this);return this.trigger(c.Events.change)}});Object.defineProperty(l.prototype,"relative",{enumerable:!0,get:function(){return this._relative},set:function(d){if(this._relative==d)return this;this._relative=!!d;return this.trigger(c.Events.change)}});m.extend(l.prototype,c.Vector.prototype,h);c.Anchor.prototype.bind=c.Anchor.prototype.on= +function(){c.Vector.prototype.bind.apply(this,arguments);m.extend(this,h)};c.Anchor.prototype.unbind=c.Anchor.prototype.off=function(){c.Vector.prototype.unbind.apply(this,arguments);m.extend(this,h)}})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=Math.cos,m=Math.sin,l=Math.tan,h=c.Utils,d=c.Matrix=function(e,a,g,d,f,t){this.elements=new c.Array(9);var n=e;h.isArray(n)||(n=h.toArray(arguments));this.identity().set(n)};h.extend(d,{Identity:[1,0,0,0,1,0,0,0,1],Multiply:function(e,a,g){if(3>=a.length){g=a[0]||0;var d=a[1]||0;a=a[2]||0;return{x:e[0]*g+e[1]*d+e[2]*a,y:e[3]*g+e[4]*d+e[5]*a,z:e[6]*g+e[7]*d+e[8]*a}}var d=e[0],f=e[1],t=e[2],v=e[3],h=e[4],k=e[5],m=e[6],l=e[7];e=e[8];var u=a[0],r=a[1],q=a[2],w=a[3],p=a[4],C=a[5],E= +a[6],H=a[7];a=a[8];g=g||new c.Array(9);g[0]=d*u+f*w+t*E;g[1]=d*r+f*p+t*H;g[2]=d*q+f*C+t*a;g[3]=v*u+h*w+k*E;g[4]=v*r+h*p+k*H;g[5]=v*q+h*C+k*a;g[6]=m*u+l*w+e*E;g[7]=m*r+l*p+e*H;g[8]=m*q+l*C+e*a;return g}});h.extend(d.prototype,c.Utils.Events,{set:function(e){var a=e;h.isArray(a)||(a=h.toArray(arguments));h.extend(this.elements,a);return this.trigger(c.Events.change)},identity:function(){this.set(d.Identity);return this},multiply:function(e,a,g,d,f,t,v,k,m){var n=arguments,B=n.length;if(1>=B)return h.each(this.elements, +function(a,c){this.elements[c]=a*e},this),this.trigger(c.Events.change);if(3>=B)return e=e||0,a=a||0,g=g||0,f=this.elements,{x:f[0]*e+f[1]*a+f[2]*g,y:f[3]*e+f[4]*a+f[5]*g,z:f[6]*e+f[7]*a+f[8]*g};var l=this.elements,B=l[0],z=l[1],q=l[2],w=l[3],p=l[4],C=l[5],E=l[6],H=l[7],l=l[8],G=n[0],F=n[1],I=n[2],O=n[3],N=n[4],P=n[5],M=n[6],L=n[7],n=n[8];this.elements[0]=B*G+z*O+q*M;this.elements[1]=B*F+z*N+q*L;this.elements[2]=B*I+z*P+q*n;this.elements[3]=w*G+p*O+C*M;this.elements[4]=w*F+p*N+C*L;this.elements[5]= +w*I+p*P+C*n;this.elements[6]=E*G+H*O+l*M;this.elements[7]=E*F+H*N+l*L;this.elements[8]=E*I+H*P+l*n;return this.trigger(c.Events.change)},inverse:function(e){var a=this.elements;e=e||new c.Matrix;var g=a[0],d=a[1],f=a[2],t=a[3],v=a[4],h=a[5],k=a[6],l=a[7],a=a[8],m=a*v-h*l,u=-a*t+h*k,r=l*t-v*k,q=g*m+d*u+f*r;if(!q)return null;q=1/q;e.elements[0]=m*q;e.elements[1]=(-a*d+f*l)*q;e.elements[2]=(h*d-f*v)*q;e.elements[3]=u*q;e.elements[4]=(a*g-f*k)*q;e.elements[5]=(-h*g+f*t)*q;e.elements[6]=r*q;e.elements[7]= +(-l*g+d*k)*q;e.elements[8]=(v*g-d*t)*q;return e},scale:function(c,a){1>=arguments.length&&(a=c);return this.multiply(c,0,0,0,a,0,0,0,1)},rotate:function(c){var a=k(c);c=m(c);return this.multiply(a,-c,0,c,a,0,0,0,1)},translate:function(c,a){return this.multiply(1,0,c,0,1,a,0,0,1)},skewX:function(c){c=l(c);return this.multiply(1,c,0,0,1,0,0,0,1)},skewY:function(c){c=l(c);return this.multiply(1,0,0,c,1,0,0,0,1)},toString:function(c){var a=[];this.toArray(c,a);return a.join(" ")},toArray:function(c,a){var g= +this.elements,e=!!a,f=parseFloat(g[0].toFixed(3)),d=parseFloat(g[1].toFixed(3)),v=parseFloat(g[2].toFixed(3)),h=parseFloat(g[3].toFixed(3)),k=parseFloat(g[4].toFixed(3)),l=parseFloat(g[5].toFixed(3));if(c){c=parseFloat(g[6].toFixed(3));var m=parseFloat(g[7].toFixed(3)),g=parseFloat(g[8].toFixed(3));if(e){a[0]=f;a[1]=h;a[2]=c;a[3]=d;a[4]=k;a[5]=m;a[6]=v;a[7]=l;a[8]=g;return}return[f,h,c,d,k,m,v,l,g]}if(e)a[0]=f,a[1]=h,a[2]=d,a[3]=k,a[4]=v,a[5]=l;else return[f,h,d,k,v,l]},clone:function(){var e=this.elements[0]; +var a=this.elements[1];var g=this.elements[2];var d=this.elements[3];var f=this.elements[4];return new c.Matrix(e,a,g,d,f,this.elements[5],this.elements[6],this.elements[7],this.elements[8])}})})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils.mod,m=c.Utils.toFixed,l=c.Utils,h={version:1.1,ns:"http://www.w3.org/2000/svg",xlink:"http://www.w3.org/1999/xlink",alignments:{left:"start",center:"middle",right:"end"},createElement:function(c,a){var g=document.createElementNS(h.ns,c);"svg"===c&&(a=l.defaults(a||{},{version:h.version}));l.isEmpty(a)||h.setAttributes(g,a);return g},setAttributes:function(c,a){for(var g=Object.keys(a),e=0;e<g.length;e++)/href/.test(g[e])?c.setAttributeNS(h.xlink,g[e],a[g[e]]):c.setAttribute(g[e], +a[g[e]]);return this},removeAttributes:function(c,a){for(var g in a)c.removeAttribute(g);return this},toString:function(e,a){for(var g=e.length,d=g-1,f,t="",h=0;h<g;h++){var l=e[h],z=a?k(h-1,g):Math.max(h-1,0);a&&k(h+1,g);var A=e[z];var x=m(l._x);var u=m(l._y);switch(l._command){case c.Commands.close:var r=c.Commands.close;break;case c.Commands.curve:var q=A.controls&&A.controls.right||c.Vector.zero;r=l.controls&&l.controls.left||c.Vector.zero;A._relative?(z=m(q.x+A.x),A=m(q.y+A.y)):(z=m(q.x),A=m(q.y)); +if(l._relative){q=m(r.x+l.x);var w=m(r.y+l.y)}else q=m(r.x),w=m(r.y);r=(0===h?c.Commands.move:c.Commands.curve)+" "+z+" "+A+" "+q+" "+w+" "+x+" "+u;break;case c.Commands.move:f=l;r=c.Commands.move+" "+x+" "+u;break;default:r=l._command+" "+x+" "+u}h>=d&&a&&(l._command===c.Commands.curve&&(u=f,A=l.controls&&l.controls.right||l,x=u.controls&&u.controls.left||u,l._relative?(z=m(A.x+l.x),A=m(A.y+l.y)):(z=m(A.x),A=m(A.y)),u._relative?(q=m(x.x+u.x),w=m(x.y+u.y)):(q=m(x.x),w=m(x.y)),x=m(u.x),u=m(u.y),r+= +" C "+z+" "+A+" "+q+" "+w+" "+x+" "+u),r+=" Z");t+=r+" "}return t},getClip:function(c){var a=c._renderer.clip;if(!a){for(var g=c;g.parent;)g=g.parent;a=c._renderer.clip=h.createElement("clipPath");g.defs.appendChild(a)}return a},group:{appendChild:function(c){var a=c._renderer.elem;if(a){var g=a.nodeName;!g||/(radial|linear)gradient/i.test(g)||c._clip||this.elem.appendChild(a)}},removeChild:function(c){var a=c._renderer.elem;a&&a.parentNode==this.elem&&a.nodeName&&(c._clip||this.elem.removeChild(a))}, +orderChild:function(c){this.elem.appendChild(c._renderer.elem)},renderChild:function(c){h[c._renderer.type].render.call(c,this)},render:function(c){this._update();if(0===this._opacity&&!this._flagOpacity)return this;this._renderer.elem||(this._renderer.elem=h.createElement("g",{id:this.id}),c.appendChild(this._renderer.elem));var a={domElement:c,elem:this._renderer.elem};(this._matrix.manual||this._flagMatrix)&&this._renderer.elem.setAttribute("transform","matrix("+this._matrix.toString()+")");for(var g= +0;g<this.children.length;g++){var e=this.children[g];h[e._renderer.type].render.call(e,c)}this._flagOpacity&&this._renderer.elem.setAttribute("opacity",this._opacity);this._flagAdditions&&this.additions.forEach(h.group.appendChild,a);this._flagSubtractions&&this.subtractions.forEach(h.group.removeChild,a);this._flagOrder&&this.children.forEach(h.group.orderChild,a);this._flagMask&&(this._mask?this._renderer.elem.setAttribute("clip-path","url(#"+this._mask.id+")"):this._renderer.elem.removeAttribute("clip-path")); +return this.flagReset()}},path:{render:function(c){this._update();if(0===this._opacity&&!this._flagOpacity)return this;var a={};if(this._matrix.manual||this._flagMatrix)a.transform="matrix("+this._matrix.toString()+")";if(this._flagVertices){var g=h.toString(this._vertices,this._closed);a.d=g}this._fill&&this._fill._renderer&&(this._fill._update(),h[this._fill._renderer.type].render.call(this._fill,c,!0));this._flagFill&&(a.fill=this._fill&&this._fill.id?"url(#"+this._fill.id+")":this._fill);this._stroke&& +this._stroke._renderer&&(this._stroke._update(),h[this._stroke._renderer.type].render.call(this._stroke,c,!0));this._flagStroke&&(a.stroke=this._stroke&&this._stroke.id?"url(#"+this._stroke.id+")":this._stroke);this._flagLinewidth&&(a["stroke-width"]=this._linewidth);this._flagOpacity&&(a["stroke-opacity"]=this._opacity,a["fill-opacity"]=this._opacity);this._flagVisible&&(a.visibility=this._visible?"visible":"hidden");this._flagCap&&(a["stroke-linecap"]=this._cap);this._flagJoin&&(a["stroke-linejoin"]= +this._join);this._flagMiter&&(a["stroke-miterlimit"]=this._miter);this._renderer.elem?h.setAttributes(this._renderer.elem,a):(a.id=this.id,this._renderer.elem=h.createElement("path",a),c.appendChild(this._renderer.elem));this._flagClip&&(c=h.getClip(this),a=this._renderer.elem,this._clip?(a.removeAttribute("id"),c.setAttribute("id",this.id),c.appendChild(a)):(c.removeAttribute("id"),a.setAttribute("id",this.id),this.parent._renderer.elem.appendChild(a)));return this.flagReset()}},text:{render:function(c){this._update(); +var a={};if(this._matrix.manual||this._flagMatrix)a.transform="matrix("+this._matrix.toString()+")";this._flagFamily&&(a["font-family"]=this._family);this._flagSize&&(a["font-size"]=this._size);this._flagLeading&&(a["line-height"]=this._leading);this._flagAlignment&&(a["text-anchor"]=h.alignments[this._alignment]||this._alignment);this._flagBaseline&&(a["alignment-baseline"]=a["dominant-baseline"]=this._baseline);this._flagStyle&&(a["font-style"]=this._style);this._flagWeight&&(a["font-weight"]=this._weight); +this._flagDecoration&&(a["text-decoration"]=this._decoration);this._fill&&this._fill._renderer&&(this._fill._update(),h[this._fill._renderer.type].render.call(this._fill,c,!0));this._flagFill&&(a.fill=this._fill&&this._fill.id?"url(#"+this._fill.id+")":this._fill);this._stroke&&this._stroke._renderer&&(this._stroke._update(),h[this._stroke._renderer.type].render.call(this._stroke,c,!0));this._flagStroke&&(a.stroke=this._stroke&&this._stroke.id?"url(#"+this._stroke.id+")":this._stroke);this._flagLinewidth&& +(a["stroke-width"]=this._linewidth);this._flagOpacity&&(a.opacity=this._opacity);this._flagVisible&&(a.visibility=this._visible?"visible":"hidden");this._renderer.elem?h.setAttributes(this._renderer.elem,a):(a.id=this.id,this._renderer.elem=h.createElement("text",a),c.defs.appendChild(this._renderer.elem));this._flagClip&&(c=h.getClip(this),a=this._renderer.elem,this._clip?(a.removeAttribute("id"),c.setAttribute("id",this.id),c.appendChild(a)):(c.removeAttribute("id"),a.setAttribute("id",this.id), +this.parent._renderer.elem.appendChild(a)));this._flagValue&&(this._renderer.elem.textContent=this._value);return this.flagReset()}},"linear-gradient":{render:function(c,a){a||this._update();a={};this._flagEndPoints&&(a.x1=this.left._x,a.y1=this.left._y,a.x2=this.right._x,a.y2=this.right._y);this._flagSpread&&(a.spreadMethod=this._spread);this._renderer.elem?h.setAttributes(this._renderer.elem,a):(a.id=this.id,a.gradientUnits="userSpaceOnUse",this._renderer.elem=h.createElement("linearGradient",a), +c.defs.appendChild(this._renderer.elem));if(this._flagStops){if(c=this._renderer.elem.childNodes.length!==this.stops.length)this._renderer.elem.childNodes.length=0;for(a=0;a<this.stops.length;a++){var g=this.stops[a],d={};g._flagOffset&&(d.offset=100*g._offset+"%");g._flagColor&&(d["stop-color"]=g._color);g._flagOpacity&&(d["stop-opacity"]=g._opacity);g._renderer.elem?h.setAttributes(g._renderer.elem,d):g._renderer.elem=h.createElement("stop",d);c&&this._renderer.elem.appendChild(g._renderer.elem); +g.flagReset()}}return this.flagReset()}},"radial-gradient":{render:function(c,a){a||this._update();a={};this._flagCenter&&(a.cx=this.center._x,a.cy=this.center._y);this._flagFocal&&(a.fx=this.focal._x,a.fy=this.focal._y);this._flagRadius&&(a.r=this._radius);this._flagSpread&&(a.spreadMethod=this._spread);this._renderer.elem?h.setAttributes(this._renderer.elem,a):(a.id=this.id,a.gradientUnits="userSpaceOnUse",this._renderer.elem=h.createElement("radialGradient",a),c.defs.appendChild(this._renderer.elem)); +if(this._flagStops){if(c=this._renderer.elem.childNodes.length!==this.stops.length)this._renderer.elem.childNodes.length=0;for(a=0;a<this.stops.length;a++){var g=this.stops[a],d={};g._flagOffset&&(d.offset=100*g._offset+"%");g._flagColor&&(d["stop-color"]=g._color);g._flagOpacity&&(d["stop-opacity"]=g._opacity);g._renderer.elem?h.setAttributes(g._renderer.elem,d):g._renderer.elem=h.createElement("stop",d);c&&this._renderer.elem.appendChild(g._renderer.elem);g.flagReset()}}return this.flagReset()}}, +texture:{render:function(d,a){a||this._update();a={};var g={x:0,y:0},e=this.image;if(this._flagLoaded&&this.loaded)switch(e.nodeName.toLowerCase()){case "canvas":g.href=g["xlink:href"]=e.toDataURL("image/png");break;case "img":case "image":g.href=g["xlink:href"]=this.src}if(this._flagOffset||this._flagLoaded||this._flagScale)a.x=this._offset.x,a.y=this._offset.y,e&&(a.x-=e.width/2,a.y-=e.height/2,this._scale instanceof c.Vector?(a.x*=this._scale.x,a.y*=this._scale.y):(a.x*=this._scale,a.y*=this._scale)), +0<a.x&&(a.x*=-1),0<a.y&&(a.y*=-1);if(this._flagScale||this._flagLoaded||this._flagRepeat)if(a.width=0,a.height=0,e){g.width=a.width=e.width;g.height=a.height=e.height;switch(this._repeat){case "no-repeat":a.width+=1,a.height+=1}this._scale instanceof c.Vector?(a.width*=this._scale.x,a.height*=this._scale.y):(a.width*=this._scale,a.height*=this._scale)}if(this._flagScale||this._flagLoaded)this._renderer.image?l.isEmpty(g)||h.setAttributes(this._renderer.image,g):this._renderer.image=h.createElement("image", +g);this._renderer.elem?l.isEmpty(a)||h.setAttributes(this._renderer.elem,a):(a.id=this.id,a.patternUnits="userSpaceOnUse",this._renderer.elem=h.createElement("pattern",a),d.defs.appendChild(this._renderer.elem));this._renderer.elem&&this._renderer.image&&!this._renderer.appended&&(this._renderer.elem.appendChild(this._renderer.image),this._renderer.appended=!0);return this.flagReset()}}},d=c[c.Types.svg]=function(d){this.domElement=d.domElement||h.createElement("svg");this.scene=new c.Group;this.scene.parent= +this;this.defs=h.createElement("defs");this.domElement.appendChild(this.defs);this.domElement.defs=this.defs;this.domElement.style.overflow="hidden"};l.extend(d,{Utils:h});l.extend(d.prototype,c.Utils.Events,{setSize:function(c,a){this.width=c;this.height=a;h.setAttributes(this.domElement,{width:c,height:a});return this},render:function(){h.group.render.call(this.scene,this.domElement);return this}})})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils.mod,m=c.Utils.toFixed,l=c.Utils.getRatio,h=c.Utils,d=function(a){return 1==a[0]&&0==a[3]&&0==a[1]&&1==a[4]&&0==a[2]&&0==a[5]},e={isHidden:/(none|transparent)/i,alignments:{left:"start",middle:"center",right:"end"},shim:function(a){a.tagName="canvas";a.nodeType=1;return a},group:{renderChild:function(a){e[a._renderer.type].render.call(a,this.ctx,!0,this.clip)},render:function(a){this._update();var c=this._matrix.elements,f=this.parent;this._renderer.opacity=this._opacity* +(f&&f._renderer?f._renderer.opacity:1);var f=d(c),g=this._mask;this._renderer.context||(this._renderer.context={});this._renderer.context.ctx=a;f||(a.save(),a.transform(c[0],c[3],c[1],c[4],c[2],c[5]));g&&e[g._renderer.type].render.call(g,a,!0);if(0<this.opacity&&0!==this.scale)for(c=0;c<this.children.length;c++)g=this.children[c],e[g._renderer.type].render.call(g,a);f||a.restore();return this.flagReset()}},path:{render:function(a,n,f){this._update();var g=this._matrix.elements;var v=this._stroke; +var l=this._linewidth;var z=this._fill;var A=this._opacity*this.parent._renderer.opacity;var x=this._visible;var u=this._cap;var r=this._join;var q=this._miter;var w=this._closed;var p=this._vertices;var C=p.length;var E=C-1;var H=d(g);var G=this._clip;if(!n&&(!x||G))return this;H||(a.save(),a.transform(g[0],g[3],g[1],g[4],g[2],g[5]));z&&(h.isString(z)?a.fillStyle=z:(e[z._renderer.type].render.call(z,a),a.fillStyle=z._renderer.effect));v&&(h.isString(v)?a.strokeStyle=v:(e[v._renderer.type].render.call(v, +a),a.strokeStyle=v._renderer.effect));l&&(a.lineWidth=l);q&&(a.miterLimit=q);r&&(a.lineJoin=r);u&&(a.lineCap=u);h.isNumber(A)&&(a.globalAlpha=A);a.beginPath();for(g=0;g<p.length;g++)switch(n=p[g],x=m(n._x),u=m(n._y),n._command){case c.Commands.close:a.closePath();break;case c.Commands.curve:A=w?k(g-1,C):Math.max(g-1,0);w&&k(g+1,C);r=p[A];q=r.controls&&r.controls.right||c.Vector.zero;var F=n.controls&&n.controls.left||c.Vector.zero;r._relative?(A=q.x+m(r._x),q=q.y+m(r._y)):(A=m(q.x),q=m(q.y));n._relative? +(r=F.x+m(n._x),F=F.y+m(n._y)):(r=m(F.x),F=m(F.y));a.bezierCurveTo(A,q,r,F,x,u);g>=E&&w&&(u=I,r=n.controls&&n.controls.right||c.Vector.zero,x=u.controls&&u.controls.left||c.Vector.zero,n._relative?(A=r.x+m(n._x),q=r.y+m(n._y)):(A=m(r.x),q=m(r.y)),u._relative?(r=x.x+m(u._x),F=x.y+m(u._y)):(r=m(x.x),F=m(x.y)),x=m(u._x),u=m(u._y),a.bezierCurveTo(A,q,r,F,x,u));break;case c.Commands.line:a.lineTo(x,u);break;case c.Commands.move:var I=n;a.moveTo(x,u)}w&&a.closePath();if(!G&&!f){if(!e.isHidden.test(z)){if(w= +z._renderer&&z._renderer.offset)a.save(),a.translate(-z._renderer.offset.x,-z._renderer.offset.y),a.scale(z._renderer.scale.x,z._renderer.scale.y);a.fill();w&&a.restore()}if(!e.isHidden.test(v)){if(w=v._renderer&&v._renderer.offset)a.save(),a.translate(-v._renderer.offset.x,-v._renderer.offset.y),a.scale(v._renderer.scale.x,v._renderer.scale.y),a.lineWidth=l/v._renderer.scale.x;a.stroke();w&&a.restore()}}H||a.restore();G&&!f&&a.clip();return this.flagReset()}},text:{render:function(a,c,f){this._update(); +var g=this._matrix.elements,n=this._stroke,k=this._linewidth,l=this._fill,A=this._opacity*this.parent._renderer.opacity,x=this._visible,u=d(g),r=l._renderer&&l._renderer.offset&&n._renderer&&n._renderer.offset,q=this._clip;if(!c&&(!x||q))return this;u||(a.save(),a.transform(g[0],g[3],g[1],g[4],g[2],g[5]));r||(a.font=[this._style,this._weight,this._size+"px/"+this._leading+"px",this._family].join(" "));a.textAlign=e.alignments[this._alignment]||this._alignment;a.textBaseline=this._baseline;l&&(h.isString(l)? +a.fillStyle=l:(e[l._renderer.type].render.call(l,a),a.fillStyle=l._renderer.effect));n&&(h.isString(n)?a.strokeStyle=n:(e[n._renderer.type].render.call(n,a),a.strokeStyle=n._renderer.effect));k&&(a.lineWidth=k);h.isNumber(A)&&(a.globalAlpha=A);q||f||(e.isHidden.test(l)||(l._renderer&&l._renderer.offset?(c=m(l._renderer.scale.x),g=m(l._renderer.scale.y),a.save(),a.translate(-m(l._renderer.offset.x),-m(l._renderer.offset.y)),a.scale(c,g),c=this._size/l._renderer.scale.y,g=this._leading/l._renderer.scale.y, +a.font=[this._style,this._weight,m(c)+"px/",m(g)+"px",this._family].join(" "),c=l._renderer.offset.x/l._renderer.scale.x,l=l._renderer.offset.y/l._renderer.scale.y,a.fillText(this.value,m(c),m(l)),a.restore()):a.fillText(this.value,0,0)),e.isHidden.test(n)||(n._renderer&&n._renderer.offset?(c=m(n._renderer.scale.x),g=m(n._renderer.scale.y),a.save(),a.translate(-m(n._renderer.offset.x),-m(n._renderer.offset.y)),a.scale(c,g),c=this._size/n._renderer.scale.y,g=this._leading/n._renderer.scale.y,a.font= +[this._style,this._weight,m(c)+"px/",m(g)+"px",this._family].join(" "),c=n._renderer.offset.x/n._renderer.scale.x,l=n._renderer.offset.y/n._renderer.scale.y,n=k/n._renderer.scale.x,a.lineWidth=m(n),a.strokeText(this.value,m(c),m(l)),a.restore()):a.strokeText(this.value,0,0)));u||a.restore();q&&!f&&a.clip();return this.flagReset()}},"linear-gradient":{render:function(a){this._update();if(!this._renderer.effect||this._flagEndPoints||this._flagStops)for(this._renderer.effect=a.createLinearGradient(this.left._x, +this.left._y,this.right._x,this.right._y),a=0;a<this.stops.length;a++){var c=this.stops[a];this._renderer.effect.addColorStop(c._offset,c._color)}return this.flagReset()}},"radial-gradient":{render:function(a){this._update();if(!this._renderer.effect||this._flagCenter||this._flagFocal||this._flagRadius||this._flagStops)for(this._renderer.effect=a.createRadialGradient(this.center._x,this.center._y,0,this.focal._x,this.focal._y,this._radius),a=0;a<this.stops.length;a++){var c=this.stops[a];this._renderer.effect.addColorStop(c._offset, +c._color)}return this.flagReset()}},texture:{render:function(a){this._update();var d=this.image;if(!this._renderer.effect||(this._flagLoaded||this._flagImage||this._flagVideo||this._flagRepeat)&&this.loaded)this._renderer.effect=a.createPattern(this.image,this._repeat);if(this._flagOffset||this._flagLoaded||this._flagScale)this._renderer.offset instanceof c.Vector||(this._renderer.offset=new c.Vector),this._renderer.offset.x=-this._offset.x,this._renderer.offset.y=-this._offset.y,d&&(this._renderer.offset.x+= +d.width/2,this._renderer.offset.y+=d.height/2,this._scale instanceof c.Vector?(this._renderer.offset.x*=this._scale.x,this._renderer.offset.y*=this._scale.y):(this._renderer.offset.x*=this._scale,this._renderer.offset.y*=this._scale));if(this._flagScale||this._flagLoaded)this._renderer.scale instanceof c.Vector||(this._renderer.scale=new c.Vector),this._scale instanceof c.Vector?this._renderer.scale.copy(this._scale):this._renderer.scale.set(this._scale,this._scale);return this.flagReset()}}},a=c[c.Types.canvas]= +function(a){var d=!1!==a.smoothing;this.domElement=a.domElement||document.createElement("canvas");this.ctx=this.domElement.getContext("2d");this.overdraw=a.overdraw||!1;h.isUndefined(this.ctx.imageSmoothingEnabled)||(this.ctx.imageSmoothingEnabled=d);this.scene=new c.Group;this.scene.parent=this};h.extend(a,{Utils:e});h.extend(a.prototype,c.Utils.Events,{setSize:function(a,c,f){this.width=a;this.height=c;this.ratio=h.isUndefined(f)?l(this.ctx):f;this.domElement.width=a*this.ratio;this.domElement.height= +c*this.ratio;this.domElement.style&&h.extend(this.domElement.style,{width:a+"px",height:c+"px"});return this},render:function(){var a=1===this.ratio;a||(this.ctx.save(),this.ctx.scale(this.ratio,this.ratio));this.overdraw||this.ctx.clearRect(0,0,this.width,this.height);e.group.render.call(this.scene,this.ctx);a||this.ctx.restore();return this}})})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.root,m=c.Matrix.Multiply,l=c.Utils.mod,h=[1,0,0,0,1,0,0,0,1],d=new c.Array(9),e=c.Utils.getRatio,a=c.Utils.toFixed,g=c.Utils,n={isHidden:/(none|transparent)/i,canvas:k.document?k.document.createElement("canvas"):{getContext:g.identity},alignments:{left:"start",middle:"center",right:"end"},matrix:new c.Matrix,uv:new c.Array([0,0,1,0,0,1,0,1,1,0,1,1]),group:{removeChild:function(a,c){if(a.children)for(var f=0;f<a.children.length;f++)n.group.removeChild(a.children[f],c);else c.deleteTexture(a._renderer.texture), +delete a._renderer.texture},renderChild:function(a){n[a._renderer.type].render.call(a,this.gl,this.program)},render:function(a,g){this._update();var f=this.parent,e=f._matrix&&f._matrix.manual||f._flagMatrix,h=this._matrix.manual||this._flagMatrix;if(e||h)this._renderer.matrix||(this._renderer.matrix=new c.Array(9)),this._matrix.toArray(!0,d),m(d,f._renderer.matrix,this._renderer.matrix),this._renderer.scale=this._scale*f._renderer.scale,e&&(this._flagMatrix=!0);this._mask&&(a.enable(a.STENCIL_TEST), +a.stencilFunc(a.ALWAYS,1,1),a.colorMask(!1,!1,!1,!0),a.stencilOp(a.KEEP,a.KEEP,a.INCR),n[this._mask._renderer.type].render.call(this._mask,a,g,this),a.colorMask(!0,!0,!0,!0),a.stencilFunc(a.NOTEQUAL,0,1),a.stencilOp(a.KEEP,a.KEEP,a.KEEP));this._flagOpacity=f._flagOpacity||this._flagOpacity;this._renderer.opacity=this._opacity*(f&&f._renderer?f._renderer.opacity:1);if(this._flagSubtractions)for(f=0;f<this.subtractions.length;f++)n.group.removeChild(this.subtractions[f],a);this.children.forEach(n.group.renderChild, +{gl:a,program:g});this._mask&&(a.colorMask(!1,!1,!1,!1),a.stencilOp(a.KEEP,a.KEEP,a.DECR),n[this._mask._renderer.type].render.call(this._mask,a,g,this),a.colorMask(!0,!0,!0,!0),a.stencilFunc(a.NOTEQUAL,0,1),a.stencilOp(a.KEEP,a.KEEP,a.KEEP),a.disable(a.STENCIL_TEST));return this.flagReset()}},path:{updateCanvas:function(f){var d=f._vertices;var e=this.canvas;var h=this.ctx;var k=f._renderer.scale;var m=f._stroke,x=f._linewidth,u=f._fill;var r=f._renderer.opacity||f._opacity;var q=f._cap;var w=f._join; +var p=f._miter;var C=f._closed,E=d.length,H=E-1;e.width=Math.max(Math.ceil(f._renderer.rect.width*k),1);e.height=Math.max(Math.ceil(f._renderer.rect.height*k),1);var G=f._renderer.rect.centroid,F=G.x,G=G.y;h.clearRect(0,0,e.width,e.height);u&&(g.isString(u)?h.fillStyle=u:(n[u._renderer.type].render.call(u,h,f),h.fillStyle=u._renderer.effect));m&&(g.isString(m)?h.strokeStyle=m:(n[m._renderer.type].render.call(m,h,f),h.strokeStyle=m._renderer.effect));x&&(h.lineWidth=x);p&&(h.miterLimit=p);w&&(h.lineJoin= +w);q&&(h.lineCap=q);g.isNumber(r)&&(h.globalAlpha=r);h.save();h.scale(k,k);h.translate(F,G);h.beginPath();for(f=0;f<d.length;f++)switch(b=d[f],k=a(b._x),r=a(b._y),b._command){case c.Commands.close:h.closePath();break;case c.Commands.curve:e=C?l(f-1,E):Math.max(f-1,0);C&&l(f+1,E);q=d[e];w=q.controls&&q.controls.right||c.Vector.zero;p=b.controls&&b.controls.left||c.Vector.zero;q._relative?(e=a(w.x+q._x),w=a(w.y+q._y)):(e=a(w.x),w=a(w.y));b._relative?(q=a(p.x+b._x),p=a(p.y+b._y)):(q=a(p.x),p=a(p.y)); +h.bezierCurveTo(e,w,q,p,k,r);f>=H&&C&&(r=I,q=b.controls&&b.controls.right||c.Vector.zero,k=r.controls&&r.controls.left||c.Vector.zero,b._relative?(e=a(q.x+b._x),w=a(q.y+b._y)):(e=a(q.x),w=a(q.y)),r._relative?(q=a(k.x+r._x),p=a(k.y+r._y)):(q=a(k.x),p=a(k.y)),k=a(r._x),r=a(r._y),h.bezierCurveTo(e,w,q,p,k,r));break;case c.Commands.line:h.lineTo(k,r);break;case c.Commands.move:var I=b;h.moveTo(k,r)}C&&h.closePath();if(!n.isHidden.test(u)){if(d=u._renderer&&u._renderer.offset)h.save(),h.translate(-u._renderer.offset.x, +-u._renderer.offset.y),h.scale(u._renderer.scale.x,u._renderer.scale.y);h.fill();d&&h.restore()}if(!n.isHidden.test(m)){if(d=m._renderer&&m._renderer.offset)h.save(),h.translate(-m._renderer.offset.x,-m._renderer.offset.y),h.scale(m._renderer.scale.x,m._renderer.scale.y),h.lineWidth=x/m._renderer.scale.x;h.stroke();d&&h.restore()}h.restore()},getBoundingClientRect:function(a,c,d){var f=Infinity,e=-Infinity,n=Infinity,h=-Infinity;a.forEach(function(a){var c=a.x,d=a.y,g=a.controls;n=Math.min(d,n);f= +Math.min(c,f);e=Math.max(c,e);h=Math.max(d,h);if(a.controls){var k=g.left;var t=g.right;k&&t&&(g=a._relative?k.x+c:k.x,k=a._relative?k.y+d:k.y,c=a._relative?t.x+c:t.x,a=a._relative?t.y+d:t.y,g&&k&&c&&a&&(n=Math.min(k,a,n),f=Math.min(g,c,f),e=Math.max(g,c,e),h=Math.max(k,a,h)))}});g.isNumber(c)&&(n-=c,f-=c,e+=c,h+=c);d.top=n;d.left=f;d.right=e;d.bottom=h;d.width=e-f;d.height=h-n;d.centroid||(d.centroid={});d.centroid.x=-f;d.centroid.y=-n},render:function(a,g,e){if(!this._visible||!this._opacity)return this; +this._update();var f=this.parent,h=this._matrix.manual||this._flagMatrix,k=this._flagVertices||this._flagFill||this._fill instanceof c.LinearGradient&&(this._fill._flagSpread||this._fill._flagStops||this._fill._flagEndPoints)||this._fill instanceof c.RadialGradient&&(this._fill._flagSpread||this._fill._flagStops||this._fill._flagRadius||this._fill._flagCenter||this._fill._flagFocal)||this._fill instanceof c.Texture&&(this._fill._flagLoaded&&this._fill.loaded||this._fill._flagOffset||this._fill._flagScale)|| +this._stroke instanceof c.LinearGradient&&(this._stroke._flagSpread||this._stroke._flagStops||this._stroke._flagEndPoints)||this._stroke instanceof c.RadialGradient&&(this._stroke._flagSpread||this._stroke._flagStops||this._stroke._flagRadius||this._stroke._flagCenter||this._stroke._flagFocal)||this._stroke instanceof c.Texture&&(this._stroke._flagLoaded&&this._stroke.loaded||this._stroke._flagOffset||this._fill._flagScale)||this._flagStroke||this._flagLinewidth||this._flagOpacity||f._flagOpacity|| +this._flagVisible||this._flagCap||this._flagJoin||this._flagMiter||this._flagScale||!this._renderer.texture;if(f._matrix.manual||f._flagMatrix||h)this._renderer.matrix||(this._renderer.matrix=new c.Array(9)),this._matrix.toArray(!0,d),m(d,f._renderer.matrix,this._renderer.matrix),this._renderer.scale=this._scale*f._renderer.scale;k&&(this._renderer.rect||(this._renderer.rect={}),this._renderer.triangles||(this._renderer.triangles=new c.Array(12)),this._renderer.opacity=this._opacity*f._renderer.opacity, +n.path.getBoundingClientRect(this._vertices,this._linewidth,this._renderer.rect),n.getTriangles(this._renderer.rect,this._renderer.triangles),n.updateBuffer.call(n,a,this,g),n.updateTexture.call(n,a,this));if(!this._clip||e)return a.bindBuffer(a.ARRAY_BUFFER,this._renderer.textureCoordsBuffer),a.vertexAttribPointer(g.textureCoords,2,a.FLOAT,!1,0,0),a.bindTexture(a.TEXTURE_2D,this._renderer.texture),a.uniformMatrix3fv(g.matrix,!1,this._renderer.matrix),a.bindBuffer(a.ARRAY_BUFFER,this._renderer.buffer), +a.vertexAttribPointer(g.position,2,a.FLOAT,!1,0,0),a.drawArrays(a.TRIANGLES,0,6),this.flagReset()}},text:{updateCanvas:function(c){var f=this.canvas,d=this.ctx,e=c._renderer.scale,h=c._stroke,k=c._linewidth*e,l=c._fill,m=c._renderer.opacity||c._opacity;f.width=Math.max(Math.ceil(c._renderer.rect.width*e),1);f.height=Math.max(Math.ceil(c._renderer.rect.height*e),1);var r=c._renderer.rect.centroid,q=r.x,r=r.y,w=l._renderer&&l._renderer.offset&&h._renderer&&h._renderer.offset;d.clearRect(0,0,f.width, +f.height);w||(d.font=[c._style,c._weight,c._size+"px/"+c._leading+"px",c._family].join(" "));d.textAlign="center";d.textBaseline="middle";l&&(g.isString(l)?d.fillStyle=l:(n[l._renderer.type].render.call(l,d,c),d.fillStyle=l._renderer.effect));h&&(g.isString(h)?d.strokeStyle=h:(n[h._renderer.type].render.call(h,d,c),d.strokeStyle=h._renderer.effect));k&&(d.lineWidth=k);g.isNumber(m)&&(d.globalAlpha=m);d.save();d.scale(e,e);d.translate(q,r);n.isHidden.test(l)||(l._renderer&&l._renderer.offset?(f=a(l._renderer.scale.x), +e=a(l._renderer.scale.y),d.save(),d.translate(-a(l._renderer.offset.x),-a(l._renderer.offset.y)),d.scale(f,e),f=c._size/l._renderer.scale.y,e=c._leading/l._renderer.scale.y,d.font=[c._style,c._weight,a(f)+"px/",a(e)+"px",c._family].join(" "),f=l._renderer.offset.x/l._renderer.scale.x,l=l._renderer.offset.y/l._renderer.scale.y,d.fillText(c.value,a(f),a(l)),d.restore()):d.fillText(c.value,0,0));n.isHidden.test(h)||(h._renderer&&h._renderer.offset?(f=a(h._renderer.scale.x),e=a(h._renderer.scale.y),d.save(), +d.translate(-a(h._renderer.offset.x),-a(h._renderer.offset.y)),d.scale(f,e),f=c._size/h._renderer.scale.y,e=c._leading/h._renderer.scale.y,d.font=[c._style,c._weight,a(f)+"px/",a(e)+"px",c._family].join(" "),f=h._renderer.offset.x/h._renderer.scale.x,l=h._renderer.offset.y/h._renderer.scale.y,h=k/h._renderer.scale.x,d.lineWidth=a(h),d.strokeText(c.value,a(f),a(l)),d.restore()):d.strokeText(c.value,0,0));d.restore()},getBoundingClientRect:function(a,c){var f=n.ctx;f.font=[a._style,a._weight,a._size+ +"px/"+a._leading+"px",a._family].join(" ");f.textAlign="center";f.textBaseline=a._baseline;var f=f.measureText(a._value).width,d=Math.max(a._size||a._leading);this._linewidth&&!n.isHidden.test(this._stroke)&&(d+=this._linewidth);var e=f/2,g=d/2;switch(n.alignments[a._alignment]||a._alignment){case n.alignments.left:c.left=0;c.right=f;break;case n.alignments.right:c.left=-f;c.right=0;break;default:c.left=-e,c.right=e}switch(a._baseline){case "bottom":c.top=-d;c.bottom=0;break;case "top":c.top=0;c.bottom= +d;break;default:c.top=-g,c.bottom=g}c.width=f;c.height=d;c.centroid||(c.centroid={});c.centroid.x=e;c.centroid.y=g},render:function(a,e,g){if(!this._visible||!this._opacity)return this;this._update();var f=this.parent,h=this._matrix.manual||this._flagMatrix,k=this._flagVertices||this._flagFill||this._fill instanceof c.LinearGradient&&(this._fill._flagSpread||this._fill._flagStops||this._fill._flagEndPoints)||this._fill instanceof c.RadialGradient&&(this._fill._flagSpread||this._fill._flagStops||this._fill._flagRadius|| +this._fill._flagCenter||this._fill._flagFocal)||this._fill instanceof c.Texture&&this._fill._flagLoaded&&this._fill.loaded||this._stroke instanceof c.LinearGradient&&(this._stroke._flagSpread||this._stroke._flagStops||this._stroke._flagEndPoints)||this._stroke instanceof c.RadialGradient&&(this._stroke._flagSpread||this._stroke._flagStops||this._stroke._flagRadius||this._stroke._flagCenter||this._stroke._flagFocal)||this._texture instanceof c.Texture&&this._texture._flagLoaded&&this._texture.loaded|| +this._flagStroke||this._flagLinewidth||this._flagOpacity||f._flagOpacity||this._flagVisible||this._flagScale||this._flagValue||this._flagFamily||this._flagSize||this._flagLeading||this._flagAlignment||this._flagBaseline||this._flagStyle||this._flagWeight||this._flagDecoration||!this._renderer.texture;if(f._matrix.manual||f._flagMatrix||h)this._renderer.matrix||(this._renderer.matrix=new c.Array(9)),this._matrix.toArray(!0,d),m(d,f._renderer.matrix,this._renderer.matrix),this._renderer.scale=this._scale* +f._renderer.scale;k&&(this._renderer.rect||(this._renderer.rect={}),this._renderer.triangles||(this._renderer.triangles=new c.Array(12)),this._renderer.opacity=this._opacity*f._renderer.opacity,n.text.getBoundingClientRect(this,this._renderer.rect),n.getTriangles(this._renderer.rect,this._renderer.triangles),n.updateBuffer.call(n,a,this,e),n.updateTexture.call(n,a,this));if(!this._clip||g)return a.bindBuffer(a.ARRAY_BUFFER,this._renderer.textureCoordsBuffer),a.vertexAttribPointer(e.textureCoords, +2,a.FLOAT,!1,0,0),a.bindTexture(a.TEXTURE_2D,this._renderer.texture),a.uniformMatrix3fv(e.matrix,!1,this._renderer.matrix),a.bindBuffer(a.ARRAY_BUFFER,this._renderer.buffer),a.vertexAttribPointer(e.position,2,a.FLOAT,!1,0,0),a.drawArrays(a.TRIANGLES,0,6),this.flagReset()}},"linear-gradient":{render:function(a,c){if(a.canvas.getContext("2d")){this._update();if(!this._renderer.effect||this._flagEndPoints||this._flagStops)for(this._renderer.effect=a.createLinearGradient(this.left._x,this.left._y,this.right._x, +this.right._y),a=0;a<this.stops.length;a++)c=this.stops[a],this._renderer.effect.addColorStop(c._offset,c._color);return this.flagReset()}}},"radial-gradient":{render:function(a,c){if(a.canvas.getContext("2d")){this._update();if(!this._renderer.effect||this._flagCenter||this._flagFocal||this._flagRadius||this._flagStops)for(this._renderer.effect=a.createRadialGradient(this.center._x,this.center._y,0,this.focal._x,this.focal._y,this._radius),a=0;a<this.stops.length;a++)c=this.stops[a],this._renderer.effect.addColorStop(c._offset, +c._color);return this.flagReset()}}},texture:{render:function(a,d){if(a.canvas.getContext("2d")){this._update();d=this.image;if(!this._renderer.effect||(this._flagLoaded||this._flagRepeat)&&this.loaded)this._renderer.effect=a.createPattern(d,this._repeat);if(this._flagOffset||this._flagLoaded||this._flagScale)this._renderer.offset instanceof c.Vector||(this._renderer.offset=new c.Vector),this._renderer.offset.x=this._offset.x,this._renderer.offset.y=this._offset.y,d&&(this._renderer.offset.x-=d.width/ +2,this._renderer.offset.y+=d.height/2,this._scale instanceof c.Vector?(this._renderer.offset.x*=this._scale.x,this._renderer.offset.y*=this._scale.y):(this._renderer.offset.x*=this._scale,this._renderer.offset.y*=this._scale));if(this._flagScale||this._flagLoaded)this._renderer.scale instanceof c.Vector||(this._renderer.scale=new c.Vector),this._scale instanceof c.Vector?this._renderer.scale.copy(this._scale):this._renderer.scale.set(this._scale,this._scale);return this.flagReset()}}},getTriangles:function(a, +c){var f=a.top,d=a.left,e=a.right;a=a.bottom;c[0]=d;c[1]=f;c[2]=e;c[3]=f;c[4]=d;c[5]=a;c[6]=d;c[7]=a;c[8]=e;c[9]=f;c[10]=e;c[11]=a},updateTexture:function(a,c){this[c._renderer.type].updateCanvas.call(n,c);c._renderer.texture&&a.deleteTexture(c._renderer.texture);a.bindBuffer(a.ARRAY_BUFFER,c._renderer.textureCoordsBuffer);c._renderer.texture=a.createTexture();a.bindTexture(a.TEXTURE_2D,c._renderer.texture);a.texParameteri(a.TEXTURE_2D,a.TEXTURE_WRAP_S,a.CLAMP_TO_EDGE);a.texParameteri(a.TEXTURE_2D, +a.TEXTURE_WRAP_T,a.CLAMP_TO_EDGE);a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MIN_FILTER,a.LINEAR);0>=this.canvas.width||0>=this.canvas.height||a.texImage2D(a.TEXTURE_2D,0,a.RGBA,a.RGBA,a.UNSIGNED_BYTE,this.canvas)},updateBuffer:function(a,c,d){g.isObject(c._renderer.buffer)&&a.deleteBuffer(c._renderer.buffer);c._renderer.buffer=a.createBuffer();a.bindBuffer(a.ARRAY_BUFFER,c._renderer.buffer);a.enableVertexAttribArray(d.position);a.bufferData(a.ARRAY_BUFFER,c._renderer.triangles,a.STATIC_DRAW);g.isObject(c._renderer.textureCoordsBuffer)&& +a.deleteBuffer(c._renderer.textureCoordsBuffer);c._renderer.textureCoordsBuffer=a.createBuffer();a.bindBuffer(a.ARRAY_BUFFER,c._renderer.textureCoordsBuffer);a.enableVertexAttribArray(d.textureCoords);a.bufferData(a.ARRAY_BUFFER,this.uv,a.STATIC_DRAW)},program:{create:function(a,d){var f=a.createProgram();g.each(d,function(c){a.attachShader(f,c)});a.linkProgram(f);if(!a.getProgramParameter(f,a.LINK_STATUS))throw d=a.getProgramInfoLog(f),a.deleteProgram(f),new c.Utils.Error("unable to link program: "+ +d);return f}},shaders:{create:function(a,d,e){e=a.createShader(a[e]);a.shaderSource(e,d);a.compileShader(e);if(!a.getShaderParameter(e,a.COMPILE_STATUS))throw d=a.getShaderInfoLog(e),a.deleteShader(e),new c.Utils.Error("unable to compile shader "+e+": "+d);return e},types:{vertex:"VERTEX_SHADER",fragment:"FRAGMENT_SHADER"},vertex:"attribute vec2 a_position;\nattribute vec2 a_textureCoords;\n\nuniform mat3 u_matrix;\nuniform vec2 u_resolution;\n\nvarying vec2 v_textureCoords;\n\nvoid main() {\n vec2 projected \x3d (u_matrix * vec3(a_position, 1.0)).xy;\n vec2 normal \x3d projected / u_resolution;\n vec2 clipspace \x3d (normal * 2.0) - 1.0;\n\n gl_Position \x3d vec4(clipspace * vec2(1.0, -1.0), 0.0, 1.0);\n v_textureCoords \x3d a_textureCoords;\n}", +fragment:"precision mediump float;\n\nuniform sampler2D u_image;\nvarying vec2 v_textureCoords;\n\nvoid main() {\n gl_FragColor \x3d texture2D(u_image, v_textureCoords);\n}"},TextureRegistry:new c.Registry};n.ctx=n.canvas.getContext("2d");k=c[c.Types.webgl]=function(a){this.domElement=a.domElement||document.createElement("canvas");this.scene=new c.Group;this.scene.parent=this;this._renderer={matrix:new c.Array(h),scale:1,opacity:1};this._flagMatrix=!0;a=g.defaults(a||{},{antialias:!1,alpha:!0,premultipliedAlpha:!0, +stencil:!0,preserveDrawingBuffer:!0,overdraw:!1});this.overdraw=a.overdraw;a=this.ctx=this.domElement.getContext("webgl",a)||this.domElement.getContext("experimental-webgl",a);if(!this.ctx)throw new c.Utils.Error("unable to create a webgl context. Try using another renderer.");var d=n.shaders.create(a,n.shaders.vertex,n.shaders.types.vertex);var f=n.shaders.create(a,n.shaders.fragment,n.shaders.types.fragment);this.program=n.program.create(a,[d,f]);a.useProgram(this.program);this.program.position= +a.getAttribLocation(this.program,"a_position");this.program.matrix=a.getUniformLocation(this.program,"u_matrix");this.program.textureCoords=a.getAttribLocation(this.program,"a_textureCoords");a.disable(a.DEPTH_TEST);a.enable(a.BLEND);a.blendEquationSeparate(a.FUNC_ADD,a.FUNC_ADD);a.blendFuncSeparate(a.SRC_ALPHA,a.ONE_MINUS_SRC_ALPHA,a.ONE,a.ONE_MINUS_SRC_ALPHA)};g.extend(k,{Utils:n});g.extend(k.prototype,c.Utils.Events,{setSize:function(a,c,d){this.width=a;this.height=c;this.ratio=g.isUndefined(d)? +e(this.ctx):d;this.domElement.width=a*this.ratio;this.domElement.height=c*this.ratio;g.extend(this.domElement.style,{width:a+"px",height:c+"px"});a*=this.ratio;c*=this.ratio;this._renderer.matrix[0]=this._renderer.matrix[4]=this._renderer.scale=this.ratio;this._flagMatrix=!0;this.ctx.viewport(0,0,a,c);d=this.ctx.getUniformLocation(this.program,"u_resolution");this.ctx.uniform2f(d,a,c);return this},render:function(){var a=this.ctx;this.overdraw||a.clear(a.COLOR_BUFFER_BIT|a.DEPTH_BUFFER_BIT);n.group.render.call(this.scene, +a,this.program);this._flagMatrix=!1;return this}})})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils,m=c.Shape=function(){this._renderer={};this._renderer.flagMatrix=k.bind(m.FlagMatrix,this);this.isShape=!0;this.id=c.Identifier+c.uniqueId();this.classList=[];this._matrix=new c.Matrix;this.translation=new c.Vector;this.rotation=0;this.scale=1};k.extend(m,{FlagMatrix:function(){this._flagMatrix=!0},MakeObservable:function(k){Object.defineProperty(k,"translation",{enumerable:!0,get:function(){return this._translation},set:function(h){this._translation&&this._translation.unbind(c.Events.change, +this._renderer.flagMatrix);this._translation=h;this._translation.bind(c.Events.change,this._renderer.flagMatrix);m.FlagMatrix.call(this)}});Object.defineProperty(k,"rotation",{enumerable:!0,get:function(){return this._rotation},set:function(c){this._rotation=c;this._flagMatrix=!0}});Object.defineProperty(k,"scale",{enumerable:!0,get:function(){return this._scale},set:function(h){this._scale instanceof c.Vector&&this._scale.unbind(c.Events.change,this._renderer.flagMatrix);this._scale=h;this._scale instanceof +c.Vector&&this._scale.bind(c.Events.change,this._renderer.flagMatrix);this._flagScale=this._flagMatrix=!0}})}});k.extend(m.prototype,c.Utils.Events,{_flagMatrix:!0,_flagScale:!1,_rotation:0,_scale:1,_translation:null,addTo:function(c){c.add(this);return this},clone:function(){var c=new m;c.translation.copy(this.translation);c.rotation=this.rotation;c.scale=this.scale;k.each(m.Properties,function(h){c[h]=this[h]},this);return c._update()},_update:function(k){!this._matrix.manual&&this._flagMatrix&& +(this._matrix.identity().translate(this.translation.x,this.translation.y),this._scale instanceof c.Vector?this._matrix.scale(this._scale.x,this._scale.y):this._matrix.scale(this._scale),this._matrix.rotate(this.rotation));k&&this.parent&&this.parent._update&&this.parent._update();return this},flagReset:function(){this._flagMatrix=this._flagScale=!1;return this}});m.MakeObservable(m.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){function k(a,d,e){var f=d.controls&&d.controls.right,g=a.controls&&a.controls.left;var n=d.x;var h=d.y;var k=(f||d).x;var l=(f||d).y;var m=(g||a).x;var t=(g||a).y;var w=a.x;var p=a.y;f&&d._relative&&(k+=d.x,l+=d.y);g&&a._relative&&(m+=a.x,t+=a.y);return c.Utils.getCurveLength(n,h,k,l,m,t,w,p,e)}function m(a,d,e){var f=d.controls&&d.controls.right,g=a.controls&&a.controls.left;var h=d.x;var n=d.y;var k=(f||d).x;var l=(f||d).y;var m=(g||a).x;var t=(g||a).y;var w=a.x;var p=a.y;f&&d._relative&& +(k+=d.x,l+=d.y);g&&a._relative&&(m+=a.x,t+=a.y);return c.Utils.subdivide(h,n,k,l,m,t,w,p,e)}var l=Math.min,h=Math.max,d=Math.round,e=c.Utils.getComputedMatrix,a=c.Utils;a.each(c.Commands,function(a,c){});var g=c.Path=function(d,f,e,h){c.Shape.call(this);this._renderer.type="path";this._renderer.flagVertices=a.bind(g.FlagVertices,this);this._renderer.bindVertices=a.bind(g.BindVertices,this);this._renderer.unbindVertices=a.bind(g.UnbindVertices,this);this._renderer.flagFill=a.bind(g.FlagFill,this); +this._renderer.flagStroke=a.bind(g.FlagStroke,this);this._closed=!!f;this._curved=!!e;this.beginning=0;this.ending=1;this.fill="#fff";this.stroke="#000";this.opacity=this.linewidth=1;this.visible=!0;this.cap="butt";this.join="miter";this.miter=4;this._vertices=[];this.vertices=d;this.automatic=!h};a.extend(g,{Properties:"fill stroke linewidth opacity visible cap join miter closed curved automatic beginning ending".split(" "),FlagVertices:function(){this._flagLength=this._flagVertices=!0},BindVertices:function(a){for(var d= +a.length;d--;)a[d].bind(c.Events.change,this._renderer.flagVertices);this._renderer.flagVertices()},UnbindVertices:function(a){for(var d=a.length;d--;)a[d].unbind(c.Events.change,this._renderer.flagVertices);this._renderer.flagVertices()},FlagFill:function(){this._flagFill=!0},FlagStroke:function(){this._flagStroke=!0},MakeObservable:function(d){c.Shape.MakeObservable(d);a.each(g.Properties.slice(2,8),c.Utils.defineProperty,d);Object.defineProperty(d,"fill",{enumerable:!0,get:function(){return this._fill}, +set:function(a){(this._fill instanceof c.Gradient||this._fill instanceof c.LinearGradient||this._fill instanceof c.RadialGradient||this._fill instanceof c.Texture)&&this._fill.unbind(c.Events.change,this._renderer.flagFill);this._fill=a;this._flagFill=!0;(this._fill instanceof c.Gradient||this._fill instanceof c.LinearGradient||this._fill instanceof c.RadialGradient||this._fill instanceof c.Texture)&&this._fill.bind(c.Events.change,this._renderer.flagFill)}});Object.defineProperty(d,"stroke",{enumerable:!0, +get:function(){return this._stroke},set:function(a){(this._stroke instanceof c.Gradient||this._stroke instanceof c.LinearGradient||this._stroke instanceof c.RadialGradient||this._stroke instanceof c.Texture)&&this._stroke.unbind(c.Events.change,this._renderer.flagStroke);this._stroke=a;this._flagStroke=!0;(this._stroke instanceof c.Gradient||this._stroke instanceof c.LinearGradient||this._stroke instanceof c.RadialGradient||this._stroke instanceof c.Texture)&&this._stroke.bind(c.Events.change,this._renderer.flagStroke)}}); +Object.defineProperty(d,"length",{get:function(){this._flagLength&&this._updateLength();return this._length}});Object.defineProperty(d,"closed",{enumerable:!0,get:function(){return this._closed},set:function(a){this._closed=!!a;this._flagVertices=!0}});Object.defineProperty(d,"curved",{enumerable:!0,get:function(){return this._curved},set:function(a){this._curved=!!a;this._flagVertices=!0}});Object.defineProperty(d,"automatic",{enumerable:!0,get:function(){return this._automatic},set:function(c){if(c!== +this._automatic){var d=(this._automatic=!!c)?"ignore":"listen";a.each(this.vertices,function(a){a[d]()})}}});Object.defineProperty(d,"beginning",{enumerable:!0,get:function(){return this._beginning},set:function(a){this._beginning=a;this._flagVertices=!0}});Object.defineProperty(d,"ending",{enumerable:!0,get:function(){return this._ending},set:function(a){this._ending=a;this._flagVertices=!0}});Object.defineProperty(d,"vertices",{enumerable:!0,get:function(){return this._collection},set:function(a){var d= +this._renderer.bindVertices,e=this._renderer.unbindVertices;this._collection&&this._collection.unbind(c.Events.insert,d).unbind(c.Events.remove,e);this._collection=new c.Utils.Collection((a||[]).slice(0));this._collection.bind(c.Events.insert,d).bind(c.Events.remove,e);d(this._collection)}});Object.defineProperty(d,"clip",{enumerable:!0,get:function(){return this._clip},set:function(a){this._clip=a;this._flagClip=!0}})}});a.extend(g.prototype,c.Shape.prototype,{_flagVertices:!0,_flagLength:!0,_flagFill:!0, +_flagStroke:!0,_flagLinewidth:!0,_flagOpacity:!0,_flagVisible:!0,_flagCap:!0,_flagJoin:!0,_flagMiter:!0,_flagClip:!1,_length:0,_fill:"#fff",_stroke:"#000",_linewidth:1,_opacity:1,_visible:!0,_cap:"round",_join:"round",_miter:4,_closed:!0,_curved:!1,_automatic:!0,_beginning:0,_ending:1,_clip:!1,clone:function(d){d=d||this.parent;var e=a.map(this.vertices,function(a){return a.clone()}),h=new g(e,this.closed,this.curved,!this.automatic);a.each(c.Path.Properties,function(a){h[a]=this[a]},this);h.translation.copy(this.translation); +h.rotation=this.rotation;h.scale=this.scale;d&&d.add(h);return h},toObject:function(){var d={vertices:a.map(this.vertices,function(a){return a.toObject()})};a.each(c.Shape.Properties,function(a){d[a]=this[a]},this);d.translation=this.translation.toObject;d.rotation=this.rotation;d.scale=this.scale;return d},noFill:function(){this.fill="transparent";return this},noStroke:function(){this.stroke="transparent";return this},corner:function(){var c=this.getBoundingClientRect(!0);c.centroid={x:c.left+c.width/ +2,y:c.top+c.height/2};a.each(this.vertices,function(a){a.addSelf(c.centroid)});return this},center:function(){var c=this.getBoundingClientRect(!0);c.centroid={x:c.left+c.width/2,y:c.top+c.height/2};a.each(this.vertices,function(a){a.subSelf(c.centroid)});return this},remove:function(){if(!this.parent)return this;this.parent.remove(this);return this},getBoundingClientRect:function(a){var c,d=Infinity,g=-Infinity,k=Infinity,n=-Infinity;this._update(!0);a=a?this._matrix:e(this);var m=this.linewidth/ +2;var x=this._vertices.length;if(0>=x){var u=a.multiply(0,0,1);return{top:u.y,left:u.x,right:u.x,bottom:u.y,width:0,height:0}}for(c=0;c<x;c++){u=this._vertices[c];var r=u.x;u=u.y;u=a.multiply(r,u,1);k=l(u.y-m,k);d=l(u.x-m,d);g=h(u.x+m,g);n=h(u.y+m,n)}return{top:k,left:d,right:g,bottom:n,width:g-d,height:n-k}},getPointAt:function(d,e){var g,f;var h=this.length*Math.min(Math.max(d,0),1);var k=this.vertices.length;var n=k-1;var l=g=null;var m=0;var r=this._lengths.length;for(f=0;m<r;m++){if(f+this._lengths[m]>= +h){this._closed?(g=c.Utils.mod(m,k),l=c.Utils.mod(m-1,k),0===m&&(g=l,l=m)):(g=m,l=Math.min(Math.max(m-1,0),n));g=this.vertices[g];l=this.vertices[l];h-=f;0!==this._lengths[m]&&(d=h/this._lengths[m]);break}f+=this._lengths[m]}if(a.isNull(g)||a.isNull(l))return null;var q=l.controls&&l.controls.right;var w=g.controls&&g.controls.left;n=l.x;h=l.y;r=(q||l).x;m=(q||l).y;var p=(w||g).x;f=(w||g).y;var C=g.x;k=g.y;q&&l._relative&&(r+=l.x,m+=l.y);w&&g._relative&&(p+=g.x,f+=g.y);g=c.Utils.getPointOnCubicBezier(d, +n,r,p,C);d=c.Utils.getPointOnCubicBezier(d,h,m,f,k);return a.isObject(e)?(e.x=g,e.y=d,e):new c.Vector(g,d)},plot:function(){if(this.curved)return c.Utils.getCurveFromPoints(this._vertices,this.closed),this;for(var a=0;a<this._vertices.length;a++)this._vertices[a]._command=0===a?c.Commands.move:c.Commands.line;return this},subdivide:function(d){this._update();var e=this.vertices.length-1,g=this.vertices[e],h=this._closed||this.vertices[e]._command===c.Commands.close,k=[];a.each(this.vertices,function(f, +n){if(!(0>=n)||h)if(f.command===c.Commands.move)k.push(new c.Anchor(g.x,g.y)),0<n&&(k[k.length-1].command=c.Commands.line);else{var l=m(f,g,d);k=k.concat(l);a.each(l,function(a,d){a.command=0>=d&&g.command===c.Commands.move?c.Commands.move:c.Commands.line});n>=e&&(this._closed&&this._automatic?(g=f,l=m(f,g,d),k=k.concat(l),a.each(l,function(a,d){a.command=0>=d&&g.command===c.Commands.move?c.Commands.move:c.Commands.line})):h&&k.push(new c.Anchor(f.x,f.y)),k[k.length-1].command=h?c.Commands.close: +c.Commands.line)}g=f},this);this._curved=this._automatic=!1;this.vertices=k;return this},_updateLength:function(d){this._update();var e=this.vertices.length,g=e-1,h=this.vertices[g],n=this._closed||this.vertices[g]._command===c.Commands.close,l=0;a.isUndefined(this._lengths)&&(this._lengths=[]);a.each(this.vertices,function(a,f){0>=f&&!n||a.command===c.Commands.move?(h=a,this._lengths[f]=0):(this._lengths[f]=k(a,h,d),l+=this._lengths[f],f>=g&&n&&(h=this.vertices[(f+1)%e],this._lengths[f+1]=k(a,h, +d),l+=this._lengths[f+1]),h=a)},this);this._length=l;return this},_update:function(){if(this._flagVertices){var a=this.vertices.length-1;var e=d(this._beginning*a);a=d(this._ending*a);this._vertices.length=0;for(var g=e;g<a+1;g++)e=this.vertices[g],this._vertices.push(e);this._automatic&&this.plot()}c.Shape.prototype._update.apply(this,arguments);return this},flagReset:function(){this._flagVertices=this._flagFill=this._flagStroke=this._flagLinewidth=this._flagOpacity=this._flagVisible=this._flagCap= +this._flagJoin=this._flagMiter=this._flagClip=!1;c.Shape.prototype.flagReset.call(this);return this}});g.MakeObservable(g.prototype)})(("undefined"!==typeof global?global:this).Two);(function(c){var k=c.Path,m=c.Utils,l=c.Line=function(h,d,e,a){e=(e-h)/2;a=(a-d)/2;k.call(this,[new c.Anchor(-e,-a),new c.Anchor(e,a)]);this.translation.set(h+e,d+a)};m.extend(l.prototype,k.prototype);k.MakeObservable(l.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Path,m=c.Utils,l=c.Rectangle=function(h,d,e,a){k.call(this,[new c.Anchor,new c.Anchor,new c.Anchor,new c.Anchor],!0);this.width=e;this.height=a;this._update();this.translation.set(h,d)};m.extend(l,{Properties:["width","height"],MakeObservable:function(h){k.MakeObservable(h);m.each(l.Properties,c.Utils.defineProperty,h)}});m.extend(l.prototype,k.prototype,{_width:0,_height:0,_flagWidth:0,_flagHeight:0,_update:function(){if(this._flagWidth||this._flagHeight){var c=this._width/2, +d=this._height/2;this.vertices[0].set(-c,-d);this.vertices[1].set(c,-d);this.vertices[2].set(c,d);this.vertices[3].set(-c,d)}k.prototype._update.call(this);return this},flagReset:function(){this._flagWidth=this._flagHeight=!1;k.prototype.flagReset.call(this);return this}});l.MakeObservable(l.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Path,m=2*Math.PI,l=Math.cos,h=Math.sin,d=c.Utils,e=c.Ellipse=function(a,e,h,f){d.isNumber(f)||(f=h);var g=d.map(d.range(c.Resolution),function(a){return new c.Anchor},this);k.call(this,g,!0,!0);this.width=2*h;this.height=2*f;this._update();this.translation.set(a,e)};d.extend(e,{Properties:["width","height"],MakeObservable:function(a){k.MakeObservable(a);d.each(e.Properties,c.Utils.defineProperty,a)}});d.extend(e.prototype,k.prototype,{_width:0,_height:0,_flagWidth:!1,_flagHeight:!1, +_update:function(){if(this._flagWidth||this._flagHeight)for(var a=0,c=this.vertices.length;a<c;a++){var d=a/c*m,e=this._width*l(d)/2,d=this._height*h(d)/2;this.vertices[a].set(e,d)}k.prototype._update.call(this);return this},flagReset:function(){this._flagWidth=this._flagHeight=!1;k.prototype.flagReset.call(this);return this}});e.MakeObservable(e.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Path,m=2*Math.PI,l=Math.cos,h=Math.sin,d=c.Utils,e=c.Circle=function(a,e,h){var g=d.map(d.range(c.Resolution),function(a){return new c.Anchor},this);k.call(this,g,!0,!0);this.radius=h;this._update();this.translation.set(a,e)};d.extend(e,{Properties:["radius"],MakeObservable:function(a){k.MakeObservable(a);d.each(e.Properties,c.Utils.defineProperty,a)}});d.extend(e.prototype,k.prototype,{_radius:0,_flagRadius:!1,_update:function(){if(this._flagRadius)for(var a=0,c=this.vertices.length;a< +c;a++){var d=a/c*m,e=this._radius*l(d),d=this._radius*h(d);this.vertices[a].set(e,d)}k.prototype._update.call(this);return this},flagReset:function(){this._flagRadius=!1;k.prototype.flagReset.call(this);return this}});e.MakeObservable(e.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Path,m=2*Math.PI,l=Math.cos,h=Math.sin,d=c.Utils,e=c.Polygon=function(a,e,h,f){f=Math.max(f||0,3);var g=d.map(d.range(f),function(a){return new c.Anchor});k.call(this,g,!0);this.width=2*h;this.height=2*h;this.sides=f;this._update();this.translation.set(a,e)};d.extend(e,{Properties:["width","height","sides"],MakeObservable:function(a){k.MakeObservable(a);d.each(e.Properties,c.Utils.defineProperty,a)}});d.extend(e.prototype,k.prototype,{_width:0,_height:0,_sides:0,_flagWidth:!1, +_flagHeight:!1,_flagSides:!1,_update:function(){if(this._flagWidth||this._flagHeight||this._flagSides){var a=this._sides,d=this.vertices.length;d>a&&this.vertices.splice(a-1,d-a);for(var e=0;e<a;e++){var f=(e+.5)/a*m+Math.PI/2,t=this._width*l(f),f=this._height*h(f);e>=d?this.vertices.push(new c.Anchor(t,f)):this.vertices[e].set(t,f)}}k.prototype._update.call(this);return this},flagReset:function(){this._flagWidth=this._flagHeight=this._flagSides=!1;k.prototype.flagReset.call(this);return this}}); +e.MakeObservable(e.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){function k(a,c){for(;0>a;)a+=c;return a%c}var m=c.Path,l=2*Math.PI,h=Math.PI/2,d=c.Utils,e=c.ArcSegment=function(a,e,h,f,k,l,B){B=d.map(d.range(B||3*c.Resolution),function(){return new c.Anchor});m.call(this,B,!1,!1,!0);this.innerRadius=h;this.outerRadius=f;this.startAngle=k;this.endAngle=l;this._update();this.translation.set(a,e)};d.extend(e,{Properties:["startAngle","endAngle","innerRadius","outerRadius"],MakeObservable:function(a){m.MakeObservable(a);d.each(e.Properties,c.Utils.defineProperty, +a)}});d.extend(e.prototype,m.prototype,{_flagStartAngle:!1,_flagEndAngle:!1,_flagInnerRadius:!1,_flagOuterRadius:!1,_startAngle:0,_endAngle:l,_innerRadius:0,_outerRadius:0,_update:function(){if(this._flagStartAngle||this._flagEndAngle||this._flagInnerRadius||this._flagOuterRadius){var a=this._startAngle,d=this._endAngle,e=this._innerRadius,f=this._outerRadius,t=k(a,l)===k(d,l),v=0<e,B=this.vertices,z=v?B.length/2:B.length,A=0;t?z--:v||(z-=2);for(var x=0,u=z-1;x<z;x++){var r=x/u;var q=B[A];r=r*(d- +a)+a;var w=(d-a)/z;var p=f*Math.cos(r);var C=f*Math.sin(r);switch(x){case 0:var E=c.Commands.move;break;default:E=c.Commands.curve}q.command=E;q.x=p;q.y=C;q.controls.left.clear();q.controls.right.clear();q.command===c.Commands.curve&&(C=f*w/Math.PI,q.controls.left.x=C*Math.cos(r-h),q.controls.left.y=C*Math.sin(r-h),q.controls.right.x=C*Math.cos(r+h),q.controls.right.y=C*Math.sin(r+h),1===x&&q.controls.left.multiplyScalar(2),x===u&&q.controls.right.multiplyScalar(2));A++}if(v)for(t?(B[A].command=c.Commands.close, +A++):(z--,u=z-1),x=0;x<z;x++)r=x/u,q=B[A],r=(1-r)*(d-a)+a,w=(d-a)/z,p=e*Math.cos(r),C=e*Math.sin(r),E=c.Commands.curve,0>=x&&(E=t?c.Commands.move:c.Commands.line),q.command=E,q.x=p,q.y=C,q.controls.left.clear(),q.controls.right.clear(),q.command===c.Commands.curve&&(C=e*w/Math.PI,q.controls.left.x=C*Math.cos(r+h),q.controls.left.y=C*Math.sin(r+h),q.controls.right.x=C*Math.cos(r-h),q.controls.right.y=C*Math.sin(r-h),1===x&&q.controls.left.multiplyScalar(2),x===u&&q.controls.right.multiplyScalar(2)), +A++;else t||(B[A].command=c.Commands.line,B[A].x=0,B[A].y=0,A++);B[A].command=c.Commands.close}m.prototype._update.call(this);return this},flagReset:function(){m.prototype.flagReset.call(this);this._flagStartAngle=this._flagEndAngle=this._flagInnerRadius=this._flagOuterRadius=!1;return this}});e.MakeObservable(e.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Path,m=2*Math.PI,l=Math.cos,h=Math.sin,d=c.Utils,e=c.Star=function(a,e,h,f,l){d.isNumber(f)||(f=h/2);if(!d.isNumber(l)||0>=l)l=5;var g=d.map(d.range(2*l),function(a){return new c.Anchor});k.call(this,g,!0);this.innerRadius=f;this.outerRadius=h;this.sides=l;this._update();this.translation.set(a,e)};d.extend(e,{Properties:["innerRadius","outerRadius","sides"],MakeObservable:function(a){k.MakeObservable(a);d.each(e.Properties,c.Utils.defineProperty,a)}});d.extend(e.prototype,k.prototype, +{_innerRadius:0,_outerRadius:0,_sides:0,_flagInnerRadius:!1,_flagOuterRadius:!1,_flagSides:!1,_update:function(){if(this._flagInnerRadius||this._flagOuterRadius||this._flagSides){var a=2*this._sides,d=this.vertices.length;d>a&&this.vertices.splice(a-1,d-a);for(var e=0;e<a;e++){var f=(e+.5)/a*m,t=e%2?this._innerRadius:this._outerRadius,v=t*l(f),f=t*h(f);e>=d?this.vertices.push(new c.Anchor(v,f)):this.vertices[e].set(v,f)}}k.prototype._update.call(this);return this},flagReset:function(){this._flagInnerRadius= +this._flagOuterRadius=this._flagSides=!1;k.prototype.flagReset.call(this);return this}});e.MakeObservable(e.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Path,m=c.Utils,l=c.RoundedRectangle=function(h,d,e,a,g){m.isNumber(g)||(g=Math.floor(Math.min(e,a)/12));var l=m.map(m.range(10),function(a){return new c.Anchor(0,0,0,0,0,0,0===a?c.Commands.move:c.Commands.curve)});l[l.length-1].command=c.Commands.close;k.call(this,l,!1,!1,!0);this.width=e;this.height=a;this.radius=g;this._update();this.translation.set(h,d)};m.extend(l,{Properties:["width","height","radius"],MakeObservable:function(h){k.MakeObservable(h);m.each(l.Properties,c.Utils.defineProperty, +h)}});m.extend(l.prototype,k.prototype,{_width:0,_height:0,_radius:0,_flagWidth:!1,_flagHeight:!1,_flagRadius:!1,_update:function(){if(this._flagWidth||this._flagHeight||this._flagRadius){var c=this._width,d=this._height,e=Math.min(Math.max(this._radius,0),Math.min(c,d)),c=c/2,a=d/2,d=this.vertices[0];d.x=-(c-e);d.y=-a;d=this.vertices[1];d.x=c-e;d.y=-a;d.controls.left.clear();d.controls.right.x=e;d.controls.right.y=0;d=this.vertices[2];d.x=c;d.y=-(a-e);d.controls.right.clear();d.controls.left.clear(); +d=this.vertices[3];d.x=c;d.y=a-e;d.controls.left.clear();d.controls.right.x=0;d.controls.right.y=e;d=this.vertices[4];d.x=c-e;d.y=a;d.controls.right.clear();d.controls.left.clear();d=this.vertices[5];d.x=-(c-e);d.y=a;d.controls.left.clear();d.controls.right.x=-e;d.controls.right.y=0;d=this.vertices[6];d.x=-c;d.y=a-e;d.controls.left.clear();d.controls.right.clear();d=this.vertices[7];d.x=-c;d.y=-(a-e);d.controls.left.clear();d.controls.right.x=0;d.controls.right.y=-e;d=this.vertices[8];d.x=-(c-e); +d.y=-a;d.controls.left.clear();d.controls.right.clear();d=this.vertices[9];d.copy(this.vertices[8])}k.prototype._update.call(this);return this},flagReset:function(){this._flagWidth=this._flagHeight=this._flagRadius=!1;k.prototype.flagReset.call(this);return this}});l.MakeObservable(l.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.root,m=c.Utils.getComputedMatrix,l=c.Utils;(k.document?k.document.createElement("canvas"):{getContext:l.identity}).getContext("2d");var h=c.Text=function(d,e,a,g){c.Shape.call(this);this._renderer.type="text";this._renderer.flagFill=l.bind(h.FlagFill,this);this._renderer.flagStroke=l.bind(h.FlagStroke,this);this.value=d;l.isNumber(e)&&(this.translation.x=e);l.isNumber(a)&&(this.translation.y=a);if(!l.isObject(g))return this;l.each(c.Text.Properties,function(a){a in g&&(this[a]= +g[a])},this)};l.extend(c.Text,{Properties:"value family size leading alignment linewidth style weight decoration baseline opacity visible fill stroke".split(" "),FlagFill:function(){this._flagFill=!0},FlagStroke:function(){this._flagStroke=!0},MakeObservable:function(d){c.Shape.MakeObservable(d);l.each(c.Text.Properties.slice(0,12),c.Utils.defineProperty,d);Object.defineProperty(d,"fill",{enumerable:!0,get:function(){return this._fill},set:function(d){(this._fill instanceof c.Gradient||this._fill instanceof +c.LinearGradient||this._fill instanceof c.RadialGradient||this._fill instanceof c.Texture)&&this._fill.unbind(c.Events.change,this._renderer.flagFill);this._fill=d;this._flagFill=!0;(this._fill instanceof c.Gradient||this._fill instanceof c.LinearGradient||this._fill instanceof c.RadialGradient||this._fill instanceof c.Texture)&&this._fill.bind(c.Events.change,this._renderer.flagFill)}});Object.defineProperty(d,"stroke",{enumerable:!0,get:function(){return this._stroke},set:function(d){(this._stroke instanceof +c.Gradient||this._stroke instanceof c.LinearGradient||this._stroke instanceof c.RadialGradient||this._stroke instanceof c.Texture)&&this._stroke.unbind(c.Events.change,this._renderer.flagStroke);this._stroke=d;this._flagStroke=!0;(this._stroke instanceof c.Gradient||this._stroke instanceof c.LinearGradient||this._stroke instanceof c.RadialGradient||this._stroke instanceof c.Texture)&&this._stroke.bind(c.Events.change,this._renderer.flagStroke)}});Object.defineProperty(d,"clip",{enumerable:!0,get:function(){return this._clip}, +set:function(c){this._clip=c;this._flagClip=!0}})}});l.extend(c.Text.prototype,c.Shape.prototype,{_flagValue:!0,_flagFamily:!0,_flagSize:!0,_flagLeading:!0,_flagAlignment:!0,_flagBaseline:!0,_flagStyle:!0,_flagWeight:!0,_flagDecoration:!0,_flagFill:!0,_flagStroke:!0,_flagLinewidth:!0,_flagOpacity:!0,_flagVisible:!0,_flagClip:!1,_value:"",_family:"sans-serif",_size:13,_leading:17,_alignment:"center",_baseline:"middle",_style:"normal",_weight:500,_decoration:"none",_fill:"#000",_stroke:"transparent", +_linewidth:1,_opacity:1,_visible:!0,_clip:!1,remove:function(){if(!this.parent)return this;this.parent.remove(this);return this},clone:function(d){d=d||this.parent;var e=new c.Text(this.value);e.translation.copy(this.translation);e.rotation=this.rotation;e.scale=this.scale;l.each(c.Text.Properties,function(a){e[a]=this[a]},this);d&&d.add(e);return e},toObject:function(){var d={translation:this.translation.toObject(),rotation:this.rotation,scale:this.scale};l.each(c.Text.Properties,function(c){d[c]= +this[c]},this);return d},noStroke:function(){this.stroke="transparent";return this},noFill:function(){this.fill="transparent";return this},getBoundingClientRect:function(c){this._update(!0);c=(c?this._matrix:m(this)).multiply(0,0,1);return{top:c.x,left:c.y,right:c.x,bottom:c.y,width:0,height:0}},flagReset:function(){this._flagValue=this._flagFamily=this._flagSize=this._flagLeading=this._flagAlignment=this._flagFill=this._flagStroke=this._flagLinewidth=this._flagOpaicty=this._flagVisible=this._flagClip= +this._flagDecoration=this._flagBaseline=!1;c.Shape.prototype.flagReset.call(this);return this}});c.Text.MakeObservable(c.Text.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils,m=c.Stop=function(c,d,e){this._renderer={};this._renderer.type="stop";this.offset=k.isNumber(c)?c:0>=m.Index?0:1;this.opacity=k.isNumber(e)?e:1;this.color=k.isString(d)?d:0>=m.Index?"#fff":"#000";m.Index=(m.Index+1)%2};k.extend(m,{Index:0,Properties:["offset","opacity","color"],MakeObservable:function(c){k.each(m.Properties,function(c){var d="_"+c,a="_flag"+c.charAt(0).toUpperCase()+c.slice(1);Object.defineProperty(this,c,{enumerable:!0,get:function(){return this[d]},set:function(c){this[d]= +c;this[a]=!0;this.parent&&(this.parent._flagStops=!0)}})},c)}});k.extend(m.prototype,c.Utils.Events,{clone:function(){var c=new m;k.each(m.Properties,function(d){c[d]=this[d]},this);return c},toObject:function(){var c={};k.each(m.Properties,function(d){c[d]=this[d]},this);return c},flagReset:function(){this._flagOffset=this._flagColor=this._flagOpacity=!1;return this}});m.MakeObservable(m.prototype);var l=c.Gradient=function(h){this._renderer={};this._renderer.type="gradient";this.id=c.Identifier+ +c.uniqueId();this.classList=[];this._renderer.flagStops=k.bind(l.FlagStops,this);this._renderer.bindStops=k.bind(l.BindStops,this);this._renderer.unbindStops=k.bind(l.UnbindStops,this);this.spread="pad";this.stops=h};k.extend(l,{Stop:m,Properties:["spread"],MakeObservable:function(h){k.each(l.Properties,c.Utils.defineProperty,h);Object.defineProperty(h,"stops",{enumerable:!0,get:function(){return this._stops},set:function(d){var e=this._renderer.bindStops,a=this._renderer.unbindStops;this._stops&& +this._stops.unbind(c.Events.insert,e).unbind(c.Events.remove,a);this._stops=new c.Utils.Collection((d||[]).slice(0));this._stops.bind(c.Events.insert,e).bind(c.Events.remove,a);e(this._stops)}})},FlagStops:function(){this._flagStops=!0},BindStops:function(h){for(var d=h.length;d--;)h[d].bind(c.Events.change,this._renderer.flagStops),h[d].parent=this;this._renderer.flagStops()},UnbindStops:function(h){for(var d=h.length;d--;)h[d].unbind(c.Events.change,this._renderer.flagStops),delete h[d].parent; +this._renderer.flagStops()}});k.extend(l.prototype,c.Utils.Events,{_flagStops:!1,_flagSpread:!1,clone:function(h){h=h||this.parent;var d=k.map(this.stops,function(a){return a.clone()}),e=new l(d);k.each(c.Gradient.Properties,function(a){e[a]=this[a]},this);h&&h.add(e);return e},toObject:function(){var c={stops:k.map(this.stops,function(c){return c.toObject()})};k.each(l.Properties,function(d){c[d]=this[d]},this);return c},_update:function(){(this._flagSpread||this._flagStops)&&this.trigger(c.Events.change); +return this},flagReset:function(){this._flagSpread=this._flagStops=!1;return this}});l.MakeObservable(l.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils,m=c.LinearGradient=function(l,h,d,e,a){c.Gradient.call(this,a);this._renderer.type="linear-gradient";a=k.bind(m.FlagEndPoints,this);this.left=(new c.Vector).bind(c.Events.change,a);this.right=(new c.Vector).bind(c.Events.change,a);k.isNumber(l)&&(this.left.x=l);k.isNumber(h)&&(this.left.y=h);k.isNumber(d)&&(this.right.x=d);k.isNumber(e)&&(this.right.y=e)};k.extend(m,{Stop:c.Gradient.Stop,MakeObservable:function(k){c.Gradient.MakeObservable(k)},FlagEndPoints:function(){this._flagEndPoints= +!0}});k.extend(m.prototype,c.Gradient.prototype,{_flagEndPoints:!1,clone:function(l){l=l||this.parent;var h=k.map(this.stops,function(c){return c.clone()}),d=new m(this.left._x,this.left._y,this.right._x,this.right._y,h);k.each(c.Gradient.Properties,function(c){d[c]=this[c]},this);l&&l.add(d);return d},toObject:function(){var k=c.Gradient.prototype.toObject.call(this);k.left=this.left.toObject();k.right=this.right.toObject();return k},_update:function(){(this._flagEndPoints||this._flagSpread||this._flagStops)&& +this.trigger(c.Events.change);return this},flagReset:function(){this._flagEndPoints=!1;c.Gradient.prototype.flagReset.call(this);return this}});m.MakeObservable(m.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils,m=c.RadialGradient=function(l,h,d,e,a,g){c.Gradient.call(this,e);this._renderer.type="radial-gradient";this.center=(new c.Vector).bind(c.Events.change,k.bind(function(){this._flagCenter=!0},this));this.radius=k.isNumber(d)?d:20;this.focal=(new c.Vector).bind(c.Events.change,k.bind(function(){this._flagFocal=!0},this));k.isNumber(l)&&(this.center.x=l);k.isNumber(h)&&(this.center.y=h);this.focal.copy(this.center);k.isNumber(a)&&(this.focal.x=a);k.isNumber(g)&&(this.focal.y= +g)};k.extend(m,{Stop:c.Gradient.Stop,Properties:["radius"],MakeObservable:function(l){c.Gradient.MakeObservable(l);k.each(m.Properties,c.Utils.defineProperty,l)}});k.extend(m.prototype,c.Gradient.prototype,{_flagRadius:!1,_flagCenter:!1,_flagFocal:!1,clone:function(l){l=l||this.parent;var h=k.map(this.stops,function(c){return c.clone()}),d=new m(this.center._x,this.center._y,this._radius,h,this.focal._x,this.focal._y);k.each(c.Gradient.Properties.concat(m.Properties),function(c){d[c]=this[c]},this); +l&&l.add(d);return d},toObject:function(){var l=c.Gradient.prototype.toObject.call(this);k.each(m.Properties,function(c){l[c]=this[c]},this);l.center=this.center.toObject();l.focal=this.focal.toObject();return l},_update:function(){(this._flagRadius||this._flatCenter||this._flagFocal||this._flagSpread||this._flagStops)&&this.trigger(c.Events.change);return this},flagReset:function(){this._flagRadius=this._flagCenter=this._flagFocal=!1;c.Gradient.prototype.flagReset.call(this);return this}});m.MakeObservable(m.prototype)})(("undefined"!== +typeof global?global:this).Two); +(function(c){var k=c.Utils,m,l=/\.(mp4|webm)$/i;this.document&&(m=document.createElement("a"));var h=c.Texture=function(d,e){this._renderer={};this._renderer.type="texture";this._renderer.flagOffset=k.bind(h.FlagOffset,this);this._renderer.flagScale=k.bind(h.FlagScale,this);this.id=c.Identifier+c.uniqueId();this.classList=[];this.offset=new c.Vector;if(k.isFunction(e)){var a=k.bind(function(){this.unbind(c.Events.load,a);k.isFunction(e)&&e()},this);this.bind(c.Events.load,a)}k.isString(d)?this.src= +d:k.isElement(d)&&(this.image=d);this._update()};k.extend(h,{Properties:["src","loaded","repeat"],ImageRegistry:new c.Registry,getAbsoluteURL:function(c){if(!m)return c;m.href=c;return m.href},getImage:function(c){c=h.getAbsoluteURL(c);if(h.ImageRegistry.contains(c))return h.ImageRegistry.get(c);c=l.test(c)?document.createElement("video"):document.createElement("img");c.crossOrigin="anonymous";return c},Register:{canvas:function(c,e){c._src="#"+c.id;h.ImageRegistry.add(c.src,c.image);k.isFunction(e)&& +e()},img:function(d,e){var a=function(c){d.image.removeEventListener("load",a,!1);d.image.removeEventListener("error",g,!1);k.isFunction(e)&&e()},g=function(e){d.image.removeEventListener("load",a,!1);d.image.removeEventListener("error",g,!1);throw new c.Utils.Error("unable to load "+d.src);};k.isNumber(d.image.width)&&0<d.image.width&&k.isNumber(d.image.height)&&0<d.image.height?a():(d.image.addEventListener("load",a,!1),d.image.addEventListener("error",g,!1));d._src=h.getAbsoluteURL(d._src);d.image&& +d.image.getAttribute("two-src")||(d.image.setAttribute("two-src",d.src),h.ImageRegistry.add(d.src,d.image),d.image.src=d.src)},video:function(d,e){var a=function(c){d.image.removeEventListener("load",a,!1);d.image.removeEventListener("error",g,!1);d.image.width=d.image.videoWidth;d.image.height=d.image.videoHeight;d.image.play();k.isFunction(e)&&e()},g=function(e){d.image.removeEventListener("load",a,!1);d.image.removeEventListener("error",g,!1);throw new c.Utils.Error("unable to load "+d.src);}; +d._src=h.getAbsoluteURL(d._src);d.image.addEventListener("canplaythrough",a,!1);d.image.addEventListener("error",g,!1);d.image&&d.image.getAttribute("two-src")||(d.image.setAttribute("two-src",d.src),h.ImageRegistry.add(d.src,d.image),d.image.src=d.src,d.image.loop=!0,d.image.load())}},load:function(c,e){var a=c.image,d=a&&a.nodeName.toLowerCase();c._flagImage&&(/canvas/i.test(d)?h.Register.canvas(c,e):(c._src=a.getAttribute("two-src")||a.src,h.Register[d](c,e)));c._flagSrc&&(a||(c.image=h.getImage(c.src)), +d=c.image.nodeName.toLowerCase(),h.Register[d](c,e))},FlagOffset:function(){this._flagOffset=!0},FlagScale:function(){this._flagScale=!0},MakeObservable:function(d){k.each(h.Properties,c.Utils.defineProperty,d);Object.defineProperty(d,"image",{enumerable:!0,get:function(){return this._image},set:function(c){switch(c&&c.nodeName.toLowerCase()){case "canvas":var a="#"+c.id;break;default:a=c.src}h.ImageRegistry.contains(a)?this._image=h.ImageRegistry.get(c.src):this._image=c;this._flagImage=!0}});Object.defineProperty(d, +"offset",{enumerable:!0,get:function(){return this._offset},set:function(d){this._offset&&this._offset.unbind(c.Events.change,this._renderer.flagOffset);this._offset=d;this._offset.bind(c.Events.change,this._renderer.flagOffset);this._flagOffset=!0}});Object.defineProperty(d,"scale",{enumerable:!0,get:function(){return this._scale},set:function(d){this._scale instanceof c.Vector&&this._scale.unbind(c.Events.change,this._renderer.flagScale);this._scale=d;this._scale instanceof c.Vector&&this._scale.bind(c.Events.change, +this._renderer.flagScale);this._flagScale=!0}})}});k.extend(h.prototype,c.Utils.Events,c.Shape.prototype,{_flagSrc:!1,_flagImage:!1,_flagVideo:!1,_flagLoaded:!1,_flagRepeat:!1,_flagOffset:!1,_flagScale:!1,_src:"",_image:null,_loaded:!1,_repeat:"no-repeat",_scale:1,_offset:null,clone:function(){return new h(this.src)},toObject:function(){return{src:this.src,image:this.image}},_update:function(){if(this._flagSrc||this._flagImage||this._flagVideo)if(this.trigger(c.Events.change),this._flagSrc||this._flagImage)this.loaded= +!1,h.load(this,k.bind(function(){this.loaded=!0;this.trigger(c.Events.change).trigger(c.Events.load)},this));this._image&&4<=this._image.readyState&&(this._flagVideo=!0);return this},flagReset:function(){this._flagSrc=this._flagImage=this._flagLoaded=this._flagVideo=this._flagScale=this._flagOffset=!1;return this}});h.MakeObservable(h.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils,m=c.Path,l=c.Rectangle,h=c.Sprite=function(d,e,a,g,h,f){m.call(this,[new c.Anchor,new c.Anchor,new c.Anchor,new c.Anchor],!0);this.noStroke();this.noFill();d instanceof c.Texture?this.texture=d:k.isString(d)&&(this.texture=new c.Texture(d));this._update();this.translation.set(e||0,a||0);k.isNumber(g)&&(this.columns=g);k.isNumber(h)&&(this.rows=h);k.isNumber(f)&&(this.frameRate=f)};k.extend(h,{Properties:["texture","columns","rows","frameRate","index"],MakeObservable:function(d){l.MakeObservable(d); +k.each(h.Properties,c.Utils.defineProperty,d)}});k.extend(h.prototype,l.prototype,{_flagTexture:!1,_flagColumns:!1,_flagRows:!1,_flagFrameRate:!1,flagIndex:!1,_amount:1,_duration:0,_startTime:0,_playing:!1,_firstFrame:0,_lastFrame:0,_loop:!0,_texture:null,_columns:1,_rows:1,_frameRate:0,_index:0,play:function(c,e,a){this._playing=!0;this._firstFrame=0;this._lastFrame=this.amount-1;this._startTime=k.performance.now();k.isNumber(c)&&(this._firstFrame=c);k.isNumber(e)&&(this._lastFrame=e);k.isFunction(a)? +this._onLastFrame=a:delete this._onLastFrame;this._index!==this._firstFrame&&(this._startTime-=1E3*Math.abs(this._index-this._firstFrame)/this._frameRate);return this},pause:function(){this._playing=!1;return this},stop:function(){this._playing=!1;this._index=0;return this},clone:function(c){c=c||this.parent;var d=new h(this.texture,this.translation.x,this.translation.y,this.columns,this.rows,this.frameRate);this.playing&&(d.play(this._firstFrame,this._lastFrame),d._loop=this._loop);c&&c.add(d);return d}, +_update:function(){var c=this._texture,e=this._columns,a=this._rows;if(this._flagColumns||this._flagRows)this._amount=this._columns*this._rows;this._flagFrameRate&&(this._duration=1E3*this._amount/this._frameRate);this._flagTexture&&(this.fill=this._texture);if(this._texture.loaded){var g=c.image.width;var h=c.image.height;var f=g/e;a=h/a;var m=this._amount;this.width!==f&&(this.width=f);this.height!==a&&(this.height=a);if(this._playing&&0<this._frameRate){k.isNaN(this._lastFrame)&&(this._lastFrame= +m-1);m=k.performance.now()-this._startTime;var v=this._lastFrame+1;var B=1E3*(v-this._firstFrame)/this._frameRate;m=this._loop?m%B:Math.min(m,B);m=k.lerp(this._firstFrame,v,m/B);m=Math.floor(m);m!==this._index&&(this._index=m,m>=this._lastFrame-1&&this._onLastFrame&&this._onLastFrame())}f=this._index%e*-f+(g-f)/2;e=-a*Math.floor(this._index/e)+(h-a)/2;f!==c.offset.x&&(c.offset.x=f);e!==c.offset.y&&(c.offset.y=e)}l.prototype._update.call(this);return this},flagReset:function(){this._flagTexture=this._flagColumns= +this._flagRows=this._flagFrameRate=!1;l.prototype.flagReset.call(this);return this}});h.MakeObservable(h.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){var k=c.Utils,m=c.Path,l=c.Rectangle,h=c.ImageSequence=function(d,e,a,g){m.call(this,[new c.Anchor,new c.Anchor,new c.Anchor,new c.Anchor],!0);this._renderer.flagTextures=k.bind(h.FlagTextures,this);this._renderer.bindTextures=k.bind(h.BindTextures,this);this._renderer.unbindTextures=k.bind(h.UnbindTextures,this);this.noStroke();this.noFill();this.textures=k.map(d,h.GenerateTexture,this);this._update();this.translation.set(e||0,a||0);k.isNumber(g)?this.frameRate=g:this.frameRate=h.DefaultFrameRate}; +k.extend(h,{Properties:["frameRate","index"],DefaultFrameRate:30,FlagTextures:function(){this._flagTextures=!0},BindTextures:function(d){for(var e=d.length;e--;)d[e].bind(c.Events.change,this._renderer.flagTextures);this._renderer.flagTextures()},UnbindTextures:function(d){for(var e=d.length;e--;)d[e].unbind(c.Events.change,this._renderer.flagTextures);this._renderer.flagTextures()},MakeObservable:function(d){l.MakeObservable(d);k.each(h.Properties,c.Utils.defineProperty,d);Object.defineProperty(d, +"textures",{enumerable:!0,get:function(){return this._textures},set:function(d){var a=this._renderer.bindTextures,e=this._renderer.unbindTextures;this._textures&&this._textures.unbind(c.Events.insert,a).unbind(c.Events.remove,e);this._textures=new c.Utils.Collection((d||[]).slice(0));this._textures.bind(c.Events.insert,a).bind(c.Events.remove,e);a(this._textures)}})},GenerateTexture:function(d){if(d instanceof c.Texture)return d;if(k.isString(d))return new c.Texture(d)}});k.extend(h.prototype,l.prototype, +{_flagTextures:!1,_flagFrameRate:!1,_flagIndex:!1,_amount:1,_duration:0,_index:0,_startTime:0,_playing:!1,_firstFrame:0,_lastFrame:0,_loop:!0,_textures:null,_frameRate:0,play:function(c,e,a){this._playing=!0;this._firstFrame=0;this._lastFrame=this.amount-1;this._startTime=k.performance.now();k.isNumber(c)&&(this._firstFrame=c);k.isNumber(e)&&(this._lastFrame=e);k.isFunction(a)?this._onLastFrame=a:delete this._onLastFrame;this._index!==this._firstFrame&&(this._startTime-=1E3*Math.abs(this._index-this._firstFrame)/ +this._frameRate);return this},pause:function(){this._playing=!1;return this},stop:function(){this._playing=!1;this._index=0;return this},clone:function(c){c=c||this.parent;var d=new h(this.textures,this.translation.x,this.translation.y,this.frameRate);d._loop=this._loop;this._playing&&d.play();c&&c.add(d);return d},_update:function(){var d=this._textures;this._flagTextures&&(this._amount=d.length);this._flagFrameRate&&(this._duration=1E3*this._amount/this._frameRate);if(this._playing&&0<this._frameRate){var e= +this._amount;k.isNaN(this._lastFrame)&&(this._lastFrame=e-1);e=k.performance.now()-this._startTime;var a=this._lastFrame+1;var g=1E3*(a-this._firstFrame)/this._frameRate;e=this._loop?e%g:Math.min(e,g);e=k.lerp(this._firstFrame,a,e/g);e=Math.floor(e);e!==this._index&&(this._index=e,a=d[this._index],a.loaded&&(d=a.image.width,g=a.image.height,this.width!==d&&(this.width=d),this.height!==g&&(this.height=g),this.fill=a,e>=this._lastFrame-1&&this._onLastFrame&&this._onLastFrame()))}else!this._flagIndex&& +this.fill instanceof c.Texture||(a=d[this._index],a.loaded&&(d=a.image.width,g=a.image.height,this.width!==d&&(this.width=d),this.height!==g&&(this.height=g)),this.fill=a);l.prototype._update.call(this);return this},flagReset:function(){this._flagTextures=this._flagFrameRate=!1;l.prototype.flagReset.call(this);return this}});h.MakeObservable(h.prototype)})(("undefined"!==typeof global?global:this).Two); +(function(c){function k(a,c){var d=a.parent;if(d===c)this.additions.push(a),this._flagAdditions=!0;else{if(d&&d.children.ids[a.id]){var e=h.indexOf(d.children,a);d.children.splice(e,1);e=h.indexOf(d.additions,a);0<=e?d.additions.splice(e,1):(d.subtractions.push(a),d._flagSubtractions=!0)}c?(a.parent=c,this.additions.push(a),this._flagAdditions=!0):(e=h.indexOf(this.additions,a),0<=e?this.additions.splice(e,1):(this.subtractions.push(a),this._flagSubtractions=!0),delete a.parent)}}var m=Math.min,l= +Math.max,h=c.Utils,d=function(){c.Utils.Collection.apply(this,arguments);Object.defineProperty(this,"_events",{value:{},enumerable:!1});this.ids={};this.on(c.Events.insert,this.attach);this.on(c.Events.remove,this.detach);d.prototype.attach.apply(this,arguments)};d.prototype=new c.Utils.Collection;d.prototype.constructor=d;h.extend(d.prototype,{attach:function(a){for(var c=0;c<a.length;c++)this.ids[a[c].id]=a[c];return this},detach:function(a){for(var c=0;c<a.length;c++)delete this.ids[a[c].id];return this}}); +var e=c.Group=function(){c.Shape.call(this,!0);this._renderer.type="group";this.additions=[];this.subtractions=[];this.children=arguments};h.extend(e,{Children:d,InsertChildren:function(a){for(var c=0;c<a.length;c++)k.call(this,a[c],this)},RemoveChildren:function(a){for(var c=0;c<a.length;c++)k.call(this,a[c])},OrderChildren:function(a){this._flagOrder=!0},MakeObservable:function(a){var g=c.Path.Properties.slice(0),k=h.indexOf(g,"opacity");0<=k&&(g.splice(k,1),Object.defineProperty(a,"opacity",{enumerable:!0, +get:function(){return this._opacity},set:function(a){this._flagOpacity=this._opacity!=a;this._opacity=a}}));c.Shape.MakeObservable(a);e.MakeGetterSetters(a,g);Object.defineProperty(a,"children",{enumerable:!0,get:function(){return this._children},set:function(a){var g=h.bind(e.InsertChildren,this),f=h.bind(e.RemoveChildren,this),k=h.bind(e.OrderChildren,this);this._children&&this._children.unbind();this._children=new d(a);this._children.bind(c.Events.insert,g);this._children.bind(c.Events.remove, +f);this._children.bind(c.Events.order,k)}});Object.defineProperty(a,"mask",{enumerable:!0,get:function(){return this._mask},set:function(a){this._mask=a;this._flagMask=!0;a.clip||(a.clip=!0)}})},MakeGetterSetters:function(a,c){h.isArray(c)||(c=[c]);h.each(c,function(c){e.MakeGetterSetter(a,c)})},MakeGetterSetter:function(a,c){var d="_"+c;Object.defineProperty(a,c,{enumerable:!0,get:function(){return this[d]},set:function(a){this[d]=a;h.each(this.children,function(d){d[c]=a})}})}});h.extend(e.prototype, +c.Shape.prototype,{_flagAdditions:!1,_flagSubtractions:!1,_flagOrder:!1,_flagOpacity:!0,_flagMask:!1,_fill:"#fff",_stroke:"#000",_linewidth:1,_opacity:1,_visible:!0,_cap:"round",_join:"round",_miter:4,_closed:!0,_curved:!1,_automatic:!0,_beginning:0,_ending:1,_mask:null,clone:function(a){a=a||this.parent;var c=new e,d=h.map(this.children,function(a){return a.clone(c)});c.add(d);c.opacity=this.opacity;this.mask&&(c.mask=this.mask);c.translation.copy(this.translation);c.rotation=this.rotation;c.scale= +this.scale;a&&a.add(c);return c},toObject:function(){var a={children:[],translation:this.translation.toObject(),rotation:this.rotation,scale:this.scale,opacity:this.opacity,mask:this.mask?this.mask.toObject():null};h.each(this.children,function(c,d){a.children[d]=c.toObject()},this);return a},corner:function(){var a=this.getBoundingClientRect(!0),c={x:a.left,y:a.top};this.children.forEach(function(a){a.translation.subSelf(c)});return this},center:function(){var a=this.getBoundingClientRect(!0);a.centroid= +{x:a.left+a.width/2,y:a.top+a.height/2};this.children.forEach(function(c){c.isShape&&c.translation.subSelf(a.centroid)});return this},getById:function(a){var c=function(a,d){if(a.id===d)return a;if(a.children)for(var e=a.children.length;e--;){var f=c(a.children[e],d);if(f)return f}};return c(this,a)||null},getByClassName:function(a){var c=[],d=function(a,e){-1!=a.classList.indexOf(e)?c.push(a):a.children&&a.children.forEach(function(a){d(a,e)});return c};return d(this,a)},getByType:function(a){var d= +[],e=function(a,g){for(var f in a.children)a.children[f]instanceof g?d.push(a.children[f]):a.children[f]instanceof c.Group&&e(a.children[f],g);return d};return e(this,a)},add:function(a){a=a instanceof Array?a.slice():h.toArray(arguments);for(var c=0;c<a.length;c++)a[c]&&a[c].id&&this.children.push(a[c]);return this},remove:function(a){var c=this.parent;if(0>=arguments.length&&c)return c.remove(this),this;a=a instanceof Array?a.slice():h.toArray(arguments);for(c=0;c<a.length;c++)a[c]&&this.children.ids[a[c].id]&& +this.children.splice(h.indexOf(this.children,a[c]),1);return this},getBoundingClientRect:function(a){var c;this._update(!0);var d=Infinity,e=-Infinity,k=Infinity,v=-Infinity;this.children.forEach(function(f){/(linear-gradient|radial-gradient|gradient)/.test(f._renderer.type)||(c=f.getBoundingClientRect(a),h.isNumber(c.top)&&h.isNumber(c.left)&&h.isNumber(c.right)&&h.isNumber(c.bottom)&&(k=m(c.top,k),d=m(c.left,d),e=l(c.right,e),v=l(c.bottom,v)))},this);return{top:k,left:d,right:e,bottom:v,width:e- +d,height:v-k}},noFill:function(){this.children.forEach(function(a){a.noFill()});return this},noStroke:function(){this.children.forEach(function(a){a.noStroke()});return this},subdivide:function(){var a=arguments;this.children.forEach(function(c){c.subdivide.apply(c,a)});return this},flagReset:function(){this._flagAdditions&&(this.additions.length=0,this._flagAdditions=!1);this._flagSubtractions&&(this.subtractions.length=0,this._flagSubtractions=!1);this._flagOrder=this._flagMask=this._flagOpacity= +!1;c.Shape.prototype.flagReset.call(this);return this}});e.MakeObservable(e.prototype)})(("undefined"!==typeof global?global:this).Two); diff --git a/src/spdk/doc/user_guides.md b/src/spdk/doc/user_guides.md new file mode 100644 index 000000000..e392c81e7 --- /dev/null +++ b/src/spdk/doc/user_guides.md @@ -0,0 +1,12 @@ +# User Guides {#user_guides} + +- @subpage system_configuration +- @subpage libraries +- @subpage app_overview +- @subpage iscsi +- @subpage nvmf +- @subpage vhost +- @subpage bdev +- @subpage blobfs +- @subpage jsonrpc +- @subpage jsonrpc_proxy diff --git a/src/spdk/doc/userspace.md b/src/spdk/doc/userspace.md new file mode 100644 index 000000000..54ba1bdfa --- /dev/null +++ b/src/spdk/doc/userspace.md @@ -0,0 +1,97 @@ +# User Space Drivers {#userspace} + +# Controlling Hardware From User Space {#userspace_control} + +Much of the documentation for SPDK talks about _user space drivers_, so it's +important to understand what that means at a technical level. First and +foremost, a _driver_ is software that directly controls a particular device +attached to a computer. Second, operating systems segregate the system's +virtual memory into two categories of addresses based on privilege level - +[kernel space and user space](https://en.wikipedia.org/wiki/User_space). This +separation is aided by features on the CPU itself that enforce memory +separation called +[protection rings](https://en.wikipedia.org/wiki/Protection_ring). Typically, +drivers run in kernel space (i.e. ring 0 on x86). SPDK contains drivers that +instead are designed to run in user space, but they still interface directly +with the hardware device that they are controlling. + +In order for SPDK to take control of a device, it must first instruct the +operating system to relinquish control. This is often referred to as unbinding +the kernel driver from the device and on Linux is done by +[writing to a file in sysfs](https://lwn.net/Articles/143397/). +SPDK then rebinds the driver to one of two special device drivers that come +bundled with Linux - +[uio](https://www.kernel.org/doc/html/latest/driver-api/uio-howto.html) or +[vfio](https://www.kernel.org/doc/Documentation/vfio.txt). These two drivers +are "dummy" drivers in the sense that they mostly indicate to the operating +system that the device has a driver bound to it so it won't automatically try +to re-bind the default driver. They don't actually initialize the hardware in +any way, nor do they even understand what type of device it is. The primary +difference between uio and vfio is that vfio is capable of programming the +platform's +[IOMMU](https://en.wikipedia.org/wiki/Input%E2%80%93output_memory_management_unit), +which is a critical piece of hardware for ensuring memory safety in user space +drivers. See @ref memory for full details. + +Once the device is unbound from the operating system kernel, the operating +system can't use it anymore. For example, if you unbind an NVMe device on Linux, +the devices corresponding to it such as /dev/nvme0n1 will disappear. It further +means that filesystems mounted on the device will also be removed and kernel +filesystems can no longer interact with the device. In fact, the entire kernel +block storage stack is no longer involved. Instead, SPDK provides re-imagined +implementations of most of the layers in a typical operating system storage +stack all as C libraries that can be directly embedded into your application. +This includes a [block device abstraction layer](@ref bdev) primarily, but +also [block allocators](@ref blob) and [filesystem-like components](@ref blobfs). + +User space drivers utilize features in uio or vfio to map the +[PCI BAR](https://en.wikipedia.org/wiki/PCI_configuration_space) for the device +into the current process, which allows the driver to perform +[MMIO](https://en.wikipedia.org/wiki/Memory-mapped_I/O) directly. The SPDK @ref +nvme, for instance, maps the BAR for the NVMe device and then follows along +with the +[NVMe Specification](http://nvmexpress.org/wp-content/uploads/NVM_Express_Revision_1.3.pdf) +to initialize the device, create queue pairs, and ultimately send I/O. + +# Interrupts {#userspace_interrupts} + +SPDK polls devices for completions instead of waiting for interrupts. There +are a number of reasons for doing this: 1) practically speaking, routing an +interrupt to a handler in a user space process just isn't feasible for most +hardware designs, 2) interrupts introduce software jitter and have significant +overhead due to forced context switches. Operations in SPDK are almost +universally asynchronous and allow the user to provide a callback on +completion. The callback is called in response to the user calling a function +to poll for completions. Polling an NVMe device is fast because only host +memory needs to be read (no MMIO) to check a queue pair for a bit flip and +technologies such as Intel's +[DDIO](https://www.intel.com/content/www/us/en/io/data-direct-i-o-technology.html) +will ensure that the host memory being checked is present in the CPU cache +after an update by the device. + +# Threading {#userspace_threading} + +NVMe devices expose multiple queues for submitting requests to the hardware. +Separate queues can be accessed without coordination, so software can send +requests to the device from multiple threads of execution in parallel without +locks. Unfortunately, kernel drivers must be designed to handle I/O coming +from lots of different places either in the operating system or in various +processes on the system, and the thread topology of those processes changes +over time. Most kernel drivers elect to map hardware queues to cores (as close +to 1:1 as possible), and then when a request is submitted they look up the +correct hardware queue for whatever core the current thread happens to be +running on. Often, they'll need to either acquire a lock around the queue or +temporarily disable interrupts to guard against preemption from threads +running on the same core, which can be expensive. This is a large improvement +from older hardware interfaces that only had a single queue or no queue at +all, but still isn't always optimal. + +A user space driver, on the other hand, is embedded into a single application. +This application knows exactly how many threads (or processes) exist +because the application created them. Therefore, the SPDK drivers choose to +expose the hardware queues directly to the application with the requirement +that a hardware queue is only ever accessed from one thread at a time. In +practice, applications assign one hardware queue to each thread (as opposed to +one hardware queue per core in kernel drivers). This guarantees that the thread +can submit requests without having to perform any sort of coordination (i.e. +locking) with the other threads in the system. diff --git a/src/spdk/doc/vagrant.md b/src/spdk/doc/vagrant.md new file mode 100644 index 000000000..f362a89d2 --- /dev/null +++ b/src/spdk/doc/vagrant.md @@ -0,0 +1,168 @@ +# Vagrant Development Environment {#vagrant} + +# Introduction {#vagrant_intro} + +[Vagrant](https://www.vagrantup.com/) provides a quick way to get a basic +NVMe enabled virtual machine sandbox running without the need for any +special hardware. +The Vagrant environment for SPDK has support for a variety of Linux distributions as well as FreeBSD. +Run scripts/vagrant/create_vbox.sh -h to see the complete list. +This environment requires Vagrant 1.9.4 or newer and +VirtualBox 5.1 or newer with the matching VirtualBox extension pack. + +Note: If you are behind a corporate firewall, set `http_proxy` and `https_proxy` in +your environment before trying to start up the VM. Also make sure that you +have installed the optional vagrant module `vagrant-proxyconf`: + +~~~{.sh} +export http_proxy=... +export https_proxy=... +vagrant plugin install vagrant-proxyconf +~~~ + +In case you want use kvm/libvirt you should also install `vagrant-libvirt` + +# VM Configuration {#vagrant_config} + +To create a configured VM with vagrant you need to run `create_vbox.sh` script. + +Basically, the script will create a new sub-directory based on distribution you choose, +copy the vagrant configuration file (a.k.a. `Vagrantfile`) to it, +and run `vagrant up` with some settings defined by the script arguments. + +By default, the VM created is configured with: + +- 2 vCPUs +- 4G of RAM +- 2 NICs (1 x NAT - host access, 1 x private network) + +In order to modify some advanced settings like provisioning and rsyncing, +you may want to change Vagrantfile source. + +For additional support, use the Vagrant help function to learn how to destroy, restart, etc. +Further below is sample output from a successful VM launch and execution of the NVMe hello +world example application. + +~~~{.sh} + vagrant --help +~~~ + +# Running An Example {#vagrant_example} + +The following shows sample output from starting up a Ubuntu18 VM, +compiling SPDK on it and running the NVMe sample application `hello_world`. +If you don't see the NVMe device as seen below in both the `lspci` output as well as the +application output, you likely have a VirtualBox and/or Vagrant +versioning issue. + +~~~{.sh} +user@dev-system:~$ cd spdk/scripts/vagrant +user@dev-system:~/spdk/scripts/vagrant$ ./create_vbox.sh ubuntu18 +mkdir: created directory '/home/user/spdk/scripts/vagrant/ubuntu18' +~/spdk/scripts/vagrant/ubuntu18 ~/spdk/scripts/vagrant +vagrant-proxyconf already installed... skipping +Bringing machine 'default' up with 'virtualbox' provider... +==> default: Box 'bento/ubuntu-18.04' could not be found. Attempting to find and install... + default: Box Provider: virtualbox + default: Box Version: 201803.24.0 +==> default: Loading metadata for box 'bento/ubuntu-18.04' + default: URL: https://vagrantcloud.com/bento/ubuntu-18.04 +==> default: Adding box 'bento/ubuntu-18.04' (v201803.24.0) for provider: virtualbox + default: Downloading: https://vagrantcloud.com/bento/boxes/ubuntu-18.04/versions/201803.24.0/providers/virtualbox.box +==> default: Box download is resuming from prior download progress +==> default: Successfully added box 'bento/ubuntu-18.04' (v201803.24.0) for 'virtualbox'! +==> default: Importing base box 'bento/ubuntu-18.04'... +==> default: Matching MAC address for NAT networking... +==> default: Setting the name of the VM: ubuntu18_default_1237088131451_82174 +==> default: Fixed port collision for 22 => 2222. Now on port 2202. +==> default: Clearing any previously set network interfaces... +==> default: Preparing network interfaces based on configuration... + default: Adapter 1: nat + default: Adapter 2: hostonly +==> default: Forwarding ports... + default: 22 (guest) => 2202 (host) (adapter 1) +==> default: Running 'pre-boot' VM customizations... +==> default: Booting VM... +==> default: Waiting for machine to boot. This may take a few minutes... + default: SSH address: 127.0.0.1:2202 + default: SSH username: vagrant + default: SSH auth method: private key + default: Warning: Remote connection disconnect. Retrying... + default: Warning: Connection reset. Retrying... +<<some output trimmed>> + default: Warning: Connection reset. Retrying... + default: Warning: Remote connection disconnect. Retrying... + default: + default: Vagrant insecure key detected. Vagrant will automatically replace + default: this with a newly generated keypair for better security. + default: + default: Inserting generated public key within guest... + default: Removing insecure key from the guest if it's present... + default: Key inserted! Disconnecting and reconnecting using new SSH key... +==> default: Machine booted and ready! +==> default: Checking for guest additions in VM... +==> default: Configuring and enabling network interfaces... +==> default: Configuring proxy for Apt... +==> default: Configuring proxy environment variables... +==> default: Rsyncing folder: /home/user/spdk/ => /home/vagrant/spdk_repo/spdk +==> default: Mounting shared folders... + default: /vagrant => /home/user/spdk/scripts/vagrant/ubuntu18 +==> default: Running provisioner: file... + + SUCCESS! + + cd to ubuntu18 and type "vagrant ssh" to use. + Use vagrant "suspend" and vagrant "resume" to stop and start. + Use vagrant "destroy" followed by "rm -rf ubuntu18" to destroy all trace of vm. +~~~ + +Check the enviroment. + +~~~{.sh} +user@dev-system:~/spdk/scripts/vagrant$ cd ubuntu18 +user@dev-system:~/spdk/scripts/vagrant/ubuntu18$ vagrant ssh +Welcome to Ubuntu Bionic Beaver (development branch) (GNU/Linux 4.15.0-12-generic x86_64) +<<some output trimmed>> +vagrant@vagrant:~$ lspci | grep "Non-Volatile" +00:0e.0 Non-Volatile memory controller: InnoTek Systemberatung GmbH Device 4e56 +vagrant@vagrant:~$ ls +spdk_repo +~~~ + +Compiling SPDK and running an example. + +~~~{.sh} +vagrant@vagrant:~/spdk_repo/spdk$ sudo apt update +<<output trimmed>> +vagrant@vagrant:~/spdk_repo/spdk$ sudo scripts/pkgdep.sh +<<output trimmed>> + +vagrant@vagrant:~/spdk_repo/spdk$ ./configure +Creating mk/config.mk...done. +Type 'make' to build. + +vagrant@vagrant:~/spdk_repo/spdk$ make +<<output trimmed>> + +vagrant@vagrant:~/spdk_repo/spdk$ sudo ./scripts/setup.sh +0000:00:0e.0 (80ee 4e56): nvme -> uio_pci_generic + +vagrant@vagrant:~/spdk_repo/spdk$ sudo build/examples/hello_world +Starting SPDK v18.10-pre / DPDK 18.05.0 initialization... +[ DPDK EAL parameters: hello_world -c 0x1 --legacy-mem --file-prefix=spdk0 --base-virtaddr=0x200000000000 --proc-type=auto ] +EAL: Detected 4 lcore(s) +EAL: Detected 1 NUMA nodes +EAL: Auto-detected process type: PRIMARY +EAL: Multi-process socket /var/run/dpdk/spdk0/mp_socket +EAL: Probing VFIO support... +Initializing NVMe Controllers +EAL: PCI device 0000:00:0e.0 on NUMA socket 0 +EAL: probe driver: 80ee:4e56 spdk_nvme +Attaching to 0000:00:0e.0 +Attached to 0000:00:0e.0 +Using controller ORCL-VBOX-NVME-VER12 (VB1234-56789 ) with 1 namespaces. + Namespace ID: 1 size: 1GB +Initialization complete. +INFO: using host memory buffer for IO +Hello world! +~~~ diff --git a/src/spdk/doc/vhost.md b/src/spdk/doc/vhost.md new file mode 100644 index 000000000..71710441d --- /dev/null +++ b/src/spdk/doc/vhost.md @@ -0,0 +1,416 @@ +# vhost Target {#vhost} + +# Table of Contents {#vhost_toc} + +- @ref vhost_intro +- @ref vhost_prereqs +- @ref vhost_start +- @ref vhost_config +- @ref vhost_qemu_config +- @ref vhost_example +- @ref vhost_advanced_topics +- @ref vhost_bugs + +# Introduction {#vhost_intro} + +A vhost target provides a local storage service as a process running on a local machine. +It is capable of exposing virtualized block devices to QEMU instances or other arbitrary +processes. + +The following diagram presents how QEMU-based VM communicates with SPDK Vhost-SCSI device. + + + +The diagram, and the vhost protocol itself is described in @ref vhost_processing doc. + +SPDK provides an accelerated vhost target by applying the same user space and polling +techniques as other components in SPDK. Since SPDK is polling for vhost submissions, +it can signal the VM to skip notifications on submission. This avoids VMEXITs on I/O +submission and can significantly reduce CPU usage in the VM on heavy I/O workloads. + +# Prerequisites {#vhost_prereqs} + +This guide assumes the SPDK has been built according to the instructions in @ref +getting_started. The SPDK vhost target is built with the default configure options. + +## Vhost Command Line Parameters {#vhost_cmd_line_args} + +Additional command line flags are available for Vhost target. + +Param | Type | Default | Description +-------- | -------- | ---------------------- | ----------- +-S | string | $PWD | directory where UNIX domain sockets will be created + +## Supported Guest Operating Systems + +The guest OS must contain virtio-scsi or virtio-blk drivers. Most Linux and FreeBSD +distributions include virtio drivers. +[Windows virtio drivers](https://fedoraproject.org/wiki/Windows_Virtio_Drivers) must be +installed separately. The SPDK vhost target has been tested with recent versions of Ubuntu, +Fedora, and Windows + +## QEMU + +Userspace vhost-scsi target support was added to upstream QEMU in v2.10.0. Run +the following command to confirm your QEMU supports userspace vhost-scsi. + +~~~{.sh} +qemu-system-x86_64 -device vhost-user-scsi-pci,help +~~~ + +Userspace vhost-blk target support was added to upstream QEMU in v2.12.0. Run +the following command to confirm your QEMU supports userspace vhost-blk. + +~~~{.sh} +qemu-system-x86_64 -device vhost-user-blk-pci,help +~~~ + +Userspace vhost-nvme target was added as experimental feature for SPDK 18.04 +release, patches for QEMU are available in SPDK's QEMU repository only. + +Run the following command to confirm your QEMU supports userspace vhost-nvme. + +~~~{.sh} +qemu-system-x86_64 -device vhost-user-nvme,help +~~~ + +# Starting SPDK vhost target {#vhost_start} + +First, run the SPDK setup.sh script to setup some hugepages for the SPDK vhost target +application. This will allocate 4096MiB (4GiB) of hugepages, enough for the SPDK +vhost target and the virtual machine. + +~~~{.sh} +HUGEMEM=4096 scripts/setup.sh +~~~ + +Next, start the SPDK vhost target application. The following command will start vhost +on CPU cores 0 and 1 (cpumask 0x3) with all future socket files placed in /var/tmp. +Vhost will fully occupy given CPU cores for I/O polling. Particular vhost devices can +be restricted to run on a subset of these CPU cores. See @ref vhost_vdev_create for +details. + +~~~{.sh} +build/bin/vhost -S /var/tmp -m 0x3 +~~~ + +To list all available vhost options use the following command. + +~~~{.sh} +build/bin/vhost -h +~~~ + +# SPDK Configuration {#vhost_config} + +## Create bdev (block device) {#vhost_bdev_create} + +SPDK bdevs are block devices which will be exposed to the guest OS. +For vhost-scsi, bdevs are exposed as SCSI LUNs on SCSI devices attached to the +vhost-scsi controller in the guest OS. +For vhost-blk, bdevs are exposed directly as block devices in the guest OS and are +not associated at all with SCSI. + +SPDK supports several different types of storage backends, including NVMe, +Linux AIO, malloc ramdisk and Ceph RBD. Refer to @ref bdev for +additional information on configuring SPDK storage backends. + +This guide will use a malloc bdev (ramdisk) named Malloc0. The following RPC +will create a 64MB malloc bdev with 512-byte block size. + +~~~{.sh} +scripts/rpc.py bdev_malloc_create 64 512 -b Malloc0 +~~~ + +## Create a vhost device {#vhost_vdev_create} + +### Vhost-SCSI + +The following RPC will create a vhost-scsi controller which can be accessed +by QEMU via /var/tmp/vhost.0. At the time of creation the controller will be +bound to a single CPU core with the smallest number of vhost controllers. +The optional `--cpumask` parameter can directly specify which cores should be +taken into account - in this case always CPU 0. To achieve optimal performance +on NUMA systems, the cpumask should specify cores on the same CPU socket as its +associated VM. + +~~~{.sh} +scripts/rpc.py vhost_create_scsi_controller --cpumask 0x1 vhost.0 +~~~ + +The following RPC will attach the Malloc0 bdev to the vhost.0 vhost-scsi +controller. Malloc0 will appear as a single LUN on a SCSI device with +target ID 0. SPDK Vhost-SCSI device currently supports only one LUN per SCSI target. +Additional LUNs can be added by specifying a different target ID. + +~~~{.sh} +scripts/rpc.py vhost_scsi_controller_add_target vhost.0 0 Malloc0 +~~~ + +To remove a bdev from a vhost-scsi controller use the following RPC: + +~~~{.sh} +scripts/rpc.py vhost_scsi_controller_remove_target vhost.0 0 +~~~ + +### Vhost-BLK + +The following RPC will create a vhost-blk device exposing Malloc0 bdev. +The device will be accessible to QEMU via /var/tmp/vhost.1. All the I/O polling +will be pinned to the least occupied CPU core within given cpumask - in this case +always CPU 0. For NUMA systems, the cpumask should specify cores on the same CPU +socket as its associated VM. + +~~~{.sh} +scripts/rpc.py vhost_create_blk_controller --cpumask 0x1 vhost.1 Malloc0 +~~~ + +It is also possible to create a read-only vhost-blk device by specifying an +extra `-r` or `--readonly` parameter. + +~~~{.sh} +scripts/rpc.py vhost_create_blk_controller --cpumask 0x1 -r vhost.1 Malloc0 +~~~ + +### Vhost-NVMe (experimental) + +The following RPC will attach the Malloc0 bdev to the vhost.0 vhost-nvme +controller. Malloc0 will appear as Namespace 1 of vhost.0 controller. Users +can use `--cpumask` parameter to specify which cores should be used for this +controller. Users must specify the maximum I/O queues supported for the +controller, at least 1 Namespace is required for each controller. + +~~~{.sh} +$rpc_py vhost_create_nvme_controller --cpumask 0x1 vhost.2 16 +$rpc_py vhost_nvme_controller_add_ns vhost.2 Malloc0 +~~~ + +Users can use the following command to remove the controller, all the block +devices attached to controller's Namespace will be removed automatically. + +~~~{.sh} +$rpc_py vhost_delete_controller vhost.2 +~~~ + +## QEMU {#vhost_qemu_config} + +Now the virtual machine can be started with QEMU. The following command-line +parameters must be added to connect the virtual machine to its vhost controller. + +First, specify the memory backend for the virtual machine. Since QEMU must +share the virtual machine's memory with the SPDK vhost target, the memory +must be specified in this format with share=on. + +~~~{.sh} +-object memory-backend-file,id=mem,size=1G,mem-path=/dev/hugepages,share=on +-numa node,memdev=mem +~~~ + +Second, ensure QEMU boots from the virtual machine image and not the +SPDK malloc block device by specifying bootindex=0 for the boot image. + +~~~{.sh} +-drive file=guest_os_image.qcow2,if=none,id=disk +-device ide-hd,drive=disk,bootindex=0 +~~~ + +Finally, specify the SPDK vhost devices: + +### Vhost-SCSI + +~~~{.sh} +-chardev socket,id=char0,path=/var/tmp/vhost.0 +-device vhost-user-scsi-pci,id=scsi0,chardev=char0 +~~~ + +### Vhost-BLK + +~~~{.sh} +-chardev socket,id=char1,path=/var/tmp/vhost.1 +-device vhost-user-blk-pci,id=blk0,chardev=char1 +~~~ + +### Vhost-NVMe (experimental) + +~~~{.sh} +-chardev socket,id=char2,path=/var/tmp/vhost.2 +-device vhost-user-nvme,id=nvme0,chardev=char2,num_io_queues=4 +~~~ + +## Example output {#vhost_example} + +This example uses an NVMe bdev alongside Mallocs. SPDK vhost application is started +on CPU cores 0 and 1, QEMU on cores 2 and 3. + +~~~{.sh} +host:~# HUGEMEM=2048 ./scripts/setup.sh +0000:01:00.0 (8086 0953): nvme -> vfio-pci +~~~ + +~~~{.sh} +host:~# ./build/bin/vhost -S /var/tmp -s 1024 -m 0x3 & +Starting DPDK 17.11.0 initialization... +[ DPDK EAL parameters: vhost -c 3 -m 1024 --master-lcore=1 --file-prefix=spdk_pid156014 ] +EAL: Detected 48 lcore(s) +EAL: Probing VFIO support... +EAL: VFIO support initialized +app.c: 369:spdk_app_start: *NOTICE*: Total cores available: 2 +reactor.c: 668:spdk_reactors_init: *NOTICE*: Occupied cpu socket mask is 0x1 +reactor.c: 424:_spdk_reactor_run: *NOTICE*: Reactor started on core 1 on socket 0 +reactor.c: 424:_spdk_reactor_run: *NOTICE*: Reactor started on core 0 on socket 0 +~~~ + +~~~{.sh} +host:~# ./scripts/rpc.py bdev_nvme_attach_controller -b Nvme0 -t pcie -a 0000:01:00.0 +EAL: PCI device 0000:01:00.0 on NUMA socket 0 +EAL: probe driver: 8086:953 spdk_nvme +EAL: using IOMMU type 1 (Type 1) +~~~ + +~~~{.sh} +host:~# ./scripts/rpc.py bdev_malloc_create 128 4096 Malloc0 +Malloc0 +~~~ + +~~~{.sh} +host:~# ./scripts/rpc.py vhost_create_scsi_controller --cpumask 0x1 vhost.0 +VHOST_CONFIG: vhost-user server: socket created, fd: 21 +VHOST_CONFIG: bind to /var/tmp/vhost.0 +vhost.c: 596:spdk_vhost_dev_construct: *NOTICE*: Controller vhost.0: new controller added +~~~ + +~~~{.sh} +host:~# ./scripts/rpc.py vhost_scsi_controller_add_target vhost.0 0 Nvme0n1 +vhost_scsi.c: 840:spdk_vhost_scsi_dev_add_tgt: *NOTICE*: Controller vhost.0: defined target 'Target 0' using lun 'Nvme0' + +~~~ + +~~~{.sh} +host:~# ./scripts/rpc.py vhost_scsi_controller_add_target vhost.0 1 Malloc0 +vhost_scsi.c: 840:spdk_vhost_scsi_dev_add_tgt: *NOTICE*: Controller vhost.0: defined target 'Target 1' using lun 'Malloc0' +~~~ + +~~~{.sh} +host:~# ./scripts/rpc.py bdev_malloc_create 64 512 -b Malloc1 +Malloc1 +~~~ + +~~~{.sh} +host:~# ./scripts/rpc.py vhost_create_blk_controller --cpumask 0x2 vhost.1 Malloc1 +vhost_blk.c: 719:spdk_vhost_blk_construct: *NOTICE*: Controller vhost.1: using bdev 'Malloc1' +~~~ + +~~~{.sh} +host:~# taskset -c 2,3 qemu-system-x86_64 \ + --enable-kvm \ + -cpu host -smp 2 \ + -m 1G -object memory-backend-file,id=mem0,size=1G,mem-path=/dev/hugepages,share=on -numa node,memdev=mem0 \ + -drive file=guest_os_image.qcow2,if=none,id=disk \ + -device ide-hd,drive=disk,bootindex=0 \ + -chardev socket,id=spdk_vhost_scsi0,path=/var/tmp/vhost.0 \ + -device vhost-user-scsi-pci,id=scsi0,chardev=spdk_vhost_scsi0,num_queues=4 \ + -chardev socket,id=spdk_vhost_blk0,path=/var/tmp/vhost.1 \ + -device vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=4 +~~~ + +Please note the following two commands are run on the guest VM. + +~~~{.sh} +guest:~# lsblk --output "NAME,KNAME,MODEL,HCTL,SIZE,VENDOR,SUBSYSTEMS" +NAME KNAME MODEL HCTL SIZE VENDOR SUBSYSTEMS +sda sda QEMU HARDDISK 1:0:0:0 80G ATA block:scsi:pci + sda1 sda1 80G block:scsi:pci +sdb sdb NVMe disk 2:0:0:0 372,6G INTEL block:scsi:virtio:pci +sdc sdc Malloc disk 2:0:1:0 128M INTEL block:scsi:virtio:pci +vda vda 128M 0x1af4 block:virtio:pci +~~~ + +~~~{.sh} +guest:~# poweroff +~~~ + +~~~{.sh} +host:~# fg +<< CTRL + C >> +vhost.c:1006:session_shutdown: *NOTICE*: Exiting +~~~ + +We can see that `sdb` and `sdc` are SPDK vhost-scsi LUNs, and `vda` is SPDK a +vhost-blk disk. + +# Advanced Topics {#vhost_advanced_topics} + +## Multi-Queue Block Layer (blk-mq) {#vhost_multiqueue} + +For best performance use the Linux kernel block multi-queue feature with vhost. +To enable it on Linux, it is required to modify kernel options inside the +virtual machine. + +Instructions below for Ubuntu OS: + +1. `vi /etc/default/grub` +2. Make sure mq is enabled: `GRUB_CMDLINE_LINUX="scsi_mod.use_blk_mq=1"` +3. `sudo update-grub` +4. Reboot virtual machine + +To achieve better performance, make sure to increase number of cores +assigned to the VM and add `num_queues` parameter to the QEMU `device`. It should be enough +to set `num_queues=4` to saturate physical device. Adding too many queues might lead to SPDK +vhost performance degradation if many vhost devices are used because each device will require +additional `num_queues` to be polled. + +## Hot-attach/hot-detach {#vhost_hotattach} + +Hotplug/hotremove within a vhost controller is called hot-attach/detach. This is to +distinguish it from SPDK bdev hotplug/hotremove. E.g. if an NVMe bdev is attached +to a vhost-scsi controller, physically hotremoving the NVMe will trigger vhost-scsi +hot-detach. It is also possible to hot-detach a bdev manually via RPC - for example +when the bdev is about to be attached to another controller. See the details below. + +Please also note that hot-attach/detach is Vhost-SCSI-specific. There are no RPCs +to hot-attach/detach the bdev from a Vhost-BLK device. If Vhost-BLK device exposes +an NVMe bdev that is hotremoved, all the I/O traffic on that Vhost-BLK device will +be aborted - possibly flooding a VM with syslog warnings and errors. + +### Hot-attach + +Hot-attach is done by simply attaching a bdev to a vhost controller with a QEMU VM +already started. No other extra action is necessary. + +~~~{.sh} +scripts/rpc.py vhost_scsi_controller_add_target vhost.0 0 Malloc0 +~~~ + +### Hot-detach + +Just like hot-attach, the hot-detach is done by simply removing bdev from a controller +when QEMU VM is already started. + +~~~{.sh} +scripts/rpc.py vhost_scsi_controller_remove_target vhost.0 0 +~~~ + +Removing an entire bdev will hot-detach it from a controller as well. + +~~~{.sh} +scripts/rpc.py bdev_malloc_delete Malloc0 +~~~ + +# Known bugs and limitations {#vhost_bugs} + +## Vhost-NVMe (experimental) can only be supported with latest Linux kernel + +Vhost-NVMe target was designed for one new feature of NVMe 1.3 specification, Doorbell +Buffer Config Admin command, which is used for emulated NVMe controller only. Linux 4.12 +added this feature, so a new Guest kernel later than 4.12 is required to test this feature. + +## Windows virtio-blk driver before version 0.1.130-1 only works with 512-byte sectors + +The Windows `viostor` driver before version 0.1.130-1 is buggy and does not +correctly support vhost-blk devices with non-512-byte block size. +See the [bug report](https://bugzilla.redhat.com/show_bug.cgi?id=1411092) for +more information. + +## QEMU vhost-user-blk + +QEMU [vhost-user-blk](https://git.qemu.org/?p=qemu.git;a=commit;h=00343e4b54ba) is +supported from version 2.12. diff --git a/src/spdk/doc/vhost_processing.md b/src/spdk/doc/vhost_processing.md new file mode 100644 index 000000000..93a11633f --- /dev/null +++ b/src/spdk/doc/vhost_processing.md @@ -0,0 +1,197 @@ +# Virtualized I/O with Vhost-user {#vhost_processing} + +# Table of Contents {#vhost_processing_toc} + +- @ref vhost_processing_intro +- @ref vhost_processing_qemu +- @ref vhost_processing_init +- @ref vhost_processing_io_path +- @ref vhost_spdk_optimizations + +# Introduction {#vhost_processing_intro} + +This document is intended to provide an overview of how Vhost works behind the +scenes. Code snippets used in this document might have been simplified for the +sake of readability and should not be used as an API or implementation +reference. + +Reading from the +[Virtio specification](http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html): + +``` +The purpose of virtio and [virtio] specification is that virtual environments +and guests should have a straightforward, efficient, standard and extensible +mechanism for virtual devices, rather than boutique per-environment or per-OS +mechanisms. +``` + +Virtio devices use virtqueues to transport data efficiently. Virtqueue is a set +of three different single-producer, single-consumer ring structures designed to +store generic scatter-gatter I/O. Virtio is most commonly used in QEMU VMs, +where the QEMU itself exposes a virtual PCI device and the guest OS communicates +with it using a specific Virtio PCI driver. With only Virtio involved, it's +always the QEMU process that handles all I/O traffic. + +Vhost is a protocol for devices accessible via inter-process communication. +It uses the same virtqueue layout as Virtio to allow Vhost devices to be mapped +directly to Virtio devices. This allows a Vhost device, exposed by an SPDK +application, to be accessed directly by a guest OS inside a QEMU process with +an existing Virtio (PCI) driver. Only the configuration, I/O submission +notification, and I/O completion interruption are piped through QEMU. +See also @ref vhost_spdk_optimizations + +The initial vhost implementation is a part of the Linux kernel and uses ioctl +interface to communicate with userspace applications. What makes it possible for +SPDK to expose a vhost device is Vhost-user protocol. + +The [Vhost-user specification](https://git.qemu.org/?p=qemu.git;a=blob_plain;f=docs/interop/vhost-user.txt;hb=HEAD) +describes the protocol as follows: + +``` +[Vhost-user protocol] is aiming to complement the ioctl interface used to +control the vhost implementation in the Linux kernel. It implements the control +plane needed to establish virtqueue sharing with a user space process on the +same host. It uses communication over a Unix domain socket to share file +descriptors in the ancillary data of the message. + +The protocol defines 2 sides of the communication, master and slave. Master is +the application that shares its virtqueues, in our case QEMU. Slave is the +consumer of the virtqueues. + +In the current implementation QEMU is the Master, and the Slave is intended to +be a software Ethernet switch running in user space, such as Snabbswitch. + +Master and slave can be either a client (i.e. connecting) or server (listening) +in the socket communication. +``` + +SPDK vhost is a Vhost-user slave server. It exposes Unix domain sockets and +allows external applications to connect. + +# QEMU {#vhost_processing_qemu} + +One of major Vhost-user use cases is networking (DPDK) or storage (SPDK) +offload in QEMU. The following diagram presents how QEMU-based VM +communicates with SPDK Vhost-SCSI device. + + + +# Device initialization {#vhost_processing_init} + +All initialization and management information is exchanged using Vhost-user +messages. The connection always starts with the feature negotiation. Both +the Master and the Slave exposes a list of their implemented features and +upon negotiation they choose a common set of those. Most of these features are +implementation-related, but also regard e.g. multiqueue support or live migration. + +After the negotiation, the Vhost-user driver shares its memory, so that the vhost +device (SPDK) can access it directly. The memory can be fragmented into multiple +physically-discontiguous regions and Vhost-user specification puts a limit on +their number - currently 8. The driver sends a single message for each region with +the following data: + + * file descriptor - for mmap + * user address - for memory translations in Vhost-user messages (e.g. + translating vring addresses) + * guest address - for buffers addresses translations in vrings (for QEMU this + is a physical address inside the guest) + * user offset - positive offset for the mmap + * size + +The Master will send new memory regions after each memory change - usually +hotplug/hotremove. The previous mappings will be removed. + +Drivers may also request a device config, consisting of e.g. disk geometry. +Vhost-SCSI drivers, however, don't need to implement this functionality +as they use common SCSI I/O to inquiry the underlying disk(s). + +Afterwards, the driver requests the number of maximum supported queues and +starts sending virtqueue data, which consists of: + + * unique virtqueue id + * index of the last processed vring descriptor + * vring addresses (from user address space) + * call descriptor (for interrupting the driver after I/O completions) + * kick descriptor (to listen for I/O requests - unused by SPDK) + +If multiqueue feature has been negotiated, the driver has to send a specific +*ENABLE* message for each extra queue it wants to be polled. Other queues are +polled as soon as they're initialized. + +# I/O path {#vhost_processing_io_path} + +The Master sends I/O by allocating proper buffers in shared memory, filling +the request data, and putting guest addresses of those buffers into virtqueues. + +A Virtio-Block request looks as follows. + +``` +struct virtio_blk_req { + uint32_t type; // READ, WRITE, FLUSH (read-only) + uint64_t offset; // offset in the disk (read-only) + struct iovec buffers[]; // scatter-gatter list (read/write) + uint8_t status; // I/O completion status (write-only) +}; +``` +And a Virtio-SCSI request as follows. + +``` +struct virtio_scsi_req_cmd { + struct virtio_scsi_cmd_req *req; // request data (read-only) + struct iovec read_only_buffers[]; // scatter-gatter list for WRITE I/Os + struct virtio_scsi_cmd_resp *resp; // response data (write-only) + struct iovec write_only_buffers[]; // scatter-gatter list for READ I/Os +} +``` + +Virtqueue generally consists of an array of descriptors and each I/O needs +to be converted into a chain of such descriptors. A single descriptor can be +either readable or writable, so each I/O request consists of at least two +(request + response). + +``` +struct virtq_desc { + /* Address (guest-physical). */ + le64 addr; + /* Length. */ + le32 len; + +/* This marks a buffer as continuing via the next field. */ +#define VIRTQ_DESC_F_NEXT 1 +/* This marks a buffer as device write-only (otherwise device read-only). */ +#define VIRTQ_DESC_F_WRITE 2 + /* The flags as indicated above. */ + le16 flags; + /* Next field if flags & NEXT */ + le16 next; +}; +``` + +Legacy Virtio implementations used the name vring alongside virtqueue, and the +name vring is still used in virtio data structures inside the code. Instead of +`struct virtq_desc`, the `struct vring_desc` is much more likely to be found. + +The device after polling this descriptor chain needs to translate and transform +it back into the original request struct. It needs to know the request layout +up-front, so each device backend (Vhost-Block/SCSI) has its own implementation +for polling virtqueues. For each descriptor, the device performs a lookup in +the Vhost-user memory region table and goes through a gpa_to_vva translation +(guest physical address to vhost virtual address). SPDK enforces the request +and response data to be contained within a single memory region. I/O buffers +do not have such limitations and SPDK may automatically perform additional +iovec splitting and gpa_to_vva translations if required. After forming the request +structs, SPDK forwards such I/O to the underlying drive and polls for the +completion. Once I/O completes, SPDK vhost fills the response buffer with +proper data and interrupts the guest by doing an eventfd_write on the call +descriptor for proper virtqueue. There are multiple interrupt coalescing +features involved, but they are not be discussed in this document. + +## SPDK optimizations {#vhost_spdk_optimizations} + +Due to its poll-mode nature, SPDK vhost removes the requirement for I/O submission +notifications, drastically increasing the vhost server throughput and decreasing +the guest overhead of submitting an I/O. A couple of different solutions exist +to mitigate the I/O completion interrupt overhead (irqfd, vDPA), but those won't +be discussed in this document. For the highest performance, a poll-mode @ref virtio +can be used, as it suppresses all I/O completion interrupts, making the I/O +path to fully bypass the QEMU/KVM overhead. diff --git a/src/spdk/doc/virtio.md b/src/spdk/doc/virtio.md new file mode 100644 index 000000000..ae498f23e --- /dev/null +++ b/src/spdk/doc/virtio.md @@ -0,0 +1,30 @@ +# Virtio driver {#virtio} + +# Introduction {#virtio_intro} + +SPDK Virtio driver is a C library that allows communicating with Virtio devices. +It allows any SPDK application to become an initiator for (SPDK) vhost targets. + +The driver supports two different usage models: + +* PCI - This is the standard mode of operation when used in a guest virtual + machine, where QEMU has presented the virtio controller as a virtual PCI device. +* vhost-user - Can be used to connect to a vhost socket directly on the same host. + +The driver, just like the SPDK @ref vhost, is using pollers instead of standard +interrupts to check for an I/O response. If used inside a VM, it bypasses interrupt +and context switching overhead of QEMU and guest kernel, significantly boosting +the overall I/O performance. + +This Virtio library is currently used to implement two bdev modules: +@ref bdev_config_virtio_scsi and @ref bdev_config_virtio_blk. +These modules will export generic SPDK block devices usable by any SPDK application. + +# 2MB hugepages {#virtio_2mb} + +vhost-user specification puts a limitation on the number of "memory regions" used (8). +Each region corresponds to one file descriptor, and DPDK - as SPDK's memory allocator - +uses one file per hugepage by default. So *by default* this makes SPDK Virtio practical +with only 1GB hugepages. To run an SPDK app using Virtio initiator with 2MB hugepages +it is required to pass '-g' command-line option . This forces DPDK to create a single +non-physically-contiguous hugetlbfs file for all its memory. diff --git a/src/spdk/doc/vmd.md b/src/spdk/doc/vmd.md new file mode 100644 index 000000000..a9572c5b8 --- /dev/null +++ b/src/spdk/doc/vmd.md @@ -0,0 +1,122 @@ +# VMD driver {#vmd} + +# In this document {#vmd_toc} + +* @ref vmd_intro +* @ref vmd_interface +* @ref vmd_key_functions +* @ref vmd_config +* @ref vmd_app_frame +* @ref vmd_app +* @ref vmd_led + +# Introduction {#vmd_intro} + +Intel Volume Management Device is a hardware logic inside processor's Root Complex +responsible for management of PCIe NVMe SSDs. It provides robust Hot Plug support +and Status LED management. + +The driver is responsible for enumeration and hooking NVMe devices behind VMD +into SPDK PCIe subsystem. It also provides API for LED management and hot plug. + +# Public Interface {#vmd_interface} + +- spdk/vmd.h + +# Key Functions {#vmd_key_functions} + +Function | Description +--------------------------------------- | ----------- +spdk_vmd_init() | @copybrief spdk_vmd_init() +spdk_vmd_pci_device_list() | @copybrief spdk_vmd_pci_device_list() +spdk_vmd_set_led_state() | @copybrief spdk_vmd_set_led_state() +spdk_vmd_get_led_state() | @copybrief spdk_vmd_get_led_state() +spdk_vmd_hotplug_monitor() | @copybrief spdk_vmd_hotplug_monitor() + +# Configuration {#vmd_config} + +To enable VMD driver enumeration, the following steps are required: + +Check for available VMD devices (VMD needs to be properly set up in BIOS first). + +Example: +``` +$ lspci | grep 201d + +$ 5d:05.5 RAID bus controller: Intel Corporation Device 201d (rev 04) +$ d7:05.5 RAID bus controller: Intel Corporation Device 201d (rev 04) +``` + +Run setup.sh script with VMD devices set in PCI_WHITELIST. + +Example: +``` +$ PCI_WHITELIST="0000:5d:05.5 0000:d7:05.5" scripts/setup.sh +``` + +Check for available devices behind the VMD with spdk_lspci. + +Example: +``` +$ ./build/bin/spdk_lspci + + 5d0505:01:00.0 (8086 a54) (NVMe disk behind VMD) + 5d0505:03:00.0 (8086 a54) (NVMe disk behind VMD) + d70505:01:00.0 (8086 a54) (NVMe disk behind VMD) + d70505:03:00.0 (8086 a54) (NVMe disk behind VMD) + 0000:5d:05.5 (8086 201d) (VMD) + 0000:d7:05.5 (8086 201d) (VMD) +``` + +VMD NVMe BDF could be used as regular NVMe BDF. + +Example: +``` +$ ./scripts/rpc.py bdev_nvme_attach_controller -b NVMe1 -t PCIe -a 5d0505:01:00.0 +``` + +# Application framework {#vmd_app_frame} + +When application framework is used, VMD section needs to be added to the configuration file: + +Text config: +``` +[VMD] + Enable True +``` + +JSON config: +``` +{ + "subsystem": "vmd", + "config": [ + { + "method": "enable_vmd", + "params": {} + } + ] +} +``` + +or use RPC call before framework starts e.g. +``` +$ ./build/bin/spdk_tgt --wait_for_rpc +$ ./scripts/rpc.py enable_vmd +$ ./scripts/rpc.py framework_start_init +``` +# Applications w/o application framework {#vmd_app} + +To enable VMD enumeration in SPDK application that are not using application framework +e.g nvme/perf, nvme/identify -V flag is required - please refer to app help if it supports VMD. + +Applications need to call spdk_vmd_init() to enumerate NVMe devices behind the VMD prior to calling +spdk_nvme_(probe|connect). +To support hot plugs spdk_vmd_hotplug_monitor() needs to be called periodically. + +# LED management {#vmd_led} + +VMD LED utility in the [examples/vmd/led](https://github.com/spdk/spdk/tree/master/examples/vmd/led) +could be used to set LED states. + +In order to verify that a platform is correctly configured to support LED management, ledctl(8) can +be utilized. For instructions on how to use it, consult the manual page of this utility. diff --git a/src/spdk/doc/vpp_integration.md b/src/spdk/doc/vpp_integration.md new file mode 100644 index 000000000..3b09e5243 --- /dev/null +++ b/src/spdk/doc/vpp_integration.md @@ -0,0 +1,237 @@ +# Vector Packet Processing {#vpp_integration} + +VPP (part of [Fast Data - Input/Output](https://fd.io/) project) is an extensible +userspace framework providing networking functionality. It is built around the concept of +packet processing graph (see [What is VPP?](https://wiki.fd.io/view/VPP/What_is_VPP?)). + +Detailed instructions for **simplified steps 1-3** below, can be found on +VPP [Quick Start Guide](https://wiki.fd.io/view/VPP). + +*SPDK supports VPP version 19.04.2.* + +# 1. Building VPP (optional) {#vpp_build} + +*Please skip this step if using already built packages.* + +Clone and checkout VPP +~~~ +git clone https://gerrit.fd.io/r/vpp && cd vpp +git checkout v19.04.2 +~~~ + +Install VPP build dependencies +~~~ +make install-dep +~~~ + +Build and create .rpm packages +~~~ +make pkg-rpm +~~~ + +Alternatively, build and create .deb packages +~~~ +make bootstrap && make pkg-deb +~~~ + +Packages can be found in `vpp/build-root/` directory. + +For more in depth instructions please see Building section in +[VPP documentation](https://wiki.fd.io/view/VPP/Pulling,_Building,_Running,_Hacking_and_Pushing_VPP_Code#Building) + +# 2. Installing VPP {#vpp_install} + +Packages can be installed from a distribution repository or built in previous step. +Minimal set of packages consists of `vpp`, `vpp-lib` and `vpp-devel`. + +*Note: Please remove or modify /etc/sysctl.d/80-vpp.conf file with appropriate values +dependent on number of hugepages that will be used on system.* + +# 3. Running VPP {#vpp_run} + +VPP takes over any network interfaces that were bound to userspace driver, +for details please see DPDK guide on +[Binding and Unbinding Network Ports to/from the Kernel Modules](http://dpdk.org/doc/guides/linux_gsg/linux_drivers.html#binding-and-unbinding-network-ports-to-from-the-kernel-modules). + +VPP is installed as service and disabled by default. To start VPP with default config: +~~~ +sudo systemctl start vpp +~~~ + +Alternatively, use `vpp` binary directly +~~~ +sudo vpp unix {cli-listen /run/vpp/cli.sock} session { evt_qs_memfd_seg } socksvr { socket-name /run/vpp-api.sock } +~~~ + +# 4. Configure VPP {#vpp_config} + +VPP can be configured using a VPP startup file and the `vppctl` command; By default, the VPP startup file is `/etc/vpp/startup.conf`, however, you can pass any file with the `-c` vpp command argument. + +## Startup configuration + +Some key values from iSCSI point of view includes: + +CPU section (`cpu`): + +- `main-core <lcore>` -- logical CPU core used for main thread. +- `corelist-workers <lcore list>` -- logical CPU cores where worker threads are running. + +DPDK section (`dpdk`): + +- `num-rx-queues <num>` -- number of receive queues. +- `num-tx-queues <num>` -- number of transmit queues. +- `dev <PCI address>` -- whitelisted device. + +Session section (`session`): + +- `evt_qs_memfd_seg` -- uses a memfd segment for event queues. This is required for SPDK. + +Socket server session (`socksvr`): + +- `socket-name <path>` -- configure API socket filename (curently SPDK uses default path `/run/vpp-api.sock`). + +Plugins section (`plugins`): + +- `plugin <plugin name> { [enable|disable] }` -- enable or disable VPP plugin. + +### Example + +~~~ +unix { + nodaemon + cli-listen /run/vpp/cli.sock +} +cpu { + main-core 1 +} +session { + evt_qs_memfd_seg +} +socksvr { + socket-name /run/vpp-api.sock +} +plugins { + plugin default { disable } + plugin dpdk_plugin.so { enable } +} +~~~ + +## vppctl command tool + +The `vppctl` command tool allows users to control VPP at runtime via a command prompt +~~~ +sudo vppctl +~~~ + +Or, by sending single command directly. For example to display interfaces within VPP: +~~~ +sudo vppctl show interface +~~~ + +Useful commands: + +- `show interface` -- show interfaces settings, state and some basic statistics. +- `show interface address` -- show interfaces state and assigned addresses. + +- `set interface ip address <VPP interface> <Address>` -- set interfaces IP address. +- `set interface state <VPP interface> [up|down]` -- bring interface up or down. + +- `show errors` -- show error counts. + +## Example: Configure two interfaces to be available via VPP + +We want to configure two DPDK ports with PCI addresses 0000:09:00.1 and 0000:0b:00.1 +to be used as portals 10.0.0.1/24 and 10.10.0.1/24. + +In the VPP startup file (e.g. `/etc/vpp/startup.conf`), whitelist the interfaces +by specifying PCI addresses in section dpdk: +~~~ + dev 0000:09:00.1 + dev 0000:0b:00.1 +~~~ + +Bind PCI NICs to UIO driver (`igb_uio` or `uio_pci_generic`). + +Restart vpp and use vppctl tool to verify interfaces. +~~~ +$ vppctl show interface + Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count + +FortyGigabitEthernet9/0/1 1 down 9000/0/0/0 +FortyGigabitEthernetb/0/1 2 down 9000/0/0/0 +~~~ + +Set appropriate addresses and bring interfaces up: +~~~ +$ vppctl set interface ip address FortyGigabitEthernet9/0/1 10.0.0.1/24 +$ vppctl set interface state FortyGigabitEthernet9/0/1 up +$ vppctl set interface ip address FortyGigabitEthernetb/0/1 10.10.0.1/24 +$ vppctl set interface state FortyGigabitEthernetb/0/1 up +~~~ + +Verify configuration: +~~~ +$ vppctl show interface address +FortyGigabitEthernet9/0/1 (up): + L3 10.0.0.1/24 +FortyGigabitEthernetb/0/1 (up): + L3 10.10.0.1/24 +~~~ + +Now, both interfaces are ready to use. To verify conectivity you can ping +10.0.0.1 and 10.10.0.1 addresses from another machine. + +## Example: Tap interfaces on single host + +For functional test purposes a virtual tap interface can be created, +so no additional network hardware is required. +This will allow network communication between SPDK iSCSI target using VPP end of tap +and kernel iSCSI initiator using the kernel part of tap. A single host is used in this scenario. + +Create tap interface via VPP +~~~ + vppctl tap connect tap0 + vppctl set interface state tapcli-0 up + vppctl set interface ip address tapcli-0 10.0.0.1/24 + vppctl show int addr +~~~ + +Assign address on kernel interface +~~~ + sudo ip addr add 10.0.0.2/24 dev tap0 + sudo ip link set tap0 up +~~~ + +To verify connectivity +~~~ + ping 10.0.0.1 +~~~ + +# 5. Building SPDK with VPP {#vpp_built_into_spdk} + +Support for VPP can be built into SPDK by using configuration option. +~~~ +configure --with-vpp +~~~ + +Alternatively, directory with built libraries can be pointed at +and will be used for compilation instead of installed packages. +~~~ +configure --with-vpp=/path/to/vpp/repo/build-root/install-vpp-native/vpp +~~~ + +# 6. Running SPDK with VPP {#vpp_running_with_spdk} + +VPP application has to be started before SPDK application, in order to enable +usage of network interfaces. For example, if you use SPDK iSCSI target or +NVMe-oF target, after the initialization finishes, interfaces configured within +VPP will be available to be configured as portal addresses. + +Moreover, you do not need to specifiy which TCP sock implementation (e.g., posix, +VPP) to be used through configuration file or RPC call. Since SPDK program +automatically determines the protocol according to the configured portal addresses +info. For example, you can specify a Listen address in NVMe-oF subsystem +configuration such as "Listen TCP 10.0.0.1:4420". SPDK programs automatically +uses different implemenation to listen this provided portal info via posix or +vpp implemenation(if compiled in SPDK program), and only one implementation can +successfully listen on the provided portal. |
