diff options
24 files changed, 2086 insertions, 0 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..feb3c1c --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,46 @@ +<!--- Provide a general summary of the issue in the Title above --> + +<!-- Formatting tips: + +GitHub supports Markdown: https://guides.github.com/features/mastering-markdown/ +Multi-line code blocks either with three back ticks, or four space indent. + +``` +Stacktrace ... +<line> +<line> +``` +--> + +## Expected Behavior +<!--- If you're describing a bug, tell us what should happen --> +<!--- If you're suggesting a change/improvement, tell us how it should work --> + +## Current Behavior +<!--- If describing a bug, tell us what happens instead of the expected behavior --> +<!--- If suggesting a change/improvement, explain the difference from current behavior --> + +## Possible Solution +<!--- Not obligatory, but suggest a fix/reason for the bug, --> +<!--- or ideas how to implement: the addition or change --> + +## Steps to Reproduce (for bugs) +<!--- Provide a link to a live example, or an unambiguous set of steps to --> +<!--- reproduce this bug. Include configuration, logs, etc. to reproduce, if relevant --> +1. +2. +3. +4. + +## Context +<!--- How has this issue affected you? What are you trying to accomplish? --> +<!--- Providing context helps us come up with a solution that is most useful in the real world --> + +## Your Environment +<!--- Include as many relevant details about the environment you experienced the problem in --> +* Module version (System - About): +* Icinga Web 2 version and modules (System - About): +* Icinga 2 version (`icinga2 --version`): +* Operating System and version: +* Webserver, PHP versions: + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3511072 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +## Editors +/.idea/ +.*.sw[op] + +## PHP vendor artifacts +/vendor/ @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..08730ad --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +Icinga Web 2 Fileshipper module +=============================== + +The main purpose of this module is to extend [Icinga Director](https://github.com/icinga/icingaweb2-module-director) +using some of it's exported hooks. Based on them it offers an `Import Source` +able to deal with `CSV`, `JSON`, `YAML` and `XML` files. It also offers the +possibility to deploy hand-crafted [Icinga 2](https://github.com/Icinga/icinga2) +config files through the `Icinga Director`. + + + +For getting started please read our [Installation instructions](doc/02-Installation.md), +and then you should be ready to dive into [Import Source](doc/03-ImportSource.md) +definitions, [supported file formats](doc/11-FileFormats.md) or and hand-crafted +[Config File Shipping](doc/04-FileShipping.md). + +Changes +------- + +### v1.2.0 + +* FEATURE: PHP 8 support +* FEATURE: Give guidance on potential misconfiguration (#34) +* FEATURE: do not fail on malformed config file (#35) + +### v1.1.0 + +* FEATURE: Added XLSX file support + +### v1.0.1 + +* FEATURE: CSV files should give NULL for columns with empty strings (#6) +* FIX: Small documentation fix diff --git a/contrib/git-hooks/post-merge b/contrib/git-hooks/post-merge new file mode 100755 index 0000000..ae59a5c --- /dev/null +++ b/contrib/git-hooks/post-merge @@ -0,0 +1,35 @@ +#/usr/bin/env bash +# Trigger Director config rendering when something changed + +# This is a git hook that will run after `git pull`. In case any file was +# changed it will instruct Director to re-render the current configuration. +# +# Add this file to your git working directory as `.git/hooks/post-merge`. It +# has to executable (`chmod +x`), otherwise it wouldn't run. The `icingacli` +# has to be on your `PATH` when running `git pull`. +# +# TODO: verify whether HEAD@{1} instead of ORIG_HEAD should be preferred + +changed_files() { + git diff-tree -r --name-only --no-commit-id ORIG_HEAD HEAD +} + +had_changes() { + local changed_lines=$(changed_files | wc -l) + if [ "$changed_lines" -gt "0" ]; then + return 0 + else + return 1 + fi +} + +render_director_config() { + `icingacli director config render` +} + +run_if() { + $1 && eval "$2" +} + +run_if had_changes render_director_config + diff --git a/doc/02-Installation.md b/doc/02-Installation.md new file mode 100644 index 0000000..607996c --- /dev/null +++ b/doc/02-Installation.md @@ -0,0 +1,38 @@ +<a id="Installation"></a>Installation +===================================== + +## Requirements + +* Icinga Director (>= 1.1.0) +* php-xml for optional XML file support +* php-yaml for optional YAML file support +* php-zip for optional XLSX file support + +When running `PHP 7.x` you'll need the latest `2.x beta` version for [php-yaml](http://pecl.php.net/package/yaml). +In case your Linux distribution offers precompiled packages they should be fine, regardless of whether they ship `php-yaml` or `php-syck`. In either case please +let me know as I didn't test them on different operatingsystems yet. + +## Install the Fileshipper module + +As with any Icinga Web 2 module, installation is pretty straight-forward. In +case you're installing it from source, all you have to do is to drop the `fileshipper` +module in one of your module paths. You can examine (and set) the module path(s) +in `Configuration / Application`. In a typical environment you'll probably drop the +module to `/usr/share/icingaweb2/modules/fileshipper`. Please note that the directory +name MUST be `fileshipper` and not `icingaweb2-module-fileshipper` or anything else. +To do so you could run the following commands on the CLI. + +```sh +cd /usr/share/icingaweb2/modules +git clone https://github.com/Icinga/icingaweb2-module-fileshipper.git fileshipper +``` + +Last but not least go to `Configuration / Modules` and enable the `fileshipper` +module, or run the following command on the CLI: + +```sh +icingacli module enable fileshipper +``` + +That's all, now you are ready to define your first [Import Source](03-ImportSource.md) +definitions and to ship hand-crafted plain Icinga 2 [Config Files](04-FileShipping.md)! diff --git a/doc/03-ImportSource.md b/doc/03-ImportSource.md new file mode 100644 index 0000000..16db474 --- /dev/null +++ b/doc/03-ImportSource.md @@ -0,0 +1,73 @@ +<a id="ImportSource"></a>Use plain files as an Import Source +============================================================ + +The following screenshot shows a bunch of `Import Source` definitions defined +in the `Icinga Director` based on the this module: + + + +Hint: This chapter assumes that you are already familiar with the `Icinga Director` +[Import and Sync](https://github.com/Icinga/icingaweb2-module-director/blob/master/doc/70-Import-and-Sync.md) mechanism. + + +<a id="fileshipper-importsource"></a>Add a new Import Source +------------------------------------------------------------ + +Given that, the next steps should be fairly easy. From the Import Source overview +shown above click `Add import source` and choose the `Import from files` option +provided by the `fileshipper` module: + + + + +<a id="fileshipper-format"></a>Choose a File Format +--------------------------------------------------- + +Next opt for a specific `File Format`: + + + +Some file formats may ask for additional settings like the `CSV` one does: + + + +In case you want to learn more about supported file formats please read the +related [documentation](11-FileFormats.md). + +<a id="fileshipper-file"></a>Select Directory and File(s) +--------------------------------------------------------- + +You are also asked to choose a `Base Directory`: + + + +Initially, this list is empty. The module doesn't allow you to freely choose any +file on your system. You have to provide a safe set of base directories in your +`fileshipper`'s module config directory, usually `/etc/icingaweb2/modules/fileshipper`. +There you need to create an `imports.ini` that could look as follows: + +```ini +[A bunch of files] +basedir = "/var/cache/various_files" + +[Puppet facts store (YAML directory)] +basedir = "/var/cache/sample-nodes" +``` + +Now you are ready to choose a specific file: + + + +For some use-cases it might also be quite useful to import `all files` in a given +directory at once: + +<a id="fileshipper-puppet"></a>Special Use Case: Puppet +------------------------------------------------------- + + + +The example on the screenshot has been configured to import all hosts from a +Puppet-based environment. If there where a PuppetDB it would have made more sense +to use the [PuppetDB module](https://github.com/Thomas-Gelf/icingaweb2-module-puppetdb). +But without such, the `facts store` on the Puppet Master makes still a good data +source for this task. diff --git a/doc/04-FileShipping.md b/doc/04-FileShipping.md new file mode 100644 index 0000000..757b784 --- /dev/null +++ b/doc/04-FileShipping.md @@ -0,0 +1,39 @@ +<a id="FileShipping"></a>Plain config file shipping +=================================================== + +Icinga 2 offers a very powerful configuration format. By definition, it isn't +a configuration format at all, it's a DSL, a Domain Specific Language. There is +no chance to implement all of it's feature in a config tool like Icinga Director, +that would make as much sense as trying to create a GUI for Ruby or Perl. + +So, sometimes one might want to ship additional hand-crafted Icinga 2 config +files not generated by the Director. Sadly, that way you loose a lot of it's +features like tracking changes, keeping deployment history and more. That's what +this part of fileshipper has been designed for. It allows you to tell Director +to deploy additional config files without even trying to understand them. + +To enable this, please create a `directories.ini` in this modules config dir, +usually `/etc/icingaweb2/modules/fileshipper`: + +```ini +[custom-rules] +source = /usr/local/src/custom-rules.git +target = zones.d/director-global/custom-rules + +[test] +source = /tmp/replication-test +target = zones.d/director-global/having-fun +extensions = .conf .md +``` + +In this example all local files from `source` will be deployed to the given target +directory. Please take care, use of this module requires advanced understanding of +Icinga2 configuration. Per default only `.conf` files are synced, you can override +this with a custom space-separated list for the `extensions` parameter. + +In case you want to trigger specific actions like re-rendering or deploying the +config on changes you might want to have a look at our sample GIT hook in +[contrib/git-hooks/post-merge](../contrib/git-hooks/post-merge). + +When working with Puppet or similar, please consider notifying an `exec` resource +with `refreshonly` set to `true` instead. diff --git a/doc/11-FileFormats.md b/doc/11-FileFormats.md new file mode 100644 index 0000000..5147ceb --- /dev/null +++ b/doc/11-FileFormats.md @@ -0,0 +1,176 @@ +<a id="FileFormats"></a> Supported File Formats +=============================================== + +Depending on the installed libraries the Import Source currently supports +multiple file formats. In case you're missing any of the following formats +in your Director frontend please re-read our [Installation instructions](02-Installation.md). + + +CSV (Comma Separated Value) +--------------------------- + +[CSV](https://en.wikipedia.org/wiki/Comma-separated_values) is a not so well +defined data format, therefore the Import Source has to make some assumptions +and ask for optional settings. + +Basically, the rules to follow are: + +* a header line is required +* each row has to have as many columns as the header line +* defining a value enclosure is mandatory, but you do not have to use it in your + CSV files. So while your import source might be asking for `"hostname";"ip"`, + it would also accept `hostname;ip` in your source files +* a field delimiter is required, this is mostly comma (`,`) or semicolon (`;`). + You could also opt for other separators to fit your very custom file format + containing tabular data + +### Sample CSV file + +```csv +"host","address","location" +"csv1.example.com","127.0.0.1","HQ" +"csv2.example.com","127.0.0.2","HQ" +"csv3.example.com","127.0.0.3","HQ" +``` + +### More complex but perfectly valid CSV sample + +```csv +"hostname","ip address","location" +csv1,"127.0.0.2","H\"ome" +"csv2",127.0.0.2,"asdf" +"csv3","127.0.0.3","Nott"", at Home" +``` + + +JSON - JavaScript Object Notation +--------------------------------- + +[JSON](https://en.wikipedia.org/wiki/JSON) is a pretty simple standarized format +with good support among most scripting and programming languages. Nothing special +to say here, as it is easy to validate. + +### Simple JSON example + +This example shows an array of objects: + +```json +[{"host": "json1", "address": "127.0.0.1"},{"host": "json2", "address": "127.0.0.2"}] +``` + +This is the easiest machine-readable form of a JSON import file. + + +### Pretty-formatted extended JSON example + +Single-line JSON files are not very human-friendly, so you'll often meet pretty- +printed JSON. Such files also make also prefectly valid import candidates: + +```json +{ + "json1.example.com": { + "host": "json1.example.com", + "address": "127.0.0.1", + "location": "HQ", + "groups": [ "Linux Servers" ] + }, + "json2.example.com": { + "host": "json2.example.com", + "address": "127.0.0.2", + "location": "HQ", + "groups": [ "Windows Servers", "Lab" ] + } +} +``` + +Microsoft Excel +--------------- + +XSLX, the Microsoft Excel 2007+ format is supported since v1.1.0. + + +XML - Extensible Markup Language +-------------------------------- + +When working with [XML](https://en.wikipedia.org/wiki/XML) please try to ship +simple files as shown in the following example. We'd love to add more features +like better attribute support or [XPath](https://en.wikipedia.org/wiki/XPATH)- +based filters. In case you need such, please let us know and ship some exmple +data, helping us to better understand your requirements! + +### Simple XML example + +```xml +<?xml version="1.0" encoding="UTF-8" ?> +<hosts> + <host> + <name>xml1</name> + <address>127.0.0.1</address> + </host> + <host> + <name>xml2</name> + <address>127.0.0.2</address> + </host> +</hosts> +``` + + +YAML (Ain't Markup Language) +---------------------------- + +[YAML](https://en.wikipedia.org/wiki/YAML) is all but simple and well defined, +it allows you to write the same data in various ways. In case you opt for it +you might have your reasons and should already be familiar with how to generate +such files. + +### Simple YAML example + +So, let's start with a simple example: + +```yaml +--- +- host: "yaml1.example.com" + address: "127.0.0.1" + location: "HQ" +- host: "yaml2.example.com" + address: "127.0.0.2" + location: "HQ" +- host: "yaml3.example.com" + address: "127.0.0.3" + location: "HQ" +``` + +### Advanced YAML example + +People who think that NoSQL solves all there data problems tend to believe that +YAML solve all their config problems. So, YAML is pretty hip and widely used +among tools in hyped niches such as configuration management. I'll pick [Puppet](https://puppet.com/) +as an example, but this might work in a similar way for many other tools. + +Instead of a single YAML file I have to deal with a directory full of files in +this case. Our [Import Source documentation](03-ImportSource.md) already shows +how to configure such, here you can see part of such a file: + +```yaml +--- !ruby/object:Puppet::Node::Facts + name: foreman.localdomain + values: + architecture: x86_64 + timezone: CEST + kernel: Linux + system_uptime: "{\x22seconds\x22=>5415, \x22hours\x22=>1, \x22days\x22=>0, \x22uptime\x22=>\x221:30 hours\x22}" + domain: localdomain + virtual: kvm + is_virtual: "true" + hardwaremodel: x86_64 + operatingsystem: CentOS + facterversion: "2.4.6" + filesystems: xfs + fqdn: foreman.localdomain + hardwareisa: x86_64 + hostname: foreman +``` + +If this looks foreign to you don't worry, most similar constructs are handled in +a smooth way by the underlying YAML parser. + diff --git a/doc/screenshot/fileshipper/01_fileshipper-imports-overview.png b/doc/screenshot/fileshipper/01_fileshipper-imports-overview.png Binary files differnew file mode 100644 index 0000000..4a10487 --- /dev/null +++ b/doc/screenshot/fileshipper/01_fileshipper-imports-overview.png diff --git a/doc/screenshot/fileshipper/02_fileshipper-add-importsource.png b/doc/screenshot/fileshipper/02_fileshipper-add-importsource.png Binary files differnew file mode 100644 index 0000000..8bfbc15 --- /dev/null +++ b/doc/screenshot/fileshipper/02_fileshipper-add-importsource.png diff --git a/doc/screenshot/fileshipper/03_fileshipper-choose-format.png b/doc/screenshot/fileshipper/03_fileshipper-choose-format.png Binary files differnew file mode 100644 index 0000000..47fa9b6 --- /dev/null +++ b/doc/screenshot/fileshipper/03_fileshipper-choose-format.png diff --git a/doc/screenshot/fileshipper/04_fileshipper-choose-basedir.png b/doc/screenshot/fileshipper/04_fileshipper-choose-basedir.png Binary files differnew file mode 100644 index 0000000..a3258e3 --- /dev/null +++ b/doc/screenshot/fileshipper/04_fileshipper-choose-basedir.png diff --git a/doc/screenshot/fileshipper/05_fileshipper-csv-details.png b/doc/screenshot/fileshipper/05_fileshipper-csv-details.png Binary files differnew file mode 100644 index 0000000..9936e78 --- /dev/null +++ b/doc/screenshot/fileshipper/05_fileshipper-csv-details.png diff --git a/doc/screenshot/fileshipper/06_fileshipper-choose-file.png b/doc/screenshot/fileshipper/06_fileshipper-choose-file.png Binary files differnew file mode 100644 index 0000000..e20ba49 --- /dev/null +++ b/doc/screenshot/fileshipper/06_fileshipper-choose-file.png diff --git a/doc/screenshot/fileshipper/07_fileshipper-whole-directory.png b/doc/screenshot/fileshipper/07_fileshipper-whole-directory.png Binary files differnew file mode 100644 index 0000000..4f66c1c --- /dev/null +++ b/doc/screenshot/fileshipper/07_fileshipper-whole-directory.png diff --git a/library/Fileshipper/ProvidedHook/Director/ImportSource.php b/library/Fileshipper/ProvidedHook/Director/ImportSource.php new file mode 100644 index 0000000..c0d5ab5 --- /dev/null +++ b/library/Fileshipper/ProvidedHook/Director/ImportSource.php @@ -0,0 +1,613 @@ +<?php + +namespace Icinga\Module\Fileshipper\ProvidedHook\Director; + +use DirectoryIterator; +use Icinga\Application\Config; +use Icinga\Exception\ConfigurationError; +use Icinga\Exception\IcingaException; +use Icinga\Module\Director\Exception\JsonException; +use Icinga\Module\Director\Hook\ImportSourceHook; +use Icinga\Module\Director\Web\Form\QuickForm; +use Icinga\Module\Fileshipper\Xlsx\Workbook; +use RuntimeException; +use Symfony\Component\Yaml\Yaml; + +class ImportSource extends ImportSourceHook +{ + protected $db; + + protected $haveSymfonyYaml; + + public function getName() + { + return 'Import from files (fileshipper)'; + } + + /** + * @return object[] + * @throws ConfigurationError + * @throws IcingaException + */ + public function fetchData() + { + $basedir = $this->getSetting('basedir'); + $filename = $this->getSetting('file_name'); + $format = $this->getSetting('file_format'); + + if ($filename === '*') { + return $this->fetchFiles($basedir, $format); + } + + return (array) $this->fetchFile($basedir, $filename, $format); + } + + /** + * @return array + * @throws ConfigurationError + * @throws IcingaException + */ + public function listColumns() + { + return array_keys((array) current($this->fetchData())); + } + + /** + * @param QuickForm $form + * @return \Icinga\Module\Director\Forms\ImportSourceForm|QuickForm + * @throws \Zend_Form_Exception + */ + public static function addSettingsFormFields(QuickForm $form) + { + $form->addElement('select', 'file_format', array( + 'label' => $form->translate('File format'), + 'description' => $form->translate( + 'Available file formats, usually CSV, JSON, YAML and XML. Whether' + . ' all of those are available eventually depends on various' + . ' libraries installed on your system. Please have a look at' + . ' the documentation in case your list is not complete.' + ), + 'required' => true, + 'class' => 'autosubmit', + 'multiOptions' => $form->optionalEnum( + static::listAvailableFormats($form) + ), + )); + + /** @var \Icinga\Module\Director\Forms\ImportSourceForm $form */ + $format = $form->getSentOrObjectSetting('file_format'); + + try { + $configFile = Config::module('fileshipper', 'imports')->getConfigFile(); + $directories = static::listBaseDirectories(); + $ignored = static::listIgnoredBaseDirectories(); + $e = null; + } catch (\Throwable $e) { + $configFile = null; + $directories = []; + $ignored = []; + } catch (\Exception $e) { + $configFile = null; + $directories = []; + $ignored = []; + } + $form->addElement('select', 'basedir', array( + 'label' => $form->translate('Base directory'), + 'description' => sprintf( + $form->translate( + 'This import rule will only work with files relative to this' + . ' directory. The content of this list depends on your' + . ' configuration in "%s"' + ), + $configFile + ), + 'required' => true, + 'class' => 'autosubmit', + 'multiOptions' => $form->optionalEnum($directories), + )); + if ($configFile === null) { + if ($e) { + $form->getElement('basedir')->addError(sprintf( + $form->translate( + 'Failed to get directories from Fileshipper configuration: %s' + ), + $e->getMessage() + )); + } + } elseif (empty($directories)) { + $dirElement = $form->getElement('basedir'); + if (! @file_exists($configFile)) { + $dirElement->addError(\sprintf( + 'The file "%s" does not exist or is not accessible', + $configFile + )); + } + } + + if (! empty($ignored)) { + $list = []; + foreach ($ignored as $ignoredDirName => $section) { + $list[] = "$section: $ignoredDirName"; + } + $ignoredString = \implode(', ', $list); + if (count($list) === 1) { + $errorString = 'The following directory has been ignored: %s'; + } else { + $errorString = 'The following directories have been ignored: %s'; + } + $form->addHtmlHint(\sprintf($errorString, $ignoredString)); + } + + if (! ($basedir = $form->getSentOrObjectSetting('basedir'))) { + return $form; + } + + $form->addElement('select', 'file_name', array( + 'label' => $form->translate('File name'), + 'description' => $form->translate( + 'Choose a file from the above directory or * to import all files' + . ' from there at once' + ), + 'required' => true, + 'class' => 'autosubmit', + 'multiOptions' => $form->optionalEnum(self::enumFiles($basedir, $form)), + )); + + $basedir = $form->getSentOrObjectSetting('basedir'); + $basename = $form->getSentOrObjectSetting('file_name'); + if ($basedir === null || $basename === null) { + return $form; + } + + $filename = sprintf('%s/%s', $basedir, $basename); + switch ($format) { + case 'csv': + static::addCsvElements($form); + break; + + case 'xslx': + static::addXslxElements($form, $filename); + break; + } + + return $form; + } + + /** + * @param QuickForm $form + * @throws \Zend_Form_Exception + */ + protected static function addCsvElements(QuickForm $form) + { + $form->addElement('text', 'csv_delimiter', array( + 'label' => $form->translate('Field delimiter'), + 'description' => $form->translate( + 'This sets the field delimiter. One character only, defaults' + . ' to comma: ,' + ), + 'value' => ',', + 'required' => true, + )); + + $form->addElement('text', 'csv_enclosure', array( + 'label' => $form->translate('Value enclosure'), + 'description' => $form->translate( + 'This sets the field enclosure character. One character only,' + . ' defaults to double quote: "' + ), + 'value' => '"', + 'required' => true, + )); + + /* + // Not configuring escape as it behaves strangely. "te""st" works fine. + // Seems that even in case we use \, it must be "manually" removed later + // on + $form->addElement('text', 'csv_escape', array( + 'label' => $form->translate('Escape character'), + 'description' => $form->translate( + 'This sets the escaping character. One character only,' + . ' defaults to backslash: \\' + ), + 'value' => '\\', + 'required' => true, + )); + */ + } + + /** + * @param QuickForm $form + * @param $filename + * @throws \Zend_Form_Exception + */ + protected static function addXslxElements(QuickForm $form, $filename) + { + $form->addElement('select', 'worksheet_addressing', array( + 'label' => $form->translate('Choose worksheet'), + 'description' => $form->translate('How to choose a worksheet'), + 'multiOptions' => array( + 'by_position' => $form->translate('by position'), + 'by_name' => $form->translate('by name'), + ), + 'value' => 'by_position', + 'class' => 'autosubmit', + 'required' => true, + )); + + /** @var \Icinga\Module\Director\Forms\ImportSourceForm $form */ + $addressing = $form->getSentOrObjectSetting('worksheet_addressing'); + switch ($addressing) { + case 'by_name': + $file = static::loadXslxFile($filename); + $names = $file->getSheetNames(); + $names = array_combine($names, $names); + $form->addElement('select', 'worksheet_name', array( + 'label' => $form->translate('Name'), + 'required' => true, + 'value' => $file->getFirstSheetName(), + 'multiOptions' => $names, + )); + break; + + case 'by_position': + default: + $form->addElement('text', 'worksheet_position', array( + 'label' => $form->translate('Position'), + 'required' => true, + 'value' => '1', + )); + break; + } + } + + /** + * @param $basedir + * @param $format + * @return array + * @throws ConfigurationError + * @throws IcingaException + */ + protected function fetchFiles($basedir, $format) + { + $result = array(); + foreach (static::listFiles($basedir) as $file) { + $result[$file] = (object) $this->fetchFile($basedir, $file, $format); + } + + return $result; + } + + /** + * @param $basedir + * @param $file + * @param $format + * @return object[] + * @throws ConfigurationError + * @throws IcingaException + */ + protected function fetchFile($basedir, $file, $format) + { + $filename = $basedir . '/' . $file; + + switch ($format) { + case 'yaml': + return $this->readYamlFile($filename); + case 'json': + return $this->readJsonFile($filename); + case 'csv': + return $this->readCsvFile($filename); + case 'xslx': + return $this->readXslxFile($filename); + case 'xml': + libxml_disable_entity_loader(true); + return $this->readXmlFile($filename); + default: + throw new ConfigurationError( + 'Unsupported file format: %s', + $format + ); + } + } + + /** + * @param $filename + * @return Workbook + */ + protected static function loadXslxFile($filename) + { + return new Workbook($filename); + } + + /** + * @param $filename + * @return array + */ + protected function readXslxFile($filename) + { + $xlsx = new Workbook($filename); + if ($this->getSetting('worksheet_addressing') === 'by_name') { + $sheet = $xlsx->getSheetByName($this->getSetting('worksheet_name')); + } else { + $sheet = $xlsx->getSheet((int) $this->getSetting('worksheet_position')); + } + + $data = $sheet->getData(); + + $headers = null; + $result = []; + foreach ($data as $line) { + if ($headers === null) { + $hasValue = false; + foreach ($line as $value) { + if ($value !== null) { + $hasValue = true; + break; + } + // For now, no value in the first column means this is no header + break; + } + if ($hasValue) { + $headers = $line; + } + + continue; + } + + $row = []; + foreach ($line as $key => $val) { + if (empty($headers[$key])) { + continue; + } + $row[$headers[$key]] = $val; + } + + $result[] = (object) $row; + } + + return $result; + } + + /** + * @param $filename + * @return object[] + */ + protected function readCsvFile($filename) + { + $fh = fopen($filename, 'r'); + $lines = array(); + $delimiter = $this->getSetting('csv_delimiter'); + $enclosure = $this->getSetting('csv_enclosure'); + // $escape = $this->getSetting('csv_escape'); + + $headers = fgetcsv($fh, 0, $delimiter, $enclosure/*, $escape*/); + $row = 1; + while ($line = fgetcsv($fh, 0, $delimiter, $enclosure/*, $escape*/)) { + if (empty($line)) { + continue; + } + if (count($headers) !== count($line)) { + throw new RuntimeException(sprintf( + 'Column count in row %d does not match columns in header row', + $row + )); + } + + $line = array_combine($headers, $line); + foreach ($line as $key => & $value) { + if ($value === '') { + $value = null; + } + } + unset($value); + $lines[] = (object) $line; + + $row ++; + } + fclose($fh); + + return $lines; + } + + /** + * @param $filename + * @return object[] + */ + protected function readJsonFile($filename) + { + $content = @file_get_contents($filename); + if ($content === false) { + throw new RuntimeException(sprintf( + 'Unable to read JSON file "%s"', + $filename + )); + } + + $data = @json_decode($content); + if ($data === null) { + throw JsonException::forLastJsonError('Unable to load JSON data'); + } + + return $data; + } + + /** + * @param $file + * @return object[] + */ + protected function readXmlFile($file) + { + $lines = array(); + $content = file_get_contents($file); + foreach (simplexml_load_string($content) as $entry) { + $line = null; + $lines[] = $this->normalizeSimpleXML($entry); + } + + return $lines; + } + + /** + * @param $object + * @return object + */ + protected function normalizeSimpleXML($object) + { + $data = $object; + if (is_object($data)) { + $data = (object) get_object_vars($data); + } + + if (is_object($data)) { + foreach ($data as $key => $value) { + $data->$key = $this->normalizeSimpleXml($value); + } + } + + if (is_array($data)) { + foreach ($data as $key => $value) { + $data[$key] = $this->normalizeSimpleXml($value); + } + } + + return $data; + } + + /** + * @param $file + * @return object[] + */ + protected function readYamlFile($file) + { + return $this->fixYamlObjects( + yaml_parse_file($file) + ); + } + + /** + * @param $what + * @return object[] + */ + protected function fixYamlObjects($what) + { + if (is_array($what)) { + foreach (array_keys($what) as $key) { + if (! is_int($key)) { + $what = (object) $what; + break; + } + } + } + + if (is_array($what) || is_object($what)) { + foreach ($what as $k => $v) { + if (! empty($v)) { + if (is_object($what)) { + $what->$k = $this->fixYamlObjects($v); + } elseif (is_array($what)) { + $what[$k] = $this->fixYamlObjects($v); + } + } + } + } + + return $what; + } + + /** + * @param QuickForm $form + * @return array + */ + protected static function listAvailableFormats(QuickForm $form) + { + $formats = array( + 'csv' => $form->translate('CSV (Comma Separated Value)'), + 'json' => $form->translate('JSON (JavaScript Object Notation)'), + ); + + if (class_exists('\\ZipArchive')) { + $formats['xslx'] = $form->translate('XSLX (Microsoft Excel 2007+)'); + } + + if (function_exists('simplexml_load_file')) { + $formats['xml'] = $form->translate('XML (Extensible Markup Language)'); + } + + if (function_exists('yaml_parse_file')) { + $formats['yaml'] = $form->translate('YAML (Ain\'t Markup Language)'); + } + + return $formats; + } + + /** + * @return array + */ + protected static function listBaseDirectories() + { + $dirs = array(); + + foreach (Config::module('fileshipper', 'imports') as $key => $section) { + if (($dir = $section->get('basedir')) && @is_dir($dir)) { + $dirs[$dir] = $key; + } + } + + return $dirs; + } + + /** + * @return array + */ + protected static function listIgnoredBaseDirectories() + { + $dirs = array(); + + foreach (Config::module('fileshipper', 'imports') as $key => $section) { + if (($dir = $section->get('basedir')) && @is_dir($dir)) { + // Ignore them + } else { + $dirs[$dir] = $key; + } + } + + return $dirs; + } + + /** + * @param $basedir + * @param QuickForm $form + * @return array + */ + protected static function enumFiles($basedir, QuickForm $form) + { + return array_merge( + array( + '*' => sprintf('* (%s)', $form->translate('all files')) + ), + static::listFiles($basedir) + ); + } + + /** + * @param $basedir + * @return array + */ + protected static function listFiles($basedir) + { + $files = array(); + + $dir = new DirectoryIterator($basedir); + foreach ($dir as $file) { + if ($file->isFile()) { + $filename = $file->getBasename(); + if ($filename[0] !== '.') { + $files[$filename] = $filename; + } + } + } + + ksort($files); + + return $files; + } +} diff --git a/library/Fileshipper/ProvidedHook/Director/ShipConfigFiles.php b/library/Fileshipper/ProvidedHook/Director/ShipConfigFiles.php new file mode 100644 index 0000000..43f6f38 --- /dev/null +++ b/library/Fileshipper/ProvidedHook/Director/ShipConfigFiles.php @@ -0,0 +1,78 @@ +<?php + +namespace Icinga\Module\Fileshipper\ProvidedHook\Director; + +use Exception; +use Icinga\Application\Config; +use Icinga\Module\Director\Hook\ShipConfigFilesHook; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; +use RegexIterator; + +class ShipConfigFiles extends ShipConfigFilesHook +{ + /** + * @return array + */ + public function fetchFiles() + { + $files = []; + foreach ($this->getDirectories() as $key => $cfg) { + $target = $cfg->get('target'); + try { + foreach ($this->listFiles($cfg->get('source'), $cfg->get('extensions')) as $file) { + try { + $files["$target/$file"] = file_get_contents($cfg->get('source') . '/' . $file); + } catch (Exception $e) { + $files["$target/$file"] = '/* ' . $e->getMessage() . ' */'; + } + } + } catch (Exception $e) { + $files["$target/ERROR.txt"] = '/* ' . $e->getMessage() . ' */'; + } + } + + return $files; + } + + /** + * @param $folder + * @param $extensions + * @return array + */ + protected function listFiles($folder, $extensions) + { + if (! $extensions) { + $pattern = '/^[^\.].+\.conf$/'; + } else { + $exts = []; + foreach (preg_split('/\s+/', $extensions, -1, PREG_SPLIT_NO_EMPTY) as $ext) { + $exts[] = preg_quote($ext, '/'); + } + + $pattern = '/^[^\.].+(?:' . implode('|', $exts) . ')$/'; + } + + $dir = new RecursiveDirectoryIterator($folder); + $ite = new RecursiveIteratorIterator($dir); + $files = new RegexIterator($ite, $pattern, RegexIterator::GET_MATCH); + $fileList = []; + $start = strlen($folder) + 1; + + foreach ($files as $file) { + foreach ($file as $f) { + $fileList[] = substr($f, $start); + } + } + + return $fileList; + } + + /** + * @return Config + */ + protected function getDirectories() + { + return Config::module('fileshipper', 'directories'); + } +} diff --git a/library/Fileshipper/Xlsx/Utils.php b/library/Fileshipper/Xlsx/Utils.php new file mode 100755 index 0000000..3ab9563 --- /dev/null +++ b/library/Fileshipper/Xlsx/Utils.php @@ -0,0 +1,37 @@ +<?php + +namespace Icinga\Module\Fileshipper\Xlsx; + +class Utils +{ + /** + * Extract text content from a rich text or inline string field + * @param null $is + * @return string + */ + public static function parseRichText($is = null) + { + $value = []; + if (isset($is->t)) { + $value[] = (string)$is->t; + } else { + foreach ($is->r as $run) { + $value[] = (string)$run->t; + } + } + + return implode(' ', $value); + } + + // converts an Excel date field (a number) to a unix timestamp (granularity: seconds) + public static function toUnixTimeStamp($excelDateTime) + { + if (! is_numeric($excelDateTime)) { + return $excelDateTime; + } + $d = floor($excelDateTime); // seconds since 1900 + $t = $excelDateTime - $d; + + return ($d > 0) ? ( $d - 25569 ) * 86400 + $t * 86400 : $t * 86400; + } +} diff --git a/library/Fileshipper/Xlsx/Workbook.php b/library/Fileshipper/Xlsx/Workbook.php new file mode 100755 index 0000000..9c20d95 --- /dev/null +++ b/library/Fileshipper/Xlsx/Workbook.php @@ -0,0 +1,300 @@ +<?php + +namespace Icinga\Module\Fileshipper\Xlsx; + +use RuntimeException; +use ZipArchive; + +/** + * Classes in this namespace have been built roughly based on various OSS + * XLSXReader implementations + */ +class Workbook +{ + // XML schemas + const SCHEMA_OFFICEDOCUMENT = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument'; + const SCHEMA_OFFICEDOCUMENT_RELATIONSHIP = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships'; + const SCHEMA_RELATIONSHIP = 'http://schemas.openxmlformats.org/package/2006/relationships'; + const SCHEMA_SHAREDSTRINGS = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings'; + const SCHEMA_WORKSHEETRELATION = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet'; + + protected static $zipErrors = [ + ZipArchive::ER_EXISTS => 'File already exists', + ZipArchive::ER_INCONS => 'Zip archive inconsistent', + ZipArchive::ER_INVAL => 'Invalid argument', + ZipArchive::ER_MEMORY => 'Malloc failure', + ZipArchive::ER_NOENT => 'No such file', + ZipArchive::ER_NOZIP => 'Not a zip archive', + ZipArchive::ER_OPEN => 'Can\'t open file', + ZipArchive::ER_READ => 'Read error', + ZipArchive::ER_SEEK => 'Seek error', + ]; + + /** @var Worksheet[] */ + protected $sheets = []; + + public $sharedStrings = []; + + protected $sheetInfo; + + protected $sheetNameIndex; + + /** @var ZipArchive */ + protected $zip; + + public $config = [ + 'removeTrailingRows' => true + ]; + + protected $mainRelation; + + public function __construct($filename, $config = []) + { + $this->config = array_merge($this->config, $config); + $this->initialize($filename); + } + + protected function initialize($filename) + { + $this->zip = new ZipArchive(); + if (true === ($result = $this->zip->open($filename))) { + $this->parse(); + } else { + throw new RuntimeException(sprintf( + 'Failed to open %s : %s', + $filename, + $this->getZipErrorString($result) + )); + } + } + + protected function getZipErrorString($errorCode) + { + if (array_key_exists($errorCode, self::$zipErrors)) { + return self::$zipErrors[$errorCode]; + } else { + return "Unknown ZIP error code $errorCode"; + } + } + + // get a file from the zip + protected function extractFile($name) + { + $data = $this->zip->getFromName($name); + if ($data === false) { + throw new RuntimeException(sprintf( + "File %s does not exist in the Excel file", + $name + )); + } else { + return $data; + } + } + + protected function loadPackageRelationshipXml() + { + return simplexml_load_string($this->extractFile('_rels/.rels')); + } + + /** + * @return \SimpleXMLElement[] + */ + protected function getPackageRelationships() + { + return $this->loadPackageRelationshipXml()->Relationship; + } + + // workbookXML + protected function getMainDocumentRelation() + { + if ($this->mainRelation === null) { + foreach ($this->getPackageRelationships() as $relation) { + if ($relation['Type'] == self::SCHEMA_OFFICEDOCUMENT) { + $this->mainRelation = $relation; + break; + } + } + + if ($this->mainRelation === null) { + throw new RuntimeException( + 'Got invalid Excel file, found no main document' + ); + } + } + + return $this->mainRelation; + } + + protected function getWorkbookXml() + { + return simplexml_load_string( + $this->extractFile( + $this->getMainDocumentRelation()['Target'] + ) + ); + } + + protected function getWorkbookDir() + { + return dirname($this->getMainDocumentRelation()['Target']); + } + + /** + * @return \SimpleXMLElement[] + */ + protected function getWorkbookRelationShips() + { + $wbDir = $this->getWorkbookDir(); + $target = basename($this->getMainDocumentRelation()['Target']); + + return simplexml_load_string( + $this->extractFile("$wbDir/_rels/$target.rels") + )->Relationship; + } + + // extract the shared string and the list of sheets + protected function parse() + { + $sheets = []; + /** @var \SimpleXMLElement $sheet */ + $pos = 0; + foreach ($this->getWorkbookXml()->sheets->sheet as $sheet) { + $pos++; + $rId = (string) $sheet->attributes('r', true)->id; + // $sheets[$pos] = [ --> check docs + $sheets[$rId] = [ + 'rId' => $rId, + 'sheetId' => (int)$sheet['sheetId'], + 'name' => (string)$sheet['name'], + ]; + } + + $workbookDir = $this->getWorkbookDir() . '/'; + foreach ($this->getWorkbookRelationShips() as $relation) { + switch ($relation['Type']) { + case self::SCHEMA_WORKSHEETRELATION: + $sheets[(string) $relation['Id']]['path'] = $workbookDir . (string)$relation['Target']; + break; + + case self::SCHEMA_SHAREDSTRINGS: + $sharedStringsXML = simplexml_load_string( + $this->extractFile($workbookDir . $relation['Target']) + ); + + foreach ($sharedStringsXML->si as $val) { + if (isset($val->t)) { + $this->sharedStrings[] = (string)$val->t; + } elseif (isset($val->r)) { + $this->sharedStrings[] = Utils::parseRichText($val); + } + } + + break; + } + } + + $this->sheetInfo = []; + foreach ($sheets as $rid => $info) { + if (! array_key_exists('path', $info)) { + var_dump($sheets); + exit; + } + $this->sheetInfo[$info['name']] = [ + 'sheetId' => $info['sheetId'], + 'rid' => $rid, + 'path' => $info['path'] + ]; + } + } + + // returns an array of sheet names, indexed by sheetId + public function getSheetNames() + { + $res = []; + foreach ($this->sheetInfo as $sheetName => $info) { + $res[$info['sheetId']] = $sheetName; + // $res[$info['rid']] = $sheetName; + } + + return $res; + } + + public function getSheetCount() + { + return count($this->sheetInfo); + } + + // instantiates a sheet object (if needed) and returns an array of its data + public function getSheetData($sheetNameOrId) + { + $sheet = $this->getSheet($sheetNameOrId); + + return $sheet->getData(); + } + + // instantiates a sheet object (if needed) and returns the sheet object + public function getSheet($sheet) + { + if (is_numeric($sheet)) { + $sheet = $this->getSheetNameById($sheet); + } elseif (!is_string($sheet)) { + throw new RuntimeException("Sheet must be a string or a sheet Id"); + } + if (!array_key_exists($sheet, $this->sheets)) { + $this->sheets[$sheet] = new Worksheet($this->getSheetXML($sheet), $sheet, $this); + } + + return $this->sheets[$sheet]; + } + + public function getSheetByName($name) + { + if (!array_key_exists($name, $this->sheets)) { + $this->sheets[$name] = new Worksheet($this->getSheetXML($name), $name, $this); + } + + return $this->sheets[$name]; + } + + public function enumRidNames() + { + $res = []; + foreach ($this->sheetInfo as $name => $info) { + $res[$name] = $name; + } + + return $res; + } + + public function getSheetNameById($sheetId) + { + foreach ($this->sheetInfo as $sheetName => $sheetInfo) { + if ($sheetInfo['sheetId'] === $sheetId) { + return $sheetName; + } + } + + throw new RuntimeException(sprintf( + "Sheet ID %s does not exist in the Excel file", + $sheetId + )); + } + + public function getFirstSheetName() + { + if (empty($this->sheetInfo)) { + throw new RuntimeException('Workbook contains no sheets'); + } + + foreach ($this->sheetInfo as $sheetName => $sheetInfo) { + return $sheetName; + } + } + + protected function getSheetXML($name) + { + return simplexml_load_string( + $this->extractFile($this->sheetInfo[$name]['path']) + ); + } +} diff --git a/library/Fileshipper/Xlsx/Worksheet.php b/library/Fileshipper/Xlsx/Worksheet.php new file mode 100755 index 0000000..1d9fc51 --- /dev/null +++ b/library/Fileshipper/Xlsx/Worksheet.php @@ -0,0 +1,245 @@ +<?php + +namespace Icinga\Module\Fileshipper\Xlsx; + +use RuntimeException; + +class Worksheet +{ + /** @var Workbook */ + protected $workbook; + + /** @var string */ + public $name; + + /** @var array */ + protected $data; + + /** @var int */ + public $rowCount; + + /** @var int */ + public $colCount; + + /** @var array */ + protected $config; + + /** @var array */ + protected $mergeTarget; + + public function __construct($xml, $sheetName, Workbook $workbook) + { + $this->config = $workbook->config; + $this->name = $sheetName; + $this->workbook = $workbook; + $this->parse($xml); + } + + // returns an array of the data from the sheet + public function getData() + { + return $this->data; + } + + protected function parse($xml) + { + $this->parseDimensions($xml->dimension); + $this->parseMergeCells($xml->mergeCells); + $this->parseData($xml->sheetData); + } + + protected function parseDimensions($dimensions) + { + $range = (string) $dimensions['ref']; + $cells = explode(':', $range); + $maxValues = $this->getColumnIndex($cells[1]); + $this->colCount = $maxValues[0] + 1; + $this->rowCount = $maxValues[1] + 1; + } + + protected function parseMergeCells($merges) + { + $result = []; + + if ($merges->mergeCell === null) { + $this->mergeTarget = $result; + return; + } + + foreach ($merges->mergeCell as $merge) { + $range = (string) $merge['ref']; + $cells = explode(':', $range); + $fromName = $cells[0]; + list($fromCol, $fromRow) = $this->getColumnIndex($fromName); + list($toCol, $toRow) = $this->getColumnIndex($cells[1]); + for ($i = $fromCol; $i <= $toCol; $i++) { + for ($j = $fromRow; $j <= $toRow; $j++) { + if ($i !== $fromCol || $j !== $fromRow) { + $result[$j][$i] = [$fromRow, $fromCol]; + } + } + } + } + + $this->mergeTarget = $result; + } + + protected function parseData($sheetData) + { + $rows = []; + $curR = 0; + $lastDataRow = -1; + + foreach ($sheetData->row as $row) { + $rowNum = (int) $row['r']; + if ($rowNum != ($curR + 1)) { + $missingRows = $rowNum - ($curR + 1); + for ($i = 0; $i < $missingRows; $i++) { + $rows[$curR] = array_pad([], $this->colCount, null); + $curR++; + } + } + $curC = 0; + $rowData = []; + + foreach ($row->c as $c) { + list($cellIndex,) = $this->getColumnIndex((string) $c['r']); + if ($cellIndex !== $curC) { + $missingCols = $cellIndex - $curC; + for ($i = 0; $i < $missingCols; $i++) { + $rowData[$curC] = null; + $curC++; + } + } + $val = $this->parseCellValue($c); + + if (!is_null($val)) { + $lastDataRow = $curR; + } + $rowData[$curC] = $val; + $curC++; + } + $rows[$curR] = array_pad($rowData, $this->colCount, null); + + // We clone merged cells, all of them will return the same value + // This behavior might eventually become optional with a related + // Config flag + if (array_key_exists($curR, $this->mergeTarget)) { + foreach ($this->mergeTarget[$curR] as $col => $cell) { + if ($rowData[$col] === null) { + $rows[$curR][$col] = $rows[$cell[0]][$cell[1]]; + } else { + throw new RuntimeException(sprintf( + '%s should merge into %s, but %s has a value: %s', + $this->makeCellName($cell[0], $cell[1]), + $this->makeCellName($curR, $col), + $this->makeCellName($curR, $col), + $rowData[$col] + )); + } + } + } + + $curR++; + } + + if ($this->config['removeTrailingRows']) { + $this->data = array_slice($rows, 0, $lastDataRow + 1); + $this->rowCount = count($this->data); + } else { + $this->data = $rows; + } + } + + protected function getColumnIndex($cell = 'A1') + { + if (preg_match('/([A-Z]+)(\d+)/', $cell, $matches)) { + $col = $matches[1]; + $row = $matches[2]; + $colLen = strlen($col); + $index = 0; + + for ($i = $colLen-1; $i >= 0; $i--) { + $index += (ord($col[$i]) - 64) * pow(26, $colLen - $i - 1); + } + + return [$index - 1, $row - 1]; + } + + throw new RuntimeException(sprintf('Invalid cell index %s', $cell)); + } + + protected function makeCellName($column, $row) + { + $str = ''; + + $rem = $column + 1; + while ($rem > 0) { + $mod = $rem % 26; + $str = chr($mod + 64) . $str; + $rem = ($rem - $mod) / 26; + } + + return $str . (string) ($row + 1); + } + + protected function parseCellValue($cell) + { + // t is the cell type + switch ((string) $cell['t']) { + // Shared string + case 's': + if ((string) $cell->v !== '') { + $value = $this->workbook->sharedStrings[intval($cell->v)]; + } else { + $value = ''; + } + break; + + // Boolean + case 'b': + $value = (string) $cell->v; + if ($value === '0') { + $value = false; + } elseif ($value === '1') { + $value = true; + } else { + $value = (bool) $cell->v; + } + break; + + // Inline rich text + case 'inlineStr': + $value = Utils::parseRichText($cell->is); + break; + + // Error message + case 'e': + if ((string) $cell->v !== '') { + $value = (string)$cell->v; + } else { + $value = ''; + } + break; + + default: + if (!isset($cell->v)) { + return null; + } + $value = (string) $cell->v; + + // Check for numeric values + if (is_numeric($value)) { + if ($value == (int) $value) { + $value = (int) $value; + } elseif ($value == (float) $value) { + $value = (float) $value; + } elseif ($value == (double) $value) { + $value = (double) $value; + } + } + } + + return $value; + } +} diff --git a/module.info b/module.info new file mode 100644 index 0000000..acb5012 --- /dev/null +++ b/module.info @@ -0,0 +1,7 @@ +Name: Fileshipper +Version: 1.2.0 +Depends: director +Description: Fileshipper for Icinga Director + The fileshipper module provides an Import Source for CSV, JSON, XML and YAML + files. It also allows to ship hand-crafted plain Icinga 2 Config files through + the Icinga Director deployment mechanism. diff --git a/phpcs.xml b/phpcs.xml new file mode 100644 index 0000000..bc41982 --- /dev/null +++ b/phpcs.xml @@ -0,0 +1,17 @@ +<?xml version="1.0"?> +<ruleset name="PHP_CodeSniffer"> + <description>Sniff our code</description> + + <file>run.php</file> + <file>library/</file> + + <arg value="wps"/> + <arg name="colors"/> + <arg name="report-width" value="auto"/> + <arg name="report-full"/> + <arg name="report-gitblame"/> + <arg name="report-summary"/> + <arg name="encoding" value="UTF-8"/> + + <rule ref="PSR2"/> +</ruleset> @@ -0,0 +1,4 @@ +<?php + +$this->provideHook('director/ShipConfigFiles'); +$this->provideHook('director/ImportSource'); |
