diff options
36 files changed, 8445 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3cc12a --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.pyc +*.swp +bin/xcapture +bin/xcap/* +*.tar.gz diff --git a/0xtools.spec b/0xtools.spec new file mode 100644 index 0000000..99ea78a --- /dev/null +++ b/0xtools.spec @@ -0,0 +1,115 @@ +%define debug_package %{nil} +# %define _unpackaged_files_terminate_build 0 + +%define ReleaseNumber 5 +%define VERSION 1.2.3 + +Name: 0xtools +Version: %{VERSION} +Release: %{ReleaseNumber}%{?dist} +Source0: 0xtools-%{VERSION}.tar.gz +URL: https://0x.tools/ +Packager: Tanel Poder <tanel@tanelpoder.com> +Summary: Always-on Profiling for Production Systems +License: GPL +Group: System Environment +BuildArch: %{_arch} + +# Creators of these rpmspec/service files are below (Tanel just merged/customized them): + +# Liyong <hungrybirder@gmail.com> +# Bart Sjerps (https://github.com/bsjerps) + +# RPM build instructions: +# Have rpmbuild, gcc, make and rpmdevtools installed +# Optionally, update Version above to the latest release +# +# Download 0xtools source archive into SOURCES: +# spectool -g -R 0xtools.spec +# +# Build package: +# rpmbuild -bb 0xtools.spec + +# Prevent compiling .py files +%define __python false + +%description +0x.tools is a set of open-source utilities for analyzing application performance on Linux. +It has a goal of deployment simplicity and minimal dependencies, to reduce friction of systematic troubleshooting. +There’s no need to upgrade the OS, install kernel modules, heavy monitoring frameworks, Java agents or databases. +These tools also work on over-decade-old Linux kernels, like version 2.6.18 from 15 years ago. + +0x.tools allow you to measure individual thread level activity, like thread sleep states, +currently executing system calls and kernel wait locations. + +%prep +%setup -q + +%build +make PREFIX=%{buildroot}/usr +make install PREFIX=%{buildroot}/usr + +%install +install -m 0755 -d -p %{buildroot}/usr/bin +install -m 0755 -d -p %{buildroot}/usr/bin/%{name} +install -m 0755 -d -p %{buildroot}/usr/lib/%{name} +install -m 0755 -d -p %{buildroot}/usr/share/%{name} +install -m 0755 -d -p %{buildroot}/var/log/xcapture + +install -m 0755 bin/run_xcpu.sh %{buildroot}/usr/bin/run_xcpu.sh +install -m 0755 bin/run_xcapture.sh %{buildroot}/usr/bin/run_xcapture.sh +install -m 0755 bin/schedlat %{buildroot}/usr/bin/schedlat +install -m 0755 bin/vmtop %{buildroot}/usr/bin/vmtop + +cp -p doc/licenses/* %{buildroot}/usr/share/%{name} +cp -p LICENSE %{buildroot}/usr/share/%{name} + + +## empty files to please %ghost section (we don't want precompiled) +## This ensures the object files also get cleaned up if we uninstall the RPM +#touch %{buildroot}/usr/lib/%{name}/{psnreport,psnproc,argparse}.pyc +#touch %{buildroot}/usr/lib/%{name}/{psnreport,psnproc,argparse}.pyo + + +# systemd service +install -Dp -m 0644 xcapture.default $RPM_BUILD_ROOT/etc/default/xcapture +install -Dp -m 0644 xcapture.service %{buildroot}/usr/lib/systemd/system/xcapture.service +install -Dp -m 0644 xcapture-restart.service %{buildroot}/usr/lib/systemd/system/xcapture-restart.service +install -Dp -m 0644 xcapture-restart.timer %{buildroot}/usr/lib/systemd/system/xcapture-restart.timer + +%clean +rm -rf %{buildroot} + +%post +/bin/systemctl daemon-reload +/bin/systemctl enable --now xcapture +/bin/systemctl enable --now xcapture-restart.timer + +%preun +if [ "$1" -eq "0" ] +then + /bin/systemctl disable --now xcapture + /bin/systemctl disable --now xcapture-restart.timer +fi + +%files +%defattr(0755,root,root,0755) +%{_bindir}/psn +%{_bindir}/run_xcapture.sh +%{_bindir}/run_xcpu.sh +%{_bindir}/schedlat +%{_bindir}/xcapture +%{_bindir}/vmtop +/usr/lib/0xtools/* +/usr/lib/systemd/system/xcapture.service +/usr/lib/systemd/system/xcapture-restart.service +/usr/lib/systemd/system/xcapture-restart.timer + +%defattr(0644,root,root,0755) +/usr/share/%{name} +%ghost /usr/lib/%{name}/*.pyc +%ghost /usr/lib/%{name}/*.pyo + +%config(noreplace) /etc/default/xcapture +%dir /var/log/xcapture/ + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f6a9eec --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,77 @@ +# 0x.tools changelog + +1.2.3 +====================== +* OS support + - add unistd.h lookup for arm64 and ppc64le platforms + +* Installation & running + - add RPMspec for creating RPMs + - add systemd service definitions + +* Features + - add more "single file descriptor" syscalls that can report filenames accessed + - report /proc/PID/ns/pid for container namespace info + - systems with python3 now print out extra newlines like intended + +1.1.0 +====================== +* general + - using semantic versioning now (major.minor.patch) + - in the future, will update version numbers in a specific tool only when it was updated + +* pSnapper + - `psn` works with python 3 now too (uses whereever the "/usr/bin/env python" command points to) + +* xcapture + - Fixed xcapture compiler warnings shown on newer gcc versions + - More precise sampling interval (account for sampling busy-time and subtract that from next sleep duration) + - Under 1 sec sleep durations supported (For example `-d 0.1` for sampling at 10 Hz) + +* make/install + - by default, executables go to `/usr/bin` now + - python libraries go under PREFIX/lib/0xtools now + - use PREFIX option in makefile to adjust the installation root + - makefile uses the `install` command instead of the `ln -s` hack for installing files + - `make uninstall` removes installed files + +0.18 +====================== +* New column + - `filenamesum` column strips numbers out of filenames to summarize events against similar files + +0.16 +====================== +* New script + - schedlat.py - show scheduling latency of a single process + +0.15 +====================== +* Minor changes only + - Handle SIGPIPE to not get `IOError: [Errno 32] Broken pipe` error when piping pSnapper output to other tools like "head" + - Change the info link tp.dev/psnapper to tanelpoder.com/psnapper + +0.14 +====================== +* report file names that are accessed with I/O syscalls with arg0 as the file descriptor + - example: `sudo psn -G syscall,filename` + - works with read, write, pread, fsync, recvmsg, sendmsg etc, but not with batch io syscalls like io_submit(), select() that may submit multiple fds per call + +* no need to install kernel-headers package anymore as pSnapper now has the unistd.h file bundled with the install + - no more exceptions complaining about missing unistd_64.h file + - pSnapper still tries to use the unistd.h file from a standard /usr/include location, but falls back to the bundled one if the file is missing. this should help with using pSnapper on other platforms too (different processor architectures, including 32bit vs 64bit versions of the same architecture have different syscall numbers + +* pSnapper can now run on RHEL5 equivalents (2.6.18 kernel), however with separately installed python26 or later, as I haven't "downgraded" pSnapper's python code to work with python 2.4 (yet) + - you could install python 2.6 or 2.7 manually in your own directory or use the EPEL package: (yum install epel-release ; yum install python26 ) + - you will also need to uncomment the 2nd line in psn script (use #!/usr/bin/env/python26 instead of python) + - note that 2.6.18 kernel doesnt provide syscall,file name and kstack sampling (but wchan is available) + + + +0.13 +====================== +* kernel stack summary reporting - new column `kstack` +* wider max column length (for kstack) +* add `--list` option to list all available columns +* replace digits from `comm` column by default to collapse different threads of the same thing into one. you can use `comm2` to see the unedited process comm. + @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3fa2427 --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +CC ?= gcc +PREFIX ?= /usr + +# build +CFLAGS ?= -Wall + +# debuginfo included +CFLAGS_DEBUG=-I include -ggdb -Wall + +# debug without compiler optimizations +CFLAGS_DEBUG0=-I include -ggdb -O0 + +all: + $(CC) $(CFLAGS) -I include -o bin/xcapture src/xcapture.c + +debug: + $(CC) $(CFLAGS_DEBUG) -o bin/xcapture src/xcapture.c + +debug0: + $(CC) $(CFLAGS_DEBUG0) -o bin/xcapture src/xcapture.c + +install: + install -m 0755 -d ${PREFIX}/bin + install -m 0755 bin/xcapture ${PREFIX}/bin/xcapture + install -m 0755 bin/psn ${PREFIX}/bin/psn + install -m 0755 bin/schedlat ${PREFIX}/bin/schedlat + install -m 0755 bin/vmtop ${PREFIX}/bin/vmtop + install -m 0755 bin/syscallargs ${PREFIX}/bin/syscallargs + install -m 0755 -d ${PREFIX}/lib/0xtools + install -m 0644 lib/0xtools/psnproc.py ${PREFIX}/lib/0xtools/psnproc.py + install -m 0644 lib/0xtools/psnreport.py ${PREFIX}/lib/0xtools/psnreport.py + install -m 0644 lib/0xtools/argparse.py ${PREFIX}/lib/0xtools/argparse.py + +uninstall: + rm -fv ${PREFIX}/bin/xcapture ${PREFIX}/bin/psn ${PREFIX}/bin/schedlat + rm -fv ${PREFIX}/lib/0xtools/psnproc.py ${PREFIX}/lib/0xtools/psnreport.py ${PREFIX}/lib/0xtools/argparse.py + rm -rfv ${PREFIX}/lib/0xtools + +clean: + rm -fv bin/xcapture diff --git a/README.md b/README.md new file mode 100644 index 0000000..07b77a3 --- /dev/null +++ b/README.md @@ -0,0 +1,66 @@ +# 0x.Tools: X-Ray vision for Linux systems + +**0x.tools** is a set of open-source utilities for analyzing application performance on Linux. It has a goal of deployment simplicity and minimal dependencies, to reduce friction of systematic troubleshooting. There’s no need to upgrade the OS, install kernel modules, heavy monitoring frameworks, Java agents or databases. Some of these tools also work on over-decade-old Linux kernels, like version 2.6.18 from 18 years ago. + +**0x.tools** allow you to measure individual thread level activity, like thread sleep states, currently executing system calls and kernel wait locations. Additionally, you can drill down into CPU usage of any thread or the system as a whole. You can be systematic in your troubleshooting - no need for guessing or genius wizard tricks with traditional system utilization stats. + +## xcapture-bpf and xtop v2.0.2 announced! (2024-07-03) + +xcapture-bpf (and xtop) are like the Linux top tool, but extended with x-ray vision and ability to view your performance data from any chosen angle (that eBPF allows to instrument). You can use it for system level overview and drill down into indivual threads' activity and soon even into individual kernel events like lock waits or memory stalls. eBPF is not only customizable, it's completely programmable and I plan to take full advantage of it. I have so far implemented less than 5% of everything this method and the new tool is capable of, stay tuned for more! + +* https://0x.tools + +### xcapture-bpf demo +This is one of the things that you get: + +[](https://asciinema.org/a/666715) + +### xcapture-bpf screenshot +A screenshot that illustrates how xcapture-bpf output and stacktiles work with terminal search/highlighting and scroll-back ability: + + + +### xcapture-bpf install instructions and info + +* Go to https://0x.tools for more info and the installation instructions of the latest eBPF-based tool + +**An example** of one of the tools `psn` (that doesn't use eBPF, just reads the usual `/proc` files) is here: + +``` +$ sudo psn -p "mysqld|kwork" -G syscall,wchan + +Linux Process Snapper v0.14 by Tanel Poder [https://0x.tools] +Sampling /proc/syscall, stat, wchan for 5 seconds... finished. + + +=== Active Threads ======================================================================================== + + samples | avg_threads | comm | state | syscall | wchan +----------------------------------------------------------------------------------------------------------- + 25 | 3.12 | (mysqld) | Disk (Uninterruptible) | fsync | _xfs_log_force_lsn + 16 | 2.00 | (mysqld) | Running (ON CPU) | [running] | 0 + 14 | 1.75 | (mysqld) | Disk (Uninterruptible) | pwrite64 | call_rwsem_down_write_failed + 8 | 1.00 | (mysqld) | Disk (Uninterruptible) | fsync | submit_bio_wait + 4 | 0.50 | (mysqld) | Disk (Uninterruptible) | pread64 | io_schedule + 4 | 0.50 | (mysqld) | Disk (Uninterruptible) | pwrite64 | io_schedule + 3 | 0.38 | (mysqld) | Disk (Uninterruptible) | pread64 | 0 + 3 | 0.38 | (mysqld) | Running (ON CPU) | [running] | io_schedule + 3 | 0.38 | (mysqld) | Running (ON CPU) | pread64 | 0 + 2 | 0.25 | (mysqld) | Disk (Uninterruptible) | [running] | 0 + 1 | 0.12 | (kworker/*:*) | Running (ON CPU) | read | worker_thread + 1 | 0.12 | (mysqld) | Disk (Uninterruptible) | fsync | io_schedule + 1 | 0.12 | (mysqld) | Disk (Uninterruptible) | futex | call_rwsem_down_write_failed + 1 | 0.12 | (mysqld) | Disk (Uninterruptible) | poll | 0 + 1 | 0.12 | (mysqld) | Disk (Uninterruptible) | pwrite64 | _xfs_log_force_lsn + 1 | 0.12 | (mysqld) | Running (ON CPU) | fsync | submit_bio_wait + 1 | 0.12 | (mysqld) | Running (ON CPU) | futex | futex_wait_queue_me +``` +**Usage info** and more details here: +* https://0x.tools + +**Twitter:** +* https://twitter.com/0xtools + +**Author:** +* https://tanelpoder.com + diff --git a/bin/cpumhz b/bin/cpumhz new file mode 100755 index 0000000..cf9ba08 --- /dev/null +++ b/bin/cpumhz @@ -0,0 +1,9 @@ +#!/bin/bash + +# cpumhz tool by Tanel Poder [https://0x.tools] + +grep MHz /proc/cpuinfo | awk '{ split($4,x,".") ; printf substr(x[1],1,2) "00..99 \n" }' \ + | sed 's/^0/ /' | sort | uniq -c \ + | awk '{ printf $0 ; for(x=0;x<$1;x++) printf "#"; printf "\n" }' \ + | sort -nbrk 2 + diff --git a/bin/cpumhzturbo b/bin/cpumhzturbo new file mode 100755 index 0000000..9f4e0cf --- /dev/null +++ b/bin/cpumhzturbo @@ -0,0 +1,11 @@ +#!/bin/bash + +# cpumhzturbo wrapper by Tanel Poder [https://0x.tools] +# run this as root or add CAP_SYS_RAWIO capability to turbostat + +turbostat -q -s Bzy_MHz -i 1 -n 1 | grep -v Bzy_MHz \ + | awk '{ printf("%04d\n",$1) }' \ + | awk '{ split($1,x,".") ; printf substr(x[1],1,2) "00..99 \n" }' | sed 's/^0/ /' | sort | uniq -c \ + | awk '{ printf $0 ; for(x=0;x<$1;x++) printf "#"; printf "\n" }' \ + | sort -nbrk 2 + diff --git a/bin/ducktop.sql b/bin/ducktop.sql new file mode 100644 index 0000000..255dcaa --- /dev/null +++ b/bin/ducktop.sql @@ -0,0 +1,43 @@ +SELECT + COUNT(*) num_samples + , ROUND(COUNT(*) / 300, 1) avg_threads -- querying 5 minutes (300 sec) of wall-clock time + , REGEXP_REPLACE(cmdline, '.*/', '') cmdline2 +-- , cmdline + , REGEXP_REPLACE(comm, '[0-9]+','*') comm2 + , task_state + , oracle_wait_event + , syscall_name +-- , syscall_arg0 +-- , profile_ustack +-- , profile_kstack +-- , REGEXP_REPLACE(REGEXP_REPLACE(offcpu_kstack, '^->0x[0-9a-f]+', ''), '\+[0-9]+','','g') offcpu_kstack + , REGEXP_REPLACE(REGEXP_REPLACE(offcpu_ustack, '^->0x[0-9a-f]+', ''), '\+[0-9]+','','g') offcpu_ustack +-- , REGEXP_REPLACE(REGEXP_REPLACE(syscall_ustack, '^->0x[0-9a-f]+', ''), '\+[0-9]+','','g') syscall_ustack +-- , REGEXP_REPLACE(REGEXP_REPLACE(syscall_ustack, '^->0x[0-9a-f]+', ''), '\+[0-9]+','','g') syscall_ustack +FROM + READ_CSV('xcapture_20231019_05.csv', auto_detect=true) samples +RIGHT OUTER JOIN + READ_CSV('syscalls.csv', auto_detect=true) syscalls +ON (samples.syscall_id = syscalls.syscall_id) +WHERE + sample_time BETWEEN TIMESTAMP'2023-10-19 05:00:00' AND TIMESTAMP'2023-10-19 05:05:00' +AND task_state IN ('R','D') +--AND cmdline LIKE 'postgres%' +AND comm != 'bpftrace' -- bpftrace is shown always active when taking a sample +GROUP BY + cmdline2 + , cmdline + , comm2 + , task_state + , oracle_wait_event + , syscall_name +-- , syscall_arg0 +-- , profile_ustack +-- , profile_kstack +-- , syscall_ustack + , offcpu_ustack +-- , offcpu_kstack +ORDER BY + num_samples DESC +LIMIT 20 +; @@ -0,0 +1,384 @@ +#!/usr/bin/env python +# +# psn -- Linux Process Snapper by Tanel Poder [https://0x.tools] +# Copyright 2019-2021 Tanel Poder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +# NOTES +# +# If you have only python3 binary in your system (no "python" command without any suffix), +# then set up the OS default alternative or symlink so that you'd have a "python" command. +# +# For example (on RHEL 8): +# sudo alternatives --set python /usr/bin/python3 +# +# One exception on RHEL 5, use the following line, without quotes (if you have the +# additional python26 package installed from EPEL): +# +# "#!/usr/bin/env python26" +# + +PSN_VERSION = '1.2.6' + +import sys, os, os.path, time, datetime +import re +import sqlite3 +import logging +from signal import signal, SIGPIPE, SIG_DFL + +sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + '/lib/0xtools') + +import argparse +import psnproc as proc, psnreport + +# don't print "broken pipe" error when piped into commands like "head" +signal(SIGPIPE,SIG_DFL) + +### Defaults ### +default_group_by='comm,state' + +### CLI handling ### +parser = argparse.ArgumentParser() +parser.add_argument('-d', dest="sample_seconds", metavar='seconds', type=int, default=5, help='number of seconds to sample for') +parser.add_argument('-p', '--pid', metavar='pid', default=None, nargs='?', help='process id to sample (including all its threads), or process name regex, or omit for system-wide sampling') +# TODO implement -t filtering below +parser.add_argument('-t', '--thread', metavar='tid', default=None, nargs='?', help='thread/task id to sample (not implemented yet)') + +parser.add_argument('-r', '--recursive', default=False, action='store_true', help='also sample and report for descendant processes') +parser.add_argument('-a', '--all-states', default=False, action='store_true', help='display threads in all states, including idle ones') + +parser.add_argument('--sample-hz', default=20, type=int, help='sample rate in Hz (default: %(default)d)') +parser.add_argument('--ps-hz', default=2, type=int, help='sample rate of new processes in Hz (default: %(default)d)') + +parser.add_argument('-o', '--output-sample-db', metavar='filename', default=':memory:', type=str, help='path of sqlite3 database to persist samples to, defaults to in-memory/transient') +parser.add_argument('-i', '--input-sample-db', metavar='filename', default=None, type=str, help='path of sqlite3 database to read samples from instead of actively sampling') + +parser.add_argument('-s', '--select', metavar='csv-columns', default='', help='additional columns to report') +parser.add_argument('-g', '--group-by', metavar='csv-columns', default=default_group_by, help='columns to aggregate by in reports') +parser.add_argument('-G', '--append-group-by', metavar='csv-columns', default=None, help='default + additional columns to aggregate by in reports') +#parser.add_argument('-f', '--filter', metavar='filter-sql', default='1=1', help="sample schema SQL to apply to filter report. ('active' and 'idle' keywords may also be used)") + +parser.add_argument('--list', default=None, action='store_true', help='list all available columns') + +# specify csv sources to be captured in full. use with -o or --output-sample-db to capture proc data for manual analysis +parser.add_argument('--sources', metavar='csv-source-names', default='', help=argparse.SUPPRESS) +# capture all sources in full. use with -o or --output-sample-db', help=argparse.SUPPRESS) +parser.add_argument('--all-sources', default=False, action='store_true', help=argparse.SUPPRESS) + +parser.add_argument('--debug', default=False, action='store_true', help=argparse.SUPPRESS) +parser.add_argument('--profile', default=False, action='store_true', help=argparse.SUPPRESS) +parser.add_argument('--show-yourself', default=False, action='store_true', help=argparse.SUPPRESS) + +args = parser.parse_args() + + +### Set up Logging +if args.debug: + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(asctime)s %(message)s') +else: + logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='%(asctime)s %(message)s') + +start_time = time.time() +sample_period_s = 1. / args.sample_hz +sample_ps_period_s = 1. / args.ps_hz + +if args.list: + for p in proc.all_sources: + print("\n" + p.name) + print("====================================") + for c in p.available_columns: + if c[1]: + print("%-30s %5s" % (c[0], c[1].__name__)) + + exit(0) + +if args.all_sources: + args.sources = proc.all_sources +else: + args.sources = [s for s in proc.all_sources if s.name in args.sources.split(',')] + +args.select = args.select.split(',') +args.group_by = args.group_by.split(',') +if args.append_group_by: + args.append_group_by = args.append_group_by.split(',') +else: + args.append_group_by = [] + +if not args.all_states: + included_states = 'active' +else: + included_states = None + +### Report definition ### + +active_process_sample = psnreport.Report( + 'Active Threads', + projection=args.select+['samples', 'avg_threads'], + dimensions=args.group_by+args.append_group_by, + order=['samples'], + where=[included_states] +) + +system_activity_by_user = psnreport.Report( + 'Active Threads by User', + projection=args.select+['uid', 'comm', 'samples', 'avg_threads'], + dimensions=args.group_by+args.append_group_by, +) + +reports = [active_process_sample]#, system_activity_by_user] + + +# consolidate sources required for all reports +sources = {} # source -> set(cols) +for r in reports: + for s, cs in r.sources.items(): + source_cols = sources.get(s, set()) + source_cols.update(cs) + sources[s] = source_cols + +# add sources requested for full sampling +for full_source in args.sources: + sources[full_source] = set([c[0] for c in full_source.available_columns]) + +# update sources with set of reqd columns +for s, cs in sources.items(): + s.set_stored_columns(cs) + + +def sqlexec(conn, sql): + logging.debug(sql) + conn.execute(sql) + logging.debug('Done') + +def sqlexecmany(conn, sql, samples): + logging.debug(sql) + conn.execute(sql, samples) + logging.debug('Done') + +### Schema setup ### +if args.input_sample_db: + conn = sqlite3.connect(args.input_sample_db) +else: + conn = sqlite3.connect(args.output_sample_db) + sqlexec(conn, 'pragma synchronous=0') + + def create_table(conn, s): + def sqlite_type(python_type): + return {int: 'INTEGER', int: 'INTEGER', float: 'REAL', str: 'TEXT', str: 'TEXT'}.get(python_type) + + sqlite_cols = [(c[0], sqlite_type(c[1])) for c in s.schema_columns] + sqlite_cols_sql = ', '.join([' '.join(c) for c in sqlite_cols]) + sql = "CREATE TABLE IF NOT EXISTS '%s' (%s)" % (s.name, sqlite_cols_sql) + sqlexec(conn,sql) + + sql = "CREATE INDEX IF NOT EXISTS '%s_event_idx' ON %s (event_time, pid, task)" % (s.name, s.name) + sqlexec(conn,sql) + + for s in sources: + create_table(conn, s) + + +### Sampling utility functions ### +def get_matching_processes(pid_arg=None, recursive=False): + # where pid_arg can be a single pid, comma-separated pids, or a regex on process executable name + + process_children = {} # pid -> [children-pids] + pid_basename = [] + + # TODO use os.listdir() when -r or process name regex is not needed + for line in os.popen('ps -A -o pid,ppid,comm', 'r').readlines()[1:]: + tokens = line.split() + pid = int(tokens[0]) + ppid = int(tokens[1]) + + children = process_children.get(ppid, []) + children.append(pid) + process_children[ppid] = children + + path_cmd = tokens[2] + #cmd = os.path.basename(path_cmd) + cmd = path_cmd + pid_basename.append((pid, cmd)) + + if pid_arg: + try: + arg_pids = [int(p) for p in pid_arg.split(',')] + selected_pids = [p for p, b in pid_basename if p in arg_pids] + except ValueError as e: + selected_pids = [p for p, b in pid_basename if re.search(pid_arg, b)] + + # recursive pid walking is not needed when looking for all system pids anyway + if recursive: + i = 0 + while i < len(selected_pids): + children = process_children.get(selected_pids[i], []) + selected_pids.extend(children) + i += 1 + else: + selected_pids = [p for p, b in pid_basename] + + # deduplicate pids & remove pSnapper pid (as pSnapper doesnt consume any resources when it's not running) + selected_pids=list(set(selected_pids)) + if not args.show_yourself: + if os.getpid() in selected_pids: + selected_pids.remove(os.getpid()) + return selected_pids + + +def get_process_tasks(pids): + tasks_by_pid = {} + for pid in pids: + try: + tasks_by_pid[pid] = os.listdir('/proc/%s/task' % pid) + except OSError as e: + pass # process may nolonger exist by the time we get here + return tasks_by_pid + + +def get_utc_now(): + if sys.version_info >= (3, 2): + return datetime.datetime.now(datetime.timezone.utc) + else: + return datetime.datetime.utcnow() + + +## Main sampling loop ### +def main(): + final_warning='' # show warning text in the end of output, if any + + if args.input_sample_db: + num_sample_events = int(sqlexec(conn,'SELECT COUNT(DISTINCT(event_time)) FROM stat').fetchone()[0]) + total_measure_s = 0. + else: + print("") + print('Linux Process Snapper v%s by Tanel Poder [https://0x.tools]' % PSN_VERSION) + print('Sampling /proc/%s for %d seconds...' % (', '.join([s.name for s in sources.keys()]), args.sample_seconds)), + sys.stdout.flush() + num_sample_events = 0 + num_ps_samples = 0 + total_measure_s = 0. + + selected_pids = None + process_tasks = {} + sample_ioerrors = 0 # detect permissions issues with /proc/access + + sample_seconds = args.sample_seconds + + try: + while time.time() < start_time + args.sample_seconds: + measure_begin_time = get_utc_now() + event_time = measure_begin_time.isoformat() + + # refresh matching pids at a much lower frequency than process sampling as the underlying ps is expensive + if not selected_pids or time.time() > start_time + num_ps_samples * sample_ps_period_s: + selected_pids = get_matching_processes(args.pid, args.recursive) + process_tasks = get_process_tasks(selected_pids) + num_ps_samples += 1 + + if not selected_pids: + print('No matching processes found:', args.pid) + sys.exit(1) + + for pid in selected_pids: + try: + # if any process-level samples fail, don't insert any sample event rows for process or tasks... + process_samples = [s.sample(event_time, pid, pid) for s in sources.keys() if s.task_level == False] + + for s, samples in zip([s for s in sources.keys() if s.task_level == False], process_samples): + sqlexecmany(conn, s.insert_sql, samples) + + for task in process_tasks.get(pid, []): + try: + # ...but if a task disappears mid-sample simply discard data for that task only + task_samples = [s.sample(event_time, pid, task) for s in sources.keys() if s.task_level == True] + + for s, samples in zip([s for s in sources.keys() if s.task_level == True], task_samples): + # TODO factor out + conn.executemany(s.insert_sql, samples) + + except IOError as e: + sample_ioerrors +=1 + logging.debug(e) + continue + + except IOError as e: + sample_ioerrors +=1 + logging.debug(e) + continue + + conn.commit() + + num_sample_events += 1 + measure_delta = get_utc_now() - measure_begin_time + total_measure_s += measure_delta.seconds + (measure_delta.microseconds * 0.000001) + + sleep_until = start_time + num_sample_events * sample_period_s + time.sleep(max(0, sleep_until - time.time())) + + print('finished.') + print + except KeyboardInterrupt as e: + sample_seconds = time.time() - start_time + print('sampling interrupted.') + print + + + ### Query and report ### + for r in reports: + r.output_report(conn) + + print('samples: %s' % num_sample_events), + if not args.input_sample_db: + print('(expected: %s)' % int(sample_seconds * args.sample_hz)) + if num_sample_events < 10: + final_warning += 'Warning: less than 10 samples captured. Reduce the amount of processes monitored or run pSnapper for longer.' + else: + print + + first_table_name = list(sources.keys())[0].name + print('total processes: %s, threads: %s' % (conn.execute('SELECT COUNT(DISTINCT(pid)) FROM ' + first_table_name).fetchone()[0], conn.execute('SELECT COUNT(DISTINCT(task)) FROM ' + first_table_name).fetchone()[0])) + print('runtime: %.2f, measure time: %.2f' % (time.time() - start_time, total_measure_s)) + print + + # TODO needs some improvement as some file IO errors come from processes/threads exiting + # before we get to sample all files of interest under the PID + if sample_ioerrors >= 100: + final_warning = 'Warning: %d /proc file accesses failed. Run as root or avoid restricted\n proc-files like "syscall" or measure only your own processes' % sample_ioerrors + + if final_warning: + print(final_warning) + print + + +## dump python self-profiling data on exit +def exit(r): + if args.profile: + import pstats + p = pstats.Stats('psn-prof.dmp') + p.sort_stats('cumulative').print_stats(30) + sys.exit(r) + + +## let's do this! +if args.profile: + import cProfile + cProfile.run('main()', 'psn-prof.dmp') +else: + main() + +exit(0) diff --git a/bin/run_xcapture.sh b/bin/run_xcapture.sh new file mode 100755 index 0000000..7506312 --- /dev/null +++ b/bin/run_xcapture.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# 0x.Tools by Tanel Poder [https://0x.tools] +# Copyright 2019-2020 Tanel Poder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +if [ $# -ne 1 ]; then + echo "Usage: $0 output_dir" + exit 1 +fi + +SUDO=sudo # change to empty string if running without sudo +NICE=-5 # set to 0 if don't want to increase priority +SLEEP=60 + +logger "$0 Starting up outdir=$1 nice=$NICE" + +while true ; do + $SUDO nice -n $NICE xcapture -o $1 -c exe,cmdline,kstack + if [ $? -eq 1 ]; then + exit 1 + fi + + # we only get here should xcapture be terminated, try to restart + logger "$0 terminated with $?, attempting to restart in $SLEEP seconds" + sleep $SLEEP +done + diff --git a/bin/run_xcpu.sh b/bin/run_xcpu.sh new file mode 100755 index 0000000..48874db --- /dev/null +++ b/bin/run_xcpu.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# 0x.Tools by Tanel Poder [https://0x.tools] +# Copyright 2019-2020 Tanel Poder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +FREQUENCY=1 # 1 Hz sampling +SUDO=sudo # change to empty string if running without sudo +NICE=-5 +SLEEP=60 +PERF=/usr/bin/perf + +if [ $# -ne 1 ]; then + echo "Usage: $0 output_dir" + exit 1 +fi + +logger "$0 Starting up outdir=$1 nice=$NICE" + +while true ; do + $SUDO nice -n $NICE $PERF record -g -F $FREQUENCY -a \ + --switch-output=1m \ + --timestamp-filename \ + --timestamp \ + -o $1/xcpu + + # we only get here should perf be terminated, try to restart + logger "$0 terminated with $?, attempting to restart in $SLEEP seconds" + sleep $SLEEP +done + diff --git a/bin/schedlat b/bin/schedlat new file mode 100755 index 0000000..9f4f1aa --- /dev/null +++ b/bin/schedlat @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +# Copyright 2020 Tanel Poder +# Licensed under the Apache License, Version 2.0 (the "License") +# +# Name: schedlat.py (v0.2) +# Purpose: display % of time a process spent in CPU runqueue +# (scheduling latency) +# Usage: ./schedlat.py PID +# +# %CPU shows % of time the task spent on CPU +# %LAT shows % of time the task spent trying to get onto CPU (in runqueue) +# %SLP shows the delta (not on CPU, not in runqueue, thus sleeping/waiting) +# +# Other: More info at https://0x.tools + +from __future__ import print_function +from datetime import datetime +import time, sys + +if len(sys.argv) != 2 or sys.argv[1] == '-h': + print("usage: " + sys.argv[0] + " PID") + exit(1) + +pid=sys.argv[1] + +with open('/proc/' + pid + '/comm', 'r') as f: + print("SchedLat by Tanel Poder [https://0x.tools]\n\nPID=" + pid + " COMM=" + f.read()) + +print("%-20s %6s %6s %6s" % ("TIMESTAMP", "%CPU", "%LAT", "%SLP")) + +while True: + with open('/proc/' + pid + '/schedstat' , 'r') as f: + t1=time.time() + (cpu_ns1, lat_ns1, dontcare) = f.read().split() + time.sleep(1) + f.seek(0) + t2=time.time() + (cpu_ns2, lat_ns2, dontcare) = f.read().split() + + cpu=(int(cpu_ns2)-int(cpu_ns1))/(t2-t1)/10000000 + lat=(int(lat_ns2)-int(lat_ns1))/(t2-t1)/10000000 + + print("%-20s %6.1f %6.1f %6.1f" % (datetime.fromtimestamp(t2).strftime("%Y-%m-%d %H:%M:%S"), cpu, lat, 100-(cpu+lat))) + diff --git a/bin/syscallargs b/bin/syscallargs new file mode 100755 index 0000000..ecc6571 --- /dev/null +++ b/bin/syscallargs @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +# syscallargs -- List Linux system calls and their arguments, by Tanel Poder [https://0x.tools] +# Copyright 2024 Tanel Poder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +# NOTES +# Basic documentation is in my blog entry: +# https://tanelpoder.com/posts/list-linux-system-call-arguments-with-syscallargs/ + +# You can list system call numbers using: +# ausyscall --dump +# +# Or check psnproc.py to see how psn reads it from /usr/include/.../unistd.h files + +import os, sys, argparse, signal + +__version__ = "1.0.1" +__author__ = "Tanel Poder" +__date__ = "2024-06-11" +__description__ = "List system call arguments from debugfs" +__url__ = "https://0x.tools" + +DEFAULT_PATH='/sys/kernel/debug/tracing/events/syscalls' + + +def parse_syscall_format(file_path): + syscall_info = {} + args_found = False + with open(file_path, 'r') as file: + lines = file.readlines() + for line in lines: + line = line.strip() + if line.startswith('name:'): + # split line to fields, also remove leading "sys_enter_" with [10:] + syscall_name = line.split(':')[1].strip()[10:] + syscall_info['name'] = syscall_name + elif line.startswith('field:'): + # syscall arguments come immediately after "__syscall_nr" line + # saving syscall_nr to populate dict as not all syscalls have arguments + if line.startswith('field:int __syscall_nr'): + args_found = True + if args_found: + # example: field:char * buf; offset:24; size:8; signed:0; + field_info = line.split(';')[0].replace(':',' ').split() + if len(field_info) >= 3: + field_type = ' '.join(field_info[1:-1]) + field_name = field_info[-1] + if 'args' not in syscall_info: + syscall_info['args'] = [] + syscall_info['args'].append((field_type, field_name)) + + return syscall_info + +def list_syscalls(syscalls_path): + syscalls = [] + for root, dirs, files in os.walk(syscalls_path): + dirs[:] = [d for d in dirs if d.startswith("sys_enter_")] + for file in files: + if file == 'format': + file_path = os.path.join(root, file) + syscall_info = parse_syscall_format(file_path) + syscalls.append(syscall_info) + + if not syscalls: + if os.geteuid() != 0: + print("Error: No syscalls found. Please run this as root or mount the debugfs with proper permissions.", file=sys.stderr) + else: + print(f"Error: No syscalls found in the specified path {syscalls_path}", file=sys.stderr) + sys.exit(1) + + return syscalls + + +def main(): + signal.signal(signal.SIGPIPE, signal.SIG_DFL) # for things like: ./syscallargs | head + + parser = argparse.ArgumentParser(description='List kernel system calls and their arguments from Linux debugfs.') + parser.add_argument('-l', '--newlines', action='store_true', help='Print each system call and its arguments on a new line') + parser.add_argument('-t', '--typeinfo', action='store_true', help='Include type information for syscall arguments in the output') + parser.add_argument('-V', '--version', action='version', version=f"%(prog)s {__version__} by {__author__} [{__url__}]", help='Show the program version and exit') + parser.add_argument('--path', type=str, default=DEFAULT_PATH, help=f'Path to the debugfs syscalls directory: {DEFAULT_PATH}') + + args = parser.parse_args() + syscalls_path = args.path + syscalls = list_syscalls(syscalls_path) + + for syscall in syscalls: + print(f"{syscall['name']}", end="\n" if args.newlines else "(") + + args_list = [] + for index, (arg_type, arg_name) in enumerate(syscall['args'][1:]): + if args.typeinfo: + argout = f"({arg_type}) {arg_name}" + else: + argout = arg_name + if args.newlines: + print(f" {index}: {argout}") + else: + args_list.append(argout) + if not args.newlines: + print(", ".join(args_list), end="") + + + print("" if args.newlines else ");") + +if __name__ == "__main__": + main() + + diff --git a/bin/syscalls.csv b/bin/syscalls.csv new file mode 100644 index 0000000..96e239a --- /dev/null +++ b/bin/syscalls.csv @@ -0,0 +1,362 @@ +SYSCALL_ID,SYSCALL_NAME +0,read +1,write +2,open +3,close +4,stat +5,fstat +6,lstat +7,poll +8,lseek +9,mmap +10,mprotect +11,munmap +12,brk +13,rt_sigaction +14,rt_sigprocmask +15,rt_sigreturn +16,ioctl +17,pread +18,pwrite +19,readv +20,writev +21,access +22,pipe +23,select +24,sched_yield +25,mremap +26,msync +27,mincore +28,madvise +29,shmget +30,shmat +31,shmctl +32,dup +33,dup2 +34,pause +35,nanosleep +36,getitimer +37,alarm +38,setitimer +39,getpid +40,sendfile +41,socket +42,connect +43,accept +44,sendto +45,recvfrom +46,sendmsg +47,recvmsg +48,shutdown +49,bind +50,listen +51,getsockname +52,getpeername +53,socketpair +54,setsockopt +55,getsockopt +56,clone +57,fork +58,vfork +59,execve +60,exit +61,wait4 +62,kill +63,uname +64,semget +65,semop +66,semctl +67,shmdt +68,msgget +69,msgsnd +70,msgrcv +71,msgctl +72,fcntl +73,flock +74,fsync +75,fdatasync +76,truncate +77,ftruncate +78,getdents +79,getcwd +80,chdir +81,fchdir +82,rename +83,mkdir +84,rmdir +85,creat +86,link +87,unlink +88,symlink +89,readlink +90,chmod +91,fchmod +92,chown +93,fchown +94,lchown +95,umask +96,gettimeofday +97,getrlimit +98,getrusage +99,sysinfo +100,times +101,ptrace +102,getuid +103,syslog +104,getgid +105,setuid +106,setgid +107,geteuid +108,getegid +109,setpgid +110,getppid +111,getpgrp +112,setsid +113,setreuid +114,setregid +115,getgroups +116,setgroups +117,setresuid +118,getresuid +119,setresgid +120,getresgid +121,getpgid +122,setfsuid +123,setfsgid +124,getsid +125,capget +126,capset +127,rt_sigpending +128,rt_sigtimedwait +129,rt_sigqueueinfo +130,rt_sigsuspend +131,sigaltstack +132,utime +133,mknod +134,uselib +135,personality +136,ustat +137,statfs +138,fstatfs +139,sysfs +140,getpriority +141,setpriority +142,sched_setparam +143,sched_getparam +144,sched_setscheduler +145,sched_getscheduler +146,sched_get_priority_max +147,sched_get_priority_min +148,sched_rr_get_interval +149,mlock +150,munlock +151,mlockall +152,munlockall +153,vhangup +154,modify_ldt +155,pivot_root +156,_sysctl +157,prctl +158,arch_prctl +159,adjtimex +160,setrlimit +161,chroot +162,sync +163,acct +164,settimeofday +165,mount +166,umount2 +167,swapon +168,swapoff +169,reboot +170,sethostname +171,setdomainname +172,iopl +173,ioperm +174,create_module +175,init_module +176,delete_module +177,get_kernel_syms +178,query_module +179,quotactl +180,nfsservctl +181,getpmsg +182,putpmsg +183,afs_syscall +184,tuxcall +185,security +186,gettid +187,readahead +188,setxattr +189,lsetxattr +190,fsetxattr +191,getxattr +192,lgetxattr +193,fgetxattr +194,listxattr +195,llistxattr +196,flistxattr +197,removexattr +198,lremovexattr +199,fremovexattr +200,tkill +201,time +202,futex +203,sched_setaffinity +204,sched_getaffinity +205,set_thread_area +206,io_setup +207,io_destroy +208,io_getevents +209,io_submit +210,io_cancel +211,get_thread_area +212,lookup_dcookie +213,epoll_create +214,epoll_ctl_old +215,epoll_wait_old +216,remap_file_pages +217,getdents64 +218,set_tid_address +219,restart_syscall +220,semtimedop +221,fadvise64 +222,timer_create +223,timer_settime +224,timer_gettime +225,timer_getoverrun +226,timer_delete +227,clock_settime +228,clock_gettime +229,clock_getres +230,clock_nanosleep +231,exit_group +232,epoll_wait +233,epoll_ctl +234,tgkill +235,utimes +236,vserver +237,mbind +238,set_mempolicy +239,get_mempolicy +240,mq_open +241,mq_unlink +242,mq_timedsend +243,mq_timedreceive +244,mq_notify +245,mq_getsetattr +246,kexec_load +247,waitid +248,add_key +249,request_key +250,keyctl +251,ioprio_set +252,ioprio_get +253,inotify_init +254,inotify_add_watch +255,inotify_rm_watch +256,migrate_pages +257,openat +258,mkdirat +259,mknodat +260,fchownat +261,futimesat +262,newfstatat +263,unlinkat +264,renameat +265,linkat +266,symlinkat +267,readlinkat +268,fchmodat +269,faccessat +270,pselect6 +271,ppoll +272,unshare +273,set_robust_list +274,get_robust_list +275,splice +276,tee +277,sync_file_range +278,vmsplice +279,move_pages +280,utimensat +281,epoll_pwait +282,signalfd +283,timerfd_create +284,eventfd +285,fallocate +286,timerfd_settime +287,timerfd_gettime +288,accept4 +289,signalfd4 +290,eventfd2 +291,epoll_create1 +292,dup3 +293,pipe2 +294,inotify_init1 +295,preadv +296,pwritev +297,rt_tgsigqueueinfo +298,perf_event_open +299,recvmmsg +300,fanotify_init +301,fanotify_mark +302,prlimit64 +303,name_to_handle_at +304,open_by_handle_at +305,clock_adjtime +306,syncfs +307,sendmmsg +308,setns +309,getcpu +310,process_vm_readv +311,process_vm_writev +312,kcmp +313,finit_module +314,sched_setattr +315,sched_getattr +316,renameat2 +317,seccomp +318,getrandom +319,memfd_create +320,kexec_file_load +321,bpf +322,execveat +323,userfaultfd +324,membarrier +325,mlock2 +326,copy_file_range +327,preadv2 +328,pwritev2 +329,pkey_mprotect +330,pkey_alloc +331,pkey_free +332,statx +333,io_pgetevents +334,rseq +424,pidfd_send_signal +425,io_uring_setup +426,io_uring_enter +427,io_uring_register +428,open_tree +429,move_mount +430,fsopen +431,fsconfig +432,fsmount +433,fspick +434,pidfd_open +435,clone3 +436,close_range +437,openat2 +438,pidfd_getfd +439,faccessat2 +440,process_madvise +441,epoll_pwait2 +442,mount_setattr +443,quotactl_fd +444,landlock_create_ruleset +445,landlock_add_rule +446,landlock_restrict_self +447,memfd_secret +448,process_mrelease +449,futex_waitv diff --git a/bin/transform.jq b/bin/transform.jq new file mode 100644 index 0000000..4e08db2 --- /dev/null +++ b/bin/transform.jq @@ -0,0 +1,44 @@ +(["SAMPLE_TIME", "TID", "PID", "COMM", "TASK_STATE", "SYSCALL_ID", + "SYSCALL_ARG0", "SYSCALL_ARG1", "SYSCALL_ARG2", + "SYSCALL_ARG3", "SYSCALL_ARG4", "SYSCALL_ARG5", + "CMDLINE", "PROFILE_USTACK", "PROFILE_KSTACK", + "SYSCALL_USTACK", "OFFCPU_USTACK", "OFFCPU_KSTACK", + "SCHED_WAKEUP", "ORACLE_WAIT_EVENT"], +(.samples[] | + .SAMPLE_TIME as $time | + .comm as $comm_map | + .task_state as $task_state_map | + .syscall_id as $syscall_id_map | + .syscall_args as $syscall_args_map | + .cmdline as $cmdline_map | + .profile_ustack as $profile_ustack_map | + .profile_kstack as $profile_kstack_map | + .syscall_ustack as $syscall_ustack_map | + .offcpu_ustack as $offcpu_ustack_map | + .offcpu_kstack as $offcpu_kstack_map | + .sched_wakeup as $sched_wakeup_map | + .oracle_wait_event as $oracle_wait_event_map | + .pid | to_entries[] | .key as $key | + [$time, $key, .value, + ($comm_map [$key] ), # // "-" + ($task_state_map [$key] ), + ($syscall_id_map [$key] ), + ($syscall_args_map [$key][0] ), + ($syscall_args_map [$key][1] ), + ($syscall_args_map [$key][2] ), + ($syscall_args_map [$key][3] ), + ($syscall_args_map [$key][4] ), + ($syscall_args_map [$key][5] ), + ($cmdline_map [$key] ), + ($profile_ustack_map [$key] ), + ($profile_kstack_map [$key] ), + ($syscall_ustack_map [$key] ), + ($offcpu_ustack_map [$key] ), + ($offcpu_kstack_map [$key] ), + ($sched_wakeup_map [$key] ), + ($oracle_wait_event_map [$key] ) + ] +)) | @csv + +# vi:syntax=zsh + diff --git a/bin/vmtop b/bin/vmtop new file mode 100755 index 0000000..af6755f --- /dev/null +++ b/bin/vmtop @@ -0,0 +1,29 @@ +#!/bin/bash + +# This tool is part of https://0x.tools + +if [ $# -ne 1 ]; then + echo "Usage: $0 SLEEP_SECONDS" + exit 1 +fi + +F1=/tmp/vmtop1.$$.tmp +F2=/tmp/vmtop2.$$.tmp + +cat /proc/vmstat > $F2 + +while true ; do + clear + echo `date` " [0x.tools vmtop]" + echo + printf "%-32s %16s %16s %16s %16s\n" "METRIC" "DELTA" "DELTA_KB" "CURRENT" "CURRENT_MB" + printf "%-32s %16s %16s %16s %16s\n" "-------------------------------" "----------------" "----------------" "----------------" "----------------" + mv $F2 $F1 + cat /proc/vmstat > $F2 + join $F1 $F2 | grep ^nr | awk '{ printf("%-32s %16d %\47 16i %\47 16i %\47 16i\n", $1,$3-$2, ($3-$2)*4, $3, $3*4/1024) }' | grep -v ' 0 ' + sleep $1 +done + + +# TODO trap CTRL-C remove file + diff --git a/bin/xcapture-bpf b/bin/xcapture-bpf new file mode 100755 index 0000000..e492094 --- /dev/null +++ b/bin/xcapture-bpf @@ -0,0 +1,732 @@ +#!/usr/bin/env python3 + +# xcapture-bpf -- Always-on profiling of Linux thread activity, by Tanel Poder [https://tanelpoder.com] +# Copyright 2024 Tanel Poder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +__version__ = "2.0.3" +__author__ = "Tanel Poder" +__date__ = "2024-06-27" +__description__ = "Always-on profiling of Linux thread activity using eBPF." +__url__ = "https://0x.tools" + +DEFAULT_GROUP_BY = "st,username,comm,syscall" # for xtop mode +DECODE_CHARSET = "utf-8" +XTOP_MAX_LINES = 25 # how many top output lines to print +BLOCK_CHARS = ['▏', '▎', '▍', '▌', '▋', '▊', '▉', '█'] # for fancy viz + +import os, sys, io, pwd, time, ctypes, platform, re, shutil, argparse, signal +from collections import defaultdict +from datetime import datetime +from bcc import BPF, PerfType, PerfSWConfig + +# distro package might not be present +try: + import distro +except ImportError: + distro = None + pass + +# all available fields with descriptions (if you add more fields to thread_state_t in BPF/C, add them here) +available_fdescr = [ ('timestamp' , 'sample timestamp') + , ('st' , 'short thread state') + , ('tid' , 'thread/task id') + , ('pid' , 'process/thread group id') + , ('username' , 'username or user id if not found') + , ('comm' , 'task comm digits deduplicated') + , ('comm2' , 'task comm actual') + , ('syscall' , 'system call') + , ('cmdline' , 'argv0 command line digits deduplicated') + , ('cmdline2' , 'argv0 command line actual') + , ('offcpu_u' , 'user stack id when thread went off CPU') + , ('offcpu_k' , 'kernel stack id when thread went off CPU') + , ('oncpu_u' , 'recent user stack id if the thread was on CPU') + , ('oncpu_k' , 'recent kernel stack id if the thread was on CPU') + , ('waker_tid' , 'thread ID that woke up this thread last') + , ('sch' , 'thread state flags for scheduler nerds') + ] + +available_fields = [] +for f in available_fdescr: + available_fields.append(f[0]) + +# default output fields for ungrouped full detail output +output_fields = [ 'timestamp', 'st', 'tid', 'pid', 'username', 'comm', 'syscall', 'cmdline' + , 'offcpu_u', 'offcpu_k', 'oncpu_u', 'oncpu_k', 'waker_tid', 'sch' ] + + +# syscall id to name translation (todo: fix aarch64 include file lookup) +def extract_system_call_ids(unistd_64_fh): + syscall_id_to_name = {} + + # strip 3264bit prefixes from syscall names + for name_prefix in ['__NR_', '__NR3264_']: + for line in unistd_64_fh.readlines(): + tokens = line.split() + if tokens and len(tokens) == 3 and tokens[0] == '#define' and tokens[2].isnumeric() is True: + _, s_name, s_id = tokens + s_id = int(s_id) + if s_name.startswith(name_prefix): + s_name = s_name[len(name_prefix):] + syscall_id_to_name[s_id] = s_name + + return syscall_id_to_name + +def get_system_call_names(): + psn_dir=os.path.dirname(os.path.realpath(__file__)) + kernel_ver=platform.release().split('-')[0] + + # this probably needs to be improved for better platform support + if platform.machine() == 'aarch64': + unistd_64_paths = ['/usr/include/asm-generic/unistd.h'] + else: + unistd_64_paths = [ '/usr/include/asm/unistd_64.h', '/usr/include/x86_64-linux-gnu/asm/unistd_64.h' + , '/usr/include/asm-x86_64/unistd.h', '/usr/include/asm/unistd.h' + , psn_dir+'/syscall_64_'+kernel_ver+'.h', psn_dir+'/syscall_64.h'] + + for path in unistd_64_paths: + try: + with open(path) as f: + return extract_system_call_ids(f) + except IOError as e: + pass + + raise Exception('unistd_64.h not found in' + ' or '.join(unistd_64_paths) + '.\n' + + ' You may need to "dnf install kernel-headers" or "apt-get install libc6-dev"\n') + +# syscall lookup table +syscall_id_to_name = get_system_call_names() + + +# task states +TASK_RUNNING = 0x00000000 +TASK_INTERRUPTIBLE = 0x00000001 +TASK_UNINTERRUPTIBLE = 0x00000002 +TASK_STOPPED = 0x00000004 +TASK_TRACED = 0x00000008 + +EXIT_DEAD = 0x00000010 +EXIT_ZOMBIE = 0x00000020 +EXIT_TRACE = (EXIT_ZOMBIE | EXIT_DEAD) + +TASK_PARKED = 0x00000040 +TASK_DEAD = 0x00000080 +TASK_WAKEKILL = 0x00000100 +TASK_WAKING = 0x00000200 +TASK_NOLOAD = 0x00000400 +TASK_NEW = 0x00000800 +TASK_RTLOCK_WAIT = 0x00001000 +TASK_FREEZABLE = 0x00002000 +TASK_FREEZABLE_UNSAFE = 0x00004000 # depends on: IS_ENABLED(CONFIG_LOCKDEP) +TASK_FROZEN = 0x00008000 +TASK_STATE_MAX = 0x00010000 # as of linux kernel 6.9 + +##define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPN" + +task_states = { + 0x00000000: "R", # "RUNNING", + 0x00000001: "S", # "INTERRUPTIBLE", + 0x00000002: "D", # UNINTERRUPTIBLE", + 0x00000004: "T", # "STOPPED", + 0x00000008: "t", # "TRACED", + 0x00000010: "X", # "EXIT_DEAD", + 0x00000020: "Z", # "EXIT_ZOMBIE", + 0x00000040: "P", # "PARKED", + 0x00000080: "dd",# "DEAD", + 0x00000100: "wk",# "WAKEKILL", + 0x00000200: "wg",# "WAKING", + 0x00000400: "I", # "NOLOAD", + 0x00000800: "N", # "NEW", + 0x00001000: "rt",# "RTLOCK_WAIT", + 0x00002000: "fe",# "FREEZABLE", + 0x00004000: "fu",# "__TASK_FREEZABLE_UNSAFE = (0x00004000 * IS_ENABLED(CONFIG_LOCKDEP))" + 0x00008000: "fo",# "FROZEN" +} + + +def get_task_state_name(task_state): + if task_state == 0: + return "R" + if task_state & TASK_NOLOAD: # idle kthread waiting for work + return "I" + + names = [] + for state, name in task_states.items(): + if task_state & state: + names.append(name) + + return "+".join(names) + + +# is task state interesting ("active") according to your rules +# mode=active: any states that should be captured and printed out (including perf/on-cpu samples) +# mode=offcpu: states that are relevant for offcpu stack printing (the BPF program doesn't clear up previous offcpu stackids) +# mode=oncpu: states that are relevant for on-cpu stack printing (don't print previous oncpu stacks if a task sample is not on CPU) +def is_interesting(st, syscall, comm, mode="active"): + if mode == "active": + if st[0] in ['R','D', 'T', 't']: + return True + if st[0] == 'S': + if current_syscall == 'io_getevents' and comm.startswith('ora'): + return True + + if mode == "offcpu": + if st[0] in ['D', 'T', 't'] or st.startswith('RQ'): # there may be occasinal states like "D+wk" reported + return True + if st[0] == 'S': + if current_syscall == 'io_getevents' and comm.startswith('ora'): + return True + + if mode == "oncpu": + if st[0] == 'R': + return True + + return False + +# translate uid to username (no container/uid namespace support right now) +def get_username(uid): + try: + username = pwd.getpwuid(uid).pw_name + return username + except KeyError: + return str(uid) + + + +def print_fields(rows, columns, linelimit=0): + columns = [col.rstrip() for col in columns] # strip as colname might have extra spaces passed in for width/formatting + col_widths = {} + # column width auto-sizing + for col in columns: + col_length = len(col) # the col may have extra trailing spaces as a formatting directive + max_value_length = max((len(str(row[col])) for row in rows if col in row), default=0) + col_widths[col] = max(col_length, max_value_length) + + header1 = "=== Active Threads " + header2 = " | ".join(f"{col:<{col_widths[col]}}" for col in columns) + + print(header1 + "=" * (len(header2) - len(header1)) + "\n") + print(header2) + print("-" * len(header2)) + + for i, row in enumerate(rows): + line = " | ".join( + f"{row[col]:>{col_widths[col]}.2f}" if col in ["seconds", "samples", "avg_thr"] + else f"{str(row[col]):<{col_widths[col]}}" + if col in row else ' ' * col_widths[col] for col in columns + ) + print(line) + + # dont break out if linelimit is at its default 0 + if linelimit and i >= linelimit - 1: + break + +def print_header_csv(columns): + header = ",".join(f"{col.upper()}" for col in columns) + print(header) + +def print_fields_csv(rows, columns): + for i, row in enumerate(rows): + line = ",".join(f"{row[col]}" for col in columns) + print(line) + +def get_ustack_traces(ustack_traces, ignore_ustacks={}, strip_args=True): + exclusions = ['__GI___clone3'] + dedup_map = {} + lines = [] + + for stack_id, pid in output_ustack: + if stack_id and stack_id >= 0 and stack_id not in ignore_ustacks: # todo: find why we have Null/none stackids in this map + line = f"ustack {stack_id:6} " + stack = list(ustack_traces.walk(stack_id)) + for addr in reversed(stack): # reversed(stack): + func_name = b.sym(addr, pid).decode(DECODE_CHARSET, 'replace') + if func_name not in exclusions: + if strip_args: + func_name = re.split('[<(]', func_name)[0] + line += "->" + (func_name if func_name != '[unknown]' else '{:x}'.format(addr)) + + dedup_map[stack_id] = line + + for stack_id in sorted(dedup_map): + lines.append(dedup_map[stack_id]) + + return lines + +def get_kstack_traces(kstack_traces, ignore_kstacks={}): + exclusions = ['entry_SYSCALL_64_after_hwframe', 'do_syscall_64', 'x64_sys_call' + , 'ret_from_fork_asm', 'ret_from_fork', '__bpf_trace_sched_switch', '__traceiter_sched_switch' + , 'el0t_64_sync', 'el0t_64_sync_handler', 'el0_svc', 'do_el0_svc', 'el0_svc_common', 'invoke_syscall' ] + lines = [] + + for k, v in kstack_traces.items(): + stack_id = k.value + if stack_id in output_kstack and stack_id not in ignore_kstacks: + line = f"kstack {stack_id:6} " + if stack_id >= 0: + stack = list(kstack_traces.walk(stack_id)) + + for addr in reversed(stack): + func = b.ksym(addr).decode(DECODE_CHARSET, 'replace') + if func not in exclusions and not func.startswith('bpf_'): + line += "->" + b.ksym(addr).decode(DECODE_CHARSET, 'replace') + + lines.append(line) + + return lines + + +def pivot_stack_traces(traces): + pivoted_traces = [] + for trace in traces: + parts = trace.split("->") + pivoted_traces.append(parts) + + max_length = max(len(trace) for trace in pivoted_traces) + for trace in pivoted_traces: + while len(trace) < max_length: + trace.append("") + + return pivoted_traces + +def calculate_columns(pivoted_traces, max_line_length): + max_length = max(len(part) for trace in pivoted_traces for part in trace) + return max(1, max_line_length // (max_length + 3)) + +def print_pivoted_dynamic(traces, max_line_length): + num_traces = len(traces) + start = 0 + + while start < num_traces: + end = start + 1 + while end <= num_traces: + subset_traces = traces[start:end] + pivoted_traces = pivot_stack_traces(subset_traces) + num_columns = calculate_columns(pivoted_traces, max_line_length) + + if num_columns < end - start: + break + + end += 1 + + end -= 1 + subset_traces = traces[start:end] + pivoted_traces = pivot_stack_traces(subset_traces) + + max_length = max(len(part) for trace in pivoted_traces for part in trace) + + print("-" * max_line_length) + for row in zip(*pivoted_traces): + print(" | ".join(f"{part:<{max_length}}" for part in row) + ' |') + + start = end + +# stack printing and formatting choice driver function +def print_stacks_if_nerdmode(): + if args.giant_nerd_mode and stackmap: + # printing stacktiles first, so the task state info is in the bottom of terminal output + (term_width, term_height) = shutil.get_terminal_size() + + print_pivoted_dynamic(get_kstack_traces(stackmap), max_line_length=term_width) + print() + + print_pivoted_dynamic(get_ustack_traces(stackmap), max_line_length=term_width) + print() + + if args.nerd_mode: + for s in get_kstack_traces(stackmap): + print(s) + print() + for s in get_ustack_traces(stackmap): + print(s) + +# group by for reporting +def group_by(records, column_names, sample_attempts_in_set, time_range_in_set): + total_records = len(records) + grouped_data = defaultdict(lambda: {'samples': 0}) + + for record in records: + key = tuple(record[col] for col in column_names) + if key not in grouped_data: + grouped_data[key].update({col: record[col] for col in column_names}) + grouped_data[key]['samples'] += 1 + + grouped_list = list(grouped_data.values()) + + for item in grouped_list: + item['avg_thr'] = round(item['samples'] / sample_attempts_in_set, 2) + item['seconds'] = round(item['samples'] * (time_range_in_set / sample_attempts_in_set), 2) + + # fancy viz + pct = item['samples'] / total_records + full_blocks = int(pct * 10) + remainder = (pct * 80) % 8 + visual = '█' * full_blocks + if remainder > 0: + visual += BLOCK_CHARS[int(remainder)] + item['visual_pct'] = visual + #ascii also possible + #item['visual_pct'] = '#' * int(pct * 10) + + + return grouped_list + + +# main() +signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +# args +parser = argparse.ArgumentParser(description=__description__) +parser.add_argument('-x', '--xtop', action='store_true', help='Run in aggregated top-thread-activity (xtop) mode') +parser.add_argument('-d', dest="report_seconds", metavar='report_seconds', type=int, default=5, help='xtop report printing interval (default: %(default)ds)') +parser.add_argument('-f', '--sample-hz', default=20, type=int, help='xtop sampling frequency in Hz (default: %(default)d)') +parser.add_argument('-g', '--group-by', metavar='csv-columns', default=DEFAULT_GROUP_BY, help='Full column list what to group by') +parser.add_argument('-G', '--append-group-by', metavar='append-csv-columns', default=None, help='List of additional columns to default cols what to group by') +parser.add_argument('-n', '--nerd-mode', action='store_true', help='Print out relevant stack traces as wide output lines') +parser.add_argument('-N', '--giant-nerd-mode', action='store_true', help='Print out relevant stack traces as stacktiles') +parser.add_argument('-c', '--clear-screen', action='store_true', help='Clear screen before printing next output') +parser.add_argument('-V', '--version', action='version', version=f"%(prog)s {__version__} by {__author__} [{__url__}]", help='Show the program version and exit') +parser.add_argument('-o', '--output-dir', type=str, default=None, help=f'Directory path where to write the output CSV files') +parser.add_argument('-l', '--list', default=None, action='store_true', help='list all available columns for display and grouping') + +args = parser.parse_args() + +if args.list: + for f in available_fdescr: + print(f'{f[0]:15} {f[1]}') + sys.exit(0) + +if args.clear_screen and args.output_dir: + print("Error: --clear-screen (interactive) and --output-dir (continuous logging) are mutually exclusive, use only one option.") + sys.exit(1) + +# handle xtop -g and -G group by columns (and same -g/-G options work for non-xtop output col addition too) +# args.group_by defaults to DEFAULT_GROUP_BY +groupby_fields = args.group_by.split(',') + +if args.xtop: + groupby_fields = groupby_fields + args.append_group_by.split(',') if args.append_group_by else groupby_fields + used_fields = groupby_fields # todo +else: + output_fields = output_fields + args.append_group_by.split(',') if args.append_group_by else output_fields + used_fields = output_fields + +if set(used_fields) - set(available_fields): + print("Error: incorrect group by field name specified, use --list option see allowed columns") + exit(1) + +# eBPF programs have be loaded as root +if os.geteuid() != 0: + print("Error: you need to run this command as root") + sys.exit(1) + +# ready to go +progname = "xtop" if args.xtop else "xcapture-bpf" +kernname = platform.release().split('-')[0] +archname = platform.machine() +distroid = distro.id().title() if distro else '' +distrover = distro.version() if distro else '' +sf = None # fd for separate stackfile in continuous csv sampling mode + +print(f'=== [0x.tools] {progname} {__version__} BETA by {__author__}. {distroid} Linux {distrover} {kernname} {archname}') + +# open and load the BPF instrumenter +with open(os.path.dirname(os.path.abspath(__file__)) + '/xcapture-bpf.c', 'r') as file: + bpf_text = file.read() + +# set up global variables for conditionally inserting stack capture code +offcpu_u = 'offcpu_u' in used_fields +offcpu_k = 'offcpu_k' in used_fields +offcpu_stacks = offcpu_u or offcpu_k +oncpu_stacks = ('oncpu_u' in used_fields or 'oncpu_k' in used_fields) +cmdline = ('cmdline' in used_fields or 'cmdline2' in used_fields) + +# dynamic compilation of features that are needed +ifdef = '' +if offcpu_u: + ifdef += '#define OFFCPU_U 1\n' +if offcpu_k: + ifdef += '#define OFFCPU_K 1\n' +if offcpu_stacks: + ifdef += '#define OFFCPU_STACKS 1\n' +if oncpu_stacks: + ifdef += '#define ONCPU_STACKS 1\n' +if cmdline: + ifdef += '#define CMDLINE 1\n' + + +print('=== Loading BPF...') +b = BPF(text= ifdef + bpf_text) + +# Software CPU_CLOCK is useful in cloud & VM environments where perf hardware events +# are not available, but software clocks don't measure what happens when CPUs are in +# critical sections when most interrupts are disabled +b.attach_perf_event(ev_type=PerfType.SOFTWARE, ev_config=PerfSWConfig.CPU_CLOCK + , fn_name="update_cpu_stack_profile" + , sample_freq=2) # args.sample_hz if args.xtop else 1 + +# start sampling the Task State Array +tsa = b.get_table("tsa") + +if oncpu_stacks or offcpu_stacks: + stackmap = b.get_table("stackmap") +else: + stackmap = {} + +# get own pid so to not display it in output +mypid = os.getpid() +print(f"=== Ready (mypid {mypid})\n") + +# regex for replacing digits in "comm" for better grouping and reporting (comm2 shows original) +trim_comm = re.compile(r'\d+') + +written_kstacks = {} # stack ids already written to csv (in -o mode) +written_ustacks = {} + +first_report_printed = False # show first xtop report quicker +csv_header_printed = False + +while True: + try: + output_kstack = {} # map of stack_ids seen so far + output_ustack = {} + output_records = [] + + sample_start = time.time() + duration = (args.report_seconds if args.xtop and first_report_printed else 1) + sample_end = sample_start + duration # todo: 1 Hz for raw/csv output for now + first_report_printed = True + samples_attempted = 0 # not all TSA samples contain active threads of interest, this tells us how many samples we really took + + while time.time() < sample_end: + samples_attempted += 1 + ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + i = tsa.items()[0] + + for i in tsa.items(): + save_record = True + # extract python values from BPF ctypes, return '-' if there's no match + fields_dict = {field[0]: getattr(i[1], field[0], '-') for field in i[1]._fields_} + + if fields_dict['tid'] == mypid: + continue + + # additional fields for adding human readable info (not using None as it would be printed out as "None") + fields_dict['st'] = '' + fields_dict['sch'] = '' # for scheduler nerds + fields_dict['state_flags'] = '' # full scheduler state bitmap + fields_dict['username'] = '' + fields_dict['syscall'] = '' + fields_dict['comm2'] = '' + fields_dict['cmdline2'] = '' + + current_syscall = syscall_id_to_name.get(fields_dict['syscall_id'], '-') if fields_dict['syscall_set'] else '-' + comm = str(fields_dict['comm'], DECODE_CHARSET) + + in_sched_migrate = fields_dict['in_sched_migrate'] + in_sched_wakeup = fields_dict['in_sched_wakeup'] + in_sched_waking = fields_dict['in_sched_waking'] + is_running_on_cpu = fields_dict['is_running_on_cpu'] + + # we use state for conditionally printing out things like offcpu_stack etc + state_suffix = '' + state = get_task_state_name(fields_dict['state']) + + if state == 'R' and not is_running_on_cpu: # runnable on runqueue + state += 'Q' + + enriched_fields = {"timestamp": ts[:-3]} + + for field_name in fields_dict: + if not field_name in used_fields: + continue + + outv = None # enriched value + if field_name in ['state', 'st']: + if is_interesting(state, current_syscall, comm): + outv = state + else: + save_record = False + break + + elif field_name.startswith('comm'): + val = fields_dict['comm'] # source field is "comm" regardless of potential comm2 output field name + if isinstance(val, bytes): + outv = str(val, DECODE_CHARSET) + else: + outv = str(val) + if field_name == 'comm': # only trim "comm", but not comm2 that is the unaltered string + outv = re.sub(trim_comm, '*', outv) + + elif field_name.startswith('cmdline'): + val = fields_dict['cmdline'] + if isinstance(val, bytes): + outv = str(val, DECODE_CHARSET) + else: + outv = str(val) + if field_name == 'cmdline': + outv = re.sub(trim_comm, '*', outv) + + elif field_name == 'syscall': + outv = current_syscall + + elif field_name == 'username': + outv = get_username(fields_dict['uid']) + + elif field_name == ('offcpu_k'): # kstack id + val = fields_dict[field_name] + # runnable state can be R or RQ: RQ is also off CPU, so will capture it + if is_interesting(state, current_syscall, comm, 'offcpu') and val > 0: + outv = val + output_kstack[val] = True + else: + outv = '-' + + elif field_name == ("offcpu_u"): # ustack id + val = fields_dict[field_name] + if is_interesting(state, current_syscall, comm, 'offcpu') and val > 0: + outv = val + # using pid/tgid here, address space is same for all threads in a process + output_ustack[val, fields_dict['pid']] = True + else: + outv = '-' + + elif field_name == ('oncpu_k'): + val = fields_dict[field_name] + # only print the perf-cpu samples when actually caught on cpu (not runqueue) for now + if is_interesting(state, current_syscall, comm, 'oncpu') and val > 0: + outv = val + output_kstack[val] = True + else: + outv = '-' + + elif field_name == ("oncpu_u"): + val = fields_dict[field_name] + if is_interesting(state, current_syscall, comm, 'oncpu') and val > 0: + outv = val + # using pid/tgid here, address space is same for all threads in a process + output_ustack[val, fields_dict['pid']] = True + else: + outv = '-' + + elif field_name == 'sch': + # (in_sched_waking, in_sched_wakeup, is_running_on_cpu) + outv = '-' if in_sched_migrate else '_' + outv += '-' if in_sched_waking else '_' + outv += '-' if in_sched_wakeup else '_' + outv += '-' if is_running_on_cpu else '_' + + else: + val = fields_dict[field_name] + if isinstance(val, bytes): + outv = str(val, DECODE_CHARSET) + else: + outv = str(val) + + enriched_fields[field_name] = outv + + if save_record: + output_records.append(enriched_fields) + + time.sleep(1 / (args.sample_hz if args.xtop else 1)) + + if output_records: + # csv output mode will not do any terminal stuff + if args.output_dir: + outfile = args.output_dir + '/threads_' + ts[:13].replace(' ', '.') + '.csv' + + if os.path.isfile(outfile): # special case if xcapture-bpf has been restarted within the same hour + csv_header_printed = True + + if sys.stdout.name != outfile: # create a new output file when the hour changes + csv_header_printed = False # new file + sys.stdout = open(outfile, 'a') + + if not csv_header_printed: + print_header_csv(output_fields) + csv_header_printed = True + + print_fields_csv(output_records, output_fields) + + # stackfile is created once and name doesn't change throughout xcapture process lifetime + if not sf: + stackfile = args.output_dir + '/stacks_' + ts[:13].replace(' ', '.') + '.csv' + sf = open(stackfile, 'a') + + if sf: + for s in get_kstack_traces(stackmap, ignore_kstacks=written_kstacks): + print(s, file=sf) + written_kstacks[int(s.split()[1])] = True + #print(written_kstacks, file=sf) + + for s in get_ustack_traces(stackmap, ignore_ustacks=written_ustacks): + print(s, file=sf) + written_ustacks[int(s.split()[1])] = True + #print(written_ustacks, file=sf) + + sf.flush() + + else: + if args.clear_screen: # interactive (xtop) + buffer = io.StringIO() + sys.stdout = buffer + + print_stacks_if_nerdmode() + print() + print() + + if args.xtop: + total_records = len(output_records) + # a new field "samples" shows up (count(*)) + grouped_list = group_by(output_records, groupby_fields, samples_attempted, sample_end - sample_start) + ordered_aggr = sorted(grouped_list, key=lambda x: x['samples'], reverse=True) + print_fields(ordered_aggr, ['seconds', 'avg_thr', 'visual_pct'] + groupby_fields, linelimit=XTOP_MAX_LINES) + + print() + print() + print(f'sampled: {samples_attempted} times, avg_thr: {round(total_records / samples_attempted, 2)}') + print(f'start: {ts[:19]}, duration: {duration}s') + + if args.clear_screen: + # terminal size may change over time + (term_width, term_height) = shutil.get_terminal_size() + + for x in range(1, term_height - min(len(ordered_aggr), XTOP_MAX_LINES) - 9): # header/footer lines + print() + else: + print() + + else: # wide raw terminal output + print_fields(output_records, output_fields) + print() + print() + + if args.clear_screen: + os.system('clear') + output = buffer.getvalue() + sys.stdout = sys.__stdout__ + print(output) + + sys.stdout.flush() + + except KeyboardInterrupt: + exit(0) + #signal.signal(signal.SIGINT, signal.SIG_IGN) + + +# That's all, folks! diff --git a/bin/xcapture-bpf.c b/bin/xcapture-bpf.c new file mode 100644 index 0000000..443e5f9 --- /dev/null +++ b/bin/xcapture-bpf.c @@ -0,0 +1,379 @@ +/* + * 0x.Tools xcapture-bpf v2.0 beta + * Sample Linux task activity using eBPF [0x.tools] + * + * Copyright 2019-2024 Tanel Poder + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + */ + +#include <uapi/linux/bpf.h> +#include <uapi/linux/ptrace.h> +#include <linux/sched.h> +#include <linux/types.h> +#include <linux/syscalls.h> + +#ifdef BCC_SEC +#define __BCC__ +#endif + +// don't need EIP value for basic stack trace analysis (deduplicate some stackids) +// unfortunately SKIP_FRAMES 1 "skips" both the EIP value and one stack frame... +// #define SKIP_FRAMES (1 & BPF_F_SKIP_FIELD_MASK) +#define SKIP_FRAMES 0 + +// need to test if using BPF_STACK_TRACE_BUILDID would optimize things +// (apparently need separate stackmaps for ustacks & kstacks then) +#if defined(ONCPU_STACKS) || defined(OFFCPU_U) || defined(OFFCPU_K) +BPF_STACK_TRACE(stackmap, 65536); +#endif + +struct thread_state_t { + u32 state; // scheduler state + u32 flags; // PF_ flags + u32 tid; + u32 pid; + u32 uid; + char comm[TASK_COMM_LEN]; + char cmdline[64]; // task->mm->argv0 command executed, unless later changed to something else, like Postgres does + + u16 syscall_id; // unsigned as we switch the value to negative on completion, to see the last syscall + // unsigned long syscall_args[6]; // IBM s390x port has only 5 syscall args + +#ifdef OFFCPU_U + s32 offcpu_u; // offcpu ustack +#endif +#ifdef OFFCPU_K + s32 offcpu_k; // offcpu kstack +#endif +#ifdef ONCPU_STACKS + s32 oncpu_u; // cpu-profile ustack + s32 oncpu_k; // cpu-profile kstack +#endif + + s32 syscall_u; + + s32 waker_tid; // who invoked the waking of the target task + bool in_sched_migrate; // migrate to another CPU + bool in_sched_waking; // invoke wakeup, potentially on another CPU via inter-processor signalling (IPI) + bool in_sched_wakeup; // actual wakeup on target CPU starts + bool is_running_on_cpu; // sched_switch (to complete the wakeup/switch) has been invoked + + // s16 waker_syscall; + // s32 waker_ustack; + // s32 oracle_wait_event; + + // internal use by python frontend + bool syscall_set; // 0 means that syscall probe has not fired yet for this task, so don't resolve syscall_id 0 +}; + + +// not using BPF_F_NO_PREALLOC here for now, trading some kernel memory for better performance +BPF_HASH(tsa, u32, struct thread_state_t, 16384); + + +TRACEPOINT_PROBE(raw_syscalls, sys_enter) { +// a rudimentary way for ignoring some syscalls we do not care about (this whole thing will change before GA release) +#if defined(__x86_64__) + if (args->id == __NR_poll || args->id == __NR_getrusage) +#elif defined(__aarch64__) + if (args->id == __NR_getrusage) +#endif + return 0; + + struct thread_state_t t_empty = {}; + + u32 tid = bpf_get_current_pid_tgid() & 0xffffffff; + u32 pid = bpf_get_current_pid_tgid() >> 32; + struct task_struct *curtask = (struct task_struct *) bpf_get_current_task(); + + struct thread_state_t *t = tsa.lookup_or_try_init(&tid, &t_empty); + if (!t) return 0; + + if (!t->syscall_set) t->syscall_set = 1; + + t->syscall_id = args->id; + + // use a conditional copy(len(args))? + // t->syscall_arg0 = args->args[0]; + // t->syscall_arg1 = args->args[1]; + // t->syscall_arg2 = args->args[2]; + // t->syscall_arg3 = args->args[3]; + // t->syscall_arg4 = args->args[4]; + // t->syscall_arg5 = args->args[5]; + // t->syscall_u = stackmap.get_stackid(args, BPF_F_USER_STACK | BPF_F_REUSE_STACKID | BPF_F_FAST_STACK_CMP); + + tsa.update(&tid, t); + return 0; +} // raw_syscalls:sys_enter + + +TRACEPOINT_PROBE(raw_syscalls, sys_exit) { + + u32 tid = bpf_get_current_pid_tgid() & 0xffffffff; + u32 pid = bpf_get_current_pid_tgid() >> 32; + struct task_struct *curtask = (struct task_struct *) bpf_get_current_task(); + + struct thread_state_t t_empty = {}; + struct thread_state_t *t = tsa.lookup_or_try_init(&tid, &t_empty); + if (!t) return 0; + + t->syscall_id = t->syscall_id * -1; // switch the syscall_id to its negative value on exit + t->syscall_u = t->syscall_u * -1; + + tsa.update(&tid, t); + + return 0; +} // raw_syscalls:sys_exit + +// sampling profiling of on-CPU threads (python frontend uses perf event with freq=1) +// update the stack id of threads currently running on (any) CPU + +int update_cpu_stack_profile(struct bpf_perf_event_data *ctx) { + + u32 tid = bpf_get_current_pid_tgid() & 0xffffffff; + + // ignore tid 0 - kernel cpuidle + if (tid) { + u32 pid = bpf_get_current_pid_tgid() >> 32; + struct task_struct *curtask = (struct task_struct *) bpf_get_current_task(); + + struct thread_state_t t_empty = {}; + struct thread_state_t *t = tsa.lookup_or_try_init(&tid, &t_empty); + if (!t) return 0; + + t->tid = tid; + t->pid = pid; + t->uid = (s32) (bpf_get_current_uid_gid() & 0xFFFFFFFF); + t->state = curtask->__state; + bpf_probe_read_str(t->comm, sizeof(t->comm), (struct task_struct *)curtask->comm); + +#ifdef CMDLINE + if (curtask->mm && curtask->mm->arg_start) { + unsigned long arg_start = curtask->mm->arg_start; + bpf_probe_read_user_str(t->cmdline, sizeof(t->cmdline), (void *)arg_start); + } +#endif +#ifdef ONCPU_STACKS + t->oncpu_u = stackmap.get_stackid(ctx, SKIP_FRAMES | BPF_F_USER_STACK | BPF_F_REUSE_STACKID | BPF_F_FAST_STACK_CMP); + t->oncpu_k = stackmap.get_stackid(ctx, 0); // BPF_F_REUSE_STACKID | BPF_F_FAST_STACK_CMP); +#endif + + tsa.update(&tid, t); + } + + return 0; +}; + +// scheduler (or someone) wants this task to migrate to another CPU +TRACEPOINT_PROBE(sched, sched_migrate_task) { + + struct thread_state_t t_empty = {}; + u32 tid = args->pid; + + struct thread_state_t *t = tsa.lookup_or_try_init(&tid, &t_empty); + if (!t) return 0; + + t->in_sched_migrate = 1; + + tsa.update(&tid, t); + + return 0; +} + +// Context enrichment example (kernel): which waker (curtask->pid) woke a wakee (args->pid) up? +TRACEPOINT_PROBE(sched, sched_waking) { + + struct task_struct *curtask = (struct task_struct *) bpf_get_current_task(); + struct thread_state_t t_empty = {}; + + u32 tid_woken = args->pid; + + struct thread_state_t *t_being_waked_up = tsa.lookup_or_try_init(&tid_woken, &t_empty); + if (!t_being_waked_up) return 0; + + t_being_waked_up->in_sched_waking = 1; + t_being_waked_up->tid = tid_woken; // this guy is being woken up + t_being_waked_up->waker_tid = curtask->pid; // this is who wakes that guy up + + tsa.update(&tid_woken, t_being_waked_up); + + return 0; +} + +// Context enrichment example (kernel): woken up task waiting in the CPU runqueue +TRACEPOINT_PROBE(sched, sched_wakeup) { + + struct task_struct *curtask = (struct task_struct *) bpf_get_current_task(); + struct thread_state_t t_empty = {}; + + u32 tid_woken = args->pid; + + struct thread_state_t *t_being_waked_up = tsa.lookup_or_try_init(&tid_woken, &t_empty); + if (!t_being_waked_up) return 0; + + t_being_waked_up->in_sched_wakeup = 1; + t_being_waked_up->tid = tid_woken; // this guy is being woken up + + tsa.update(&tid_woken, t_being_waked_up); + + return 0; +} + +// newly started task woken up +TRACEPOINT_PROBE(sched, sched_wakeup_new) { + + struct task_struct *curtask = (struct task_struct *) bpf_get_current_task(); // curtask is who does the waking-up! + struct thread_state_t t_empty = {}; + + u32 tid_woken = args->pid; + + struct thread_state_t *t_new = tsa.lookup_or_try_init(&tid_woken, &t_empty); + if (!t_new) return 0; + + t_new->in_sched_wakeup = 1; + t_new->tid = tid_woken; // this guy is being woken up + t_new->waker_tid = curtask->pid; // this is who wakes that guy up (todo: is this valid here?) + + bpf_probe_read_str(t_new->comm, sizeof(t_new->comm), args->comm); + + // dont read cmdline here, will get cmdline of the task that ran clone()? + // bpf_probe_read_user_str(t_new->cmdline, sizeof(t_new->cmdline), (struct task_struct *)curtask->mm->arg_start); + + tsa.update(&tid_woken, t_new); + + return 0; +} + +// "next" is about to be put on CPU, "prev" goes off-CPU +RAW_TRACEPOINT_PROBE(sched_switch) { + + // from https://github.com/torvalds/linux/blob/master/include/trace/events/sched.h (sched_switch trace event) + bool *preempt = (bool *)ctx->args[0]; + struct task_struct *prev = (struct task_struct *)ctx->args[1]; + struct task_struct *next = (struct task_struct *)ctx->args[2]; + unsigned int prev_state = prev->__state; // ctx->args[3] won't work in older configs due to breaking change in sched_switch tracepoint + + s32 prev_tid = prev->pid; // task (tid in user tools) + s32 prev_pid = prev->tgid; // tgid (pid in user tools) + s32 next_tid = next->pid; // task + s32 next_pid = next->tgid; // tgid + + struct thread_state_t t_empty_prev = {0}; + struct thread_state_t t_empty_next = {0}; + + // we don't want to capture/report the previous cpuidle "task" during actual task wakeups (tid 0) + if (prev_tid) { + struct thread_state_t *t_prev = tsa.lookup_or_try_init(&prev_tid, &t_empty_prev); + if (!t_prev) return 0; + + t_prev->tid = prev_tid; + t_prev->pid = prev_pid; + t_prev->flags = prev->flags; + + // if (!t_prev->comm[0]) + bpf_probe_read_str(t_prev->comm, sizeof(t_prev->comm), prev->comm); + + // switch finished, clear waking/wakeup flags + t_prev->in_sched_migrate = 0; // todo: these 3 are probably not needed here + t_prev->in_sched_waking = 0; + t_prev->in_sched_wakeup = 0; + t_prev->is_running_on_cpu = 0; + t_prev->state = prev_state; + + t_prev->uid = prev->cred->euid.val; + +#ifdef OFFCPU_U + if (!(prev->flags & PF_KTHREAD)) + t_prev->offcpu_u = stackmap.get_stackid(ctx, SKIP_FRAMES | BPF_F_USER_STACK | BPF_F_REUSE_STACKID | BPF_F_FAST_STACK_CMP); +#endif +#ifdef OFFCPU_K + t_prev->offcpu_k = stackmap.get_stackid(ctx, BPF_F_REUSE_STACKID | BPF_F_FAST_STACK_CMP); +#endif +#ifdef CMDLINE + if (prev->mm && prev->mm->arg_start) { + unsigned long arg_start = prev->mm->arg_start; + bpf_probe_read_user_str(t_prev->cmdline, sizeof(t_prev->cmdline), (void *)arg_start); + } +#endif + tsa.update(&prev_tid, t_prev); + } + + // we don't want to capture/report the cpuidle "task" (tid 0) when CPU goes to cpuidle + if (next_tid) { + struct thread_state_t *t_next = tsa.lookup_or_try_init(&next_tid, &t_empty_next); + if (!t_next) return 0; + + t_next->tid = next_tid; + t_next->pid = next_pid; + t_next->flags = next->flags; + + //if (!t_next->comm[0]) + bpf_probe_read_str(t_next->comm, sizeof(t_next->comm), next->comm); + + t_next->state = next->__state; + t_next->in_sched_migrate = 0; + t_next->in_sched_waking = 0; + t_next->in_sched_wakeup = 0; + t_next->is_running_on_cpu = 1; + + t_next->uid = next->cred->euid.val; + +// previous off-cpu stacks are oncpu stacks when getting back onto cpu (kernel stack slightly different) +#ifdef ONCPU_STACKS +#ifdef OFFCPU_U + t_next->oncpu_u = t_next->offcpu_u; +#endif +#ifdef OFFCPU_K + t_next->oncpu_k = t_next->offcpu_k; +#endif +#endif + +#ifdef CMDLINE + if (next->mm && next->mm->arg_start) { + unsigned long arg_start = next->mm->arg_start; + bpf_probe_read_user_str(t_next->cmdline, sizeof(t_next->cmdline), (void *)arg_start); + } +#endif + + tsa.update(&next_tid, t_next); + } + + return 0; +} + + +// remove hashmap elements on task exit +static inline int cleanup_tid(u32 tid_exiting) { + tsa.delete(&tid_exiting); + return 0; +} + +TRACEPOINT_PROBE(sched, sched_process_exit) { + return cleanup_tid(args->pid); +} + +TRACEPOINT_PROBE(sched, sched_process_free) { + return cleanup_tid(args->pid); +} + +TRACEPOINT_PROBE(sched, sched_kthread_stop) { + return cleanup_tid(args->pid); +} + +// vim:syntax=c diff --git a/bin/xcapture.bt b/bin/xcapture.bt new file mode 100644 index 0000000..a39bca9 --- /dev/null +++ b/bin/xcapture.bt @@ -0,0 +1,201 @@ +/* + * 0x.Tools xcapture.bt v0.4 - Proof-of-concept prototype for sampling + * Linux thread activity using eBPF [0x.tools] + * + * Copyright 2019-2023 Tanel Poder + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + */ + +// This is a PoC prototype for demonstrating feasibility of the custom, programmable +// task state object populating + sampling approach. This script is not complete and +// it probably has bugs. I have plenty of improvements in It's not a finished tool or a product. +// +// To avoid the extremely slow stack address to symbol resolution in bpftrace, enable +// symbol caching, for example: +// +// sudo BPFTRACE_CACHE_USER_SYMBOLS=1 bpftrace xcapture.bt +// or +// sudo BPFTRACE_CACHE_USER_SYMBOLS=1 bpftrace -f json xcapture.bt > out.json + +BEGIN { + @TASK_STATES[0x00] = "R"; // "(running)" + @TASK_STATES[0x01] = "S"; // "(sleeping)" + @TASK_STATES[0x02] = "D"; // "(disk sleep)" + @TASK_STATES[0x04] = "T"; // "(stopped)" + @TASK_STATES[0x08] = "t"; // "(tracing stop)" + @TASK_STATES[0x10] = "X"; // "(dead)" + @TASK_STATES[0x20] = "Z"; // "(zombie)" + @TASK_STATES[0x40] = "P"; // "(parked)" + @TASK_STATES[0x80] = "I"; // "(idle)" +} + + +// record system calls by threads into the thread state array +// ideally/eventually need to move pid/uid/gid (and perhaps comm) assignment out of the syscall probe +tracepoint:raw_syscalls:sys_enter { + // [tid] uses thread local storage, cleaned out automatically on thread exit + @pid [tid] = pid; // *in bpftrace* tid means thread ID (task ID), pid means Process ID (thread group ID) + @uid [tid] = uid; + @gid [tid] = gid; + @comm [tid] = comm; + @cmdline [tid] = str(uptr(curtask->mm->arg_start)); + @task_state [tid] = @TASK_STATES[curtask->__state & 0xff]; + @syscall_id [tid] = args->id; + @syscall_args [tid] = (args->args[0], args->args[1], args->args[2], args->args[3], args->args[4], args->args[5]); + @syscall_ustack [tid] = ustack(); +} + +tracepoint:raw_syscalls:sys_exit { + delete(@syscall_id[tid]) // @syscall_id [tid] = -1; +} + + +// thread requests going off CPU +// by the time schedule() is called, the caller has set the new task state +kprobe:schedule { + @task_state [tid] = @TASK_STATES[curtask->__state & 0xff]; + @offcpu_ustack [tid] = ustack(); + @offcpu_kstack [tid] = kstack(); +} + +// thread has been put back on CPU +// newer kernels have the "isra" version of this function name, thus the * wildcard +kprobe:finish_task_switch* { + @task_state [tid] = @TASK_STATES[curtask->__state & 0xff]; + delete(@offcpu_ustack[tid]); + delete(@offcpu_kstack[tid]); +} + +// sampled profiling of on-CPU threads +// update the stack id of threads currently running on (any) cpu +profile:hz:1 { + @task_state [tid] = @TASK_STATES[curtask->__state & 0xff]; + @profile_ustack[tid] = ustack(); + @profile_kstack[tid] = kstack(); +} + +// Context enrichment example (kernel): tasks waiting in the CPU runqueue +tracepoint:sched:sched_wakeup, +tracepoint:sched:sched_wakeup_new { + @sched_wakeup[args->pid] = 1; +} + +tracepoint:sched:sched_switch { + delete(@sched_wakeup[args->next_pid]); // or: @sched_wakeup[args->next_pid] = -1; +} + +tracepoint:sched:sched_process_exit { + delete(@pid [args->pid]); + delete(@uid [args->pid]); + delete(@gid [args->pid]); + delete(@comm [args->pid]); + delete(@cmdline [args->pid]); + delete(@task_state [args->pid]); + delete(@syscall_id [args->pid]); + delete(@syscall_args [args->pid]); + delete(@syscall_ustack [args->pid]); + delete(@sched_wakeup [args->pid]); +} + + +// Context enrichment example (application): Oracle database wait events +uprobe:/u01/app/oracle/product/19.0.0/dbhome_1/bin/oracle:kskthbwt { + $EVENT_NAME_ARRAY_START=(uint64 *) *uptr(0x600069f0); // uaddr("ksledt_") gave error... + $EVENT_NAME_SLOT_SIZE=(uint64) 56; // sizeof(struct) + + @oracle_wait_event[tid] = str(*uptr($EVENT_NAME_ARRAY_START + ($EVENT_NAME_SLOT_SIZE * arg1)/8)); +} + +uprobe:/u01/app/oracle/product/19.0.0/dbhome_1/bin/oracle:kskthewt { + delete(@oracle_wait_event[tid]); // @oracle_wait_event[tid] = -1; +} + + +// write out SAMPLES of thread states & activity +// interval is executed on 1 CPU only, so we won't emit duplicates +interval:hz:1 { + @SAMPLE_TIME=strftime("\"%Y-%m-%dT%H:%M:%S.%f\"", nsecs); // extra "" for json output + print(@SAMPLE_TIME); + + print(@pid); + print(@comm); + print(@cmdline); + print(@task_state); + print(@syscall_id); + print(@syscall_args); + print(@profile_ustack); + print(@profile_kstack); + print(@syscall_ustack); + print(@offcpu_ustack); + print(@offcpu_kstack); + print(@sched_wakeup); + print(@oracle_wait_event); +} + +END { + clear(@SAMPLE_TIME); + clear(@TASK_STATES); + clear(@pid); + clear(@uid); + clear(@gid); + clear(@comm); + clear(@cmdline); + clear(@profile_ustack); + clear(@profile_kstack); + clear(@syscall_ustack); + clear(@offcpu_ustack); + clear(@offcpu_kstack); + clear(@sched_wakeup); + clear(@syscall_id); + clear(@syscall_args); + clear(@task_state); + clear(@oracle_wait_event); +} + +// TODO: +// --------------------------------------------------------------------------------------------- +// There's *plenty* to do! If you know bcc/libbpf and are interested in helping out, ping me :-) +// +// Email: tanel@tanelpoder.com +// +// PRINTOUT NOTES: +// ---------------------------------------------------------------------------------------------- +// "Kernel: 5.3 bpftrace supports C style while loops: +// bpftrace -e 'i:ms:100 { $i = 0; while ($i <= 100) { printf("%d ", $i); $i++} exit(); }' +// Loops can be short circuited by using the continue and break keywords." +// +// Unfortunately bpftrace doesn't (yet?) support iterating through only the existing (populated) +// elements in hash maps, we don't want to loop from 1 to pid_max every time we emit output! +// +// Thus, we need to use bcc/libbpf for the sampling loops or use bpftool to dump or mount +// the kernel ebpf maps as files and do our reading / sampling from there. +// +// Since we don't want to always emit/print every single task, but would rather have some +// conditional logic & intelligence of what threads are interesting (a'la only print R & D states +// and some specific syscalls under S state), it's better to push this decision logic down to +// kernel. This means bcc or more likely libbpf as mentioned above. +// +// DATA STRUCTURE NOTES: +// ---------------------------------------------------------------------------------------------- +// With bcc/libbpf it's likely possible to use a hashmap of structs (or hashmap of maps) for +// storing each thread's complete state in a single thread state "array", under a single TID key. +// This should reduce any timing & "read consistency" issues when sampling/emitting records too. +// +/* vi:syntax=c */ +/* vi:filetype=c */ diff --git a/bin/xcapture_20231019_05.csv.bz2 b/bin/xcapture_20231019_05.csv.bz2 Binary files differnew file mode 100644 index 0000000..af9cfaf --- /dev/null +++ b/bin/xcapture_20231019_05.csv.bz2 diff --git a/bin/xtop b/bin/xtop new file mode 100755 index 0000000..42a8887 --- /dev/null +++ b/bin/xtop @@ -0,0 +1,6 @@ +#!/usr/bin/bash + +CURDIR="$(dirname "$(realpath "$0")")" + +${CURDIR}/xcapture-bpf --xtop --clear-screen $* + diff --git a/doc/licenses/Python-license.txt b/doc/licenses/Python-license.txt new file mode 100644 index 0000000..5cdb01e --- /dev/null +++ b/doc/licenses/Python-license.txt @@ -0,0 +1,279 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations (now Zope +Corporation, see http://www.zope.com). In 2001, the Python Software +Foundation (PSF, see http://www.python.org/psf/) was formed, a +non-profit organization created specifically to own Python-related +Intellectual Property. Zope Corporation is a sponsoring member of +the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.2 2.1.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2.1 2.2 2002 PSF yes + 2.2.2 2.2.1 2002 PSF yes + 2.2.3 2.2.2 2003 PSF yes + 2.3 2.2.2 2002-2003 PSF yes + 2.3.1 2.3 2002-2003 PSF yes + 2.3.2 2.3.1 2002-2003 PSF yes + 2.3.3 2.3.2 2002-2003 PSF yes + 2.3.4 2.3.3 2004 PSF yes + 2.3.5 2.3.4 2005 PSF yes + 2.4 2.3 2004 PSF yes + 2.4.1 2.4 2005 PSF yes + 2.4.2 2.4.1 2005 PSF yes + 2.4.3 2.4.2 2006 PSF yes + 2.4.4 2.4.3 2006 PSF yes + 2.5 2.4 2006 PSF yes + 2.5.1 2.5 2007 PSF yes + 2.5.2 2.5.1 2008 PSF yes + 2.5.3 2.5.2 2008 PSF yes + 2.6 2.5 2008 PSF yes + 2.6.1 2.6 2008 PSF yes + 2.6.2 2.6.1 2009 PSF yes + 2.6.3 2.6.2 2009 PSF yes + 2.6.4 2.6.3 2009 PSF yes + 2.6.5 2.6.4 2010 PSF yes + 2.7 2.6 2010 PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 +Python Software Foundation; All Rights Reserved" are retained in Python alone or +in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/include/syscall_64.h b/include/syscall_64.h new file mode 100644 index 0000000..6b1ade7 --- /dev/null +++ b/include/syscall_64.h @@ -0,0 +1,301 @@ +#define __NR_read 0 +#define __NR_write 1 +#define __NR_open 2 +#define __NR_close 3 +#define __NR_stat 4 +#define __NR_fstat 5 +#define __NR_lstat 6 +#define __NR_poll 7 +#define __NR_lseek 8 +#define __NR_mmap 9 +#define __NR_mprotect 10 +#define __NR_munmap 11 +#define __NR_brk 12 +#define __NR_rt_sigaction 13 +#define __NR_rt_sigprocmask 14 +#define __NR_rt_sigreturn 15 +#define __NR_ioctl 16 +#define __NR_pread64 17 +#define __NR_pwrite64 18 +#define __NR_readv 19 +#define __NR_writev 20 +#define __NR_access 21 +#define __NR_pipe 22 +#define __NR_select 23 +#define __NR_sched_yield 24 +#define __NR_mremap 25 +#define __NR_msync 26 +#define __NR_mincore 27 +#define __NR_madvise 28 +#define __NR_shmget 29 +#define __NR_shmat 30 +#define __NR_shmctl 31 +#define __NR_dup 32 +#define __NR_dup2 33 +#define __NR_pause 34 +#define __NR_nanosleep 35 +#define __NR_getitimer 36 +#define __NR_alarm 37 +#define __NR_setitimer 38 +#define __NR_getpid 39 +#define __NR_sendfile 40 +#define __NR_socket 41 +#define __NR_connect 42 +#define __NR_accept 43 +#define __NR_sendto 44 +#define __NR_recvfrom 45 +#define __NR_sendmsg 46 +#define __NR_recvmsg 47 +#define __NR_shutdown 48 +#define __NR_bind 49 +#define __NR_listen 50 +#define __NR_getsockname 51 +#define __NR_getpeername 52 +#define __NR_socketpair 53 +#define __NR_setsockopt 54 +#define __NR_getsockopt 55 +#define __NR_clone 56 +#define __NR_fork 57 +#define __NR_vfork 58 +#define __NR_execve 59 +#define __NR_exit 60 +#define __NR_wait4 61 +#define __NR_kill 62 +#define __NR_uname 63 +#define __NR_semget 64 +#define __NR_semop 65 +#define __NR_semctl 66 +#define __NR_shmdt 67 +#define __NR_msgget 68 +#define __NR_msgsnd 69 +#define __NR_msgrcv 70 +#define __NR_msgctl 71 +#define __NR_fcntl 72 +#define __NR_flock 73 +#define __NR_fsync 74 +#define __NR_fdatasync 75 +#define __NR_truncate 76 +#define __NR_ftruncate 77 +#define __NR_getdents 78 +#define __NR_getcwd 79 +#define __NR_chdir 80 +#define __NR_fchdir 81 +#define __NR_rename 82 +#define __NR_mkdir 83 +#define __NR_rmdir 84 +#define __NR_creat 85 +#define __NR_link 86 +#define __NR_unlink 87 +#define __NR_symlink 88 +#define __NR_readlink 89 +#define __NR_chmod 90 +#define __NR_fchmod 91 +#define __NR_chown 92 +#define __NR_fchown 93 +#define __NR_lchown 94 +#define __NR_umask 95 +#define __NR_gettimeofday 96 +#define __NR_getrlimit 97 +#define __NR_getrusage 98 +#define __NR_sysinfo 99 +#define __NR_times 100 +#define __NR_ptrace 101 +#define __NR_getuid 102 +#define __NR_syslog 103 +#define __NR_getgid 104 +#define __NR_setuid 105 +#define __NR_setgid 106 +#define __NR_geteuid 107 +#define __NR_getegid 108 +#define __NR_setpgid 109 +#define __NR_getppid 110 +#define __NR_getpgrp 111 +#define __NR_setsid 112 +#define __NR_setreuid 113 +#define __NR_setregid 114 +#define __NR_getgroups 115 +#define __NR_setgroups 116 +#define __NR_setresuid 117 +#define __NR_getresuid 118 +#define __NR_setresgid 119 +#define __NR_getresgid 120 +#define __NR_getpgid 121 +#define __NR_setfsuid 122 +#define __NR_setfsgid 123 +#define __NR_getsid 124 +#define __NR_capget 125 +#define __NR_capset 126 +#define __NR_rt_sigpending 127 +#define __NR_rt_sigtimedwait 128 +#define __NR_rt_sigqueueinfo 129 +#define __NR_rt_sigsuspend 130 +#define __NR_sigaltstack 131 +#define __NR_utime 132 +#define __NR_mknod 133 +#define __NR_uselib 134 +#define __NR_personality 135 +#define __NR_ustat 136 +#define __NR_statfs 137 +#define __NR_fstatfs 138 +#define __NR_sysfs 139 +#define __NR_getpriority 140 +#define __NR_setpriority 141 +#define __NR_sched_setparam 142 +#define __NR_sched_getparam 143 +#define __NR_sched_setscheduler 144 +#define __NR_sched_getscheduler 145 +#define __NR_sched_get_priority_max 146 +#define __NR_sched_get_priority_min 147 +#define __NR_sched_rr_get_interval 148 +#define __NR_mlock 149 +#define __NR_munlock 150 +#define __NR_mlockall 151 +#define __NR_munlockall 152 +#define __NR_vhangup 153 +#define __NR_modify_ldt 154 +#define __NR_pivot_root 155 +#define __NR__sysctl 156 +#define __NR_prctl 157 +#define __NR_arch_prctl 158 +#define __NR_adjtimex 159 +#define __NR_setrlimit 160 +#define __NR_chroot 161 +#define __NR_sync 162 +#define __NR_acct 163 +#define __NR_settimeofday 164 +#define __NR_mount 165 +#define __NR_umount2 166 +#define __NR_swapon 167 +#define __NR_swapoff 168 +#define __NR_reboot 169 +#define __NR_sethostname 170 +#define __NR_setdomainname 171 +#define __NR_iopl 172 +#define __NR_ioperm 173 +#define __NR_create_module 174 +#define __NR_init_module 175 +#define __NR_delete_module 176 +#define __NR_get_kernel_syms 177 +#define __NR_query_module 178 +#define __NR_quotactl 179 +#define __NR_nfsservctl 180 +#define __NR_getpmsg 181 /* reserved for LiS/STREAMS */ +#define __NR_putpmsg 182 /* reserved for LiS/STREAMS */ +#define __NR_afs_syscall 183 /* reserved for AFS */ +#define __NR_tuxcall 184 /* reserved for tux */ +#define __NR_security 185 +#define __NR_gettid 186 +#define __NR_readahead 187 +#define __NR_setxattr 188 +#define __NR_lsetxattr 189 +#define __NR_fsetxattr 190 +#define __NR_getxattr 191 +#define __NR_lgetxattr 192 +#define __NR_fgetxattr 193 +#define __NR_listxattr 194 +#define __NR_llistxattr 195 +#define __NR_flistxattr 196 +#define __NR_removexattr 197 +#define __NR_lremovexattr 198 +#define __NR_fremovexattr 199 +#define __NR_tkill 200 +#define __NR_time 201 +#define __NR_futex 202 +#define __NR_sched_setaffinity 203 +#define __NR_sched_getaffinity 204 +#define __NR_set_thread_area 205 +#define __NR_io_setup 206 +#define __NR_io_destroy 207 +#define __NR_io_getevents 208 +#define __NR_io_submit 209 +#define __NR_io_cancel 210 +#define __NR_get_thread_area 211 +#define __NR_lookup_dcookie 212 +#define __NR_epoll_create 213 +#define __NR_epoll_ctl_old 214 +#define __NR_epoll_wait_old 215 +#define __NR_remap_file_pages 216 +#define __NR_getdents64 217 +#define __NR_set_tid_address 218 +#define __NR_restart_syscall 219 +#define __NR_semtimedop 220 +#define __NR_fadvise64 221 +#define __NR_timer_create 222 +#define __NR_timer_settime 223 +#define __NR_timer_gettime 224 +#define __NR_timer_getoverrun 225 +#define __NR_timer_delete 226 +#define __NR_clock_settime 227 +#define __NR_clock_gettime 228 +#define __NR_clock_getres 229 +#define __NR_clock_nanosleep 230 +#define __NR_exit_group 231 +#define __NR_epoll_wait 232 +#define __NR_epoll_ctl 233 +#define __NR_tgkill 234 +#define __NR_utimes 235 +#define __NR_vserver 236 +#define __NR_mbind 237 +#define __NR_set_mempolicy 238 +#define __NR_get_mempolicy 239 +#define __NR_mq_open 240 +#define __NR_mq_unlink 241 +#define __NR_mq_timedsend 242 +#define __NR_mq_timedreceive 243 +#define __NR_mq_notify 244 +#define __NR_mq_getsetattr 245 +#define __NR_kexec_load 246 +#define __NR_waitid 247 +#define __NR_add_key 248 +#define __NR_request_key 249 +#define __NR_keyctl 250 +#define __NR_ioprio_set 251 +#define __NR_ioprio_get 252 +#define __NR_inotify_init 253 +#define __NR_inotify_add_watch 254 +#define __NR_inotify_rm_watch 255 +#define __NR_migrate_pages 256 +#define __NR_openat 257 +#define __NR_mkdirat 258 +#define __NR_mknodat 259 +#define __NR_fchownat 260 +#define __NR_futimesat 261 +#define __NR_newfstatat 262 +#define __NR_unlinkat 263 +#define __NR_renameat 264 +#define __NR_linkat 265 +#define __NR_symlinkat 266 +#define __NR_readlinkat 267 +#define __NR_fchmodat 268 +#define __NR_faccessat 269 +#define __NR_pselect6 270 +#define __NR_ppoll 271 +#define __NR_unshare 272 +#define __NR_set_robust_list 273 +#define __NR_get_robust_list 274 +#define __NR_splice 275 +#define __NR_tee 276 +#define __NR_sync_file_range 277 +#define __NR_vmsplice 278 +#define __NR_move_pages 279 +#define __NR_utimensat 280 +#define __NR_epoll_pwait 281 +#define __NR_signalfd 282 +#define __NR_timerfd_create 283 +#define __NR_eventfd 284 +#define __NR_fallocate 285 +#define __NR_timerfd_settime 286 +#define __NR_timerfd_gettime 287 +#define __NR_accept4 288 +#define __NR_signalfd4 289 +#define __NR_eventfd2 290 +#define __NR_epoll_create1 291 +#define __NR_dup3 292 +#define __NR_pipe2 293 +#define __NR_inotify_init1 294 +#define __NR_preadv 295 +#define __NR_pwritev 296 +#define __NR_rt_tgsigqueueinfo 297 +#define __NR_perf_event_open 298 +#define __NR_recvmmsg 299 +#define __NR_prlimit64 300 diff --git a/include/syscall_64_2.6.18.h b/include/syscall_64_2.6.18.h new file mode 100644 index 0000000..6b1ade7 --- /dev/null +++ b/include/syscall_64_2.6.18.h @@ -0,0 +1,301 @@ +#define __NR_read 0 +#define __NR_write 1 +#define __NR_open 2 +#define __NR_close 3 +#define __NR_stat 4 +#define __NR_fstat 5 +#define __NR_lstat 6 +#define __NR_poll 7 +#define __NR_lseek 8 +#define __NR_mmap 9 +#define __NR_mprotect 10 +#define __NR_munmap 11 +#define __NR_brk 12 +#define __NR_rt_sigaction 13 +#define __NR_rt_sigprocmask 14 +#define __NR_rt_sigreturn 15 +#define __NR_ioctl 16 +#define __NR_pread64 17 +#define __NR_pwrite64 18 +#define __NR_readv 19 +#define __NR_writev 20 +#define __NR_access 21 +#define __NR_pipe 22 +#define __NR_select 23 +#define __NR_sched_yield 24 +#define __NR_mremap 25 +#define __NR_msync 26 +#define __NR_mincore 27 +#define __NR_madvise 28 +#define __NR_shmget 29 +#define __NR_shmat 30 +#define __NR_shmctl 31 +#define __NR_dup 32 +#define __NR_dup2 33 +#define __NR_pause 34 +#define __NR_nanosleep 35 +#define __NR_getitimer 36 +#define __NR_alarm 37 +#define __NR_setitimer 38 +#define __NR_getpid 39 +#define __NR_sendfile 40 +#define __NR_socket 41 +#define __NR_connect 42 +#define __NR_accept 43 +#define __NR_sendto 44 +#define __NR_recvfrom 45 +#define __NR_sendmsg 46 +#define __NR_recvmsg 47 +#define __NR_shutdown 48 +#define __NR_bind 49 +#define __NR_listen 50 +#define __NR_getsockname 51 +#define __NR_getpeername 52 +#define __NR_socketpair 53 +#define __NR_setsockopt 54 +#define __NR_getsockopt 55 +#define __NR_clone 56 +#define __NR_fork 57 +#define __NR_vfork 58 +#define __NR_execve 59 +#define __NR_exit 60 +#define __NR_wait4 61 +#define __NR_kill 62 +#define __NR_uname 63 +#define __NR_semget 64 +#define __NR_semop 65 +#define __NR_semctl 66 +#define __NR_shmdt 67 +#define __NR_msgget 68 +#define __NR_msgsnd 69 +#define __NR_msgrcv 70 +#define __NR_msgctl 71 +#define __NR_fcntl 72 +#define __NR_flock 73 +#define __NR_fsync 74 +#define __NR_fdatasync 75 +#define __NR_truncate 76 +#define __NR_ftruncate 77 +#define __NR_getdents 78 +#define __NR_getcwd 79 +#define __NR_chdir 80 +#define __NR_fchdir 81 +#define __NR_rename 82 +#define __NR_mkdir 83 +#define __NR_rmdir 84 +#define __NR_creat 85 +#define __NR_link 86 +#define __NR_unlink 87 +#define __NR_symlink 88 +#define __NR_readlink 89 +#define __NR_chmod 90 +#define __NR_fchmod 91 +#define __NR_chown 92 +#define __NR_fchown 93 +#define __NR_lchown 94 +#define __NR_umask 95 +#define __NR_gettimeofday 96 +#define __NR_getrlimit 97 +#define __NR_getrusage 98 +#define __NR_sysinfo 99 +#define __NR_times 100 +#define __NR_ptrace 101 +#define __NR_getuid 102 +#define __NR_syslog 103 +#define __NR_getgid 104 +#define __NR_setuid 105 +#define __NR_setgid 106 +#define __NR_geteuid 107 +#define __NR_getegid 108 +#define __NR_setpgid 109 +#define __NR_getppid 110 +#define __NR_getpgrp 111 +#define __NR_setsid 112 +#define __NR_setreuid 113 +#define __NR_setregid 114 +#define __NR_getgroups 115 +#define __NR_setgroups 116 +#define __NR_setresuid 117 +#define __NR_getresuid 118 +#define __NR_setresgid 119 +#define __NR_getresgid 120 +#define __NR_getpgid 121 +#define __NR_setfsuid 122 +#define __NR_setfsgid 123 +#define __NR_getsid 124 +#define __NR_capget 125 +#define __NR_capset 126 +#define __NR_rt_sigpending 127 +#define __NR_rt_sigtimedwait 128 +#define __NR_rt_sigqueueinfo 129 +#define __NR_rt_sigsuspend 130 +#define __NR_sigaltstack 131 +#define __NR_utime 132 +#define __NR_mknod 133 +#define __NR_uselib 134 +#define __NR_personality 135 +#define __NR_ustat 136 +#define __NR_statfs 137 +#define __NR_fstatfs 138 +#define __NR_sysfs 139 +#define __NR_getpriority 140 +#define __NR_setpriority 141 +#define __NR_sched_setparam 142 +#define __NR_sched_getparam 143 +#define __NR_sched_setscheduler 144 +#define __NR_sched_getscheduler 145 +#define __NR_sched_get_priority_max 146 +#define __NR_sched_get_priority_min 147 +#define __NR_sched_rr_get_interval 148 +#define __NR_mlock 149 +#define __NR_munlock 150 +#define __NR_mlockall 151 +#define __NR_munlockall 152 +#define __NR_vhangup 153 +#define __NR_modify_ldt 154 +#define __NR_pivot_root 155 +#define __NR__sysctl 156 +#define __NR_prctl 157 +#define __NR_arch_prctl 158 +#define __NR_adjtimex 159 +#define __NR_setrlimit 160 +#define __NR_chroot 161 +#define __NR_sync 162 +#define __NR_acct 163 +#define __NR_settimeofday 164 +#define __NR_mount 165 +#define __NR_umount2 166 +#define __NR_swapon 167 +#define __NR_swapoff 168 +#define __NR_reboot 169 +#define __NR_sethostname 170 +#define __NR_setdomainname 171 +#define __NR_iopl 172 +#define __NR_ioperm 173 +#define __NR_create_module 174 +#define __NR_init_module 175 +#define __NR_delete_module 176 +#define __NR_get_kernel_syms 177 +#define __NR_query_module 178 +#define __NR_quotactl 179 +#define __NR_nfsservctl 180 +#define __NR_getpmsg 181 /* reserved for LiS/STREAMS */ +#define __NR_putpmsg 182 /* reserved for LiS/STREAMS */ +#define __NR_afs_syscall 183 /* reserved for AFS */ +#define __NR_tuxcall 184 /* reserved for tux */ +#define __NR_security 185 +#define __NR_gettid 186 +#define __NR_readahead 187 +#define __NR_setxattr 188 +#define __NR_lsetxattr 189 +#define __NR_fsetxattr 190 +#define __NR_getxattr 191 +#define __NR_lgetxattr 192 +#define __NR_fgetxattr 193 +#define __NR_listxattr 194 +#define __NR_llistxattr 195 +#define __NR_flistxattr 196 +#define __NR_removexattr 197 +#define __NR_lremovexattr 198 +#define __NR_fremovexattr 199 +#define __NR_tkill 200 +#define __NR_time 201 +#define __NR_futex 202 +#define __NR_sched_setaffinity 203 +#define __NR_sched_getaffinity 204 +#define __NR_set_thread_area 205 +#define __NR_io_setup 206 +#define __NR_io_destroy 207 +#define __NR_io_getevents 208 +#define __NR_io_submit 209 +#define __NR_io_cancel 210 +#define __NR_get_thread_area 211 +#define __NR_lookup_dcookie 212 +#define __NR_epoll_create 213 +#define __NR_epoll_ctl_old 214 +#define __NR_epoll_wait_old 215 +#define __NR_remap_file_pages 216 +#define __NR_getdents64 217 +#define __NR_set_tid_address 218 +#define __NR_restart_syscall 219 +#define __NR_semtimedop 220 +#define __NR_fadvise64 221 +#define __NR_timer_create 222 +#define __NR_timer_settime 223 +#define __NR_timer_gettime 224 +#define __NR_timer_getoverrun 225 +#define __NR_timer_delete 226 +#define __NR_clock_settime 227 +#define __NR_clock_gettime 228 +#define __NR_clock_getres 229 +#define __NR_clock_nanosleep 230 +#define __NR_exit_group 231 +#define __NR_epoll_wait 232 +#define __NR_epoll_ctl 233 +#define __NR_tgkill 234 +#define __NR_utimes 235 +#define __NR_vserver 236 +#define __NR_mbind 237 +#define __NR_set_mempolicy 238 +#define __NR_get_mempolicy 239 +#define __NR_mq_open 240 +#define __NR_mq_unlink 241 +#define __NR_mq_timedsend 242 +#define __NR_mq_timedreceive 243 +#define __NR_mq_notify 244 +#define __NR_mq_getsetattr 245 +#define __NR_kexec_load 246 +#define __NR_waitid 247 +#define __NR_add_key 248 +#define __NR_request_key 249 +#define __NR_keyctl 250 +#define __NR_ioprio_set 251 +#define __NR_ioprio_get 252 +#define __NR_inotify_init 253 +#define __NR_inotify_add_watch 254 +#define __NR_inotify_rm_watch 255 +#define __NR_migrate_pages 256 +#define __NR_openat 257 +#define __NR_mkdirat 258 +#define __NR_mknodat 259 +#define __NR_fchownat 260 +#define __NR_futimesat 261 +#define __NR_newfstatat 262 +#define __NR_unlinkat 263 +#define __NR_renameat 264 +#define __NR_linkat 265 +#define __NR_symlinkat 266 +#define __NR_readlinkat 267 +#define __NR_fchmodat 268 +#define __NR_faccessat 269 +#define __NR_pselect6 270 +#define __NR_ppoll 271 +#define __NR_unshare 272 +#define __NR_set_robust_list 273 +#define __NR_get_robust_list 274 +#define __NR_splice 275 +#define __NR_tee 276 +#define __NR_sync_file_range 277 +#define __NR_vmsplice 278 +#define __NR_move_pages 279 +#define __NR_utimensat 280 +#define __NR_epoll_pwait 281 +#define __NR_signalfd 282 +#define __NR_timerfd_create 283 +#define __NR_eventfd 284 +#define __NR_fallocate 285 +#define __NR_timerfd_settime 286 +#define __NR_timerfd_gettime 287 +#define __NR_accept4 288 +#define __NR_signalfd4 289 +#define __NR_eventfd2 290 +#define __NR_epoll_create1 291 +#define __NR_dup3 292 +#define __NR_pipe2 293 +#define __NR_inotify_init1 294 +#define __NR_preadv 295 +#define __NR_pwritev 296 +#define __NR_rt_tgsigqueueinfo 297 +#define __NR_perf_event_open 298 +#define __NR_recvmmsg 299 +#define __NR_prlimit64 300 diff --git a/include/syscall_64_2.6.32.h b/include/syscall_64_2.6.32.h new file mode 100644 index 0000000..decfe3f --- /dev/null +++ b/include/syscall_64_2.6.32.h @@ -0,0 +1,312 @@ +#define __NR_read 0 +#define __NR_write 1 +#define __NR_open 2 +#define __NR_close 3 +#define __NR_stat 4 +#define __NR_fstat 5 +#define __NR_lstat 6 +#define __NR_poll 7 +#define __NR_lseek 8 +#define __NR_mmap 9 +#define __NR_mprotect 10 +#define __NR_munmap 11 +#define __NR_brk 12 +#define __NR_rt_sigaction 13 +#define __NR_rt_sigprocmask 14 +#define __NR_rt_sigreturn 15 +#define __NR_ioctl 16 +#define __NR_pread64 17 +#define __NR_pwrite64 18 +#define __NR_readv 19 +#define __NR_writev 20 +#define __NR_access 21 +#define __NR_pipe 22 +#define __NR_select 23 +#define __NR_sched_yield 24 +#define __NR_mremap 25 +#define __NR_msync 26 +#define __NR_mincore 27 +#define __NR_madvise 28 +#define __NR_shmget 29 +#define __NR_shmat 30 +#define __NR_shmctl 31 +#define __NR_dup 32 +#define __NR_dup2 33 +#define __NR_pause 34 +#define __NR_nanosleep 35 +#define __NR_getitimer 36 +#define __NR_alarm 37 +#define __NR_setitimer 38 +#define __NR_getpid 39 +#define __NR_sendfile 40 +#define __NR_socket 41 +#define __NR_connect 42 +#define __NR_accept 43 +#define __NR_sendto 44 +#define __NR_recvfrom 45 +#define __NR_sendmsg 46 +#define __NR_recvmsg 47 +#define __NR_shutdown 48 +#define __NR_bind 49 +#define __NR_listen 50 +#define __NR_getsockname 51 +#define __NR_getpeername 52 +#define __NR_socketpair 53 +#define __NR_setsockopt 54 +#define __NR_getsockopt 55 +#define __NR_clone 56 +#define __NR_fork 57 +#define __NR_vfork 58 +#define __NR_execve 59 +#define __NR_exit 60 +#define __NR_wait4 61 +#define __NR_kill 62 +#define __NR_uname 63 +#define __NR_semget 64 +#define __NR_semop 65 +#define __NR_semctl 66 +#define __NR_shmdt 67 +#define __NR_msgget 68 +#define __NR_msgsnd 69 +#define __NR_msgrcv 70 +#define __NR_msgctl 71 +#define __NR_fcntl 72 +#define __NR_flock 73 +#define __NR_fsync 74 +#define __NR_fdatasync 75 +#define __NR_truncate 76 +#define __NR_ftruncate 77 +#define __NR_getdents 78 +#define __NR_getcwd 79 +#define __NR_chdir 80 +#define __NR_fchdir 81 +#define __NR_rename 82 +#define __NR_mkdir 83 +#define __NR_rmdir 84 +#define __NR_creat 85 +#define __NR_link 86 +#define __NR_unlink 87 +#define __NR_symlink 88 +#define __NR_readlink 89 +#define __NR_chmod 90 +#define __NR_fchmod 91 +#define __NR_chown 92 +#define __NR_fchown 93 +#define __NR_lchown 94 +#define __NR_umask 95 +#define __NR_gettimeofday 96 +#define __NR_getrlimit 97 +#define __NR_getrusage 98 +#define __NR_sysinfo 99 +#define __NR_times 100 +#define __NR_ptrace 101 +#define __NR_getuid 102 +#define __NR_syslog 103 +#define __NR_getgid 104 +#define __NR_setuid 105 +#define __NR_setgid 106 +#define __NR_geteuid 107 +#define __NR_getegid 108 +#define __NR_setpgid 109 +#define __NR_getppid 110 +#define __NR_getpgrp 111 +#define __NR_setsid 112 +#define __NR_setreuid 113 +#define __NR_setregid 114 +#define __NR_getgroups 115 +#define __NR_setgroups 116 +#define __NR_setresuid 117 +#define __NR_getresuid 118 +#define __NR_setresgid 119 +#define __NR_getresgid 120 +#define __NR_getpgid 121 +#define __NR_setfsuid 122 +#define __NR_setfsgid 123 +#define __NR_getsid 124 +#define __NR_capget 125 +#define __NR_capset 126 +#define __NR_rt_sigpending 127 +#define __NR_rt_sigtimedwait 128 +#define __NR_rt_sigqueueinfo 129 +#define __NR_rt_sigsuspend 130 +#define __NR_sigaltstack 131 +#define __NR_utime 132 +#define __NR_mknod 133 +#define __NR_uselib 134 +#define __NR_personality 135 +#define __NR_ustat 136 +#define __NR_statfs 137 +#define __NR_fstatfs 138 +#define __NR_sysfs 139 +#define __NR_getpriority 140 +#define __NR_setpriority 141 +#define __NR_sched_setparam 142 +#define __NR_sched_getparam 143 +#define __NR_sched_setscheduler 144 +#define __NR_sched_getscheduler 145 +#define __NR_sched_get_priority_max 146 +#define __NR_sched_get_priority_min 147 +#define __NR_sched_rr_get_interval 148 +#define __NR_mlock 149 +#define __NR_munlock 150 +#define __NR_mlockall 151 +#define __NR_munlockall 152 +#define __NR_vhangup 153 +#define __NR_modify_ldt 154 +#define __NR_pivot_root 155 +#define __NR__sysctl 156 +#define __NR_prctl 157 +#define __NR_arch_prctl 158 +#define __NR_adjtimex 159 +#define __NR_setrlimit 160 +#define __NR_chroot 161 +#define __NR_sync 162 +#define __NR_acct 163 +#define __NR_settimeofday 164 +#define __NR_mount 165 +#define __NR_umount2 166 +#define __NR_swapon 167 +#define __NR_swapoff 168 +#define __NR_reboot 169 +#define __NR_sethostname 170 +#define __NR_setdomainname 171 +#define __NR_iopl 172 +#define __NR_ioperm 173 +#define __NR_create_module 174 +#define __NR_init_module 175 +#define __NR_delete_module 176 +#define __NR_get_kernel_syms 177 +#define __NR_query_module 178 +#define __NR_quotactl 179 +#define __NR_nfsservctl 180 +#define __NR_getpmsg 181 +#define __NR_putpmsg 182 +#define __NR_afs_syscall 183 +#define __NR_tuxcall 184 +#define __NR_security 185 +#define __NR_gettid 186 +#define __NR_readahead 187 +#define __NR_setxattr 188 +#define __NR_lsetxattr 189 +#define __NR_fsetxattr 190 +#define __NR_getxattr 191 +#define __NR_lgetxattr 192 +#define __NR_fgetxattr 193 +#define __NR_listxattr 194 +#define __NR_llistxattr 195 +#define __NR_flistxattr 196 +#define __NR_removexattr 197 +#define __NR_lremovexattr 198 +#define __NR_fremovexattr 199 +#define __NR_tkill 200 +#define __NR_time 201 +#define __NR_futex 202 +#define __NR_sched_setaffinity 203 +#define __NR_sched_getaffinity 204 +#define __NR_set_thread_area 205 +#define __NR_io_setup 206 +#define __NR_io_destroy 207 +#define __NR_io_getevents 208 +#define __NR_io_submit 209 +#define __NR_io_cancel 210 +#define __NR_get_thread_area 211 +#define __NR_lookup_dcookie 212 +#define __NR_epoll_create 213 +#define __NR_epoll_ctl_old 214 +#define __NR_epoll_wait_old 215 +#define __NR_remap_file_pages 216 +#define __NR_getdents64 217 +#define __NR_set_tid_address 218 +#define __NR_restart_syscall 219 +#define __NR_semtimedop 220 +#define __NR_fadvise64 221 +#define __NR_timer_create 222 +#define __NR_timer_settime 223 +#define __NR_timer_gettime 224 +#define __NR_timer_getoverrun 225 +#define __NR_timer_delete 226 +#define __NR_clock_settime 227 +#define __NR_clock_gettime 228 +#define __NR_clock_getres 229 +#define __NR_clock_nanosleep 230 +#define __NR_exit_group 231 +#define __NR_epoll_wait 232 +#define __NR_epoll_ctl 233 +#define __NR_tgkill 234 +#define __NR_utimes 235 +#define __NR_vserver 236 +#define __NR_mbind 237 +#define __NR_set_mempolicy 238 +#define __NR_get_mempolicy 239 +#define __NR_mq_open 240 +#define __NR_mq_unlink 241 +#define __NR_mq_timedsend 242 +#define __NR_mq_timedreceive 243 +#define __NR_mq_notify 244 +#define __NR_mq_getsetattr 245 +#define __NR_kexec_load 246 +#define __NR_waitid 247 +#define __NR_add_key 248 +#define __NR_request_key 249 +#define __NR_keyctl 250 +#define __NR_ioprio_set 251 +#define __NR_ioprio_get 252 +#define __NR_inotify_init 253 +#define __NR_inotify_add_watch 254 +#define __NR_inotify_rm_watch 255 +#define __NR_migrate_pages 256 +#define __NR_openat 257 +#define __NR_mkdirat 258 +#define __NR_mknodat 259 +#define __NR_fchownat 260 +#define __NR_futimesat 261 +#define __NR_newfstatat 262 +#define __NR_unlinkat 263 +#define __NR_renameat 264 +#define __NR_linkat 265 +#define __NR_symlinkat 266 +#define __NR_readlinkat 267 +#define __NR_fchmodat 268 +#define __NR_faccessat 269 +#define __NR_pselect6 270 +#define __NR_ppoll 271 +#define __NR_unshare 272 +#define __NR_set_robust_list 273 +#define __NR_get_robust_list 274 +#define __NR_splice 275 +#define __NR_tee 276 +#define __NR_sync_file_range 277 +#define __NR_vmsplice 278 +#define __NR_move_pages 279 +#define __NR_utimensat 280 +#define __NR_epoll_pwait 281 +#define __NR_signalfd 282 +#define __NR_timerfd_create 283 +#define __NR_eventfd 284 +#define __NR_fallocate 285 +#define __NR_timerfd_settime 286 +#define __NR_timerfd_gettime 287 +#define __NR_accept4 288 +#define __NR_signalfd4 289 +#define __NR_eventfd2 290 +#define __NR_epoll_create1 291 +#define __NR_dup3 292 +#define __NR_pipe2 293 +#define __NR_inotify_init1 294 +#define __NR_preadv 295 +#define __NR_pwritev 296 +#define __NR_rt_tgsigqueueinfo 297 +#define __NR_perf_event_open 298 +#define __NR_recvmmsg 299 +#define __NR_fanotify_init 300 +#define __NR_fanotify_mark 301 +#define __NR_prlimit64 302 +#define __NR_name_to_handle_at 303 +#define __NR_open_by_handle_at 304 +#define __NR_clock_adjtime 305 +#define __NR_syncfs 306 +#define __NR_sendmmsg 307 +#define __NR_setns 308 +#define __NR_get_cpu 309 +#define __NR_process_vm_readv 310 +#define __NR_process_vm_writev 311 diff --git a/include/syscall_names.h b/include/syscall_names.h new file mode 100644 index 0000000..7146a1f --- /dev/null +++ b/include/syscall_names.h @@ -0,0 +1,438 @@ +/* + * 0x.Tools xCapture - sample thread activity from Linux procfs [https://0x.tools] + * Copyright 2019-2021 Tanel Poder + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + * NOTES: + * + * full architecture-specific list linux kernel source: + * arch/x86/entry/syscalls/syscall_64.tbl + * or run ausyscall --dump or parse unistd.h + * + * awk '/^[0-9]/{ printf("[%4d] = {\"%s\", \"%s\"},\n", $1, $2, $3); }' < ./arch/x86/entry/syscalls/syscall_64.tbl + */ + +typedef struct sysent { + const char *abi; + const char *name; +} struct_sysent; + +const struct_sysent sysent0[] = { +[ 0] = {"common", "read"}, +[ 1] = {"common", "write"}, +[ 2] = {"common", "open"}, +[ 3] = {"common", "close"}, +[ 4] = {"common", "stat"}, +[ 5] = {"common", "fstat"}, +[ 6] = {"common", "lstat"}, +[ 7] = {"common", "poll"}, +[ 8] = {"common", "lseek"}, +[ 9] = {"common", "mmap"}, +[ 10] = {"common", "mprotect"}, +[ 11] = {"common", "munmap"}, +[ 12] = {"common", "brk"}, +[ 13] = {"64", "rt_sigaction"}, +[ 14] = {"common", "rt_sigprocmask"}, +[ 15] = {"64", "rt_sigreturn"}, +[ 16] = {"64", "ioctl"}, +[ 17] = {"common", "pread64"}, +[ 18] = {"common", "pwrite64"}, +[ 19] = {"64", "readv"}, +[ 20] = {"64", "writev"}, +[ 21] = {"common", "access"}, +[ 22] = {"common", "pipe"}, +[ 23] = {"common", "select"}, +[ 24] = {"common", "sched_yield"}, +[ 25] = {"common", "mremap"}, +[ 26] = {"common", "msync"}, +[ 27] = {"common", "mincore"}, +[ 28] = {"common", "madvise"}, +[ 29] = {"common", "shmget"}, +[ 30] = {"common", "shmat"}, +[ 31] = {"common", "shmctl"}, +[ 32] = {"common", "dup"}, +[ 33] = {"common", "dup2"}, +[ 34] = {"common", "pause"}, +[ 35] = {"common", "nanosleep"}, +[ 36] = {"common", "getitimer"}, +[ 37] = {"common", "alarm"}, +[ 38] = {"common", "setitimer"}, +[ 39] = {"common", "getpid"}, +[ 40] = {"common", "sendfile"}, +[ 41] = {"common", "socket"}, +[ 42] = {"common", "connect"}, +[ 43] = {"common", "accept"}, +[ 44] = {"common", "sendto"}, +[ 45] = {"64", "recvfrom"}, +[ 46] = {"64", "sendmsg"}, +[ 47] = {"64", "recvmsg"}, +[ 48] = {"common", "shutdown"}, +[ 49] = {"common", "bind"}, +[ 50] = {"common", "listen"}, +[ 51] = {"common", "getsockname"}, +[ 52] = {"common", "getpeername"}, +[ 53] = {"common", "socketpair"}, +[ 54] = {"64", "setsockopt"}, +[ 55] = {"64", "getsockopt"}, +[ 56] = {"common", "clone"}, +[ 57] = {"common", "fork"}, +[ 58] = {"common", "vfork"}, +[ 59] = {"64", "execve"}, +[ 60] = {"common", "exit"}, +[ 61] = {"common", "wait4"}, +[ 62] = {"common", "kill"}, +[ 63] = {"common", "uname"}, +[ 64] = {"common", "semget"}, +[ 65] = {"common", "semop"}, +[ 66] = {"common", "semctl"}, +[ 67] = {"common", "shmdt"}, +[ 68] = {"common", "msgget"}, +[ 69] = {"common", "msgsnd"}, +[ 70] = {"common", "msgrcv"}, +[ 71] = {"common", "msgctl"}, +[ 72] = {"common", "fcntl"}, +[ 73] = {"common", "flock"}, +[ 74] = {"common", "fsync"}, +[ 75] = {"common", "fdatasync"}, +[ 76] = {"common", "truncate"}, +[ 77] = {"common", "ftruncate"}, +[ 78] = {"common", "getdents"}, +[ 79] = {"common", "getcwd"}, +[ 80] = {"common", "chdir"}, +[ 81] = {"common", "fchdir"}, +[ 82] = {"common", "rename"}, +[ 83] = {"common", "mkdir"}, +[ 84] = {"common", "rmdir"}, +[ 85] = {"common", "creat"}, +[ 86] = {"common", "link"}, +[ 87] = {"common", "unlink"}, +[ 88] = {"common", "symlink"}, +[ 89] = {"common", "readlink"}, +[ 90] = {"common", "chmod"}, +[ 91] = {"common", "fchmod"}, +[ 92] = {"common", "chown"}, +[ 93] = {"common", "fchown"}, +[ 94] = {"common", "lchown"}, +[ 95] = {"common", "umask"}, +[ 96] = {"common", "gettimeofday"}, +[ 97] = {"common", "getrlimit"}, +[ 98] = {"common", "getrusage"}, +[ 99] = {"common", "sysinfo"}, +[ 100] = {"common", "times"}, +[ 101] = {"64", "ptrace"}, +[ 102] = {"common", "getuid"}, +[ 103] = {"common", "syslog"}, +[ 104] = {"common", "getgid"}, +[ 105] = {"common", "setuid"}, +[ 106] = {"common", "setgid"}, +[ 107] = {"common", "geteuid"}, +[ 108] = {"common", "getegid"}, +[ 109] = {"common", "setpgid"}, +[ 110] = {"common", "getppid"}, +[ 111] = {"common", "getpgrp"}, +[ 112] = {"common", "setsid"}, +[ 113] = {"common", "setreuid"}, +[ 114] = {"common", "setregid"}, +[ 115] = {"common", "getgroups"}, +[ 116] = {"common", "setgroups"}, +[ 117] = {"common", "setresuid"}, +[ 118] = {"common", "getresuid"}, +[ 119] = {"common", "setresgid"}, +[ 120] = {"common", "getresgid"}, +[ 121] = {"common", "getpgid"}, +[ 122] = {"common", "setfsuid"}, +[ 123] = {"common", "setfsgid"}, +[ 124] = {"common", "getsid"}, +[ 125] = {"common", "capget"}, +[ 126] = {"common", "capset"}, +[ 127] = {"64", "rt_sigpending"}, +[ 128] = {"64", "rt_sigtimedwait"}, +[ 129] = {"64", "rt_sigqueueinfo"}, +[ 130] = {"common", "rt_sigsuspend"}, +[ 131] = {"64", "sigaltstack"}, +[ 132] = {"common", "utime"}, +[ 133] = {"common", "mknod"}, +[ 134] = {"64", "uselib"}, +[ 135] = {"common", "personality"}, +[ 136] = {"common", "ustat"}, +[ 137] = {"common", "statfs"}, +[ 138] = {"common", "fstatfs"}, +[ 139] = {"common", "sysfs"}, +[ 140] = {"common", "getpriority"}, +[ 141] = {"common", "setpriority"}, +[ 142] = {"common", "sched_setparam"}, +[ 143] = {"common", "sched_getparam"}, +[ 144] = {"common", "sched_setscheduler"}, +[ 145] = {"common", "sched_getscheduler"}, +[ 146] = {"common", "sched_get_priority_max"}, +[ 147] = {"common", "sched_get_priority_min"}, +[ 148] = {"common", "sched_rr_get_interval"}, +[ 149] = {"common", "mlock"}, +[ 150] = {"common", "munlock"}, +[ 151] = {"common", "mlockall"}, +[ 152] = {"common", "munlockall"}, +[ 153] = {"common", "vhangup"}, +[ 154] = {"common", "modify_ldt"}, +[ 155] = {"common", "pivot_root"}, +[ 156] = {"64", "_sysctl"}, +[ 157] = {"common", "prctl"}, +[ 158] = {"common", "arch_prctl"}, +[ 159] = {"common", "adjtimex"}, +[ 160] = {"common", "setrlimit"}, +[ 161] = {"common", "chroot"}, +[ 162] = {"common", "sync"}, +[ 163] = {"common", "acct"}, +[ 164] = {"common", "settimeofday"}, +[ 165] = {"common", "mount"}, +[ 166] = {"common", "umount2"}, +[ 167] = {"common", "swapon"}, +[ 168] = {"common", "swapoff"}, +[ 169] = {"common", "reboot"}, +[ 170] = {"common", "sethostname"}, +[ 171] = {"common", "setdomainname"}, +[ 172] = {"common", "iopl"}, +[ 173] = {"common", "ioperm"}, +[ 174] = {"64", "create_module"}, +[ 175] = {"common", "init_module"}, +[ 176] = {"common", "delete_module"}, +[ 177] = {"64", "get_kernel_syms"}, +[ 178] = {"64", "query_module"}, +[ 179] = {"common", "quotactl"}, +[ 180] = {"64", "nfsservctl"}, +[ 181] = {"common", "getpmsg"}, +[ 182] = {"common", "putpmsg"}, +[ 183] = {"common", "afs_syscall"}, +[ 184] = {"common", "tuxcall"}, +[ 185] = {"common", "security"}, +[ 186] = {"common", "gettid"}, +[ 187] = {"common", "readahead"}, +[ 188] = {"common", "setxattr"}, +[ 189] = {"common", "lsetxattr"}, +[ 190] = {"common", "fsetxattr"}, +[ 191] = {"common", "getxattr"}, +[ 192] = {"common", "lgetxattr"}, +[ 193] = {"common", "fgetxattr"}, +[ 194] = {"common", "listxattr"}, +[ 195] = {"common", "llistxattr"}, +[ 196] = {"common", "flistxattr"}, +[ 197] = {"common", "removexattr"}, +[ 198] = {"common", "lremovexattr"}, +[ 199] = {"common", "fremovexattr"}, +[ 200] = {"common", "tkill"}, +[ 201] = {"common", "time"}, +[ 202] = {"common", "futex"}, +[ 203] = {"common", "sched_setaffinity"}, +[ 204] = {"common", "sched_getaffinity"}, +[ 205] = {"64", "set_thread_area"}, +[ 206] = {"64", "io_setup"}, +[ 207] = {"common", "io_destroy"}, +[ 208] = {"common", "io_getevents"}, +[ 209] = {"64", "io_submit"}, +[ 210] = {"common", "io_cancel"}, +[ 211] = {"64", "get_thread_area"}, +[ 212] = {"common", "lookup_dcookie"}, +[ 213] = {"common", "epoll_create"}, +[ 214] = {"64", "epoll_ctl_old"}, +[ 215] = {"64", "epoll_wait_old"}, +[ 216] = {"common", "remap_file_pages"}, +[ 217] = {"common", "getdents64"}, +[ 218] = {"common", "set_tid_address"}, +[ 219] = {"common", "restart_syscall"}, +[ 220] = {"common", "semtimedop"}, +[ 221] = {"common", "fadvise64"}, +[ 222] = {"64", "timer_create"}, +[ 223] = {"common", "timer_settime"}, +[ 224] = {"common", "timer_gettime"}, +[ 225] = {"common", "timer_getoverrun"}, +[ 226] = {"common", "timer_delete"}, +[ 227] = {"common", "clock_settime"}, +[ 228] = {"common", "clock_gettime"}, +[ 229] = {"common", "clock_getres"}, +[ 230] = {"common", "clock_nanosleep"}, +[ 231] = {"common", "exit_group"}, +[ 232] = {"common", "epoll_wait"}, +[ 233] = {"common", "epoll_ctl"}, +[ 234] = {"common", "tgkill"}, +[ 235] = {"common", "utimes"}, +[ 236] = {"64", "vserver"}, +[ 237] = {"common", "mbind"}, +[ 238] = {"common", "set_mempolicy"}, +[ 239] = {"common", "get_mempolicy"}, +[ 240] = {"common", "mq_open"}, +[ 241] = {"common", "mq_unlink"}, +[ 242] = {"common", "mq_timedsend"}, +[ 243] = {"common", "mq_timedreceive"}, +[ 244] = {"64", "mq_notify"}, +[ 245] = {"common", "mq_getsetattr"}, +[ 246] = {"64", "kexec_load"}, +[ 247] = {"64", "waitid"}, +[ 248] = {"common", "add_key"}, +[ 249] = {"common", "request_key"}, +[ 250] = {"common", "keyctl"}, +[ 251] = {"common", "ioprio_set"}, +[ 252] = {"common", "ioprio_get"}, +[ 253] = {"common", "inotify_init"}, +[ 254] = {"common", "inotify_add_watch"}, +[ 255] = {"common", "inotify_rm_watch"}, +[ 256] = {"common", "migrate_pages"}, +[ 257] = {"common", "openat"}, +[ 258] = {"common", "mkdirat"}, +[ 259] = {"common", "mknodat"}, +[ 260] = {"common", "fchownat"}, +[ 261] = {"common", "futimesat"}, +[ 262] = {"common", "newfstatat"}, +[ 263] = {"common", "unlinkat"}, +[ 264] = {"common", "renameat"}, +[ 265] = {"common", "linkat"}, +[ 266] = {"common", "symlinkat"}, +[ 267] = {"common", "readlinkat"}, +[ 268] = {"common", "fchmodat"}, +[ 269] = {"common", "faccessat"}, +[ 270] = {"common", "pselect6"}, +[ 271] = {"common", "ppoll"}, +[ 272] = {"common", "unshare"}, +[ 273] = {"64", "set_robust_list"}, +[ 274] = {"64", "get_robust_list"}, +[ 275] = {"common", "splice"}, +[ 276] = {"common", "tee"}, +[ 277] = {"common", "sync_file_range"}, +[ 278] = {"64", "vmsplice"}, +[ 279] = {"64", "move_pages"}, +[ 280] = {"common", "utimensat"}, +[ 281] = {"common", "epoll_pwait"}, +[ 282] = {"common", "signalfd"}, +[ 283] = {"common", "timerfd_create"}, +[ 284] = {"common", "eventfd"}, +[ 285] = {"common", "fallocate"}, +[ 286] = {"common", "timerfd_settime"}, +[ 287] = {"common", "timerfd_gettime"}, +[ 288] = {"common", "accept4"}, +[ 289] = {"common", "signalfd4"}, +[ 290] = {"common", "eventfd2"}, +[ 291] = {"common", "epoll_create1"}, +[ 292] = {"common", "dup3"}, +[ 293] = {"common", "pipe2"}, +[ 294] = {"common", "inotify_init1"}, +[ 295] = {"64", "preadv"}, +[ 296] = {"64", "pwritev"}, +[ 297] = {"64", "rt_tgsigqueueinfo"}, +[ 298] = {"common", "perf_event_open"}, +[ 299] = {"64", "recvmmsg"}, +[ 300] = {"common", "fanotify_init"}, +[ 301] = {"common", "fanotify_mark"}, +[ 302] = {"common", "prlimit64"}, +[ 303] = {"common", "name_to_handle_at"}, +[ 304] = {"common", "open_by_handle_at"}, +[ 305] = {"common", "clock_adjtime"}, +[ 306] = {"common", "syncfs"}, +[ 307] = {"64", "sendmmsg"}, +[ 308] = {"common", "setns"}, +[ 309] = {"common", "getcpu"}, +[ 310] = {"64", "process_vm_readv"}, +[ 311] = {"64", "process_vm_writev"}, +[ 312] = {"common", "kcmp"}, +[ 313] = {"common", "finit_module"}, +[ 314] = {"common", "sched_setattr"}, +[ 315] = {"common", "sched_getattr"}, +[ 316] = {"common", "renameat2"}, +[ 317] = {"common", "seccomp"}, +[ 318] = {"common", "getrandom"}, +[ 319] = {"common", "memfd_create"}, +[ 320] = {"common", "kexec_file_load"}, +[ 321] = {"common", "bpf"}, +[ 322] = {"64", "execveat"}, +[ 323] = {"common", "userfaultfd"}, +[ 324] = {"common", "membarrier"}, +[ 325] = {"common", "mlock2"}, +[ 326] = {"common", "copy_file_range"}, +[ 327] = {"64", "preadv2"}, +[ 328] = {"64", "pwritev2"}, +[ 329] = {"common", "pkey_mprotect"}, +[ 330] = {"common", "pkey_alloc"}, +[ 331] = {"common", "pkey_free"}, +[ 332] = {"common", "statx"}, +[ 333] = {"common", "io_pgetevents"}, +[ 334] = {"common", "rseq"}, +/* gap */ +[ 424] = {"common", "pidfd_send_signal"}, +[ 425] = {"common", "io_uring_setup"}, +[ 426] = {"common", "io_uring_enter"}, +[ 427] = {"common", "io_uring_register"}, +[ 428] = {"common", "open_tree"}, +[ 429] = {"common", "move_mount"}, +[ 430] = {"common", "fsopen"}, +[ 431] = {"common", "fsconfig"}, +[ 432] = {"common", "fsmount"}, +[ 433] = {"common", "fspick"}, +[ 434] = {"common", "pidfd_open"}, +[ 435] = {"common", "clone3"}, +/* gap */ +[ 437] = {"common", "openat2"}, +[ 438] = {"common", "pidfd_getfd"}, +[ 439] = {"common", "faccessat2"}, +[ 440] = {"common", "process_madvise"}, +[ 441] = {"common", "epoll_pwait2"}, +[ 442] = {"common", "mount_setattr"}, +[ 443] = {"common", "quotactl_fd"}, +[ 444] = {"common", "landlock_create_ruleset"}, +[ 445] = {"common", "landlock_add_rule"}, +[ 446] = {"common", "landlock_restrict_self"}, +[ 447] = {"common", "memfd_secret"}, +[ 448] = {"common", "process_mrelease"}, +[ 449] = {"common", "futex_waitv"}, +/* gap */ +[ 512] = {"x32", "rt_sigaction"}, +[ 513] = {"x32", "rt_sigreturn"}, +[ 514] = {"x32", "ioctl"}, +[ 515] = {"x32", "readv"}, +[ 516] = {"x32", "writev"}, +[ 517] = {"x32", "recvfrom"}, +[ 518] = {"x32", "sendmsg"}, +[ 519] = {"x32", "recvmsg"}, +[ 520] = {"x32", "execve"}, +[ 521] = {"x32", "ptrace"}, +[ 522] = {"x32", "rt_sigpending"}, +[ 523] = {"x32", "rt_sigtimedwait"}, +[ 524] = {"x32", "rt_sigqueueinfo"}, +[ 525] = {"x32", "sigaltstack"}, +[ 526] = {"x32", "timer_create"}, +[ 527] = {"x32", "mq_notify"}, +[ 528] = {"x32", "kexec_load"}, +[ 529] = {"x32", "waitid"}, +[ 530] = {"x32", "set_robust_list"}, +[ 531] = {"x32", "get_robust_list"}, +[ 532] = {"x32", "vmsplice"}, +[ 533] = {"x32", "move_pages"}, +[ 534] = {"x32", "preadv"}, +[ 535] = {"x32", "pwritev"}, +[ 536] = {"x32", "rt_tgsigqueueinfo"}, +[ 537] = {"x32", "recvmmsg"}, +[ 538] = {"x32", "sendmmsg"}, +[ 539] = {"x32", "process_vm_readv"}, +[ 540] = {"x32", "process_vm_writev"}, +[ 541] = {"x32", "setsockopt"}, +[ 542] = {"x32", "getsockopt"}, +[ 543] = {"x32", "io_setup"}, +[ 544] = {"x32", "io_submit"}, +[ 545] = {"x32", "execveat"}, +[ 546] = {"x32", "preadv2"}, +[ 547] = {"x32", "pwritev2"}, +}; + +#define NR_SYSCALLS sizeof(sysent0) / sizeof(sysent0[0]) + diff --git a/lib/0xtools/argparse.py b/lib/0xtools/argparse.py new file mode 100644 index 0000000..cccc9ac --- /dev/null +++ b/lib/0xtools/argparse.py @@ -0,0 +1,2383 @@ +# Author: Steven J. Bethard <steven.bethard@gmail.com>. +# Maintainer: Thomas Waldmann <tw@waldmann-edv.de> + +# argparse is (c) 2006-2009 Steven J. Bethard <steven.bethard@gmail.com>. +# +# The argparse module was contributed to Python as of Python 2.7 and thus +# was licensed under the Python license. Same license applies to all files in +# the argparse package project. +# +# For details about the Python License, please see doc/Python-License.txt. +# +# History +# ------- +# +# Before (and including) argparse 1.1, the argparse package was licensed under +# Apache License v2.0. +# +# After argparse 1.1, all project files from the argparse project were deleted +# due to license compatibility issues between Apache License 2.0 and GNU GPL v2. +# +# The project repository then had a clean start with some files taken from +# Python 2.7.1, so definitely all files are under Python License now. + + +"""Command-line parsing library + +This module is an optparse-inspired command-line parsing library that: + + - handles both optional and positional arguments + - produces highly informative usage messages + - supports parsers that dispatch to sub-parsers + +The following is a simple usage example that sums integers from the +command-line and writes the result to a file:: + + parser = argparse.ArgumentParser( + description='sum the integers at the command line') + parser.add_argument( + 'integers', metavar='int', nargs='+', type=int, + help='an integer to be summed') + parser.add_argument( + '--log', default=sys.stdout, type=argparse.FileType('w'), + help='the file where the sum should be written') + args = parser.parse_args() + args.log.write('%s' % sum(args.integers)) + args.log.close() + +The module contains the following public classes: + + - ArgumentParser -- The main entry point for command-line parsing. As the + example above shows, the add_argument() method is used to populate + the parser with actions for optional and positional arguments. Then + the parse_args() method is invoked to convert the args at the + command-line into an object with attributes. + + - ArgumentError -- The exception raised by ArgumentParser objects when + there are errors with the parser's actions. Errors raised while + parsing the command-line are caught by ArgumentParser and emitted + as command-line messages. + + - FileType -- A factory for defining types of files to be created. As the + example above shows, instances of FileType are typically passed as + the type= argument of add_argument() calls. + + - Action -- The base class for parser actions. Typically actions are + selected by passing strings like 'store_true' or 'append_const' to + the action= argument of add_argument(). However, for greater + customization of ArgumentParser actions, subclasses of Action may + be defined and passed as the action= argument. + + - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, + ArgumentDefaultsHelpFormatter -- Formatter classes which + may be passed as the formatter_class= argument to the + ArgumentParser constructor. HelpFormatter is the default, + RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser + not to change the formatting for help text, and + ArgumentDefaultsHelpFormatter adds information about argument defaults + to the help. + +All other classes in this module are considered implementation details. +(Also note that HelpFormatter and RawDescriptionHelpFormatter are only +considered public as object names -- the API of the formatter objects is +still considered an implementation detail.) +""" + +__version__ = '1.1' +__all__ = [ + 'ArgumentParser', + 'ArgumentError', + 'ArgumentTypeError', + 'FileType', + 'HelpFormatter', + 'ArgumentDefaultsHelpFormatter', + 'RawDescriptionHelpFormatter', + 'RawTextHelpFormatter', + 'Namespace', + 'Action', + 'ONE_OR_MORE', + 'OPTIONAL', + 'PARSER', + 'REMAINDER', + 'SUPPRESS', + 'ZERO_OR_MORE', +] + + +import collections as _collections +import copy as _copy +import os as _os +import re as _re +import sys as _sys +import textwrap as _textwrap + +from gettext import gettext as _ + + +def _callable(obj): + return hasattr(obj, '__call__') or hasattr(obj, '__bases__') + + +SUPPRESS = '==SUPPRESS==' + +OPTIONAL = '?' +ZERO_OR_MORE = '*' +ONE_OR_MORE = '+' +PARSER = 'A...' +REMAINDER = '...' +_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' + +# ============================= +# Utility functions and classes +# ============================= + +class _AttributeHolder(object): + """Abstract base class that provides __repr__. + + The __repr__ method returns a string in the format:: + ClassName(attr=name, attr=name, ...) + The attributes are determined either by a class-level attribute, + '_kwarg_names', or by inspecting the instance __dict__. + """ + + def __repr__(self): + type_name = type(self).__name__ + arg_strings = [] + for arg in self._get_args(): + arg_strings.append(repr(arg)) + for name, value in self._get_kwargs(): + arg_strings.append('%s=%r' % (name, value)) + return '%s(%s)' % (type_name, ', '.join(arg_strings)) + + def _get_kwargs(self): + return sorted(self.__dict__.items()) + + def _get_args(self): + return [] + + +def _ensure_value(namespace, name, value): + if getattr(namespace, name, None) is None: + setattr(namespace, name, value) + return getattr(namespace, name) + + +# =============== +# Formatting Help +# =============== + +class HelpFormatter(object): + """Formatter for generating usage messages and argument help strings. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def __init__(self, + prog, + indent_increment=2, + max_help_position=24, + width=None): + + # default setting for width + if width is None: + try: + width = int(_os.environ['COLUMNS']) + except (KeyError, ValueError): + width = 80 + width -= 2 + + self._prog = prog + self._indent_increment = indent_increment + self._max_help_position = max_help_position + self._width = width + + self._current_indent = 0 + self._level = 0 + self._action_max_length = 0 + + self._root_section = self._Section(self, None) + self._current_section = self._root_section + + self._whitespace_matcher = _re.compile(r'\s+') + self._long_break_matcher = _re.compile(r'\n\n\n+') + + # =============================== + # Section and indentation methods + # =============================== + def _indent(self): + self._current_indent += self._indent_increment + self._level += 1 + + def _dedent(self): + self._current_indent -= self._indent_increment + assert self._current_indent >= 0, 'Indent decreased below 0.' + self._level -= 1 + + class _Section(object): + + def __init__(self, formatter, parent, heading=None): + self.formatter = formatter + self.parent = parent + self.heading = heading + self.items = [] + + def format_help(self): + # format the indented section + if self.parent is not None: + self.formatter._indent() + join = self.formatter._join_parts + for func, args in self.items: + func(*args) + item_help = join([func(*args) for func, args in self.items]) + if self.parent is not None: + self.formatter._dedent() + + # return nothing if the section was empty + if not item_help: + return '' + + # add the heading if the section was non-empty + if self.heading is not SUPPRESS and self.heading is not None: + current_indent = self.formatter._current_indent + heading = '%*s%s:\n' % (current_indent, '', self.heading) + else: + heading = '' + + # join the section-initial newline, the heading and the help + return join(['\n', heading, item_help, '\n']) + + def _add_item(self, func, args): + self._current_section.items.append((func, args)) + + # ======================== + # Message building methods + # ======================== + def start_section(self, heading): + self._indent() + section = self._Section(self, self._current_section, heading) + self._add_item(section.format_help, []) + self._current_section = section + + def end_section(self): + self._current_section = self._current_section.parent + self._dedent() + + def add_text(self, text): + if text is not SUPPRESS and text is not None: + self._add_item(self._format_text, [text]) + + def add_usage(self, usage, actions, groups, prefix=None): + if usage is not SUPPRESS: + args = usage, actions, groups, prefix + self._add_item(self._format_usage, args) + + def add_argument(self, action): + if action.help is not SUPPRESS: + + # find all invocations + get_invocation = self._format_action_invocation + invocations = [get_invocation(action)] + for subaction in self._iter_indented_subactions(action): + invocations.append(get_invocation(subaction)) + + # update the maximum item length + invocation_length = max([len(s) for s in invocations]) + action_length = invocation_length + self._current_indent + self._action_max_length = max(self._action_max_length, + action_length) + + # add the item to the list + self._add_item(self._format_action, [action]) + + def add_arguments(self, actions): + for action in actions: + self.add_argument(action) + + # ======================= + # Help-formatting methods + # ======================= + def format_help(self): + help = self._root_section.format_help() + if help: + help = self._long_break_matcher.sub('\n\n', help) + help = help.strip('\n') + '\n' + return help + + def _join_parts(self, part_strings): + return ''.join([part + for part in part_strings + if part and part is not SUPPRESS]) + + def _format_usage(self, usage, actions, groups, prefix): + if prefix is None: + prefix = _('usage: ') + + # if usage is specified, use that + if usage is not None: + usage = usage % dict(prog=self._prog) + + # if no optionals or positionals are available, usage is just prog + elif usage is None and not actions: + usage = '%(prog)s' % dict(prog=self._prog) + + # if optionals and positionals are available, calculate usage + elif usage is None: + prog = '%(prog)s' % dict(prog=self._prog) + + # split optionals from positionals + optionals = [] + positionals = [] + for action in actions: + if action.option_strings: + optionals.append(action) + else: + positionals.append(action) + + # build full usage string + format = self._format_actions_usage + action_usage = format(optionals + positionals, groups) + usage = ' '.join([s for s in [prog, action_usage] if s]) + + # wrap the usage parts if it's too long + text_width = self._width - self._current_indent + if len(prefix) + len(usage) > text_width: + + # break usage into wrappable parts + part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' + opt_usage = format(optionals, groups) + pos_usage = format(positionals, groups) + opt_parts = _re.findall(part_regexp, opt_usage) + pos_parts = _re.findall(part_regexp, pos_usage) + assert ' '.join(opt_parts) == opt_usage + assert ' '.join(pos_parts) == pos_usage + + # helper for wrapping lines + def get_lines(parts, indent, prefix=None): + lines = [] + line = [] + if prefix is not None: + line_len = len(prefix) - 1 + else: + line_len = len(indent) - 1 + for part in parts: + if line_len + 1 + len(part) > text_width: + lines.append(indent + ' '.join(line)) + line = [] + line_len = len(indent) - 1 + line.append(part) + line_len += len(part) + 1 + if line: + lines.append(indent + ' '.join(line)) + if prefix is not None: + lines[0] = lines[0][len(indent):] + return lines + + # if prog is short, follow it with optionals or positionals + if len(prefix) + len(prog) <= 0.75 * text_width: + indent = ' ' * (len(prefix) + len(prog) + 1) + if opt_parts: + lines = get_lines([prog] + opt_parts, indent, prefix) + lines.extend(get_lines(pos_parts, indent)) + elif pos_parts: + lines = get_lines([prog] + pos_parts, indent, prefix) + else: + lines = [prog] + + # if prog is long, put it on its own line + else: + indent = ' ' * len(prefix) + parts = opt_parts + pos_parts + lines = get_lines(parts, indent) + if len(lines) > 1: + lines = [] + lines.extend(get_lines(opt_parts, indent)) + lines.extend(get_lines(pos_parts, indent)) + lines = [prog] + lines + + # join lines into usage + usage = '\n'.join(lines) + + # prefix with 'usage:' + return '%s%s\n\n' % (prefix, usage) + + def _format_actions_usage(self, actions, groups): + # find group indices and identify actions in groups + group_actions = set() + inserts = {} + for group in groups: + try: + start = actions.index(group._group_actions[0]) + except ValueError: + continue + else: + end = start + len(group._group_actions) + if actions[start:end] == group._group_actions: + for action in group._group_actions: + group_actions.add(action) + if not group.required: + if start in inserts: + inserts[start] += ' [' + else: + inserts[start] = '[' + inserts[end] = ']' + else: + if start in inserts: + inserts[start] += ' (' + else: + inserts[start] = '(' + inserts[end] = ')' + for i in range(start + 1, end): + inserts[i] = '|' + + # collect all actions format strings + parts = [] + for i, action in enumerate(actions): + + # suppressed arguments are marked with None + # remove | separators for suppressed arguments + if action.help is SUPPRESS: + parts.append(None) + if inserts.get(i) == '|': + inserts.pop(i) + elif inserts.get(i + 1) == '|': + inserts.pop(i + 1) + + # produce all arg strings + elif not action.option_strings: + part = self._format_args(action, action.dest) + + # if it's in a group, strip the outer [] + if action in group_actions: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + + # add the action string to the list + parts.append(part) + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = '%s' % option_string + + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + part = '%s %s' % (option_string, args_string) + + # make it look optional if it's not required or in a group + if not action.required and action not in group_actions: + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + # insert things at the necessary indices + for i in sorted(inserts, reverse=True): + parts[i:i] = [inserts[i]] + + # join all the action items with spaces + text = ' '.join([item for item in parts if item is not None]) + + # clean up separators for mutually exclusive groups + open = r'[\[(]' + close = r'[\])]' + text = _re.sub(r'(%s) ' % open, r'\1', text) + text = _re.sub(r' (%s)' % close, r'\1', text) + text = _re.sub(r'%s *%s' % (open, close), r'', text) + text = _re.sub(r'\(([^|]*)\)', r'\1', text) + text = text.strip() + + # return the text + return text + + def _format_text(self, text): + if '%(prog)' in text: + text = text % dict(prog=self._prog) + text_width = self._width - self._current_indent + indent = ' ' * self._current_indent + return self._fill_text(text, text_width, indent) + '\n\n' + + def _format_action(self, action): + # determine the required width and the entry label + help_position = min(self._action_max_length + 2, + self._max_help_position) + help_width = self._width - help_position + action_width = help_position - self._current_indent - 2 + action_header = self._format_action_invocation(action) + + # ho nelp; start on same line and add a final newline + if not action.help: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + + # short action name; start on the same line and pad two spaces + elif len(action_header) <= action_width: + tup = self._current_indent, '', action_width, action_header + action_header = '%*s%-*s ' % tup + indent_first = 0 + + # long action name; start on the next line + else: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + indent_first = help_position + + # collect the pieces of the action help + parts = [action_header] + + # if there was help for the action, add lines of help text + if action.help: + help_text = self._expand_help(action) + help_lines = self._split_lines(help_text, help_width) + parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) + for line in help_lines[1:]: + parts.append('%*s%s\n' % (help_position, '', line)) + + # or add a newline if the description doesn't end with one + elif not action_header.endswith('\n'): + parts.append('\n') + + # if there are any sub-actions, add their help as well + for subaction in self._iter_indented_subactions(action): + parts.append(self._format_action(subaction)) + + # return a single string + return self._join_parts(parts) + + def _format_action_invocation(self, action): + if not action.option_strings: + metavar, = self._metavar_formatter(action, action.dest)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend(action.option_strings) + + # if the Optional takes a value, format is: + # -s ARGS, --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s %s' % (option_string, args_string)) + + return ', '.join(parts) + + def _metavar_formatter(self, action, default_metavar): + if action.metavar is not None: + result = action.metavar + elif action.choices is not None: + choice_strs = [str(choice) for choice in action.choices] + result = '{%s}' % ','.join(choice_strs) + else: + result = default_metavar + + def format(tuple_size): + if isinstance(result, tuple): + return result + else: + return (result, ) * tuple_size + return format + + def _format_args(self, action, default_metavar): + get_metavar = self._metavar_formatter(action, default_metavar) + if action.nargs is None: + result = '%s' % get_metavar(1) + elif action.nargs == OPTIONAL: + result = '[%s]' % get_metavar(1) + elif action.nargs == ZERO_OR_MORE: + result = '[%s [%s ...]]' % get_metavar(2) + elif action.nargs == ONE_OR_MORE: + result = '%s [%s ...]' % get_metavar(2) + elif action.nargs == REMAINDER: + result = '...' + elif action.nargs == PARSER: + result = '%s ...' % get_metavar(1) + else: + formats = ['%s' for _ in range(action.nargs)] + result = ' '.join(formats) % get_metavar(action.nargs) + return result + + def _expand_help(self, action): + params = dict(vars(action), prog=self._prog) + for name in list(params): + if params[name] is SUPPRESS: + del params[name] + for name in list(params): + if hasattr(params[name], '__name__'): + params[name] = params[name].__name__ + if params.get('choices') is not None: + choices_str = ', '.join([str(c) for c in params['choices']]) + params['choices'] = choices_str + return self._get_help_string(action) % params + + def _iter_indented_subactions(self, action): + try: + get_subactions = action._get_subactions + except AttributeError: + pass + else: + self._indent() + for subaction in get_subactions(): + yield subaction + self._dedent() + + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.wrap(text, width) + + def _fill_text(self, text, width, indent): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.fill(text, width, initial_indent=indent, + subsequent_indent=indent) + + def _get_help_string(self, action): + return action.help + + +class RawDescriptionHelpFormatter(HelpFormatter): + """Help message formatter which retains any formatting in descriptions. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _fill_text(self, text, width, indent): + return ''.join([indent + line for line in text.splitlines(True)]) + + +class RawTextHelpFormatter(RawDescriptionHelpFormatter): + """Help message formatter which retains formatting of all help text. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _split_lines(self, text, width): + return text.splitlines() + + +class ArgumentDefaultsHelpFormatter(HelpFormatter): + """Help message formatter which adds default values to argument help. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _get_help_string(self, action): + help = action.help + if '%(default)' not in action.help: + if action.default is not SUPPRESS: + defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + help += ' (default: %(default)s)' + return help + + +# ===================== +# Options and Arguments +# ===================== + +def _get_action_name(argument): + if argument is None: + return None + elif argument.option_strings: + return '/'.join(argument.option_strings) + elif argument.metavar not in (None, SUPPRESS): + return argument.metavar + elif argument.dest not in (None, SUPPRESS): + return argument.dest + else: + return None + + +class ArgumentError(Exception): + """An error from creating or using an argument (optional or positional). + + The string value of this exception is the message, augmented with + information about the argument that caused it. + """ + + def __init__(self, argument, message): + self.argument_name = _get_action_name(argument) + self.message = message + + def __str__(self): + if self.argument_name is None: + format = '%(message)s' + else: + format = 'argument %(argument_name)s: %(message)s' + return format % dict(message=self.message, + argument_name=self.argument_name) + + +class ArgumentTypeError(Exception): + """An error from trying to convert a command line string to a type.""" + pass + + +# ============== +# Action classes +# ============== + +class Action(_AttributeHolder): + """Information about how to convert command line strings to Python objects. + + Action objects are used by an ArgumentParser to represent the information + needed to parse a single argument from one or more strings from the + command line. The keyword arguments to the Action constructor are also + all attributes of Action instances. + + Keyword Arguments: + + - option_strings -- A list of command-line option strings which + should be associated with this action. + + - dest -- The name of the attribute to hold the created object(s) + + - nargs -- The number of command-line arguments that should be + consumed. By default, one argument will be consumed and a single + value will be produced. Other values include: + - N (an integer) consumes N arguments (and produces a list) + - '?' consumes zero or one arguments + - '*' consumes zero or more arguments (and produces a list) + - '+' consumes one or more arguments (and produces a list) + Note that the difference between the default and nargs=1 is that + with the default, a single value will be produced, while with + nargs=1, a list containing a single value will be produced. + + - const -- The value to be produced if the option is specified and the + option uses an action that takes no values. + + - default -- The value to be produced if the option is not specified. + + - type -- A callable that accepts a single string argument, and + returns the converted value. The standard Python types str, int, + float, and complex are useful examples of such callables. If None, + str is used. + + - choices -- A container of values that should be allowed. If not None, + after a command-line argument has been converted to the appropriate + type, an exception will be raised if it is not a member of this + collection. + + - required -- True if the action must always be specified at the + command line. This is only meaningful for optional command-line + arguments. + + - help -- The help string describing the argument. + + - metavar -- The name to be used for the option's argument with the + help string. If None, the 'dest' value will be used as the name. + """ + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + self.option_strings = option_strings + self.dest = dest + self.nargs = nargs + self.const = const + self.default = default + self.type = type + self.choices = choices + self.required = required + self.help = help + self.metavar = metavar + + def _get_kwargs(self): + names = [ + 'option_strings', + 'dest', + 'nargs', + 'const', + 'default', + 'type', + 'choices', + 'help', + 'metavar', + ] + return [(name, getattr(self, name)) for name in names] + + def __call__(self, parser, namespace, values, option_string=None): + raise NotImplementedError(_('.__call__() not defined')) + + +class _StoreAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for store actions must be > 0; if you ' + 'have nothing to store, actions such as store ' + 'true or store const may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_StoreAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + + +class _StoreConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_StoreConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) + + +class _StoreTrueAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=False, + required=False, + help=None): + super(_StoreTrueAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + required=required, + help=help) + + +class _StoreFalseAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=True, + required=False, + help=None): + super(_StoreFalseAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=False, + default=default, + required=required, + help=help) + + +class _AppendAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for append actions must be > 0; if arg ' + 'strings are not supplying the value to append, ' + 'the append const action may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_AppendAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(values) + setattr(namespace, self.dest, items) + + +class _AppendConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_AppendConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(self.const) + setattr(namespace, self.dest, items) + + +class _CountAction(Action): + + def __init__(self, + option_strings, + dest, + default=None, + required=False, + help=None): + super(_CountAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + new_count = _ensure_value(namespace, self.dest, 0) + 1 + setattr(namespace, self.dest, new_count) + + +class _HelpAction(Action): + + def __init__(self, + option_strings, + dest=SUPPRESS, + default=SUPPRESS, + help=None): + super(_HelpAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + parser.exit() + + +class _VersionAction(Action): + + def __init__(self, + option_strings, + version=None, + dest=SUPPRESS, + default=SUPPRESS, + help="show program's version number and exit"): + super(_VersionAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + self.version = version + + def __call__(self, parser, namespace, values, option_string=None): + version = self.version + if version is None: + version = parser.version + formatter = parser._get_formatter() + formatter.add_text(version) + parser.exit(message=formatter.format_help()) + + +class _SubParsersAction(Action): + + class _ChoicesPseudoAction(Action): + + def __init__(self, name, help): + sup = super(_SubParsersAction._ChoicesPseudoAction, self) + sup.__init__(option_strings=[], dest=name, help=help) + + def __init__(self, + option_strings, + prog, + parser_class, + dest=SUPPRESS, + help=None, + metavar=None): + + self._prog_prefix = prog + self._parser_class = parser_class + self._name_parser_map = _collections.OrderedDict() + self._choices_actions = [] + + super(_SubParsersAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=PARSER, + choices=self._name_parser_map, + help=help, + metavar=metavar) + + def add_parser(self, name, **kwargs): + # set prog from the existing prefix + if kwargs.get('prog') is None: + kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + + # create a pseudo-action to hold the choice help + if 'help' in kwargs: + help = kwargs.pop('help') + choice_action = self._ChoicesPseudoAction(name, help) + self._choices_actions.append(choice_action) + + # create the parser and add it to the map + parser = self._parser_class(**kwargs) + self._name_parser_map[name] = parser + return parser + + def _get_subactions(self): + return self._choices_actions + + def __call__(self, parser, namespace, values, option_string=None): + parser_name = values[0] + arg_strings = values[1:] + + # set the parser name if requested + if self.dest is not SUPPRESS: + setattr(namespace, self.dest, parser_name) + + # select the parser + try: + parser = self._name_parser_map[parser_name] + except KeyError: + tup = parser_name, ', '.join(self._name_parser_map) + msg = _('unknown parser %r (choices: %s)') % tup + raise ArgumentError(self, msg) + + # parse all the remaining options into the namespace + # store any unrecognized options on the object, so that the top + # level parser can decide what to do with them + namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + if arg_strings: + vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) + + +# ============== +# Type classes +# ============== + +class FileType(object): + """Factory for creating file object types + + Instances of FileType are typically passed as type= arguments to the + ArgumentParser add_argument() method. + + Keyword Arguments: + - mode -- A string indicating how the file is to be opened. Accepts the + same values as the builtin open() function. + - bufsize -- The file's desired buffer size. Accepts the same values as + the builtin open() function. + """ + + def __init__(self, mode='r', bufsize=-1): + self._mode = mode + self._bufsize = bufsize + + def __call__(self, string): + # the special argument "-" means sys.std{in,out} + if string == '-': + if 'r' in self._mode: + return _sys.stdin + elif 'w' in self._mode: + return _sys.stdout + else: + msg = _('argument "-" with mode %r') % self._mode + raise ValueError(msg) + + # all other arguments are used as file names + try: + return open(string, self._mode, self._bufsize) + except IOError as e: + message = _("can't open '%s': %s") + raise ArgumentTypeError(message % (string, e)) + + def __repr__(self): + args = self._mode, self._bufsize + args_str = ', '.join(repr(arg) for arg in args if arg != -1) + return '%s(%s)' % (type(self).__name__, args_str) + +# =========================== +# Optional and Positional Parsing +# =========================== + +class Namespace(_AttributeHolder): + """Simple object for storing attributes. + + Implements equality by attribute names and values, and provides a simple + string representation. + """ + + def __init__(self, **kwargs): + for name in kwargs: + setattr(self, name, kwargs[name]) + + __hash__ = None + + def __eq__(self, other): + return vars(self) == vars(other) + + def __ne__(self, other): + return not (self == other) + + def __contains__(self, key): + return key in self.__dict__ + + +class _ActionsContainer(object): + + def __init__(self, + description, + prefix_chars, + argument_default, + conflict_handler): + super(_ActionsContainer, self).__init__() + + self.description = description + self.argument_default = argument_default + self.prefix_chars = prefix_chars + self.conflict_handler = conflict_handler + + # set up registries + self._registries = {} + + # register actions + self.register('action', None, _StoreAction) + self.register('action', 'store', _StoreAction) + self.register('action', 'store_const', _StoreConstAction) + self.register('action', 'store_true', _StoreTrueAction) + self.register('action', 'store_false', _StoreFalseAction) + self.register('action', 'append', _AppendAction) + self.register('action', 'append_const', _AppendConstAction) + self.register('action', 'count', _CountAction) + self.register('action', 'help', _HelpAction) + self.register('action', 'version', _VersionAction) + self.register('action', 'parsers', _SubParsersAction) + + # raise an exception if the conflict handler is invalid + self._get_handler() + + # action storage + self._actions = [] + self._option_string_actions = {} + + # groups + self._action_groups = [] + self._mutually_exclusive_groups = [] + + # defaults storage + self._defaults = {} + + # determines whether an "option" looks like a negative number + self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + + # whether or not there are any optionals that look like negative + # numbers -- uses a list so it can be shared and edited + self._has_negative_number_optionals = [] + + # ==================== + # Registration methods + # ==================== + def register(self, registry_name, value, object): + registry = self._registries.setdefault(registry_name, {}) + registry[value] = object + + def _registry_get(self, registry_name, value, default=None): + return self._registries[registry_name].get(value, default) + + # ================================== + # Namespace default accessor methods + # ================================== + def set_defaults(self, **kwargs): + self._defaults.update(kwargs) + + # if these defaults match any existing arguments, replace + # the previous default on the object with the new one + for action in self._actions: + if action.dest in kwargs: + action.default = kwargs[action.dest] + + def get_default(self, dest): + for action in self._actions: + if action.dest == dest and action.default is not None: + return action.default + return self._defaults.get(dest, None) + + + # ======================= + # Adding argument actions + # ======================= + def add_argument(self, *args, **kwargs): + """ + add_argument(dest, ..., name=value, ...) + add_argument(option_string, option_string, ..., name=value, ...) + """ + + # if no positional args are supplied or only one is supplied and + # it doesn't look like an option string, parse a positional + # argument + chars = self.prefix_chars + if not args or len(args) == 1 and args[0][0] not in chars: + if args and 'dest' in kwargs: + raise ValueError('dest supplied twice for positional argument') + kwargs = self._get_positional_kwargs(*args, **kwargs) + + # otherwise, we're adding an optional argument + else: + kwargs = self._get_optional_kwargs(*args, **kwargs) + + # if no default was supplied, use the parser-level default + if 'default' not in kwargs: + dest = kwargs['dest'] + if dest in self._defaults: + kwargs['default'] = self._defaults[dest] + elif self.argument_default is not None: + kwargs['default'] = self.argument_default + + # create the action object, and add it to the parser + action_class = self._pop_action_class(kwargs) + if not _callable(action_class): + raise ValueError('unknown action "%s"' % (action_class,)) + action = action_class(**kwargs) + + # raise an error if the action type is not callable + type_func = self._registry_get('type', action.type, action.type) + if not _callable(type_func): + raise ValueError('%r is not callable' % (type_func,)) + + # raise an error if the metavar does not match the type + if hasattr(self, "_get_formatter"): + try: + self._get_formatter()._format_args(action, None) + except TypeError: + raise ValueError("length of metavar tuple does not match nargs") + + return self._add_action(action) + + def add_argument_group(self, *args, **kwargs): + group = _ArgumentGroup(self, *args, **kwargs) + self._action_groups.append(group) + return group + + def add_mutually_exclusive_group(self, **kwargs): + group = _MutuallyExclusiveGroup(self, **kwargs) + self._mutually_exclusive_groups.append(group) + return group + + def _add_action(self, action): + # resolve any conflicts + self._check_conflict(action) + + # add to actions list + self._actions.append(action) + action.container = self + + # index the action by any option strings it has + for option_string in action.option_strings: + self._option_string_actions[option_string] = action + + # set the flag if any option strings look like negative numbers + for option_string in action.option_strings: + if self._negative_number_matcher.match(option_string): + if not self._has_negative_number_optionals: + self._has_negative_number_optionals.append(True) + + # return the created action + return action + + def _remove_action(self, action): + self._actions.remove(action) + + def _add_container_actions(self, container): + # collect groups by titles + title_group_map = {} + for group in self._action_groups: + if group.title in title_group_map: + msg = _('cannot merge actions - two groups are named %r') + raise ValueError(msg % (group.title)) + title_group_map[group.title] = group + + # map each action to its group + group_map = {} + for group in container._action_groups: + + # if a group with the title exists, use that, otherwise + # create a new group matching the container's group + if group.title not in title_group_map: + title_group_map[group.title] = self.add_argument_group( + title=group.title, + description=group.description, + conflict_handler=group.conflict_handler) + + # map the actions to their new group + for action in group._group_actions: + group_map[action] = title_group_map[group.title] + + # add container's mutually exclusive groups + # NOTE: if add_mutually_exclusive_group ever gains title= and + # description= then this code will need to be expanded as above + for group in container._mutually_exclusive_groups: + mutex_group = self.add_mutually_exclusive_group( + required=group.required) + + # map the actions to their new mutex group + for action in group._group_actions: + group_map[action] = mutex_group + + # add all actions to this container or their group + for action in container._actions: + group_map.get(action, self)._add_action(action) + + def _get_positional_kwargs(self, dest, **kwargs): + # make sure required is not specified + if 'required' in kwargs: + msg = _("'required' is an invalid argument for positionals") + raise TypeError(msg) + + # mark positional arguments as required if at least one is + # always required + if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: + kwargs['required'] = True + if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + kwargs['required'] = True + + # return the keyword arguments with no option strings + return dict(kwargs, dest=dest, option_strings=[]) + + def _get_optional_kwargs(self, *args, **kwargs): + # determine short and long option strings + option_strings = [] + long_option_strings = [] + for option_string in args: + # error on strings that don't start with an appropriate prefix + if not option_string[0] in self.prefix_chars: + msg = _('invalid option string %r: ' + 'must start with a character %r') + tup = option_string, self.prefix_chars + raise ValueError(msg % tup) + + # strings starting with two prefix characters are long options + option_strings.append(option_string) + if option_string[0] in self.prefix_chars: + if len(option_string) > 1: + if option_string[1] in self.prefix_chars: + long_option_strings.append(option_string) + + # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' + dest = kwargs.pop('dest', None) + if dest is None: + if long_option_strings: + dest_option_string = long_option_strings[0] + else: + dest_option_string = option_strings[0] + dest = dest_option_string.lstrip(self.prefix_chars) + if not dest: + msg = _('dest= is required for options like %r') + raise ValueError(msg % option_string) + dest = dest.replace('-', '_') + + # return the updated keyword arguments + return dict(kwargs, dest=dest, option_strings=option_strings) + + def _pop_action_class(self, kwargs, default=None): + action = kwargs.pop('action', default) + return self._registry_get('action', action, action) + + def _get_handler(self): + # determine function from conflict handler string + handler_func_name = '_handle_conflict_%s' % self.conflict_handler + try: + return getattr(self, handler_func_name) + except AttributeError: + msg = _('invalid conflict_resolution value: %r') + raise ValueError(msg % self.conflict_handler) + + def _check_conflict(self, action): + + # find all options that conflict with this option + confl_optionals = [] + for option_string in action.option_strings: + if option_string in self._option_string_actions: + confl_optional = self._option_string_actions[option_string] + confl_optionals.append((option_string, confl_optional)) + + # resolve any conflicts + if confl_optionals: + conflict_handler = self._get_handler() + conflict_handler(action, confl_optionals) + + def _handle_conflict_error(self, action, conflicting_actions): + message = _('conflicting option string(s): %s') + conflict_string = ', '.join([option_string + for option_string, action + in conflicting_actions]) + raise ArgumentError(action, message % conflict_string) + + def _handle_conflict_resolve(self, action, conflicting_actions): + + # remove all conflicting options + for option_string, action in conflicting_actions: + + # remove the conflicting option + action.option_strings.remove(option_string) + self._option_string_actions.pop(option_string, None) + + # if the option now has no option string, remove it from the + # container holding it + if not action.option_strings: + action.container._remove_action(action) + + +class _ArgumentGroup(_ActionsContainer): + + def __init__(self, container, title=None, description=None, **kwargs): + # add any missing keyword arguments by checking the container + update = kwargs.setdefault + update('conflict_handler', container.conflict_handler) + update('prefix_chars', container.prefix_chars) + update('argument_default', container.argument_default) + super_init = super(_ArgumentGroup, self).__init__ + super_init(description=description, **kwargs) + + # group attributes + self.title = title + self._group_actions = [] + + # share most attributes with the container + self._registries = container._registries + self._actions = container._actions + self._option_string_actions = container._option_string_actions + self._defaults = container._defaults + self._has_negative_number_optionals = \ + container._has_negative_number_optionals + self._mutually_exclusive_groups = container._mutually_exclusive_groups + + def _add_action(self, action): + action = super(_ArgumentGroup, self)._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + super(_ArgumentGroup, self)._remove_action(action) + self._group_actions.remove(action) + + +class _MutuallyExclusiveGroup(_ArgumentGroup): + + def __init__(self, container, required=False): + super(_MutuallyExclusiveGroup, self).__init__(container) + self.required = required + self._container = container + + def _add_action(self, action): + if action.required: + msg = _('mutually exclusive arguments must be optional') + raise ValueError(msg) + action = self._container._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + self._container._remove_action(action) + self._group_actions.remove(action) + + +class ArgumentParser(_AttributeHolder, _ActionsContainer): + """Object for parsing command line strings into Python objects. + + Keyword Arguments: + - prog -- The name of the program (default: sys.argv[0]) + - usage -- A usage message (default: auto-generated from arguments) + - description -- A description of what the program does + - epilog -- Text following the argument descriptions + - parents -- Parsers whose arguments should be copied into this one + - formatter_class -- HelpFormatter class for printing help messages + - prefix_chars -- Characters that prefix optional arguments + - fromfile_prefix_chars -- Characters that prefix files containing + additional arguments + - argument_default -- The default value for all arguments + - conflict_handler -- String indicating how to handle conflicts + - add_help -- Add a -h/-help option + """ + + def __init__(self, + prog=None, + usage=None, + description=None, + epilog=None, + version=None, + parents=[], + formatter_class=HelpFormatter, + prefix_chars='-', + fromfile_prefix_chars=None, + argument_default=None, + conflict_handler='error', + add_help=True): + + if version is not None: + import warnings + warnings.warn( + """The "version" argument to ArgumentParser is deprecated. """ + """Please use """ + """"add_argument(..., action='version', version="N", ...)" """ + """instead""", DeprecationWarning) + + superinit = super(ArgumentParser, self).__init__ + superinit(description=description, + prefix_chars=prefix_chars, + argument_default=argument_default, + conflict_handler=conflict_handler) + + # default setting for prog + if prog is None: + prog = _os.path.basename(_sys.argv[0]) + + self.prog = prog + self.usage = usage + self.epilog = epilog + self.version = version + self.formatter_class = formatter_class + self.fromfile_prefix_chars = fromfile_prefix_chars + self.add_help = add_help + + add_group = self.add_argument_group + self._positionals = add_group(_('positional arguments')) + self._optionals = add_group(_('optional arguments')) + self._subparsers = None + + # register types + def identity(string): + return string + self.register('type', None, identity) + + # add help and version arguments if necessary + # (using explicit default to override global argument_default) + default_prefix = '-' if '-' in prefix_chars else prefix_chars[0] + if self.add_help: + self.add_argument( + default_prefix+'h', default_prefix*2+'help', + action='help', default=SUPPRESS, + help=_('show this help message and exit')) + if self.version: + self.add_argument( + default_prefix+'v', default_prefix*2+'version', + action='version', default=SUPPRESS, + version=self.version, + help=_("show program's version number and exit")) + + # add parent arguments and defaults + for parent in parents: + self._add_container_actions(parent) + try: + defaults = parent._defaults + except AttributeError: + pass + else: + self._defaults.update(defaults) + + # ======================= + # Pretty __repr__ methods + # ======================= + def _get_kwargs(self): + names = [ + 'prog', + 'usage', + 'description', + 'version', + 'formatter_class', + 'conflict_handler', + 'add_help', + ] + return [(name, getattr(self, name)) for name in names] + + # ================================== + # Optional/Positional adding methods + # ================================== + def add_subparsers(self, **kwargs): + if self._subparsers is not None: + self.error(_('cannot have multiple subparser arguments')) + + # add the parser class to the arguments if it's not present + kwargs.setdefault('parser_class', type(self)) + + if 'title' in kwargs or 'description' in kwargs: + title = _(kwargs.pop('title', 'subcommands')) + description = _(kwargs.pop('description', None)) + self._subparsers = self.add_argument_group(title, description) + else: + self._subparsers = self._positionals + + # prog defaults to the usage message of this parser, skipping + # optional arguments and with no "usage:" prefix + if kwargs.get('prog') is None: + formatter = self._get_formatter() + positionals = self._get_positional_actions() + groups = self._mutually_exclusive_groups + formatter.add_usage(self.usage, positionals, groups, '') + kwargs['prog'] = formatter.format_help().strip() + + # create the parsers action and add it to the positionals list + parsers_class = self._pop_action_class(kwargs, 'parsers') + action = parsers_class(option_strings=[], **kwargs) + self._subparsers._add_action(action) + + # return the created parsers action + return action + + def _add_action(self, action): + if action.option_strings: + self._optionals._add_action(action) + else: + self._positionals._add_action(action) + return action + + def _get_optional_actions(self): + return [action + for action in self._actions + if action.option_strings] + + def _get_positional_actions(self): + return [action + for action in self._actions + if not action.option_strings] + + # ===================================== + # Command line argument parsing methods + # ===================================== + def parse_args(self, args=None, namespace=None): + args, argv = self.parse_known_args(args, namespace) + if argv: + msg = _('unrecognized arguments: %s') + self.error(msg % ' '.join(argv)) + return args + + def parse_known_args(self, args=None, namespace=None): + if args is None: + # args default to the system args + args = _sys.argv[1:] + else: + # make sure that args are mutable + args = list(args) + + # default Namespace built from parser defaults + if namespace is None: + namespace = Namespace() + + # add any action defaults that aren't present + for action in self._actions: + if action.dest is not SUPPRESS: + if not hasattr(namespace, action.dest): + if action.default is not SUPPRESS: + setattr(namespace, action.dest, action.default) + + # add any parser defaults that aren't present + for dest in self._defaults: + if not hasattr(namespace, dest): + setattr(namespace, dest, self._defaults[dest]) + + # parse the arguments and exit if there are any errors + try: + namespace, args = self._parse_known_args(args, namespace) + if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) + delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) + return namespace, args + except ArgumentError: + err = _sys.exc_info()[1] + self.error(str(err)) + + def _parse_known_args(self, arg_strings, namespace): + # replace arg strings that are file references + if self.fromfile_prefix_chars is not None: + arg_strings = self._read_args_from_files(arg_strings) + + # map all mutually exclusive arguments to the other arguments + # they can't occur with + action_conflicts = {} + for mutex_group in self._mutually_exclusive_groups: + group_actions = mutex_group._group_actions + for i, mutex_action in enumerate(mutex_group._group_actions): + conflicts = action_conflicts.setdefault(mutex_action, []) + conflicts.extend(group_actions[:i]) + conflicts.extend(group_actions[i + 1:]) + + # find all option indices, and determine the arg_string_pattern + # which has an 'O' if there is an option at an index, + # an 'A' if there is an argument, or a '-' if there is a '--' + option_string_indices = {} + arg_string_pattern_parts = [] + arg_strings_iter = iter(arg_strings) + for i, arg_string in enumerate(arg_strings_iter): + + # all args after -- are non-options + if arg_string == '--': + arg_string_pattern_parts.append('-') + for arg_string in arg_strings_iter: + arg_string_pattern_parts.append('A') + + # otherwise, add the arg to the arg strings + # and note the index if it was an option + else: + option_tuple = self._parse_optional(arg_string) + if option_tuple is None: + pattern = 'A' + else: + option_string_indices[i] = option_tuple + pattern = 'O' + arg_string_pattern_parts.append(pattern) + + # join the pieces together to form the pattern + arg_strings_pattern = ''.join(arg_string_pattern_parts) + + # converts arg strings to the appropriate and then takes the action + seen_actions = set() + seen_non_default_actions = set() + + def take_action(action, argument_strings, option_string=None): + seen_actions.add(action) + argument_values = self._get_values(action, argument_strings) + + # error if this argument is not allowed with other previously + # seen arguments, assuming that actions that use the default + # value don't really count as "present" + if argument_values is not action.default: + seen_non_default_actions.add(action) + for conflict_action in action_conflicts.get(action, []): + if conflict_action in seen_non_default_actions: + msg = _('not allowed with argument %s') + action_name = _get_action_name(conflict_action) + raise ArgumentError(action, msg % action_name) + + # take the action if we didn't receive a SUPPRESS value + # (e.g. from a default) + if argument_values is not SUPPRESS: + action(self, namespace, argument_values, option_string) + + # function to convert arg_strings into an optional action + def consume_optional(start_index): + + # get the optional identified at this index + option_tuple = option_string_indices[start_index] + action, option_string, explicit_arg = option_tuple + + # identify additional optionals in the same arg string + # (e.g. -xyz is the same as -x -y -z if no args are required) + match_argument = self._match_argument + action_tuples = [] + while True: + + # if we found no optional action, skip it + if action is None: + extras.append(arg_strings[start_index]) + return start_index + 1 + + # if there is an explicit argument, try to match the + # optional's string arguments to only this + if explicit_arg is not None: + arg_count = match_argument(action, 'A') + + # if the action is a single-dash option and takes no + # arguments, try to parse more single-dash options out + # of the tail of the option string + chars = self.prefix_chars + if arg_count == 0 and option_string[1] not in chars: + action_tuples.append((action, [], option_string)) + char = option_string[0] + option_string = char + explicit_arg[0] + new_explicit_arg = explicit_arg[1:] or None + optionals_map = self._option_string_actions + if option_string in optionals_map: + action = optionals_map[option_string] + explicit_arg = new_explicit_arg + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if the action expect exactly one argument, we've + # successfully matched the option; exit the loop + elif arg_count == 1: + stop = start_index + 1 + args = [explicit_arg] + action_tuples.append((action, args, option_string)) + break + + # error if a double-dash option did not use the + # explicit argument + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if there is no explicit argument, try to match the + # optional's string arguments with the following strings + # if successful, exit the loop + else: + start = start_index + 1 + selected_patterns = arg_strings_pattern[start:] + arg_count = match_argument(action, selected_patterns) + stop = start + arg_count + args = arg_strings[start:stop] + action_tuples.append((action, args, option_string)) + break + + # add the Optional to the list and return the index at which + # the Optional's string args stopped + assert action_tuples + for action, args, option_string in action_tuples: + take_action(action, args, option_string) + return stop + + # the list of Positionals left to be parsed; this is modified + # by consume_positionals() + positionals = self._get_positional_actions() + + # function to convert arg_strings into positional actions + def consume_positionals(start_index): + # match as many Positionals as possible + match_partial = self._match_arguments_partial + selected_pattern = arg_strings_pattern[start_index:] + arg_counts = match_partial(positionals, selected_pattern) + + # slice off the appropriate arg strings for each Positional + # and add the Positional and its args to the list + for action, arg_count in zip(positionals, arg_counts): + args = arg_strings[start_index: start_index + arg_count] + start_index += arg_count + take_action(action, args) + + # slice off the Positionals that we just parsed and return the + # index at which the Positionals' string args stopped + positionals[:] = positionals[len(arg_counts):] + return start_index + + # consume Positionals and Optionals alternately, until we have + # passed the last option string + extras = [] + start_index = 0 + if option_string_indices: + max_option_string_index = max(option_string_indices) + else: + max_option_string_index = -1 + while start_index <= max_option_string_index: + + # consume any Positionals preceding the next option + next_option_string_index = min([ + index + for index in option_string_indices + if index >= start_index]) + if start_index != next_option_string_index: + positionals_end_index = consume_positionals(start_index) + + # only try to parse the next optional if we didn't consume + # the option string during the positionals parsing + if positionals_end_index > start_index: + start_index = positionals_end_index + continue + else: + start_index = positionals_end_index + + # if we consumed all the positionals we could and we're not + # at the index of an option string, there were extra arguments + if start_index not in option_string_indices: + strings = arg_strings[start_index:next_option_string_index] + extras.extend(strings) + start_index = next_option_string_index + + # consume the next optional and any arguments for it + start_index = consume_optional(start_index) + + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) + + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + + # if we didn't use all the Positional objects, there were too few + # arg strings supplied. + if positionals: + self.error(_('too few arguments')) + + # make sure all required actions were present, and convert defaults. + for action in self._actions: + if action not in seen_actions: + if action.required: + name = _get_action_name(action) + self.error(_('argument %s is required') % name) + else: + # Convert action default now instead of doing it before + # parsing arguments to avoid calling convert functions + # twice (which may fail) if the argument was given, but + # only if it was defined already in the namespace + if (action.default is not None and + isinstance(action.default, basestring) and + hasattr(namespace, action.dest) and + action.default is getattr(namespace, action.dest)): + setattr(namespace, action.dest, + self._get_value(action, action.default)) + + # make sure all required groups had one option present + for group in self._mutually_exclusive_groups: + if group.required: + for action in group._group_actions: + if action in seen_non_default_actions: + break + + # if no actions were used, report the error + else: + names = [_get_action_name(action) + for action in group._group_actions + if action.help is not SUPPRESS] + msg = _('one of the arguments %s is required') + self.error(msg % ' '.join(names)) + + # return the updated namespace and the extra arguments + return namespace, extras + + def _read_args_from_files(self, arg_strings): + # expand arguments referencing files + new_arg_strings = [] + for arg_string in arg_strings: + + # for regular arguments, just add them back into the list + if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: + new_arg_strings.append(arg_string) + + # replace arguments referencing files with the file content + else: + try: + args_file = open(arg_string[1:]) + try: + arg_strings = [] + for arg_line in args_file.read().splitlines(): + for arg in self.convert_arg_line_to_args(arg_line): + arg_strings.append(arg) + arg_strings = self._read_args_from_files(arg_strings) + new_arg_strings.extend(arg_strings) + finally: + args_file.close() + except IOError: + err = _sys.exc_info()[1] + self.error(str(err)) + + # return the modified argument list + return new_arg_strings + + def convert_arg_line_to_args(self, arg_line): + return [arg_line] + + def _match_argument(self, action, arg_strings_pattern): + # match the pattern for this action to the arg strings + nargs_pattern = self._get_nargs_pattern(action) + match = _re.match(nargs_pattern, arg_strings_pattern) + + # raise an exception if we weren't able to find a match + if match is None: + nargs_errors = { + None: _('expected one argument'), + OPTIONAL: _('expected at most one argument'), + ONE_OR_MORE: _('expected at least one argument'), + } + default = _('expected %s argument(s)') % action.nargs + msg = nargs_errors.get(action.nargs, default) + raise ArgumentError(action, msg) + + # return the number of arguments matched + return len(match.group(1)) + + def _match_arguments_partial(self, actions, arg_strings_pattern): + # progressively shorten the actions list by slicing off the + # final actions until we find a match + result = [] + for i in range(len(actions), 0, -1): + actions_slice = actions[:i] + pattern = ''.join([self._get_nargs_pattern(action) + for action in actions_slice]) + match = _re.match(pattern, arg_strings_pattern) + if match is not None: + result.extend([len(string) for string in match.groups()]) + break + + # return the list of arg string counts + return result + + def _parse_optional(self, arg_string): + # if it's an empty string, it was meant to be a positional + if not arg_string: + return None + + # if it doesn't start with a prefix, it was meant to be positional + if not arg_string[0] in self.prefix_chars: + return None + + # if the option string is present in the parser, return the action + if arg_string in self._option_string_actions: + action = self._option_string_actions[arg_string] + return action, arg_string, None + + # if it's just a single character, it was meant to be positional + if len(arg_string) == 1: + return None + + # if the option string before the "=" is present, return the action + if '=' in arg_string: + option_string, explicit_arg = arg_string.split('=', 1) + if option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, explicit_arg + + # search through all possible prefixes of the option string + # and all actions in the parser for possible interpretations + option_tuples = self._get_option_tuples(arg_string) + + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, explicit_arg in option_tuples]) + tup = arg_string, options + self.error(_('ambiguous option: %s could match %s') % tup) + + # if exactly one action matched, this segmentation is good, + # so return the parsed action + elif len(option_tuples) == 1: + option_tuple, = option_tuples + return option_tuple + + # if it was not found as an option, but it looks like a negative + # number, it was meant to be positional + # unless there are negative-number-like options + if self._negative_number_matcher.match(arg_string): + if not self._has_negative_number_optionals: + return None + + # if it contains a space, it was meant to be a positional + if ' ' in arg_string: + return None + + # it was meant to be an optional but there is no such option + # in this parser (though it might be a valid option in a subparser) + return None, arg_string, None + + def _get_option_tuples(self, option_string): + result = [] + + # option strings starting with two prefix characters are only + # split at the '=' + chars = self.prefix_chars + if option_string[0] in chars and option_string[1] in chars: + if '=' in option_string: + option_prefix, explicit_arg = option_string.split('=', 1) + else: + option_prefix = option_string + explicit_arg = None + for option_string in self._option_string_actions: + if option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # single character options can be concatenated with their arguments + # but multiple character options always have to have their argument + # separate + elif option_string[0] in chars and option_string[1] not in chars: + option_prefix = option_string + explicit_arg = None + short_option_prefix = option_string[:2] + short_explicit_arg = option_string[2:] + + for option_string in self._option_string_actions: + if option_string == short_option_prefix: + action = self._option_string_actions[option_string] + tup = action, option_string, short_explicit_arg + result.append(tup) + elif option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # shouldn't ever get here + else: + self.error(_('unexpected option string: %s') % option_string) + + # return the collected option tuples + return result + + def _get_nargs_pattern(self, action): + # in all examples below, we have to allow for '--' args + # which are represented as '-' in the pattern + nargs = action.nargs + + # the default (None) is assumed to be a single argument + if nargs is None: + nargs_pattern = '(-*A-*)' + + # allow zero or one arguments + elif nargs == OPTIONAL: + nargs_pattern = '(-*A?-*)' + + # allow zero or more arguments + elif nargs == ZERO_OR_MORE: + nargs_pattern = '(-*[A-]*)' + + # allow one or more arguments + elif nargs == ONE_OR_MORE: + nargs_pattern = '(-*A[A-]*)' + + # allow any number of options or arguments + elif nargs == REMAINDER: + nargs_pattern = '([-AO]*)' + + # allow one argument followed by any number of options or arguments + elif nargs == PARSER: + nargs_pattern = '(-*A[-AO]*)' + + # all others should be integers + else: + nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) + + # if this is an optional action, -- is not allowed + if action.option_strings: + nargs_pattern = nargs_pattern.replace('-*', '') + nargs_pattern = nargs_pattern.replace('-', '') + + # return the pattern + return nargs_pattern + + # ======================== + # Value conversion methods + # ======================== + def _get_values(self, action, arg_strings): + # for everything but PARSER, REMAINDER args, strip out first '--' + if action.nargs not in [PARSER, REMAINDER]: + try: + arg_strings.remove('--') + except ValueError: + pass + + # optional argument produces a default when not present + if not arg_strings and action.nargs == OPTIONAL: + if action.option_strings: + value = action.const + else: + value = action.default + if isinstance(value, basestring): + value = self._get_value(action, value) + self._check_value(action, value) + + # when nargs='*' on a positional, if there were no command-line + # args, use the default if it is anything other than None + elif (not arg_strings and action.nargs == ZERO_OR_MORE and + not action.option_strings): + if action.default is not None: + value = action.default + else: + value = arg_strings + self._check_value(action, value) + + # single argument or optional argument produces a single value + elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: + arg_string, = arg_strings + value = self._get_value(action, arg_string) + self._check_value(action, value) + + # REMAINDER arguments convert all values, checking none + elif action.nargs == REMAINDER: + value = [self._get_value(action, v) for v in arg_strings] + + # PARSER arguments convert all values, but check only the first + elif action.nargs == PARSER: + value = [self._get_value(action, v) for v in arg_strings] + self._check_value(action, value[0]) + + # all other types of nargs produce a list + else: + value = [self._get_value(action, v) for v in arg_strings] + for v in value: + self._check_value(action, v) + + # return the converted value + return value + + def _get_value(self, action, arg_string): + type_func = self._registry_get('type', action.type, action.type) + if not _callable(type_func): + msg = _('%r is not callable') + raise ArgumentError(action, msg % type_func) + + # convert the value to the appropriate type + try: + result = type_func(arg_string) + + # ArgumentTypeErrors indicate errors + except ArgumentTypeError: + name = getattr(action.type, '__name__', repr(action.type)) + msg = str(_sys.exc_info()[1]) + raise ArgumentError(action, msg) + + # TypeErrors or ValueErrors also indicate errors + except (TypeError, ValueError): + name = getattr(action.type, '__name__', repr(action.type)) + msg = _('invalid %s value: %r') + raise ArgumentError(action, msg % (name, arg_string)) + + # return the converted value + return result + + def _check_value(self, action, value): + # converted value must be one of the choices (if specified) + if action.choices is not None and value not in action.choices: + tup = value, ', '.join(map(repr, action.choices)) + msg = _('invalid choice: %r (choose from %s)') % tup + raise ArgumentError(action, msg) + + # ======================= + # Help-formatting methods + # ======================= + def format_usage(self): + formatter = self._get_formatter() + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + return formatter.format_help() + + def format_help(self): + formatter = self._get_formatter() + + # usage + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + + # description + formatter.add_text(self.description) + + # positionals, optionals and user-defined groups + for action_group in self._action_groups: + formatter.start_section(action_group.title) + formatter.add_text(action_group.description) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + + # epilog + formatter.add_text(self.epilog) + + # determine help from format above + return formatter.format_help() + + def format_version(self): + import warnings + warnings.warn( + 'The format_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + formatter = self._get_formatter() + formatter.add_text(self.version) + return formatter.format_help() + + def _get_formatter(self): + return self.formatter_class(prog=self.prog) + + # ===================== + # Help-printing methods + # ===================== + def print_usage(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_usage(), file) + + def print_help(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_help(), file) + + def print_version(self, file=None): + import warnings + warnings.warn( + 'The print_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + self._print_message(self.format_version(), file) + + def _print_message(self, message, file=None): + if message: + if file is None: + file = _sys.stderr + file.write(message) + + # =============== + # Exiting methods + # =============== + def exit(self, status=0, message=None): + if message: + self._print_message(message, _sys.stderr) + _sys.exit(status) + + def error(self, message): + """error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + + If you override this in a subclass, it should not return -- it + should either exit or raise an exception. + """ + self.print_usage(_sys.stderr) + self.exit(2, _('%s: error: %s\n') % (self.prog, message)) diff --git a/lib/0xtools/psnproc.py b/lib/0xtools/psnproc.py new file mode 100644 index 0000000..fe3c624 --- /dev/null +++ b/lib/0xtools/psnproc.py @@ -0,0 +1,603 @@ +# psn -- Linux Process Snapper by Tanel Poder [https://0x.tools] +# Copyright 2019-2021 Tanel Poder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +# structures defining /proc +import os, os.path +import re +import platform + +system_timer_hz = os.sysconf('SC_CLK_TCK') + +class ProcSource: + def __init__(self, name, path, available_columns, stored_column_names, task_level=False, read_samples=lambda f: [f.read()], parse_sample=lambda self, sample: sample.split()): + self.name = name + self.path = path + self.available_columns = available_columns + self.task_level = task_level + self.read_samples = read_samples + self.parse_sample = parse_sample + + self.set_stored_columns(stored_column_names) + + + + def set_stored_columns(self, stored_column_names): + col_name_i, schema_type_i, source_i, transform_i = range(4) + self.stored_column_names = stored_column_names or [c[0] for c in self.available_columns] + + # find schema columns + sample_cols = [('event_time', str), ('pid', int), ('task', int)] + source_cols = [c for c in self.available_columns if c[col_name_i] in self.stored_column_names and c[col_name_i] not in dict(sample_cols) and c[1] is not None] + self.schema_columns = sample_cols + source_cols + + column_indexes = dict([(c[col_name_i], c[source_i]) for c in self.available_columns]) + + schema_extract_idx = [column_indexes[c[col_name_i]] for c in source_cols] + schema_extract_convert = [c[schema_type_i] if len(c) == 3 else c[transform_i] for c in source_cols] + self.schema_extract = list(zip(schema_extract_idx, schema_extract_convert)) + + self.insert_sql = "INSERT INTO '%s' VALUES (%s)" % (self.name, ','.join(['?' for i in self.schema_columns])) + + + # knowing the bit length, we can decide if it's a large positive number or just a (small or large) negative one + def hex_to_signed_int(self, hex_str, bit_length): + unsigned_int = int(hex_str, 16) + + if unsigned_int >= 2**(bit_length - 1): + return unsigned_int - 2**bit_length + return unsigned_int + + + def sample(self, event_time, pid, task): + sample_path = self.path % (pid, task) if self.task_level else self.path % pid + + with open(sample_path) as f: + full_sample = None + raw_samples = self.read_samples(f) + + def create_row_sample(raw_sample): + full_sample = self.parse_sample(self, raw_sample) + + # some syscall-specific code pushed down to general sampling function + # call readlink() to get the file name for system calls that have a file descriptor as arg0 + filename = '' + if self.name == 'syscall': + # special case: kernel threads show all-zero "syscall" on newer kernels like 4.x + # otherwise it incorrectly looks like that kernel is in a "read" syscall (id=0 on x86_64) + if full_sample[0] == '-1' or full_sample == ['0', '0x0', '0x0', '0x0', '0x0', '0x0', '0x0', '0x0', '0x0']: + full_sample = ['kernel_thread', '0x0', '0x0', '0x0', '0x0', '0x0', '0x0', '0x0', '0x0'] + + try: + syscall_id = full_sample[0] # get string version of syscall number or "running" or "-1" + except (ValueError, IndexError) as e: + print('problem extracting syscall id', self.name, 'sample:') + printr(full_sample) + print + raise + + if syscall_id in syscalls_with_fd_arg: + try: + arg0 = int(full_sample[1], 16) + # a hacky way for avoiding reading false file descriptors for kernel threads on older kernels + # (like 2.6.32) that show "syscall 0x0" for kernel threads + some random false arguments. + # TODO refactor this and kernel_thread translation above + if arg0 <= 65536: + filename = os.readlink("/proc/%s/fd/%s" % (pid, arg0)) + " " + special_fds.get(arg0, '') + else: + filename = 'fd over 65536' + + except (OSError) as e: + # file has been closed or process has disappeared + #print 'problem with translating fd to name /proc/%s/fd/%s' % (pid, arg0), 'sample:' + #print full_sample + #print + filename = '-' + + elif syscall_id in syscalls_with_sint_arg: + waitpid = self.hex_to_signed_int(full_sample[1], 32) + if waitpid == -1: + filename = 'pid:[child]' + else: + filename = 'pid:[' + str(waitpid) + ']' + + full_sample += (filename,) + + r = [event_time, pid, task] + [convert(full_sample[idx]) for idx, convert in self.schema_extract] + return r + + try: + return [create_row_sample(rs) for rs in raw_samples] + except (ValueError, IndexError) as e: + print('problem parsing', self.name, 'sample:') + print(raw_samples) + print + raise + + +### stat ### +# process_state_name = { +# 'R': 'Running (ON CPU)', +# 'S': 'Sleeping (Interruptible)', +# 'D': 'Disk (Uninterruptible)', +# 'Z': 'Zombie', +# 'T': 'Traced/Stopped', +# 'W': 'Paging' +# } + +# https://github.com/torvalds/linux/blob/master/fs/proc/array.c +# State W (paging) is not used in kernels 2.6.x onwards +process_state_name = { + 'R': 'Running (ON CPU)', #/* 0x00 */ + 'S': 'Sleep (Interruptible)', #/* 0x01 */ + 'D': 'Disk (Uninterruptible)', #/* 0x02 */ + 'T': '(stopped)', #/* 0x04 */ + 't': '(tracing stop)', #/* 0x08 */ + 'X': '(dead)', #/* 0x10 */ + 'Z': '(zombie)', #/* 0x20 */ + 'P': '(parked)', #/* 0x40 */ + #/* states beyond TASK_REPORT: */ + 'I': '(idle)', #/* 0x80 */ +} + +def parse_stat_sample(proc_source, sample): + tokens = raw_tokens = sample.split() + + # stitch together comm field of the form (word word) + if raw_tokens[1][0] == '(' and raw_tokens[1][-1] != ')': + tokens = raw_tokens[:2] + raw_tokens = raw_tokens[2:] + while tokens[-1][-1] != ')': + tokens[-1] += ' ' + raw_tokens.pop(0) + tokens.extend(raw_tokens) + + return tokens + + +trim_comm = re.compile(r'\d+') + + +stat = ProcSource('stat', '/proc/%s/task/%s/stat', [ + ('pid', int, 0), + ('comm', str, 1, lambda c: re.sub(trim_comm, '*', c)), + ('comm2', str, 1), + ('state_id', str, 2), + ('state', str, 2, lambda state_id: process_state_name.get(state_id, state_id)), + ('ppid', int, 3), + ('pgrp', int, 4), + ('session', int, 5), + ('tty_nr', int, 6), + ('tpgid', int, 7), + ('flags', None, 8), + ('minflt', int, 9), + ('cminflt', int, 10), + ('majflt', int, 11), + ('cmajflt', int, 12), + ('utime', int, 13), + ('stime', int, 14), + ('cutime', int, 15), + ('cstime', int, 16), + ('utime_sec', int, 13, lambda v: int(v) / system_timer_hz), + ('stime_sec', int, 14, lambda v: int(v) / system_timer_hz), + ('cutime_sec', int, 15, lambda v: int(v) / system_timer_hz), + ('cstime_sec', int, 16, lambda v: int(v) / system_timer_hz), + ('priority', int, 17), + ('nice', int, 18), + ('num_threads', int, 19), + ('itrealvalue', None, 20), + ('starttime', int, 21), + ('vsize', int, 22), + ('rss', int, 23), + ('rsslim', str, 24), + ('startcode', None, 25), + ('endcode', None, 26), + ('startstack', None, 27), + ('kstkesp', None, 28), + ('kstkeip', None, 29), + ('signal', None, 30), + ('blocked', None, 31), + ('sigignore', None, 32), + ('sigcatch', None, 33), + ('wchan', None, 34), + ('nswap', None, 35), + ('cnswap', None, 36), + ('exit_signal', int, 37), + ('processor', int, 38), + ('rt_priority', int, 39), + ('policy', None, 40), + ('delayacct_blkio_ticks', int, 41), + ('guest_time', int, 42), + ('cgust_time', int, 43), + ('start_data', None, 44), + ('end_data', None, 45), + ('start_brk', None, 46), + ('arg_start', None, 47), + ('arg_end', None, 48), + ('env_start', None, 49), + ('env_end', None, 50), + ('exit_code', int, 51), +], None, +task_level=True, +parse_sample=parse_stat_sample) + + + +### status ### +def parse_status_sample(proc_source, sample): + lines = sample.split('\n') + + sample_values = [] + + for line in [l for l in lines if l]: + line_tokens = line.split() + n, v = line_tokens[0][:-1].lower(), ' '.join(line_tokens[1:]) + n_kb = n + '_kb' + + # missing values take default parse function value: assume no order change, and that available_columns contains all possible field names + while len(sample_values) < len(proc_source.available_columns) and proc_source.available_columns[len(sample_values)][0] not in (n, n_kb): + parse_fn = proc_source.available_columns[len(sample_values)][1] + sample_values.append(parse_fn()) + + if len(sample_values) < len(proc_source.available_columns): + sample_values.append(v) + + return sample_values + + +status = ProcSource('status', '/proc/%s/status', [ + ('name', str, 0), + ('umask', str, 1), + ('state', str, 2), # remove duplicate with stat + ('tgid', int, 3), + ('ngid', int, 4), + ('pid', int, 5), + ('ppid', int, 6), # remove duplicate with stat + ('tracerpid', int, 7), + ('uid', int, 8, lambda v: int(v.split()[0])), + ('gid', int, 9, lambda v: int(v.split()[0])), + ('fdsize', int, 10), + ('groups', str, 11), + ('nstgid', str, 12), + ('nspid', str, 13), + ('nspgid', str, 14), + ('nssid', str, 15), + ('vmpeak_kb', int, 16, lambda v: int(v.split()[0])), + ('vmsize_kb', int, 17, lambda v: int(v.split()[0])), + ('vmlck_kb', int, 18, lambda v: int(v.split()[0])), + ('vmpin_kb', int, 19, lambda v: int(v.split()[0])), + ('vmhwm_kb', int, 20, lambda v: int(v.split()[0])), + ('vmrss_kb', int, 21, lambda v: int(v.split()[0])), + ('rssanon_kb', int, 22, lambda v: int(v.split()[0])), + ('rssfile_kb', int, 23, lambda v: int(v.split()[0])), + ('rssshmem_kb', int, 24, lambda v: int(v.split()[0])), + ('vmdata_kb', int, 25, lambda v: int(v.split()[0])), + ('vmstk_kb', int, 26, lambda v: int(v.split()[0])), + ('vmexe_kb', int, 27, lambda v: int(v.split()[0])), + ('vmlib_kb', int, 28, lambda v: int(v.split()[0])), + ('vmpte_kb', int, 29, lambda v: int(v.split()[0])), + ('vmpmd_kb', int, 30, lambda v: int(v.split()[0])), + ('vmswap_kb', int, 31, lambda v: int(v.split()[0])), + ('hugetlbpages_kb', int, 32, lambda v: int(v.split()[0])), + ('threads', int, 33), + ('sigq', str, 34), + ('sigpnd', str, 35), + ('shdpnd', str, 36), + ('sigblk', str, 37), + ('sigign', str, 38), + ('sigcgt', str, 39), + ('capinh', str, 40), + ('capprm', str, 41), + ('capeff', str, 42), + ('capbnd', str, 43), + ('capamb', str, 44), + ('seccomp', int, 45), + ('cpus_allowed', str, 46), + ('cpus_allowed_list', str, 47), + ('mems_allowed', str, 48), + ('mems_allowed_list', str, 49), + ('voluntary_ctxt_switches', int, 50), + ('nonvoluntary_ctxt_switches', int, 51) +], None, task_level=False, parse_sample=parse_status_sample) + + +### syscall ### +def extract_system_call_ids(unistd_64_fh): + syscall_id_to_name = {'running': '[running]', '-1': '[kernel_direct]', 'kernel_thread':'[kernel_thread]'} + + # examples from a unistd.h file + # #define __NR_mount 40 + # #define __NR3264_truncate 45 + + for name_prefix in ['__NR_', '__NR3264_']: + for line in unistd_64_fh.readlines(): + tokens = line.split() + if tokens and len(tokens) == 3 and tokens[0] == '#define': + _, s_name, s_id = tokens + if s_name.startswith(name_prefix): + s_name = s_name[len(name_prefix):] + syscall_id_to_name[s_id] = s_name + + return syscall_id_to_name + +# currently assuming all platforms are x86_64 +def get_system_call_names(): + psn_dir=os.path.dirname(os.path.realpath(__file__)) + kernel_ver=platform.release().split('-')[0] + + # this probably needds to be improved for better platform support + if platform.machine() == 'aarch64': + unistd_64_paths = ['/usr/include/asm-generic/unistd.h'] + else: + unistd_64_paths = ['/usr/include/asm/unistd_64.h', '/usr/include/x86_64-linux-gnu/asm/unistd_64.h', '/usr/include/asm-x86_64/unistd.h', '/usr/include/asm/unistd.h', psn_dir+'/syscall_64_'+kernel_ver+'.h', psn_dir+'/syscall_64.h'] + + for path in unistd_64_paths: + try: + with open(path) as f: + return extract_system_call_ids(f) + except IOError as e: + pass + + raise Exception('unistd_64.h not found in' + ' or '.join(unistd_64_paths) + '.\n You may need to "yum install kernel-headers" or "apt-get install libc6-dev"\n until this dependency is removed in a newer pSnapper version') + + +syscall_id_to_name = get_system_call_names() + +# define syscalls for which we can look up filename from fd argument +# before the change for Python 3 +#syscall_name_to_id = dict((y,x) for x,y in syscall_id_to_name.iteritems()) +syscall_name_to_id = dict((y,x) for x,y in syscall_id_to_name.items()) + +syscalls_with_fd_arg = set([ + syscall_name_to_id.get('read' , 'N/A') + , syscall_name_to_id.get('write' , 'N/A') + , syscall_name_to_id.get('pread64' , 'N/A') + , syscall_name_to_id.get('pwrite64' , 'N/A') + , syscall_name_to_id.get('fsync' , 'N/A') + , syscall_name_to_id.get('fdatasync' , 'N/A') + , syscall_name_to_id.get('recvfrom' , 'N/A') + , syscall_name_to_id.get('sendto' , 'N/A') + , syscall_name_to_id.get('recvmsg' , 'N/A') + , syscall_name_to_id.get('sendmsg' , 'N/A') + , syscall_name_to_id.get('epoll_wait' , 'N/A') + , syscall_name_to_id.get('ioctl' , 'N/A') + , syscall_name_to_id.get('accept' , 'N/A') + , syscall_name_to_id.get('accept4' , 'N/A') + , syscall_name_to_id.get('getdents' , 'N/A') + , syscall_name_to_id.get('getdents64' , 'N/A') + , syscall_name_to_id.get('unlinkat' , 'N/A') + , syscall_name_to_id.get('fstat' , 'N/A') + , syscall_name_to_id.get('fstatfs' , 'N/A') + , syscall_name_to_id.get('newfstatat' , 'N/A') +# , syscall_name_to_id.get('openat' , 'N/A') +# , syscall_name_to_id.get('openat2' , 'N/A') + , syscall_name_to_id.get('readv' , 'N/A') + , syscall_name_to_id.get('writev' , 'N/A') + , syscall_name_to_id.get('preadv' , 'N/A') + , syscall_name_to_id.get('pwritev' , 'N/A') + , syscall_name_to_id.get('preadv2' , 'N/A') + , syscall_name_to_id.get('pwritev2' , 'N/A') + , syscall_name_to_id.get('splice' , 'N/A') +]) + +special_fds = { 0:'(stdin) ', 1:'(stdout)', 2:'(stderr)' } + +syscalls_with_sint_arg = set([ + syscall_name_to_id.get('wait4' , 'N/A') # arg0 is pid_t + , syscall_name_to_id.get('waitpid' , 'N/A') +]) + +def parse_syscall_sample(proc_source, sample): + tokens = sample.split() + if tokens[0] == 'running': + return (tokens[0], '', '', '', '', '', '', None, None) + else: + return tokens + + +trim_socket = re.compile(r'\d+') + +syscall = ProcSource('syscall', '/proc/%s/task/%s/syscall', [ + ('syscall_id', int, 0, lambda sn: -2 if sn == 'running' else int(sn)), + ('syscall', str, 0, lambda sn: syscall_id_to_name[sn]), # convert syscall_id via unistd_64.h into call name + ('arg0', str, 1), + ('arg1', str, 2), + ('arg2', str, 3), + ('arg3', str, 4), + ('arg4', str, 5), + ('arg5', str, 6), + ('esp', None, 7), # stack pointer + ('eip', None, 8), # program counter/instruction pointer + ('filename', str, 9, lambda fn: re.sub(trim_socket, '*', fn) if fn.split(':')[0] in ['socket','pipe'] else fn), + ('filename2', str, 9), + ('filenamesum',str, 9, lambda fn: re.sub(trim_socket, '*', fn)), + ('basename', str, 9, lambda fn: re.sub(trim_socket, '*', fn) if fn.split(':')[0] in ['socket','pipe'] else os.path.basename(fn)), # filename if syscall has fd as arg0 + ('dirname', str, 9, lambda fn: re.sub(trim_socket, '*', fn) if fn.split(':')[0] in ['socket','pipe'] else os.path.dirname(fn)), # filename if syscall has fd as arg0 +], None, +task_level=True, parse_sample=parse_syscall_sample) + + +### get file name from file descriptor ### +#filename = ProcSource('fd', '/proc/%s/task/%s/fd', [('wchan', str, 0)], ['wchan'], task_level=True) + +### process cmdline args ### +def parse_cmdline_sample(proc_source,sample): + # the cmdline entry may have spaces in it and happens to have a \000 in the end + # the split [] hack is due to postgres having some extra spaces in its cmdlines + return [sample.split('\000')[0].strip()] + +cmdline = ProcSource('cmdline', '/proc/%s/task/%s/cmdline', [('cmdline', str, 0)], ['cmdline'], task_level=True, parse_sample=parse_cmdline_sample) + +### wchan ### +wchan = ProcSource('wchan', '/proc/%s/task/%s/wchan', [('wchan', str, 0)], ['wchan'], task_level=True) + + +### io ### +def parse_io_sample(proc_source, sample): + return [line.split()[1] if line else '' for line in sample.split('\n')] + +io = ProcSource('io', '/proc/%s/task/%s/io', [ + ('rchar', int, 0), + ('wchar', int, 1), + ('syscr', int, 2), + ('syscw', int, 3), + ('read_bytes', int, 4), + ('write_bytes', int, 5), + ('cancelled_write_bytes', int, 6), +], None, +task_level=True, +parse_sample=parse_io_sample) + + + +### net/dev ### (not accounted at process level) +def read_net_samples(fh): + return fh.readlines()[2:] + + +def parse_net_sample(proc_source, sample): + fields = sample.split() + fields[0] = fields[0][:-1] + return fields + + +net = ProcSource('net', '/proc/%s/task/%s/net/dev', [ + ('iface', str, 0), + ('rx_bytes', str, 1), + ('rx_packets', str, 2), + ('rx_errs', str, 3), + ('rx_drop', str, 4), + ('rx_fifo', str, 5), + ('rx_frame', str, 6), + ('rx_compressed', str, 7), + ('rx_multicast', str, 8), + ('tx_bytes', str, 9), + ('tx_packets', str, 10), + ('tx_errs', str, 11), + ('tx_drop', str, 12), + ('tx_fifo', str, 13), + ('tx_colls', str, 14), + ('tx_carrier', str, 15), + ('tx_compressed', str, 16), +], None, +read_samples=read_net_samples, +parse_sample=parse_net_sample) + + + +### stack ### +def read_stack_samples(fh): + result = '' + + # reverse stack and ignore the (reversed) top frame 0xfffffffffffff + # | | + # v v + for x in fh.readlines()[::-1][1:]: + func = x.split(' ')[1].split('+')[0] + if func not in ['entry_SYSCALL_64_after_hwframe','do_syscall_64','el0t_64_sync_handler', + 'el0_svc', 'do_el0_svc', 'el0_svc_common.constprop.0', 'invoke_syscall.constprop.0' ]: + if result: # skip writing the 1st "->" + result += '->' + result += func + '()' + + return [result or '-'] + + +stack = ProcSource('stack', '/proc/%s/task/%s/stack', [ + ('kstack', str, 0), +], None, +task_level=True, +read_samples=read_stack_samples) + + + +### smaps ### +def read_smaps_samples(fh): + samples = [] + current_sample = '' + for line in fh.readlines(): + current_sample += line + if line[:7] == 'VmFlags': + samples.append(current_sample) + current_sample = '' + return samples + + +def parse_smaps_sample(proc_source, sample): + sample_values = [] + sample_lines = [l for l in sample.split('\n') if l != ''] + + header_tokens = sample_lines[0].split() + sample_values.extend(header_tokens[:5]) + sample_values.append(' '.join(header_tokens[5:])) + + for line in sample_lines[1:-1]: + n, kb, _ = line.split() + n = n[:-1].lower() + '_kb' + + # missing values take default parse function value: assume no order change, and that available_columns contains all possible field names + while len(sample_values) < len(proc_source.available_columns) and n != proc_source.available_columns[len(sample_values)][0]: + parse_fn = proc_source.available_columns[len(sample_values)][1] + sample_values.append(parse_fn()) + + if len(sample_values) < len(proc_source.available_columns): + sample_values.append(kb) + + while len(sample_values) < len(proc_source.available_columns) - 1: + parse_fn = proc_source.available_columns[len(sample_values)][1] + sample_values.append(parse_fn()) + + sample_values.append(' '.join(sample_lines[-1].split()[1:])) + return sample_values + + +smaps = ProcSource('smaps', '/proc/%s/smaps', [ + ('address_range', str, 0), + ('perms', str, 1), + ('offset', str, 2), + ('dev', str, 3), + ('inode', int, 4), + ('pathname', str, 5), + ('size_kb', int, 6), + ('rss_kb', int, 7), + ('pss_kb', int, 8), + ('shared_clean_kb', int, 9), + ('shared_dirty_kb', int, 10), + ('private_clean_kb', int, 11), + ('private_dirty_kb', int, 12), + ('referenced_kb', int, 13), + ('anonymous_kb', int, 14), + ('anonhugepages_kb', int, 15), + ('shmempmdmapped_kb', int, 16), + ('shared_hugetld_kb', int, 17), + ('private_hugetld_kb', int, 18), + ('swap_kb', int, 19), + ('swappss_kb', int, 20), + ('kernelpagesize_kb', int, 21), + ('mmupagesize_kb', int, 22), + ('locked_kb', int, 23), + ('vmflags', str, 24), +], None, +task_level=False, +read_samples=read_smaps_samples, +parse_sample=parse_smaps_sample) + + + + +all_sources = [stat, status, syscall, wchan, io, smaps, stack, cmdline] + diff --git a/lib/0xtools/psnreport.py b/lib/0xtools/psnreport.py new file mode 100644 index 0000000..b5f6bbf --- /dev/null +++ b/lib/0xtools/psnreport.py @@ -0,0 +1,201 @@ +# psn -- Linux Process Snapper by Tanel Poder [https://0x.tools] +# Copyright 2019-2021 Tanel Poder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +# query/report code + +from itertools import groupby +from datetime import datetime + +import psnproc as proc +import logging + +def flatten(li): + return [item for sublist in li for item in sublist] + + +### ASCII table output ### +def output_table_report(report, dataset): + max_field_width = 500 + header_fmts, field_fmts = [], [] + total_field_width = 0 + total_field_width_without_kstack = 0 + + if dataset: + col_idx = 0 + for source, cols, expr, token in report.full_projection(): + if token in ('pid', 'task', 'samples'): + col_type = int + elif token == 'event_time': + col_type = str + elif token == 'avg_threads': + col_type = float + elif cols: + col = [c for c in source.available_columns if c[0] == cols[0]][0] + col_type = col[1] + else: + col_type = str + + if col_type in (str, int, int): + max_field_length = max([len(str(row[col_idx])) for row in dataset]) + elif col_idx == float: + max_field_length = max([len(str(int(row[col_idx]))) for row in dataset]) + 3 # arbitrary! + + field_width = min(max_field_width, max(len(token), max_field_length)) + + # left-align strings both in header and data + if col_type == str: + header_fmts.append('%%-%s.%ss' % (field_width, field_width)) + else: + header_fmts.append('%%%s.%ss' % (field_width, field_width)) + + if col_type == str: + field_fmts.append('%%-%s.%ss' % (field_width, field_width)) + elif col_type in (int, int): + field_fmts.append('%%%sd' % field_width) + elif col_type == float: + field_fmts.append('%%%s.%sf' % (field_width, 2)) # arbitrary + + total_field_width += field_width + total_field_width_without_kstack += field_width if token != 'kstack' else 0 + col_idx += 1 + + report_width = total_field_width + (3 * (len(header_fmts) -1)) + 2 + hr = '-' * report_width + title_pad = report_width - len(report.name) - 2 + #title = '=== ' + report.name + ' ' + '=' * (title_pad - 29) + ' [' + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + '] ===' + title = '=== ' + report.name + ' ' + '=' * (title_pad - 3) + header_fmt = ' ' + ' | '.join(header_fmts) + ' ' + field_fmt = ' ' + ' | '.join(field_fmts) + ' ' + + print("") + print(title) + print("") + if dataset: + print(header_fmt % tuple([c[3] for c in report.full_projection()])) + print(hr) + for row in dataset: + print(field_fmt % row) + else: + print('query returned no rows') + print("") + print("") + + + +class Report: + def __init__(self, name, projection, dimensions=[], where=[], order=[], output_fn=output_table_report): + def reify_column_token(col_token): + if col_token == 'samples': + return (None, [], 'COUNT(1)', col_token) + elif col_token == 'avg_threads': + return (None, [], 'CAST(COUNT(1) AS REAL) / %(num_sample_events)s', col_token) + elif col_token in ('pid', 'task', 'event_time'): + return ('first_source', [col_token], col_token, col_token) + + for t in proc.all_sources: + for c in t.schema_columns: + if col_token.lower() == c[0].lower(): + return (t, [c[0]], c[0], c[0]) + + raise Exception('projection/dimension column %s not found.\nUse psn --list to see all available columns' % col_token) + + def process_filter_sql(filter_sql): + idle_filter = "stat.state_id IN ('S', 'Z', 'I', 'P')" + + if filter_sql == 'active': + return (proc.stat, ['state_id'], 'not(%s)' % idle_filter, filter_sql) + elif filter_sql == 'idle': + return (proc.stat, ['state_id'], idle_filter, filter_sql) + else: + raise Exception('arbitrary filtering not implemented') + + self.name = name + self.projection = [reify_column_token(t) for t in projection if t] + self.dimensions = [reify_column_token(t) for t in dimensions if t] + self.order = [reify_column_token(t) for t in order if t] + self.where = [process_filter_sql(t) for t in where if t] + self.output_fn = output_fn + + # columns without a specific source are assigned the first source + first_source = [c[0] for c in (self.projection + self.dimensions + self.order + self.where) if c[0] and c[0] != 'first_source'][0] + self.projection = [(first_source if c[0] == 'first_source' else c[0], c[1], c[2], c[3]) for c in self.projection] + self.dimensions = [(first_source if c[0] == 'first_source' else c[0], c[1], c[2], c[3]) for c in self.dimensions] + self.order = [(first_source if c[0] == 'first_source' else c[0], c[1], c[2], c[3]) for c in self.order] + self.where = [(first_source if c[0] == 'first_source' else c[0], c[1], c[2], c[3]) for c in self.where] + + self.sources = {} # source -> [cols] + for d in [self.projection, self.dimensions, self.order, self.where]: + for source, column_names, expr, token in d: + source_columns = self.sources.get(source, ['pid', 'task', 'event_time']) + source_columns.extend(column_names) + self.sources[source] = source_columns + if None in self.sources: + del self.sources[None] + + + def full_projection(self): + return self.projection + [c for c in self.dimensions if c not in self.projection] + + + def query(self): + def render_col(c): + return '%s.%s' % (c[0].name, c[2]) if c[0] else c[2] + + # build join conditions + first_source_name = list(self.sources.keys())[0].name + join_where = flatten([['%s.%s = %s.%s' % (s.name, c, first_source_name, c) for c in ['pid', 'task', 'event_time']] for s in list(self.sources.keys())[1:]]) + + attr = { + 'projection': '\t' + ',\n\t'.join([render_col(c) for c in self.full_projection()]), + 'tables': '\t' + ',\n\t'.join([s.name for s in self.sources]), + 'where': '\t' + ' AND\n\t'.join([c[2] for c in self.where] + join_where), + 'dimensions': '\t' + ',\n\t'.join([render_col(c) for c in self.dimensions]), + 'order': '\t' + ',\n\t'.join([render_col(c) + ' DESC' for c in self.order]), + 'num_sample_events': '(SELECT COUNT(DISTINCT(event_time)) FROM %s)' % first_source_name + } + + logging.debug('attr where=%s#end' % attr['where']) + + sql = 'SELECT\n%(projection)s\nFROM\n%(tables)s' % attr + # tanel changed from self.where to attr['where'] + # TODO think through the logic of using self.where vs attr.where (in the context of allowing pid/tid to be not part of group by) + if attr['where'].strip(): + sql += '\nWHERE\n%(where)s' % attr + if attr['dimensions']: + sql += '\nGROUP BY\n%(dimensions)s' % attr + if attr['order']: + sql += '\nORDER BY\n%(order)s' % attr + + # final substitution allows things like avg_threads to work + return sql % attr + + + def dataset(self, conn): + logging.debug(self.query()) + r = conn.execute(self.query()).fetchall() + logging.debug('Done') + return r + + def output_report(self, conn): + self.output_fn(self, self.dataset(conn)) + + + + + diff --git a/release.sh b/release.sh new file mode 100755 index 0000000..a2aa230 --- /dev/null +++ b/release.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +PROJECT_NAME="0xtools" + +if [ 0 -eq $# ]; then + echo "" + echo " Usage: ./release.sh tag_or_commitid [tag_or_commitid...]" + echo "" + exit 1 +fi + +for name in "$@"; do + target_type=$(git cat-file -t "${name}" 2>/dev/null) + if [[ -z "${target_type}" ]]; then + echo "${name} is invalid, ignored." + continue + fi + + suffix="" + if expr "${target_type}" : "^commit" >/dev/null; then + suffix=$(git rev-parse --short=8 "${name}") + elif expr "${target_type}" : "^tag" >/dev/null; then + suffix="${name}" + else + echo "${name} is neither a commit nor a tag!" + continue + fi + target_name="${PROJECT_NAME}-${suffix}" + echo "archiving ${target_name}" + git archive -9 --format=tar.gz --prefix="${target_name}"/ "${name}" >"${target_name}".tar.gz + echo "finish ${target_name}" +done diff --git a/src/xcapture.c b/src/xcapture.c new file mode 100644 index 0000000..65cb2e4 --- /dev/null +++ b/src/xcapture.c @@ -0,0 +1,464 @@ +/* + * 0x.Tools xCapture - sample thread activity from Linux procfs [https://0x.tools] + * Copyright 2019-2021 Tanel Poder + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + */ + +#define XCAP_VERSION "1.2.6" + +#define _GNU_SOURCE + +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <string.h> +#include <errno.h> +#include <time.h> +#include <sys/time.h> +#include <dirent.h> +#include <fcntl.h> +#include <asm/unistd.h> +#include <linux/limits.h> +#include <pwd.h> +#include <sys/stat.h> +#include <ctype.h> +#include <unistd.h> + +#include <syscall_names.h> + +#define WSP " \n" // whitespace +#define MAXFILEBUF 4096 + +int DEBUG = 0; + +char filebuf[MAXFILEBUF]; // filebuf global temp area by design +char statbuf[MAXFILEBUF]; // filebuf global temp area by design (special for /proc/PID/stat value passing optimization) +char exclude_states[10] = "XZIS"; // do not show tasks in Sleeping state by default + +char *output_dir = NULL; // use stdout if output_dir is not set +int header_printed = 0; +char output_format = 'S'; // S -> space-delimited fixed output format, C -> CSV +char outsep = ' '; +int pad = 1; // output field padding (for space-delimited fixed-width output) + +const char *getusername(uid_t uid) +{ + struct passwd *pw = getpwuid(uid); + if (pw) + { + return pw->pw_name; + } + + return "-"; +} + + +int readfile(int pid, int tid, const char *name, char *buf) +{ + int fd, bytes = 0; + char path[256]; + + tid ? sprintf(path, "/proc/%d/task/%d/%s", pid, tid, name) : sprintf(path, "/proc/%d/%s", pid, name); + + fd = open(path, O_RDONLY); + if (fd == -1) { + if (DEBUG) fprintf(stderr, "error opening file %s\n", path); + return -1; + } + + bytes = read(fd, buf, MAXFILEBUF); + close(fd); + + // handle errors, empty records and missing string terminators in input + assert(bytes >= -1); + switch (bytes) { + case -1: + if (DEBUG) fprintf(stderr, "read(%s) returned %d\n", path, bytes); + buf[bytes] = '-'; + buf[bytes + 1] = 0; + bytes = 2; + break; + case 0: + buf[bytes] = '-'; + buf[bytes + 1] = 0; + bytes = 2; + break; + case 1: + buf[bytes] = 0; + bytes = 2; + break; + default: // bytes >= 2 + if (bytes < MAXFILEBUF) + buf[bytes] = 0; + else + buf[MAXFILEBUF-1] = 0; + } + return bytes; +} + +int outputstack(char *str) { + int i; + + // find the end and start of function name in the stack + // example input lines (different number of fields): + // [<ffffffff8528428c>] vfs_read+0x8c/0x130 + // [<ffffffffc03b03f4>] xfs_file_fsync+0x224/0x240 [xfs] + for (i=strlen(str)-1; i>=0; i--) { + if (str[i] == '+') str[i] = '\0'; + if (str[i] == ' ' && str[i-1] == ']') { // ignore spaces _after_ the function name + if (strcmp(str+i+1, "entry_SYSCALL_64_after_hwframe") && + strcmp(str+i+1, "do_syscall_64") && + strcmp(str+i+1, "0xffffffffffffffff\n") + ) { + fprintf(stdout, "->%s()", str+i+1); + } + } + } + return 0; +} + +// this function changes the input str (tokenizes it in place) +int outputfields(char *str, char *mask, char *sep) { + int i; + char *field, *pos; + + // special case for stack trace handling, we don't want to split the input string before calling outputstack() + if (mask[0] == 't') + return outputstack(str); + + for (i=0; i<strlen(mask); i++) { + if ((field = strsep(&str, sep)) != NULL) { + switch (mask[i]) { + case '.': // skip field + break; + case 'e': // extract Executable file name from full path + pos = strrchr(field, '/'); + if (pos) + fprintf(stdout, "%s%c", pos, outsep); + else + fprintf(stdout, "%s%c", field, outsep); + break; + case 'E': // same as above, but wider output + pos = strrchr(field, '/'); + if (pos) + fprintf(stdout, pad ? "%-20s%c" : "%s%c", pos+1, outsep); + else + fprintf(stdout, pad ? "%-20s%c" : "%s%c", field, outsep); + break; + case 'o': // just output string as is + fprintf(stdout, "%s%c", field, outsep); + break; + case 'O': // just output string as is, padded to 30 chars + fprintf(stdout, pad ? "%-30s%c" : "%s%c", field, outsep); + break; + case 'x': // print in hex + fprintf(stdout, pad ? "0x%llx " : "0x%llx%c", atoll(field), outsep); + break; + case 's': // convert syscall number to name, the input starts with either: + // >= 0 (syscall), -1 (in kernel without syscall) or 'running' (likely userspace) + fprintf(stdout, "%s%c", field[0]=='r' ? "[running]" : field[0]=='-' ? "[no_syscall]" : sysent0[atoi(field)].name, outsep); + break; + case 'S': // same as above, but wider output + fprintf(stdout, pad ? "%-30s%c" : "%s%c", field[0]=='r' ? "[running]" : field[0]=='-' ? "[no_syscall]" : sysent0[atoi(field)].name, outsep); + break; + case 't': // we shouldn't get here thanks to the if statement above + break; + default: + fprintf(stderr, "Error: Wrong char '%c' in mask %s\n", mask[i], mask); + exit(1); + } + } + else break; + } + + return i; +} + +// currently a fixed string, will make this dynamic together with command line option support +int outputheader(char *add_columns) { + + fprintf(stdout, pad ? "%-23s %7s %7s %-16s %-2s %-30s %-30s %-30s" : "%s,%s,%s,%s,%s,%s,%s,%s", + output_dir ? "TS" : "DATE TIME", "PID", "TID", "USERNAME", "ST", "COMMAND", "SYSCALL", "WCHAN"); + if (strcasestr(add_columns, "exe")) fprintf(stdout, pad ? " %-20s" : ",%s", "EXE"); + if (strcasestr(add_columns, "nspid")) fprintf(stdout, pad ? " %12s" : ",%s", "NSPID"); + if (strcasestr(add_columns, "cmdline")) fprintf(stdout, pad ? " %-30s" : ",%s", "CMDLINE"); + if (strcasestr(add_columns, "kstack")) fprintf(stdout, pad ? " %s" : ",%s", "KSTACK"); + fprintf(stdout, "\n"); + return 1; +} + +// partial entry happens when /proc/PID/stat disappears before we manage to read it +void outputprocpartial(int pid, int tid, char *sampletime, uid_t proc_uid, long nspid, char *add_columns, char *message) { + + header_printed = header_printed ? 1 : outputheader(add_columns); + + fprintf(stdout, pad ? "%-23s %7d %7d %-16s %-2c %-30s %-30s %-30s" : "%s,%d,%d,%s,%c,%s,%s,%s", + sampletime, pid, tid, getusername(proc_uid), '-', message, "-", "-"); + + if (strcasestr(add_columns, "exe")) fprintf(stdout, pad ? " %-20s" : ",%s", "-"); + if (strcasestr(add_columns, "nspid")) fprintf(stdout, pad ? " %12s" : ",%s", "-"); + if (strcasestr(add_columns, "cmdline")) fprintf(stdout, pad ? " %-30s" : ",%s", "-"); + if (strcasestr(add_columns, "kstack")) fprintf(stdout, pad ? " %s" : ",%s", "-"); + fprintf(stdout, "\n"); +} + +int outputprocentry(int pid, int tid, char *sampletime, uid_t proc_uid, long nspid, char *add_columns) { + + int b; + char task_status; // used for early bailout, filtering by task status + char sympath[64]; + char *fieldend; + + // if printing out only the /proc/PID entry (not TID), then we have just read the relevant stat file into filebuf + // in the calling function. this callflow-dependent optimization avoids an 'expensive' /proc/PID/stat read + b = tid ? readfile(pid, tid, "stat", statbuf) : strlen(statbuf); + fieldend = strstr(statbuf, ") "); + + if (b > 0 && fieldend) { // the 1st field end "not null" check is due to /proc not having read consistency (rarely in-flux values are shown as \0\0\0\0\0\0\0... + + // this task_status check operation has to come before any outputfields() calls as they modify filebuf global var + task_status = *(fieldend + 2); // find where the 3rd field - after a ")" starts + + if (!strchr(exclude_states, task_status)) { // task status is not in X,Z,I (S) + + // only print header (in stdout mode) when there are any samples to report + header_printed = header_printed ? 1 : outputheader(add_columns); + + fprintf(stdout, pad ? "%-23s %7d %7d %-16s %-2c " : "%s,%d,%d,%s,%c,", sampletime, pid, tid, getusername(proc_uid), task_status); + outputfields(statbuf, ".O", WSP); // .O......x for PF_ flags + + b = readfile(pid, tid, "syscall", filebuf); + if (b > 0) { outputfields(filebuf, "S", WSP); } else { fprintf(stdout, pad ? "%-30s " : "%s,", "-"); } + + b = readfile(pid, tid, "wchan", filebuf); + if (b > 0) { outputfields(filebuf, "O", ". \n"); } else { fprintf(stdout, pad ? "%-30s " : "%s,", "-"); } + + if (strcasestr(add_columns, "exe")) { + tid ? sprintf(sympath, "/proc/%d/task/%d/exe", pid, tid) : sprintf(sympath, "/proc/%d/exe", pid); + b = readlink(sympath, filebuf, PATH_MAX); + if (b > 0) { filebuf[b] = 0 ; outputfields(filebuf, "E", WSP); } else { fprintf(stdout, pad ? "%-20s " : "%s,", "-"); } + } + + if (strcasestr(add_columns, "nspid")) { + fprintf(stdout, pad ? "%12ld%c" : "%ld%c", nspid, outsep); + } + + if (strcasestr(add_columns, "cmdline")) { + b = readfile(pid, tid, "cmdline", filebuf); // contains spaces and \0s within data TODO escape (or just print argv[0]) + if (b > 0) { fprintf(stdout, pad ? "%-30s%c" : "%s%c", filebuf, outsep); } else { fprintf(stdout, pad ? "%-30s%c" : "%s%c", "-", outsep); } + } + + if (strcasestr(add_columns, "kstack")) { + b = readfile(pid, tid, "stack", filebuf); + if (b > 0) { outputfields(filebuf, "t", WSP); } else { fprintf(stdout, "-"); } + } + + fprintf(stdout, "\n"); + } + } + else { + outputprocpartial(pid, tid, sampletime, proc_uid, nspid, add_columns, "[task_entry_lost(read)]"); + return 1; + } + + return 0; +} + +void printhelp() { + const char *helptext = + "by Tanel Poder [https://0x.tools]\n\n" + "Usage:\n" + " xcapture [options]\n\n" + " By default, sample all /proc tasks in states R, D every second and print to stdout\n\n" + " Options:\n" + " -a capture tasks in additional states, even the ones Sleeping (S)\n" + " -A capture tasks in All states, including Zombie (Z), Exiting (X), Idle (I)\n" + " -c <c1,c2> print additional columns (for example: -c exe,cmdline,nspid,kstack)\n" + " -d <N> seconds between samples (default: 1.0)\n" + " -E <string> custom task state Exclusion filter (default: XZIS)\n" + " -h display this help message\n" + " -o <dirname> write wide output into hourly CSV files in this directory instead of stdout\n"; + + fprintf(stderr, "\n0x.Tools xcapture v%s %s\n", XCAP_VERSION, helptext); +} + +float timedifference_msec(struct timeval t0, struct timeval t1) +{ + return (t1.tv_sec - t0.tv_sec) * 1000.0f + (t1.tv_usec - t0.tv_usec) / 1000.0f; +} + +int main(int argc, char **argv) +{ + char outbuf[BUFSIZ]; + char outpath[PATH_MAX]; + char dirpath[PATH_MAX]; // used for /proc stuff only, so no long paths + DIR *pd, *td; + struct dirent *pde, *tde; // process level and thread/task level directory entries in /proc + + char timebuf[80], usec_buf[9]; + struct timeval tmnow,loop_iteration_start_time,loop_iteration_end_time; + float loop_iteration_msec; + float sleep_for_msec; + struct tm *tm; + int prevhour = -1; // used for detecting switch to a new hour for creating a new output file + int interval_msec = 1000; + + struct stat pidstat, nspstat; + uid_t proc_uid; + long nspid; + + int nthreads = 0; + int mypid = getpid(); + + // argument handling + char *add_columns = ""; // keep "" as a default value and not NULL + int c; + + while ((c = getopt (argc, argv, "aAc:d:E:ho:")) != -1) + switch (c) { + case 'a': + strncpy(exclude_states, "XZI", sizeof(exclude_states)); + break; + case 'A': + strncpy(exclude_states, "", sizeof(exclude_states)); + break; + case 'c': + add_columns = optarg; + break; + case 'd': + interval_msec = atof(optarg) * 1000; + if (interval_msec <= 0 || interval_msec > 3600000) { + fprintf(stderr, "Option -d has invalid value for capture interval - %s (%d)\n", optarg, interval_msec); + return 1; + } + break; + case 'E': + strncpy(exclude_states, optarg, sizeof(exclude_states)); + break; + case 'h': + printhelp(); + exit(1); + break; + case 'o': + output_dir = optarg; + output_format = 'C'; // CSV + outsep = ','; + pad = 0; + if (!strlen(add_columns)) add_columns = "nspid,exe,kstack"; + break; + case '?': + if (strchr("cEd", optopt)) + fprintf(stderr, "Option -%c requires an argument.\n", optopt); + else if (isprint (optopt)) + fprintf(stderr, "Unknown option `-%c'.\n", optopt); + else + fprintf(stderr, "Unknown option character `\\x%x'.\n", optopt); + return 1; + default: + abort(); + } + // end argument handling + + setbuf(stdout, outbuf); + + fprintf(stderr, "\n0xTools xcapture v%s by Tanel Poder [https://0x.tools]\n\nSampling /proc...\n\n", XCAP_VERSION); + + while (1) { + + gettimeofday(&tmnow, NULL); + gettimeofday(&loop_iteration_start_time, NULL); + tm = localtime(&tmnow.tv_sec); + + if (output_dir) { + if (prevhour != tm->tm_hour) { + strftime(timebuf, 30, "%Y-%m-%d.%H", tm); + snprintf(outpath, sizeof(outpath), "%s/%s.csv", output_dir, timebuf); + if (!freopen(outpath, "a", stdout)) { fprintf(stderr, "Error opening output file\n"); exit(1); }; + setbuf(stdout, outbuf); // is this needed after freopen? + prevhour = tm->tm_hour; + header_printed = outputheader(add_columns); + } + } + else { + header_printed = 0; // dynamic stdout header printing decision is made later on + } + + strftime(timebuf, 30, pad ? "%Y-%m-%d %H:%M:%S" : "%Y-%m-%d %H:%M:%S", tm); // currently same format for both outputs + strcat(timebuf, "."); + sprintf(usec_buf, "%03d", (int)tmnow.tv_usec/1000); // ms resolution should be ok for infrequent sampling + strcat(timebuf, usec_buf); + + pd = opendir("/proc"); + if (!pd) { fprintf(stderr, "/proc listing error='%s', this shouldn't happen\n", strerror(errno)); exit(1); } + + while ((pde = readdir(pd))) { // /proc/PID + if (pde->d_name[0] >= '0' && pde->d_name[0] <= '9' && atoi(pde->d_name) != mypid) { + sprintf(dirpath, "/proc/%s", pde->d_name); + proc_uid = stat(dirpath, &pidstat) ? -1 : pidstat.st_uid; + sprintf(dirpath, "/proc/%s/ns/pid", pde->d_name); + nspid = stat(dirpath, &nspstat) ? -1 : nspstat.st_ino; + + // if not multithreaded, read current /proc/PID/x files for efficiency. "nthreads" is 20th field in proc/PID/stat + if (readfile(atoi(pde->d_name), 0, "stat", statbuf) > 0) { + sscanf(statbuf, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %u", &nthreads); + + if (nthreads > 1) { + sprintf(dirpath, "/proc/%s/task", pde->d_name); + + td = opendir(dirpath); + if (td) { + + while ((tde = readdir(td))) { // proc/PID/task/TID + if (tde->d_name[0] >= '0' && tde->d_name[0] <= '9') { + outputprocentry(atoi(pde->d_name), atoi(tde->d_name), timebuf, proc_uid, nspid, add_columns); + } + } + } + else { + outputprocpartial(atoi(pde->d_name), -1, timebuf, proc_uid, nspid, add_columns, "[task_entry_lost(list)]"); + } + closedir(td); + } + else { // nthreads <= 1, therefore pid == tid + outputprocentry(atoi(pde->d_name), atoi(pde->d_name), timebuf, proc_uid, nspid, add_columns); + } + + } // readfile(statbuf) + else { + outputprocpartial(atoi(pde->d_name), -1, timebuf, proc_uid, nspid, add_columns, "[proc_entry_lost(list)]"); + if (DEBUG) fprintf(stderr, "proc entry disappeared /proc/%s/stat, len=%zu, errno=%s\n", pde->d_name, strlen(statbuf), strerror(errno)); + } + } + } + closedir(pd); + + if (!output_dir && header_printed) fprintf(stdout, "\n"); + + fflush(stdout); + + // sleep for the requested interval minus time spent taking the previous sample + gettimeofday(&loop_iteration_end_time, NULL); + loop_iteration_msec = timedifference_msec(loop_iteration_start_time, loop_iteration_end_time); + sleep_for_msec = interval_msec - loop_iteration_msec; + if (sleep_for_msec > 0) usleep(sleep_for_msec * 1000); + + } + + return 0; +} diff --git a/xcapture-restart.service b/xcapture-restart.service new file mode 100644 index 0000000..938aebd --- /dev/null +++ b/xcapture-restart.service @@ -0,0 +1,6 @@ +[Unit] +Description=restart xcapture + +[Service] +Type=oneshot +ExecStart=/usr/bin/systemctl restart xcapture diff --git a/xcapture-restart.timer b/xcapture-restart.timer new file mode 100644 index 0000000..dc3d9ff --- /dev/null +++ b/xcapture-restart.timer @@ -0,0 +1,8 @@ +[Unit] +Description=xcapture restart + +[Timer] +OnCalendar=hourly + +[Install] +WantedBy=timers.target diff --git a/xcapture.default b/xcapture.default new file mode 100644 index 0000000..180d4cd --- /dev/null +++ b/xcapture.default @@ -0,0 +1,4 @@ +SAMPLEINTERVAL=10 +LOGDIRPATH=/var/log/xcapture +ADDITIONALOPTIONS="exe,cmdline,syscall,wchan,kstack" +MINUTES=1440 diff --git a/xcapture.service b/xcapture.service new file mode 100644 index 0000000..4410187 --- /dev/null +++ b/xcapture.service @@ -0,0 +1,16 @@ +[Unit] +Description=0x.Tools xcapture + +[Service] +Environment="SAMPLEINTERVAL=1" +Environment="LOGDIRPATH=/var/log/xcapture" +Environment="ADDITIONALOPTIONS=syscall,wchan,exe,cmdline" +Environment="MINUTES=59520" +EnvironmentFile=/etc/default/xcapture +ExecStartPre=/bin/sh -c 'test -d "$LOGDIRPATH"' +ExecStartPre=/bin/sh -c 'test "$SAMPLEINTERVAL" -ge 1' +ExecStart=/bin/sh -c '/usr/bin/xcapture -d ${SAMPLEINTERVAL} -c ${ADDITIONALOPTIONS} -o ${LOGDIRPATH}' +KillSignal=SIGTERM + +[Install] +WantedBy=multi-user.target |