Adding upstream version 0~20240209.upstream/0_20240209

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

12 files changed, 2158 insertions, 0 deletions

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 0000000..83eb40c
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,48 @@
+---
+name: test
+on:
+  push:
+  pull_request:
+concurrency:
+  # Cancels pending runs when a PR gets updated.
+  group: ${{ github.head_ref || github.run_id }}-${{ github.actor }}
+  cancel-in-progress: true
+jobs:
+
+  build-with-gcc:
+    strategy:
+      matrix:
+        cfg: [debug, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt-get install -y build-essential
+      - run: make CFG=${{ matrix.cfg}} -j$(nproc)
+      - run: _${{ matrix.cfg }}/inotify-info
+
+  build-with-zig:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        with:
+          key: zig-sdk-and-cache-${{ hashFiles('.github/workflows/ci.yaml') }}
+          path: ~/.cache/zig
+      - run: |
+          wget --progress=dot:mega \
+            https://ziglang.org/download/0.11.0/zig-linux-$(uname -m)-0.11.0.tar.xz
+          tar -xJf zig-linux-*.tar.xz
+          rm zig-linux-*.xz
+          mv zig-linux-* zig-sdk
+      - run: |
+          make -j$(nproc) \
+            CC="zig-sdk/zig cc -target $(uname -m)-linux-musl" \
+            CXX="zig-sdk/zig c++ -target $(uname -m)-linux-musl"
+      - run: _release/inotify-info
+
+  build-with-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: docker build -t inotify-info .
+      - run: docker run --rm --privileged -v /proc:/proc inotify-info
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..08376e7
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+_debug/
+_release/
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..8acb222
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,25 @@
+FROM alpine
+
+# zig is installed from the upstream tarball, because:
+# - as of writing, alpine has zig only in testing (which is cumbersome to use)
+# - apk get zig pulls in libllvm, which is huge.
+#
+# Upstream tarball is statically linked, making it small and convenient to use.
+RUN apk add make \
+ && wget https://ziglang.org/download/0.11.0/zig-linux-$(uname -m)-0.11.0.tar.xz \
+ && tar -xJf zig-linux-*.tar.xz \
+ && rm zig-linux-*.xz \
+ && mv zig-linux-* zig
+
+WORKDIR inotify-info
+
+COPY . .
+
+RUN CC="/zig/zig cc -target $(uname -m)-linux-musl" \
+    CXX="/zig/zig c++ -target $(uname -m)-linux-musl" \
+    make
+
+FROM scratch
+COPY --from=0 /inotify-info/_release/inotify-info /inotify-info
+
+ENTRYPOINT ["/inotify-info"]
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..fc10152
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,23 @@
+The MIT License (MIT)
+
+Copyright 2021 Michael Sartain
+
+All Rights Reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..643c951
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,111 @@
+# $@: name of the target file (one before colon)
+# $<: name of first prerequisite file (first one after colon)
+# $^: names of all prerequisite files (space separated)
+# $*: stem (bit which matches the % wildcard in rule definition)
+#
+# VAR = val: Normal setting - values within are recursively expand when var used.
+# VAR := val: Setting of var with simple expansion of values inside - values are expanded at decl time.
+# VAR ?= val: Set var only if it doesn't have a value.
+# VAR += val: Append val to existing value (or set if var didn't exist).
+
+# To use static analyzer:
+#   http://clang-analyzer.llvm.org/scan-build.html
+# Ie:
+#   scan-build -k -V --use-analyzer ~/bin/clang make
+
+NAME = inotify-info
+
+CFG ?= release
+ifeq ($(CFG), debug)
+    ASAN ?= 1
+endif
+
+LD = $(CC)
+RM = rm -f
+MKDIR = mkdir -p
+VERBOSE ?= 0
+
+COMPILER = $(shell $(CC) -v 2>&1 | grep -q "clang version" && echo clang || echo gcc)
+
+WARNINGS = -Wall -Wextra -Wpedantic -Wmissing-include-dirs -Wformat=2 -Wshadow
+ifneq ($(COMPILER),clang)
+  # https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html
+  WARNINGS += -Wsuggest-attribute=format -Wall
+endif
+
+CFLAGS = $(WARNINGS) -march=native -fno-exceptions -gdwarf-4 -g2 -ggnu-pubnames -gsplit-dwarf
+CFLAGS += -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64
+CXXFLAGS = -fno-rtti -Woverloaded-virtual
+LDFLAGS = -march=native -gdwarf-4 -g2 -Wl,--build-id=sha1
+LIBS = -Wl,--no-as-needed -lm -ldl -lpthread -lstdc++
+
+CFILES = \
+	inotify-info.cpp \
+	lfqueue/lfqueue.c
+
+# Useful GCC address sanitizer checks not enabled by default
+# https://kristerw.blogspot.com/2018/06/useful-gcc-address-sanitizer-checks-not.html
+
+ifeq ($(ASAN), 1)
+	# https://gcc.gnu.org/gcc-5/changes.html
+	#  -fsanitize=float-cast-overflow: check that the result of floating-point type to integer conversions do not overflow;
+	#  -fsanitize=vptr: enable checking of C++ member function calls, member accesses and some conversions between pointers to base and derived classes, detect if the referenced object does not have the correct dynamic type.
+	ASAN_FLAGS = -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	ASAN_FLAGS += -fsanitize=address # fast memory error detector (heap, stack, global buffer overflow, and use-after free)
+	ASAN_FLAGS += -fsanitize=leak # detect leaks
+	ASAN_FLAGS += -fsanitize=undefined # fast undefined behavior detector
+	ASAN_FLAGS += -fsanitize=float-divide-by-zero # detect floating-point division by zero;
+	ASAN_FLAGS += -fsanitize=bounds # enable instrumentation of array bounds and detect out-of-bounds accesses;
+	ASAN_FLAGS += -fsanitize=object-size # enable object size checking, detect various out-of-bounds accesses.
+	ASAN_FLAGS += -fsanitize=alignment # enable alignment checking, detect various misaligned objects;
+	CFLAGS += $(ASAN_FLAGS)
+	LDFLAGS += $(ASAN_FLAGS)
+endif
+
+ifeq ($(CFG), debug)
+	ODIR=_debug
+	CFLAGS += -O0 -DDEBUG
+	CFLAGS += -D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_GLIBCXX_SANITIZE_VECTOR -D_LIBCPP_DEBUG=1 -D_LIBCPP_ENABLE_DEBUG_MODE=1
+else
+	ODIR=_release
+	CFLAGS += -O2 -DNDEBUG
+endif
+
+PROJ = $(ODIR)/$(NAME)
+$(info Building $(ODIR)/$(NAME)...)
+
+ifeq ($(VERBOSE), 1)
+	VERBOSE_PREFIX=
+else
+	VERBOSE_PREFIX=@
+endif
+
+C_OBJS = ${CFILES:%.c=${ODIR}/%.o}
+OBJS = ${C_OBJS:%.cpp=${ODIR}/%.o}
+
+all: $(PROJ)
+
+$(ODIR)/$(NAME): $(OBJS)
+	@echo "Linking $@...";
+	$(VERBOSE_PREFIX)$(LD) $(LDFLAGS) $^ $(LIBS) -o $@
+
+-include $(OBJS:.o=.d)
+
+$(ODIR)/%.o: %.c Makefile
+	$(VERBOSE_PREFIX)echo "---- $< ----";
+	@$(MKDIR) $(dir $@)
+	$(VERBOSE_PREFIX)$(CC) -MMD -MP -std=gnu99 $(CFLAGS) -o $@ -c $<
+
+$(ODIR)/%.o: %.cpp Makefile
+	$(VERBOSE_PREFIX)echo "---- $< ----";
+	@$(MKDIR) $(dir $@)
+	$(VERBOSE_PREFIX)$(CXX) -MMD -MP -std=c++11 $(CFLAGS) $(CXXFLAGS) -o $@ -c $<
+
+.PHONY: clean
+
+clean:
+	@echo Cleaning...
+	$(VERBOSE_PREFIX)$(RM) $(PROJ)
+	$(VERBOSE_PREFIX)$(RM) $(OBJS)
+	$(VERBOSE_PREFIX)$(RM) $(OBJS:.o=.d)
+	$(VERBOSE_PREFIX)$(RM) $(OBJS:.o=.dwo)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..a652a65
--- /dev/null
+++ b/README.md
@@ -0,0 +1,183 @@
+# inotify-info
+> Easily track down the number of inotify watches, instances, and which files are being watched.
+
+The Linux inotify system has a few issues [1][problem1][2][problem2] and it can be difficult to debug when you for instance run out of watches. Using this app should hopefully aid you in tracking down how many inotify watches, instances, and what files are being watched.
+
+## Screenshot
+![Alt text](images/inotify-info.png?raw=true "inotify-info")
+
+## Build
+```
+$ make
+Building _release/inotify-info...
+---- inotify-info.cpp ----
+---- lfqueue/lfqueue.c ----
+Linking _release/inotify-info...
+```
+```
+$ CFG=debug make
+Building _debug/inotify-info...
+---- inotify-info.cpp ----
+---- lfqueue/lfqueue.c ----
+Linking _debug/inotify-info...
+```
+
+Or you can build Docker image.
+
+```sh
+docker build . -t inotify-info
+```
+
+## Install
+You are free to copy the resulting executable to any suitable location in your `$PATH`.
+```
+cp _release/inotify-info /usr/local/bin/
+```
+
+## Run (Prints Summary)
+```
+$ _release/inotify-info 
+------------------------------------------------------------------------------
+INotify Limits:
+  max_queued_events:  16384
+  max_user_instances: 128
+  max_user_watches:   65536
+------------------------------------------------------------------------------
+     Pid  App                        Watches   Instances
+     2632 systemd                         23   3
+     2653 pulseaudio                       2   2
+     2656 dbus-daemon                      2   1
+     2987 dbus-daemon                      1   1
+     3056 xfsettingsd                     56   1
+     3068 xfdesktop                       10   1
+     3072 wrapper-2.0                      6   1
+     3091 xfce4-clipman                    1   1
+     3099 xiccd                            1   1
+     3343 xfce4-terminal                   1   1
+     3997 xfce4-appfinder                 11   1
+     4048 xdg-desktop-portal               1   1
+     4086 xdg-desktop-portal-gtk          56   1
+   205668 vivaldi-bin                      8   1
+   205705 vivaldi-bin                      2   1
+------------------------------------------------------------------------------
+Total inotify Watches:   181
+Total inotify Instances: 18
+------------------------------------------------------------------------------
+```
+
+## Run with Appname Filter
+```
+$ _release/inotify-info xfce4
+------------------------------------------------------------------------------
+INotify Limits:
+  max_queued_events:  16384
+  max_user_instances: 128
+  max_user_watches:   65536
+------------------------------------------------------------------------------
+     Pid  App                        Watches   Instances
+     2632 systemd                         23   3
+     2653 pulseaudio                       2   2
+     2656 dbus-daemon                      2   1
+     2987 dbus-daemon                      1   1
+     3056 xfsettingsd                     56   1
+     3068 xfdesktop                       10   1
+     3072 wrapper-2.0                      6   1
+     3091 xfce4-clipman                    1   1
+      94111050 [10304h]
+     3099 xiccd                            1   1
+     3343 xfce4-terminal                   1   1
+      71048655 [10304h]
+     3997 xfce4-appfinder                 11   1
+      94111468 [10304h]  15339430 [10304h]  14554799 [10304h]  70254617 [10304h]  70254684 [10304h]  16786993 [10304h]  14551253 [10304h]  14550430 [10304h]  70254647 [10304h]  70254646 [10304h]
+      92275589 [10304h]
+     4048 xdg-desktop-portal               1   1
+     4086 xdg-desktop-portal-gtk          56   1
+   205668 vivaldi-bin                      8   1
+   205705 vivaldi-bin                      2   1
+------------------------------------------------------------------------------
+Total inotify Watches:   181
+Total inotify Instances: 18
+------------------------------------------------------------------------------
+
+Searching '/' for listed inodes... (8 threads)
+ 14550430 [10304h] /usr/share/applications/
+ 14551253 [10304h] /usr/local/share/
+ 14554799 [10304h] /usr/share/xfce4/
+ 15339430 [10304h] /usr/share/desktop-directories/
+ 16786993 [10304h] /usr/share/xfce4/applications/
+ 70254617 [10304h] /home/mikesart/.local/share/
+ 70254646 [10304h] /home/mikesart/.config/menus/
+ 70254647 [10304h] /home/mikesart/.config/menus/applications-merged/
+ 70254684 [10304h] /home/mikesart/.local/share/applications/
+ 71048655 [10304h] /home/mikesart/.config/xfce4/terminal/
+ 92275589 [10304h] /etc/xdg/menus/
+ 94111050 [10304h] /home/mikesart/.config/xfce4/panel/
+ 94111468 [10304h] /home/mikesart/.cache/xfce4/xfce4-appfinder/
+```
+## Run with Specific Pid(s)
+```
+$ _release/inotify-info 3997
+------------------------------------------------------------------------------
+INotify Limits:
+  max_queued_events:  16384
+  max_user_instances: 128
+  max_user_watches:   65536
+------------------------------------------------------------------------------
+     Pid  App                        Watches   Instances
+     2632 systemd                         23   3
+     2653 pulseaudio                       2   2
+     2656 dbus-daemon                      2   1
+     2987 dbus-daemon                      1   1
+     3056 xfsettingsd                     56   1
+     3068 xfdesktop                       10   1
+     3072 wrapper-2.0                      6   1
+     3091 xfce4-clipman                    1   1
+     3099 xiccd                            1   1
+     3343 xfce4-terminal                   1   1
+     3997 xfce4-appfinder                 11   1
+      94111468 [10304h]  15339430 [10304h]  14554799 [10304h]  70254617 [10304h]  70254684 [10304h]  16786993 [10304h]  14551253 [10304h]  14550430 [10304h]  70254647 [10304h]  70254646 [10304h]
+      92275589 [10304h]
+     4048 xdg-desktop-portal               1   1
+     4086 xdg-desktop-portal-gtk          56   1
+   205668 vivaldi-bin                      8   1
+   205705 vivaldi-bin                      2   1
+------------------------------------------------------------------------------
+Total inotify Watches:   181
+Total inotify Instances: 18
+------------------------------------------------------------------------------
+
+Searching '/' for listed inodes... (8 threads)
+ 14550430 [10304h] /usr/share/applications/
+ 14551253 [10304h] /usr/local/share/
+ 14554799 [10304h] /usr/share/xfce4/
+ 15339430 [10304h] /usr/share/desktop-directories/
+ 16786993 [10304h] /usr/share/xfce4/applications/
+ 70254617 [10304h] /home/mikesart/.local/share/
+ 70254646 [10304h] /home/mikesart/.config/menus/
+ 70254647 [10304h] /home/mikesart/.config/menus/applications-merged/
+ 70254684 [10304h] /home/mikesart/.local/share/applications/
+ 92275589 [10304h] /etc/xdg/menus/
+ 94111468 [10304h] /home/mikesart/.cache/xfce4/xfce4-appfinder/
+```
+
+## Run on Docker
+
+```sh
+docker run --rm --privileged -v /proc:/proc inotify-info
+```
+
+## Run on Nix(OS)
+
+```
+nix run nixpkgs#inotify-info
+```
+
+## Credits
+
+[lfqueue][lfqueue] is [BSD-2-Clause License][bsd]
+
+
+[problem1]: https://code.visualstudio.com/docs/setup/linux#_visual-studio-code-is-unable-to-watch-for-file-changes-in-this-large-workspace-error-enospc  
+[problem2]: https://unix.stackexchange.com/questions/15509/whos-consuming-my-inotify-resources  
+[lfqueue]:  https://github.com/Taymindis/lfqueue
+[bsd]:      https://github.com/Taymindis/lfqueue/blob/master/LICENSE
diff --git a/images/inotify-info.png b/images/inotify-info.png
new file mode 100644
index 0000000..507554f
--- /dev/null
+++ b/images/inotify-info.png
diff --git a/inotify-info.cpp b/inotify-info.cpp
new file mode 100644
index 0000000..71ed3a1
--- /dev/null
+++ b/inotify-info.cpp
@@ -0,0 +1,1218 @@
+/*
+ * Copyright 2021 Michael Sartain
+ *
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#define _GNU_SOURCE 1
+
+#include <limits.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <pthread.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <libgen.h>
+#include <locale.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <syscall.h>
+#include <unistd.h>
+#include <sys/sysmacros.h>
+#include <sys/statfs.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <unordered_set>
+#include <unordered_map>
+
+#include "inotify-info.h"
+#include "lfqueue/lfqueue.h"
+
+/*
+ * TODO
+ *  - Comments
+ *  - Disable color
+ */
+
+static int g_verbose = 0;
+static size_t g_numthreads = 32;
+
+/* true if at least one inotify watch is found in fdinfo files
+ * On a system with no active inotify watches, but which otherwise
+ * supports inotify watch info, this will prevent the watches column
+ * from being displayed.
+ * This case is indistinguishable from the case where the kernel does
+ * not support inotify watch info.
+ */
+static int g_kernel_provides_watches_info = 0;
+
+static char thousands_sep = ',';
+
+static std::vector< std::string > ignore_dirs;
+
+/*
+ * filename info
+ */
+struct filename_info_t
+{
+    ino64_t inode;  // Inode number
+    dev_t dev;      // Device ID containing file
+    std::string filename;
+};
+
+/*
+ * inotify process info
+ */
+struct procinfo_t
+{
+    pid_t pid = 0;
+
+    // uid
+    uid_t uid = 0;
+
+    // Count of inotify watches and instances
+    uint32_t watches = 0;
+    uint32_t instances = 0;
+
+    // This appname or pid found in command line?
+    bool in_cmd_line = false;
+
+    // Full executable path
+    std::string executable;
+
+    // Executable basename
+    std::string appname;
+
+    // Inotify fdset filenames
+    std::vector< std::string > fdset_filenames;
+
+    // Device id map -> set of inodes for that device id
+    std::unordered_map< dev_t, std::unordered_set< ino64_t > > dev_map;
+};
+
+class lfqueue_wrapper_t
+{
+public:
+    lfqueue_wrapper_t()                { lfqueue_init( &queue ); }
+    ~lfqueue_wrapper_t()               { lfqueue_destroy( &queue ); }
+
+    void queue_directory( char *path ) { lfqueue_enq( &queue, path ); }
+    char *dequeue_directory()          { return ( char * )lfqueue_deq( &queue ); }
+
+public:
+    typedef long long my_m256i __attribute__((__vector_size__(32), __aligned__(32)));
+
+    union
+    {
+        lfqueue_t queue;
+        my_m256i align_buf[4]; // Align to 128 bytes
+    };
+};
+
+/*
+ * shared thread data
+ */
+class thread_shared_data_t
+{
+public:
+    bool init( uint32_t numthreads, const std::vector< procinfo_t > &inotify_proclist );
+
+public:
+    // Array of queues - one per thread
+    std::vector< lfqueue_wrapper_t > dirqueues;
+    // Map of all inotify inodes watched to the set of devices they are on
+    std::unordered_map< ino64_t, std::unordered_set< dev_t > > inode_set;
+};
+
+/*
+ * thread info
+ */
+class thread_info_t
+{
+public:
+    thread_info_t( thread_shared_data_t &tdata_in ) : tdata( tdata_in ) {}
+    ~thread_info_t() {}
+
+    void queue_directory( char *path );
+    char *dequeue_directory();
+
+    // Returns -1: queue empty, 0: open error, > 0 success
+    int parse_dirqueue_entry();
+
+    void add_filename( ino64_t inode, const char *path, const char *d_name, bool is_dir );
+
+public:
+    uint32_t idx = 0;
+    pthread_t pthread_id = 0;
+
+    thread_shared_data_t &tdata;
+
+    // Total dirs scanned by this thread
+    uint32_t scanned_dirs = 0;
+    // Files found by this thread
+    std::vector< filename_info_t > found_files;
+};
+
+/*
+ * getdents64 syscall
+ */
+GCC_DIAG_PUSH_OFF( pedantic )
+struct linux_dirent64
+{
+    uint64_t d_ino;          // Inode number
+    int64_t d_off;           // Offset to next linux_dirent
+    unsigned short d_reclen; // Length of this linux_dirent
+    unsigned char d_type;    // File type
+    char d_name[];           // Filename (null-terminated)
+};
+GCC_DIAG_POP()
+
+int sys_getdents64( int fd, char *dirp, int count )
+{
+    return syscall( SYS_getdents64, fd, dirp, count );
+}
+
+static double gettime()
+{
+    struct timespec ts;
+
+    clock_gettime( CLOCK_MONOTONIC, &ts );
+    return ( double )ts.tv_sec + ( double )ts.tv_nsec / 1e9;
+}
+
+std::string string_formatv( const char *fmt, va_list ap )
+{
+    std::string str;
+    int size = 512;
+
+    for ( ;; )
+    {
+        str.resize( size );
+        int n = vsnprintf( ( char * )str.c_str(), size, fmt, ap );
+
+        if ( ( n > -1 ) && ( n < size ) )
+        {
+            str.resize( n );
+            return str;
+        }
+
+        size = ( n > -1 ) ? ( n + 1 ) : ( size * 2 );
+    }
+}
+
+std::string string_format( const char *fmt, ... )
+{
+    va_list ap;
+    std::string str;
+
+    va_start( ap, fmt );
+    str = string_formatv( fmt, ap );
+    va_end( ap );
+
+    return str;
+}
+
+static std::string get_link_name( const char *pathname )
+{
+    std::string Result;
+    char filename[ PATH_MAX + 1 ];
+
+    ssize_t ret = readlink( pathname, filename, sizeof( filename ) );
+    if ( ( ret > 0 ) && ( ret < ( ssize_t )sizeof( filename ) ) )
+    {
+        filename[ ret ] = 0;
+        Result = filename;
+    }
+    return Result;
+}
+
+static uid_t get_uid(const char *pathname)
+{
+    int fd = open( pathname, O_RDONLY, 0 );
+
+    if ( fd >= 0 )
+    {
+        char buf[ 16 * 1024 ];
+
+        ssize_t len = read( fd, buf, sizeof( buf ) );
+
+        close( fd );
+        fd = -1;
+
+        if ( len > 0 )
+        {
+            buf[ len - 1 ] = 0;
+
+            const char *uidstr = strstr( buf, "\nUid:" );
+            if ( uidstr )
+            {
+                return atoll( uidstr + 5 );
+            }
+        }
+    }
+
+    return -1;
+}
+
+static uint64_t get_token_val( const char *line, const char *token )
+{
+    const char *str = strstr( line, token );
+
+    return str ? strtoull( str + strlen( token ), nullptr, 16 ) : 0;
+}
+
+static uint32_t inotify_parse_fdinfo_file( procinfo_t &procinfo, const char *fdset_name )
+{
+    uint32_t watch_count = 0;
+
+    FILE *fp = fopen( fdset_name, "r" );
+    if ( fp )
+    {
+        char line_buf[ 256 ];
+
+        procinfo.fdset_filenames.push_back( fdset_name );
+
+        for ( ;; )
+        {
+            if ( !fgets( line_buf, sizeof( line_buf ), fp ) )
+                break;
+
+            /* sample fdinfo; inotify line added in linux 3.8, available if
+             * kernel compiled with CONFIG_INOTIFY_USER and CONFIG_PROC_FS
+             *   pos:    0
+             *   flags:  00
+             *   mnt_id: 15
+             *   ino:    5865
+             *   inotify wd:1 ino:80001 sdev:800011 mask:100 ignored_mask:0 fhandle-bytes:8 fhandle-type:1 f_handle:01000800bc1b8c7c
+             */
+            if ( !strncmp( line_buf, "inotify ", 8 ) )
+            {
+                watch_count++;
+
+                uint64_t inode_val = get_token_val( line_buf, "ino:" );
+                uint64_t sdev_val = get_token_val( line_buf, "sdev:" );
+
+                if ( inode_val )
+                {
+                    // https://unix.stackexchange.com/questions/645937/listing-the-files-that-are-being-watched-by-inotify-instances
+                    //   Assuming that the sdev field is encoded according to Linux's so-called "huge
+                    //   encoding", which uses 20 bits (instead of 8) for minor numbers, in bitwise
+                    //   parlance the major number is sdev >> 20 while the minor is sdev & 0xfffff.
+                    unsigned int major = sdev_val >> 20;
+                    unsigned int minor = sdev_val & 0xfffff;
+
+                    // Add inode to this device map
+                    procinfo.dev_map[ makedev( major, minor ) ].insert( inode_val );
+                }
+            }
+        }
+
+        fclose( fp );
+    }
+
+    return watch_count;
+}
+
+static void inotify_parse_fddir( procinfo_t &procinfo )
+{
+    std::string filename = string_format( "/proc/%d/fd", procinfo.pid );
+
+    DIR *dir_fd = opendir( filename.c_str() );
+    if ( !dir_fd )
+        return;
+
+    for ( ;; )
+    {
+        struct dirent *dp_fd = readdir( dir_fd );
+        if ( !dp_fd )
+            break;
+
+        if ( ( dp_fd->d_type == DT_LNK ) && isdigit( dp_fd->d_name[ 0 ] ) )
+        {
+            filename = string_format( "/proc/%d/fd/%s", procinfo.pid, dp_fd->d_name );
+            filename = get_link_name( filename.c_str() );
+
+            if ( filename == "anon_inode:inotify" || filename == "inotify" )
+            {
+                filename = string_format( "/proc/%d/fdinfo/%s", procinfo.pid, dp_fd->d_name );
+
+                procinfo.instances++;
+                procinfo.watches += inotify_parse_fdinfo_file( procinfo, filename.c_str() );
+
+                /* If any watches have been found, enable the stats display */
+                g_kernel_provides_watches_info |= !!procinfo.watches;
+            }
+        }
+    }
+
+    closedir( dir_fd );
+}
+
+void thread_info_t::queue_directory( char *path )
+{
+    tdata.dirqueues[ idx ].queue_directory( path );
+}
+
+char *thread_info_t::dequeue_directory()
+{
+    char *path = tdata.dirqueues[ idx ].dequeue_directory();
+
+    if ( !path )
+    {
+        // Nothing on our queue, check queues on other threads
+        for ( lfqueue_wrapper_t &dirq : tdata.dirqueues )
+        {
+            path = dirq.dequeue_directory();
+            if ( path )
+                break;
+        }
+    }
+
+    return path;
+}
+
+// statx() was added to Linux in kernel 4.11; library support was added in glibc 2.28.
+#if defined( __linux__ ) && ( ( __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 28 ) || ( __GLIBC__ > 2 ) )
+
+struct statx mystatx( const char *filename, unsigned int mask = 0 )
+{
+    struct statx statxbuf;
+    int flags = AT_NO_AUTOMOUNT | AT_SYMLINK_NOFOLLOW | AT_STATX_DONT_SYNC;
+
+    if ( statx( 0, filename, flags, mask, &statxbuf ) == -1 )
+    {
+        printf( "ERROR: statx-ino( %s ) failed. Errno: %d\n", filename, errno );
+        memset( &statxbuf, 0, sizeof( statxbuf ) );
+    }
+
+    return statxbuf;
+}
+
+static dev_t stat_get_dev_t( const char *filename )
+{
+    struct statx statxbuf = mystatx( filename );
+
+    return makedev( statxbuf.stx_dev_major, statxbuf.stx_dev_minor );
+}
+
+static uint64_t stat_get_ino( const char *filename )
+{
+    return mystatx( filename, STATX_INO ).stx_ino;
+}
+
+#else
+
+// Fall back to using stat() functions. Should work but be slower than using statx().
+
+static dev_t stat_get_dev_t( const char *filename )
+{
+    struct stat statbuf;
+
+    int ret = stat( filename, &statbuf );
+    if ( ret == -1 )
+    {
+        printf( "ERROR: stat-dev_t( %s ) failed. Errno: %d\n", filename, errno );
+        return 0;
+    }
+    return statbuf.st_dev;
+}
+
+static uint64_t stat_get_ino( const char *filename )
+{
+    struct stat statbuf;
+
+    int ret = stat( filename, &statbuf );
+    if ( ret == -1 )
+    {
+        printf( "ERROR: stat-ino( %s ) failed. Errno: %d\n", filename, errno );
+        return 0;
+    }
+
+    return statbuf.st_ino;
+}
+
+#endif
+
+void thread_info_t::add_filename( ino64_t inode, const char *path, const char *d_name, bool is_dir )
+{
+    auto it = tdata.inode_set.find( inode );
+
+    if ( it != tdata.inode_set.end() )
+    {
+        const std::unordered_set< dev_t > &dev_set = it->second;
+
+        std::string filename = std::string( path ) + d_name;
+        dev_t dev = stat_get_dev_t( filename.c_str() );
+
+        // Make sure the inode AND device ID match before adding.
+        if ( dev_set.find( dev ) != dev_set.end() )
+        {
+            filename_info_t fname;
+
+            fname.filename = is_dir ? filename + "/" : filename;
+            fname.inode = inode;
+            fname.dev = dev;
+
+            found_files.push_back( fname );
+        }
+    }
+}
+
+static bool is_dot_dir( const char *dname )
+{
+    if ( dname[ 0 ] == '.' )
+    {
+        if ( !dname[ 1 ] )
+            return true;
+
+        if ( ( dname[ 1 ] == '.' ) && !dname[ 2 ] )
+            return true;
+    }
+
+    return false;
+}
+
+// From "linux/magic.h"
+#define PROC_SUPER_MAGIC    0x9fa0
+#define SMB_SUPER_MAGIC     0x517B
+#define CIFS_SUPER_MAGIC    0xFF534D42      /* the first four bytes of SMB PDUs */
+#define SMB2_SUPER_MAGIC    0xFE534D42
+#define FUSE_SUPER_MAGIC    0x65735546
+
+// Detect proc and fuse directories and skip them.
+//   https://github.com/mikesart/inotify-info/issues/6
+// Could use setmntent("/proc/mounts", "r") + getmntent if speed is an issue?
+static bool is_proc_dir( const char *path, const char *d_name )
+{
+    struct statfs s;
+    std::string filename = std::string( path ) + d_name;
+
+    if ( statfs(filename.c_str(), &s ) == 0 )
+    {
+        switch ( s.f_type )
+        {
+        case PROC_SUPER_MAGIC:
+        case FUSE_SUPER_MAGIC:
+            return true;
+        }
+    }
+
+    return false;
+}
+
+// Returns -1: queue empty, 0: open error, > 0 success
+int thread_info_t::parse_dirqueue_entry()
+{
+    char __attribute__( ( aligned( 16 ) ) ) buf[ 1024 ];
+
+    char *path = dequeue_directory();
+    if ( !path )
+    {
+        return -1;
+    }
+
+    for ( std::string &dname : ignore_dirs )
+    {
+        if ( dname == path )
+        {
+            if ( g_verbose > 1 )
+            {
+                printf( "Ignoring '%s'\n", path );
+            }
+            return 0;
+        }
+    }
+
+    int fd = open( path, O_RDONLY | O_DIRECTORY, 0 );
+    if ( fd < 0 )
+    {
+        free( path );
+        return 0;
+    }
+
+    scanned_dirs++;
+
+    size_t pathlen = strlen( path );
+
+    for ( ;; )
+    {
+        int ret = sys_getdents64( fd, buf, sizeof( buf ) );
+
+        if ( ret < 0 )
+        {
+            bool spew_error = true;
+
+            if ( ( errno == 5 ) && !strncmp( path, "/sys/kernel/", 12 ) )
+            {
+                // In docker container we can get permission denied errors in /sys/kernel. Ignore them.
+                // https://github.com/mikesart/inotify-info/issues/16
+                spew_error = false;
+            }
+
+            if ( spew_error )
+            {
+                printf( "ERROR: sys_getdents64 failed on '%s': %d errno:%d\n", path, ret, errno );
+            }
+            break;
+        }
+        if ( ret == 0 )
+            break;
+
+        for ( int bpos = 0; bpos < ret; )
+        {
+            struct linux_dirent64 *dirp = ( struct linux_dirent64 * )( buf + bpos );
+            const char *d_name = dirp->d_name;
+
+            // DT_BLK      This is a block device.
+            // DT_CHR      This is a character device.
+            // DT_FIFO     This is a named pipe (FIFO).
+            // DT_SOCK     This is a UNIX domain socket.
+            // DT_UNKNOWN  The file type could not be determined.
+
+            // DT_REG      This is a regular file.
+            // DT_LNK      This is a symbolic link.
+            if ( dirp->d_type == DT_REG || dirp->d_type == DT_LNK )
+            {
+                add_filename( dirp->d_ino, path, d_name, false );
+            }
+            // DT_DIR      This is a directory.
+            else if ( dirp->d_type == DT_DIR )
+            {
+                if ( !is_dot_dir( d_name ) && !is_proc_dir( path, d_name ) )
+                {
+                    add_filename( dirp->d_ino, path, d_name, true );
+
+                    size_t len = strlen( d_name );
+                    char *newpath = ( char * )malloc( pathlen + len + 2 );
+
+                    if ( newpath )
+                    {
+                        strcpy( newpath, path );
+                        strcpy( newpath + pathlen, d_name );
+                        newpath[ pathlen + len ] = '/';
+                        newpath[ pathlen + len + 1 ] = 0;
+
+                        queue_directory( newpath );
+                    }
+                }
+            }
+
+            bpos += dirp->d_reclen;
+        }
+    }
+
+    close( fd );
+    free( path );
+    return 1;
+}
+
+static void *parse_dirqueue_threadproc( void *arg )
+{
+    thread_info_t *pthread_info = ( thread_info_t * )arg;
+
+    for ( ;; )
+    {
+        // Loop until all the dequeue(s) fail
+        if ( pthread_info->parse_dirqueue_entry() == -1 )
+            break;
+    }
+
+    return nullptr;
+}
+
+static bool is_proc_in_cmdline_applist( const procinfo_t &procinfo, std::vector< std::string > &cmdline_applist )
+{
+    for ( const std::string &str : cmdline_applist )
+    {
+        // Check if our command line string is a subset of this appname
+        if ( strstr( procinfo.appname.c_str(), str.c_str() ) )
+            return true;
+
+        // Check if the PIDs match
+        if ( atoll( str.c_str() ) == procinfo.pid )
+            return true;
+    }
+
+    return false;
+}
+
+static bool watch_count_is_greater ( procinfo_t elem1, procinfo_t elem2 )
+{
+   return elem1.watches > elem2.watches;
+}
+
+
+static bool init_inotify_proclist( std::vector< procinfo_t > &inotify_proclist )
+{
+    DIR *dir_proc = opendir( "/proc" );
+
+    if ( !dir_proc )
+    {
+        printf( "ERROR: opendir /proc failed: %d\n", errno );
+        return false;
+    }
+
+    for ( ;; )
+    {
+        struct dirent *dp_proc = readdir( dir_proc );
+        if ( !dp_proc )
+            break;
+
+        if ( ( dp_proc->d_type == DT_DIR ) && isdigit( dp_proc->d_name[ 0 ] ) )
+        {
+            procinfo_t procinfo;
+
+            procinfo.pid = atoll( dp_proc->d_name );
+
+            std::string executable = string_format( "/proc/%d/exe", procinfo.pid );
+            std::string status = string_format( "/proc/%d/status", procinfo.pid );
+            procinfo.uid = get_uid( status.c_str() );
+            procinfo.executable = get_link_name( executable.c_str() );
+            if ( !procinfo.executable.empty() )
+            {
+                procinfo.appname = basename( (char*)procinfo.executable.c_str() );
+
+                inotify_parse_fddir( procinfo );
+
+                if ( procinfo.instances )
+                {
+                    inotify_proclist.push_back( procinfo );
+                }
+            }
+        }
+    }
+    std::sort(inotify_proclist.begin(), inotify_proclist.end(), watch_count_is_greater);
+
+    closedir( dir_proc );
+    return true;
+}
+
+// From:
+//  https://stackoverflow.com/questions/1449805/how-to-format-a-number-using-comma-as-thousands-separator-in-c
+size_t str_format_uint32( char dst[16], uint32_t num )
+{
+    if ( thousands_sep )
+    {
+        char src[16];
+        char *p_src = src;
+        char *p_dst = dst;
+        int num_len, commas;
+
+        num_len = sprintf(src, "%u", num);
+
+        for (commas = 2 - num_len % 3; *p_src; commas = (commas + 1) % 3)
+        {
+            *p_dst++ = *p_src++;
+            if (commas == 1) {
+                *p_dst++ = thousands_sep;
+            }
+        }
+        *--p_dst = '\0';
+
+        return (size_t)(p_dst - dst);
+    }
+
+    return sprintf(dst, "%u", num);
+}
+
+static void print_inotify_proclist( std::vector< procinfo_t > &inotify_proclist )
+{
+#if 0
+    // test data
+    procinfo_t proc_info = {};
+    proc_info.pid = 100;
+    proc_info.appname = "fsnotifier";
+    proc_info.watches = 2;
+    proc_info.instances = 1;
+    inotify_proclist.push_back(proc_info);
+
+    proc_info.pid = 1000;
+    proc_info.appname = "evolution-addressbook-factor";
+    proc_info.watches = 116;
+    proc_info.instances = 10;
+    inotify_proclist.push_back(proc_info);
+
+    proc_info.pid = 22154;
+    proc_info.appname = "evolution-addressbook-factor blah blah";
+    proc_info.watches = 28200;
+    proc_info.instances = 100;
+    inotify_proclist.push_back(proc_info);
+
+    proc_info.pid = 0x7fffffff;
+    proc_info.appname = "evolution-addressbook-factor blah blah2";
+    proc_info.watches = 999999;
+    proc_info.instances = 999999999;
+    inotify_proclist.push_back(proc_info);
+#endif
+
+    int lenPid = 10;
+    int lenUid = 10;
+    int lenApp = 10;
+    int lenWatches = 8;
+    int lenInstances = 10;
+
+    for ( procinfo_t &procinfo : inotify_proclist )
+        lenApp = std::max<int>( procinfo.appname.length(), lenApp );
+
+    /* If the number of watches is negative, the kernel doesn't support this info. omit the header*/
+    if (g_kernel_provides_watches_info)
+        printf( "%s%*s %-*s %-*s %*s %*s%s\n",
+            BCYAN, lenPid, "Pid", lenUid, "Uid", lenApp, "App", lenWatches, "Watches", lenInstances, "Instances", RESET);
+    else
+        printf( "%s%*s %-*s %*s %*s%s\n",
+            BCYAN, lenPid, "Pid", lenUid, "Uid", lenApp, "App", lenInstances, "Instances", RESET);
+
+
+    for ( procinfo_t &procinfo : inotify_proclist )
+    {
+        char watches_str[16];
+
+        str_format_uint32(watches_str, procinfo.watches);
+
+        if (g_kernel_provides_watches_info)
+            printf( "%*d %-*d %s%-*s%s %*s %*u\n",
+                lenPid, procinfo.pid,
+                lenUid, procinfo.uid,
+                BYELLOW, lenApp, procinfo.appname.c_str(), RESET,
+                lenWatches, watches_str,
+                lenInstances, procinfo.instances );
+        else
+            printf( "%*d %-*d %s%-*s%s %*u\n",
+                lenPid, procinfo.pid,
+                lenUid, procinfo.uid,
+                BYELLOW, lenApp, procinfo.appname.c_str(), RESET,
+                lenInstances, procinfo.instances );
+
+        if ( g_verbose > 1 )
+        {
+            for ( std::string &fname : procinfo.fdset_filenames )
+            {
+                printf( "    %s%s%s\n", CYAN, fname.c_str(), RESET );
+            }
+        }
+
+        if ( procinfo.in_cmd_line  )
+        {
+            for ( const auto &it1 : procinfo.dev_map )
+            {
+                dev_t dev = it1.first;
+
+                printf( "%s[%u.%u]:%s", BGRAY, major( dev ), minor( dev ), RESET );
+                for ( const auto &it2 : it1.second )
+                {
+                    std::string inode_device_str = string_format( "%lu", it2 );
+
+                    printf( " %s%s%s", BGRAY, inode_device_str.c_str(), RESET );
+                }
+                printf( "\n" );
+            }
+        }
+    }
+}
+
+bool thread_shared_data_t::init( uint32_t numthreads, const std::vector< procinfo_t > &inotify_proclist )
+{
+    for ( const procinfo_t &procinfo : inotify_proclist )
+    {
+        if ( !procinfo.in_cmd_line  )
+            continue;
+
+        for ( const auto &it1 : procinfo.dev_map )
+        {
+            dev_t dev = it1.first;
+
+            for ( const auto &inode : it1.second )
+            {
+                inode_set[ inode ].insert( dev );
+            }
+        }
+    }
+
+    if ( !inode_set.empty() )
+    {
+        dirqueues.resize( numthreads );
+    }
+
+    return !inode_set.empty();
+}
+
+static uint32_t find_files_in_inode_set( const std::vector< procinfo_t > &inotify_proclist,
+    std::vector< filename_info_t > &all_found_files )
+{
+    thread_shared_data_t tdata;
+
+    g_numthreads = std::max< size_t >( 1, g_numthreads );
+
+    if ( !tdata.init( g_numthreads, inotify_proclist ) )
+        return 0;
+
+    printf( "\n%sSearching '/' for listed inodes...%s (%lu threads)\n", BCYAN, RESET, g_numthreads );
+
+    // Initialize thread_info_t array
+    std::vector< class thread_info_t > thread_array( g_numthreads, thread_info_t( tdata ) );
+
+    for ( uint32_t idx = 0; idx < thread_array.size(); idx++ )
+    {
+        thread_info_t &thread_info = thread_array[ idx ];
+
+        thread_info.idx = idx;
+
+        if ( idx == 0 )
+        {
+            // Add root dir in case someone is watching it
+            thread_info.add_filename( stat_get_ino( "/" ), "/", "", false );
+            // Add and parse root
+            thread_info.queue_directory( strdup( "/" ) );
+            thread_info.parse_dirqueue_entry();
+        }
+        else if ( pthread_create( &thread_info.pthread_id, NULL, &parse_dirqueue_threadproc, &thread_info ) )
+        {
+            printf( "Warning: pthread_create failed. errno: %d\n", errno );
+            thread_info.pthread_id = 0;
+        }
+    }
+
+    // Put main thread to work
+    parse_dirqueue_threadproc( &thread_array[ 0 ] );
+
+    uint32_t total_scanned_dirs = 0;
+    for ( const thread_info_t &thread_info : thread_array )
+    {
+        if ( thread_info.pthread_id )
+        {
+            if ( g_verbose > 1 )
+            {
+                printf( "Waiting for thread #%zu\n", thread_info.pthread_id );
+            }
+
+            void *status = NULL;
+            int rc = pthread_join( thread_info.pthread_id, &status );
+
+            if ( g_verbose > 1 )
+            {
+                printf( "Thread #%zu rc=%d status=%d\n", thread_info.pthread_id, rc, ( int )( intptr_t )status );
+            }
+        }
+
+        // Snag data from this thread
+        total_scanned_dirs += thread_info.scanned_dirs;
+
+        all_found_files.insert( all_found_files.end(),
+                thread_info.found_files.begin(), thread_info.found_files.end() );
+
+        if ( g_verbose > 1 )
+        {
+            printf( "Thread #%zu: %u dirs, %zu files found\n",
+                    thread_info.pthread_id, thread_info.scanned_dirs, thread_info.found_files.size() );
+        }
+    }
+
+    struct
+    {
+        bool operator()( const filename_info_t &a, const filename_info_t &b ) const
+        {
+            if ( a.dev == b.dev )
+                return a.inode < b.inode;
+            return a.dev < b.dev;
+        }
+    } filename_info_less_func;
+
+    std::sort( all_found_files.begin(), all_found_files.end(), filename_info_less_func );
+
+    return total_scanned_dirs;
+}
+
+static uint32_t get_inotify_procfs_value( const std::string &fname )
+{
+    char buf[ 64 ];
+    uint32_t val = 0;
+    std::string filename = "/proc/sys/fs/inotify/" + fname;
+
+    int fd = open( filename.c_str(), O_RDONLY );
+    if ( fd >= 0 )
+    {
+        if ( read( fd, buf, sizeof( buf ) ) > 0 )
+        {
+            val = strtoul( buf, nullptr, 10 );
+        }
+
+        close( fd );
+    }
+
+    return val;
+}
+
+static void print_inotify_limits()
+{
+    const std::vector< std::string > filenames =
+    {
+        "max_queued_events",
+        "max_user_instances",
+        "max_user_watches"
+    };
+
+    printf( "%sINotify Limits:%s\n", BCYAN, RESET );
+    for ( const std::string &fname : filenames )
+    {
+        char str[16];
+        uint32_t val = get_inotify_procfs_value( fname );
+
+        str_format_uint32(str, val);
+
+        printf( "  %-20s %s%s%s\n", fname.c_str(), BGREEN, str, RESET );
+    }
+}
+
+static uint32_t parse_config_file( const char *config_file )
+{
+    uint32_t dir_count = 0;
+
+    FILE *fp = fopen( config_file, "r" );
+    if ( fp )
+    {
+        char line_buf[ 8192 ];
+        bool in_ignore_dirs_section = false;
+
+        for ( ;; )
+        {
+            if ( !fgets( line_buf, sizeof( line_buf ) - 1, fp ) )
+                break;
+
+            if ( line_buf[0] == '#' )
+            {
+                // comment
+            }
+            else if ( !in_ignore_dirs_section )
+            {
+                size_t len = strcspn( line_buf, "\r\n" );
+
+                if ( ( len == 12 ) && !strncmp( "[ignoredirs]", line_buf, 12 ) )
+                {
+                    in_ignore_dirs_section = true;
+                }
+            }
+            else if ( line_buf[ 0 ] == '[' )
+            {
+                in_ignore_dirs_section = false;
+            }
+            else if ( in_ignore_dirs_section && ( line_buf[ 0 ] == '/' ) )
+            {
+                size_t len = strcspn( line_buf, "\r\n" );
+
+                if ( len > 1 )
+                {
+                    line_buf[ len ] = 0;
+                    if ( line_buf[ len - 1 ] != '/' )
+                    {
+                        line_buf[ len ] = '/';
+                        line_buf[ len + 1 ] = '\0';
+                    }
+
+                    ignore_dirs.push_back( line_buf );
+                    dir_count++;
+                }
+            }
+        }
+
+        fclose( fp );
+    }
+
+    return dir_count;
+}
+
+static bool parse_ignore_dirs_file()
+{
+    const std::string filename = "inotify-info.config";
+
+    const char *xdg_config_dir = getenv( "XDG_CONFIG_HOME" );
+    if ( xdg_config_dir )
+    {
+        std::string config_file = std::string( xdg_config_dir ) + "/" + filename;
+        if ( parse_config_file( config_file.c_str() ) )
+            return true;
+
+        config_file = std::string( xdg_config_dir) + "/.config/" + filename;
+        if ( parse_config_file( config_file.c_str() ) )
+            return true;
+    }
+
+    const char *home_dir = getenv( "HOME" );
+    if ( home_dir )
+    {
+        std::string config_file = std::string( home_dir ) + "/" + filename;
+        if ( parse_config_file( config_file.c_str() ) )
+            return true;
+    }
+
+    std::string config_file = "/etc/" + filename;
+    if ( parse_config_file( config_file.c_str() ) )
+        return true;
+
+    return false;
+}
+
+static void print_usage( const char *appname )
+{
+    printf( "Usage: %s [--threads=##] [appname | pid...]\n", appname );
+    printf( "    [-vv]\n" );
+    printf( "    [-?|-h|--help]\n" );
+
+    exit( -1 );
+}
+
+static void parse_cmdline( int argc, char **argv, std::vector< std::string > &cmdline_applist )
+{
+    static struct option long_opts[] =
+    {
+        { "verbose", no_argument, 0, 0 },
+        { "threads", required_argument, 0, 0 },
+        { "ignoredir", required_argument, 0, 0 },
+        { 0, 0, 0, 0 }
+    };
+
+    // Let's pick the number of processors online (with a max of 32) for a default.
+    g_numthreads = std::min< uint32_t >( g_numthreads, sysconf( _SC_NPROCESSORS_ONLN ) );
+
+    int c;
+    int opt_ind = 0;
+    while ( ( c = getopt_long( argc, argv, "m:s:?hv", long_opts, &opt_ind ) ) != -1 )
+    {
+        switch ( c )
+        {
+        case 0:
+            if ( !strcasecmp( "verbose", long_opts[ opt_ind ].name ) )
+                g_verbose++;
+            else if ( !strcasecmp( "threads", long_opts[ opt_ind ].name ) )
+                g_numthreads = atoi( optarg );
+            else if ( !strcasecmp( "ignoredir", long_opts[ opt_ind ].name ) )
+            {
+                std::string dirname = optarg;
+                if ( dirname.size() > 1 )
+                {
+                    if ( optarg[ dirname.size() - 1 ] != '/' )
+                        dirname += "/";
+                    ignore_dirs.push_back( dirname );
+                }
+            }
+            break;
+        case 'v':
+            g_verbose++;
+            break;
+        case 'h':
+        case '?':
+        default:
+            print_usage( argv[ 0 ] );
+            break;
+        }
+    }
+
+    for ( ; optind < argc; optind++ )
+    {
+        cmdline_applist.push_back( argv[ optind ] );
+    }
+
+    parse_ignore_dirs_file();
+
+    if ( g_verbose > 1 )
+    {
+        printf( "%lu ignore_dirs:\n", ignore_dirs.size() );
+
+        for ( std::string &dname : ignore_dirs )
+        {
+            printf( "  '%s'\n", dname.c_str() );
+        }
+    }
+}
+
+static void print_separator()
+{
+    printf( "%s%s%s\n", YELLOW, std::string( 78, '-' ).c_str(), RESET );
+}
+
+int main( int argc, char *argv[] )
+{
+    std::vector< std::string > cmdline_applist;
+    std::vector< procinfo_t > inotify_proclist;
+
+    struct lconv *env = localeconv();
+    if (env && env->thousands_sep && env->thousands_sep[0])
+    {
+        thousands_sep = env->thousands_sep[0];
+    }
+
+    parse_cmdline( argc, argv, cmdline_applist );
+    print_separator();
+
+    print_inotify_limits();
+    print_separator();
+
+    if ( init_inotify_proclist( inotify_proclist ) )
+    {
+        uint32_t total_watches = 0;
+        uint32_t total_instances = 0;
+        std::vector< filename_info_t > all_found_files;
+
+        for ( procinfo_t &procinfo : inotify_proclist )
+        {
+            procinfo.in_cmd_line = is_proc_in_cmdline_applist( procinfo, cmdline_applist );
+
+            total_watches += procinfo.watches;
+            total_instances += procinfo.instances;
+        }
+
+        if (inotify_proclist.size()) {
+            print_inotify_proclist( inotify_proclist );
+            print_separator();
+        }
+
+        if (g_kernel_provides_watches_info)
+            printf( "Total inotify Watches:   %s%u%s\n", BGREEN, total_watches, RESET );
+        printf( "Total inotify Instances: %s%u%s\n", BGREEN, total_instances, RESET );
+        print_separator();
+
+        double search_time = gettime();
+        uint32_t total_scanned_dirs = find_files_in_inode_set( inotify_proclist, all_found_files );
+        if ( total_scanned_dirs )
+        {
+            search_time = gettime() - search_time;
+
+            for ( const filename_info_t &fname_info : all_found_files )
+            {
+                printf( "%s%9lu%s [%u:%u] %s\n", BGREEN, fname_info.inode, RESET,
+                        major( fname_info.dev ), minor( fname_info.dev ),
+                        fname_info.filename.c_str() );
+            }
+
+            setlocale( LC_NUMERIC, "" );
+GCC_DIAG_PUSH_OFF( format )
+            printf( "\n%'u dirs scanned (%.2f seconds)\n", total_scanned_dirs, search_time );
+GCC_DIAG_POP()
+        }
+    }
+
+    return 0;
+}
diff --git a/inotify-info.h b/inotify-info.h
new file mode 100644
index 0000000..fe42811
--- /dev/null
+++ b/inotify-info.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2021 Michael Sartain
+ *
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#define RESET    "\x1b[0m"
+#define GRAY     "\x1b[0;30m"
+#define RED      "\x1b[0;31m"
+#define GREEN    "\x1b[0;32m"
+#define YELLOW   "\x1b[0;33m"
+#define BLUE     "\x1b[0;34m"
+#define MAGENTA  "\x1b[0;35m"
+#define CYAN     "\x1b[0;36m"
+#define WHITE    "\x1b[0;37m"
+#define BGRAY    "\x1b[1;30m"
+#define BRED     "\x1b[1;31m"
+#define BGREEN   "\x1b[1;32m"
+#define BYELLOW  "\x1b[1;33m"
+#define BBLUE    "\x1b[1;34m"
+#define BMAGENTA "\x1b[1;35m"
+#define BCYAN    "\x1b[1;36m"
+#define BWHITE   "\x1b[1;37m"
+
+#define TO_STR( x ) #x
+#define TO_STR_VALUE( x ) TO_STR( x )
+
+#define ATTRIBUTE_PRINTF( _x, _y ) __attribute__( ( __format__( __printf__, _x, _y ) ) )
+
+#define GCC_DIAG_STR( s ) #s
+#define GCC_DIAG_JOINSTR( x, y ) GCC_DIAG_STR( x##y )
+#define GCC_DIAG_DO_PRAGMA( x ) _Pragma( #x )
+#define GCC_DIAG_PRAGMA( x ) GCC_DIAG_DO_PRAGMA( GCC diagnostic x )
+
+#define GCC_DIAG_PUSH_OFF( x ) \
+    GCC_DIAG_PRAGMA( push )    \
+    GCC_DIAG_PRAGMA( ignored GCC_DIAG_JOINSTR( -W, x ) )
+#define GCC_DIAG_POP() GCC_DIAG_PRAGMA( pop )
+
+std::string string_formatv(const char *fmt, va_list ap) ATTRIBUTE_PRINTF(1, 0);
+std::string string_format(const char *fmt, ...) ATTRIBUTE_PRINTF(1, 2);
+
diff --git a/lfqueue/LICENSE b/lfqueue/LICENSE
new file mode 100644
index 0000000..76f17ad
--- /dev/null
+++ b/lfqueue/LICENSE
@@ -0,0 +1,25 @@
+BSD 2-Clause License
+
+Copyright (c) 2018, Taymindis Woon
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/lfqueue/lfqueue.c b/lfqueue/lfqueue.c
new file mode 100644
index 0000000..17b8ae3
--- /dev/null
+++ b/lfqueue/lfqueue.c
@@ -0,0 +1,379 @@
+/*
+*
+* BSD 2-Clause License
+*
+* Copyright (c) 2018, Taymindis Woon
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are met:
+*
+* * Redistributions of source code must retain the above copyright notice, this
+*   list of conditions and the following disclaimer.
+*
+* * Redistributions in binary form must reproduce the above copyright notice,
+*   this list of conditions and the following disclaimer in the documentation
+*   and/or other materials provided with the distribution.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#if defined __GNUC__ || defined __CYGWIN__ || defined __MINGW32__ || defined __APPLE__
+
+#include <sys/time.h>
+#include <unistd.h> // for usleep
+#include <sched.h>
+
+#define __LFQ_VAL_COMPARE_AND_SWAP __sync_val_compare_and_swap
+#define __LFQ_BOOL_COMPARE_AND_SWAP __sync_bool_compare_and_swap
+#define __LFQ_FETCH_AND_ADD __sync_fetch_and_add
+#define __LFQ_ADD_AND_FETCH __sync_add_and_fetch
+#define __LFQ_YIELD_THREAD sched_yield
+#define __LFQ_SYNC_MEMORY __sync_synchronize
+
+#else
+
+#include <Windows.h>
+#include <time.h>
+#ifdef _WIN64
+inline BOOL __SYNC_BOOL_CAS(LONG64 volatile *dest, LONG64 input, LONG64 comparand) {
+	return InterlockedCompareExchangeNoFence64(dest, input, comparand) == comparand;
+}
+#define __LFQ_VAL_COMPARE_AND_SWAP(dest, comparand, input) \
+    InterlockedCompareExchangeNoFence64((LONG64 volatile *)dest, (LONG64)input, (LONG64)comparand)
+#define __LFQ_BOOL_COMPARE_AND_SWAP(dest, comparand, input) \
+    __SYNC_BOOL_CAS((LONG64 volatile *)dest, (LONG64)input, (LONG64)comparand)
+#define __LFQ_FETCH_AND_ADD InterlockedExchangeAddNoFence64
+#define __LFQ_ADD_AND_FETCH InterlockedAddNoFence64
+#define __LFQ_SYNC_MEMORY MemoryBarrier
+
+#else
+#ifndef asm
+#define asm __asm
+#endif
+inline BOOL __SYNC_BOOL_CAS(LONG volatile *dest, LONG input, LONG comparand) {
+	return InterlockedCompareExchangeNoFence(dest, input, comparand) == comparand;
+}
+#define __LFQ_VAL_COMPARE_AND_SWAP(dest, comparand, input) \
+    InterlockedCompareExchangeNoFence((LONG volatile *)dest, (LONG)input, (LONG)comparand)
+#define __LFQ_BOOL_COMPARE_AND_SWAP(dest, comparand, input) \
+    __SYNC_BOOL_CAS((LONG volatile *)dest, (LONG)input, (LONG)comparand)
+#define __LFQ_FETCH_AND_ADD InterlockedExchangeAddNoFence
+#define __LFQ_ADD_AND_FETCH InterlockedAddNoFence
+#define __LFQ_SYNC_MEMORY() asm mfence
+
+#endif
+#include <windows.h>
+#define __LFQ_YIELD_THREAD SwitchToThread
+#endif
+
+#include "lfqueue.h"
+#define DEF_LFQ_ASSIGNED_SPIN 2048
+
+#if defined __GNUC__ || defined __CYGWIN__ || defined __MINGW32__ || defined __APPLE__
+#define lfq_time_t long
+#define lfq_get_curr_time(_time_sec) \
+struct timeval _time_; \
+gettimeofday(&_time_, NULL); \
+*_time_sec = _time_.tv_sec
+#define lfq_diff_time(_etime_, _stime_) _etime_ - _stime_
+#else
+#define lfq_time_t time_t
+#define lfq_get_curr_time(_time_sec) time(_time_sec)
+#define lfq_diff_time(_etime_, _stime_) difftime(_etime_, _stime_)
+#endif
+
+struct lfqueue_cas_node_s {
+	void * value;
+	struct lfqueue_cas_node_s *next, *nextfree;
+	lfq_time_t _deactivate_tm;
+};
+
+//static lfqueue_cas_node_t* __lfq_assigned(lfqueue_t *);
+static void __lfq_recycle_free(lfqueue_t *, lfqueue_cas_node_t*);
+static void __lfq_check_free(lfqueue_t *);
+static void *_dequeue(lfqueue_t *);
+static void *_single_dequeue(lfqueue_t *);
+static int _enqueue(lfqueue_t *, void* );
+static inline void* _lfqueue_malloc(__attribute__ ((unused)) void* pl, size_t sz) {
+	return malloc(sz);
+}
+static inline void _lfqueue_free(__attribute__ ((unused)) void* pl, void* ptr) {
+	free(ptr);
+}
+
+static void *
+_dequeue(lfqueue_t *lfqueue) {
+	lfqueue_cas_node_t *head, *next;
+	void *val;
+
+	for (;;) {
+		head = lfqueue->head;
+		if (__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->head, head, head)) {
+			next = head->next;
+			if (__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->tail, head, head)) {
+				if (next == NULL) {
+					val = NULL;
+					goto _done;
+				}
+			}
+			else {
+				if (next) {
+					val = next->value;
+					if (__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->head, head, next)) {
+						break;
+					}
+				} else {
+					val = NULL;
+					goto _done;
+				}
+			}
+		}
+	}
+
+	__lfq_recycle_free(lfqueue, head);
+_done:
+	// __asm volatile("" ::: "memory");
+	__LFQ_SYNC_MEMORY();
+	__lfq_check_free(lfqueue);
+	return val;
+}
+
+static void *
+_single_dequeue(lfqueue_t *lfqueue) {
+	lfqueue_cas_node_t *head, *next;
+	void *val;
+
+	for (;;) {
+		head = lfqueue->head;
+		if (__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->head, head, head)) {
+			next = head->next;
+			if (__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->tail, head, head)) {
+				if (next == NULL) {
+					return NULL;
+				}
+			}
+			else {
+				if (next) {
+					val = next->value;
+					if (__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->head, head, next)) {
+						lfqueue->_free(lfqueue->pl, head);
+						break;
+					}
+				} else {
+					return NULL;
+				}
+			}
+		}
+	}
+	return val;
+}
+
+static int
+_enqueue(lfqueue_t *lfqueue, void* value) {
+	lfqueue_cas_node_t *tail, *node;
+	node = (lfqueue_cas_node_t*) lfqueue->_malloc(lfqueue->pl, sizeof(lfqueue_cas_node_t));
+	if (node == NULL) {
+		perror("malloc");
+		return errno;
+	}
+	node->value = value;
+	node->next = NULL;
+	node->nextfree = NULL;
+	for (;;) {
+		__LFQ_SYNC_MEMORY();
+		tail = lfqueue->tail;
+		if (__LFQ_BOOL_COMPARE_AND_SWAP(&tail->next, NULL, node)) {
+			// compulsory swap as tail->next is no NULL anymore, it has fenced on other thread
+			__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->tail, tail, node);
+			__lfq_check_free(lfqueue);
+			return 0;
+		}
+	}
+
+	/*It never be here*/
+	return -1;
+}
+
+static void
+__lfq_recycle_free(lfqueue_t *lfqueue, lfqueue_cas_node_t* freenode) {
+	lfqueue_cas_node_t *freed;
+	do {
+		freed = lfqueue->move_free;
+	} while (!__LFQ_BOOL_COMPARE_AND_SWAP(&freed->nextfree, NULL, freenode) );
+
+	lfq_get_curr_time(&freenode->_deactivate_tm);
+
+	__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->move_free, freed, freenode);
+}
+
+static void
+__lfq_check_free(lfqueue_t *lfqueue) {
+	lfq_time_t curr_time;
+	if (__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->in_free_mode, 0, 1)) {
+		lfq_get_curr_time(&curr_time);
+		lfqueue_cas_node_t *rtfree = lfqueue->root_free, *nextfree;
+		while ( rtfree && (rtfree != lfqueue->move_free) ) {
+			nextfree = rtfree->nextfree;
+			if ( lfq_diff_time(curr_time, rtfree->_deactivate_tm) > 2) {
+				//	printf("%p\n", rtfree);
+				lfqueue->_free(lfqueue->pl, rtfree);
+				rtfree = nextfree;
+			} else {
+				break;
+			}
+		}
+		lfqueue->root_free = rtfree;
+		__LFQ_BOOL_COMPARE_AND_SWAP(&lfqueue->in_free_mode, 1, 0);
+	}
+	__LFQ_SYNC_MEMORY();
+}
+
+int
+lfqueue_init(lfqueue_t *lfqueue) {
+	return lfqueue_init_mf(lfqueue, NULL, _lfqueue_malloc, _lfqueue_free);
+}
+
+int
+lfqueue_init_mf(lfqueue_t *lfqueue, void* pl, lfqueue_malloc_fn lfqueue_malloc, lfqueue_free_fn lfqueue_free) {
+	lfqueue->_malloc = lfqueue_malloc;
+	lfqueue->_free = lfqueue_free;
+	lfqueue->pl = pl;
+
+	lfqueue_cas_node_t *base = lfqueue_malloc(pl, sizeof(lfqueue_cas_node_t));
+	lfqueue_cas_node_t *freebase = lfqueue_malloc(pl, sizeof(lfqueue_cas_node_t));
+	if (base == NULL || freebase == NULL) {
+		perror("malloc");
+		return errno;
+	}
+	base->value = NULL;
+	base->next = NULL;
+	base->nextfree = NULL;
+	base->_deactivate_tm = 0;
+
+	freebase->value = NULL;
+	freebase->next = NULL;
+	freebase->nextfree = NULL;
+	freebase->_deactivate_tm = 0;
+
+	lfqueue->head = lfqueue->tail = base; // Not yet to be free for first node only
+	lfqueue->root_free = lfqueue->move_free = freebase; // Not yet to be free for first node only
+	lfqueue->size = 0;
+	lfqueue->in_free_mode = 0;
+
+	return 0;
+}
+
+void
+lfqueue_destroy(lfqueue_t *lfqueue) {
+	void* p;
+	while ((p = lfqueue_deq(lfqueue))) {
+		lfqueue->_free(lfqueue->pl, p);
+	}
+	// Clear the recycle chain nodes
+	lfqueue_cas_node_t *rtfree = lfqueue->root_free, *nextfree;
+	while (rtfree && (rtfree != lfqueue->move_free) ) {
+		nextfree = rtfree->nextfree;
+		lfqueue->_free(lfqueue->pl, rtfree);
+		rtfree = nextfree;
+	}
+	if (rtfree) {
+		lfqueue->_free(lfqueue->pl, rtfree);
+	}
+
+	lfqueue->_free(lfqueue->pl, lfqueue->tail); // Last free
+
+	lfqueue->size = 0;
+}
+
+int
+lfqueue_enq(lfqueue_t *lfqueue, void *value) {
+	if (_enqueue(lfqueue, value)) {
+		return -1;
+	}
+	__LFQ_ADD_AND_FETCH(&lfqueue->size, 1);
+	return 0;
+}
+
+void*
+lfqueue_deq(lfqueue_t *lfqueue) {
+	void *v;
+	if (//__LFQ_ADD_AND_FETCH(&lfqueue->size, 0) &&
+	    (v = _dequeue(lfqueue))
+	) {
+
+		__LFQ_FETCH_AND_ADD(&lfqueue->size, -1);
+		return v;
+	}
+	return NULL;
+}
+
+void*
+lfqueue_deq_must(lfqueue_t *lfqueue) {
+	void *v;
+	while ( !(v = _dequeue(lfqueue)) ) {
+		// Rest the thread for other thread, to avoid keep looping force
+		lfqueue_sleep(1);
+	}
+	__LFQ_FETCH_AND_ADD(&lfqueue->size, -1);
+	return v;
+}
+
+/**This is only applicable when only single thread consume only**/
+void*
+lfqueue_single_deq(lfqueue_t *lfqueue) {
+	void *v;
+	if (//__LFQ_ADD_AND_FETCH(&lfqueue->size, 0) &&
+	    (v = _single_dequeue(lfqueue))
+	) {
+
+		__LFQ_FETCH_AND_ADD(&lfqueue->size, -1);
+		return v;
+	}
+	return NULL;
+}
+
+/**This is only applicable when only single thread consume only**/
+void*
+lfqueue_single_deq_must(lfqueue_t *lfqueue) {
+	void *v;
+	while ( !(v = _single_dequeue(lfqueue)) ) {
+		// Rest the thread for other thread, to avoid keep looping force
+		lfqueue_sleep(1);
+	}
+	__LFQ_FETCH_AND_ADD(&lfqueue->size, -1);
+	return v;
+}
+
+size_t
+lfqueue_size(lfqueue_t *lfqueue) {
+	return __LFQ_ADD_AND_FETCH(&lfqueue->size, 0);
+}
+
+void
+lfqueue_sleep(unsigned int milisec) {
+#if defined __GNUC__ || defined __CYGWIN__ || defined __MINGW32__ || defined __APPLE__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wimplicit-function-declaration"
+	usleep(milisec * 1000);
+#pragma GCC diagnostic pop
+#else
+	Sleep(milisec);
+#endif
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/lfqueue/lfqueue.h b/lfqueue/lfqueue.h
new file mode 100644
index 0000000..1eafc7b
--- /dev/null
+++ b/lfqueue/lfqueue.h
@@ -0,0 +1,84 @@
+/*
+*
+* BSD 2-Clause License
+*
+* Copyright (c) 2018, Taymindis Woon
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are met:
+*
+* * Redistributions of source code must retain the above copyright notice, this
+*   list of conditions and the following disclaimer.
+*
+* * Redistributions in binary form must reproduce the above copyright notice,
+*   this list of conditions and the following disclaimer in the documentation
+*   and/or other materials provided with the distribution.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+*/
+
+#ifndef LFQUEUE_H
+#define LFQUEUE_H
+
+#include <stdlib.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct lfqueue_cas_node_s lfqueue_cas_node_t;
+typedef void* (*lfqueue_malloc_fn)(void*, size_t);
+typedef void (*lfqueue_free_fn)(void*, void*);
+
+#if defined __GNUC__ || defined __CYGWIN__ || defined __MINGW32__ || defined __APPLE__
+#define lfq_bool_t int
+#else
+#ifdef _WIN64
+#define lfq_bool_t int64_t
+#else
+#define lfq_bool_t int
+#endif
+#endif
+
+typedef struct {
+	lfqueue_cas_node_t *head, *tail, *root_free, *move_free;
+	volatile size_t size;
+	volatile lfq_bool_t in_free_mode;
+	lfqueue_malloc_fn _malloc;
+	lfqueue_free_fn _free;
+	void *pl;
+} lfqueue_t;
+
+extern int   lfqueue_init(lfqueue_t *lfqueue);
+extern int   lfqueue_init_mf(lfqueue_t *lfqueue, void* pl, lfqueue_malloc_fn lfqueue_malloc, lfqueue_free_fn lfqueue_free);
+extern int   lfqueue_enq(lfqueue_t *lfqueue, void *value);
+extern void* lfqueue_deq(lfqueue_t *lfqueue);
+extern void* lfqueue_single_deq(lfqueue_t *lfqueue);
+
+/** loop until value been dequeue, it sleeps 1ms if not found to reduce cpu high usage **/
+extern void* lfqueue_deq_must(lfqueue_t *lfqueue);
+extern void* lfqueue_single_deq_must(lfqueue_t *lfqueue);
+
+extern void lfqueue_destroy(lfqueue_t *lfqueue);
+extern size_t lfqueue_size(lfqueue_t *lfqueue);
+extern void lfqueue_sleep(unsigned int milisec);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+