summaryrefslogtreecommitdiffstats
path: root/collectors
diff options
context:
space:
mode:
Diffstat (limited to 'collectors')
-rw-r--r--collectors/COLLECTORS.md518
-rw-r--r--collectors/Makefile.am40
-rw-r--r--collectors/README.md48
-rw-r--r--collectors/REFERENCE.md170
-rw-r--r--collectors/all.h367
-rw-r--r--collectors/apps.plugin/Makefile.am12
-rw-r--r--collectors/apps.plugin/README.md399
-rw-r--r--collectors/apps.plugin/apps_groups.conf422
-rw-r--r--collectors/apps.plugin/apps_plugin.c5015
-rw-r--r--collectors/cgroups.plugin/Makefile.am13
-rw-r--r--collectors/cgroups.plugin/README.md308
-rwxr-xr-xcollectors/cgroups.plugin/cgroup-name.sh597
-rwxr-xr-xcollectors/cgroups.plugin/cgroup-network-helper.sh302
-rw-r--r--collectors/cgroups.plugin/cgroup-network.c723
-rw-r--r--collectors/cgroups.plugin/sys_fs_cgroup.c4887
-rw-r--r--collectors/cgroups.plugin/sys_fs_cgroup.h44
-rw-r--r--collectors/cgroups.plugin/tests/test_cgroups_plugin.c131
-rw-r--r--collectors/cgroups.plugin/tests/test_cgroups_plugin.h16
-rw-r--r--collectors/cgroups.plugin/tests/test_doubles.c157
-rw-r--r--collectors/charts.d.plugin/Makefile.am50
-rw-r--r--collectors/charts.d.plugin/README.md198
-rw-r--r--collectors/charts.d.plugin/ap/Makefile.inc13
-rw-r--r--collectors/charts.d.plugin/ap/README.md99
-rw-r--r--collectors/charts.d.plugin/ap/ap.chart.sh179
-rw-r--r--collectors/charts.d.plugin/ap/ap.conf23
-rw-r--r--collectors/charts.d.plugin/apcupsd/Makefile.inc13
-rw-r--r--collectors/charts.d.plugin/apcupsd/README.md21
-rw-r--r--collectors/charts.d.plugin/apcupsd/apcupsd.chart.sh223
-rw-r--r--collectors/charts.d.plugin/apcupsd/apcupsd.conf25
-rw-r--r--collectors/charts.d.plugin/charts.d.conf48
-rwxr-xr-xcollectors/charts.d.plugin/charts.d.dryrun-helper.sh72
-rwxr-xr-xcollectors/charts.d.plugin/charts.d.plugin.in732
-rw-r--r--collectors/charts.d.plugin/example/Makefile.inc13
-rw-r--r--collectors/charts.d.plugin/example/README.md10
-rw-r--r--collectors/charts.d.plugin/example/example.chart.sh123
-rw-r--r--collectors/charts.d.plugin/example/example.conf21
-rw-r--r--collectors/charts.d.plugin/libreswan/Makefile.inc13
-rw-r--r--collectors/charts.d.plugin/libreswan/README.md56
-rw-r--r--collectors/charts.d.plugin/libreswan/libreswan.chart.sh187
-rw-r--r--collectors/charts.d.plugin/libreswan/libreswan.conf29
-rw-r--r--collectors/charts.d.plugin/loopsleepms.sh.inc227
-rw-r--r--collectors/charts.d.plugin/nut/Makefile.inc13
-rw-r--r--collectors/charts.d.plugin/nut/README.md74
-rw-r--r--collectors/charts.d.plugin/nut/nut.chart.sh232
-rw-r--r--collectors/charts.d.plugin/nut/nut.conf33
-rw-r--r--collectors/charts.d.plugin/opensips/Makefile.inc13
-rw-r--r--collectors/charts.d.plugin/opensips/README.md19
-rw-r--r--collectors/charts.d.plugin/opensips/opensips.chart.sh325
-rw-r--r--collectors/charts.d.plugin/opensips/opensips.conf21
-rw-r--r--collectors/charts.d.plugin/sensors/Makefile.inc13
-rw-r--r--collectors/charts.d.plugin/sensors/README.md79
-rw-r--r--collectors/charts.d.plugin/sensors/sensors.chart.sh250
-rw-r--r--collectors/charts.d.plugin/sensors/sensors.conf32
-rw-r--r--collectors/cups.plugin/Makefile.am8
-rw-r--r--collectors/cups.plugin/README.md64
-rw-r--r--collectors/cups.plugin/cups_plugin.c439
-rw-r--r--collectors/diskspace.plugin/Makefile.am8
-rw-r--r--collectors/diskspace.plugin/README.md43
-rw-r--r--collectors/diskspace.plugin/plugin_diskspace.c690
-rw-r--r--collectors/ebpf.plugin/Makefile.am42
-rw-r--r--collectors/ebpf.plugin/README.md970
-rw-r--r--collectors/ebpf.plugin/ebpf.c2249
-rw-r--r--collectors/ebpf.plugin/ebpf.d.conf69
-rw-r--r--collectors/ebpf.plugin/ebpf.d/cachestat.conf36
-rw-r--r--collectors/ebpf.plugin/ebpf.d/dcstat.conf34
-rw-r--r--collectors/ebpf.plugin/ebpf.d/disk.conf9
-rw-r--r--collectors/ebpf.plugin/ebpf.d/ebpf_kernel_reject_list.txt1
-rw-r--r--collectors/ebpf.plugin/ebpf.d/fd.conf21
-rw-r--r--collectors/ebpf.plugin/ebpf.d/filesystem.conf20
-rw-r--r--collectors/ebpf.plugin/ebpf.d/hardirq.conf8
-rw-r--r--collectors/ebpf.plugin/ebpf.d/mdflush.conf7
-rw-r--r--collectors/ebpf.plugin/ebpf.d/mount.conf19
-rw-r--r--collectors/ebpf.plugin/ebpf.d/network.conf53
-rw-r--r--collectors/ebpf.plugin/ebpf.d/oomkill.conf7
-rw-r--r--collectors/ebpf.plugin/ebpf.d/process.conf25
-rw-r--r--collectors/ebpf.plugin/ebpf.d/shm.conf36
-rw-r--r--collectors/ebpf.plugin/ebpf.d/softirq.conf8
-rw-r--r--collectors/ebpf.plugin/ebpf.d/swap.conf28
-rw-r--r--collectors/ebpf.plugin/ebpf.d/sync.conf36
-rw-r--r--collectors/ebpf.plugin/ebpf.d/vfs.conf19
-rw-r--r--collectors/ebpf.plugin/ebpf.h299
-rw-r--r--collectors/ebpf.plugin/ebpf_apps.c1155
-rw-r--r--collectors/ebpf.plugin/ebpf_apps.h441
-rw-r--r--collectors/ebpf.plugin/ebpf_cachestat.c1335
-rw-r--r--collectors/ebpf.plugin/ebpf_cachestat.h90
-rw-r--r--collectors/ebpf.plugin/ebpf_cgroup.c317
-rw-r--r--collectors/ebpf.plugin/ebpf_cgroup.h69
-rw-r--r--collectors/ebpf.plugin/ebpf_dcstat.c1225
-rw-r--r--collectors/ebpf.plugin/ebpf_dcstat.h84
-rw-r--r--collectors/ebpf.plugin/ebpf_disk.c865
-rw-r--r--collectors/ebpf.plugin/ebpf_disk.h78
-rw-r--r--collectors/ebpf.plugin/ebpf_fd.c1183
-rw-r--r--collectors/ebpf.plugin/ebpf_fd.h85
-rw-r--r--collectors/ebpf.plugin/ebpf_filesystem.c638
-rw-r--r--collectors/ebpf.plugin/ebpf_filesystem.h49
-rw-r--r--collectors/ebpf.plugin/ebpf_hardirq.c507
-rw-r--r--collectors/ebpf.plugin/ebpf_hardirq.h73
-rw-r--r--collectors/ebpf.plugin/ebpf_mdflush.c334
-rw-r--r--collectors/ebpf.plugin/ebpf_mdflush.h42
-rw-r--r--collectors/ebpf.plugin/ebpf_mount.c517
-rw-r--r--collectors/ebpf.plugin/ebpf_mount.h44
-rw-r--r--collectors/ebpf.plugin/ebpf_oomkill.c402
-rw-r--r--collectors/ebpf.plugin/ebpf_oomkill.h32
-rw-r--r--collectors/ebpf.plugin/ebpf_process.c1327
-rw-r--r--collectors/ebpf.plugin/ebpf_process.h107
-rw-r--r--collectors/ebpf.plugin/ebpf_shm.c1137
-rw-r--r--collectors/ebpf.plugin/ebpf_shm.h63
-rw-r--r--collectors/ebpf.plugin/ebpf_socket.c3976
-rw-r--r--collectors/ebpf.plugin/ebpf_socket.h370
-rw-r--r--collectors/ebpf.plugin/ebpf_softirq.c290
-rw-r--r--collectors/ebpf.plugin/ebpf_softirq.h34
-rw-r--r--collectors/ebpf.plugin/ebpf_swap.c915
-rw-r--r--collectors/ebpf.plugin/ebpf_swap.h53
-rw-r--r--collectors/ebpf.plugin/ebpf_sync.c608
-rw-r--r--collectors/ebpf.plugin/ebpf_sync.h59
-rw-r--r--collectors/ebpf.plugin/ebpf_vfs.c1983
-rw-r--r--collectors/ebpf.plugin/ebpf_vfs.h177
-rw-r--r--collectors/fping.plugin/Makefile.am24
-rw-r--r--collectors/fping.plugin/README.md110
-rw-r--r--collectors/fping.plugin/fping.conf44
-rwxr-xr-xcollectors/fping.plugin/fping.plugin.in202
-rw-r--r--collectors/freebsd.plugin/Makefile.am8
-rw-r--r--collectors/freebsd.plugin/README.md12
-rw-r--r--collectors/freebsd.plugin/freebsd_devstat.c759
-rw-r--r--collectors/freebsd.plugin/freebsd_getifaddrs.c599
-rw-r--r--collectors/freebsd.plugin/freebsd_getmntinfo.c299
-rw-r--r--collectors/freebsd.plugin/freebsd_ipfw.c359
-rw-r--r--collectors/freebsd.plugin/freebsd_kstat_zfs.c304
-rw-r--r--collectors/freebsd.plugin/freebsd_sysctl.c3062
-rw-r--r--collectors/freebsd.plugin/plugin_freebsd.c136
-rw-r--r--collectors/freebsd.plugin/plugin_freebsd.h55
-rw-r--r--collectors/freeipmi.plugin/Makefile.am8
-rw-r--r--collectors/freeipmi.plugin/README.md202
-rw-r--r--collectors/freeipmi.plugin/freeipmi_plugin.c1857
-rw-r--r--collectors/idlejitter.plugin/Makefile.am8
-rw-r--r--collectors/idlejitter.plugin/README.md32
-rw-r--r--collectors/idlejitter.plugin/plugin_idlejitter.c93
-rw-r--r--collectors/ioping.plugin/Makefile.am24
-rw-r--r--collectors/ioping.plugin/README.md86
-rw-r--r--collectors/ioping.plugin/ioping.conf40
-rwxr-xr-xcollectors/ioping.plugin/ioping.plugin.in210
-rw-r--r--collectors/macos.plugin/Makefile.am8
-rw-r--r--collectors/macos.plugin/README.md12
-rw-r--r--collectors/macos.plugin/macos_fw.c648
-rw-r--r--collectors/macos.plugin/macos_mach_smi.c227
-rw-r--r--collectors/macos.plugin/macos_sysctl.c1424
-rw-r--r--collectors/macos.plugin/plugin_macos.c81
-rw-r--r--collectors/macos.plugin/plugin_macos.h12
-rw-r--r--collectors/nfacct.plugin/Makefile.am8
-rw-r--r--collectors/nfacct.plugin/README.md54
-rw-r--r--collectors/nfacct.plugin/plugin_nfacct.c886
-rw-r--r--collectors/perf.plugin/Makefile.am8
-rw-r--r--collectors/perf.plugin/README.md80
-rw-r--r--collectors/perf.plugin/perf_plugin.c1353
-rw-r--r--collectors/plugins.d/Makefile.am11
-rw-r--r--collectors/plugins.d/README.md599
-rw-r--r--collectors/plugins.d/plugins_d.c290
-rw-r--r--collectors/plugins.d/plugins_d.h103
-rw-r--r--collectors/plugins.d/pluginsd_parser.c1360
-rw-r--r--collectors/plugins.d/pluginsd_parser.h39
-rw-r--r--collectors/proc.plugin/Makefile.am8
-rw-r--r--collectors/proc.plugin/README.md607
-rw-r--r--collectors/proc.plugin/ipc.c554
-rw-r--r--collectors/proc.plugin/plugin_proc.c189
-rw-r--r--collectors/proc.plugin/plugin_proc.h66
-rw-r--r--collectors/proc.plugin/proc_diskstats.c2045
-rw-r--r--collectors/proc.plugin/proc_interrupts.c245
-rw-r--r--collectors/proc.plugin/proc_loadavg.c126
-rw-r--r--collectors/proc.plugin/proc_mdstat.c640
-rw-r--r--collectors/proc.plugin/proc_meminfo.c513
-rw-r--r--collectors/proc.plugin/proc_net_dev.c1522
-rw-r--r--collectors/proc.plugin/proc_net_ip_vs_stats.c123
-rw-r--r--collectors/proc.plugin/proc_net_netstat.c3110
-rw-r--r--collectors/proc.plugin/proc_net_rpc_nfs.c439
-rw-r--r--collectors/proc.plugin/proc_net_rpc_nfsd.c763
-rw-r--r--collectors/proc.plugin/proc_net_sctp_snmp.c367
-rw-r--r--collectors/proc.plugin/proc_net_sockstat.c529
-rw-r--r--collectors/proc.plugin/proc_net_sockstat6.c278
-rw-r--r--collectors/proc.plugin/proc_net_softnet_stat.c149
-rw-r--r--collectors/proc.plugin/proc_net_stat_conntrack.c345
-rw-r--r--collectors/proc.plugin/proc_net_stat_synproxy.c153
-rw-r--r--collectors/proc.plugin/proc_net_wireless.c432
-rw-r--r--collectors/proc.plugin/proc_pagetypeinfo.c338
-rw-r--r--collectors/proc.plugin/proc_pressure.c209
-rw-r--r--collectors/proc.plugin/proc_pressure.h43
-rw-r--r--collectors/proc.plugin/proc_self_mountinfo.c458
-rw-r--r--collectors/proc.plugin/proc_self_mountinfo.h61
-rw-r--r--collectors/proc.plugin/proc_softirqs.c243
-rw-r--r--collectors/proc.plugin/proc_spl_kstat_zfs.c423
-rw-r--r--collectors/proc.plugin/proc_stat.c1064
-rw-r--r--collectors/proc.plugin/proc_sys_kernel_random_entropy_avail.c47
-rw-r--r--collectors/proc.plugin/proc_uptime.c42
-rw-r--r--collectors/proc.plugin/proc_vmstat.c311
-rw-r--r--collectors/proc.plugin/sys_block_zram.c279
-rw-r--r--collectors/proc.plugin/sys_class_infiniband.c703
-rw-r--r--collectors/proc.plugin/sys_class_power_supply.c414
-rw-r--r--collectors/proc.plugin/sys_devices_system_edac_mc.c204
-rw-r--r--collectors/proc.plugin/sys_devices_system_node.c165
-rw-r--r--collectors/proc.plugin/sys_fs_btrfs.c743
-rw-r--r--collectors/proc.plugin/sys_kernel_mm_ksm.c194
-rw-r--r--collectors/proc.plugin/zfs_common.c960
-rw-r--r--collectors/proc.plugin/zfs_common.h115
-rw-r--r--collectors/python.d.plugin/Makefile.am236
-rw-r--r--collectors/python.d.plugin/README.md270
-rw-r--r--collectors/python.d.plugin/adaptec_raid/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/adaptec_raid/README.md80
-rw-r--r--collectors/python.d.plugin/adaptec_raid/adaptec_raid.chart.py247
-rw-r--r--collectors/python.d.plugin/adaptec_raid/adaptec_raid.conf53
-rw-r--r--collectors/python.d.plugin/alarms/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/alarms/README.md66
-rw-r--r--collectors/python.d.plugin/alarms/alarms.chart.py95
-rw-r--r--collectors/python.d.plugin/alarms/alarms.conf60
-rw-r--r--collectors/python.d.plugin/am2320/Makefile.inc8
-rw-r--r--collectors/python.d.plugin/am2320/README.md53
-rw-r--r--collectors/python.d.plugin/am2320/am2320.chart.py68
-rw-r--r--collectors/python.d.plugin/am2320/am2320.conf68
-rw-r--r--collectors/python.d.plugin/anomalies/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/anomalies/README.md245
-rw-r--r--collectors/python.d.plugin/anomalies/anomalies.chart.py426
-rw-r--r--collectors/python.d.plugin/anomalies/anomalies.conf184
-rw-r--r--collectors/python.d.plugin/beanstalk/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/beanstalk/README.md133
-rw-r--r--collectors/python.d.plugin/beanstalk/beanstalk.chart.py252
-rw-r--r--collectors/python.d.plugin/beanstalk/beanstalk.conf78
-rw-r--r--collectors/python.d.plugin/bind_rndc/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/bind_rndc/README.md79
-rw-r--r--collectors/python.d.plugin/bind_rndc/bind_rndc.chart.py252
-rw-r--r--collectors/python.d.plugin/bind_rndc/bind_rndc.conf110
-rw-r--r--collectors/python.d.plugin/boinc/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/boinc/README.md41
-rw-r--r--collectors/python.d.plugin/boinc/boinc.chart.py168
-rw-r--r--collectors/python.d.plugin/boinc/boinc.conf66
-rw-r--r--collectors/python.d.plugin/ceph/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/ceph/README.md48
-rw-r--r--collectors/python.d.plugin/ceph/ceph.chart.py374
-rw-r--r--collectors/python.d.plugin/ceph/ceph.conf75
-rw-r--r--collectors/python.d.plugin/changefinder/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/changefinder/README.md217
-rw-r--r--collectors/python.d.plugin/changefinder/changefinder.chart.py185
-rw-r--r--collectors/python.d.plugin/changefinder/changefinder.conf74
-rw-r--r--collectors/python.d.plugin/dockerd/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/dockerd/README.md46
-rw-r--r--collectors/python.d.plugin/dockerd/dockerd.chart.py86
-rw-r--r--collectors/python.d.plugin/dockerd/dockerd.conf77
-rw-r--r--collectors/python.d.plugin/dovecot/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/dovecot/README.md105
-rw-r--r--collectors/python.d.plugin/dovecot/dovecot.chart.py143
-rw-r--r--collectors/python.d.plugin/dovecot/dovecot.conf98
-rw-r--r--collectors/python.d.plugin/example/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/example/README.md14
-rw-r--r--collectors/python.d.plugin/example/example.chart.py51
-rw-r--r--collectors/python.d.plugin/example/example.conf87
-rw-r--r--collectors/python.d.plugin/exim/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/exim/README.md41
-rw-r--r--collectors/python.d.plugin/exim/exim.chart.py39
-rw-r--r--collectors/python.d.plugin/exim/exim.conf91
-rw-r--r--collectors/python.d.plugin/fail2ban/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/fail2ban/README.md82
-rw-r--r--collectors/python.d.plugin/fail2ban/fail2ban.chart.py217
-rw-r--r--collectors/python.d.plugin/fail2ban/fail2ban.conf68
-rw-r--r--collectors/python.d.plugin/gearman/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/gearman/README.md50
-rw-r--r--collectors/python.d.plugin/gearman/gearman.chart.py243
-rw-r--r--collectors/python.d.plugin/gearman/gearman.conf72
-rw-r--r--collectors/python.d.plugin/go_expvar/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/go_expvar/README.md319
-rw-r--r--collectors/python.d.plugin/go_expvar/go_expvar.chart.py253
-rw-r--r--collectors/python.d.plugin/go_expvar/go_expvar.conf108
-rw-r--r--collectors/python.d.plugin/haproxy/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/haproxy/README.md67
-rw-r--r--collectors/python.d.plugin/haproxy/haproxy.chart.py360
-rw-r--r--collectors/python.d.plugin/haproxy/haproxy.conf83
-rw-r--r--collectors/python.d.plugin/hddtemp/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/hddtemp/README.md38
-rw-r--r--collectors/python.d.plugin/hddtemp/hddtemp.chart.py99
-rw-r--r--collectors/python.d.plugin/hddtemp/hddtemp.conf95
-rw-r--r--collectors/python.d.plugin/hpssa/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/hpssa/README.md83
-rw-r--r--collectors/python.d.plugin/hpssa/hpssa.chart.py396
-rw-r--r--collectors/python.d.plugin/hpssa/hpssa.conf61
-rw-r--r--collectors/python.d.plugin/icecast/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/icecast/README.md44
-rw-r--r--collectors/python.d.plugin/icecast/icecast.chart.py94
-rw-r--r--collectors/python.d.plugin/icecast/icecast.conf81
-rw-r--r--collectors/python.d.plugin/ipfs/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/ipfs/README.md51
-rw-r--r--collectors/python.d.plugin/ipfs/ipfs.chart.py149
-rw-r--r--collectors/python.d.plugin/ipfs/ipfs.conf82
-rw-r--r--collectors/python.d.plugin/litespeed/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/litespeed/README.md72
-rw-r--r--collectors/python.d.plugin/litespeed/litespeed.chart.py188
-rw-r--r--collectors/python.d.plugin/litespeed/litespeed.conf72
-rw-r--r--collectors/python.d.plugin/logind/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/logind/README.md86
-rw-r--r--collectors/python.d.plugin/logind/logind.chart.py85
-rw-r--r--collectors/python.d.plugin/logind/logind.conf60
-rw-r--r--collectors/python.d.plugin/megacli/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/megacli/README.md86
-rw-r--r--collectors/python.d.plugin/megacli/megacli.chart.py278
-rw-r--r--collectors/python.d.plugin/megacli/megacli.conf60
-rw-r--r--collectors/python.d.plugin/memcached/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/memcached/README.md99
-rw-r--r--collectors/python.d.plugin/memcached/memcached.chart.py197
-rw-r--r--collectors/python.d.plugin/memcached/memcached.conf90
-rw-r--r--collectors/python.d.plugin/mongodb/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/mongodb/README.md210
-rw-r--r--collectors/python.d.plugin/mongodb/mongodb.chart.py786
-rw-r--r--collectors/python.d.plugin/mongodb/mongodb.conf102
-rw-r--r--collectors/python.d.plugin/monit/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/monit/README.md52
-rw-r--r--collectors/python.d.plugin/monit/monit.chart.py360
-rw-r--r--collectors/python.d.plugin/monit/monit.conf86
-rw-r--r--collectors/python.d.plugin/nsd/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/nsd/README.md68
-rw-r--r--collectors/python.d.plugin/nsd/nsd.chart.py105
-rw-r--r--collectors/python.d.plugin/nsd/nsd.conf91
-rw-r--r--collectors/python.d.plugin/ntpd/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/ntpd/README.md90
-rw-r--r--collectors/python.d.plugin/ntpd/ntpd.chart.py385
-rw-r--r--collectors/python.d.plugin/ntpd/ntpd.conf89
-rw-r--r--collectors/python.d.plugin/nvidia_smi/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/nvidia_smi/README.md66
-rw-r--r--collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py589
-rw-r--r--collectors/python.d.plugin/nvidia_smi/nvidia_smi.conf68
-rw-r--r--collectors/python.d.plugin/openldap/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/openldap/README.md79
-rw-r--r--collectors/python.d.plugin/openldap/openldap.chart.py216
-rw-r--r--collectors/python.d.plugin/openldap/openldap.conf75
-rw-r--r--collectors/python.d.plugin/oracledb/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/oracledb/README.md97
-rw-r--r--collectors/python.d.plugin/oracledb/oracledb.chart.py831
-rw-r--r--collectors/python.d.plugin/oracledb/oracledb.conf84
-rw-r--r--collectors/python.d.plugin/pandas/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/pandas/README.md92
-rw-r--r--collectors/python.d.plugin/pandas/pandas.chart.py89
-rw-r--r--collectors/python.d.plugin/pandas/pandas.conf191
-rw-r--r--collectors/python.d.plugin/postfix/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/postfix/README.md36
-rw-r--r--collectors/python.d.plugin/postfix/postfix.chart.py52
-rw-r--r--collectors/python.d.plugin/postfix/postfix.conf72
-rw-r--r--collectors/python.d.plugin/proxysql/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/proxysql/README.md106
-rw-r--r--collectors/python.d.plugin/proxysql/proxysql.chart.py352
-rw-r--r--collectors/python.d.plugin/proxysql/proxysql.conf116
-rw-r--r--collectors/python.d.plugin/puppet/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/puppet/README.md67
-rw-r--r--collectors/python.d.plugin/puppet/puppet.chart.py121
-rw-r--r--collectors/python.d.plugin/puppet/puppet.conf94
-rw-r--r--collectors/python.d.plugin/python.d.conf84
-rw-r--r--collectors/python.d.plugin/python.d.plugin.in922
-rw-r--r--collectors/python.d.plugin/python_modules/__init__.py0
-rw-r--r--collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py91
-rw-r--r--collectors/python.d.plugin/python_modules/bases/FrameworkServices/LogService.py82
-rw-r--r--collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py163
-rw-r--r--collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py261
-rw-r--r--collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py336
-rw-r--r--collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py207
-rw-r--r--collectors/python.d.plugin/python_modules/bases/FrameworkServices/__init__.py0
-rw-r--r--collectors/python.d.plugin/python_modules/bases/__init__.py0
-rw-r--r--collectors/python.d.plugin/python_modules/bases/charts.py431
-rw-r--r--collectors/python.d.plugin/python_modules/bases/collection.py117
-rw-r--r--collectors/python.d.plugin/python_modules/bases/loaders.py46
-rw-r--r--collectors/python.d.plugin/python_modules/bases/loggers.py206
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/__init__.py316
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/composer.py140
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/constructor.py676
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/cyaml.py86
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/dumper.py63
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/emitter.py1141
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/error.py76
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/events.py87
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/loader.py41
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/nodes.py50
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/parser.py590
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/reader.py191
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/representer.py485
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/resolver.py225
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/scanner.py1458
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/serializer.py112
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml2/tokens.py105
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/__init__.py313
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/composer.py140
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/constructor.py687
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/cyaml.py86
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/dumper.py63
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/emitter.py1138
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/error.py76
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/events.py87
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/loader.py41
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/nodes.py50
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/parser.py590
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/reader.py193
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/representer.py375
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/resolver.py225
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/scanner.py1449
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/serializer.py112
-rw-r--r--collectors/python.d.plugin/python_modules/pyyaml3/tokens.py105
-rw-r--r--collectors/python.d.plugin/python_modules/third_party/__init__.py0
-rw-r--r--collectors/python.d.plugin/python_modules/third_party/boinc_client.py515
-rw-r--r--collectors/python.d.plugin/python_modules/third_party/filelock.py451
-rw-r--r--collectors/python.d.plugin/python_modules/third_party/lm_sensors.py327
-rw-r--r--collectors/python.d.plugin/python_modules/third_party/mcrcon.py74
-rw-r--r--collectors/python.d.plugin/python_modules/third_party/monotonic.py201
-rw-r--r--collectors/python.d.plugin/python_modules/third_party/ordereddict.py110
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/__init__.py98
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/_collections.py320
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/connection.py374
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/connectionpool.py900
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/__init__.py0
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/__init__.py0
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/bindings.py591
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/low_level.py344
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/appengine.py297
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/ntlmpool.py113
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/pyopenssl.py458
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/securetransport.py808
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/contrib/socks.py189
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/exceptions.py247
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/fields.py179
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/filepost.py95
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/packages/__init__.py5
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/packages/backports/__init__.py0
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/packages/backports/makefile.py54
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/packages/ordered_dict.py260
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/packages/six.py852
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/packages/ssl_match_hostname/__init__.py20
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/packages/ssl_match_hostname/_implementation.py156
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/poolmanager.py441
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/request.py149
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/response.py623
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/__init__.py55
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/connection.py131
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/request.py119
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/response.py82
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/retry.py402
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/selectors.py588
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/ssl_.py338
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/timeout.py243
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/url.py231
-rw-r--r--collectors/python.d.plugin/python_modules/urllib3/util/wait.py41
-rw-r--r--collectors/python.d.plugin/rabbitmq/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/rabbitmq/README.md138
-rw-r--r--collectors/python.d.plugin/rabbitmq/rabbitmq.chart.py443
-rw-r--r--collectors/python.d.plugin/rabbitmq/rabbitmq.conf86
-rw-r--r--collectors/python.d.plugin/rethinkdbs/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/rethinkdbs/README.md53
-rw-r--r--collectors/python.d.plugin/rethinkdbs/rethinkdbs.chart.py247
-rw-r--r--collectors/python.d.plugin/rethinkdbs/rethinkdbs.conf76
-rw-r--r--collectors/python.d.plugin/retroshare/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/retroshare/README.md47
-rw-r--r--collectors/python.d.plugin/retroshare/retroshare.chart.py78
-rw-r--r--collectors/python.d.plugin/retroshare/retroshare.conf72
-rw-r--r--collectors/python.d.plugin/riakkv/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/riakkv/README.md126
-rw-r--r--collectors/python.d.plugin/riakkv/riakkv.chart.py334
-rw-r--r--collectors/python.d.plugin/riakkv/riakkv.conf68
-rw-r--r--collectors/python.d.plugin/samba/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/samba/README.md121
-rw-r--r--collectors/python.d.plugin/samba/samba.chart.py144
-rw-r--r--collectors/python.d.plugin/samba/samba.conf60
-rw-r--r--collectors/python.d.plugin/sensors/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/sensors/README.md33
-rw-r--r--collectors/python.d.plugin/sensors/sensors.chart.py179
-rw-r--r--collectors/python.d.plugin/sensors/sensors.conf61
-rw-r--r--collectors/python.d.plugin/smartd_log/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/smartd_log/README.md125
-rw-r--r--collectors/python.d.plugin/smartd_log/smartd_log.chart.py772
-rw-r--r--collectors/python.d.plugin/smartd_log/smartd_log.conf75
-rw-r--r--collectors/python.d.plugin/spigotmc/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/spigotmc/README.md38
-rw-r--r--collectors/python.d.plugin/spigotmc/spigotmc.chart.py184
-rw-r--r--collectors/python.d.plugin/spigotmc/spigotmc.conf66
-rw-r--r--collectors/python.d.plugin/springboot/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/springboot/README.md145
-rw-r--r--collectors/python.d.plugin/springboot/springboot.chart.py160
-rw-r--r--collectors/python.d.plugin/springboot/springboot.conf118
-rw-r--r--collectors/python.d.plugin/squid/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/squid/README.md58
-rw-r--r--collectors/python.d.plugin/squid/squid.chart.py123
-rw-r--r--collectors/python.d.plugin/squid/squid.conf167
-rw-r--r--collectors/python.d.plugin/tomcat/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/tomcat/README.md53
-rw-r--r--collectors/python.d.plugin/tomcat/tomcat.chart.py199
-rw-r--r--collectors/python.d.plugin/tomcat/tomcat.conf89
-rw-r--r--collectors/python.d.plugin/tor/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/tor/README.md66
-rw-r--r--collectors/python.d.plugin/tor/tor.chart.py107
-rw-r--r--collectors/python.d.plugin/tor/tor.conf79
-rw-r--r--collectors/python.d.plugin/traefik/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/traefik/README.md74
-rw-r--r--collectors/python.d.plugin/traefik/traefik.chart.py198
-rw-r--r--collectors/python.d.plugin/traefik/traefik.conf77
-rw-r--r--collectors/python.d.plugin/uwsgi/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/uwsgi/README.md52
-rw-r--r--collectors/python.d.plugin/uwsgi/uwsgi.chart.py177
-rw-r--r--collectors/python.d.plugin/uwsgi/uwsgi.conf92
-rw-r--r--collectors/python.d.plugin/varnish/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/varnish/README.md65
-rw-r--r--collectors/python.d.plugin/varnish/varnish.chart.py385
-rw-r--r--collectors/python.d.plugin/varnish/varnish.conf66
-rw-r--r--collectors/python.d.plugin/w1sensor/Makefile.inc13
-rw-r--r--collectors/python.d.plugin/w1sensor/README.md28
-rw-r--r--collectors/python.d.plugin/w1sensor/w1sensor.chart.py97
-rw-r--r--collectors/python.d.plugin/w1sensor/w1sensor.conf72
-rw-r--r--collectors/python.d.plugin/zscores/Makefile.inc12
-rw-r--r--collectors/python.d.plugin/zscores/README.md144
-rw-r--r--collectors/python.d.plugin/zscores/zscores.chart.py146
-rw-r--r--collectors/python.d.plugin/zscores/zscores.conf108
-rw-r--r--collectors/slabinfo.plugin/Makefile.am14
-rw-r--r--collectors/slabinfo.plugin/README.md29
-rw-r--r--collectors/slabinfo.plugin/slabinfo.c393
-rw-r--r--collectors/statsd.plugin/Makefile.am23
-rw-r--r--collectors/statsd.plugin/README.md699
-rw-r--r--collectors/statsd.plugin/asterisk.conf208
-rw-r--r--collectors/statsd.plugin/asterisk.md61
-rw-r--r--collectors/statsd.plugin/example.conf64
-rw-r--r--collectors/statsd.plugin/k6.conf110
-rw-r--r--collectors/statsd.plugin/k6.md76
-rw-r--r--collectors/statsd.plugin/statsd.c2844
-rw-r--r--collectors/tc.plugin/Makefile.am20
-rw-r--r--collectors/tc.plugin/README.md205
-rw-r--r--collectors/tc.plugin/plugin_tc.c1182
-rwxr-xr-xcollectors/tc.plugin/tc-qos-helper.sh.in297
-rw-r--r--collectors/timex.plugin/Makefile.am8
-rw-r--r--collectors/timex.plugin/README.md31
-rw-r--r--collectors/timex.plugin/plugin_timex.c176
-rw-r--r--collectors/xenstat.plugin/Makefile.am8
-rw-r--r--collectors/xenstat.plugin/README.md53
-rw-r--r--collectors/xenstat.plugin/xenstat_plugin.c1071
529 files changed, 136206 insertions, 0 deletions
diff --git a/collectors/COLLECTORS.md b/collectors/COLLECTORS.md
new file mode 100644
index 0000000..7f66076
--- /dev/null
+++ b/collectors/COLLECTORS.md
@@ -0,0 +1,518 @@
+<!--
+title: "Supported collectors list"
+description: "Netdata gathers real-time metrics from hundreds of data sources using collectors. Most require zero configuration and are pre-configured out of the box."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/COLLECTORS.md
+-->
+
+# Supported collectors list
+
+Netdata uses collectors to help you gather metrics from your favorite applications and services and view them in
+real-time, interactive charts. The following list includes collectors for both external services/applications and
+internal system metrics.
+
+Learn more about [how collectors work](/docs/collect/how-collectors-work.md), and then learn how to [enable or
+configure](/docs/collect/enable-configure.md) any of the below collectors using the same process.
+
+Some collectors have both Go and Python versions as we continue our effort to migrate all collectors to Go. In these
+cases, _Netdata always prioritizes the Go version_, and we highly recommend you use the Go versions for the best
+experience.
+
+If you want to use a Python version of a collector, you need to explicitly [disable the Go
+version](/docs/collect/enable-configure.md), and enable the Python version. Netdata then skips the Go version and
+attempts to load the Python version and its accompanying configuration file.
+
+If you don't see the app/service you'd like to monitor in this list:
+
+- Check out our [GitHub issues](https://github.com/netdata/netdata/issues). Use the search bar to look for previous
+ discussions about that collector—we may be looking for assistance from users such as yourself!
+- If you don't see the collector there, you can make
+ a [feature request](https://github.com/netdata/netdata/issues/new/choose) on GitHub.
+- If you have basic software development skills, you can add your own plugin
+ in [Go](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin#how-to-develop-a-collector)
+ or [Python](https://learn.netdata.cloud/guides/python-collector)
+
+Supported Collectors List:
+
+- [Service and application collectors](#service-and-application-collectors)
+ - [Generic](#generic)
+ - [APM (application performance monitoring)](#apm-application-performance-monitoring)
+ - [Containers and VMs](#containers-and-vms)
+ - [Data stores](#data-stores)
+ - [Distributed computing](#distributed-computing)
+ - [Email](#email)
+ - [Kubernetes](#kubernetes)
+ - [Logs](#logs)
+ - [Messaging](#messaging)
+ - [Network](#network)
+ - [Provisioning](#provisioning)
+ - [Remote devices](#remote-devices)
+ - [Search](#search)
+ - [Storage](#storage)
+ - [Web](#web)
+- [System collectors](#system-collectors)
+ - [Applications](#applications)
+ - [Disks and filesystems](#disks-and-filesystems)
+ - [eBPF](#ebpf)
+ - [Hardware](#hardware)
+ - [Memory](#memory)
+ - [Networks](#networks)
+ - [Operating systems](#operating-systems)
+ - [Processes](#processes)
+ - [Resources](#resources)
+ - [Users](#users)
+- [Netdata collectors](#netdata-collectors)
+- [Orchestrators](#orchestrators)
+- [Third-party collectors](#third-party-collectors)
+- [Etc](#etc)
+
+## Service and application collectors
+
+The Netdata Agent auto-detects and collects metrics from all of the services and applications below. You can also
+configure any of these collectors according to your setup and infrastructure.
+
+### Generic
+
+- [Prometheus endpoints](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/prometheus): Gathers
+ metrics from any number of Prometheus endpoints, with support to autodetect more than 600 services and applications.
+- [Pandas](https://learn.netdata.cloud/docs/agent/collectors/python.d.plugin/pandas): A Python collector that gathers
+ metrics from a [pandas](https://pandas.pydata.org/) dataframe. Pandas is a high level data processing library in
+ Python that can read various formats of data from local files or web endpoints. Custom processing and transformation
+ logic can also be expressed as part of the collector configuration.
+
+### APM (application performance monitoring)
+
+- [Go applications](/collectors/python.d.plugin/go_expvar/README.md): Monitor any Go application that exposes its
+ metrics with the `expvar` package from the Go standard library.
+- [Java Spring Boot 2
+ applications](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/springboot2/) (Go version):
+ Monitor running Java Spring Boot 2 applications that expose their metrics with the use of the Spring Boot Actuator.
+- [Java Spring Boot 2 applications](/collectors/python.d.plugin/springboot/README.md) (Python version): Monitor
+ running Java Spring Boot applications that expose their metrics with the use of the Spring Boot Actuator.
+- [statsd](/collectors/statsd.plugin/README.md): Implement a high performance `statsd` server for Netdata.
+- [phpDaemon](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/phpdaemon/): Collect worker
+ statistics (total, active, idle), and uptime for web and network applications.
+- [uWSGI](/collectors/python.d.plugin/uwsgi/README.md): Monitor performance metrics exposed by the uWSGI Stats
+ Server.
+
+### Containers and VMs
+
+- [Docker containers](/collectors/cgroups.plugin/README.md): Monitor the health and performance of individual Docker
+ containers using the cgroups collector plugin.
+- [DockerD](/collectors/python.d.plugin/dockerd/README.md): Collect container health statistics.
+- [Docker Engine](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/docker_engine/): Collect
+ runtime statistics from the `docker` daemon using the `metrics-address` feature.
+- [Docker Hub](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/dockerhub/): Collect statistics
+ about Docker repositories, such as pulls, starts, status, time since last update, and more.
+- [Libvirt](/collectors/cgroups.plugin/README.md): Monitor the health and performance of individual Libvirt containers
+ using the cgroups collector plugin.
+- [LXC](/collectors/cgroups.plugin/README.md): Monitor the health and performance of individual LXC containers using
+ the cgroups collector plugin.
+- [LXD](/collectors/cgroups.plugin/README.md): Monitor the health and performance of individual LXD containers using
+ the cgroups collector plugin.
+- [systemd-nspawn](/collectors/cgroups.plugin/README.md): Monitor the health and performance of individual
+ systemd-nspawn containers using the cgroups collector plugin.
+- [vCenter Server Appliance](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/vcsa/): Monitor
+ appliance system, components, and software update health statuses via the Health API.
+- [vSphere](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/vsphere/): Collect host and virtual
+ machine performance metrics.
+- [Xen/XCP-ng](/collectors/xenstat.plugin/README.md): Collect XenServer and XCP-ng metrics using `libxenstat`.
+
+### Data stores
+
+- [CockroachDB](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/cockroachdb/): Monitor various
+ database components using `_status/vars` endpoint.
+- [Consul](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/consul/): Capture service and unbound
+ checks status (passing, warning, critical, maintenance).
+- [Couchbase](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/couchbase/): Gather per-bucket
+ metrics from any number of instances of the distributed JSON document database.
+- [CouchDB](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/couchdb): Monitor database health and
+ performance metrics
+ (reads/writes, HTTP traffic, replication status, etc).
+- [MongoDB](/collectors/python.d.plugin/mongodb/README.md): Collect memory-caching system performance metrics and
+ reads the server's response to `stats` command (stats interface).
+- [MySQL](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/mysql/): Collect database global,
+ replication and per user statistics.
+- [OracleDB](/collectors/python.d.plugin/oracledb/README.md): Monitor database performance and health metrics.
+- [Pika](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/pika/): Gather metric, such as clients,
+ memory usage, queries, and more from the Redis interface-compatible database.
+- [Postgres](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/postgres): Collect database health
+ and performance metrics.
+- [ProxySQL](/collectors/python.d.plugin/proxysql/README.md): Monitor database backend and frontend performance
+ metrics.
+- [Redis](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/redis/): Monitor status from any
+ number of database instances by reading the server's response to the `INFO ALL` command.
+- [RethinkDB](/collectors/python.d.plugin/rethinkdbs/README.md): Collect database server and cluster statistics.
+- [Riak KV](/collectors/python.d.plugin/riakkv/README.md): Collect database stats from the `/stats` endpoint.
+- [Zookeeper](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/zookeeper/): Monitor application
+ health metrics reading the server's response to the `mntr` command.
+- [Memcached](/collectors/python.d.plugin/memcached/README.md): Collect memory-caching system performance metrics.
+
+### Distributed computing
+
+- [BOINC](/collectors/python.d.plugin/boinc/README.md): Monitor the total number of tasks, open tasks, and task
+ states for the distributed computing client.
+- [Gearman](/collectors/python.d.plugin/gearman/README.md): Collect application summary (queued, running) and per-job
+ worker statistics (queued, idle, running).
+
+### Email
+
+- [Dovecot](/collectors/python.d.plugin/dovecot/README.md): Collect email server performance metrics by reading the
+ server's response to the `EXPORT global` command.
+- [EXIM](/collectors/python.d.plugin/exim/README.md): Uses the `exim` tool to monitor the queue length of a
+ mail/message transfer agent (MTA).
+- [Postfix](/collectors/python.d.plugin/postfix/README.md): Uses the `postqueue` tool to monitor the queue length of a
+ mail/message transfer agent (MTA).
+
+### Kubernetes
+
+- [Kubelet](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/k8s_kubelet/): Monitor one or more
+ instances of the Kubelet agent and collects metrics on number of pods/containers running, volume of Docker
+ operations, and more.
+- [kube-proxy](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/k8s_kubeproxy/): Collect
+ metrics, such as syncing proxy rules and REST client requests, from one or more instances of `kube-proxy`.
+- [Service discovery](https://github.com/netdata/agent-service-discovery/): Find what services are running on a
+ cluster's pods, converts that into configuration files, and exports them so they can be monitored by Netdata.
+
+### Logs
+
+- [Fluentd](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/fluentd/): Gather application
+ plugins metrics from an endpoint provided by `in_monitor plugin`.
+- [Logstash](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/logstash/): Monitor JVM threads,
+ memory usage, garbage collection statistics, and more.
+- [OpenVPN status logs](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/openvpn_status_log): Parse
+ server log files and provide summary (client, traffic) metrics.
+- [Squid web server logs](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/squidlog/): Tail Squid
+ access logs to return the volume of requests, types of requests, bandwidth, and much more.
+- [Web server logs (Go version for Apache,
+ NGINX)](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/weblog/): Tail access logs and provide
+ very detailed web server performance statistics. This module is able to parse 200k+ rows in less than half a second.
+- [Web server logs (Apache, NGINX)](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/weblog): Tail
+ access log
+ file and collect web server/caching proxy metrics.
+
+### Messaging
+
+- [ActiveMQ](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/activemq/): Collect message broker
+ queues and topics statistics using the ActiveMQ Console API.
+- [Beanstalk](/collectors/python.d.plugin/beanstalk/README.md): Collect server and tube-level statistics, such as CPU
+ usage, jobs rates, commands, and more.
+- [Pulsar](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/pulsar/): Collect summary,
+ namespaces, and topics performance statistics.
+- [RabbitMQ (Go)](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/rabbitmq/): Collect message
+ broker overview, system and per virtual host metrics.
+- [RabbitMQ (Python)](/collectors/python.d.plugin/rabbitmq/README.md): Collect message broker global and per virtual
+ host metrics.
+- [VerneMQ](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/vernemq/): Monitor MQTT broker
+ health and performance metrics. It collects all available info for both MQTTv3 and v5 communication
+
+### Network
+
+- [Bind 9](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/bind/): Collect nameserver summary
+ performance statistics via a web interface (`statistics-channels` feature).
+- [Chrony](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/chrony): Monitor the precision and
+ statistics of a local `chronyd` server.
+- [CoreDNS](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/coredns/): Measure DNS query round
+ trip time.
+- [Dnsmasq](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/dnsmasq_dhcp/): Automatically
+ detects all configured `Dnsmasq` DHCP ranges and Monitor their utilization.
+- [DNSdist](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/dnsdist/): Collect
+ load-balancer performance and health metrics.
+- [Dnsmasq DNS Forwarder](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/dnsmasq/): Gather
+ queries, entries, operations, and events for the lightweight DNS forwarder.
+- [DNS Query Time](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/dnsquery/): Monitor the round
+ trip time for DNS queries in milliseconds.
+- [Freeradius](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/freeradius/): Collect
+ server authentication and accounting statistics from the `status server`.
+- [Libreswan](/collectors/charts.d.plugin/libreswan/README.md): Collect bytes-in, bytes-out, and uptime metrics.
+- [Icecast](/collectors/python.d.plugin/icecast/README.md): Monitor the number of listeners for active sources.
+- [ISC Bind (RDNC)](/collectors/python.d.plugin/bind_rndc/README.md): Collect nameserver summary performance
+ statistics using the `rndc` tool.
+- [ISC DHCP](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/isc_dhcpd): Reads a
+ `dhcpd.leases` file and collects metrics on total active leases, pool active leases, and pool utilization.
+- [OpenLDAP](/collectors/python.d.plugin/openldap/README.md): Provides statistics information from the OpenLDAP
+ (`slapd`) server.
+- [NSD](/collectors/python.d.plugin/nsd/README.md): Monitor nameserver performance metrics using the `nsd-control`
+ tool.
+- [NTP daemon](/collectors/python.d.plugin/ntpd/README.md): Monitor the system variables of the local `ntpd` daemon
+ (optionally including variables of the polled peers) using the NTP Control Message Protocol via a UDP socket.
+- [OpenSIPS](/collectors/charts.d.plugin/opensips/README.md): Collect server health and performance metrics using the
+ `opensipsctl` tool.
+- [OpenVPN](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/openvpn/): Gather server summary
+ (client, traffic) and per user metrics (traffic, connection time) stats using `management-interface`.
+- [Pi-hole](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/pihole/): Monitor basic (DNS
+ queries, clients, blocklist) and extended (top clients, top permitted, and blocked domains) statistics using the PHP
+ API.
+- [PowerDNS Authoritative Server](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/powerdns):
+ Monitor one or more instances of the nameserver software to collect questions, events, and latency metrics.
+- [PowerDNS Recursor](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/powerdns_recursor):
+ Gather incoming/outgoing questions, drops, timeouts, and cache usage from any number of DNS recursor instances.
+- [RetroShare](/collectors/python.d.plugin/retroshare/README.md): Monitor application bandwidth, peers, and DHT
+ metrics.
+- [Tor](/collectors/python.d.plugin/tor/README.md): Capture traffic usage statistics using the Tor control port.
+- [Unbound](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/unbound/): Collect DNS resolver
+ summary and extended system and per thread metrics via the `remote-control` interface.
+
+### Provisioning
+
+- [Puppet](/collectors/python.d.plugin/puppet/README.md): Monitor the status of Puppet Server and Puppet DB.
+
+### Remote devices
+
+- [AM2320](/collectors/python.d.plugin/am2320/README.md): Monitor sensor temperature and humidity.
+- [Access point](/collectors/charts.d.plugin/ap/README.md): Monitor client, traffic and signal metrics using the `aw`
+ tool.
+- [APC UPS](/collectors/charts.d.plugin/apcupsd/README.md): Capture status information using the `apcaccess` tool.
+- [Energi Core](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/energid): Monitor
+ blockchain indexes, memory usage, network usage, and transactions of wallet instances.
+- [UPS/PDU](/collectors/charts.d.plugin/nut/README.md): Read the status of UPS/PDU devices using the `upsc` tool.
+- [SNMP devices](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/snmp): Gather data using the SNMP
+ protocol.
+- [1-Wire sensors](/collectors/python.d.plugin/w1sensor/README.md): Monitor sensor temperature.
+
+### Search
+
+- [Elasticsearch](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/elasticsearch): Collect
+ dozens of metrics on search engine performance from local nodes and local indices. Includes cluster health and
+ statistics.
+- [Solr](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/solr/): Collect application search
+ requests, search errors, update requests, and update errors statistics.
+
+### Storage
+
+- [Ceph](/collectors/python.d.plugin/ceph/README.md): Monitor the Ceph cluster usage and server data consumption.
+- [HDFS](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/hdfs/): Monitor health and performance
+ metrics for filesystem datanodes and namenodes.
+- [IPFS](/collectors/python.d.plugin/ipfs/README.md): Collect file system bandwidth, peers, and repo metrics.
+- [Scaleio](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/scaleio/): Monitor storage system,
+ storage pools, and SDCS health and performance metrics via VxFlex OS Gateway API.
+- [Samba](/collectors/python.d.plugin/samba/README.md): Collect file sharing metrics using the `smbstatus` tool.
+
+### Web
+
+- [Apache](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/apache/): Collect Apache web
+ server performance metrics via the `server-status?auto` endpoint.
+- [HAProxy](/collectors/python.d.plugin/haproxy/README.md): Collect frontend, backend, and health metrics.
+- [HTTP endpoints](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/httpcheck/): Monitor
+ any HTTP endpoint's availability and response time.
+- [Lighttpd](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/lighttpd/): Collect web server
+ performance metrics using the `server-status?auto` endpoint.
+- [Lighttpd2](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/lighttpd2/): Collect web server
+ performance metrics using the `server-status?format=plain` endpoint.
+- [Litespeed](/collectors/python.d.plugin/litespeed/README.md): Collect web server data (network, connection,
+ requests, cache) by reading `.rtreport*` files.
+- [Nginx](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/nginx/): Monitor web server
+ status information by gathering metrics via `ngx_http_stub_status_module`.
+- [Nginx VTS](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/nginxvts/): Gathers metrics from
+ any Nginx deployment with the _virtual host traffic status module_ enabled, including metrics on uptime, memory
+ usage, and cache, and more.
+- [PHP-FPM](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/phpfpm/): Collect application
+ summary and processes health metrics by scraping the status page (`/status?full`).
+- [TCP endpoints](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/portcheck/): Monitor any
+ TCP endpoint's availability and response time.
+- [Spigot Minecraft servers](/collectors/python.d.plugin/spigotmc/README.md): Monitor average ticket rate and number
+ of users.
+- [Squid](/collectors/python.d.plugin/squid/README.md): Monitor client and server bandwidth/requests by gathering
+ data from the Cache Manager component.
+- [Tengine](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/tengine/): Monitor web server
+ statistics using information provided by `ngx_http_reqstat_module`.
+- [Tomcat](/collectors/python.d.plugin/tomcat/README.md): Collect web server performance metrics from the Manager App
+ (`/manager/status?XML=true`).
+- [Traefik](/collectors/python.d.plugin/traefik/README.md): Uses Traefik's Health API to provide statistics.
+- [Varnish](/collectors/python.d.plugin/varnish/README.md): Provides HTTP accelerator global, backends (VBE), and
+ disks (SMF) statistics using the `varnishstat` tool.
+- [x509 check](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/x509check/): Monitor certificate
+ expiration time.
+- [Whois domain expiry](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/whoisquery/): Checks the
+ remaining time until a given domain is expired.
+
+## System collectors
+
+The Netdata Agent can collect these system- and hardware-level metrics using a variety of collectors, some of which
+(such as `proc.plugin`) collect multiple types of metrics simultaneously.
+
+### Applications
+
+- [Fail2ban](/collectors/python.d.plugin/fail2ban/README.md): Parses configuration files to detect all jails, then
+ uses log files to report ban rates and volume of banned IPs.
+- [Monit](/collectors/python.d.plugin/monit/README.md): Monitor statuses of targets (service-checks) using the XML
+ stats interface.
+- [WMI (Windows Management Instrumentation)
+ exporter](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/wmi/): Collect CPU, memory,
+ network, disk, OS, system, and log-in metrics scraping `wmi_exporter`.
+
+### Disks and filesystems
+
+- [BCACHE](/collectors/proc.plugin/README.md): Monitor BCACHE statistics with the the `proc.plugin` collector.
+- [Block devices](/collectors/proc.plugin/README.md): Gather metrics about the health and performance of block
+ devices using the the `proc.plugin` collector.
+- [Btrfs](/collectors/proc.plugin/README.md): Monitors Btrfs filesystems with the the `proc.plugin` collector.
+- [Device mapper](/collectors/proc.plugin/README.md): Gather metrics about the Linux device mapper with the proc
+ collector.
+- [Disk space](/collectors/diskspace.plugin/README.md): Collect disk space usage metrics on Linux mount points.
+- [Clock synchronization](/collectors/timex.plugin/README.md): Collect the system clock synchronization status on Linux.
+- [Files and directories](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/filecheck): Gather
+ metrics about the existence, modification time, and size of files or directories.
+- [ioping.plugin](/collectors/ioping.plugin/README.md): Measure disk read/write latency.
+- [NFS file servers and clients](/collectors/proc.plugin/README.md): Gather operations, utilization, and space usage
+ using the the `proc.plugin` collector.
+- [RAID arrays](/collectors/proc.plugin/README.md): Collect health, disk status, operation status, and more with the
+ the `proc.plugin` collector.
+- [Veritas Volume Manager](/collectors/proc.plugin/README.md): Gather metrics about the Veritas Volume Manager (VVM).
+- [ZFS](/collectors/proc.plugin/README.md): Monitor bandwidth and utilization of ZFS disks/partitions using the proc
+ collector.
+
+### eBPF
+
+- [Files](/collectors/ebpf.plugin/README.md): Provides information about how often a system calls kernel
+ functions related to file descriptors using the eBPF collector.
+- [Virtual file system (VFS)](/collectors/ebpf.plugin/README.md): Monitor IO, errors, deleted objects, and
+ more for kernel virtual file systems (VFS) using the eBPF collector.
+- [Processes](/collectors/ebpf.plugin/README.md): Monitor threads, task exits, and errors using the eBPF collector.
+
+### Hardware
+
+- [Adaptec RAID](/collectors/python.d.plugin/adaptec_raid/README.md): Monitor logical and physical devices health
+ metrics using the `arcconf` tool.
+- [CUPS](/collectors/cups.plugin/README.md): Monitor CUPS.
+- [FreeIPMI](/collectors/freeipmi.plugin/README.md): Uses `libipmimonitoring-dev` or `libipmimonitoring-devel` to
+ monitor the number of sensors, temperatures, voltages, currents, and more.
+- [Hard drive temperature](/collectors/python.d.plugin/hddtemp/README.md): Monitor the temperature of storage
+ devices.
+- [HP Smart Storage Arrays](/collectors/python.d.plugin/hpssa/README.md): Monitor controller, cache module, logical
+ and physical drive state, and temperature using the `ssacli` tool.
+- [MegaRAID controllers](/collectors/python.d.plugin/megacli/README.md): Collect adapter, physical drives, and
+ battery stats using the `megacli` tool.
+- [NVIDIA GPU](/collectors/python.d.plugin/nvidia_smi/README.md): Monitor performance metrics (memory usage, fan
+ speed, pcie bandwidth utilization, temperature, and more) using the `nvidia-smi` tool.
+- [Sensors](/collectors/python.d.plugin/sensors/README.md): Reads system sensors information (temperature, voltage,
+ electric current, power, and more) from `/sys/devices/`.
+- [S.M.A.R.T](/collectors/python.d.plugin/smartd_log/README.md): Reads SMART Disk Monitoring daemon logs.
+
+### Memory
+
+- [Available memory](/collectors/proc.plugin/README.md): Tracks changes in available RAM using the the `proc.plugin`
+ collector.
+- [Committed memory](/collectors/proc.plugin/README.md): Monitor committed memory using the `proc.plugin` collector.
+- [Huge pages](/collectors/proc.plugin/README.md): Gather metrics about huge pages in Linux and FreeBSD with the
+ `proc.plugin` collector.
+- [KSM](/collectors/proc.plugin/README.md): Measure the amount of merging, savings, and effectiveness using the
+ `proc.plugin` collector.
+- [Numa](/collectors/proc.plugin/README.md): Gather metrics on the number of non-uniform memory access (NUMA) events
+ every second using the `proc.plugin` collector.
+- [Page faults](/collectors/proc.plugin/README.md): Collect the number of memory page faults per second using the
+ `proc.plugin` collector.
+- [RAM](/collectors/proc.plugin/README.md): Collect metrics on system RAM, available RAM, and more using the
+ `proc.plugin` collector.
+- [SLAB](/collectors/slabinfo.plugin/README.md): Collect kernel SLAB details on Linux systems.
+- [swap](/collectors/proc.plugin/README.md): Monitor the amount of free and used swap at every second using the
+ `proc.plugin` collector.
+- [Writeback memory](/collectors/proc.plugin/README.md): Collect how much memory is actively being written to disk at
+ every second using the `proc.plugin` collector.
+
+### Networks
+
+- [Access points](/collectors/charts.d.plugin/ap/README.md): Visualizes data related to access points.
+- [fping.plugin](fping.plugin/README.md): Measure network latency, jitter and packet loss between the monitored node
+ and any number of remote network end points.
+- [Netfilter](/collectors/nfacct.plugin/README.md): Collect netfilter firewall, connection tracker, and accounting
+ metrics using `libmnl` and `libnetfilter_acct`.
+- [Network stack](/collectors/proc.plugin/README.md): Monitor the networking stack for errors, TCP connection aborts,
+ bandwidth, and more.
+- [Network QoS](/collectors/tc.plugin/README.md): Collect traffic QoS metrics (`tc`) of Linux network interfaces.
+- [SYNPROXY](/collectors/proc.plugin/README.md): Monitor entries uses, SYN packets received, TCP cookies, and more.
+
+### Operating systems
+
+- [freebsd.plugin](freebsd.plugin/README.md): Collect resource usage and performance data on FreeBSD systems.
+- [macOS](/collectors/macos.plugin/README.md): Collect resource usage and performance data on macOS systems.
+
+### Processes
+
+- [Applications](/collectors/apps.plugin/README.md): Gather CPU, disk, memory, network, eBPF, and other metrics per
+ application using the `apps.plugin` collector.
+- [systemd](/collectors/cgroups.plugin/README.md): Monitor the CPU and memory usage of systemd services using the
+ `cgroups.plugin` collector.
+- [systemd unit states](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/systemdunits): See the
+ state (active, inactive, activating, deactivating, failed) of various systemd unit types.
+- [System processes](/collectors/proc.plugin/README.md): Collect metrics on system load and total processes running
+ using `/proc/loadavg` and the `proc.plugin` collector.
+- [Uptime](/collectors/proc.plugin/README.md): Monitor the uptime of a system using the `proc.plugin` collector.
+
+### Resources
+
+- [CPU frequency](/collectors/proc.plugin/README.md): Monitor CPU frequency, as set by the `cpufreq` kernel module,
+ using the `proc.plugin` collector.
+- [CPU idle](/collectors/proc.plugin/README.md): Measure CPU idle every second using the `proc.plugin` collector.
+- [CPU performance](/collectors/perf.plugin/README.md): Collect CPU performance metrics using performance monitoring
+ units (PMU).
+- [CPU throttling](/collectors/proc.plugin/README.md): Gather metrics about thermal throttling using the `/proc/stat`
+ module and the `proc.plugin` collector.
+- [CPU utilization](/collectors/proc.plugin/README.md): Capture CPU utilization, both system-wide and per-core, using
+ the `/proc/stat` module and the `proc.plugin` collector.
+- [Entropy](/collectors/proc.plugin/README.md): Monitor the available entropy on a system using the `proc.plugin`
+ collector.
+- [Interprocess Communication (IPC)](/collectors/proc.plugin/README.md): Monitor IPC semaphores and shared memory
+ using the `proc.plugin` collector.
+- [Interrupts](/collectors/proc.plugin/README.md): Monitor interrupts per second using the `proc.plugin` collector.
+- [IdleJitter](/collectors/idlejitter.plugin/README.md): Measure CPU latency and jitter on all operating systems.
+- [SoftIRQs](/collectors/proc.plugin/README.md): Collect metrics on SoftIRQs, both system-wide and per-core, using the
+ `proc.plugin` collector.
+- [SoftNet](/collectors/proc.plugin/README.md): Capture SoftNet events per second, both system-wide and per-core,
+ using the `proc.plugin` collector.
+
+### Users
+
+- [systemd-logind](/collectors/python.d.plugin/logind/README.md): Monitor active sessions, users, and seats tracked
+ by `systemd-logind` or `elogind`.
+- [User/group usage](/collectors/apps.plugin/README.md): Gather CPU, disk, memory, network, and other metrics per user
+ and user group using the `apps.plugin` collector.
+
+## Netdata collectors
+
+These collectors are recursive in nature, in that they monitor some function of the Netdata Agent itself. Some
+collectors are described only in code and associated charts in Netdata dashboards.
+
+- [ACLK (code only)](https://github.com/netdata/netdata/blob/master/aclk/legacy/aclk_stats.c): View whether a Netdata
+ Agent is connected to Netdata Cloud via the [ACLK](/aclk/README.md), the volume of queries, process times, and more.
+- [Alarms](https://learn.netdata.cloud/docs/agent/collectors/python.d.plugin/alarms): This collector creates an
+ **Alarms** menu with one line plot showing the alarm states of a Netdata Agent over time.
+- [Anomalies](https://learn.netdata.cloud/docs/agent/collectors/python.d.plugin/anomalies): This collector uses the
+ Python PyOD library to perform unsupervised anomaly detection on your Netdata charts and/or dimensions.
+- [Exporting (code only)](https://github.com/netdata/netdata/blob/master/exporting/send_internal_metrics.c): Gather
+ metrics on CPU utilization for the [exporting engine](/exporting/README.md), and specific metrics for each enabled
+ exporting connector.
+- [Global statistics (code only)](https://github.com/netdata/netdata/blob/master/daemon/global_statistics.c): See
+ metrics on the CPU utilization, network traffic, volume of web clients, API responses, database engine usage, and
+ more.
+
+## Orchestrators
+
+Plugin orchestrators organize and run many of the above collectors.
+
+If you're interested in developing a new collector that you'd like to contribute to Netdata, we highly recommend using
+the `go.d.plugin`.
+
+- [go.d.plugin](https://github.com/netdata/go.d.plugin): An orchestrator for data collection modules written in `go`.
+- [python.d.plugin](python.d.plugin/README.md): An orchestrator for data collection modules written in `python` v2/v3.
+- [charts.d.plugin](charts.d.plugin/README.md): An orchestrator for data collection modules written in `bash` v4+.
+
+## Third-party collectors
+
+These collectors are developed and maintained by third parties and, unlike the other collectors, are not installed by
+default. To use a third-party collector, visit their GitHub/documentation page and follow their installation procedures.
+
+- [CyberPower UPS](https://github.com/HawtDogFlvrWtr/netdata_cyberpwrups_plugin): Polls CyberPower UPS data using
+ PowerPanel® Personal Linux.
+- [Logged-in users](https://github.com/veksh/netdata-numsessions): Collect the number of currently logged-on users.
+- [nextcloud](https://github.com/arnowelzel/netdata-nextcloud): Monitor Nextcloud servers.
+- [nim-netdata-plugin](https://github.com/FedericoCeratto/nim-netdata-plugin): A helper to create native Netdata
+ plugins using Nim.
+- [Nvidia GPUs](https://github.com/coraxx/netdata_nv_plugin): Monitor Nvidia GPUs.
+- [Teamspeak 3](https://github.com/coraxx/netdata_ts3_plugin): Pulls active users and bandwidth from TeamSpeak 3
+ servers.
+- [SSH](https://github.com/Yaser-Amiri/netdata-ssh-module): Monitor failed authentication requests of an SSH server.
+
+## Etc
+
+- [charts.d example](charts.d.plugin/example/README.md): An example `charts.d` collector.
+- [python.d example](python.d.plugin/example/README.md): An example `python.d` collector.
diff --git a/collectors/Makefile.am b/collectors/Makefile.am
new file mode 100644
index 0000000..9f8bf52
--- /dev/null
+++ b/collectors/Makefile.am
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+SUBDIRS = \
+ plugins.d \
+ apps.plugin \
+ cgroups.plugin \
+ charts.d.plugin \
+ cups.plugin \
+ diskspace.plugin \
+ timex.plugin \
+ fping.plugin \
+ ioping.plugin \
+ freebsd.plugin \
+ freeipmi.plugin \
+ idlejitter.plugin \
+ macos.plugin \
+ nfacct.plugin \
+ xenstat.plugin \
+ perf.plugin \
+ proc.plugin \
+ python.d.plugin \
+ slabinfo.plugin \
+ statsd.plugin \
+ ebpf.plugin \
+ tc.plugin \
+ $(NULL)
+
+usercustompluginsconfigdir=$(configdir)/custom-plugins.d
+usergoconfigdir=$(configdir)/go.d
+
+# Explicitly install directories to avoid permission issues due to umask
+install-exec-local:
+ $(INSTALL) -d $(DESTDIR)$(usercustompluginsconfigdir)
+ $(INSTALL) -d $(DESTDIR)$(usergoconfigdir)
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/README.md b/collectors/README.md
new file mode 100644
index 0000000..de46a72
--- /dev/null
+++ b/collectors/README.md
@@ -0,0 +1,48 @@
+<!--
+title: "Collecting metrics"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/README.md
+id: "collectors-ref"
+-->
+
+# Collecting metrics
+
+Netdata can collect metrics from hundreds of different sources, be they internal data created by the system itself, or
+external data created by services or applications. To see _all_ of the sources Netdata collects from, view our [list of
+supported collectors](/collectors/COLLECTORS.md).
+
+There are two essential points to understand about how collecting metrics works in Netdata:
+
+- All collectors are **installed by default** with every installation of Netdata. You do not need to install
+ collectors manually to collect metrics from new sources.
+- Upon startup, Netdata will **auto-detect** any application or service that has a
+ [collector](/collectors/COLLECTORS.md), as long as both the collector and the app/service are configured correctly.
+
+Most users will want to enable a new Netdata collector for their app/service. For those details, see
+our [collectors' configuration reference](/collectors/REFERENCE.md).
+
+## Take your next steps with collectors
+
+[Supported collectors list](/collectors/COLLECTORS.md)
+
+[Collectors configuration reference](/collectors/REFERENCE.md)
+
+## Guides
+
+[Monitor Nginx or Apache web server log files with Netdata](/docs/guides/collect-apache-nginx-web-logs.md)
+
+[Monitor CockroachDB metrics with Netdata](/docs/guides/monitor-cockroachdb.md)
+
+[Monitor Unbound DNS servers with Netdata](/docs/guides/collect-unbound-metrics.md)
+
+[Monitor a Hadoop cluster with Netdata](/docs/guides/monitor-hadoop-cluster.md)
+
+## Related features
+
+**[Dashboards](/web/README.md)**: Visualize your newly-collect metrics in real-time using Netdata's [built-in
+dashboard](/web/gui/README.md).
+
+**[Exporting](/exporting/README.md)**: Extend our built-in [database engine](/database/engine/README.md), which supports
+long-term metrics storage, by archiving metrics to external databases like Graphite, Prometheus, MongoDB, TimescaleDB, and more.
+It can export metrics to multiple databases simultaneously.
+
+
diff --git a/collectors/REFERENCE.md b/collectors/REFERENCE.md
new file mode 100644
index 0000000..939b189
--- /dev/null
+++ b/collectors/REFERENCE.md
@@ -0,0 +1,170 @@
+<!--
+title: "Collectors configuration reference"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/REFERENCE.md
+-->
+
+# Collectors configuration reference
+
+Welcome to the collector configuration reference guide.
+
+This guide contains detailed information about enabling/disabling plugins or modules, in addition a quick reference to
+the internal plugins API.
+
+## Netdata's collector architecture
+
+Netdata has an intricate system for organizing and managing its collectors. **Collectors** are the processes/programs
+that actually gather metrics from various sources. Collectors are organized by **plugins**, which help manage all the
+independent processes in a variety of programming languages based on their purpose and performance requirements.
+**Modules** are a type of collector, used primarily to connect to external applications, such as an Nginx web server or
+MySQL database, among many others.
+
+For most users, enabling individual collectors for the application/service you're interested in is far more important
+than knowing which plugin it uses. See our [collectors list](/collectors/COLLECTORS.md) to see whether your favorite app/service has
+a collector, and then read the documentation for that specific collector to figure out how to enable it.
+
+There are three types of plugins:
+
+- **Internal** plugins organize collectors that gather metrics from `/proc`, `/sys` and other Linux kernel sources.
+ They are written in `C`, and run as threads within the Netdata daemon.
+- **External** plugins organize collectors that gather metrics from external processes, such as a MySQL database or
+ Nginx web server. They can be written in any language, and the `netdata` daemon spawns them as long-running
+ independent processes. They communicate with the daemon via pipes.
+- **Plugin orchestrators**, which are external plugins that instead support a number of **modules**. Modules are a
+ type of collector. We have a few plugin orchestrators available for those who want to develop their own collectors,
+ but focus most of our efforts on the [Go plugin](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/).
+
+## Enable, configure, and disable modules
+
+Most collector modules come with **auto-detection**, configured to work out-of-the-box on popular operating systems with
+the default settings.
+
+However, there are cases that auto-detection fails. Usually, the reason is that the applications to be monitored do not
+allow Netdata to connect. In most of the cases, allowing the user `netdata` from `localhost` to connect and collect
+metrics, will automatically enable data collection for the application in question (it will require a Netdata restart).
+
+
+## Troubleshoot a collector
+
+First, navigate to your plugins directory, which is usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case
+on your system, open `netdata.conf` and look for the setting `plugins directory`. Once you're in the plugins directory,
+switch to the `netdata` user.
+
+```bash
+cd /usr/libexec/netdata/plugins.d/
+sudo su -s /bin/bash netdata
+```
+
+The next step is based on the collector's orchestrator. You can figure out which orchestrator the collector uses by
+
+uses either
+by viewing the [collectors list](COLLECTORS.md) and referencing the _configuration file_ field. For example, if that
+field contains `go.d`, that collector uses the Go orchestrator.
+
+```bash
+# Go orchestrator (go.d.plugin)
+./go.d.plugin -d -m <MODULE_NAME>
+
+# Python orchestrator (python.d.plugin)
+./python.d.plugin <MODULE_NAME> debug trace
+
+# Bash orchestrator (bash.d.plugin)
+./charts.d.plugin debug 1 <MODULE_NAME>
+```
+
+The output from the relevant command will provide valuable troubleshooting information. If you can't figure out how to
+enable the collector using the details from this output, feel free to [create an issue on our
+GitHub](https://github.com/netdata/netdata/issues/new?assignees=&labels=bug%2Cneeds+triage&template=BUG_REPORT.yml) to get some
+help from our collectors experts.
+
+## Enable and disable plugins
+
+You can enable or disable individual plugins by opening `netdata.conf` and scrolling down to the `[plugins]` section.
+This section features a list of Netdata's plugins, with a boolean setting to enable or disable them. The exception is
+`statsd.plugin`, which has its own `[statsd]` section. Your `[plugins]` section should look similar to this:
+
+```conf
+[plugins]
+ # proc = yes
+ # diskspace = yes
+ # timex = yes
+ # cgroups = yes
+ # tc = yes
+ # idlejitter = yes
+ # enable running new plugins = yes
+ # check for new plugins every = 60
+ # slabinfo = no
+ # fping = yes
+ # ioping = yes
+ # python.d = yes
+ # go.d = yes
+ # apps = yes
+ # perf = yes
+ # charts.d = yes
+```
+
+By default, most plugins are enabled, so you don't need to enable them explicitly to use their collectors. To enable or
+disable any specific plugin, remove the comment (`#`) and change the boolean setting to `yes` or `no`.
+
+All **external plugins** are managed by [plugins.d](plugins.d/README.md), which provides additional management options.
+
+## Internal plugins
+
+Each of the internal plugins runs as a thread inside the `netdata` daemon. Once this thread has started, the plugin may
+spawn additional threads according to its design.
+
+### Internal plugins API
+
+The internal data collection API consists of the following calls:
+
+```c
+collect_data() {
+ // collect data here (one iteration)
+
+ collected_number collected_value = collect_a_value();
+
+ // give the metrics to Netdata
+
+ static RRDSET *st = NULL; // the chart
+ static RRDDIM *rd = NULL; // a dimension attached to this chart
+
+ if(unlikely(!st)) {
+ // we haven't created this chart before
+ // create it now
+ st = rrdset_create_localhost(
+ "type"
+ , "id"
+ , "name"
+ , "family"
+ , "context"
+ , "Chart Title"
+ , "units"
+ , "plugin-name"
+ , "module-name"
+ , priority
+ , update_every
+ , chart_type
+ );
+
+ // attach a metric to it
+ rd = rrddim_add(st, "id", "name", multiplier, divider, algorithm);
+ }
+
+ // give the collected value(s) to the chart
+ rrddim_set_by_pointer(st, rd, collected_value);
+
+ // signal Netdata we are done with this iteration
+ rrdset_done(st);
+}
+```
+
+Of course, Netdata has a lot of libraries to help you also in collecting the metrics. The best way to find your way
+through this, is to examine what other similar plugins do.
+
+## External Plugins
+
+**External plugins** use the API and are managed by [plugins.d](plugins.d/README.md).
+
+## Write a custom collector
+
+You can add custom collectors by following the [external plugins documentation](/collectors/plugins.d/README.md).
+
diff --git a/collectors/all.h b/collectors/all.h
new file mode 100644
index 0000000..85a7ac8
--- /dev/null
+++ b/collectors/all.h
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_ALL_H
+#define NETDATA_ALL_H 1
+
+#include "daemon/common.h"
+
+// netdata internal data collection plugins
+
+#include "plugins.d/plugins_d.h"
+
+// ----------------------------------------------------------------------------
+// netdata chart priorities
+
+// This is a work in progress - to scope is to collect here all chart priorities.
+// These should be based on the CONTEXT of the charts + the chart id when needed
+// - for each SECTION +1000 (or +X000 for big sections)
+// - for each FAMILY +100
+// - for each CHART +10
+
+#define NETDATA_CHART_PRIO_SYSTEM_CPU 100
+#define NETDATA_CHART_PRIO_SYSTEM_LOAD 100
+#define NETDATA_CHART_PRIO_SYSTEM_IO 150
+#define NETDATA_CHART_PRIO_SYSTEM_PGPGIO 151
+#define NETDATA_CHART_PRIO_SYSTEM_RAM 200
+#define NETDATA_CHART_PRIO_SYSTEM_SWAP 201
+#define NETDATA_CHART_PRIO_SYSTEM_SWAPIO 250
+#define NETDATA_CHART_PRIO_SYSTEM_NET 500
+#define NETDATA_CHART_PRIO_SYSTEM_IPV4 500 // freebsd only
+#define NETDATA_CHART_PRIO_SYSTEM_IP 501
+#define NETDATA_CHART_PRIO_SYSTEM_IPV6 502
+#define NETDATA_CHART_PRIO_SYSTEM_PROCESSES 600
+#define NETDATA_CHART_PRIO_SYSTEM_PROCESS_STATES 601
+#define NETDATA_CHART_PRIO_SYSTEM_FORKS 700
+#define NETDATA_CHART_PRIO_SYSTEM_ACTIVE_PROCESSES 750
+#define NETDATA_CHART_PRIO_SYSTEM_CTXT 800
+#define NETDATA_CHART_PRIO_SYSTEM_IDLEJITTER 800
+#define NETDATA_CHART_PRIO_SYSTEM_INTR 900
+#define NETDATA_CHART_PRIO_SYSTEM_SOFTIRQS 950
+#define NETDATA_CHART_PRIO_SYSTEM_SOFTNET_STAT 955
+#define NETDATA_CHART_PRIO_SYSTEM_INTERRUPTS 1000
+#define NETDATA_CHART_PRIO_SYSTEM_DEV_INTR 1000 // freebsd only
+#define NETDATA_CHART_PRIO_SYSTEM_SOFT_INTR 1100 // freebsd only
+#define NETDATA_CHART_PRIO_SYSTEM_ENTROPY 1000
+#define NETDATA_CHART_PRIO_SYSTEM_UPTIME 1000
+#define NETDATA_CHART_PRIO_CLOCK_SYNC_STATE 1100
+#define NETDATA_CHART_PRIO_CLOCK_STATUS 1105
+#define NETDATA_CHART_PRIO_CLOCK_SYNC_OFFSET 1110
+#define NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_QUEUES 1200 // freebsd only
+#define NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_MESSAGES 1201
+#define NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_SIZE 1202
+#define NETDATA_CHART_PRIO_SYSTEM_IPC_SEMAPHORES 1203
+#define NETDATA_CHART_PRIO_SYSTEM_IPC_SEM_ARRAYS 1204
+#define NETDATA_CHART_PRIO_SYSTEM_IPC_SHARED_MEM_SEGS 1205
+#define NETDATA_CHART_PRIO_SYSTEM_IPC_SHARED_MEM_SIZE 1206
+#define NETDATA_CHART_PRIO_SYSTEM_IPC_SHARED_MEM_CALLS 1207
+#define NETDATA_CHART_PRIO_SYSTEM_PACKETS 7001 // freebsd only
+
+
+// CPU per core
+
+#define NETDATA_CHART_PRIO_CPU_PER_CORE 1000 // +1 per core
+#define NETDATA_CHART_PRIO_CPU_TEMPERATURE 1050 // freebsd only
+#define NETDATA_CHART_PRIO_CPUFREQ_SCALING_CUR_FREQ 5003 // freebsd only
+#define NETDATA_CHART_PRIO_CPUIDLE 6000
+
+#define NETDATA_CHART_PRIO_CORE_THROTTLING 5001
+#define NETDATA_CHART_PRIO_PACKAGE_THROTTLING 5002
+
+// Interrupts per core
+
+#define NETDATA_CHART_PRIO_INTERRUPTS_PER_CORE 1100 // +1 per core
+
+// Memory Section - 1xxx
+
+#define NETDATA_CHART_PRIO_MEM_SYSTEM_AVAILABLE 1010
+#define NETDATA_CHART_PRIO_MEM_SYSTEM_OOM_KILL 1020
+#define NETDATA_CHART_PRIO_MEM_SYSTEM_COMMITTED 1030
+#define NETDATA_CHART_PRIO_MEM_SYSTEM_PGFAULTS 1040
+#define NETDATA_CHART_PRIO_MEM_KERNEL 1100
+#define NETDATA_CHART_PRIO_MEM_SLAB 1200
+#define NETDATA_CHART_PRIO_MEM_HUGEPAGES 1250
+#define NETDATA_CHART_PRIO_MEM_KSM 1300
+#define NETDATA_CHART_PRIO_MEM_KSM_SAVINGS 1301
+#define NETDATA_CHART_PRIO_MEM_KSM_RATIOS 1302
+#define NETDATA_CHART_PRIO_MEM_NUMA 1400
+#define NETDATA_CHART_PRIO_MEM_NUMA_NODES 1410
+#define NETDATA_CHART_PRIO_MEM_PAGEFRAG 1450
+#define NETDATA_CHART_PRIO_MEM_HW 1500
+#define NETDATA_CHART_PRIO_MEM_HW_ECC_CE 1550
+#define NETDATA_CHART_PRIO_MEM_HW_ECC_UE 1560
+#define NETDATA_CHART_PRIO_MEM_ZRAM 1600
+#define NETDATA_CHART_PRIO_MEM_ZRAM_SAVINGS 1601
+#define NETDATA_CHART_PRIO_MEM_ZRAM_RATIO 1602
+#define NETDATA_CHART_PRIO_MEM_ZRAM_EFFICIENCY 1603
+
+// Disks
+
+#define NETDATA_CHART_PRIO_DISK_IO 2000
+#define NETDATA_CHART_PRIO_DISK_OPS 2010
+#define NETDATA_CHART_PRIO_DISK_QOPS 2015
+#define NETDATA_CHART_PRIO_DISK_BACKLOG 2020
+#define NETDATA_CHART_PRIO_DISK_BUSY 2030
+#define NETDATA_CHART_PRIO_DISK_UTIL 2040
+#define NETDATA_CHART_PRIO_DISK_AWAIT 2050
+#define NETDATA_CHART_PRIO_DISK_AVGSZ 2060
+#define NETDATA_CHART_PRIO_DISK_SVCTM 2070
+#define NETDATA_CHART_PRIO_DISK_MOPS 2080
+#define NETDATA_CHART_PRIO_DISK_IOTIME 2090
+#define NETDATA_CHART_PRIO_DISK_LATENCY 2095
+#define NETDATA_CHART_PRIO_BCACHE_CACHE_ALLOC 2120
+#define NETDATA_CHART_PRIO_BCACHE_HIT_RATIO 2120
+#define NETDATA_CHART_PRIO_BCACHE_RATES 2121
+#define NETDATA_CHART_PRIO_BCACHE_SIZE 2122
+#define NETDATA_CHART_PRIO_BCACHE_USAGE 2123
+#define NETDATA_CHART_PRIO_BCACHE_OPS 2124
+#define NETDATA_CHART_PRIO_BCACHE_BYPASS 2125
+#define NETDATA_CHART_PRIO_BCACHE_CACHE_READ_RACES 2126
+
+#define NETDATA_CHART_PRIO_DISKSPACE_SPACE 2023
+#define NETDATA_CHART_PRIO_DISKSPACE_INODES 2024
+
+// MDSTAT
+
+#define NETDATA_CHART_PRIO_MDSTAT_HEALTH 2100
+#define NETDATA_CHART_PRIO_MDSTAT_FLUSH 2101
+#define NETDATA_CHART_PRIO_MDSTAT_NONREDUNDANT 2105
+#define NETDATA_CHART_PRIO_MDSTAT_DISKS 2106 // 5 charts per raid
+#define NETDATA_CHART_PRIO_MDSTAT_MISMATCH 2107
+#define NETDATA_CHART_PRIO_MDSTAT_OPERATION 2108
+#define NETDATA_CHART_PRIO_MDSTAT_FINISH 2109
+#define NETDATA_CHART_PRIO_MDSTAT_SPEED 2110
+
+// Filesystem
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_CLEAN 2150
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_COUNT 2151
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_BYTES 2152
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EBYTES 2153
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_FSYNC 2154
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EFSYNC 2155
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_OPEN 2156
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EOPEN 2157
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_CREATE 2158
+#define NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_ECREATE 2159
+
+#define NETDATA_CHART_PRIO_EBPF_FILESYSTEM_CHARTS 2160
+
+// Mount Points
+#define NETDATA_CHART_PRIO_EBPF_MOUNT_CHARTS 2190
+
+// File descriptor
+#define NETDATA_CHART_PRIO_EBPF_FD_CHARTS 2195
+
+
+// NFS (server)
+
+#define NETDATA_CHART_PRIO_NFSD_READCACHE 2200
+#define NETDATA_CHART_PRIO_NFSD_FILEHANDLES 2201
+#define NETDATA_CHART_PRIO_NFSD_IO 2202
+#define NETDATA_CHART_PRIO_NFSD_THREADS 2203
+#define NETDATA_CHART_PRIO_NFSD_THREADS_FULLCNT 2204
+#define NETDATA_CHART_PRIO_NFSD_THREADS_HISTOGRAM 2205
+#define NETDATA_CHART_PRIO_NFSD_READAHEAD 2205
+#define NETDATA_CHART_PRIO_NFSD_NET 2207
+#define NETDATA_CHART_PRIO_NFSD_RPC 2208
+#define NETDATA_CHART_PRIO_NFSD_PROC2 2209
+#define NETDATA_CHART_PRIO_NFSD_PROC3 2210
+#define NETDATA_CHART_PRIO_NFSD_PROC4 2211
+#define NETDATA_CHART_PRIO_NFSD_PROC4OPS 2212
+
+// NFS (client)
+
+#define NETDATA_CHART_PRIO_NFS_NET 2307
+#define NETDATA_CHART_PRIO_NFS_RPC 2308
+#define NETDATA_CHART_PRIO_NFS_PROC2 2309
+#define NETDATA_CHART_PRIO_NFS_PROC3 2310
+#define NETDATA_CHART_PRIO_NFS_PROC4 2311
+
+// BTRFS
+
+#define NETDATA_CHART_PRIO_BTRFS_DISK 2400
+#define NETDATA_CHART_PRIO_BTRFS_DATA 2401
+#define NETDATA_CHART_PRIO_BTRFS_METADATA 2402
+#define NETDATA_CHART_PRIO_BTRFS_SYSTEM 2403
+
+// ZFS
+
+#define NETDATA_CHART_PRIO_ZFS_ARC_SIZE 2500
+#define NETDATA_CHART_PRIO_ZFS_L2_SIZE 2500
+#define NETDATA_CHART_PRIO_ZFS_READS 2510
+#define NETDATA_CHART_PRIO_ZFS_ACTUAL_HITS 2519
+#define NETDATA_CHART_PRIO_ZFS_ARC_SIZE_BREAKDOWN 2520
+#define NETDATA_CHART_PRIO_ZFS_IMPORTANT_OPS 2522
+#define NETDATA_CHART_PRIO_ZFS_MEMORY_OPS 2523
+#define NETDATA_CHART_PRIO_ZFS_IO 2700
+#define NETDATA_CHART_PRIO_ZFS_HITS 2520
+#define NETDATA_CHART_PRIO_ZFS_DHITS 2530
+#define NETDATA_CHART_PRIO_ZFS_DEMAND_DATA_HITS 2540
+#define NETDATA_CHART_PRIO_ZFS_PREFETCH_DATA_HITS 2550
+#define NETDATA_CHART_PRIO_ZFS_PHITS 2560
+#define NETDATA_CHART_PRIO_ZFS_MHITS 2570
+#define NETDATA_CHART_PRIO_ZFS_L2HITS 2580
+#define NETDATA_CHART_PRIO_ZFS_LIST_HITS 2600
+#define NETDATA_CHART_PRIO_ZFS_HASH_ELEMENTS 2800
+#define NETDATA_CHART_PRIO_ZFS_HASH_CHAINS 2810
+
+#define NETDATA_CHART_PRIO_ZFS_POOL_STATE 2820
+
+// HARDIRQS
+
+#define NETDATA_CHART_PRIO_HARDIRQ_LATENCY 2900
+
+// SOFTIRQs
+
+#define NETDATA_CHART_PRIO_SOFTIRQS_PER_CORE 3000 // +1 per core
+
+// IPFW (freebsd)
+
+#define NETDATA_CHART_PRIO_IPFW_PACKETS 3001
+#define NETDATA_CHART_PRIO_IPFW_BYTES 3002
+#define NETDATA_CHART_PRIO_IPFW_ACTIVE 3003
+#define NETDATA_CHART_PRIO_IPFW_EXPIRED 3004
+#define NETDATA_CHART_PRIO_IPFW_MEM 3005
+
+
+// IPVS
+
+#define NETDATA_CHART_PRIO_IPVS_NET 3100
+#define NETDATA_CHART_PRIO_IPVS_SOCKETS 3101
+#define NETDATA_CHART_PRIO_IPVS_PACKETS 3102
+
+// Softnet
+
+#define NETDATA_CHART_PRIO_SOFTNET_PER_CORE 4101 // +1 per core
+
+// IP STACK
+
+#define NETDATA_CHART_PRIO_IP_ERRORS 4100
+#define NETDATA_CHART_PRIO_IP_TCP_CONNABORTS 4210
+#define NETDATA_CHART_PRIO_IP_TCP_SYN_QUEUE 4215
+#define NETDATA_CHART_PRIO_IP_TCP_ACCEPT_QUEUE 4216
+#define NETDATA_CHART_PRIO_IP_TCP_REORDERS 4220
+#define NETDATA_CHART_PRIO_IP_TCP_OFO 4250
+#define NETDATA_CHART_PRIO_IP_TCP_SYNCOOKIES 4260
+#define NETDATA_CHART_PRIO_IP_TCP_MEM 4290
+#define NETDATA_CHART_PRIO_IP_BCAST 4500
+#define NETDATA_CHART_PRIO_IP_BCAST_PACKETS 4510
+#define NETDATA_CHART_PRIO_IP_MCAST 4600
+#define NETDATA_CHART_PRIO_IP_MCAST_PACKETS 4610
+#define NETDATA_CHART_PRIO_IP_ECN 4700
+
+// IPv4
+
+#define NETDATA_CHART_PRIO_IPV4_SOCKETS 5100
+#define NETDATA_CHART_PRIO_IPV4_PACKETS 5130
+#define NETDATA_CHART_PRIO_IPV4_ERRORS 5150
+#define NETDATA_CHART_PRIO_IPV4_ICMP 5170
+#define NETDATA_CHART_PRIO_IPV4_TCP 5200
+#define NETDATA_CHART_PRIO_IPV4_TCP_SOCKETS 5201
+#define NETDATA_CHART_PRIO_IPV4_TCP_MEM 5290
+#define NETDATA_CHART_PRIO_IPV4_UDP 5300
+#define NETDATA_CHART_PRIO_IPV4_UDP_MEM 5390
+#define NETDATA_CHART_PRIO_IPV4_UDPLITE 5400
+#define NETDATA_CHART_PRIO_IPV4_RAW 5450
+#define NETDATA_CHART_PRIO_IPV4_FRAGMENTS 5460
+#define NETDATA_CHART_PRIO_IPV4_FRAGMENTS_MEM 5470
+
+// IPv6
+
+#define NETDATA_CHART_PRIO_IPV6_PACKETS 6200
+#define NETDATA_CHART_PRIO_IPV6_ECT 6210
+#define NETDATA_CHART_PRIO_IPV6_ERRORS 6300
+#define NETDATA_CHART_PRIO_IPV6_FRAGMENTS 6400
+#define NETDATA_CHART_PRIO_IPV6_FRAGSOUT 6401
+#define NETDATA_CHART_PRIO_IPV6_FRAGSIN 6402
+#define NETDATA_CHART_PRIO_IPV6_TCP 6500
+#define NETDATA_CHART_PRIO_IPV6_UDP 6600
+#define NETDATA_CHART_PRIO_IPV6_UDP_PACKETS 6601
+#define NETDATA_CHART_PRIO_IPV6_UDP_ERRORS 6610
+#define NETDATA_CHART_PRIO_IPV6_UDPLITE 6700
+#define NETDATA_CHART_PRIO_IPV6_UDPLITE_PACKETS 6701
+#define NETDATA_CHART_PRIO_IPV6_UDPLITE_ERRORS 6710
+#define NETDATA_CHART_PRIO_IPV6_RAW 6800
+#define NETDATA_CHART_PRIO_IPV6_BCAST 6840
+#define NETDATA_CHART_PRIO_IPV6_MCAST 6850
+#define NETDATA_CHART_PRIO_IPV6_MCAST_PACKETS 6851
+#define NETDATA_CHART_PRIO_IPV6_ICMP 6900
+#define NETDATA_CHART_PRIO_IPV6_ICMP_REDIR 6910
+#define NETDATA_CHART_PRIO_IPV6_ICMP_ERRORS 6920
+#define NETDATA_CHART_PRIO_IPV6_ICMP_ECHOS 6930
+#define NETDATA_CHART_PRIO_IPV6_ICMP_GROUPMEMB 6940
+#define NETDATA_CHART_PRIO_IPV6_ICMP_ROUTER 6950
+#define NETDATA_CHART_PRIO_IPV6_ICMP_NEIGHBOR 6960
+#define NETDATA_CHART_PRIO_IPV6_ICMP_LDV2 6970
+#define NETDATA_CHART_PRIO_IPV6_ICMP_TYPES 6980
+
+
+// Network interfaces
+
+#define NETDATA_CHART_PRIO_FIRST_NET_IFACE 7000 // 6 charts per interface
+#define NETDATA_CHART_PRIO_FIRST_NET_PACKETS 7001
+#define NETDATA_CHART_PRIO_FIRST_NET_ERRORS 7002
+#define NETDATA_CHART_PRIO_FIRST_NET_DROPS 7003
+#define NETDATA_CHART_PRIO_FIRST_NET_EVENTS 7006
+#define NETDATA_CHART_PRIO_CGROUP_NET_IFACE 43000
+
+// SCTP
+
+#define NETDATA_CHART_PRIO_SCTP 7000
+
+// QoS
+
+#define NETDATA_CHART_PRIO_TC_QOS 7000
+#define NETDATA_CHART_PRIO_TC_QOS_PACKETS 7010
+#define NETDATA_CHART_PRIO_TC_QOS_DROPPED 7020
+#define NETDATA_CHART_PRIO_TC_QOS_TOKENS 7030
+#define NETDATA_CHART_PRIO_TC_QOS_CTOKENS 7040
+
+// Infiniband
+#define NETDATA_CHART_PRIO_INFINIBAND 7100
+
+// Netfilter
+
+#define NETDATA_CHART_PRIO_NETFILTER_SOCKETS 8700
+#define NETDATA_CHART_PRIO_NETFILTER_NEW 8701
+#define NETDATA_CHART_PRIO_NETFILTER_CHANGES 8702
+#define NETDATA_CHART_PRIO_NETFILTER_EXPECT 8703
+#define NETDATA_CHART_PRIO_NETFILTER_ERRORS 8705
+#define NETDATA_CHART_PRIO_NETFILTER_SEARCH 8710
+
+// SYNPROXY
+
+#define NETDATA_CHART_PRIO_SYNPROXY_SYN_RECEIVED 8751
+#define NETDATA_CHART_PRIO_SYNPROXY_COOKIES 8752
+#define NETDATA_CHART_PRIO_SYNPROXY_CONN_OPEN 8753
+#define NETDATA_CHART_PRIO_SYNPROXY_ENTRIES 8754
+
+// Linux Power Supply
+
+#define NETDATA_CHART_PRIO_POWER_SUPPLY_CAPACITY 9500 // 4 charts per power supply
+#define NETDATA_CHART_PRIO_POWER_SUPPLY_CHARGE 9501
+#define NETDATA_CHART_PRIO_POWER_SUPPLY_ENERGY 9502
+#define NETDATA_CHART_PRIO_POWER_SUPPLY_VOLTAGE 9503
+
+
+// Wireless
+
+#define NETDATA_CHART_PRIO_WIRELESS_IFACE 7110
+
+// CGROUPS
+
+#define NETDATA_CHART_PRIO_CGROUPS_SYSTEMD 19000 // many charts
+#define NETDATA_CHART_PRIO_CGROUPS_CONTAINERS 40000 // many charts
+
+// STATSD
+
+#define NETDATA_CHART_PRIO_STATSD_PRIVATE 90000 // many charts
+
+// INTERNAL NETDATA INFO
+
+#define NETDATA_CHART_PRIO_CHECKS 99999
+
+#define NETDATA_CHART_PRIO_NETDATA_TIMEX 132030
+#define NETDATA_CHART_PRIO_NETDATA_TC_TIME 1000100
+
+
+#endif //NETDATA_ALL_H
diff --git a/collectors/apps.plugin/Makefile.am b/collectors/apps.plugin/Makefile.am
new file mode 100644
index 0000000..533b14d
--- /dev/null
+++ b/collectors/apps.plugin/Makefile.am
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
+
+dist_libconfig_DATA = \
+ apps_groups.conf \
+ $(NULL)
diff --git a/collectors/apps.plugin/README.md b/collectors/apps.plugin/README.md
new file mode 100644
index 0000000..150889d
--- /dev/null
+++ b/collectors/apps.plugin/README.md
@@ -0,0 +1,399 @@
+<!--
+title: "apps.plugin"
+sidebar_label: "Application monitoring (apps.plugin)"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/apps.plugin/README.md
+-->
+
+# apps.plugin
+
+`apps.plugin` breaks down system resource usage to **processes**, **users** and **user groups**.
+
+To achieve this task, it iterates through the whole process tree, collecting resource usage information
+for every process found running.
+
+Since Netdata needs to present this information in charts and track them through time,
+instead of presenting a `top` like list, `apps.plugin` uses a pre-defined list of **process groups**
+to which it assigns all running processes. This list is customizable via `apps_groups.conf`, and Netdata
+ships with a good default for most cases (to edit it on your system run `/etc/netdata/edit-config apps_groups.conf`).
+
+So, `apps.plugin` builds a process tree (much like `ps fax` does in Linux), and groups
+processes together (evaluating both child and parent processes) so that the result is always a list with
+a predefined set of members (of course, only process groups found running are reported).
+
+> If you find that `apps.plugin` categorizes standard applications as `other`, we would be
+> glad to accept pull requests improving the defaults shipped with Netdata in `apps_groups.conf`.
+
+Unlike traditional process monitoring tools (like `top`), `apps.plugin` is able to account the resource
+utilization of exit processes. Their utilization is accounted at their currently running parents.
+So, `apps.plugin` is perfectly able to measure the resources used by shell scripts and other processes
+that fork/spawn other short lived processes hundreds of times per second.
+
+## Charts
+
+`apps.plugin` provides charts for 3 sections:
+
+1. Per application charts as **Applications** at Netdata dashboards
+2. Per user charts as **Users** at Netdata dashboards
+3. Per user group charts as **User Groups** at Netdata dashboards
+
+Each of these sections provides the same number of charts:
+
+- CPU utilization (`apps.cpu`)
+ - Total CPU usage
+ - User/system CPU usage (`apps.cpu_user`/`apps.cpu_system`)
+- Disk I/O
+ - Physical reads/writes (`apps.preads`/`apps.pwrites`)
+ - Logical reads/writes (`apps.lreads`/`apps.lwrites`)
+ - Open unique files (if a file is found open multiple times, it is counted just once, `apps.files`)
+- Memory
+ - Real Memory Used (non-shared, `apps.mem`)
+ - Virtual Memory Allocated (`apps.vmem`)
+ - Minor page faults (i.e. memory activity, `apps.minor_faults`)
+- Processes
+ - Threads running (`apps.threads`)
+ - Processes running (`apps.processes`)
+ - Carried over uptime (since the last Netdata Agent restart, `apps.uptime`)
+ - Minimum uptime (`apps.uptime_min`)
+ - Average uptime (`apps.uptime_average`)
+ - Maximum uptime (`apps.uptime_max`)
+ - Pipes open (`apps.pipes`)
+- Swap memory
+ - Swap memory used (`apps.swap`)
+ - Major page faults (i.e. swap activity, `apps.major_faults`)
+- Network
+ - Sockets open (`apps.sockets`)
+
+In addition, if the [eBPF collector](/collectors/ebpf.plugin/README.md) is running, your dashboard will also show an
+additional [list of charts](/collectors/ebpf.plugin/README.md#integration-with-appsplugin) using low-level Linux
+metrics.
+
+The above are reported:
+
+- For **Applications** per target configured.
+- For **Users** per username or UID (when the username is not available).
+- For **User Groups** per groupname or GID (when groupname is not available).
+
+## Performance
+
+`apps.plugin` is a complex piece of software and has a lot of work to do
+We are proud that `apps.plugin` is a lot faster compared to any other similar tool,
+while collecting a lot more information for the processes, however the fact is that
+this plugin requires more CPU resources than the `netdata` daemon itself.
+
+Under Linux, for each process running, `apps.plugin` reads several `/proc` files
+per process. Doing this work per-second, especially on hosts with several thousands
+of processes, may increase the CPU resources consumed by the plugin.
+
+In such cases, you many need to lower its data collection frequency.
+
+To do this, edit `/etc/netdata/netdata.conf` and find this section:
+
+```
+[plugin:apps]
+ # update every = 1
+ # command options =
+```
+
+Uncomment the line `update every` and set it to a higher number. If you just set it to `2`,
+its CPU resources will be cut in half, and data collection will be once every 2 seconds.
+
+## Configuration
+
+The configuration file is `/etc/netdata/apps_groups.conf`. To edit it on your system, run `/etc/netdata/edit-config apps_groups.conf`.
+
+The configuration file works accepts multiple lines, each having this format:
+
+```txt
+group: process1 process2 ...
+```
+
+Each group can be given multiple times, to add more processes to it.
+
+For the **Applications** section, only groups configured in this file are reported.
+All other processes will be reported as `other`.
+
+For each process given, its whole process tree will be grouped, not just the process matched.
+The plugin will include both parents and children. If including the parents into the group is
+undesirable, the line `other: *` should be appended to the `apps_groups.conf`.
+
+The process names are the ones returned by:
+
+- `ps -e` or `cat /proc/PID/stat`
+- in case of substring mode (see below): `/proc/PID/cmdline`
+
+To add process names with spaces, enclose them in quotes (single or double)
+example: `'Plex Media Serv'` or `"my other process"`.
+
+You can add an asterisk `*` at the beginning and/or the end of a process:
+
+- `*name` _suffix_ mode: will search for processes ending with `name` (at `/proc/PID/stat`)
+- `name*` _prefix_ mode: will search for processes beginning with `name` (at `/proc/PID/stat`)
+- `*name*` _substring_ mode: will search for `name` in the whole command line (at `/proc/PID/cmdline`)
+
+If you enter even just one _name_ (substring), `apps.plugin` will process
+`/proc/PID/cmdline` for all processes (of course only once per process: when they are first seen).
+
+To add processes with single quotes, enclose them in double quotes: `"process with this ' single quote"`
+
+To add processes with double quotes, enclose them in single quotes: `'process with this " double quote'`
+
+If a group or process name starts with a `-`, the dimension will be hidden from the chart (cpu chart only).
+
+If a process starts with a `+`, debugging will be enabled for it (debugging produces a lot of output - do not enable it in production systems).
+
+You can add any number of groups. Only the ones found running will affect the charts generated.
+However, producing charts with hundreds of dimensions may slow down your web browser.
+
+The order of the entries in this list is important: the first that matches a process is used, so put important
+ones at the top. Processes not matched by any row, will inherit it from their parents or children.
+
+The order also controls the order of the dimensions on the generated charts (although applications started
+after apps.plugin is started, will be appended to the existing list of dimensions the `netdata` daemon maintains).
+
+There are a few command line options you can pass to `apps.plugin`. The list of available options can be acquired with the `--help` flag. The options can be set in the `netdata.conf` file. For example, to disable user and user group charts you should set
+
+```
+[plugin:apps]
+ command options = without-users without-groups
+```
+
+### Integration with eBPF
+
+If you don't see charts under the **eBPF syscall** or **eBPF net** sections, you should edit your
+[`ebpf.d.conf`](/collectors/ebpf.plugin/README.md#configure-the-ebpf-collector) file to ensure the eBPF program is enabled.
+
+Also see our [guide on troubleshooting apps with eBPF
+metrics](/docs/guides/troubleshoot/monitor-debug-applications-ebpf.md) for ideas on how to interpret these charts in a
+few scenarios.
+
+## Permissions
+
+`apps.plugin` requires additional privileges to collect all the information it needs.
+The problem is described in issue #157.
+
+When Netdata is installed, `apps.plugin` is given the capabilities `cap_dac_read_search,cap_sys_ptrace+ep`.
+If this fails (i.e. `setcap` fails), `apps.plugin` is setuid to `root`.
+
+### linux capabilities in containers
+
+There are a few cases, like `docker` and `virtuozzo` containers, where `setcap` succeeds, but the capabilities
+are silently ignored (in `lxc` containers `setcap` fails).
+
+In these cases ()`setcap` succeeds but capabilities do not work), you will have to setuid
+to root `apps.plugin` by running these commands:
+
+```sh
+chown root:netdata /usr/libexec/netdata/plugins.d/apps.plugin
+chmod 4750 /usr/libexec/netdata/plugins.d/apps.plugin
+```
+
+You will have to run these, every time you update Netdata.
+
+## Security
+
+`apps.plugin` performs a hard-coded function of building the process tree in memory,
+iterating forever, collecting metrics for each running process and sending them to Netdata.
+This is a one-way communication, from `apps.plugin` to Netdata.
+
+So, since `apps.plugin` cannot be instructed by Netdata for the actions it performs,
+we think it is pretty safe to allow it have these increased privileges.
+
+Keep in mind that `apps.plugin` will still run without escalated permissions,
+but it will not be able to collect all the information.
+
+## Application Badges
+
+You can create badges that you can embed anywhere you like, with URLs like this:
+
+```
+https://your.netdata.ip:19999/api/v1/badge.svg?chart=apps.processes&dimensions=myapp&value_color=green%3E0%7Cred
+```
+
+The color expression unescaped is this: `value_color=green>0|red`.
+
+Here is an example for the process group `sql` at `https://registry.my-netdata.io`:
+
+![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.processes&dimensions=sql&value_color=green%3E0%7Cred)
+
+Netdata is able give you a lot more badges for your app.
+Examples below for process group `sql`:
+
+- CPU usage: ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.cpu&dimensions=sql&value_color=green=0%7Corange%3C50%7Cred)
+- Disk Physical Reads ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.preads&dimensions=sql&value_color=green%3C100%7Corange%3C1000%7Cred)
+- Disk Physical Writes ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.pwrites&dimensions=sql&value_color=green%3C100%7Corange%3C1000%7Cred)
+- Disk Logical Reads ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.lreads&dimensions=sql&value_color=green%3C100%7Corange%3C1000%7Cred)
+- Disk Logical Writes ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.lwrites&dimensions=sql&value_color=green%3C100%7Corange%3C1000%7Cred)
+- Open Files ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.files&dimensions=sql&value_color=green%3E30%7Cred)
+- Real Memory ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.mem&dimensions=sql&value_color=green%3C100%7Corange%3C200%7Cred)
+- Virtual Memory ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.vmem&dimensions=sql&value_color=green%3C100%7Corange%3C1000%7Cred)
+- Swap Memory ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.swap&dimensions=sql&value_color=green=0%7Cred)
+- Minor Page Faults ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.minor_faults&dimensions=sql&value_color=green%3C100%7Corange%3C1000%7Cred)
+- Processes ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.processes&dimensions=sql&value_color=green%3E0%7Cred)
+- Threads ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.threads&dimensions=sql&value_color=green%3E=28%7Cred)
+- Major Faults (swap activity) ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.major_faults&dimensions=sql&value_color=green=0%7Cred)
+- Open Pipes ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.pipes&dimensions=sql&value_color=green=0%7Cred)
+- Open Sockets ![image](https://registry.my-netdata.io/api/v1/badge.svg?chart=apps.sockets&dimensions=sql&value_color=green%3E=3%7Cred)
+
+For more information about badges check [Generating Badges](/web/api/badges/README.md)
+
+## Comparison with console tools
+
+SSH to a server running Netdata and execute this:
+
+```sh
+while true; do ls -l /var/run >/dev/null; done
+```
+
+In most systems `/var/run` is a `tmpfs` device, so there is nothing that can stop this command
+from consuming entirely one of the CPU cores of the machine.
+
+As we will see below, **none** of the console performance monitoring tools can report that this
+command is using 100% CPU. They do report of course that the CPU is busy, but **they fail to
+identify the process that consumes so much CPU**.
+
+Here is what common Linux console monitoring tools report:
+
+### top
+
+`top` reports that `bash` is using just 14%.
+
+If you check the total system CPU utilization, it says there is no idle CPU at all, but `top`
+fails to provide a breakdown of the CPU consumption in the system. The sum of the CPU utilization
+of all processes reported by `top`, is 15.6%.
+
+```
+top - 18:46:28 up 3 days, 20:14, 2 users, load average: 0.22, 0.05, 0.02
+Tasks: 76 total, 2 running, 74 sleeping, 0 stopped, 0 zombie
+%Cpu(s): 32.8 us, 65.6 sy, 0.0 ni, 0.0 id, 0.0 wa, 1.3 hi, 0.3 si, 0.0 st
+KiB Mem : 1016576 total, 244112 free, 52012 used, 720452 buff/cache
+KiB Swap: 0 total, 0 free, 0 used. 753712 avail Mem
+
+ PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
+12789 root 20 0 14980 4180 3020 S 14.0 0.4 0:02.82 bash
+ 9 root 20 0 0 0 0 S 1.0 0.0 0:22.36 rcuos/0
+ 642 netdata 20 0 132024 20112 2660 S 0.3 2.0 14:26.29 netdata
+12522 netdata 20 0 9508 2476 1828 S 0.3 0.2 0:02.26 apps.plugin
+ 1 root 20 0 67196 10216 7500 S 0.0 1.0 0:04.83 systemd
+ 2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd
+```
+
+### htop
+
+Exactly like `top`, `htop` is providing an incomplete breakdown of the system CPU utilization.
+
+```
+ CPU[||||||||||||||||||||||||100.0%] Tasks: 27, 11 thr; 2 running
+ Mem[||||||||||||||||||||85.4M/993M] Load average: 1.16 0.88 0.90
+ Swp[ 0K/0K] Uptime: 3 days, 21:37:03
+
+ PID USER PRI NI VIRT RES SHR S CPU% MEM% TIME+ Command
+12789 root 20 0 15104 4484 3208 S 14.0 0.4 10:57.15 -bash
+ 7024 netdata 20 0 9544 2480 1744 S 0.7 0.2 0:00.88 /usr/libexec/netd
+ 7009 netdata 20 0 138M 21016 2712 S 0.7 2.1 0:00.89 /usr/sbin/netdata
+ 7012 netdata 20 0 138M 21016 2712 S 0.0 2.1 0:00.31 /usr/sbin/netdata
+ 563 root 20 0 308M 202M 202M S 0.0 20.4 1:00.81 /usr/lib/systemd/
+ 7019 netdata 20 0 138M 21016 2712 S 0.0 2.1 0:00.14 /usr/sbin/netdata
+```
+
+### atop
+
+`atop` also fails to break down CPU usage.
+
+```
+ATOP - localhost 2016/12/10 20:11:27 ----------- 10s elapsed
+PRC | sys 1.13s | user 0.43s | #proc 75 | #zombie 0 | #exit 5383 |
+CPU | sys 67% | user 31% | irq 2% | idle 0% | wait 0% |
+CPL | avg1 1.34 | avg5 1.05 | avg15 0.96 | csw 51346 | intr 10508 |
+MEM | tot 992.8M | free 211.5M | cache 470.0M | buff 87.2M | slab 164.7M |
+SWP | tot 0.0M | free 0.0M | | vmcom 207.6M | vmlim 496.4M |
+DSK | vda | busy 0% | read 0 | write 4 | avio 1.50 ms |
+NET | transport | tcpi 16 | tcpo 15 | udpi 0 | udpo 0 |
+NET | network | ipi 16 | ipo 15 | ipfrw 0 | deliv 16 |
+NET | eth0 ---- | pcki 16 | pcko 15 | si 1 Kbps | so 4 Kbps |
+
+ PID SYSCPU USRCPU VGROW RGROW RDDSK WRDSK ST EXC S CPU CMD 1/600
+12789 0.98s 0.40s 0K 0K 0K 336K -- - S 14% bash
+ 9 0.08s 0.00s 0K 0K 0K 0K -- - S 1% rcuos/0
+ 7024 0.03s 0.00s 0K 0K 0K 0K -- - S 0% apps.plugin
+ 7009 0.01s 0.01s 0K 0K 0K 4K -- - S 0% netdata
+```
+
+### glances
+
+And the same is true for `glances`. The system runs at 100%, but `glances` reports only 17%
+per process utilization.
+
+Note also, that being a `python` program, `glances` uses 1.6% CPU while it runs.
+
+```
+localhost Uptime: 3 days, 21:42:00
+
+CPU [100.0%] CPU 100.0% MEM 23.7% SWAP 0.0% LOAD 1-core
+MEM [ 23.7%] user: 30.9% total: 993M total: 0 1 min: 1.18
+SWAP [ 0.0%] system: 67.8% used: 236M used: 0 5 min: 1.08
+ idle: 0.0% free: 757M free: 0 15 min: 1.00
+
+NETWORK Rx/s Tx/s TASKS 75 (90 thr), 1 run, 74 slp, 0 oth
+eth0 168b 2Kb
+eth1 0b 0b CPU% MEM% PID USER NI S Command
+lo 0b 0b 13.5 0.4 12789 root 0 S -bash
+ 1.6 2.2 7025 root 0 R /usr/bin/python /u
+DISK I/O R/s W/s 1.0 0.0 9 root 0 S rcuos/0
+vda1 0 4K 0.3 0.2 7024 netdata 0 S /usr/libexec/netda
+ 0.3 0.0 7 root 0 S rcu_sched
+FILE SYS Used Total 0.3 2.1 7009 netdata 0 S /usr/sbin/netdata
+/ (vda1) 1.56G 29.5G 0.0 0.0 17 root 0 S oom_reaper
+```
+
+### why does this happen?
+
+All the console tools report usage based on the processes found running *at the moment they
+examine the process tree*. So, they see just one `ls` command, which is actually very quick
+with minor CPU utilization. But the shell, is spawning hundreds of them, one after another
+(much like shell scripts do).
+
+### What does Netdata report?
+
+The total CPU utilization of the system:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/21076212/9198e5a6-bf2e-11e6-9bc0-6bdea25befb2.png)
+<br/>***Figure 1**: The system overview section at Netdata, just a few seconds after the command was run*
+
+And at the applications `apps.plugin` breaks down CPU usage per application:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/21076220/c9687848-bf2e-11e6-8d81-348592c5aca2.png)
+<br/>***Figure 2**: The Applications section at Netdata, just a few seconds after the command was run*
+
+So, the `ssh` session is using 95% CPU time.
+
+Why `ssh`?
+
+`apps.plugin` groups all processes based on its configuration file.
+The default configuration has nothing for `bash`, but it has for `sshd`, so Netdata accumulates
+all ssh sessions to a dimension on the charts, called `ssh`. This includes all the processes in
+the process tree of `sshd`, **including the exited children**.
+
+> Distributions based on `systemd`, provide another way to get cpu utilization per user session
+> or service running: control groups, or cgroups, commonly used as part of containers
+> `apps.plugin` does not use these mechanisms. The process grouping made by `apps.plugin` works
+> on any Linux, `systemd` based or not.
+
+#### a more technical description of how Netdata works
+
+Netdata reads `/proc/<pid>/stat` for all processes, once per second and extracts `utime` and
+`stime` (user and system cpu utilization), much like all the console tools do.
+
+But it also extracts `cutime` and `cstime` that account the user and system time of the exit children of each process.
+By keeping a map in memory of the whole process tree, it is capable of assigning the right time to every process, taking
+into account all its exited children.
+
+It is tricky, since a process may be running for 1 hour and once it exits, its parent should not
+receive the whole 1 hour of cpu time in just 1 second - you have to subtract the cpu time that has
+been reported for it prior to this iteration.
+
+It is even trickier, because walking through the entire process tree takes some time itself. So,
+if you sum the CPU utilization of all processes, you might have more CPU time than the reported
+total cpu time of the system. Netdata solves this, by adapting the per process cpu utilization to
+the total of the system. [Netdata adds charts that document this normalization](https://london.my-netdata.io/default.html#menu_netdata_submenu_apps_plugin).
+
+
diff --git a/collectors/apps.plugin/apps_groups.conf b/collectors/apps.plugin/apps_groups.conf
new file mode 100644
index 0000000..dd45c5a
--- /dev/null
+++ b/collectors/apps.plugin/apps_groups.conf
@@ -0,0 +1,422 @@
+#
+# apps.plugin process grouping
+#
+# The apps.plugin displays charts with information about the processes running.
+# This config allows grouping processes together, so that several processes
+# will be reported as one.
+#
+# Only groups in this file are reported. All other processes will be reported
+# as 'other'.
+#
+# For each process given, its whole process tree will be grouped, not just
+# the process matched. The plugin will include both parents and childs.
+#
+# The format is:
+#
+# group: process1 process2 process3 ...
+#
+# Each group can be given multiple times, to add more processes to it.
+#
+# The process names are the ones returned by:
+#
+# - ps -e or /proc/PID/stat
+# - in case of substring mode (see below): /proc/PID/cmdline
+#
+# To add process names with spaces, enclose them in quotes (single or double)
+# example: 'Plex Media Serv' "my other process".
+#
+# Note that spaces are not supported for process groups. Use a dash "-" instead.
+# example-process-group: process1 process2
+#
+# Wildcard support:
+# You can add an asterisk (*) at the beginning and/or the end of a process:
+#
+# *name suffix mode: will search for processes ending with 'name'
+# (/proc/PID/stat)
+#
+# name* prefix mode: will search for processes beginning with 'name'
+# (/proc/PID/stat)
+#
+# *name* substring mode: will search for 'name' in the whole command line
+# (/proc/PID/cmdline)
+#
+# If you enter even just one *name* (substring), apps.plugin will process
+# /proc/PID/cmdline for all processes, just once (when they are first seen).
+#
+# To add processes with single quotes, enclose them in double quotes
+# example: "process with this ' single quote"
+#
+# To add processes with double quotes, enclose them in single quotes:
+# example: 'process with this " double quote'
+#
+# If a group or process name starts with a -, the dimension will be hidden
+# (cpu chart only).
+#
+# If a process starts with a +, debugging will be enabled for it
+# (debugging produces a lot of output - do not enable it in production systems)
+#
+# You can add any number of groups you like. Only the ones found running will
+# affect the charts generated. However, producing charts with hundreds of
+# dimensions may slow down your web browser.
+#
+# The order of the entries in this list is important: the first that matches
+# a process is used, so put important ones at the top. Processes not matched
+# by any row, will inherit it from their parents or children.
+#
+# The order also controls the order of the dimensions on the generated charts
+# (although applications started after apps.plugin is started, will be appended
+# to the existing list of dimensions the netdata daemon maintains).
+
+# -----------------------------------------------------------------------------
+# NETDATA processes accounting
+
+# netdata main process
+netdata: netdata
+
+# netdata known plugins
+# plugins not defined here will be accumulated in netdata, above
+apps.plugin: apps.plugin
+freeipmi.plugin: freeipmi.plugin
+nfacct.plugin: nfacct.plugin
+cups.plugin: cups.plugin
+xenstat.plugin: xenstat.plugin
+perf.plugin: perf.plugin
+charts.d.plugin: *charts.d.plugin*
+python.d.plugin: *python.d.plugin*
+tc-qos-helper: *tc-qos-helper.sh*
+fping: fping
+ioping: ioping
+go.d.plugin: *go.d.plugin*
+slabinfo.plugin: slabinfo.plugin
+ebpf.plugin: *ebpf.plugin*
+
+# agent-service-discovery
+agent_sd: agent_sd
+
+# -----------------------------------------------------------------------------
+# authentication/authorization related servers
+
+auth: radius* openldap* ldap* slapd authelia sssd saslauthd polkitd gssproxy
+fail2ban: fail2ban*
+
+# -----------------------------------------------------------------------------
+# web/ftp servers
+
+httpd: apache* httpd nginx* lighttpd hiawatha caddy h2o
+proxy: squid* c-icap squidGuard varnish*
+php: php* lsphp*
+ftpd: proftpd in.tftpd vsftpd
+uwsgi: uwsgi
+unicorn: *unicorn*
+puma: *puma*
+
+# -----------------------------------------------------------------------------
+# database servers
+
+sql: mysqld* mariad* postgres* postmaster* oracle_* ora_* sqlservr
+nosql: mongod redis* memcached *couchdb*
+timedb: prometheus *carbon-cache.py* *carbon-aggregator.py* *graphite/manage.py* *net.opentsdb.tools.TSDMain* influxd*
+columndb: clickhouse-server*
+
+# -----------------------------------------------------------------------------
+# email servers
+
+mta: amavis* zmstat-* zmdiaglog zmmailboxdmgr opendkim postfwd2 smtp* lmtp* sendmail postfix master pickup qmgr showq tlsmgr postscreen oqmgr msmtp* nullmailer*
+mda: dovecot *imapd *pop3d *popd
+
+# -----------------------------------------------------------------------------
+# network, routing, VPN
+
+ppp: ppp*
+vpn: openvpn pptp* cjdroute gvpe tincd wireguard tailscaled
+wifi: hostapd wpa_supplicant
+routing: ospfd* ospf6d* bgpd bfdd fabricd isisd eigrpd sharpd staticd ripd ripngd pimd pbrd nhrpd ldpd zebra vrrpd vtysh bird*
+modem: ModemManager
+netmanager: NetworkManager nm* systemd-networkd networkctl netplan connmand wicked* avahi-autoipd networkd-dispatcher
+firewall: firewalld ufw nft
+tor: tor
+bluetooth: bluetooth bluez bluedevil obexd
+
+# -----------------------------------------------------------------------------
+# high availability and balancers
+
+camo: *camo*
+balancer: ipvs_* haproxy
+ha: corosync hs_logd ha_logd stonithd pacemakerd lrmd crmd keepalived ucarp*
+
+# -----------------------------------------------------------------------------
+# telephony
+
+pbx: asterisk safe_asterisk *vicidial*
+sip: opensips* stund
+
+# -----------------------------------------------------------------------------
+# chat
+
+chat: irssi *vines* *prosody* murmurd
+
+# -----------------------------------------------------------------------------
+# monitoring
+
+logs: ulogd* syslog* rsyslog* logrotate systemd-journald rotatelogs sysklogd metalog
+nms: snmpd vnstatd smokeping zabbix* munin* mon openhpid tailon nrpe
+monit: monit
+splunk: splunkd
+azure: mdsd *waagent* *omiserver* *omiagent* hv_kvp_daemon hv_vss_daemon *auoms* *omsagent*
+datadog: *datadog*
+edgedelta: edgedelta
+newrelic: newrelic*
+google-agent: *google_guest_agent* *google_osconfig_agent*
+nvidia-smi: nvidia-smi
+htop: htop
+watchdog: watchdog
+
+# -----------------------------------------------------------------------------
+# storage, file systems and file servers
+
+ceph: ceph-* ceph_* radosgw* rbd-* cephfs-* osdmaptool crushtool
+samba: smbd nmbd winbindd ctdbd ctdb-* ctdb_*
+nfs: rpcbind rpc.* nfs*
+zfs: spl_* z_* txg_* zil_* arc_* l2arc*
+btrfs: btrfs*
+iscsi: iscsid iscsi_eh
+afp: netatalk afpd cnid_dbd cnid_metad
+ntfs-3g: ntfs-3g
+
+# -----------------------------------------------------------------------------
+# kubernetes
+
+kubelet: kubelet
+kube-dns: kube-dns
+kube-proxy: kube-proxy
+metrics-server: metrics-server
+heapster: heapster
+
+# -----------------------------------------------------------------------------
+# AWS
+
+aws-s3: '*aws s3*' s3cmd s5cmd
+aws: aws
+
+# -----------------------------------------------------------------------------
+# virtualization platform
+
+proxmox-ve: pve* spiceproxy
+
+# -----------------------------------------------------------------------------
+# containers & virtual machines
+
+containers: lxc* docker* balena*
+VMs: vbox* VBox* qemu* kvm*
+libvirt: virtlogd virtqemud virtstoraged virtnetworkd virtlockd virtinterfaced
+libvirt: virtnodedevd virtproxyd virtsecretd libvirtd
+guest-agent: qemu-ga spice-vdagent cloud-init*
+
+# -----------------------------------------------------------------------------
+# ssh servers and clients
+
+ssh: ssh* scp sftp* dropbear
+
+# -----------------------------------------------------------------------------
+# print servers and clients
+
+print: cups* lpd lpq
+
+# -----------------------------------------------------------------------------
+# time servers and clients
+
+time: ntp* systemd-timesyn* chronyd ptp*
+
+# -----------------------------------------------------------------------------
+# dhcp servers and clients
+
+dhcp: *dhcp* dhclient
+
+# -----------------------------------------------------------------------------
+# name servers and clients
+
+dns: named unbound nsd pdns_server knotd gdnsd yadifad dnsmasq systemd-resolve* pihole* avahi-daemon avahi-dnsconfd
+dnsdist: dnsdist
+
+# -----------------------------------------------------------------------------
+# installation / compilation / debugging
+
+build: cc1 cc1plus as gcc* cppcheck ld make cmake automake autoconf autoreconf
+build: cargo rustc bazel buck git gdb valgrind* rpmbuild dpkg-buildpackage
+
+# -----------------------------------------------------------------------------
+# package management
+
+packagemanager: apt* dpkg* dselect dnf yum rpm zypp* yast* pacman xbps* swupd* emerge*
+packagemanager: packagekitd pkgin pkg apk snapd slackpkg slapt-get
+
+# -----------------------------------------------------------------------------
+# antivirus
+
+antivirus: clam* *clam imunify360*
+
+# -----------------------------------------------------------------------------
+# torrent clients
+
+torrents: *deluge* transmission* *SickBeard* *CouchPotato* *rtorrent*
+
+# -----------------------------------------------------------------------------
+# backup servers and clients
+
+backup: rsync lsyncd bacula* borg rclone
+
+# -----------------------------------------------------------------------------
+# cron
+
+cron: cron* atd anacron systemd-cron* incrond
+
+# -----------------------------------------------------------------------------
+# UPS
+
+ups: upsmon upsd */nut/* apcupsd
+
+# -----------------------------------------------------------------------------
+# media players, servers, clients
+
+media: mplayer vlc xine mediatomb omxplayer* kodi* xbmc* mediacenter eventlircd
+media: mpd minidlnad mt-daapd Plex* jellyfin squeeze* jackett Ombi
+media: strawberry* clementine*
+
+audio: pulse* pipewire wireplumber jack*
+
+# -----------------------------------------------------------------------------
+# java applications
+
+hdfsdatanode: *org.apache.hadoop.hdfs.server.datanode.DataNode*
+hdfsnamenode: *org.apache.hadoop.hdfs.server.namenode.NameNode*
+hdfsjournalnode: *org.apache.hadoop.hdfs.qjournal.server.JournalNode*
+hdfszkfc: *org.apache.hadoop.hdfs.tools.DFSZKFailoverController*
+
+yarnnode: *org.apache.hadoop.yarn.server.nodemanager.NodeManager*
+yarnmgr: *org.apache.hadoop.yarn.server.resourcemanager.ResourceManager*
+yarnproxy: *org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer*
+
+sparkworker: *org.apache.spark.deploy.worker.Worker*
+sparkmaster: *org.apache.spark.deploy.master.Master*
+
+hbaseregion: *org.apache.hadoop.hbase.regionserver.HRegionServer*
+hbaserest: *org.apache.hadoop.hbase.rest.RESTServer*
+hbasethrift: *org.apache.hadoop.hbase.thrift.ThriftServer*
+hbasemaster: *org.apache.hadoop.hbase.master.HMaster*
+
+zookeeper: *org.apache.zookeeper.server.quorum.QuorumPeerMain*
+
+hive2: *org.apache.hive.service.server.HiveServer2*
+hivemetastore: *org.apache.hadoop.hive.metastore.HiveMetaStore*
+
+solr: *solr.install.dir*
+
+airflow: *airflow*
+
+# -----------------------------------------------------------------------------
+# GUI
+
+X: X Xorg xinit xdm Xwayland xsettingsd
+wayland: swaylock swayidle waypipe wayvnc
+kde: *kdeinit* kdm sddm plasmashell startplasma-* kwin* kwallet* krunner kactivitymanager*
+gnome: gnome-* gdm gconf* mutter
+mate: mate-* msd-* marco*
+cinnamon: cinnamon* muffin
+xfce: xfwm4 xfdesktop xfce* Thunar xfsettingsd xfconf*
+lxde: lxde* startlxde lxdm lxappearance* lxlauncher* lxpanel* lxsession* lxsettings*
+lxqt: lxqt* startlxqt
+enlightenment: entrance enlightenment*
+i3: i3*
+awesome: awesome awesome-client
+dwm: dwm.*
+sway: sway
+weston: weston
+cage: cage
+wayfire: wayfire
+gui: lightdm colord seatd greetd gkrellm slim qingy dconf* *gvfs gvfs*
+gui: '*systemd --user*' xdg-* at-spi-*
+
+webbrowser: *chrome-sandbox* *google-chrome* *chromium* *firefox* vivaldi* opera* epiphany chrome*
+webbrowser: lynx elinks w3m w3mmee links
+mua: evolution-* thunderbird* mutt neomutt pine mailx alpine
+
+# -----------------------------------------------------------------------------
+# Kernel / System
+
+ksmd: ksmd
+khugepaged: khugepaged
+kdamond: kdamond
+kswapd: kswapd
+zswap: zswap
+kcompactd: kcompactd
+
+system: systemd-* udisks* udevd* *udevd ipv6_addrconf dbus-* rtkit*
+system: mdadm acpid uuidd upowerd elogind* eudev mdev lvmpolld dmeventd
+system: accounts-daemon rngd haveged rasdaemon irqbalance start-stop-daemon
+system: supervise-daemon openrc* init runit runsvdir runsv auditd lsmd
+system: abrt* nscd rtkit-daemon gpg-agent usbguard*
+
+kernel: kworker kthreadd kauditd lockd khelper kdevtmpfs khungtaskd rpciod
+kernel: fsnotify_mark kthrotld deferwq scsi_* kdmflush oom_reaper kdevtempfs
+kernel: ksoftirqd
+
+# -----------------------------------------------------------------------------
+# inetd
+
+inetd: inetd xinetd
+
+# -----------------------------------------------------------------------------
+# other application servers
+
+consul: consul
+
+kafka: *kafka.Kafka*
+
+rabbitmq: *rabbitmq*
+
+sidekiq: *sidekiq*
+java: java
+ipfs: ipfs
+
+node: node*
+factorio: factorio
+
+p4: p4*
+
+git-services: gitea gitlab-runner
+
+freeswitch: freeswitch*
+
+# -------- web3 / blockchains ----------
+
+go-ethereum: geth*
+nethermind-ethereum: nethermind*
+besu-ethereum: besu*
+openEthereum: openethereum*
+urbit: urbit*
+bitcoin-node: *bitcoind* lnd*
+filecoin: lotus* lotus-miner* lotus-worker*
+solana: solana*
+web3: *hardhat* *ganache* *truffle* *brownie* *waffle*
+terra: terra* mantle*
+
+# -----------------------------------------------------------------------------
+# chaos engineering tools
+
+stress: stress stress-ng*
+gremlin: gremlin*
+
+# -----------------------------------------------------------------------------
+# load testing tools
+
+locust: locust
+
+# -----------------------------------------------------------------------------
+# data science and machine learning tools
+
+jupyter: jupyter*
+
+# -----------------------------------------------------------------------------
+# File synchronization tools
+
+filesync: dropbox syncthing
diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c
new file mode 100644
index 0000000..89b8333
--- /dev/null
+++ b/collectors/apps.plugin/apps_plugin.c
@@ -0,0 +1,5015 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+/*
+ * netdata apps.plugin
+ * (C) Copyright 2016-2017 Costa Tsaousis <costa@tsaousis.gr>
+ * Released under GPL v3+
+ */
+
+#include "collectors/all.h"
+#include "libnetdata/libnetdata.h"
+#include "libnetdata/required_dummies.h"
+
+#define APPS_PLUGIN_FUNCTIONS() do { \
+ fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" 10 \"Detailed information on the currently running processes on this node\"\n"); \
+ } while(0)
+
+
+// ----------------------------------------------------------------------------
+// debugging
+
+static int debug_enabled = 0;
+static inline void debug_log_int(const char *fmt, ... ) {
+ va_list args;
+
+ fprintf( stderr, "apps.plugin: ");
+ va_start( args, fmt );
+ vfprintf( stderr, fmt, args );
+ va_end( args );
+
+ fputc('\n', stderr);
+}
+
+#ifdef NETDATA_INTERNAL_CHECKS
+
+#define debug_log(fmt, args...) do { if(unlikely(debug_enabled)) debug_log_int(fmt, ##args); } while(0)
+
+#else
+
+static inline void debug_log_dummy(void) {}
+#define debug_log(fmt, args...) debug_log_dummy()
+
+#endif
+
+
+// ----------------------------------------------------------------------------
+
+#ifdef __FreeBSD__
+#include <sys/user.h>
+#endif
+
+// ----------------------------------------------------------------------------
+// per O/S configuration
+
+// the minimum PID of the system
+// this is also the pid of the init process
+#define INIT_PID 1
+
+// if the way apps.plugin will work, will read the entire process list,
+// including the resource utilization of each process, instantly
+// set this to 1
+// when set to 0, apps.plugin builds a sort list of processes, in order
+// to process children processes, before parent processes
+#ifdef __FreeBSD__
+#define ALL_PIDS_ARE_READ_INSTANTLY 1
+#else
+#define ALL_PIDS_ARE_READ_INSTANTLY 0
+#endif
+
+// ----------------------------------------------------------------------------
+// string lengths
+
+#define MAX_COMPARE_NAME 100
+#define MAX_NAME 100
+#define MAX_CMDLINE 16384
+
+// ----------------------------------------------------------------------------
+// the rates we are going to send to netdata will have this detail a value of:
+// - 1 will send just integer parts to netdata
+// - 100 will send 2 decimal points
+// - 1000 will send 3 decimal points
+// etc.
+#define RATES_DETAIL 10000ULL
+
+// ----------------------------------------------------------------------------
+// factor for calculating correct CPU time values depending on units of raw data
+static unsigned int time_factor = 0;
+
+// ----------------------------------------------------------------------------
+// to avoid reallocating too frequently, we can increase the number of spare
+// file descriptors used by processes.
+// IMPORTANT:
+// having a lot of spares, increases the CPU utilization of the plugin.
+#define MAX_SPARE_FDS 1
+
+// ----------------------------------------------------------------------------
+// command line options
+
+static int
+ update_every = 1,
+ enable_guest_charts = 0,
+#ifdef __FreeBSD__
+ enable_file_charts = 0,
+#else
+ enable_file_charts = 1,
+ max_fds_cache_seconds = 60,
+#endif
+ enable_detailed_uptime_charts = 0,
+ enable_users_charts = 1,
+ enable_groups_charts = 1,
+ include_exited_childs = 1;
+
+// will be changed to getenv(NETDATA_USER_CONFIG_DIR) if it exists
+static char *user_config_dir = CONFIG_DIR;
+static char *stock_config_dir = LIBCONFIG_DIR;
+
+// some variables for keeping track of processes count by states
+typedef enum {
+ PROC_STATUS_RUNNING = 0,
+ PROC_STATUS_SLEEPING_D, // uninterruptible sleep
+ PROC_STATUS_SLEEPING, // interruptible sleep
+ PROC_STATUS_ZOMBIE,
+ PROC_STATUS_STOPPED,
+ PROC_STATUS_END, //place holder for ending enum fields
+} proc_state;
+
+#ifndef __FreeBSD__
+static proc_state proc_state_count[PROC_STATUS_END];
+static const char *proc_states[] = {
+ [PROC_STATUS_RUNNING] = "running",
+ [PROC_STATUS_SLEEPING] = "sleeping_interruptible",
+ [PROC_STATUS_SLEEPING_D] = "sleeping_uninterruptible",
+ [PROC_STATUS_ZOMBIE] = "zombie",
+ [PROC_STATUS_STOPPED] = "stopped",
+ };
+#endif
+
+// ----------------------------------------------------------------------------
+// internal flags
+// handled in code (automatically set)
+
+static int
+ show_guest_time = 0, // 1 when guest values are collected
+ show_guest_time_old = 0,
+ proc_pid_cmdline_is_needed = 0; // 1 when we need to read /proc/cmdline
+
+
+// ----------------------------------------------------------------------------
+// internal counters
+
+static size_t
+ global_iterations_counter = 1,
+ calls_counter = 0,
+ file_counter = 0,
+ filenames_allocated_counter = 0,
+ inodes_changed_counter = 0,
+ links_changed_counter = 0,
+ targets_assignment_counter = 0;
+
+
+// ----------------------------------------------------------------------------
+// Normalization
+//
+// With normalization we lower the collected metrics by a factor to make them
+// match the total utilization of the system.
+// The discrepancy exists because apps.plugin needs some time to collect all
+// the metrics. This results in utilization that exceeds the total utilization
+// of the system.
+//
+// With normalization we align the per-process utilization, to the total of
+// the system. We first consume the exited children utilization and it the
+// collected values is above the total, we proportionally scale each reported
+// metric.
+
+// the total system time, as reported by /proc/stat
+static kernel_uint_t
+ global_utime = 0,
+ global_stime = 0,
+ global_gtime = 0;
+
+// the normalization ratios, as calculated by normalize_utilization()
+NETDATA_DOUBLE
+ utime_fix_ratio = 1.0,
+ stime_fix_ratio = 1.0,
+ gtime_fix_ratio = 1.0,
+ minflt_fix_ratio = 1.0,
+ majflt_fix_ratio = 1.0,
+ cutime_fix_ratio = 1.0,
+ cstime_fix_ratio = 1.0,
+ cgtime_fix_ratio = 1.0,
+ cminflt_fix_ratio = 1.0,
+ cmajflt_fix_ratio = 1.0;
+
+
+struct pid_on_target {
+ int32_t pid;
+ struct pid_on_target *next;
+};
+
+struct openfds {
+ kernel_uint_t files;
+ kernel_uint_t pipes;
+ kernel_uint_t sockets;
+ kernel_uint_t inotifies;
+ kernel_uint_t eventfds;
+ kernel_uint_t timerfds;
+ kernel_uint_t signalfds;
+ kernel_uint_t eventpolls;
+ kernel_uint_t other;
+};
+
+// ----------------------------------------------------------------------------
+// target
+//
+// target is the structure that processes are aggregated to be reported
+// to netdata.
+//
+// - Each entry in /etc/apps_groups.conf creates a target.
+// - Each user and group used by a process in the system, creates a target.
+
+struct target {
+ char compare[MAX_COMPARE_NAME + 1];
+ uint32_t comparehash;
+ size_t comparelen;
+
+ char id[MAX_NAME + 1];
+ uint32_t idhash;
+
+ char name[MAX_NAME + 1];
+
+ uid_t uid;
+ gid_t gid;
+
+ kernel_uint_t minflt;
+ kernel_uint_t cminflt;
+ kernel_uint_t majflt;
+ kernel_uint_t cmajflt;
+ kernel_uint_t utime;
+ kernel_uint_t stime;
+ kernel_uint_t gtime;
+ kernel_uint_t cutime;
+ kernel_uint_t cstime;
+ kernel_uint_t cgtime;
+ kernel_uint_t num_threads;
+ // kernel_uint_t rss;
+
+ kernel_uint_t status_vmsize;
+ kernel_uint_t status_vmrss;
+ kernel_uint_t status_vmshared;
+ kernel_uint_t status_rssfile;
+ kernel_uint_t status_rssshmem;
+ kernel_uint_t status_vmswap;
+
+ kernel_uint_t io_logical_bytes_read;
+ kernel_uint_t io_logical_bytes_written;
+ kernel_uint_t io_read_calls;
+ kernel_uint_t io_write_calls;
+ kernel_uint_t io_storage_bytes_read;
+ kernel_uint_t io_storage_bytes_written;
+ kernel_uint_t io_cancelled_write_bytes;
+
+ int *target_fds;
+ int target_fds_size;
+
+ struct openfds openfds;
+
+ kernel_uint_t starttime;
+ kernel_uint_t collected_starttime;
+ kernel_uint_t uptime_min;
+ kernel_uint_t uptime_sum;
+ kernel_uint_t uptime_max;
+
+ unsigned int processes; // how many processes have been merged to this
+ int exposed; // if set, we have sent this to netdata
+ int hidden; // if set, we set the hidden flag on the dimension
+ int debug_enabled;
+ int ends_with;
+ int starts_with; // if set, the compare string matches only the
+ // beginning of the command
+
+ struct pid_on_target *root_pid; // list of aggregated pids for target debugging
+
+ struct target *target; // the one that will be reported to netdata
+ struct target *next;
+};
+
+struct target
+ *apps_groups_default_target = NULL, // the default target
+ *apps_groups_root_target = NULL, // apps_groups.conf defined
+ *users_root_target = NULL, // users
+ *groups_root_target = NULL; // user groups
+
+size_t
+ apps_groups_targets_count = 0; // # of apps_groups.conf targets
+
+
+// ----------------------------------------------------------------------------
+// pid_stat
+//
+// structure to store data for each process running
+// see: man proc for the description of the fields
+
+struct pid_fd {
+ int fd;
+
+#ifndef __FreeBSD__
+ ino_t inode;
+ char *filename;
+ uint32_t link_hash;
+ size_t cache_iterations_counter;
+ size_t cache_iterations_reset;
+#endif
+};
+
+struct pid_stat {
+ int32_t pid;
+ char comm[MAX_COMPARE_NAME + 1];
+ char *cmdline;
+
+ uint32_t log_thrown;
+
+ char state;
+ int32_t ppid;
+ // int32_t pgrp;
+ // int32_t session;
+ // int32_t tty_nr;
+ // int32_t tpgid;
+ // uint64_t flags;
+
+ // these are raw values collected
+ kernel_uint_t minflt_raw;
+ kernel_uint_t cminflt_raw;
+ kernel_uint_t majflt_raw;
+ kernel_uint_t cmajflt_raw;
+ kernel_uint_t utime_raw;
+ kernel_uint_t stime_raw;
+ kernel_uint_t gtime_raw; // guest_time
+ kernel_uint_t cutime_raw;
+ kernel_uint_t cstime_raw;
+ kernel_uint_t cgtime_raw; // cguest_time
+
+ // these are rates
+ kernel_uint_t minflt;
+ kernel_uint_t cminflt;
+ kernel_uint_t majflt;
+ kernel_uint_t cmajflt;
+ kernel_uint_t utime;
+ kernel_uint_t stime;
+ kernel_uint_t gtime;
+ kernel_uint_t cutime;
+ kernel_uint_t cstime;
+ kernel_uint_t cgtime;
+
+ // int64_t priority;
+ // int64_t nice;
+ int32_t num_threads;
+ // int64_t itrealvalue;
+ kernel_uint_t collected_starttime;
+ // kernel_uint_t vsize;
+ // kernel_uint_t rss;
+ // kernel_uint_t rsslim;
+ // kernel_uint_t starcode;
+ // kernel_uint_t endcode;
+ // kernel_uint_t startstack;
+ // kernel_uint_t kstkesp;
+ // kernel_uint_t kstkeip;
+ // uint64_t signal;
+ // uint64_t blocked;
+ // uint64_t sigignore;
+ // uint64_t sigcatch;
+ // uint64_t wchan;
+ // uint64_t nswap;
+ // uint64_t cnswap;
+ // int32_t exit_signal;
+ // int32_t processor;
+ // uint32_t rt_priority;
+ // uint32_t policy;
+ // kernel_uint_t delayacct_blkio_ticks;
+
+ uid_t uid;
+ gid_t gid;
+
+ kernel_uint_t status_vmsize;
+ kernel_uint_t status_vmrss;
+ kernel_uint_t status_vmshared;
+ kernel_uint_t status_rssfile;
+ kernel_uint_t status_rssshmem;
+ kernel_uint_t status_vmswap;
+#ifndef __FreeBSD__
+ ARL_BASE *status_arl;
+#endif
+
+ kernel_uint_t io_logical_bytes_read_raw;
+ kernel_uint_t io_logical_bytes_written_raw;
+ kernel_uint_t io_read_calls_raw;
+ kernel_uint_t io_write_calls_raw;
+ kernel_uint_t io_storage_bytes_read_raw;
+ kernel_uint_t io_storage_bytes_written_raw;
+ kernel_uint_t io_cancelled_write_bytes_raw;
+
+ kernel_uint_t io_logical_bytes_read;
+ kernel_uint_t io_logical_bytes_written;
+ kernel_uint_t io_read_calls;
+ kernel_uint_t io_write_calls;
+ kernel_uint_t io_storage_bytes_read;
+ kernel_uint_t io_storage_bytes_written;
+ kernel_uint_t io_cancelled_write_bytes;
+
+ struct pid_fd *fds; // array of fds it uses
+ size_t fds_size; // the size of the fds array
+
+ struct openfds openfds;
+
+ int children_count; // number of processes directly referencing this
+ unsigned char keep:1; // 1 when we need to keep this process in memory even after it exited
+ int keeploops; // increases by 1 every time keep is 1 and updated 0
+ unsigned char updated:1; // 1 when the process is currently running
+ unsigned char merged:1; // 1 when it has been merged to its parent
+ unsigned char read:1; // 1 when we have already read this process for this iteration
+
+ int sortlist; // higher numbers = top on the process tree
+ // each process gets a unique number
+
+ struct target *target; // app_groups.conf targets
+ struct target *user_target; // uid based targets
+ struct target *group_target; // gid based targets
+
+ usec_t stat_collected_usec;
+ usec_t last_stat_collected_usec;
+
+ usec_t io_collected_usec;
+ usec_t last_io_collected_usec;
+
+ kernel_uint_t uptime;
+
+ char *fds_dirname; // the full directory name in /proc/PID/fd
+
+ char *stat_filename;
+ char *status_filename;
+ char *io_filename;
+ char *cmdline_filename;
+
+ struct pid_stat *parent;
+ struct pid_stat *prev;
+ struct pid_stat *next;
+};
+
+size_t pagesize;
+
+kernel_uint_t global_uptime;
+
+// log each problem once per process
+// log flood protection flags (log_thrown)
+#define PID_LOG_IO 0x00000001
+#define PID_LOG_STATUS 0x00000002
+#define PID_LOG_CMDLINE 0x00000004
+#define PID_LOG_FDS 0x00000008
+#define PID_LOG_STAT 0x00000010
+
+static struct pid_stat
+ *root_of_pids = NULL, // global list of all processes running
+ **all_pids = NULL; // to avoid allocations, we pre-allocate
+ // a pointer for each pid in the entire pid space.
+
+static size_t
+ all_pids_count = 0; // the number of processes running
+
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
+// Another pre-allocated list of all possible pids.
+// We need it to pids and assign them a unique sortlist id, so that we
+// read parents before children. This is needed to prevent a situation where
+// a child is found running, but until we read its parent, it has exited and
+// its parent has accumulated its resources.
+static pid_t
+ *all_pids_sortlist = NULL;
+#endif
+
+// ----------------------------------------------------------------------------
+// file descriptor
+//
+// this is used to keep a global list of all open files of the system.
+// it is needed in order to calculate the unique files processes have open.
+
+#define FILE_DESCRIPTORS_INCREASE_STEP 100
+
+// types for struct file_descriptor->type
+typedef enum fd_filetype {
+ FILETYPE_OTHER,
+ FILETYPE_FILE,
+ FILETYPE_PIPE,
+ FILETYPE_SOCKET,
+ FILETYPE_INOTIFY,
+ FILETYPE_EVENTFD,
+ FILETYPE_EVENTPOLL,
+ FILETYPE_TIMERFD,
+ FILETYPE_SIGNALFD
+} FD_FILETYPE;
+
+struct file_descriptor {
+ avl_t avl;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ uint32_t magic;
+#endif /* NETDATA_INTERNAL_CHECKS */
+
+ const char *name;
+ uint32_t hash;
+
+ FD_FILETYPE type;
+ int count;
+ int pos;
+} *all_files = NULL;
+
+static int
+ all_files_len = 0,
+ all_files_size = 0;
+
+long currentmaxfds = 0;
+
+// ----------------------------------------------------------------------------
+// read users and groups from files
+
+struct user_or_group_id {
+ avl_t avl;
+
+ union {
+ uid_t uid;
+ gid_t gid;
+ } id;
+
+ char *name;
+
+ int updated;
+
+ struct user_or_group_id * next;
+};
+
+enum user_or_group_id_type {
+ USER_ID,
+ GROUP_ID
+};
+
+struct user_or_group_ids{
+ enum user_or_group_id_type type;
+
+ avl_tree_type index;
+ struct user_or_group_id *root;
+
+ char filename[FILENAME_MAX + 1];
+};
+
+int user_id_compare(void* a, void* b) {
+ if(((struct user_or_group_id *)a)->id.uid < ((struct user_or_group_id *)b)->id.uid)
+ return -1;
+
+ else if(((struct user_or_group_id *)a)->id.uid > ((struct user_or_group_id *)b)->id.uid)
+ return 1;
+
+ else
+ return 0;
+}
+
+struct user_or_group_ids all_user_ids = {
+ .type = USER_ID,
+
+ .index = {
+ NULL,
+ user_id_compare
+ },
+
+ .root = NULL,
+
+ .filename = "",
+};
+
+int group_id_compare(void* a, void* b) {
+ if(((struct user_or_group_id *)a)->id.gid < ((struct user_or_group_id *)b)->id.gid)
+ return -1;
+
+ else if(((struct user_or_group_id *)a)->id.gid > ((struct user_or_group_id *)b)->id.gid)
+ return 1;
+
+ else
+ return 0;
+}
+
+struct user_or_group_ids all_group_ids = {
+ .type = GROUP_ID,
+
+ .index = {
+ NULL,
+ group_id_compare
+ },
+
+ .root = NULL,
+
+ .filename = "",
+};
+
+int file_changed(const struct stat *statbuf, struct timespec *last_modification_time) {
+ if(likely(statbuf->st_mtim.tv_sec == last_modification_time->tv_sec &&
+ statbuf->st_mtim.tv_nsec == last_modification_time->tv_nsec)) return 0;
+
+ last_modification_time->tv_sec = statbuf->st_mtim.tv_sec;
+ last_modification_time->tv_nsec = statbuf->st_mtim.tv_nsec;
+
+ return 1;
+}
+
+int read_user_or_group_ids(struct user_or_group_ids *ids, struct timespec *last_modification_time) {
+ struct stat statbuf;
+ if(unlikely(stat(ids->filename, &statbuf)))
+ return 1;
+ else
+ if(likely(!file_changed(&statbuf, last_modification_time))) return 0;
+
+ procfile *ff = procfile_open(ids->filename, " :\t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 1;
+
+ size_t line, lines = procfile_lines(ff);
+
+ for(line = 0; line < lines ;line++) {
+ size_t words = procfile_linewords(ff, line);
+ if(unlikely(words < 3)) continue;
+
+ char *name = procfile_lineword(ff, line, 0);
+ if(unlikely(!name || !*name)) continue;
+
+ char *id_string = procfile_lineword(ff, line, 2);
+ if(unlikely(!id_string || !*id_string)) continue;
+
+
+ struct user_or_group_id *user_or_group_id = callocz(1, sizeof(struct user_or_group_id));
+
+ if(ids->type == USER_ID)
+ user_or_group_id->id.uid = (uid_t)str2ull(id_string);
+ else
+ user_or_group_id->id.gid = (uid_t)str2ull(id_string);
+
+ user_or_group_id->name = strdupz(name);
+ user_or_group_id->updated = 1;
+
+ struct user_or_group_id *existing_user_id = NULL;
+
+ if(likely(ids->root))
+ existing_user_id = (struct user_or_group_id *)avl_search(&ids->index, (avl_t *) user_or_group_id);
+
+ if(unlikely(existing_user_id)) {
+ freez(existing_user_id->name);
+ existing_user_id->name = user_or_group_id->name;
+ existing_user_id->updated = 1;
+ freez(user_or_group_id);
+ }
+ else {
+ if(unlikely(avl_insert(&ids->index, (avl_t *) user_or_group_id) != (void *) user_or_group_id)) {
+ error("INTERNAL ERROR: duplicate indexing of id during realloc");
+ };
+
+ user_or_group_id->next = ids->root;
+ ids->root = user_or_group_id;
+ }
+ }
+
+ procfile_close(ff);
+
+ // remove unused ids
+ struct user_or_group_id *user_or_group_id = ids->root, *prev_user_id = NULL;
+
+ while(user_or_group_id) {
+ if(unlikely(!user_or_group_id->updated)) {
+ if(unlikely((struct user_or_group_id *)avl_remove(&ids->index, (avl_t *) user_or_group_id) != user_or_group_id))
+ error("INTERNAL ERROR: removal of unused id from index, removed a different id");
+
+ if(prev_user_id)
+ prev_user_id->next = user_or_group_id->next;
+ else
+ ids->root = user_or_group_id->next;
+
+ freez(user_or_group_id->name);
+ freez(user_or_group_id);
+
+ if(prev_user_id)
+ user_or_group_id = prev_user_id->next;
+ else
+ user_or_group_id = ids->root;
+ }
+ else {
+ user_or_group_id->updated = 0;
+
+ prev_user_id = user_or_group_id;
+ user_or_group_id = user_or_group_id->next;
+ }
+ }
+
+ return 0;
+}
+
+// ----------------------------------------------------------------------------
+// apps_groups.conf
+// aggregate all processes in groups, to have a limited number of dimensions
+
+static struct target *get_users_target(uid_t uid) {
+ struct target *w;
+ for(w = users_root_target ; w ; w = w->next)
+ if(w->uid == uid) return w;
+
+ w = callocz(sizeof(struct target), 1);
+ snprintfz(w->compare, MAX_COMPARE_NAME, "%u", uid);
+ w->comparehash = simple_hash(w->compare);
+ w->comparelen = strlen(w->compare);
+
+ snprintfz(w->id, MAX_NAME, "%u", uid);
+ w->idhash = simple_hash(w->id);
+
+ struct user_or_group_id user_id_to_find, *user_or_group_id = NULL;
+ user_id_to_find.id.uid = uid;
+
+ if(*netdata_configured_host_prefix) {
+ static struct timespec last_passwd_modification_time;
+ int ret = read_user_or_group_ids(&all_user_ids, &last_passwd_modification_time);
+
+ if(likely(!ret && all_user_ids.index.root))
+ user_or_group_id = (struct user_or_group_id *)avl_search(&all_user_ids.index, (avl_t *) &user_id_to_find);
+ }
+
+ if(user_or_group_id && user_or_group_id->name && *user_or_group_id->name) {
+ snprintfz(w->name, MAX_NAME, "%s", user_or_group_id->name);
+ }
+ else {
+ struct passwd *pw = getpwuid(uid);
+ if(!pw || !pw->pw_name || !*pw->pw_name)
+ snprintfz(w->name, MAX_NAME, "%u", uid);
+ else
+ snprintfz(w->name, MAX_NAME, "%s", pw->pw_name);
+ }
+
+ netdata_fix_chart_name(w->name);
+
+ w->uid = uid;
+
+ w->next = users_root_target;
+ users_root_target = w;
+
+ debug_log("added uid %u ('%s') target", w->uid, w->name);
+
+ return w;
+}
+
+struct target *get_groups_target(gid_t gid)
+{
+ struct target *w;
+ for(w = groups_root_target ; w ; w = w->next)
+ if(w->gid == gid) return w;
+
+ w = callocz(sizeof(struct target), 1);
+ snprintfz(w->compare, MAX_COMPARE_NAME, "%u", gid);
+ w->comparehash = simple_hash(w->compare);
+ w->comparelen = strlen(w->compare);
+
+ snprintfz(w->id, MAX_NAME, "%u", gid);
+ w->idhash = simple_hash(w->id);
+
+ struct user_or_group_id group_id_to_find, *group_id = NULL;
+ group_id_to_find.id.gid = gid;
+
+ if(*netdata_configured_host_prefix) {
+ static struct timespec last_group_modification_time;
+ int ret = read_user_or_group_ids(&all_group_ids, &last_group_modification_time);
+
+ if(likely(!ret && all_group_ids.index.root))
+ group_id = (struct user_or_group_id *)avl_search(&all_group_ids.index, (avl_t *) &group_id_to_find);
+ }
+
+ if(group_id && group_id->name && *group_id->name) {
+ snprintfz(w->name, MAX_NAME, "%s", group_id->name);
+ }
+ else {
+ struct group *gr = getgrgid(gid);
+ if(!gr || !gr->gr_name || !*gr->gr_name)
+ snprintfz(w->name, MAX_NAME, "%u", gid);
+ else
+ snprintfz(w->name, MAX_NAME, "%s", gr->gr_name);
+ }
+
+ netdata_fix_chart_name(w->name);
+
+ w->gid = gid;
+
+ w->next = groups_root_target;
+ groups_root_target = w;
+
+ debug_log("added gid %u ('%s') target", w->gid, w->name);
+
+ return w;
+}
+
+// find or create a new target
+// there are targets that are just aggregated to other target (the second argument)
+static struct target *get_apps_groups_target(const char *id, struct target *target, const char *name) {
+ int tdebug = 0, thidden = target?target->hidden:0, ends_with = 0;
+ const char *nid = id;
+
+ // extract the options
+ while(nid[0] == '-' || nid[0] == '+' || nid[0] == '*') {
+ if(nid[0] == '-') thidden = 1;
+ if(nid[0] == '+') tdebug = 1;
+ if(nid[0] == '*') ends_with = 1;
+ nid++;
+ }
+ uint32_t hash = simple_hash(id);
+
+ // find if it already exists
+ struct target *w, *last = apps_groups_root_target;
+ for(w = apps_groups_root_target ; w ; w = w->next) {
+ if(w->idhash == hash && strncmp(nid, w->id, MAX_NAME) == 0)
+ return w;
+
+ last = w;
+ }
+
+ // find an existing target
+ if(unlikely(!target)) {
+ while(*name == '-') {
+ if(*name == '-') thidden = 1;
+ name++;
+ }
+
+ for(target = apps_groups_root_target ; target != NULL ; target = target->next) {
+ if(!target->target && strcmp(name, target->name) == 0)
+ break;
+ }
+
+ if(unlikely(debug_enabled)) {
+ if(unlikely(target))
+ debug_log("REUSING TARGET NAME '%s' on ID '%s'", target->name, target->id);
+ else
+ debug_log("NEW TARGET NAME '%s' on ID '%s'", name, id);
+ }
+ }
+
+ if(target && target->target)
+ fatal("Internal Error: request to link process '%s' to target '%s' which is linked to target '%s'", id, target->id, target->target->id);
+
+ w = callocz(sizeof(struct target), 1);
+ strncpyz(w->id, nid, MAX_NAME);
+ w->idhash = simple_hash(w->id);
+
+ if(unlikely(!target))
+ // copy the name
+ strncpyz(w->name, name, MAX_NAME);
+ else
+ // copy the id
+ strncpyz(w->name, nid, MAX_NAME);
+
+ strncpyz(w->compare, nid, MAX_COMPARE_NAME);
+ size_t len = strlen(w->compare);
+ if(w->compare[len - 1] == '*') {
+ w->compare[len - 1] = '\0';
+ w->starts_with = 1;
+ }
+ w->ends_with = ends_with;
+
+ if(w->starts_with && w->ends_with)
+ proc_pid_cmdline_is_needed = 1;
+
+ w->comparehash = simple_hash(w->compare);
+ w->comparelen = strlen(w->compare);
+
+ w->hidden = thidden;
+#ifdef NETDATA_INTERNAL_CHECKS
+ w->debug_enabled = tdebug;
+#else
+ if(tdebug)
+ fprintf(stderr, "apps.plugin has been compiled without debugging\n");
+#endif
+ w->target = target;
+
+ // append it, to maintain the order in apps_groups.conf
+ if(last) last->next = w;
+ else apps_groups_root_target = w;
+
+ debug_log("ADDING TARGET ID '%s', process name '%s' (%s), aggregated on target '%s', options: %s %s"
+ , w->id
+ , w->compare, (w->starts_with && w->ends_with)?"substring":((w->starts_with)?"prefix":((w->ends_with)?"suffix":"exact"))
+ , w->target?w->target->name:w->name
+ , (w->hidden)?"hidden":"-"
+ , (w->debug_enabled)?"debug":"-"
+ );
+
+ return w;
+}
+
+// read the apps_groups.conf file
+static int read_apps_groups_conf(const char *path, const char *file)
+{
+ char filename[FILENAME_MAX + 1];
+
+ snprintfz(filename, FILENAME_MAX, "%s/apps_%s.conf", path, file);
+
+ debug_log("process groups file: '%s'", filename);
+
+ // ----------------------------------------
+
+ procfile *ff = procfile_open(filename, " :\t", PROCFILE_FLAG_DEFAULT);
+ if(!ff) return 1;
+
+ procfile_set_quotes(ff, "'\"");
+
+ ff = procfile_readall(ff);
+ if(!ff)
+ return 1;
+
+ size_t line, lines = procfile_lines(ff);
+
+ for(line = 0; line < lines ;line++) {
+ size_t word, words = procfile_linewords(ff, line);
+ if(!words) continue;
+
+ char *name = procfile_lineword(ff, line, 0);
+ if(!name || !*name) continue;
+
+ // find a possibly existing target
+ struct target *w = NULL;
+
+ // loop through all words, skipping the first one (the name)
+ for(word = 0; word < words ;word++) {
+ char *s = procfile_lineword(ff, line, word);
+ if(!s || !*s) continue;
+ if(*s == '#') break;
+
+ // is this the first word? skip it
+ if(s == name) continue;
+
+ // add this target
+ struct target *n = get_apps_groups_target(s, w, name);
+ if(!n) {
+ error("Cannot create target '%s' (line %zu, word %zu)", s, line, word);
+ continue;
+ }
+
+ // just some optimization
+ // to avoid searching for a target for each process
+ if(!w) w = n->target?n->target:n;
+ }
+ }
+
+ procfile_close(ff);
+
+ apps_groups_default_target = get_apps_groups_target("p+!o@w#e$i^r&7*5(-i)l-o_", NULL, "other"); // match nothing
+ if(!apps_groups_default_target)
+ fatal("Cannot create default target");
+
+ // allow the user to override group 'other'
+ if(apps_groups_default_target->target)
+ apps_groups_default_target = apps_groups_default_target->target;
+
+ return 0;
+}
+
+
+// ----------------------------------------------------------------------------
+// struct pid_stat management
+static inline void init_pid_fds(struct pid_stat *p, size_t first, size_t size);
+
+static inline struct pid_stat *get_pid_entry(pid_t pid) {
+ if(unlikely(all_pids[pid]))
+ return all_pids[pid];
+
+ struct pid_stat *p = callocz(sizeof(struct pid_stat), 1);
+ p->fds = mallocz(sizeof(struct pid_fd) * MAX_SPARE_FDS);
+ p->fds_size = MAX_SPARE_FDS;
+ init_pid_fds(p, 0, p->fds_size);
+ p->pid = pid;
+
+ DOUBLE_LINKED_LIST_APPEND_UNSAFE(root_of_pids, p, prev, next);
+
+ all_pids[pid] = p;
+ all_pids_count++;
+
+ return p;
+}
+
+static inline void del_pid_entry(pid_t pid) {
+ struct pid_stat *p = all_pids[pid];
+
+ if(unlikely(!p)) {
+ error("attempted to free pid %d that is not allocated.", pid);
+ return;
+ }
+
+ debug_log("process %d %s exited, deleting it.", pid, p->comm);
+
+ DOUBLE_LINKED_LIST_REMOVE_UNSAFE(root_of_pids, p, prev, next);
+
+ // free the filename
+#ifndef __FreeBSD__
+ {
+ size_t i;
+ for(i = 0; i < p->fds_size; i++)
+ if(p->fds[i].filename)
+ freez(p->fds[i].filename);
+ }
+#endif
+ freez(p->fds);
+
+ freez(p->fds_dirname);
+ freez(p->stat_filename);
+ freez(p->status_filename);
+#ifndef __FreeBSD__
+ arl_free(p->status_arl);
+#endif
+ freez(p->io_filename);
+ freez(p->cmdline_filename);
+ freez(p->cmdline);
+ freez(p);
+
+ all_pids[pid] = NULL;
+ all_pids_count--;
+}
+
+// ----------------------------------------------------------------------------
+
+static inline int managed_log(struct pid_stat *p, uint32_t log, int status) {
+ if(unlikely(!status)) {
+ // error("command failed log %u, errno %d", log, errno);
+
+ if(unlikely(debug_enabled || errno != ENOENT)) {
+ if(unlikely(debug_enabled || !(p->log_thrown & log))) {
+ p->log_thrown |= log;
+ switch(log) {
+ case PID_LOG_IO:
+ #ifdef __FreeBSD__
+ error("Cannot fetch process %d I/O info (command '%s')", p->pid, p->comm);
+ #else
+ error("Cannot process %s/proc/%d/io (command '%s')", netdata_configured_host_prefix, p->pid, p->comm);
+ #endif
+ break;
+
+ case PID_LOG_STATUS:
+ #ifdef __FreeBSD__
+ error("Cannot fetch process %d status info (command '%s')", p->pid, p->comm);
+ #else
+ error("Cannot process %s/proc/%d/status (command '%s')", netdata_configured_host_prefix, p->pid, p->comm);
+ #endif
+ break;
+
+ case PID_LOG_CMDLINE:
+ #ifdef __FreeBSD__
+ error("Cannot fetch process %d command line (command '%s')", p->pid, p->comm);
+ #else
+ error("Cannot process %s/proc/%d/cmdline (command '%s')", netdata_configured_host_prefix, p->pid, p->comm);
+ #endif
+ break;
+
+ case PID_LOG_FDS:
+ #ifdef __FreeBSD__
+ error("Cannot fetch process %d files (command '%s')", p->pid, p->comm);
+ #else
+ error("Cannot process entries in %s/proc/%d/fd (command '%s')", netdata_configured_host_prefix, p->pid, p->comm);
+ #endif
+ break;
+
+ case PID_LOG_STAT:
+ break;
+
+ default:
+ error("unhandled error for pid %d, command '%s'", p->pid, p->comm);
+ break;
+ }
+ }
+ }
+ errno = 0;
+ }
+ else if(unlikely(p->log_thrown & log)) {
+ // error("unsetting log %u on pid %d", log, p->pid);
+ p->log_thrown &= ~log;
+ }
+
+ return status;
+}
+
+static inline void assign_target_to_pid(struct pid_stat *p) {
+ targets_assignment_counter++;
+
+ uint32_t hash = simple_hash(p->comm);
+ size_t pclen = strlen(p->comm);
+
+ struct target *w;
+ for(w = apps_groups_root_target; w ; w = w->next) {
+ // if(debug_enabled || (p->target && p->target->debug_enabled)) debug_log_int("\t\tcomparing '%s' with '%s'", w->compare, p->comm);
+
+ // find it - 4 cases:
+ // 1. the target is not a pattern
+ // 2. the target has the prefix
+ // 3. the target has the suffix
+ // 4. the target is something inside cmdline
+
+ if(unlikely(( (!w->starts_with && !w->ends_with && w->comparehash == hash && !strcmp(w->compare, p->comm))
+ || (w->starts_with && !w->ends_with && !strncmp(w->compare, p->comm, w->comparelen))
+ || (!w->starts_with && w->ends_with && pclen >= w->comparelen && !strcmp(w->compare, &p->comm[pclen - w->comparelen]))
+ || (proc_pid_cmdline_is_needed && w->starts_with && w->ends_with && p->cmdline && strstr(p->cmdline, w->compare))
+ ))) {
+
+ if(w->target) p->target = w->target;
+ else p->target = w;
+
+ if(debug_enabled || (p->target && p->target->debug_enabled))
+ debug_log_int("%s linked to target %s", p->comm, p->target->name);
+
+ break;
+ }
+ }
+}
+
+
+// ----------------------------------------------------------------------------
+// update pids from proc
+
+static inline int read_proc_pid_cmdline(struct pid_stat *p) {
+ static char cmdline[MAX_CMDLINE + 1];
+
+#ifdef __FreeBSD__
+ size_t i, bytes = MAX_CMDLINE;
+ int mib[4];
+
+ mib[0] = CTL_KERN;
+ mib[1] = KERN_PROC;
+ mib[2] = KERN_PROC_ARGS;
+ mib[3] = p->pid;
+ if (unlikely(sysctl(mib, 4, cmdline, &bytes, NULL, 0)))
+ goto cleanup;
+#else
+ if(unlikely(!p->cmdline_filename)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/%d/cmdline", netdata_configured_host_prefix, p->pid);
+ p->cmdline_filename = strdupz(filename);
+ }
+
+ int fd = open(p->cmdline_filename, procfile_open_flags, 0666);
+ if(unlikely(fd == -1)) goto cleanup;
+
+ ssize_t i, bytes = read(fd, cmdline, MAX_CMDLINE);
+ close(fd);
+
+ if(unlikely(bytes < 0)) goto cleanup;
+#endif
+
+ cmdline[bytes] = '\0';
+ for(i = 0; i < bytes ; i++) {
+ if(unlikely(!cmdline[i])) cmdline[i] = ' ';
+ }
+
+ if(p->cmdline) freez(p->cmdline);
+ p->cmdline = strdupz(cmdline);
+
+ debug_log("Read file '%s' contents: %s", p->cmdline_filename, p->cmdline);
+
+ return 1;
+
+cleanup:
+ // copy the command to the command line
+ if(p->cmdline) freez(p->cmdline);
+ p->cmdline = strdupz(p->comm);
+ return 0;
+}
+
+// ----------------------------------------------------------------------------
+// macro to calculate the incremental rate of a value
+// each parameter is accessed only ONCE - so it is safe to pass function calls
+// or other macros as parameters
+
+#define incremental_rate(rate_variable, last_kernel_variable, new_kernel_value, collected_usec, last_collected_usec) { \
+ kernel_uint_t _new_tmp = new_kernel_value; \
+ (rate_variable) = (_new_tmp - (last_kernel_variable)) * (USEC_PER_SEC * RATES_DETAIL) / ((collected_usec) - (last_collected_usec)); \
+ (last_kernel_variable) = _new_tmp; \
+ }
+
+// the same macro for struct pid members
+#define pid_incremental_rate(type, var, value) \
+ incremental_rate(var, var##_raw, value, p->type##_collected_usec, p->last_##type##_collected_usec)
+
+
+// ----------------------------------------------------------------------------
+
+#ifndef __FreeBSD__
+struct arl_callback_ptr {
+ struct pid_stat *p;
+ procfile *ff;
+ size_t line;
+};
+
+void arl_callback_status_uid(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name; (void)hash; (void)value;
+ struct arl_callback_ptr *aptr = (struct arl_callback_ptr *)dst;
+ if(unlikely(procfile_linewords(aptr->ff, aptr->line) < 5)) return;
+
+ //const char *real_uid = procfile_lineword(aptr->ff, aptr->line, 1);
+ const char *effective_uid = procfile_lineword(aptr->ff, aptr->line, 2);
+ //const char *saved_uid = procfile_lineword(aptr->ff, aptr->line, 3);
+ //const char *filesystem_uid = procfile_lineword(aptr->ff, aptr->line, 4);
+
+ if(likely(effective_uid && *effective_uid))
+ aptr->p->uid = (uid_t)str2l(effective_uid);
+}
+
+void arl_callback_status_gid(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name; (void)hash; (void)value;
+ struct arl_callback_ptr *aptr = (struct arl_callback_ptr *)dst;
+ if(unlikely(procfile_linewords(aptr->ff, aptr->line) < 5)) return;
+
+ //const char *real_gid = procfile_lineword(aptr->ff, aptr->line, 1);
+ const char *effective_gid = procfile_lineword(aptr->ff, aptr->line, 2);
+ //const char *saved_gid = procfile_lineword(aptr->ff, aptr->line, 3);
+ //const char *filesystem_gid = procfile_lineword(aptr->ff, aptr->line, 4);
+
+ if(likely(effective_gid && *effective_gid))
+ aptr->p->gid = (uid_t)str2l(effective_gid);
+}
+
+void arl_callback_status_vmsize(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name; (void)hash; (void)value;
+ struct arl_callback_ptr *aptr = (struct arl_callback_ptr *)dst;
+ if(unlikely(procfile_linewords(aptr->ff, aptr->line) < 3)) return;
+
+ aptr->p->status_vmsize = str2kernel_uint_t(procfile_lineword(aptr->ff, aptr->line, 1));
+}
+
+void arl_callback_status_vmswap(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name; (void)hash; (void)value;
+ struct arl_callback_ptr *aptr = (struct arl_callback_ptr *)dst;
+ if(unlikely(procfile_linewords(aptr->ff, aptr->line) < 3)) return;
+
+ aptr->p->status_vmswap = str2kernel_uint_t(procfile_lineword(aptr->ff, aptr->line, 1));
+}
+
+void arl_callback_status_vmrss(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name; (void)hash; (void)value;
+ struct arl_callback_ptr *aptr = (struct arl_callback_ptr *)dst;
+ if(unlikely(procfile_linewords(aptr->ff, aptr->line) < 3)) return;
+
+ aptr->p->status_vmrss = str2kernel_uint_t(procfile_lineword(aptr->ff, aptr->line, 1));
+}
+
+void arl_callback_status_rssfile(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name; (void)hash; (void)value;
+ struct arl_callback_ptr *aptr = (struct arl_callback_ptr *)dst;
+ if(unlikely(procfile_linewords(aptr->ff, aptr->line) < 3)) return;
+
+ aptr->p->status_rssfile = str2kernel_uint_t(procfile_lineword(aptr->ff, aptr->line, 1));
+}
+
+void arl_callback_status_rssshmem(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name; (void)hash; (void)value;
+ struct arl_callback_ptr *aptr = (struct arl_callback_ptr *)dst;
+ if(unlikely(procfile_linewords(aptr->ff, aptr->line) < 3)) return;
+
+ aptr->p->status_rssshmem = str2kernel_uint_t(procfile_lineword(aptr->ff, aptr->line, 1));
+}
+
+static void update_proc_state_count(char proc_state) {
+ switch (proc_state) {
+ case 'S':
+ proc_state_count[PROC_STATUS_SLEEPING] += 1;
+ break;
+ case 'R':
+ proc_state_count[PROC_STATUS_RUNNING] += 1;
+ break;
+ case 'D':
+ proc_state_count[PROC_STATUS_SLEEPING_D] += 1;
+ break;
+ case 'Z':
+ proc_state_count[PROC_STATUS_ZOMBIE] += 1;
+ break;
+ case 'T':
+ proc_state_count[PROC_STATUS_STOPPED] += 1;
+ break;
+ default:
+ break;
+ }
+}
+#endif // !__FreeBSD__
+
+static inline int read_proc_pid_status(struct pid_stat *p, void *ptr) {
+ p->status_vmsize = 0;
+ p->status_vmrss = 0;
+ p->status_vmshared = 0;
+ p->status_rssfile = 0;
+ p->status_rssshmem = 0;
+ p->status_vmswap = 0;
+
+#ifdef __FreeBSD__
+ struct kinfo_proc *proc_info = (struct kinfo_proc *)ptr;
+
+ p->uid = proc_info->ki_uid;
+ p->gid = proc_info->ki_groups[0];
+ p->status_vmsize = proc_info->ki_size / 1024; // in KiB
+ p->status_vmrss = proc_info->ki_rssize * pagesize / 1024; // in KiB
+ // TODO: what about shared and swap memory on FreeBSD?
+ return 1;
+#else
+ (void)ptr;
+
+ static struct arl_callback_ptr arl_ptr;
+ static procfile *ff = NULL;
+
+ if(unlikely(!p->status_arl)) {
+ p->status_arl = arl_create("/proc/pid/status", NULL, 60);
+ arl_expect_custom(p->status_arl, "Uid", arl_callback_status_uid, &arl_ptr);
+ arl_expect_custom(p->status_arl, "Gid", arl_callback_status_gid, &arl_ptr);
+ arl_expect_custom(p->status_arl, "VmSize", arl_callback_status_vmsize, &arl_ptr);
+ arl_expect_custom(p->status_arl, "VmRSS", arl_callback_status_vmrss, &arl_ptr);
+ arl_expect_custom(p->status_arl, "RssFile", arl_callback_status_rssfile, &arl_ptr);
+ arl_expect_custom(p->status_arl, "RssShmem", arl_callback_status_rssshmem, &arl_ptr);
+ arl_expect_custom(p->status_arl, "VmSwap", arl_callback_status_vmswap, &arl_ptr);
+ }
+
+
+ if(unlikely(!p->status_filename)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/%d/status", netdata_configured_host_prefix, p->pid);
+ p->status_filename = strdupz(filename);
+ }
+
+ ff = procfile_reopen(ff, p->status_filename, (!ff)?" \t:,-()/":NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
+ if(unlikely(!ff)) return 0;
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0;
+
+ calls_counter++;
+
+ // let ARL use this pid
+ arl_ptr.p = p;
+ arl_ptr.ff = ff;
+
+ size_t lines = procfile_lines(ff), l;
+ arl_begin(p->status_arl);
+
+ for(l = 0; l < lines ;l++) {
+ // debug_log("CHECK: line %zu of %zu, key '%s' = '%s'", l, lines, procfile_lineword(ff, l, 0), procfile_lineword(ff, l, 1));
+ arl_ptr.line = l;
+ if(unlikely(arl_check(p->status_arl,
+ procfile_lineword(ff, l, 0),
+ procfile_lineword(ff, l, 1)))) break;
+ }
+
+ p->status_vmshared = p->status_rssfile + p->status_rssshmem;
+
+ // debug_log("%s uid %d, gid %d, VmSize %zu, VmRSS %zu, RssFile %zu, RssShmem %zu, shared %zu", p->comm, (int)p->uid, (int)p->gid, p->status_vmsize, p->status_vmrss, p->status_rssfile, p->status_rssshmem, p->status_vmshared);
+
+ return 1;
+#endif
+}
+
+
+// ----------------------------------------------------------------------------
+
+static inline int read_proc_pid_stat(struct pid_stat *p, void *ptr) {
+ (void)ptr;
+
+#ifdef __FreeBSD__
+ struct kinfo_proc *proc_info = (struct kinfo_proc *)ptr;
+ if (unlikely(proc_info->ki_tdflags & TDF_IDLETD))
+ goto cleanup;
+#else
+ static procfile *ff = NULL;
+
+ if(unlikely(!p->stat_filename)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/%d/stat", netdata_configured_host_prefix, p->pid);
+ p->stat_filename = strdupz(filename);
+ }
+
+ int set_quotes = (!ff)?1:0;
+
+ ff = procfile_reopen(ff, p->stat_filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
+ if(unlikely(!ff)) goto cleanup;
+
+ // if(set_quotes) procfile_set_quotes(ff, "()");
+ if(unlikely(set_quotes))
+ procfile_set_open_close(ff, "(", ")");
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) goto cleanup;
+#endif
+
+ p->last_stat_collected_usec = p->stat_collected_usec;
+ p->stat_collected_usec = now_monotonic_usec();
+ calls_counter++;
+
+#ifdef __FreeBSD__
+ char *comm = proc_info->ki_comm;
+ p->ppid = proc_info->ki_ppid;
+#else
+ // p->pid = str2pid_t(procfile_lineword(ff, 0, 0));
+ char *comm = procfile_lineword(ff, 0, 1);
+ p->state = *(procfile_lineword(ff, 0, 2));
+ p->ppid = (int32_t)str2pid_t(procfile_lineword(ff, 0, 3));
+ // p->pgrp = (int32_t)str2pid_t(procfile_lineword(ff, 0, 4));
+ // p->session = (int32_t)str2pid_t(procfile_lineword(ff, 0, 5));
+ // p->tty_nr = (int32_t)str2pid_t(procfile_lineword(ff, 0, 6));
+ // p->tpgid = (int32_t)str2pid_t(procfile_lineword(ff, 0, 7));
+ // p->flags = str2uint64_t(procfile_lineword(ff, 0, 8));
+#endif
+ if(strcmp(p->comm, comm) != 0) {
+ if(unlikely(debug_enabled)) {
+ if(p->comm[0])
+ debug_log("\tpid %d (%s) changed name to '%s'", p->pid, p->comm, comm);
+ else
+ debug_log("\tJust added %d (%s)", p->pid, comm);
+ }
+
+ strncpyz(p->comm, comm, MAX_COMPARE_NAME);
+
+ // /proc/<pid>/cmdline
+ if(likely(proc_pid_cmdline_is_needed))
+ managed_log(p, PID_LOG_CMDLINE, read_proc_pid_cmdline(p));
+
+ assign_target_to_pid(p);
+ }
+
+#ifdef __FreeBSD__
+ pid_incremental_rate(stat, p->minflt, (kernel_uint_t)proc_info->ki_rusage.ru_minflt);
+ pid_incremental_rate(stat, p->cminflt, (kernel_uint_t)proc_info->ki_rusage_ch.ru_minflt);
+ pid_incremental_rate(stat, p->majflt, (kernel_uint_t)proc_info->ki_rusage.ru_majflt);
+ pid_incremental_rate(stat, p->cmajflt, (kernel_uint_t)proc_info->ki_rusage_ch.ru_majflt);
+ pid_incremental_rate(stat, p->utime, (kernel_uint_t)proc_info->ki_rusage.ru_utime.tv_sec * 100 + proc_info->ki_rusage.ru_utime.tv_usec / 10000);
+ pid_incremental_rate(stat, p->stime, (kernel_uint_t)proc_info->ki_rusage.ru_stime.tv_sec * 100 + proc_info->ki_rusage.ru_stime.tv_usec / 10000);
+ pid_incremental_rate(stat, p->cutime, (kernel_uint_t)proc_info->ki_rusage_ch.ru_utime.tv_sec * 100 + proc_info->ki_rusage_ch.ru_utime.tv_usec / 10000);
+ pid_incremental_rate(stat, p->cstime, (kernel_uint_t)proc_info->ki_rusage_ch.ru_stime.tv_sec * 100 + proc_info->ki_rusage_ch.ru_stime.tv_usec / 10000);
+
+ p->num_threads = proc_info->ki_numthreads;
+
+ if(enable_guest_charts) {
+ enable_guest_charts = 0;
+ info("Guest charts aren't supported by FreeBSD");
+ }
+#else
+ pid_incremental_rate(stat, p->minflt, str2kernel_uint_t(procfile_lineword(ff, 0, 9)));
+ pid_incremental_rate(stat, p->cminflt, str2kernel_uint_t(procfile_lineword(ff, 0, 10)));
+ pid_incremental_rate(stat, p->majflt, str2kernel_uint_t(procfile_lineword(ff, 0, 11)));
+ pid_incremental_rate(stat, p->cmajflt, str2kernel_uint_t(procfile_lineword(ff, 0, 12)));
+ pid_incremental_rate(stat, p->utime, str2kernel_uint_t(procfile_lineword(ff, 0, 13)));
+ pid_incremental_rate(stat, p->stime, str2kernel_uint_t(procfile_lineword(ff, 0, 14)));
+ pid_incremental_rate(stat, p->cutime, str2kernel_uint_t(procfile_lineword(ff, 0, 15)));
+ pid_incremental_rate(stat, p->cstime, str2kernel_uint_t(procfile_lineword(ff, 0, 16)));
+ // p->priority = str2kernel_uint_t(procfile_lineword(ff, 0, 17));
+ // p->nice = str2kernel_uint_t(procfile_lineword(ff, 0, 18));
+ p->num_threads = (int32_t)str2uint32_t(procfile_lineword(ff, 0, 19));
+ // p->itrealvalue = str2kernel_uint_t(procfile_lineword(ff, 0, 20));
+ p->collected_starttime = str2kernel_uint_t(procfile_lineword(ff, 0, 21)) / system_hz;
+ p->uptime = (global_uptime > p->collected_starttime)?(global_uptime - p->collected_starttime):0;
+ // p->vsize = str2kernel_uint_t(procfile_lineword(ff, 0, 22));
+ // p->rss = str2kernel_uint_t(procfile_lineword(ff, 0, 23));
+ // p->rsslim = str2kernel_uint_t(procfile_lineword(ff, 0, 24));
+ // p->starcode = str2kernel_uint_t(procfile_lineword(ff, 0, 25));
+ // p->endcode = str2kernel_uint_t(procfile_lineword(ff, 0, 26));
+ // p->startstack = str2kernel_uint_t(procfile_lineword(ff, 0, 27));
+ // p->kstkesp = str2kernel_uint_t(procfile_lineword(ff, 0, 28));
+ // p->kstkeip = str2kernel_uint_t(procfile_lineword(ff, 0, 29));
+ // p->signal = str2kernel_uint_t(procfile_lineword(ff, 0, 30));
+ // p->blocked = str2kernel_uint_t(procfile_lineword(ff, 0, 31));
+ // p->sigignore = str2kernel_uint_t(procfile_lineword(ff, 0, 32));
+ // p->sigcatch = str2kernel_uint_t(procfile_lineword(ff, 0, 33));
+ // p->wchan = str2kernel_uint_t(procfile_lineword(ff, 0, 34));
+ // p->nswap = str2kernel_uint_t(procfile_lineword(ff, 0, 35));
+ // p->cnswap = str2kernel_uint_t(procfile_lineword(ff, 0, 36));
+ // p->exit_signal = str2kernel_uint_t(procfile_lineword(ff, 0, 37));
+ // p->processor = str2kernel_uint_t(procfile_lineword(ff, 0, 38));
+ // p->rt_priority = str2kernel_uint_t(procfile_lineword(ff, 0, 39));
+ // p->policy = str2kernel_uint_t(procfile_lineword(ff, 0, 40));
+ // p->delayacct_blkio_ticks = str2kernel_uint_t(procfile_lineword(ff, 0, 41));
+
+ if(enable_guest_charts) {
+
+ pid_incremental_rate(stat, p->gtime, str2kernel_uint_t(procfile_lineword(ff, 0, 42)));
+ pid_incremental_rate(stat, p->cgtime, str2kernel_uint_t(procfile_lineword(ff, 0, 43)));
+
+ if (show_guest_time || p->gtime || p->cgtime) {
+ p->utime -= (p->utime >= p->gtime) ? p->gtime : p->utime;
+ p->cutime -= (p->cutime >= p->cgtime) ? p->cgtime : p->cutime;
+ show_guest_time = 1;
+ }
+ }
+#endif
+
+ if(unlikely(debug_enabled || (p->target && p->target->debug_enabled)))
+ debug_log_int("READ PROC/PID/STAT: %s/proc/%d/stat, process: '%s' on target '%s' (dt=%llu) VALUES: utime=" KERNEL_UINT_FORMAT ", stime=" KERNEL_UINT_FORMAT ", cutime=" KERNEL_UINT_FORMAT ", cstime=" KERNEL_UINT_FORMAT ", minflt=" KERNEL_UINT_FORMAT ", majflt=" KERNEL_UINT_FORMAT ", cminflt=" KERNEL_UINT_FORMAT ", cmajflt=" KERNEL_UINT_FORMAT ", threads=%d", netdata_configured_host_prefix, p->pid, p->comm, (p->target)?p->target->name:"UNSET", p->stat_collected_usec - p->last_stat_collected_usec, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt, p->num_threads);
+
+ if(unlikely(global_iterations_counter == 1)) {
+ p->minflt = 0;
+ p->cminflt = 0;
+ p->majflt = 0;
+ p->cmajflt = 0;
+ p->utime = 0;
+ p->stime = 0;
+ p->gtime = 0;
+ p->cutime = 0;
+ p->cstime = 0;
+ p->cgtime = 0;
+ }
+#ifndef __FreeBSD__
+ update_proc_state_count(p->state);
+#endif
+ return 1;
+
+cleanup:
+ p->minflt = 0;
+ p->cminflt = 0;
+ p->majflt = 0;
+ p->cmajflt = 0;
+ p->utime = 0;
+ p->stime = 0;
+ p->gtime = 0;
+ p->cutime = 0;
+ p->cstime = 0;
+ p->cgtime = 0;
+ p->num_threads = 0;
+ // p->rss = 0;
+ return 0;
+}
+
+// ----------------------------------------------------------------------------
+
+static inline int read_proc_pid_io(struct pid_stat *p, void *ptr) {
+ (void)ptr;
+#ifdef __FreeBSD__
+ struct kinfo_proc *proc_info = (struct kinfo_proc *)ptr;
+#else
+ static procfile *ff = NULL;
+
+ if(unlikely(!p->io_filename)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/%d/io", netdata_configured_host_prefix, p->pid);
+ p->io_filename = strdupz(filename);
+ }
+
+ // open the file
+ ff = procfile_reopen(ff, p->io_filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
+ if(unlikely(!ff)) goto cleanup;
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) goto cleanup;
+#endif
+
+ calls_counter++;
+
+ p->last_io_collected_usec = p->io_collected_usec;
+ p->io_collected_usec = now_monotonic_usec();
+
+#ifdef __FreeBSD__
+ pid_incremental_rate(io, p->io_storage_bytes_read, proc_info->ki_rusage.ru_inblock);
+ pid_incremental_rate(io, p->io_storage_bytes_written, proc_info->ki_rusage.ru_oublock);
+#else
+ pid_incremental_rate(io, p->io_logical_bytes_read, str2kernel_uint_t(procfile_lineword(ff, 0, 1)));
+ pid_incremental_rate(io, p->io_logical_bytes_written, str2kernel_uint_t(procfile_lineword(ff, 1, 1)));
+ pid_incremental_rate(io, p->io_read_calls, str2kernel_uint_t(procfile_lineword(ff, 2, 1)));
+ pid_incremental_rate(io, p->io_write_calls, str2kernel_uint_t(procfile_lineword(ff, 3, 1)));
+ pid_incremental_rate(io, p->io_storage_bytes_read, str2kernel_uint_t(procfile_lineword(ff, 4, 1)));
+ pid_incremental_rate(io, p->io_storage_bytes_written, str2kernel_uint_t(procfile_lineword(ff, 5, 1)));
+ pid_incremental_rate(io, p->io_cancelled_write_bytes, str2kernel_uint_t(procfile_lineword(ff, 6, 1)));
+#endif
+
+ if(unlikely(global_iterations_counter == 1)) {
+ p->io_logical_bytes_read = 0;
+ p->io_logical_bytes_written = 0;
+ p->io_read_calls = 0;
+ p->io_write_calls = 0;
+ p->io_storage_bytes_read = 0;
+ p->io_storage_bytes_written = 0;
+ p->io_cancelled_write_bytes = 0;
+ }
+
+ return 1;
+
+#ifndef __FreeBSD__
+cleanup:
+ p->io_logical_bytes_read = 0;
+ p->io_logical_bytes_written = 0;
+ p->io_read_calls = 0;
+ p->io_write_calls = 0;
+ p->io_storage_bytes_read = 0;
+ p->io_storage_bytes_written = 0;
+ p->io_cancelled_write_bytes = 0;
+ return 0;
+#endif
+}
+
+#ifndef __FreeBSD__
+static inline int read_global_time() {
+ static char filename[FILENAME_MAX + 1] = "";
+ static procfile *ff = NULL;
+ static kernel_uint_t utime_raw = 0, stime_raw = 0, gtime_raw = 0, gntime_raw = 0, ntime_raw = 0;
+ static usec_t collected_usec = 0, last_collected_usec = 0;
+
+ if(unlikely(!ff)) {
+ snprintfz(filename, FILENAME_MAX, "%s/proc/stat", netdata_configured_host_prefix);
+ ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) goto cleanup;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) goto cleanup;
+
+ last_collected_usec = collected_usec;
+ collected_usec = now_monotonic_usec();
+
+ calls_counter++;
+
+ // temporary - it is added global_ntime;
+ kernel_uint_t global_ntime = 0;
+
+ incremental_rate(global_utime, utime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 1)), collected_usec, last_collected_usec);
+ incremental_rate(global_ntime, ntime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 2)), collected_usec, last_collected_usec);
+ incremental_rate(global_stime, stime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 3)), collected_usec, last_collected_usec);
+ incremental_rate(global_gtime, gtime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 10)), collected_usec, last_collected_usec);
+
+ global_utime += global_ntime;
+
+ if(enable_guest_charts) {
+ // temporary - it is added global_ntime;
+ kernel_uint_t global_gntime = 0;
+
+ // guest nice time, on guest time
+ incremental_rate(global_gntime, gntime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 11)), collected_usec, last_collected_usec);
+
+ global_gtime += global_gntime;
+
+ // remove guest time from user time
+ global_utime -= (global_utime > global_gtime) ? global_gtime : global_utime;
+ }
+
+ if(unlikely(global_iterations_counter == 1)) {
+ global_utime = 0;
+ global_stime = 0;
+ global_gtime = 0;
+ }
+
+ return 1;
+
+cleanup:
+ global_utime = 0;
+ global_stime = 0;
+ global_gtime = 0;
+ return 0;
+}
+#else
+static inline int read_global_time() {
+ static kernel_uint_t utime_raw = 0, stime_raw = 0, ntime_raw = 0;
+ static usec_t collected_usec = 0, last_collected_usec = 0;
+ long cp_time[CPUSTATES];
+
+ if (unlikely(CPUSTATES != 5)) {
+ goto cleanup;
+ } else {
+ static int mib[2] = {0, 0};
+
+ if (unlikely(GETSYSCTL_SIMPLE("kern.cp_time", mib, cp_time))) {
+ goto cleanup;
+ }
+ }
+
+ last_collected_usec = collected_usec;
+ collected_usec = now_monotonic_usec();
+
+ calls_counter++;
+
+ // temporary - it is added global_ntime;
+ kernel_uint_t global_ntime = 0;
+
+ incremental_rate(global_utime, utime_raw, cp_time[0] * 100LLU / system_hz, collected_usec, last_collected_usec);
+ incremental_rate(global_ntime, ntime_raw, cp_time[1] * 100LLU / system_hz, collected_usec, last_collected_usec);
+ incremental_rate(global_stime, stime_raw, cp_time[2] * 100LLU / system_hz, collected_usec, last_collected_usec);
+
+ global_utime += global_ntime;
+
+ if(unlikely(global_iterations_counter == 1)) {
+ global_utime = 0;
+ global_stime = 0;
+ global_gtime = 0;
+ }
+
+ return 1;
+
+cleanup:
+ global_utime = 0;
+ global_stime = 0;
+ global_gtime = 0;
+ return 0;
+}
+#endif /* !__FreeBSD__ */
+
+// ----------------------------------------------------------------------------
+
+int file_descriptor_compare(void* a, void* b) {
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(((struct file_descriptor *)a)->magic != 0x0BADCAFE || ((struct file_descriptor *)b)->magic != 0x0BADCAFE)
+ error("Corrupted index data detected. Please report this.");
+#endif /* NETDATA_INTERNAL_CHECKS */
+
+ if(((struct file_descriptor *)a)->hash < ((struct file_descriptor *)b)->hash)
+ return -1;
+
+ else if(((struct file_descriptor *)a)->hash > ((struct file_descriptor *)b)->hash)
+ return 1;
+
+ else
+ return strcmp(((struct file_descriptor *)a)->name, ((struct file_descriptor *)b)->name);
+}
+
+// int file_descriptor_iterator(avl_t *a) { if(a) {}; return 0; }
+
+avl_tree_type all_files_index = {
+ NULL,
+ file_descriptor_compare
+};
+
+static struct file_descriptor *file_descriptor_find(const char *name, uint32_t hash) {
+ struct file_descriptor tmp;
+ tmp.hash = (hash)?hash:simple_hash(name);
+ tmp.name = name;
+ tmp.count = 0;
+ tmp.pos = 0;
+#ifdef NETDATA_INTERNAL_CHECKS
+ tmp.magic = 0x0BADCAFE;
+#endif /* NETDATA_INTERNAL_CHECKS */
+
+ return (struct file_descriptor *)avl_search(&all_files_index, (avl_t *) &tmp);
+}
+
+#define file_descriptor_add(fd) avl_insert(&all_files_index, (avl_t *)(fd))
+#define file_descriptor_remove(fd) avl_remove(&all_files_index, (avl_t *)(fd))
+
+// ----------------------------------------------------------------------------
+
+static inline void file_descriptor_not_used(int id)
+{
+ if(id > 0 && id < all_files_size) {
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(all_files[id].magic != 0x0BADCAFE) {
+ error("Ignoring request to remove empty file id %d.", id);
+ return;
+ }
+#endif /* NETDATA_INTERNAL_CHECKS */
+
+ debug_log("decreasing slot %d (count = %d).", id, all_files[id].count);
+
+ if(all_files[id].count > 0) {
+ all_files[id].count--;
+
+ if(!all_files[id].count) {
+ debug_log(" >> slot %d is empty.", id);
+
+ if(unlikely(file_descriptor_remove(&all_files[id]) != (void *)&all_files[id]))
+ error("INTERNAL ERROR: removal of unused fd from index, removed a different fd");
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ all_files[id].magic = 0x00000000;
+#endif /* NETDATA_INTERNAL_CHECKS */
+ all_files_len--;
+ }
+ }
+ else
+ error("Request to decrease counter of fd %d (%s), while the use counter is 0", id, all_files[id].name);
+ }
+ else error("Request to decrease counter of fd %d, which is outside the array size (1 to %d)", id, all_files_size);
+}
+
+static inline void all_files_grow() {
+ void *old = all_files;
+ int i;
+
+ // there is no empty slot
+ debug_log("extending fd array to %d entries", all_files_size + FILE_DESCRIPTORS_INCREASE_STEP);
+
+ all_files = reallocz(all_files, (all_files_size + FILE_DESCRIPTORS_INCREASE_STEP) * sizeof(struct file_descriptor));
+
+ // if the address changed, we have to rebuild the index
+ // since all pointers are now invalid
+
+ if(unlikely(old && old != (void *)all_files)) {
+ debug_log(" >> re-indexing.");
+
+ all_files_index.root = NULL;
+ for(i = 0; i < all_files_size; i++) {
+ if(!all_files[i].count) continue;
+ if(unlikely(file_descriptor_add(&all_files[i]) != (void *)&all_files[i]))
+ error("INTERNAL ERROR: duplicate indexing of fd during realloc.");
+ }
+
+ debug_log(" >> re-indexing done.");
+ }
+
+ // initialize the newly added entries
+
+ for(i = all_files_size; i < (all_files_size + FILE_DESCRIPTORS_INCREASE_STEP); i++) {
+ all_files[i].count = 0;
+ all_files[i].name = NULL;
+#ifdef NETDATA_INTERNAL_CHECKS
+ all_files[i].magic = 0x00000000;
+#endif /* NETDATA_INTERNAL_CHECKS */
+ all_files[i].pos = i;
+ }
+
+ if(unlikely(!all_files_size)) all_files_len = 1;
+ all_files_size += FILE_DESCRIPTORS_INCREASE_STEP;
+}
+
+static inline int file_descriptor_set_on_empty_slot(const char *name, uint32_t hash, FD_FILETYPE type) {
+ // check we have enough memory to add it
+ if(!all_files || all_files_len == all_files_size)
+ all_files_grow();
+
+ debug_log(" >> searching for empty slot.");
+
+ // search for an empty slot
+
+ static int last_pos = 0;
+ int i, c;
+ for(i = 0, c = last_pos ; i < all_files_size ; i++, c++) {
+ if(c >= all_files_size) c = 0;
+ if(c == 0) continue;
+
+ if(!all_files[c].count) {
+ debug_log(" >> Examining slot %d.", c);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(all_files[c].magic == 0x0BADCAFE && all_files[c].name && file_descriptor_find(all_files[c].name, all_files[c].hash))
+ error("fd on position %d is not cleared properly. It still has %s in it.", c, all_files[c].name);
+#endif /* NETDATA_INTERNAL_CHECKS */
+
+ debug_log(" >> %s fd position %d for %s (last name: %s)", all_files[c].name?"re-using":"using", c, name, all_files[c].name);
+
+ freez((void *)all_files[c].name);
+ all_files[c].name = NULL;
+ last_pos = c;
+ break;
+ }
+ }
+
+ all_files_len++;
+
+ if(i == all_files_size) {
+ fatal("We should find an empty slot, but there isn't any");
+ exit(1);
+ }
+ // else we have an empty slot in 'c'
+
+ debug_log(" >> updating slot %d.", c);
+
+ all_files[c].name = strdupz(name);
+ all_files[c].hash = hash;
+ all_files[c].type = type;
+ all_files[c].pos = c;
+ all_files[c].count = 1;
+#ifdef NETDATA_INTERNAL_CHECKS
+ all_files[c].magic = 0x0BADCAFE;
+#endif /* NETDATA_INTERNAL_CHECKS */
+ if(unlikely(file_descriptor_add(&all_files[c]) != (void *)&all_files[c]))
+ error("INTERNAL ERROR: duplicate indexing of fd.");
+
+ debug_log("using fd position %d (name: %s)", c, all_files[c].name);
+
+ return c;
+}
+
+static inline int file_descriptor_find_or_add(const char *name, uint32_t hash) {
+ if(unlikely(!hash))
+ hash = simple_hash(name);
+
+ debug_log("adding or finding name '%s' with hash %u", name, hash);
+
+ struct file_descriptor *fd = file_descriptor_find(name, hash);
+ if(fd) {
+ // found
+ debug_log(" >> found on slot %d", fd->pos);
+
+ fd->count++;
+ return fd->pos;
+ }
+ // not found
+
+ FD_FILETYPE type;
+ if(likely(name[0] == '/')) type = FILETYPE_FILE;
+ else if(likely(strncmp(name, "pipe:", 5) == 0)) type = FILETYPE_PIPE;
+ else if(likely(strncmp(name, "socket:", 7) == 0)) type = FILETYPE_SOCKET;
+ else if(likely(strncmp(name, "anon_inode:", 11) == 0)) {
+ const char *t = &name[11];
+
+ if(strcmp(t, "inotify") == 0) type = FILETYPE_INOTIFY;
+ else if(strcmp(t, "[eventfd]") == 0) type = FILETYPE_EVENTFD;
+ else if(strcmp(t, "[eventpoll]") == 0) type = FILETYPE_EVENTPOLL;
+ else if(strcmp(t, "[timerfd]") == 0) type = FILETYPE_TIMERFD;
+ else if(strcmp(t, "[signalfd]") == 0) type = FILETYPE_SIGNALFD;
+ else {
+ debug_log("UNKNOWN anonymous inode: %s", name);
+ type = FILETYPE_OTHER;
+ }
+ }
+ else if(likely(strcmp(name, "inotify") == 0)) type = FILETYPE_INOTIFY;
+ else {
+ debug_log("UNKNOWN linkname: %s", name);
+ type = FILETYPE_OTHER;
+ }
+
+ return file_descriptor_set_on_empty_slot(name, hash, type);
+}
+
+static inline void clear_pid_fd(struct pid_fd *pfd) {
+ pfd->fd = 0;
+
+ #ifndef __FreeBSD__
+ pfd->link_hash = 0;
+ pfd->inode = 0;
+ pfd->cache_iterations_counter = 0;
+ pfd->cache_iterations_reset = 0;
+#endif
+}
+
+static inline void make_all_pid_fds_negative(struct pid_stat *p) {
+ struct pid_fd *pfd = p->fds, *pfdend = &p->fds[p->fds_size];
+ while(pfd < pfdend) {
+ pfd->fd = -(pfd->fd);
+ pfd++;
+ }
+}
+
+static inline void cleanup_negative_pid_fds(struct pid_stat *p) {
+ struct pid_fd *pfd = p->fds, *pfdend = &p->fds[p->fds_size];
+
+ while(pfd < pfdend) {
+ int fd = pfd->fd;
+
+ if(unlikely(fd < 0)) {
+ file_descriptor_not_used(-(fd));
+ clear_pid_fd(pfd);
+ }
+
+ pfd++;
+ }
+}
+
+static inline void init_pid_fds(struct pid_stat *p, size_t first, size_t size) {
+ struct pid_fd *pfd = &p->fds[first], *pfdend = &p->fds[first + size];
+
+ while(pfd < pfdend) {
+#ifndef __FreeBSD__
+ pfd->filename = NULL;
+#endif
+ clear_pid_fd(pfd);
+ pfd++;
+ }
+}
+
+static inline int read_pid_file_descriptors(struct pid_stat *p, void *ptr) {
+ (void)ptr;
+#ifdef __FreeBSD__
+ int mib[4];
+ size_t size;
+ struct kinfo_file *fds;
+ static char *fdsbuf;
+ char *bfdsbuf, *efdsbuf;
+ char fdsname[FILENAME_MAX + 1];
+#define SHM_FORMAT_LEN 31 // format: 21 + size: 10
+ char shm_name[FILENAME_MAX - SHM_FORMAT_LEN + 1];
+
+ // we make all pid fds negative, so that
+ // we can detect unused file descriptors
+ // at the end, to free them
+ make_all_pid_fds_negative(p);
+
+ mib[0] = CTL_KERN;
+ mib[1] = KERN_PROC;
+ mib[2] = KERN_PROC_FILEDESC;
+ mib[3] = p->pid;
+
+ if (unlikely(sysctl(mib, 4, NULL, &size, NULL, 0))) {
+ error("sysctl error: Can't get file descriptors data size for pid %d", p->pid);
+ return 0;
+ }
+ if (likely(size > 0))
+ fdsbuf = reallocz(fdsbuf, size);
+ if (unlikely(sysctl(mib, 4, fdsbuf, &size, NULL, 0))) {
+ error("sysctl error: Can't get file descriptors data for pid %d", p->pid);
+ return 0;
+ }
+
+ bfdsbuf = fdsbuf;
+ efdsbuf = fdsbuf + size;
+ while (bfdsbuf < efdsbuf) {
+ fds = (struct kinfo_file *)(uintptr_t)bfdsbuf;
+ if (unlikely(fds->kf_structsize == 0))
+ break;
+
+ // do not process file descriptors for current working directory, root directory,
+ // jail directory, ktrace vnode, text vnode and controlling terminal
+ if (unlikely(fds->kf_fd < 0)) {
+ bfdsbuf += fds->kf_structsize;
+ continue;
+ }
+
+ // get file descriptors array index
+ size_t fdid = fds->kf_fd;
+
+ // check if the fds array is small
+ if (unlikely(fdid >= p->fds_size)) {
+ // it is small, extend it
+
+ debug_log("extending fd memory slots for %s from %d to %d", p->comm, p->fds_size, fdid + MAX_SPARE_FDS);
+
+ p->fds = reallocz(p->fds, (fdid + MAX_SPARE_FDS) * sizeof(struct pid_fd));
+
+ // and initialize it
+ init_pid_fds(p, p->fds_size, (fdid + MAX_SPARE_FDS) - p->fds_size);
+ p->fds_size = fdid + MAX_SPARE_FDS;
+ }
+
+ if (unlikely(p->fds[fdid].fd == 0)) {
+ // we don't know this fd, get it
+
+ switch (fds->kf_type) {
+ case KF_TYPE_FIFO:
+ case KF_TYPE_VNODE:
+ if (unlikely(!fds->kf_path[0])) {
+ sprintf(fdsname, "other: inode: %lu", fds->kf_un.kf_file.kf_file_fileid);
+ break;
+ }
+ sprintf(fdsname, "%s", fds->kf_path);
+ break;
+ case KF_TYPE_SOCKET:
+ switch (fds->kf_sock_domain) {
+ case AF_INET:
+ case AF_INET6:
+ if (fds->kf_sock_protocol == IPPROTO_TCP)
+ sprintf(fdsname, "socket: %d %lx", fds->kf_sock_protocol, fds->kf_un.kf_sock.kf_sock_inpcb);
+ else
+ sprintf(fdsname, "socket: %d %lx", fds->kf_sock_protocol, fds->kf_un.kf_sock.kf_sock_pcb);
+ break;
+ case AF_UNIX:
+ /* print address of pcb and connected pcb */
+ sprintf(fdsname, "socket: %lx %lx", fds->kf_un.kf_sock.kf_sock_pcb, fds->kf_un.kf_sock.kf_sock_unpconn);
+ break;
+ default:
+ /* print protocol number and socket address */
+#if __FreeBSD_version < 1200031
+ sprintf(fdsname, "socket: other: %d %s %s", fds->kf_sock_protocol, fds->kf_sa_local.__ss_pad1, fds->kf_sa_local.__ss_pad2);
+#else
+ sprintf(fdsname, "socket: other: %d %s %s", fds->kf_sock_protocol, fds->kf_un.kf_sock.kf_sa_local.__ss_pad1, fds->kf_un.kf_sock.kf_sa_local.__ss_pad2);
+#endif
+ }
+ break;
+ case KF_TYPE_PIPE:
+ sprintf(fdsname, "pipe: %lu %lu", fds->kf_un.kf_pipe.kf_pipe_addr, fds->kf_un.kf_pipe.kf_pipe_peer);
+ break;
+ case KF_TYPE_PTS:
+#if __FreeBSD_version < 1200031
+ sprintf(fdsname, "other: pts: %u", fds->kf_un.kf_pts.kf_pts_dev);
+#else
+ sprintf(fdsname, "other: pts: %lu", fds->kf_un.kf_pts.kf_pts_dev);
+#endif
+ break;
+ case KF_TYPE_SHM:
+ strncpyz(shm_name, fds->kf_path, FILENAME_MAX - SHM_FORMAT_LEN);
+ sprintf(fdsname, "other: shm: %s size: %lu", shm_name, fds->kf_un.kf_file.kf_file_size);
+ break;
+ case KF_TYPE_SEM:
+ sprintf(fdsname, "other: sem: %u", fds->kf_un.kf_sem.kf_sem_value);
+ break;
+ default:
+ sprintf(fdsname, "other: pid: %d fd: %d", fds->kf_un.kf_proc.kf_pid, fds->kf_fd);
+ }
+
+ // if another process already has this, we will get
+ // the same id
+ p->fds[fdid].fd = file_descriptor_find_or_add(fdsname, 0);
+ }
+
+ // else make it positive again, we need it
+ // of course, the actual file may have changed
+
+ else
+ p->fds[fdid].fd = -p->fds[fdid].fd;
+
+ bfdsbuf += fds->kf_structsize;
+ }
+#else
+ if(unlikely(!p->fds_dirname)) {
+ char dirname[FILENAME_MAX+1];
+ snprintfz(dirname, FILENAME_MAX, "%s/proc/%d/fd", netdata_configured_host_prefix, p->pid);
+ p->fds_dirname = strdupz(dirname);
+ }
+
+ DIR *fds = opendir(p->fds_dirname);
+ if(unlikely(!fds)) return 0;
+
+ struct dirent *de;
+ char linkname[FILENAME_MAX + 1];
+
+ // we make all pid fds negative, so that
+ // we can detect unused file descriptors
+ // at the end, to free them
+ make_all_pid_fds_negative(p);
+
+ while((de = readdir(fds))) {
+ // we need only files with numeric names
+
+ if(unlikely(de->d_name[0] < '0' || de->d_name[0] > '9'))
+ continue;
+
+ // get its number
+ int fdid = (int) str2l(de->d_name);
+ if(unlikely(fdid < 0)) continue;
+
+ // check if the fds array is small
+ if(unlikely((size_t)fdid >= p->fds_size)) {
+ // it is small, extend it
+
+ debug_log("extending fd memory slots for %s from %d to %d"
+ , p->comm
+ , p->fds_size
+ , fdid + MAX_SPARE_FDS
+ );
+
+ p->fds = reallocz(p->fds, (fdid + MAX_SPARE_FDS) * sizeof(struct pid_fd));
+
+ // and initialize it
+ init_pid_fds(p, p->fds_size, (fdid + MAX_SPARE_FDS) - p->fds_size);
+ p->fds_size = (size_t)fdid + MAX_SPARE_FDS;
+ }
+
+ if(unlikely(p->fds[fdid].fd < 0 && de->d_ino != p->fds[fdid].inode)) {
+ // inodes do not match, clear the previous entry
+ inodes_changed_counter++;
+ file_descriptor_not_used(-p->fds[fdid].fd);
+ clear_pid_fd(&p->fds[fdid]);
+ }
+
+ if(p->fds[fdid].fd < 0 && p->fds[fdid].cache_iterations_counter > 0) {
+ p->fds[fdid].fd = -p->fds[fdid].fd;
+ p->fds[fdid].cache_iterations_counter--;
+ continue;
+ }
+
+ if(unlikely(!p->fds[fdid].filename)) {
+ filenames_allocated_counter++;
+ char fdname[FILENAME_MAX + 1];
+ snprintfz(fdname, FILENAME_MAX, "%s/proc/%d/fd/%s", netdata_configured_host_prefix, p->pid, de->d_name);
+ p->fds[fdid].filename = strdupz(fdname);
+ }
+
+ file_counter++;
+ ssize_t l = readlink(p->fds[fdid].filename, linkname, FILENAME_MAX);
+ if(unlikely(l == -1)) {
+ // cannot read the link
+
+ if(debug_enabled || (p->target && p->target->debug_enabled))
+ error("Cannot read link %s", p->fds[fdid].filename);
+
+ if(unlikely(p->fds[fdid].fd < 0)) {
+ file_descriptor_not_used(-p->fds[fdid].fd);
+ clear_pid_fd(&p->fds[fdid]);
+ }
+
+ continue;
+ }
+ else
+ linkname[l] = '\0';
+
+ uint32_t link_hash = simple_hash(linkname);
+
+ if(unlikely(p->fds[fdid].fd < 0 && p->fds[fdid].link_hash != link_hash)) {
+ // the link changed
+ links_changed_counter++;
+ file_descriptor_not_used(-p->fds[fdid].fd);
+ clear_pid_fd(&p->fds[fdid]);
+ }
+
+ if(unlikely(p->fds[fdid].fd == 0)) {
+ // we don't know this fd, get it
+
+ // if another process already has this, we will get
+ // the same id
+ p->fds[fdid].fd = file_descriptor_find_or_add(linkname, link_hash);
+ p->fds[fdid].inode = de->d_ino;
+ p->fds[fdid].link_hash = link_hash;
+ }
+ else {
+ // else make it positive again, we need it
+ p->fds[fdid].fd = -p->fds[fdid].fd;
+ }
+
+ // caching control
+ // without this we read all the files on every iteration
+ if(max_fds_cache_seconds > 0) {
+ size_t spread = ((size_t)max_fds_cache_seconds > 10) ? 10 : (size_t)max_fds_cache_seconds;
+
+ // cache it for a few iterations
+ size_t max = ((size_t) max_fds_cache_seconds + (fdid % spread)) / (size_t) update_every;
+ p->fds[fdid].cache_iterations_reset++;
+
+ if(unlikely(p->fds[fdid].cache_iterations_reset % spread == (size_t) fdid % spread))
+ p->fds[fdid].cache_iterations_reset++;
+
+ if(unlikely((fdid <= 2 && p->fds[fdid].cache_iterations_reset > 5) ||
+ p->fds[fdid].cache_iterations_reset > max)) {
+ // for stdin, stdout, stderr (fdid <= 2) we have checked a few times, or if it goes above the max, goto max
+ p->fds[fdid].cache_iterations_reset = max;
+ }
+
+ p->fds[fdid].cache_iterations_counter = p->fds[fdid].cache_iterations_reset;
+ }
+ }
+
+ closedir(fds);
+#endif
+ cleanup_negative_pid_fds(p);
+
+ return 1;
+}
+
+// ----------------------------------------------------------------------------
+
+static inline int debug_print_process_and_parents(struct pid_stat *p, usec_t time) {
+ char *prefix = "\\_ ";
+ int indent = 0;
+
+ if(p->parent)
+ indent = debug_print_process_and_parents(p->parent, p->stat_collected_usec);
+ else
+ prefix = " > ";
+
+ char buffer[indent + 1];
+ int i;
+
+ for(i = 0; i < indent ;i++) buffer[i] = ' ';
+ buffer[i] = '\0';
+
+ fprintf(stderr, " %s %s%s (%d %s %llu"
+ , buffer
+ , prefix
+ , p->comm
+ , p->pid
+ , p->updated?"running":"exited"
+ , p->stat_collected_usec - time
+ );
+
+ if(p->utime) fprintf(stderr, " utime=" KERNEL_UINT_FORMAT, p->utime);
+ if(p->stime) fprintf(stderr, " stime=" KERNEL_UINT_FORMAT, p->stime);
+ if(p->gtime) fprintf(stderr, " gtime=" KERNEL_UINT_FORMAT, p->gtime);
+ if(p->cutime) fprintf(stderr, " cutime=" KERNEL_UINT_FORMAT, p->cutime);
+ if(p->cstime) fprintf(stderr, " cstime=" KERNEL_UINT_FORMAT, p->cstime);
+ if(p->cgtime) fprintf(stderr, " cgtime=" KERNEL_UINT_FORMAT, p->cgtime);
+ if(p->minflt) fprintf(stderr, " minflt=" KERNEL_UINT_FORMAT, p->minflt);
+ if(p->cminflt) fprintf(stderr, " cminflt=" KERNEL_UINT_FORMAT, p->cminflt);
+ if(p->majflt) fprintf(stderr, " majflt=" KERNEL_UINT_FORMAT, p->majflt);
+ if(p->cmajflt) fprintf(stderr, " cmajflt=" KERNEL_UINT_FORMAT, p->cmajflt);
+ fprintf(stderr, ")\n");
+
+ return indent + 1;
+}
+
+static inline void debug_print_process_tree(struct pid_stat *p, char *msg __maybe_unused) {
+ debug_log("%s: process %s (%d, %s) with parents:", msg, p->comm, p->pid, p->updated?"running":"exited");
+ debug_print_process_and_parents(p, p->stat_collected_usec);
+}
+
+static inline void debug_find_lost_child(struct pid_stat *pe, kernel_uint_t lost, int type) {
+ int found = 0;
+ struct pid_stat *p = NULL;
+
+ for(p = root_of_pids; p ; p = p->next) {
+ if(p == pe) continue;
+
+ switch(type) {
+ case 1:
+ if(p->cminflt > lost) {
+ fprintf(stderr, " > process %d (%s) could use the lost exited child minflt " KERNEL_UINT_FORMAT " of process %d (%s)\n", p->pid, p->comm, lost, pe->pid, pe->comm);
+ found++;
+ }
+ break;
+
+ case 2:
+ if(p->cmajflt > lost) {
+ fprintf(stderr, " > process %d (%s) could use the lost exited child majflt " KERNEL_UINT_FORMAT " of process %d (%s)\n", p->pid, p->comm, lost, pe->pid, pe->comm);
+ found++;
+ }
+ break;
+
+ case 3:
+ if(p->cutime > lost) {
+ fprintf(stderr, " > process %d (%s) could use the lost exited child utime " KERNEL_UINT_FORMAT " of process %d (%s)\n", p->pid, p->comm, lost, pe->pid, pe->comm);
+ found++;
+ }
+ break;
+
+ case 4:
+ if(p->cstime > lost) {
+ fprintf(stderr, " > process %d (%s) could use the lost exited child stime " KERNEL_UINT_FORMAT " of process %d (%s)\n", p->pid, p->comm, lost, pe->pid, pe->comm);
+ found++;
+ }
+ break;
+
+ case 5:
+ if(p->cgtime > lost) {
+ fprintf(stderr, " > process %d (%s) could use the lost exited child gtime " KERNEL_UINT_FORMAT " of process %d (%s)\n", p->pid, p->comm, lost, pe->pid, pe->comm);
+ found++;
+ }
+ break;
+ }
+ }
+
+ if(!found) {
+ switch(type) {
+ case 1:
+ fprintf(stderr, " > cannot find any process to use the lost exited child minflt " KERNEL_UINT_FORMAT " of process %d (%s)\n", lost, pe->pid, pe->comm);
+ break;
+
+ case 2:
+ fprintf(stderr, " > cannot find any process to use the lost exited child majflt " KERNEL_UINT_FORMAT " of process %d (%s)\n", lost, pe->pid, pe->comm);
+ break;
+
+ case 3:
+ fprintf(stderr, " > cannot find any process to use the lost exited child utime " KERNEL_UINT_FORMAT " of process %d (%s)\n", lost, pe->pid, pe->comm);
+ break;
+
+ case 4:
+ fprintf(stderr, " > cannot find any process to use the lost exited child stime " KERNEL_UINT_FORMAT " of process %d (%s)\n", lost, pe->pid, pe->comm);
+ break;
+
+ case 5:
+ fprintf(stderr, " > cannot find any process to use the lost exited child gtime " KERNEL_UINT_FORMAT " of process %d (%s)\n", lost, pe->pid, pe->comm);
+ break;
+ }
+ }
+}
+
+static inline kernel_uint_t remove_exited_child_from_parent(kernel_uint_t *field, kernel_uint_t *pfield) {
+ kernel_uint_t absorbed = 0;
+
+ if(*field > *pfield) {
+ absorbed += *pfield;
+ *field -= *pfield;
+ *pfield = 0;
+ }
+ else {
+ absorbed += *field;
+ *pfield -= *field;
+ *field = 0;
+ }
+
+ return absorbed;
+}
+
+static inline void process_exited_processes() {
+ struct pid_stat *p;
+
+ for(p = root_of_pids; p ; p = p->next) {
+ if(p->updated || !p->stat_collected_usec)
+ continue;
+
+ kernel_uint_t utime = (p->utime_raw + p->cutime_raw) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);
+ kernel_uint_t stime = (p->stime_raw + p->cstime_raw) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);
+ kernel_uint_t gtime = (p->gtime_raw + p->cgtime_raw) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);
+ kernel_uint_t minflt = (p->minflt_raw + p->cminflt_raw) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);
+ kernel_uint_t majflt = (p->majflt_raw + p->cmajflt_raw) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);
+
+ if(utime + stime + gtime + minflt + majflt == 0)
+ continue;
+
+ if(unlikely(debug_enabled)) {
+ debug_log("Absorb %s (%d %s total resources: utime=" KERNEL_UINT_FORMAT " stime=" KERNEL_UINT_FORMAT " gtime=" KERNEL_UINT_FORMAT " minflt=" KERNEL_UINT_FORMAT " majflt=" KERNEL_UINT_FORMAT ")"
+ , p->comm
+ , p->pid
+ , p->updated?"running":"exited"
+ , utime
+ , stime
+ , gtime
+ , minflt
+ , majflt
+ );
+ debug_print_process_tree(p, "Searching parents");
+ }
+
+ struct pid_stat *pp;
+ for(pp = p->parent; pp ; pp = pp->parent) {
+ if(!pp->updated) continue;
+
+ kernel_uint_t absorbed;
+ absorbed = remove_exited_child_from_parent(&utime, &pp->cutime);
+ if(unlikely(debug_enabled && absorbed))
+ debug_log(" > process %s (%d %s) absorbed " KERNEL_UINT_FORMAT " utime (remaining: " KERNEL_UINT_FORMAT ")", pp->comm, pp->pid, pp->updated?"running":"exited", absorbed, utime);
+
+ absorbed = remove_exited_child_from_parent(&stime, &pp->cstime);
+ if(unlikely(debug_enabled && absorbed))
+ debug_log(" > process %s (%d %s) absorbed " KERNEL_UINT_FORMAT " stime (remaining: " KERNEL_UINT_FORMAT ")", pp->comm, pp->pid, pp->updated?"running":"exited", absorbed, stime);
+
+ absorbed = remove_exited_child_from_parent(&gtime, &pp->cgtime);
+ if(unlikely(debug_enabled && absorbed))
+ debug_log(" > process %s (%d %s) absorbed " KERNEL_UINT_FORMAT " gtime (remaining: " KERNEL_UINT_FORMAT ")", pp->comm, pp->pid, pp->updated?"running":"exited", absorbed, gtime);
+
+ absorbed = remove_exited_child_from_parent(&minflt, &pp->cminflt);
+ if(unlikely(debug_enabled && absorbed))
+ debug_log(" > process %s (%d %s) absorbed " KERNEL_UINT_FORMAT " minflt (remaining: " KERNEL_UINT_FORMAT ")", pp->comm, pp->pid, pp->updated?"running":"exited", absorbed, minflt);
+
+ absorbed = remove_exited_child_from_parent(&majflt, &pp->cmajflt);
+ if(unlikely(debug_enabled && absorbed))
+ debug_log(" > process %s (%d %s) absorbed " KERNEL_UINT_FORMAT " majflt (remaining: " KERNEL_UINT_FORMAT ")", pp->comm, pp->pid, pp->updated?"running":"exited", absorbed, majflt);
+ }
+
+ if(unlikely(utime + stime + gtime + minflt + majflt > 0)) {
+ if(unlikely(debug_enabled)) {
+ if(utime) debug_find_lost_child(p, utime, 3);
+ if(stime) debug_find_lost_child(p, stime, 4);
+ if(gtime) debug_find_lost_child(p, gtime, 5);
+ if(minflt) debug_find_lost_child(p, minflt, 1);
+ if(majflt) debug_find_lost_child(p, majflt, 2);
+ }
+
+ p->keep = 1;
+
+ debug_log(" > remaining resources - KEEP - for another loop: %s (%d %s total resources: utime=" KERNEL_UINT_FORMAT " stime=" KERNEL_UINT_FORMAT " gtime=" KERNEL_UINT_FORMAT " minflt=" KERNEL_UINT_FORMAT " majflt=" KERNEL_UINT_FORMAT ")"
+ , p->comm
+ , p->pid
+ , p->updated?"running":"exited"
+ , utime
+ , stime
+ , gtime
+ , minflt
+ , majflt
+ );
+
+ for(pp = p->parent; pp ; pp = pp->parent) {
+ if(pp->updated) break;
+ pp->keep = 1;
+
+ debug_log(" > - KEEP - parent for another loop: %s (%d %s)"
+ , pp->comm
+ , pp->pid
+ , pp->updated?"running":"exited"
+ );
+ }
+
+ p->utime_raw = utime * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL);
+ p->stime_raw = stime * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL);
+ p->gtime_raw = gtime * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL);
+ p->minflt_raw = minflt * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL);
+ p->majflt_raw = majflt * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL);
+ p->cutime_raw = p->cstime_raw = p->cgtime_raw = p->cminflt_raw = p->cmajflt_raw = 0;
+
+ debug_log(" ");
+ }
+ else
+ debug_log(" > totally absorbed - DONE - %s (%d %s)"
+ , p->comm
+ , p->pid
+ , p->updated?"running":"exited"
+ );
+ }
+}
+
+static inline void link_all_processes_to_their_parents(void) {
+ struct pid_stat *p, *pp;
+
+ // link all children to their parents
+ // and update children count on parents
+ for(p = root_of_pids; p ; p = p->next) {
+ // for each process found
+
+ p->sortlist = 0;
+ p->parent = NULL;
+
+ if(unlikely(!p->ppid)) {
+ //unnecessary code from apps_plugin.c
+ //p->parent = NULL;
+ continue;
+ }
+
+ pp = all_pids[p->ppid];
+ if(likely(pp)) {
+ p->parent = pp;
+ pp->children_count++;
+
+ if(unlikely(debug_enabled || (p->target && p->target->debug_enabled)))
+ debug_log_int("child %d (%s, %s) on target '%s' has parent %d (%s, %s). Parent: utime=" KERNEL_UINT_FORMAT ", stime=" KERNEL_UINT_FORMAT ", gtime=" KERNEL_UINT_FORMAT ", minflt=" KERNEL_UINT_FORMAT ", majflt=" KERNEL_UINT_FORMAT ", cutime=" KERNEL_UINT_FORMAT ", cstime=" KERNEL_UINT_FORMAT ", cgtime=" KERNEL_UINT_FORMAT ", cminflt=" KERNEL_UINT_FORMAT ", cmajflt=" KERNEL_UINT_FORMAT "", p->pid, p->comm, p->updated?"running":"exited", (p->target)?p->target->name:"UNSET", pp->pid, pp->comm, pp->updated?"running":"exited", pp->utime, pp->stime, pp->gtime, pp->minflt, pp->majflt, pp->cutime, pp->cstime, pp->cgtime, pp->cminflt, pp->cmajflt);
+ }
+ else {
+ p->parent = NULL;
+ error("pid %d %s states parent %d, but the later does not exist.", p->pid, p->comm, p->ppid);
+ }
+ }
+}
+
+// ----------------------------------------------------------------------------
+
+// 1. read all files in /proc
+// 2. for each numeric directory:
+// i. read /proc/pid/stat
+// ii. read /proc/pid/status
+// iii. read /proc/pid/io (requires root access)
+// iii. read the entries in directory /proc/pid/fd (requires root access)
+// for each entry:
+// a. find or create a struct file_descriptor
+// b. cleanup any old/unused file_descriptors
+
+// after all these, some pids may be linked to targets, while others may not
+
+// in case of errors, only 1 every 1000 errors is printed
+// to avoid filling up all disk space
+// if debug is enabled, all errors are printed
+
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
+static int compar_pid(const void *pid1, const void *pid2) {
+
+ struct pid_stat *p1 = all_pids[*((pid_t *)pid1)];
+ struct pid_stat *p2 = all_pids[*((pid_t *)pid2)];
+
+ if(p1->sortlist > p2->sortlist)
+ return -1;
+ else
+ return 1;
+}
+#endif
+
+static inline int collect_data_for_pid(pid_t pid, void *ptr) {
+ if(unlikely(pid < 0 || pid > pid_max)) {
+ error("Invalid pid %d read (expected %d to %d). Ignoring process.", pid, 0, pid_max);
+ return 0;
+ }
+
+ struct pid_stat *p = get_pid_entry(pid);
+ if(unlikely(!p || p->read)) return 0;
+ p->read = 1;
+
+ // debug_log("Reading process %d (%s), sortlist %d", p->pid, p->comm, p->sortlist);
+
+ // --------------------------------------------------------------------
+ // /proc/<pid>/stat
+
+ if(unlikely(!managed_log(p, PID_LOG_STAT, read_proc_pid_stat(p, ptr))))
+ // there is no reason to proceed if we cannot get its status
+ return 0;
+
+ // check its parent pid
+ if(unlikely(p->ppid < 0 || p->ppid > pid_max)) {
+ error("Pid %d (command '%s') states invalid parent pid %d. Using 0.", pid, p->comm, p->ppid);
+ p->ppid = 0;
+ }
+
+ // --------------------------------------------------------------------
+ // /proc/<pid>/io
+
+ managed_log(p, PID_LOG_IO, read_proc_pid_io(p, ptr));
+
+ // --------------------------------------------------------------------
+ // /proc/<pid>/status
+
+ if(unlikely(!managed_log(p, PID_LOG_STATUS, read_proc_pid_status(p, ptr))))
+ // there is no reason to proceed if we cannot get its status
+ return 0;
+
+ // --------------------------------------------------------------------
+ // /proc/<pid>/fd
+
+ if(enable_file_charts)
+ managed_log(p, PID_LOG_FDS, read_pid_file_descriptors(p, ptr));
+
+ // --------------------------------------------------------------------
+ // done!
+
+ if(unlikely(debug_enabled && include_exited_childs && all_pids_count && p->ppid && all_pids[p->ppid] && !all_pids[p->ppid]->read))
+ debug_log("Read process %d (%s) sortlisted %d, but its parent %d (%s) sortlisted %d, is not read", p->pid, p->comm, p->sortlist, all_pids[p->ppid]->pid, all_pids[p->ppid]->comm, all_pids[p->ppid]->sortlist);
+
+ // mark it as updated
+ p->updated = 1;
+ p->keep = 0;
+ p->keeploops = 0;
+
+ return 1;
+}
+
+static int collect_data_for_all_processes(void) {
+ struct pid_stat *p = NULL;
+
+#ifndef __FreeBSD__
+ // clear process state counter
+ memset(proc_state_count, 0, sizeof proc_state_count);
+#else
+ int i, procnum;
+
+ static size_t procbase_size = 0;
+ static struct kinfo_proc *procbase = NULL;
+
+ size_t new_procbase_size;
+
+ int mib[3] = { CTL_KERN, KERN_PROC, KERN_PROC_PROC };
+ if (unlikely(sysctl(mib, 3, NULL, &new_procbase_size, NULL, 0))) {
+ error("sysctl error: Can't get processes data size");
+ return 0;
+ }
+
+ // give it some air for processes that may be started
+ // during this little time.
+ new_procbase_size += 100 * sizeof(struct kinfo_proc);
+
+ // increase the buffer if needed
+ if(new_procbase_size > procbase_size) {
+ procbase_size = new_procbase_size;
+ procbase = reallocz(procbase, procbase_size);
+ }
+
+ // sysctl() gets from new_procbase_size the buffer size
+ // and also returns to it the amount of data filled in
+ new_procbase_size = procbase_size;
+
+ // get the processes from the system
+ if (unlikely(sysctl(mib, 3, procbase, &new_procbase_size, NULL, 0))) {
+ error("sysctl error: Can't get processes data");
+ return 0;
+ }
+
+ // based on the amount of data filled in
+ // calculate the number of processes we got
+ procnum = new_procbase_size / sizeof(struct kinfo_proc);
+
+#endif
+
+ if(all_pids_count) {
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
+ size_t slc = 0;
+#endif
+ for(p = root_of_pids; p ; p = p->next) {
+ p->read = 0; // mark it as not read, so that collect_data_for_pid() will read it
+ p->updated = 0;
+ p->merged = 0;
+ p->children_count = 0;
+ p->parent = NULL;
+
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
+ all_pids_sortlist[slc++] = p->pid;
+#endif
+ }
+
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
+ if(unlikely(slc != all_pids_count)) {
+ error("Internal error: I was thinking I had %zu processes in my arrays, but it seems there are %zu.", all_pids_count, slc);
+ all_pids_count = slc;
+ }
+
+ if(include_exited_childs) {
+ // Read parents before childs
+ // This is needed to prevent a situation where
+ // a child is found running, but until we read
+ // its parent, it has exited and its parent
+ // has accumulated its resources.
+
+ qsort((void *)all_pids_sortlist, (size_t)all_pids_count, sizeof(pid_t), compar_pid);
+
+ // we forward read all running processes
+ // collect_data_for_pid() is smart enough,
+ // not to read the same pid twice per iteration
+ for(slc = 0; slc < all_pids_count; slc++) {
+ collect_data_for_pid(all_pids_sortlist[slc], NULL);
+ }
+ }
+#endif
+ }
+
+#ifdef __FreeBSD__
+ for (i = 0 ; i < procnum ; ++i) {
+ pid_t pid = procbase[i].ki_pid;
+ collect_data_for_pid(pid, &procbase[i]);
+ }
+#else
+ static char uptime_filename[FILENAME_MAX + 1] = "";
+ if(*uptime_filename == '\0')
+ snprintfz(uptime_filename, FILENAME_MAX, "%s/proc/uptime", netdata_configured_host_prefix);
+
+ global_uptime = (kernel_uint_t)(uptime_msec(uptime_filename) / MSEC_PER_SEC);
+
+ char dirname[FILENAME_MAX + 1];
+
+ snprintfz(dirname, FILENAME_MAX, "%s/proc", netdata_configured_host_prefix);
+ DIR *dir = opendir(dirname);
+ if(!dir) return 0;
+
+ struct dirent *de = NULL;
+
+ while((de = readdir(dir))) {
+ char *endptr = de->d_name;
+
+ if(unlikely(de->d_type != DT_DIR || de->d_name[0] < '0' || de->d_name[0] > '9'))
+ continue;
+
+ pid_t pid = (pid_t) strtoul(de->d_name, &endptr, 10);
+
+ // make sure we read a valid number
+ if(unlikely(endptr == de->d_name || *endptr != '\0'))
+ continue;
+
+ collect_data_for_pid(pid, NULL);
+ }
+ closedir(dir);
+#endif
+
+ if(!all_pids_count)
+ return 0;
+
+ // we need /proc/stat to normalize the cpu consumption of the exited childs
+ read_global_time();
+
+ // build the process tree
+ link_all_processes_to_their_parents();
+
+ // normally this is done
+ // however we may have processes exited while we collected values
+ // so let's find the exited ones
+ // we do this by collecting the ownership of process
+ // if we manage to get the ownership, the process still runs
+ process_exited_processes();
+ return 1;
+}
+
+// ----------------------------------------------------------------------------
+// update statistics on the targets
+
+// 1. link all childs to their parents
+// 2. go from bottom to top, marking as merged all childs to their parents
+// this step links all parents without a target to the child target, if any
+// 3. link all top level processes (the ones not merged) to the default target
+// 4. go from top to bottom, linking all childs without a target, to their parent target
+// after this step, all processes have a target
+// [5. for each killed pid (updated = 0), remove its usage from its target]
+// 6. zero all apps_groups_targets
+// 7. concentrate all values on the apps_groups_targets
+// 8. remove all killed processes
+// 9. find the unique file count for each target
+// check: update_apps_groups_statistics()
+
+static void cleanup_exited_pids(void) {
+ size_t c;
+ struct pid_stat *p = NULL;
+
+ for(p = root_of_pids; p ;) {
+ if(!p->updated && (!p->keep || p->keeploops > 0)) {
+ if(unlikely(debug_enabled && (p->keep || p->keeploops)))
+ debug_log(" > CLEANUP cannot keep exited process %d (%s) anymore - removing it.", p->pid, p->comm);
+
+ for(c = 0; c < p->fds_size; c++)
+ if(p->fds[c].fd > 0) {
+ file_descriptor_not_used(p->fds[c].fd);
+ clear_pid_fd(&p->fds[c]);
+ }
+
+ pid_t r = p->pid;
+ p = p->next;
+ del_pid_entry(r);
+ }
+ else {
+ if(unlikely(p->keep)) p->keeploops++;
+ p->keep = 0;
+ p = p->next;
+ }
+ }
+}
+
+static void apply_apps_groups_targets_inheritance(void) {
+ struct pid_stat *p = NULL;
+
+ // children that do not have a target
+ // inherit their target from their parent
+ int found = 1, loops = 0;
+ while(found) {
+ if(unlikely(debug_enabled)) loops++;
+ found = 0;
+ for(p = root_of_pids; p ; p = p->next) {
+ // if this process does not have a target
+ // and it has a parent
+ // and its parent has a target
+ // then, set the parent's target to this process
+ if(unlikely(!p->target && p->parent && p->parent->target)) {
+ p->target = p->parent->target;
+ found++;
+
+ if(debug_enabled || (p->target && p->target->debug_enabled))
+ debug_log_int("TARGET INHERITANCE: %s is inherited by %d (%s) from its parent %d (%s).", p->target->name, p->pid, p->comm, p->parent->pid, p->parent->comm);
+ }
+ }
+ }
+
+ // find all the procs with 0 childs and merge them to their parents
+ // repeat, until nothing more can be done.
+ int sortlist = 1;
+ found = 1;
+ while(found) {
+ if(unlikely(debug_enabled)) loops++;
+ found = 0;
+
+ for(p = root_of_pids; p ; p = p->next) {
+ if(unlikely(!p->sortlist && !p->children_count))
+ p->sortlist = sortlist++;
+
+ if(unlikely(
+ !p->children_count // if this process does not have any children
+ && !p->merged // and is not already merged
+ && p->parent // and has a parent
+ && p->parent->children_count // and its parent has children
+ // and the target of this process and its parent is the same,
+ // or the parent does not have a target
+ && (p->target == p->parent->target || !p->parent->target)
+ && p->ppid != INIT_PID // and its parent is not init
+ )) {
+ // mark it as merged
+ p->parent->children_count--;
+ p->merged = 1;
+
+ // the parent inherits the child's target, if it does not have a target itself
+ if(unlikely(p->target && !p->parent->target)) {
+ p->parent->target = p->target;
+
+ if(debug_enabled || (p->target && p->target->debug_enabled))
+ debug_log_int("TARGET INHERITANCE: %s is inherited by %d (%s) from its child %d (%s).", p->target->name, p->parent->pid, p->parent->comm, p->pid, p->comm);
+ }
+
+ found++;
+ }
+ }
+
+ debug_log("TARGET INHERITANCE: merged %d processes", found);
+ }
+
+ // init goes always to default target
+ if(all_pids[INIT_PID])
+ all_pids[INIT_PID]->target = apps_groups_default_target;
+
+ // pid 0 goes always to default target
+ if(all_pids[0])
+ all_pids[0]->target = apps_groups_default_target;
+
+ // give a default target on all top level processes
+ if(unlikely(debug_enabled)) loops++;
+ for(p = root_of_pids; p ; p = p->next) {
+ // if the process is not merged itself
+ // then is is a top level process
+ if(unlikely(!p->merged && !p->target))
+ p->target = apps_groups_default_target;
+
+ // make sure all processes have a sortlist
+ if(unlikely(!p->sortlist))
+ p->sortlist = sortlist++;
+ }
+
+ if(all_pids[1])
+ all_pids[1]->sortlist = sortlist++;
+
+ // give a target to all merged child processes
+ found = 1;
+ while(found) {
+ if(unlikely(debug_enabled)) loops++;
+ found = 0;
+ for(p = root_of_pids; p ; p = p->next) {
+ if(unlikely(!p->target && p->merged && p->parent && p->parent->target)) {
+ p->target = p->parent->target;
+ found++;
+
+ if(debug_enabled || (p->target && p->target->debug_enabled))
+ debug_log_int("TARGET INHERITANCE: %s is inherited by %d (%s) from its parent %d (%s) at phase 2.", p->target->name, p->pid, p->comm, p->parent->pid, p->parent->comm);
+ }
+ }
+ }
+
+ debug_log("apply_apps_groups_targets_inheritance() made %d loops on the process tree", loops);
+}
+
+static size_t zero_all_targets(struct target *root) {
+ struct target *w;
+ size_t count = 0;
+
+ for (w = root; w ; w = w->next) {
+ count++;
+
+ w->minflt = 0;
+ w->majflt = 0;
+ w->utime = 0;
+ w->stime = 0;
+ w->gtime = 0;
+ w->cminflt = 0;
+ w->cmajflt = 0;
+ w->cutime = 0;
+ w->cstime = 0;
+ w->cgtime = 0;
+ w->num_threads = 0;
+ // w->rss = 0;
+ w->processes = 0;
+
+ w->status_vmsize = 0;
+ w->status_vmrss = 0;
+ w->status_vmshared = 0;
+ w->status_rssfile = 0;
+ w->status_rssshmem = 0;
+ w->status_vmswap = 0;
+
+ w->io_logical_bytes_read = 0;
+ w->io_logical_bytes_written = 0;
+ w->io_read_calls = 0;
+ w->io_write_calls = 0;
+ w->io_storage_bytes_read = 0;
+ w->io_storage_bytes_written = 0;
+ w->io_cancelled_write_bytes = 0;
+
+ // zero file counters
+ if(w->target_fds) {
+ memset(w->target_fds, 0, sizeof(int) * w->target_fds_size);
+ w->openfds.files = 0;
+ w->openfds.pipes = 0;
+ w->openfds.sockets = 0;
+ w->openfds.inotifies = 0;
+ w->openfds.eventfds = 0;
+ w->openfds.timerfds = 0;
+ w->openfds.signalfds = 0;
+ w->openfds.eventpolls = 0;
+ w->openfds.other = 0;
+ }
+
+ w->collected_starttime = 0;
+ w->uptime_min = 0;
+ w->uptime_sum = 0;
+ w->uptime_max = 0;
+
+ if(unlikely(w->root_pid)) {
+ struct pid_on_target *pid_on_target_to_free, *pid_on_target = w->root_pid;
+
+ while(pid_on_target) {
+ pid_on_target_to_free = pid_on_target;
+ pid_on_target = pid_on_target->next;
+ freez(pid_on_target_to_free);
+ }
+
+ w->root_pid = NULL;
+ }
+ }
+
+ return count;
+}
+
+static inline void reallocate_target_fds(struct target *w) {
+ if(unlikely(!w))
+ return;
+
+ if(unlikely(!w->target_fds || w->target_fds_size < all_files_size)) {
+ w->target_fds = reallocz(w->target_fds, sizeof(int) * all_files_size);
+ memset(&w->target_fds[w->target_fds_size], 0, sizeof(int) * (all_files_size - w->target_fds_size));
+ w->target_fds_size = all_files_size;
+ }
+}
+
+static void aggregage_fd_type_on_openfds(FD_FILETYPE type, struct openfds *openfds) {
+ switch(type) {
+ case FILETYPE_FILE:
+ openfds->files++;
+ break;
+
+ case FILETYPE_PIPE:
+ openfds->pipes++;
+ break;
+
+ case FILETYPE_SOCKET:
+ openfds->sockets++;
+ break;
+
+ case FILETYPE_INOTIFY:
+ openfds->inotifies++;
+ break;
+
+ case FILETYPE_EVENTFD:
+ openfds->eventfds++;
+ break;
+
+ case FILETYPE_TIMERFD:
+ openfds->timerfds++;
+ break;
+
+ case FILETYPE_SIGNALFD:
+ openfds->signalfds++;
+ break;
+
+ case FILETYPE_EVENTPOLL:
+ openfds->eventpolls++;
+ break;
+
+ case FILETYPE_OTHER:
+ openfds->other++;
+ break;
+ }
+}
+
+static inline void aggregate_fd_on_target(int fd, struct target *w) {
+ if(unlikely(!w))
+ return;
+
+ if(unlikely(w->target_fds[fd])) {
+ // it is already aggregated
+ // just increase its usage counter
+ w->target_fds[fd]++;
+ return;
+ }
+
+ // increase its usage counter
+ // so that we will not add it again
+ w->target_fds[fd]++;
+
+ aggregage_fd_type_on_openfds(all_files[fd].type, &w->openfds);
+}
+
+static inline void aggregate_pid_fds_on_targets(struct pid_stat *p) {
+
+ if(unlikely(!p->updated)) {
+ // the process is not running
+ return;
+ }
+
+ struct target *w = p->target, *u = p->user_target, *g = p->group_target;
+
+ reallocate_target_fds(w);
+ reallocate_target_fds(u);
+ reallocate_target_fds(g);
+
+ p->openfds.files = 0;
+ p->openfds.pipes = 0;
+ p->openfds.sockets = 0;
+ p->openfds.inotifies = 0;
+ p->openfds.eventfds = 0;
+ p->openfds.timerfds = 0;
+ p->openfds.signalfds = 0;
+ p->openfds.eventpolls = 0;
+ p->openfds.other = 0;
+
+ long currentfds = 0;
+ size_t c, size = p->fds_size;
+ struct pid_fd *fds = p->fds;
+ for(c = 0; c < size ;c++) {
+ int fd = fds[c].fd;
+
+ if(likely(fd <= 0 || fd >= all_files_size))
+ continue;
+
+ currentfds++;
+ aggregage_fd_type_on_openfds(all_files[fd].type, &p->openfds);
+
+ aggregate_fd_on_target(fd, w);
+ aggregate_fd_on_target(fd, u);
+ aggregate_fd_on_target(fd, g);
+ }
+
+ if (currentfds >= currentmaxfds)
+ currentmaxfds = currentfds;
+}
+
+static inline void aggregate_pid_on_target(struct target *w, struct pid_stat *p, struct target *o) {
+ (void)o;
+
+ if(unlikely(!p->updated)) {
+ // the process is not running
+ return;
+ }
+
+ if(unlikely(!w)) {
+ error("pid %d %s was left without a target!", p->pid, p->comm);
+ return;
+ }
+
+ w->cutime += p->cutime;
+ w->cstime += p->cstime;
+ w->cgtime += p->cgtime;
+ w->cminflt += p->cminflt;
+ w->cmajflt += p->cmajflt;
+
+ w->utime += p->utime;
+ w->stime += p->stime;
+ w->gtime += p->gtime;
+ w->minflt += p->minflt;
+ w->majflt += p->majflt;
+
+ // w->rss += p->rss;
+
+ w->status_vmsize += p->status_vmsize;
+ w->status_vmrss += p->status_vmrss;
+ w->status_vmshared += p->status_vmshared;
+ w->status_rssfile += p->status_rssfile;
+ w->status_rssshmem += p->status_rssshmem;
+ w->status_vmswap += p->status_vmswap;
+
+ w->io_logical_bytes_read += p->io_logical_bytes_read;
+ w->io_logical_bytes_written += p->io_logical_bytes_written;
+ w->io_read_calls += p->io_read_calls;
+ w->io_write_calls += p->io_write_calls;
+ w->io_storage_bytes_read += p->io_storage_bytes_read;
+ w->io_storage_bytes_written += p->io_storage_bytes_written;
+ w->io_cancelled_write_bytes += p->io_cancelled_write_bytes;
+
+ w->processes++;
+ w->num_threads += p->num_threads;
+
+ if(!w->collected_starttime || p->collected_starttime < w->collected_starttime) w->collected_starttime = p->collected_starttime;
+ if(!w->uptime_min || p->uptime < w->uptime_min) w->uptime_min = p->uptime;
+ w->uptime_sum += p->uptime;
+ if(!w->uptime_max || w->uptime_max < p->uptime) w->uptime_max = p->uptime;
+
+ if(unlikely(debug_enabled || w->debug_enabled)) {
+ debug_log_int("aggregating '%s' pid %d on target '%s' utime=" KERNEL_UINT_FORMAT ", stime=" KERNEL_UINT_FORMAT ", gtime=" KERNEL_UINT_FORMAT ", cutime=" KERNEL_UINT_FORMAT ", cstime=" KERNEL_UINT_FORMAT ", cgtime=" KERNEL_UINT_FORMAT ", minflt=" KERNEL_UINT_FORMAT ", majflt=" KERNEL_UINT_FORMAT ", cminflt=" KERNEL_UINT_FORMAT ", cmajflt=" KERNEL_UINT_FORMAT "", p->comm, p->pid, w->name, p->utime, p->stime, p->gtime, p->cutime, p->cstime, p->cgtime, p->minflt, p->majflt, p->cminflt, p->cmajflt);
+
+ struct pid_on_target *pid_on_target = mallocz(sizeof(struct pid_on_target));
+ pid_on_target->pid = p->pid;
+ pid_on_target->next = w->root_pid;
+ w->root_pid = pid_on_target;
+ }
+}
+
+static inline void post_aggregate_targets(struct target *root) {
+ struct target *w;
+ for (w = root; w ; w = w->next) {
+ if(w->collected_starttime) {
+ if (!w->starttime || w->collected_starttime < w->starttime) {
+ w->starttime = w->collected_starttime;
+ }
+ } else {
+ w->starttime = 0;
+ }
+ }
+}
+
+static void calculate_netdata_statistics(void) {
+
+ apply_apps_groups_targets_inheritance();
+
+ zero_all_targets(users_root_target);
+ zero_all_targets(groups_root_target);
+ apps_groups_targets_count = zero_all_targets(apps_groups_root_target);
+
+ // this has to be done, before the cleanup
+ struct pid_stat *p = NULL;
+ struct target *w = NULL, *o = NULL;
+
+ // concentrate everything on the targets
+ for(p = root_of_pids; p ; p = p->next) {
+
+ // --------------------------------------------------------------------
+ // apps_groups target
+
+ aggregate_pid_on_target(p->target, p, NULL);
+
+
+ // --------------------------------------------------------------------
+ // user target
+
+ o = p->user_target;
+ if(likely(p->user_target && p->user_target->uid == p->uid))
+ w = p->user_target;
+ else {
+ if(unlikely(debug_enabled && p->user_target))
+ debug_log("pid %d (%s) switched user from %u (%s) to %u.", p->pid, p->comm, p->user_target->uid, p->user_target->name, p->uid);
+
+ w = p->user_target = get_users_target(p->uid);
+ }
+
+ aggregate_pid_on_target(w, p, o);
+
+
+ // --------------------------------------------------------------------
+ // user group target
+
+ o = p->group_target;
+ if(likely(p->group_target && p->group_target->gid == p->gid))
+ w = p->group_target;
+ else {
+ if(unlikely(debug_enabled && p->group_target))
+ debug_log("pid %d (%s) switched group from %u (%s) to %u.", p->pid, p->comm, p->group_target->gid, p->group_target->name, p->gid);
+
+ w = p->group_target = get_groups_target(p->gid);
+ }
+
+ aggregate_pid_on_target(w, p, o);
+
+
+ // --------------------------------------------------------------------
+ // aggregate all file descriptors
+
+ if(enable_file_charts)
+ aggregate_pid_fds_on_targets(p);
+ }
+
+ post_aggregate_targets(apps_groups_root_target);
+ post_aggregate_targets(users_root_target);
+ post_aggregate_targets(groups_root_target);
+
+ cleanup_exited_pids();
+}
+
+// ----------------------------------------------------------------------------
+// update chart dimensions
+
+static inline void send_BEGIN(const char *type, const char *id, usec_t usec) {
+ fprintf(stdout, "BEGIN %s.%s %llu\n", type, id, usec);
+}
+
+static inline void send_SET(const char *name, kernel_uint_t value) {
+ fprintf(stdout, "SET %s = " KERNEL_UINT_FORMAT "\n", name, value);
+}
+
+static inline void send_END(void) {
+ fprintf(stdout, "END\n");
+}
+
+void send_resource_usage_to_netdata(usec_t dt) {
+ static struct timeval last = { 0, 0 };
+ static struct rusage me_last;
+
+ struct timeval now;
+ struct rusage me;
+
+ usec_t cpuuser;
+ usec_t cpusyst;
+
+ if(!last.tv_sec) {
+ now_monotonic_timeval(&last);
+ getrusage(RUSAGE_SELF, &me_last);
+
+ cpuuser = 0;
+ cpusyst = 0;
+ }
+ else {
+ now_monotonic_timeval(&now);
+ getrusage(RUSAGE_SELF, &me);
+
+ cpuuser = me.ru_utime.tv_sec * USEC_PER_SEC + me.ru_utime.tv_usec;
+ cpusyst = me.ru_stime.tv_sec * USEC_PER_SEC + me.ru_stime.tv_usec;
+
+ memmove(&last, &now, sizeof(struct timeval));
+ memmove(&me_last, &me, sizeof(struct rusage));
+ }
+
+ static char created_charts = 0;
+ if(unlikely(!created_charts)) {
+ created_charts = 1;
+
+ fprintf(stdout,
+ "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' apps.plugin netdata.apps_cpu stacked 140000 %1$d\n"
+ "DIMENSION user '' incremental 1 1000\n"
+ "DIMENSION system '' incremental 1 1000\n"
+ "CHART netdata.apps_sizes '' 'Apps Plugin Files' 'files/s' apps.plugin netdata.apps_sizes line 140001 %1$d\n"
+ "DIMENSION calls '' incremental 1 1\n"
+ "DIMENSION files '' incremental 1 1\n"
+ "DIMENSION filenames '' incremental 1 1\n"
+ "DIMENSION inode_changes '' incremental 1 1\n"
+ "DIMENSION link_changes '' incremental 1 1\n"
+ "DIMENSION pids '' absolute 1 1\n"
+ "DIMENSION fds '' absolute 1 1\n"
+ "DIMENSION targets '' absolute 1 1\n"
+ "DIMENSION new_pids 'new pids' incremental 1 1\n"
+ , update_every
+ );
+
+ fprintf(stdout,
+ "CHART netdata.apps_fix '' 'Apps Plugin Normalization Ratios' 'percentage' apps.plugin netdata.apps_fix line 140002 %1$d\n"
+ "DIMENSION utime '' absolute 1 %2$llu\n"
+ "DIMENSION stime '' absolute 1 %2$llu\n"
+ "DIMENSION gtime '' absolute 1 %2$llu\n"
+ "DIMENSION minflt '' absolute 1 %2$llu\n"
+ "DIMENSION majflt '' absolute 1 %2$llu\n"
+ , update_every
+ , RATES_DETAIL
+ );
+
+ if(include_exited_childs)
+ fprintf(stdout,
+ "CHART netdata.apps_children_fix '' 'Apps Plugin Exited Children Normalization Ratios' 'percentage' apps.plugin netdata.apps_children_fix line 140003 %1$d\n"
+ "DIMENSION cutime '' absolute 1 %2$llu\n"
+ "DIMENSION cstime '' absolute 1 %2$llu\n"
+ "DIMENSION cgtime '' absolute 1 %2$llu\n"
+ "DIMENSION cminflt '' absolute 1 %2$llu\n"
+ "DIMENSION cmajflt '' absolute 1 %2$llu\n"
+ , update_every
+ , RATES_DETAIL
+ );
+
+ }
+
+ fprintf(stdout,
+ "BEGIN netdata.apps_cpu %llu\n"
+ "SET user = %llu\n"
+ "SET system = %llu\n"
+ "END\n"
+ "BEGIN netdata.apps_sizes %llu\n"
+ "SET calls = %zu\n"
+ "SET files = %zu\n"
+ "SET filenames = %zu\n"
+ "SET inode_changes = %zu\n"
+ "SET link_changes = %zu\n"
+ "SET pids = %zu\n"
+ "SET fds = %d\n"
+ "SET targets = %zu\n"
+ "SET new_pids = %zu\n"
+ "END\n"
+ , dt
+ , cpuuser
+ , cpusyst
+ , dt
+ , calls_counter
+ , file_counter
+ , filenames_allocated_counter
+ , inodes_changed_counter
+ , links_changed_counter
+ , all_pids_count
+ , all_files_len
+ , apps_groups_targets_count
+ , targets_assignment_counter
+ );
+
+ fprintf(stdout,
+ "BEGIN netdata.apps_fix %llu\n"
+ "SET utime = %u\n"
+ "SET stime = %u\n"
+ "SET gtime = %u\n"
+ "SET minflt = %u\n"
+ "SET majflt = %u\n"
+ "END\n"
+ , dt
+ , (unsigned int)(utime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(stime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(gtime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(minflt_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(majflt_fix_ratio * 100 * RATES_DETAIL)
+ );
+
+ if(include_exited_childs)
+ fprintf(stdout,
+ "BEGIN netdata.apps_children_fix %llu\n"
+ "SET cutime = %u\n"
+ "SET cstime = %u\n"
+ "SET cgtime = %u\n"
+ "SET cminflt = %u\n"
+ "SET cmajflt = %u\n"
+ "END\n"
+ , dt
+ , (unsigned int)(cutime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(cstime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(cgtime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(cminflt_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(cmajflt_fix_ratio * 100 * RATES_DETAIL)
+ );
+}
+
+static void normalize_utilization(struct target *root) {
+ struct target *w;
+
+ // childs processing introduces spikes
+ // here we try to eliminate them by disabling childs processing either for specific dimensions
+ // or entirely. Of course, either way, we disable it just a single iteration.
+
+ kernel_uint_t max_time = processors * time_factor * RATES_DETAIL;
+ kernel_uint_t utime = 0, cutime = 0, stime = 0, cstime = 0, gtime = 0, cgtime = 0, minflt = 0, cminflt = 0, majflt = 0, cmajflt = 0;
+
+ if(global_utime > max_time) global_utime = max_time;
+ if(global_stime > max_time) global_stime = max_time;
+ if(global_gtime > max_time) global_gtime = max_time;
+
+ for(w = root; w ; w = w->next) {
+ if(w->target || (!w->processes && !w->exposed)) continue;
+
+ utime += w->utime;
+ stime += w->stime;
+ gtime += w->gtime;
+ cutime += w->cutime;
+ cstime += w->cstime;
+ cgtime += w->cgtime;
+
+ minflt += w->minflt;
+ majflt += w->majflt;
+ cminflt += w->cminflt;
+ cmajflt += w->cmajflt;
+ }
+
+ if(global_utime || global_stime || global_gtime) {
+ if(global_utime + global_stime + global_gtime > utime + cutime + stime + cstime + gtime + cgtime) {
+ // everything we collected fits
+ utime_fix_ratio =
+ stime_fix_ratio =
+ gtime_fix_ratio =
+ cutime_fix_ratio =
+ cstime_fix_ratio =
+ cgtime_fix_ratio = 1.0; //(NETDATA_DOUBLE)(global_utime + global_stime) / (NETDATA_DOUBLE)(utime + cutime + stime + cstime);
+ }
+ else if((global_utime + global_stime > utime + stime) && (cutime || cstime)) {
+ // children resources are too high
+ // lower only the children resources
+ utime_fix_ratio =
+ stime_fix_ratio =
+ gtime_fix_ratio = 1.0;
+ cutime_fix_ratio =
+ cstime_fix_ratio =
+ cgtime_fix_ratio = (NETDATA_DOUBLE)((global_utime + global_stime) - (utime + stime)) / (NETDATA_DOUBLE)(cutime + cstime);
+ }
+ else if(utime || stime) {
+ // even running processes are unrealistic
+ // zero the children resources
+ // lower the running processes resources
+ utime_fix_ratio =
+ stime_fix_ratio =
+ gtime_fix_ratio = (NETDATA_DOUBLE)(global_utime + global_stime) / (NETDATA_DOUBLE)(utime + stime);
+ cutime_fix_ratio =
+ cstime_fix_ratio =
+ cgtime_fix_ratio = 0.0;
+ }
+ else {
+ utime_fix_ratio =
+ stime_fix_ratio =
+ gtime_fix_ratio =
+ cutime_fix_ratio =
+ cstime_fix_ratio =
+ cgtime_fix_ratio = 0.0;
+ }
+ }
+ else {
+ utime_fix_ratio =
+ stime_fix_ratio =
+ gtime_fix_ratio =
+ cutime_fix_ratio =
+ cstime_fix_ratio =
+ cgtime_fix_ratio = 0.0;
+ }
+
+ if(utime_fix_ratio > 1.0) utime_fix_ratio = 1.0;
+ if(cutime_fix_ratio > 1.0) cutime_fix_ratio = 1.0;
+ if(stime_fix_ratio > 1.0) stime_fix_ratio = 1.0;
+ if(cstime_fix_ratio > 1.0) cstime_fix_ratio = 1.0;
+ if(gtime_fix_ratio > 1.0) gtime_fix_ratio = 1.0;
+ if(cgtime_fix_ratio > 1.0) cgtime_fix_ratio = 1.0;
+
+ // if(utime_fix_ratio < 0.0) utime_fix_ratio = 0.0;
+ // if(cutime_fix_ratio < 0.0) cutime_fix_ratio = 0.0;
+ // if(stime_fix_ratio < 0.0) stime_fix_ratio = 0.0;
+ // if(cstime_fix_ratio < 0.0) cstime_fix_ratio = 0.0;
+ // if(gtime_fix_ratio < 0.0) gtime_fix_ratio = 0.0;
+ // if(cgtime_fix_ratio < 0.0) cgtime_fix_ratio = 0.0;
+
+ // TODO
+ // we use cpu time to normalize page faults
+ // the problem is that to find the proper max values
+ // for page faults we have to parse /proc/vmstat
+ // which is quite big to do it again (netdata does it already)
+ //
+ // a better solution could be to somehow have netdata
+ // do this normalization for us
+
+ if(utime || stime || gtime)
+ majflt_fix_ratio =
+ minflt_fix_ratio = (NETDATA_DOUBLE)(utime * utime_fix_ratio + stime * stime_fix_ratio + gtime * gtime_fix_ratio) / (NETDATA_DOUBLE)(utime + stime + gtime);
+ else
+ minflt_fix_ratio =
+ majflt_fix_ratio = 1.0;
+
+ if(cutime || cstime || cgtime)
+ cmajflt_fix_ratio =
+ cminflt_fix_ratio = (NETDATA_DOUBLE)(cutime * cutime_fix_ratio + cstime * cstime_fix_ratio + cgtime * cgtime_fix_ratio) / (NETDATA_DOUBLE)(cutime + cstime + cgtime);
+ else
+ cminflt_fix_ratio =
+ cmajflt_fix_ratio = 1.0;
+
+ // the report
+
+ debug_log(
+ "SYSTEM: u=" KERNEL_UINT_FORMAT " s=" KERNEL_UINT_FORMAT " g=" KERNEL_UINT_FORMAT " "
+ "COLLECTED: u=" KERNEL_UINT_FORMAT " s=" KERNEL_UINT_FORMAT " g=" KERNEL_UINT_FORMAT " cu=" KERNEL_UINT_FORMAT " cs=" KERNEL_UINT_FORMAT " cg=" KERNEL_UINT_FORMAT " "
+ "DELTA: u=" KERNEL_UINT_FORMAT " s=" KERNEL_UINT_FORMAT " g=" KERNEL_UINT_FORMAT " "
+ "FIX: u=%0.2f s=%0.2f g=%0.2f cu=%0.2f cs=%0.2f cg=%0.2f "
+ "FINALLY: u=" KERNEL_UINT_FORMAT " s=" KERNEL_UINT_FORMAT " g=" KERNEL_UINT_FORMAT " cu=" KERNEL_UINT_FORMAT " cs=" KERNEL_UINT_FORMAT " cg=" KERNEL_UINT_FORMAT " "
+ , global_utime
+ , global_stime
+ , global_gtime
+ , utime
+ , stime
+ , gtime
+ , cutime
+ , cstime
+ , cgtime
+ , utime + cutime - global_utime
+ , stime + cstime - global_stime
+ , gtime + cgtime - global_gtime
+ , utime_fix_ratio
+ , stime_fix_ratio
+ , gtime_fix_ratio
+ , cutime_fix_ratio
+ , cstime_fix_ratio
+ , cgtime_fix_ratio
+ , (kernel_uint_t)(utime * utime_fix_ratio)
+ , (kernel_uint_t)(stime * stime_fix_ratio)
+ , (kernel_uint_t)(gtime * gtime_fix_ratio)
+ , (kernel_uint_t)(cutime * cutime_fix_ratio)
+ , (kernel_uint_t)(cstime * cstime_fix_ratio)
+ , (kernel_uint_t)(cgtime * cgtime_fix_ratio)
+ );
+}
+
+static void send_collected_data_to_netdata(struct target *root, const char *type, usec_t dt) {
+ struct target *w;
+
+ send_BEGIN(type, "cpu", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, (kernel_uint_t)(w->utime * utime_fix_ratio) + (kernel_uint_t)(w->stime * stime_fix_ratio) + (kernel_uint_t)(w->gtime * gtime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cutime * cutime_fix_ratio) + (kernel_uint_t)(w->cstime * cstime_fix_ratio) + (kernel_uint_t)(w->cgtime * cgtime_fix_ratio)):0ULL));
+ }
+ send_END();
+
+ send_BEGIN(type, "cpu_user", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, (kernel_uint_t)(w->utime * utime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cutime * cutime_fix_ratio)):0ULL));
+ }
+ send_END();
+
+ send_BEGIN(type, "cpu_system", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, (kernel_uint_t)(w->stime * stime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cstime * cstime_fix_ratio)):0ULL));
+ }
+ send_END();
+
+ if(show_guest_time) {
+ send_BEGIN(type, "cpu_guest", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, (kernel_uint_t)(w->gtime * gtime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cgtime * cgtime_fix_ratio)):0ULL));
+ }
+ send_END();
+ }
+
+ send_BEGIN(type, "threads", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ send_SET(w->name, w->num_threads);
+ }
+ send_END();
+
+ send_BEGIN(type, "processes", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ send_SET(w->name, w->processes);
+ }
+ send_END();
+
+#ifndef __FreeBSD__
+ send_BEGIN(type, "uptime", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, (global_uptime > w->starttime)?(global_uptime - w->starttime):0);
+ }
+ send_END();
+
+ if (enable_detailed_uptime_charts) {
+ send_BEGIN(type, "uptime_min", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->uptime_min);
+ }
+ send_END();
+
+ send_BEGIN(type, "uptime_avg", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->uptime_sum / w->processes);
+ }
+ send_END();
+
+ send_BEGIN(type, "uptime_max", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->uptime_max);
+ }
+ send_END();
+ }
+#endif
+
+ send_BEGIN(type, "mem", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, (w->status_vmrss > w->status_vmshared)?(w->status_vmrss - w->status_vmshared):0ULL);
+ }
+ send_END();
+
+ send_BEGIN(type, "vmem", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->status_vmsize);
+ }
+ send_END();
+
+#ifndef __FreeBSD__
+ send_BEGIN(type, "swap", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->status_vmswap);
+ }
+ send_END();
+#endif
+
+ send_BEGIN(type, "minor_faults", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, (kernel_uint_t)(w->minflt * minflt_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cminflt * cminflt_fix_ratio)):0ULL));
+ }
+ send_END();
+
+ send_BEGIN(type, "major_faults", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, (kernel_uint_t)(w->majflt * majflt_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cmajflt * cmajflt_fix_ratio)):0ULL));
+ }
+ send_END();
+
+#ifndef __FreeBSD__
+ send_BEGIN(type, "lreads", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->io_logical_bytes_read);
+ }
+ send_END();
+
+ send_BEGIN(type, "lwrites", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->io_logical_bytes_written);
+ }
+ send_END();
+#endif
+
+ send_BEGIN(type, "preads", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->io_storage_bytes_read);
+ }
+ send_END();
+
+ send_BEGIN(type, "pwrites", dt);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->io_storage_bytes_written);
+ }
+ send_END();
+
+ if(enable_file_charts) {
+ send_BEGIN(type, "files", dt);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->openfds.files);
+ }
+ if (!strcmp("apps", type)){
+ kernel_uint_t usedfdpercentage = (kernel_uint_t) ((currentmaxfds * 100) / sysconf(_SC_OPEN_MAX));
+ fprintf(stdout, "VARIABLE fdperc = " KERNEL_UINT_FORMAT "\n", usedfdpercentage);
+ }
+ send_END();
+
+ send_BEGIN(type, "sockets", dt);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->openfds.sockets);
+ }
+ send_END();
+
+ send_BEGIN(type, "pipes", dt);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes))
+ send_SET(w->name, w->openfds.pipes);
+ }
+ send_END();
+ }
+}
+
+
+// ----------------------------------------------------------------------------
+// generate the charts
+
+static void send_charts_updates_to_netdata(struct target *root, const char *type, const char *title)
+{
+ struct target *w;
+ int newly_added = 0;
+
+ for(w = root ; w ; w = w->next) {
+ if (w->target) continue;
+
+ if(unlikely(w->processes && (debug_enabled || w->debug_enabled))) {
+ struct pid_on_target *pid_on_target;
+
+ fprintf(stderr, "apps.plugin: target '%s' has aggregated %u process%s:", w->name, w->processes, (w->processes == 1)?"":"es");
+
+ for(pid_on_target = w->root_pid; pid_on_target; pid_on_target = pid_on_target->next) {
+ fprintf(stderr, " %d", pid_on_target->pid);
+ }
+
+ fputc('\n', stderr);
+ }
+
+ if (!w->exposed && w->processes) {
+ newly_added++;
+ w->exposed = 1;
+ if (debug_enabled || w->debug_enabled)
+ debug_log_int("%s just added - regenerating charts.", w->name);
+ }
+ }
+
+ // nothing more to show
+ if(!newly_added && show_guest_time == show_guest_time_old) return;
+
+ // we have something new to show
+ // update the charts
+ fprintf(stdout, "CHART %s.cpu '' '%s CPU Time (100%% = 1 core)' 'percentage' cpu %s.cpu stacked 20001 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu %s\n", w->name, time_factor * RATES_DETAIL / 100, w->hidden ? "hidden" : "");
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.mem '' '%s Real Memory (w/o shared)' 'MiB' mem %s.mem stacked 20003 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute %ld %ld\n", w->name, 1L, 1024L);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+
+ fprintf(stdout, "CHART %s.vmem '' '%s Virtual Memory Size' 'MiB' mem %s.vmem stacked 20005 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute %ld %ld\n", w->name, 1L, 1024L);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.threads '' '%s Threads' 'threads' processes %s.threads stacked 20006 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.processes '' '%s Processes' 'processes' processes %s.processes stacked 20007 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+#ifndef __FreeBSD__
+ fprintf(stdout, "CHART %s.uptime '' '%s Carried Over Uptime' 'seconds' processes %s.uptime line 20008 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ if (enable_detailed_uptime_charts) {
+ fprintf(stdout, "CHART %s.uptime_min '' '%s Minimum Uptime' 'seconds' processes %s.uptime_min line 20009 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.uptime_avg '' '%s Average Uptime' 'seconds' processes %s.uptime_avg line 20010 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.uptime_max '' '%s Maximum Uptime' 'seconds' processes %s.uptime_max line 20011 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+ }
+#endif
+
+ fprintf(stdout, "CHART %s.cpu_user '' '%s CPU User Time (100%% = 1 core)' 'percentage' cpu %s.cpu_user stacked 20020 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, time_factor * RATES_DETAIL / 100LLU);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.cpu_system '' '%s CPU System Time (100%% = 1 core)' 'percentage' cpu %s.cpu_system stacked 20021 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, time_factor * RATES_DETAIL / 100LLU);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ if(show_guest_time) {
+ fprintf(stdout, "CHART %s.cpu_guest '' '%s CPU Guest Time (100%% = 1 core)' 'percentage' cpu %s.cpu_guest stacked 20022 %d\n", type, title, type, update_every);
+ for (w = root; w; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, time_factor * RATES_DETAIL / 100LLU);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+ }
+
+#ifndef __FreeBSD__
+ fprintf(stdout, "CHART %s.swap '' '%s Swap Memory' 'MiB' swap %s.swap stacked 20011 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute %ld %ld\n", w->name, 1L, 1024L);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+#endif
+
+ fprintf(stdout, "CHART %s.major_faults '' '%s Major Page Faults (swap read)' 'page faults/s' swap %s.major_faults stacked 20012 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, RATES_DETAIL);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.minor_faults '' '%s Minor Page Faults' 'page faults/s' mem %s.minor_faults stacked 20011 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, RATES_DETAIL);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+#ifdef __FreeBSD__
+ fprintf(stdout, "CHART %s.preads '' '%s Disk Reads' 'blocks/s' disk %s.preads stacked 20002 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, RATES_DETAIL);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.pwrites '' '%s Disk Writes' 'blocks/s' disk %s.pwrites stacked 20002 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, RATES_DETAIL);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+#else
+ fprintf(stdout, "CHART %s.preads '' '%s Disk Reads' 'KiB/s' disk %s.preads stacked 20002 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.pwrites '' '%s Disk Writes' 'KiB/s' disk %s.pwrites stacked 20002 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.lreads '' '%s Disk Logical Reads' 'KiB/s' disk %s.lreads stacked 20042 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.lwrites '' '%s I/O Logical Writes' 'KiB/s' disk %s.lwrites stacked 20042 %d\n", type, title, type, update_every);
+ for (w = root; w ; w = w->next) {
+ if(unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+#endif
+
+ if(enable_file_charts) {
+ fprintf(stdout, "CHART %s.files '' '%s Open Files' 'open files' disk %s.files stacked 20050 %d\n", type,
+ title, type, update_every);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.sockets '' '%s Open Sockets' 'open sockets' net %s.sockets stacked 20051 %d\n",
+ type, title, type, update_every);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+
+ fprintf(stdout, "CHART %s.pipes '' '%s Pipes' 'open pipes' processes %s.pipes stacked 20053 %d\n", type,
+ title, type, update_every);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
+ }
+ APPS_PLUGIN_FUNCTIONS();
+ }
+}
+
+
+#ifndef __FreeBSD__
+static void send_proc_states_count(usec_t dt)
+{
+ static bool chart_added = false;
+ // create chart for count of processes in different states
+ if (!chart_added) {
+ fprintf(
+ stdout,
+ "CHART system.processes_state '' 'System Processes State' 'processes' processes system.processes_state line %d %d\n",
+ NETDATA_CHART_PRIO_SYSTEM_PROCESS_STATES,
+ update_every);
+ for (proc_state i = PROC_STATUS_RUNNING; i < PROC_STATUS_END; i++) {
+ fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", proc_states[i]);
+ }
+ chart_added = true;
+ }
+
+ // send process state count
+ send_BEGIN("system", "processes_state", dt);
+ for (proc_state i = PROC_STATUS_RUNNING; i < PROC_STATUS_END; i++) {
+ send_SET(proc_states[i], proc_state_count[i]);
+ }
+ send_END();
+}
+#endif
+
+// ----------------------------------------------------------------------------
+// parse command line arguments
+
+int check_proc_1_io() {
+ int ret = 0;
+
+ procfile *ff = procfile_open("/proc/1/io", NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
+ if(!ff) goto cleanup;
+
+ ff = procfile_readall(ff);
+ if(!ff) goto cleanup;
+
+ ret = 1;
+
+cleanup:
+ procfile_close(ff);
+ return ret;
+}
+
+static void parse_args(int argc, char **argv)
+{
+ int i, freq = 0;
+
+ for(i = 1; i < argc; i++) {
+ if(!freq) {
+ int n = (int)str2l(argv[i]);
+ if(n > 0) {
+ freq = n;
+ continue;
+ }
+ }
+
+ if(strcmp("version", argv[i]) == 0 || strcmp("-version", argv[i]) == 0 || strcmp("--version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) {
+ printf("apps.plugin %s\n", VERSION);
+ exit(0);
+ }
+
+ if(strcmp("test-permissions", argv[i]) == 0 || strcmp("-t", argv[i]) == 0) {
+ if(!check_proc_1_io()) {
+ perror("Tried to read /proc/1/io and it failed");
+ exit(1);
+ }
+ printf("OK\n");
+ exit(0);
+ }
+
+ if(strcmp("debug", argv[i]) == 0) {
+ debug_enabled = 1;
+#ifndef NETDATA_INTERNAL_CHECKS
+ fprintf(stderr, "apps.plugin has been compiled without debugging\n");
+#endif
+ continue;
+ }
+
+#ifndef __FreeBSD__
+ if(strcmp("fds-cache-secs", argv[i]) == 0) {
+ if(argc <= i + 1) {
+ fprintf(stderr, "Parameter 'fds-cache-secs' requires a number as argument.\n");
+ exit(1);
+ }
+ i++;
+ max_fds_cache_seconds = str2i(argv[i]);
+ if(max_fds_cache_seconds < 0) max_fds_cache_seconds = 0;
+ continue;
+ }
+#endif
+
+ if(strcmp("no-childs", argv[i]) == 0 || strcmp("without-childs", argv[i]) == 0) {
+ include_exited_childs = 0;
+ continue;
+ }
+
+ if(strcmp("with-childs", argv[i]) == 0) {
+ include_exited_childs = 1;
+ continue;
+ }
+
+ if(strcmp("with-guest", argv[i]) == 0) {
+ enable_guest_charts = 1;
+ continue;
+ }
+
+ if(strcmp("no-guest", argv[i]) == 0 || strcmp("without-guest", argv[i]) == 0) {
+ enable_guest_charts = 0;
+ continue;
+ }
+
+ if(strcmp("with-files", argv[i]) == 0) {
+ enable_file_charts = 1;
+ continue;
+ }
+
+ if(strcmp("no-files", argv[i]) == 0 || strcmp("without-files", argv[i]) == 0) {
+ enable_file_charts = 0;
+ continue;
+ }
+
+ if(strcmp("no-users", argv[i]) == 0 || strcmp("without-users", argv[i]) == 0) {
+ enable_users_charts = 0;
+ continue;
+ }
+
+ if(strcmp("no-groups", argv[i]) == 0 || strcmp("without-groups", argv[i]) == 0) {
+ enable_groups_charts = 0;
+ continue;
+ }
+
+ if(strcmp("with-detailed-uptime", argv[i]) == 0) {
+ enable_detailed_uptime_charts = 1;
+ continue;
+ }
+
+ if(strcmp("-h", argv[i]) == 0 || strcmp("--help", argv[i]) == 0) {
+ fprintf(stderr,
+ "\n"
+ " netdata apps.plugin %s\n"
+ " Copyright (C) 2016-2017 Costa Tsaousis <costa@tsaousis.gr>\n"
+ " Released under GNU General Public License v3 or later.\n"
+ " All rights reserved.\n"
+ "\n"
+ " This program is a data collector plugin for netdata.\n"
+ "\n"
+ " Available command line options:\n"
+ "\n"
+ " SECONDS set the data collection frequency\n"
+ "\n"
+ " debug enable debugging (lot of output)\n"
+ "\n"
+ " with-childs\n"
+ " without-childs enable / disable aggregating exited\n"
+ " children resources into parents\n"
+ " (default is enabled)\n"
+ "\n"
+ " with-guest\n"
+ " without-guest enable / disable reporting guest charts\n"
+ " (default is disabled)\n"
+ "\n"
+ " with-files\n"
+ " without-files enable / disable reporting files, sockets, pipes\n"
+ " (default is enabled)\n"
+ "\n"
+ " without-users disable reporting per user charts\n"
+ "\n"
+ " without-groups disable reporting per user group charts\n"
+ "\n"
+ " with-detailed-uptime enable reporting min/avg/max uptime charts\n"
+ "\n"
+#ifndef __FreeBSD__
+ " fds-cache-secs N cache the files of processed for N seconds\n"
+ " caching is adaptive per file (when a file\n"
+ " is found, it starts at 0 and while the file\n"
+ " remains open, it is incremented up to the\n"
+ " max given)\n"
+ " (default is %d seconds)\n"
+ "\n"
+#endif
+ " version or -v or -V print program version and exit\n"
+ "\n"
+ , VERSION
+#ifndef __FreeBSD__
+ , max_fds_cache_seconds
+#endif
+ );
+ exit(1);
+ }
+
+ error("Cannot understand option %s", argv[i]);
+ exit(1);
+ }
+
+ if(freq > 0) update_every = freq;
+
+ if(read_apps_groups_conf(user_config_dir, "groups")) {
+ info("Cannot read process groups configuration file '%s/apps_groups.conf'. Will try '%s/apps_groups.conf'", user_config_dir, stock_config_dir);
+
+ if(read_apps_groups_conf(stock_config_dir, "groups")) {
+ error("Cannot read process groups '%s/apps_groups.conf'. There are no internal defaults. Failing.", stock_config_dir);
+ exit(1);
+ }
+ else
+ info("Loaded config file '%s/apps_groups.conf'", stock_config_dir);
+ }
+ else
+ info("Loaded config file '%s/apps_groups.conf'", user_config_dir);
+}
+
+static int am_i_running_as_root() {
+ uid_t uid = getuid(), euid = geteuid();
+
+ if(uid == 0 || euid == 0) {
+ if(debug_enabled) info("I am running with escalated privileges, uid = %u, euid = %u.", uid, euid);
+ return 1;
+ }
+
+ if(debug_enabled) info("I am not running with escalated privileges, uid = %u, euid = %u.", uid, euid);
+ return 0;
+}
+
+#ifdef HAVE_CAPABILITY
+static int check_capabilities() {
+ cap_t caps = cap_get_proc();
+ if(!caps) {
+ error("Cannot get current capabilities.");
+ return 0;
+ }
+ else if(debug_enabled)
+ info("Received my capabilities from the system.");
+
+ int ret = 1;
+
+ cap_flag_value_t cfv = CAP_CLEAR;
+ if(cap_get_flag(caps, CAP_DAC_READ_SEARCH, CAP_EFFECTIVE, &cfv) == -1) {
+ error("Cannot find if CAP_DAC_READ_SEARCH is effective.");
+ ret = 0;
+ }
+ else {
+ if(cfv != CAP_SET) {
+ error("apps.plugin should run with CAP_DAC_READ_SEARCH.");
+ ret = 0;
+ }
+ else if(debug_enabled)
+ info("apps.plugin runs with CAP_DAC_READ_SEARCH.");
+ }
+
+ cfv = CAP_CLEAR;
+ if(cap_get_flag(caps, CAP_SYS_PTRACE, CAP_EFFECTIVE, &cfv) == -1) {
+ error("Cannot find if CAP_SYS_PTRACE is effective.");
+ ret = 0;
+ }
+ else {
+ if(cfv != CAP_SET) {
+ error("apps.plugin should run with CAP_SYS_PTRACE.");
+ ret = 0;
+ }
+ else if(debug_enabled)
+ info("apps.plugin runs with CAP_SYS_PTRACE.");
+ }
+
+ cap_free(caps);
+
+ return ret;
+}
+#else
+static int check_capabilities() {
+ return 0;
+}
+#endif
+
+netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
+
+#define PROCESS_FILTER_CATEGORY "category:"
+#define PROCESS_FILTER_USER "user:"
+#define PROCESS_FILTER_GROUP "group:"
+#define PROCESS_FILTER_PROCESS "process:"
+#define PROCESS_FILTER_PID "pid:"
+#define PROCESS_FILTER_UID "uid:"
+#define PROCESS_FILTER_GID "gid:"
+
+static struct target *find_target_by_name(struct target *base, const char *name) {
+ struct target *t;
+ for(t = base; t ; t = t->next) {
+ if (strcmp(t->name, name) == 0)
+ return t;
+ }
+
+ return NULL;
+}
+
+static kernel_uint_t MemTotal = 0;
+
+static void get_MemTotal(void) {
+#ifdef __FreeBSD__
+ // TODO - fix this for FreeBSD
+ return;
+#else
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/meminfo", netdata_configured_host_prefix);
+
+ procfile *ff = procfile_open(filename, ": \t", PROCFILE_FLAG_DEFAULT);
+ if(!ff)
+ return;
+
+ ff = procfile_readall(ff);
+ if(!ff)
+ return;
+
+ size_t line, lines = procfile_lines(ff);
+
+ for(line = 0; line < lines ;line++) {
+ size_t words = procfile_linewords(ff, line);
+ if(words == 3 && strcmp(procfile_lineword(ff, line, 0), "MemTotal") == 0 && strcmp(procfile_lineword(ff, line, 2), "kB") == 0) {
+ kernel_uint_t n = str2ull(procfile_lineword(ff, line, 1));
+ if(n) MemTotal = n;
+ break;
+ }
+ }
+
+ procfile_close(ff);
+#endif
+}
+
+static void apps_plugin_function_error(const char *transaction, int code, const char *msg) {
+ char buffer[PLUGINSD_LINE_MAX + 1];
+ json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
+
+ pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());
+ fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);
+ pluginsd_function_result_end_to_stdout();
+}
+
+static void apps_plugin_function_processes_help(const char *transaction) {
+ pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);
+ fprintf(stdout, "%s",
+ "apps.plugin / processes\n"
+ "\n"
+ "Function `processes` presents all the currently running processes of the system.\n"
+ "\n"
+ "The following filters are supported:\n"
+ "\n"
+ " category:NAME\n"
+ " Shows only processes that are assigned the category `NAME` in apps_groups.conf\n"
+ "\n"
+ " user:NAME\n"
+ " Shows only processes that are running as user name `NAME`.\n"
+ "\n"
+ " group:NAME\n"
+ " Shows only processes that are running as group name `NAME`.\n"
+ "\n"
+ " process:NAME\n"
+ " Shows only processes that their Command is `NAME` or their parent's Command is `NAME`.\n"
+ "\n"
+ " pid:NUMBER\n"
+ " Shows only processes that their PID is `NUMBER` or their parent's PID is `NUMBER`\n"
+ "\n"
+ " uid:NUMBER\n"
+ " Shows only processes that their UID is `NUMBER`\n"
+ "\n"
+ " gid:NUMBER\n"
+ " Shows only processes that their GID is `NUMBER`\n"
+ "\n"
+ "Filters can be combined. Each filter can be given only one time.\n"
+ );
+ pluginsd_function_result_end_to_stdout();
+}
+
+#define add_table_field(wb, key, name, visible, type, units, max, sort, sortable, sticky, unique_key, pointer_to, summary) do { \
+ if(fields_added) buffer_strcat(wb, ","); \
+ buffer_sprintf(wb, "\n \"%s\": {", key); \
+ buffer_sprintf(wb, "\n \"index\":%d,", fields_added); \
+ buffer_sprintf(wb, "\n \"unique_key\":%s,", (unique_key)?"true":"false"); \
+ buffer_sprintf(wb, "\n \"name\":\"%s\",", name); \
+ buffer_sprintf(wb, "\n \"visible\":%s,", (visible)?"true":"false"); \
+ buffer_sprintf(wb, "\n \"type\":\"%s\",", type); \
+ if(units) \
+ buffer_sprintf(wb, "\n \"units\":\"%s\",", (char*)(units)); \
+ if(!isnan((NETDATA_DOUBLE)(max))) \
+ buffer_sprintf(wb, "\n \"max\":%f,", (NETDATA_DOUBLE)(max)); \
+ if(pointer_to) \
+ buffer_sprintf(wb, "\n \"pointer_to\":\"%s\",", (char *)(pointer_to)); \
+ buffer_sprintf(wb, "\n \"sort\":\"%s\",", sort); \
+ buffer_sprintf(wb, "\n \"sortable\":%s,", (sortable)?"true":"false"); \
+ buffer_sprintf(wb, "\n \"sticky\":%s,", (sticky)?"true":"false"); \
+ buffer_sprintf(wb, "\n \"summary\":\"%s\"", summary); \
+ buffer_sprintf(wb, "\n }"); \
+ fields_added++; \
+} while(0)
+
+#define add_value_field_llu_with_max(wb, key, value) do { \
+ unsigned long long _tmp = (value); \
+ key ## _max = (rows == 0) ? (_tmp) : MAX(key ## _max, _tmp); \
+ buffer_fast_strcat(wb, ",", 1); \
+ buffer_print_llu(wb, _tmp); \
+} while(0)
+
+#define add_value_field_ndd_with_max(wb, key, value) do { \
+ NETDATA_DOUBLE _tmp = (value); \
+ key ## _max = (rows == 0) ? (_tmp) : MAX(key ## _max, _tmp); \
+ buffer_fast_strcat(wb, ",", 1); \
+ buffer_rrd_value(wb, _tmp); \
+} while(0)
+
+static void apps_plugin_function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
+ struct pid_stat *p;
+
+ char *words[PLUGINSD_MAX_WORDS] = { NULL };
+ size_t num_words = pluginsd_split_words(function, words, PLUGINSD_MAX_WORDS, NULL, NULL, 0);
+
+ struct target *category = NULL, *user = NULL, *group = NULL;
+ const char *process_name = NULL;
+ pid_t pid = 0;
+ uid_t uid = 0;
+ gid_t gid = 0;
+
+ bool filter_pid = false, filter_uid = false, filter_gid = false;
+
+ for(int i = 1; i < PLUGINSD_MAX_WORDS ;i++) {
+ const char *keyword = get_word(words, num_words, i);
+ if(!keyword) break;
+
+ if(!category && strncmp(keyword, PROCESS_FILTER_CATEGORY, strlen(PROCESS_FILTER_CATEGORY)) == 0) {
+ category = find_target_by_name(apps_groups_root_target, &keyword[strlen(PROCESS_FILTER_CATEGORY)]);
+ if(!category) {
+ apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");
+ return;
+ }
+ }
+ else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) {
+ user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]);
+ if(!user) {
+ apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");
+ return;
+ }
+ }
+ else if(strncmp(keyword, PROCESS_FILTER_GROUP, strlen(PROCESS_FILTER_GROUP)) == 0) {
+ group = find_target_by_name(groups_root_target, &keyword[strlen(PROCESS_FILTER_GROUP)]);
+ if(!group) {
+ apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");
+ return;
+ }
+ }
+ else if(!process_name && strncmp(keyword, PROCESS_FILTER_PROCESS, strlen(PROCESS_FILTER_PROCESS)) == 0) {
+ process_name = &keyword[strlen(PROCESS_FILTER_PROCESS)];
+ }
+ else if(!pid && strncmp(keyword, PROCESS_FILTER_PID, strlen(PROCESS_FILTER_PID)) == 0) {
+ pid = str2i(&keyword[strlen(PROCESS_FILTER_PID)]);
+ filter_pid = true;
+ }
+ else if(!uid && strncmp(keyword, PROCESS_FILTER_UID, strlen(PROCESS_FILTER_UID)) == 0) {
+ uid = str2i(&keyword[strlen(PROCESS_FILTER_UID)]);
+ filter_uid = true;
+ }
+ else if(!gid && strncmp(keyword, PROCESS_FILTER_GID, strlen(PROCESS_FILTER_GID)) == 0) {
+ gid = str2i(&keyword[strlen(PROCESS_FILTER_GID)]);
+ filter_gid = true;
+ }
+ else if(strcmp(keyword, "help") == 0) {
+ apps_plugin_function_processes_help(transaction);
+ return;
+ }
+ else {
+ char msg[PLUGINSD_LINE_MAX];
+ snprintfz(msg, PLUGINSD_LINE_MAX, "Invalid parameter '%s'", keyword);
+ apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, msg);
+ return;
+ }
+ }
+
+ time_t expires = now_realtime_sec() + update_every;
+ pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires);
+
+ unsigned int cpu_divisor = time_factor * RATES_DETAIL / 100;
+ unsigned int memory_divisor = 1024;
+ unsigned int io_divisor = 1024 * RATES_DETAIL;
+
+ BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX);
+ buffer_sprintf(wb,
+ "{"
+ "\n \"status\":%d"
+ ",\n \"type\":\"table\""
+ ",\n \"update_every\":%d"
+ ",\n \"data\":["
+ "\n"
+ , HTTP_RESP_OK
+ , update_every
+ );
+
+ NETDATA_DOUBLE
+ UserCPU_max = 0.0
+ , SysCPU_max = 0.0
+ , GuestCPU_max = 0.0
+ , CUserCPU_max = 0.0
+ , CSysCPU_max = 0.0
+ , CGuestCPU_max = 0.0
+ , CPU_max = 0.0
+ , VMSize_max = 0.0
+ , RSS_max = 0.0
+ , Shared_max = 0.0
+ , Swap_max = 0.0
+ , MemPcnt_max = 0.0
+ ;
+
+ unsigned long long
+ Processes_max = 0
+ , Threads_max = 0
+ , Uptime_max = 0
+ , MinFlt_max = 0
+ , CMinFlt_max = 0
+ , TMinFlt_max = 0
+ , MajFlt_max = 0
+ , CMajFlt_max = 0
+ , TMajFlt_max = 0
+ , PReads_max = 0
+ , PWrites_max = 0
+ , RCalls_max = 0
+ , WCalls_max = 0
+ , Files_max = 0
+ , Pipes_max = 0
+ , Sockets_max = 0
+ , iNotiFDs_max = 0
+ , EventFDs_max = 0
+ , TimerFDs_max = 0
+ , SigFDs_max = 0
+ , EvPollFDs_max = 0
+ , OtherFDs_max = 0
+ , FDs_max = 0
+ ;
+
+#ifndef __FreeBSD__
+ unsigned long long
+ LReads_max = 0
+ , LWrites_max = 0
+ ;
+#endif
+
+ int rows= 0;
+ for(p = root_of_pids; p ; p = p->next) {
+ if(!p->updated)
+ continue;
+
+ if(category && p->target != category)
+ continue;
+
+ if(user && p->user_target != user)
+ continue;
+
+ if(group && p->group_target != group)
+ continue;
+
+ if(process_name && ((strcmp(p->comm, process_name) != 0 && !p->parent) || (p->parent && strcmp(p->comm, process_name) != 0 && strcmp(p->parent->comm, process_name) != 0)))
+ continue;
+
+ if(filter_pid && p->pid != pid && p->ppid != pid)
+ continue;
+
+ if(filter_uid && p->uid != uid)
+ continue;
+
+ if(filter_gid && p->gid != gid)
+ continue;
+
+ if(rows) buffer_fast_strcat(wb, ",\n", 2);
+ rows++;
+
+ buffer_strcat(wb, " [");
+
+ // IMPORTANT!
+ // THE ORDER SHOULD BE THE SAME WITH THE FIELDS!
+
+ // pid
+ buffer_print_llu(wb, p->pid);
+
+ // cmd
+ buffer_fast_strcat(wb, ",\"", 2);
+ buffer_strcat_jsonescape(wb, p->comm);
+ buffer_fast_strcat(wb, "\"", 1);
+
+#ifdef NETDATA_DEV_MODE
+ // cmdline
+ buffer_fast_strcat(wb, ",\"", 2);
+ buffer_strcat_jsonescape(wb, (p->cmdline && *p->cmdline) ? p->cmdline : p->comm);
+ buffer_fast_strcat(wb, "\"", 1);
+#endif
+
+ // ppid
+ buffer_fast_strcat(wb, ",", 1); buffer_print_llu(wb, p->ppid);
+
+ // category
+ buffer_fast_strcat(wb, ",\"", 2);
+ buffer_strcat_jsonescape(wb, p->target ? p->target->name : "-");
+ buffer_fast_strcat(wb, "\"", 1);
+
+ // user
+ buffer_fast_strcat(wb, ",\"", 2);
+ buffer_strcat_jsonescape(wb, p->user_target ? p->user_target->name : "-");
+ buffer_fast_strcat(wb, "\"", 1);
+
+ // uid
+ buffer_fast_strcat(wb, ",", 1); buffer_print_llu(wb, p->uid);
+
+ // group
+ buffer_fast_strcat(wb, ",\"", 2);
+ buffer_strcat_jsonescape(wb, p->group_target ? p->group_target->name : "-");
+ buffer_fast_strcat(wb, "\"", 1);
+
+ // gid
+ buffer_fast_strcat(wb, ",", 1); buffer_print_llu(wb, p->gid);
+
+ // procs
+ add_value_field_llu_with_max(wb, Processes, p->children_count);
+
+ // threads
+ add_value_field_llu_with_max(wb, Threads, p->num_threads);
+
+ // uptime
+ add_value_field_llu_with_max(wb, Uptime, p->uptime);
+
+ // minor page faults
+ add_value_field_llu_with_max(wb, MinFlt, p->minflt / RATES_DETAIL);
+ add_value_field_llu_with_max(wb, CMinFlt, p->cminflt / RATES_DETAIL);
+ add_value_field_llu_with_max(wb, TMinFlt, (p->minflt + p->cminflt) / RATES_DETAIL);
+
+ // major page faults
+ add_value_field_llu_with_max(wb, MajFlt, p->majflt / RATES_DETAIL);
+ add_value_field_llu_with_max(wb, CMajFlt, p->cmajflt / RATES_DETAIL);
+ add_value_field_llu_with_max(wb, TMajFlt, (p->majflt + p->cmajflt) / RATES_DETAIL);
+
+ // CPU utilization %
+ add_value_field_ndd_with_max(wb, UserCPU, (NETDATA_DOUBLE)(p->utime) / cpu_divisor);
+ add_value_field_ndd_with_max(wb, SysCPU, (NETDATA_DOUBLE)(p->stime) / cpu_divisor);
+ add_value_field_ndd_with_max(wb, GuestCPU, (NETDATA_DOUBLE)(p->gtime) / cpu_divisor);
+ add_value_field_ndd_with_max(wb, CUserCPU, (NETDATA_DOUBLE)(p->cutime) / cpu_divisor);
+ add_value_field_ndd_with_max(wb, CSysCPU, (NETDATA_DOUBLE)(p->cstime) / cpu_divisor);
+ add_value_field_ndd_with_max(wb, CGuestCPU, (NETDATA_DOUBLE)(p->cgtime) / cpu_divisor);
+ add_value_field_ndd_with_max(wb, CPU, (NETDATA_DOUBLE)(p->utime + p->stime + p->gtime + p->cutime + p->cstime + p->cgtime) / cpu_divisor);
+
+ // memory MiB
+ add_value_field_ndd_with_max(wb, VMSize, (NETDATA_DOUBLE)p->status_vmsize / memory_divisor);
+ add_value_field_ndd_with_max(wb, RSS, (NETDATA_DOUBLE)p->status_vmrss / memory_divisor);
+ add_value_field_ndd_with_max(wb, Shared, (NETDATA_DOUBLE)p->status_vmshared / memory_divisor);
+ add_value_field_ndd_with_max(wb, Swap, (NETDATA_DOUBLE)p->status_vmswap / memory_divisor);
+
+ if(MemTotal)
+ add_value_field_ndd_with_max(wb, MemPcnt, (NETDATA_DOUBLE)p->status_vmrss * 100.0 / (NETDATA_DOUBLE)MemTotal);
+
+ // Logical I/O
+#ifndef __FreeBSD__
+ add_value_field_llu_with_max(wb, LReads, p->io_logical_bytes_read / io_divisor);
+ add_value_field_llu_with_max(wb, LWrites, p->io_logical_bytes_written / io_divisor);
+#endif
+
+ // Physical I/O
+ add_value_field_llu_with_max(wb, PReads, p->io_storage_bytes_read / io_divisor);
+ add_value_field_llu_with_max(wb, PWrites, p->io_storage_bytes_written / io_divisor);
+
+ // I/O calls
+ add_value_field_llu_with_max(wb, RCalls, p->io_read_calls / RATES_DETAIL);
+ add_value_field_llu_with_max(wb, WCalls, p->io_write_calls / RATES_DETAIL);
+
+ // open file descriptors
+ add_value_field_llu_with_max(wb, Files, p->openfds.files);
+ add_value_field_llu_with_max(wb, Pipes, p->openfds.pipes);
+ add_value_field_llu_with_max(wb, Sockets, p->openfds.sockets);
+ add_value_field_llu_with_max(wb, iNotiFDs, p->openfds.inotifies);
+ add_value_field_llu_with_max(wb, EventFDs, p->openfds.eventfds);
+ add_value_field_llu_with_max(wb, TimerFDs, p->openfds.timerfds);
+ add_value_field_llu_with_max(wb, SigFDs, p->openfds.signalfds);
+ add_value_field_llu_with_max(wb, EvPollFDs, p->openfds.eventpolls);
+ add_value_field_llu_with_max(wb, OtherFDs, p->openfds.other);
+ add_value_field_llu_with_max(wb, FDs, p->openfds.files + p->openfds.pipes + p->openfds.sockets + p->openfds.inotifies + p->openfds.eventfds + p->openfds.timerfds + p->openfds.signalfds + p->openfds.eventpolls + p->openfds.other);
+
+ buffer_fast_strcat(wb, "]", 1);
+
+ fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
+ buffer_flush(wb);
+ }
+
+ {
+ int fields_added = 0;
+
+ buffer_flush(wb);
+ buffer_sprintf(wb, "\n ],\n \"columns\": {");
+
+ // IMPORTANT!
+ // THE ORDER SHOULD BE THE SAME WITH THE VALUES!
+ add_table_field(wb, "Pid", "Process ID", true, "integer", NULL, NAN, "ascending", true, true, true, NULL, "count_unique");
+ add_table_field(wb, "Cmd", "Process Name", true, "string", NULL, NAN, "ascending", true, true, false, NULL, "count_unique");
+
+#ifdef NETDATA_DEV_MODE
+ add_table_field(wb, "CmdLine", "Command Line", false, "detail-string:Cmd", NULL, NAN, "ascending", true, false, false, NULL, "count_unique");
+#endif
+ add_table_field(wb, "PPid", "Parent Process ID", false, "integer", NULL, NAN, "ascending", true, false, false, "Pid", "count_unique");
+ add_table_field(wb, "Category", "Category (apps_groups.conf)", true, "string", NULL, NAN, "ascending", true, true, false, NULL, "count_unique");
+ add_table_field(wb, "User", "User Owner", true, "string", NULL, NAN, "ascending", true, false, false, NULL, "count_unique");
+ add_table_field(wb, "Uid", "User ID", false, "integer", NULL, NAN, "ascending", true, false, false, NULL, "count_unique");
+ add_table_field(wb, "Group", "Group Owner", false, "string", NULL, NAN, "ascending", true, false, false, NULL, "count_unique");
+ add_table_field(wb, "Gid", "Group ID", false, "integer", NULL, NAN, "ascending", true, false, false, NULL, "count_unique");
+ add_table_field(wb, "Processes", "Processes", true, "bar-with-integer", "processes", Processes_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "Threads", "Threads", true, "bar-with-integer", "threads", Threads_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "Uptime", "Uptime in seconds", true, "duration", "seconds", Uptime_max, "descending", true, false, false, NULL, "max");
+
+ // minor page faults
+ add_table_field(wb, "MinFlt", "Minor Page Faults/s", false, "bar", "pgflts/s", MinFlt_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "CMinFlt", "Children Minor Page Faults/s", false, "bar", "pgflts/s", CMinFlt_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "TMinFlt", "Total Minor Page Faults/s", false, "bar", "pgflts/s", TMinFlt_max, "descending", true, false, false, NULL, "sum");
+
+ // major page faults
+ add_table_field(wb, "MajFlt", "Major Page Faults/s", false, "bar", "pgflts/s", MajFlt_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "CMajFlt", "Children Major Page Faults/s", false, "bar", "pgflts/s", CMajFlt_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "TMajFlt", "Total Major Page Faults/s", true, "bar", "pgflts/s", TMajFlt_max, "descending", true, false, false, NULL, "sum");
+
+ // CPU utilization
+ add_table_field(wb, "UserCPU", "User CPU time", false, "bar", "%", UserCPU_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "SysCPU", "System CPU Time", false, "bar", "%", SysCPU_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "GuestCPU", "Guest CPU Time", false, "bar", "%", GuestCPU_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "CUserCPU", "Children User CPU Time", false, "bar", "%", CUserCPU_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "CSysCPU", "Children System CPU Time", false, "bar", "%", CSysCPU_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "CGuestCPU", "Children Guest CPU Time", false, "bar", "%", CGuestCPU_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "CPU", "Total CPU Time", true, "bar", "%", CPU_max, "descending", true, false, false, NULL, "sum");
+
+ // memory
+ add_table_field(wb, "VMSize", "Virtual Memory Size", false, "bar", "MiB", VMSize_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "RSS", "Resident Set Size", MemTotal ? false : true, "bar", "MiB", RSS_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "Shared", "Shared Pages", false, "bar", "MiB", Shared_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "Swap", "Swap Memory", false, "bar", "MiB", Swap_max, "descending", true, false, false, NULL, "sum");
+
+ if(MemTotal)
+ add_table_field(wb, "MemPcnt", "Memory Percentage", true, "bar", "%", 100.0, "descending", true, false, false, NULL, "sum");
+
+ // Logical I/O
+#ifndef __FreeBSD__
+ add_table_field(wb, "LReads", "Logical I/O Reads", false, "bar", "KiB/s", LReads_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "LWrites", "Logical I/O Writes", false, "bar", "KiB/s", LWrites_max, "descending", true, false, false, NULL, "sum");
+#endif
+
+ // Physical I/O
+ add_table_field(wb, "PReads", "Physical I/O Reads", true, "bar", "KiB/s", PReads_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "PWrites", "Physical I/O Writes", true, "bar", "KiB/s", PWrites_max, "descending", true, false, false, NULL, "sum");
+
+ // I/O calls
+ add_table_field(wb, "RCalls", "I/O Read Calls", false, "bar", "calls/s", RCalls_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "WCalls", "I/O Write Calls", false, "bar", "calls/s", WCalls_max, "descending", true, false, false, NULL, "sum");
+
+ // open file descriptors
+ add_table_field(wb, "Files", "Open Files", false, "bar", "fds", Files_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "Pipes", "Open Pipes", false, "bar", "fds", Pipes_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "Sockets", "Open Sockets", false, "bar", "fds", Sockets_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "iNotiFDs", "Open iNotify Descriptors", false, "bar", "fds", iNotiFDs_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "EventFDs", "Open Event Descriptors", false, "bar", "fds", EventFDs_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "TimerFDs", "Open Timer Descriptors", false, "bar", "fds", TimerFDs_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "SigFDs", "Open Signal Descriptors", false, "bar", "fds", SigFDs_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "EvPollFDs", "Open Event Poll Descriptors", false, "bar", "fds", EvPollFDs_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "OtherFDs", "Other Open Descriptors", false, "bar", "fds", OtherFDs_max, "descending", true, false, false, NULL, "sum");
+ add_table_field(wb, "FDs", "All Open File Descriptors", true, "bar", "fds", FDs_max, "descending", true, false, false, NULL, "sum");
+
+ buffer_strcat(
+ wb,
+ ""
+ "\n },"
+ "\n \"default_sort_column\": \"CPU\","
+ "\n \"charts\": {"
+ "\n \"CPU\": {"
+ "\n \"name\":\"CPU Utilization\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"UserCPU\", \"SysCPU\", \"GuestCPU\", \"CUserCPU\", \"CSysCPU\", \"CGuestCPU\" ]"
+ "\n },"
+ "\n \"Memory\": {"
+ "\n \"name\":\"Memory\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"VMSize\", \"RSS\", \"Shared\", \"Swap\" ]"
+ "\n },"
+ );
+
+ if(MemTotal)
+ buffer_strcat(
+ wb,
+ ""
+ "\n \"MemoryPercent\": {"
+ "\n \"name\":\"Memory Percentage\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"MemPcnt\" ]"
+ "\n },"
+ );
+
+ buffer_strcat(
+ wb, ""
+ #ifndef __FreeBSD__
+ "\n \"Reads\": {"
+ "\n \"name\":\"I/O Reads\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"LReads\", \"PReads\" ]"
+ "\n },"
+ "\n \"Writes\": {"
+ "\n \"name\":\"I/O Writes\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"LWrites\", \"PWrites\" ]"
+ "\n },"
+ "\n \"LogicalIO\": {"
+ "\n \"name\":\"Logical I/O\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"LReads\", \"LWrites\" ]"
+ "\n },"
+ #endif
+ "\n \"PhysicalIO\": {"
+ "\n \"name\":\"Physical I/O\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"PReads\", \"PWrites\" ]"
+ "\n },"
+ "\n \"IOCalls\": {"
+ "\n \"name\":\"I/O Calls\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"RCalls\", \"WCalls\" ]"
+ "\n },"
+ "\n \"MinFlt\": {"
+ "\n \"name\":\"Minor Page Faults\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"MinFlt\", \"CMinFlt\" ]"
+ "\n },"
+ "\n \"MajFlt\": {"
+ "\n \"name\":\"Major Page Faults\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"MajFlt\", \"CMajFlt\" ]"
+ "\n },"
+ "\n \"Threads\": {"
+ "\n \"name\":\"Threads\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"Threads\" ]"
+ "\n },"
+ "\n \"Processes\": {"
+ "\n \"name\":\"Processes\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"Processes\" ]"
+ "\n },"
+ "\n \"FDs\": {"
+ "\n \"name\":\"File Descriptors\","
+ "\n \"type\":\"stacked-bar\","
+ "\n \"columns\": [ \"Files\", \"Pipes\", \"Sockets\", \"iNotiFDs\", \"EventFDs\", \"TimerFDs\", \"SigFDs\", \"EvPollFDs\", \"OtherFDs\" ]"
+ "\n }"
+ "\n },"
+ "\n \"group_by\": {"
+ "\n \"pid\": {"
+ "\n \"name\":\"Process Tree by PID\","
+ "\n \"columns\":[ \"PPid\" ]"
+ "\n },"
+ "\n \"category\": {"
+ "\n \"name\":\"Process Tree by Category\","
+ "\n \"columns\":[ \"Category\", \"PPid\" ]"
+ "\n },"
+ "\n \"user\": {"
+ "\n \"name\":\"Process Tree by User\","
+ "\n \"columns\":[ \"User\", \"PPid\" ]"
+ "\n },"
+ "\n \"group\": {"
+ "\n \"name\":\"Process Tree by Group\","
+ "\n \"columns\":[ \"Group\", \"PPid\" ]"
+ "\n }"
+ "\n }"
+ );
+
+ fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
+ }
+
+ buffer_free(wb);
+
+ fprintf(stdout, ",\n \"expires\":%lld", (long long)expires);
+ fprintf(stdout, "\n}");
+
+ pluginsd_function_result_end_to_stdout();
+}
+
+bool apps_plugin_exit = false;
+
+void *reader_main(void *arg __maybe_unused) {
+ char buffer[PLUGINSD_LINE_MAX + 1];
+
+ char *s = NULL;
+ while(!apps_plugin_exit && (s = fgets(buffer, PLUGINSD_LINE_MAX, stdin))) {
+
+ char *words[PLUGINSD_MAX_WORDS] = { NULL };
+ size_t num_words = pluginsd_split_words(buffer, words, PLUGINSD_MAX_WORDS, NULL, NULL, 0);
+
+ const char *keyword = get_word(words, num_words, 0);
+
+ if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0) {
+ char *transaction = get_word(words, num_words, 1);
+ char *timeout_s = get_word(words, num_words, 2);
+ char *function = get_word(words, num_words, 3);
+
+ if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
+ error("Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
+ keyword,
+ transaction?transaction:"(unset)",
+ timeout_s?timeout_s:"(unset)",
+ function?function:"(unset)");
+ }
+ else {
+ int timeout = str2i(timeout_s);
+ if(timeout <= 0) timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
+
+// internal_error(true, "Received function '%s', transaction '%s', timeout %d", function, transaction, timeout);
+
+ netdata_mutex_lock(&mutex);
+
+ if(strncmp(function, "processes", strlen("processes")) == 0)
+ apps_plugin_function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
+ else
+ apps_plugin_function_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");
+
+ fflush(stdout);
+ netdata_mutex_unlock(&mutex);
+
+// internal_error(true, "Done with function '%s', transaction '%s', timeout %d", function, transaction, timeout);
+ }
+ }
+ else
+ error("Received unknown command: %s", keyword?keyword:"(unset)");
+ }
+
+ if(!s || feof(stdin) || ferror(stdin)) {
+ apps_plugin_exit = true;
+ error("Received error on stdin.");
+ }
+
+ exit(1);
+ return NULL;
+}
+
+int main(int argc, char **argv) {
+ // debug_flags = D_PROCFILE;
+
+ clocks_init();
+
+ pagesize = (size_t)sysconf(_SC_PAGESIZE);
+
+ // set the name for logging
+ program_name = "apps.plugin";
+
+ // disable syslog for apps.plugin
+ error_log_syslog = 0;
+
+ // set errors flood protection to 100 logs per hour
+ error_log_errors_per_period = 100;
+ error_log_throttle_period = 3600;
+
+ bool send_resource_usage = true;
+ {
+ const char *s = getenv("NETDATA_INTERNALS_MONITORING");
+ if(s && *s && strcmp(s, "NO") == 0)
+ send_resource_usage = false;
+ }
+
+ // since apps.plugin runs as root, prevent it from opening symbolic links
+ procfile_open_flags = O_RDONLY|O_NOFOLLOW;
+
+ netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
+ if(verify_netdata_host_prefix() == -1) exit(1);
+
+ user_config_dir = getenv("NETDATA_USER_CONFIG_DIR");
+ if(user_config_dir == NULL) {
+ // info("NETDATA_CONFIG_DIR is not passed from netdata");
+ user_config_dir = CONFIG_DIR;
+ }
+ // else info("Found NETDATA_USER_CONFIG_DIR='%s'", user_config_dir);
+
+ stock_config_dir = getenv("NETDATA_STOCK_CONFIG_DIR");
+ if(stock_config_dir == NULL) {
+ // info("NETDATA_CONFIG_DIR is not passed from netdata");
+ stock_config_dir = LIBCONFIG_DIR;
+ }
+ // else info("Found NETDATA_USER_CONFIG_DIR='%s'", user_config_dir);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(debug_flags != 0) {
+ struct rlimit rl = { RLIM_INFINITY, RLIM_INFINITY };
+ if(setrlimit(RLIMIT_CORE, &rl) != 0)
+ info("Cannot request unlimited core dumps for debugging... Proceeding anyway...");
+#ifdef HAVE_SYS_PRCTL_H
+ prctl(PR_SET_DUMPABLE, 1, 0, 0, 0);
+#endif
+ }
+#endif /* NETDATA_INTERNAL_CHECKS */
+
+ procfile_adaptive_initial_allocation = 1;
+
+ get_system_HZ();
+#ifdef __FreeBSD__
+ time_factor = 1000000ULL / RATES_DETAIL; // FreeBSD uses usecs
+#else
+ time_factor = system_hz; // Linux uses clock ticks
+#endif
+
+ get_system_pid_max();
+ get_system_cpus();
+
+ parse_args(argc, argv);
+
+ if(!check_capabilities() && !am_i_running_as_root() && !check_proc_1_io()) {
+ uid_t uid = getuid(), euid = geteuid();
+#ifdef HAVE_CAPABILITY
+ error("apps.plugin should either run as root (now running with uid %u, euid %u) or have special capabilities. "
+ "Without these, apps.plugin cannot report disk I/O utilization of other processes. "
+ "To enable capabilities run: sudo setcap cap_dac_read_search,cap_sys_ptrace+ep %s; "
+ "To enable setuid to root run: sudo chown root:netdata %s; sudo chmod 4750 %s; "
+ , uid, euid, argv[0], argv[0], argv[0]
+ );
+#else
+ error("apps.plugin should either run as root (now running with uid %u, euid %u) or have special capabilities. "
+ "Without these, apps.plugin cannot report disk I/O utilization of other processes. "
+ "Your system does not support capabilities. "
+ "To enable setuid to root run: sudo chown root:netdata %s; sudo chmod 4750 %s; "
+ , uid, euid, argv[0], argv[0]
+ );
+#endif
+ }
+
+ info("started on pid %d", getpid());
+
+ snprintfz(all_user_ids.filename, FILENAME_MAX, "%s/etc/passwd", netdata_configured_host_prefix);
+ debug_log("passwd file: '%s'", all_user_ids.filename);
+
+ snprintfz(all_group_ids.filename, FILENAME_MAX, "%s/etc/group", netdata_configured_host_prefix);
+ debug_log("group file: '%s'", all_group_ids.filename);
+
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
+ all_pids_sortlist = callocz(sizeof(pid_t), (size_t)pid_max + 1);
+#endif
+
+ all_pids = callocz(sizeof(struct pid_stat *), (size_t) pid_max + 1);
+
+ netdata_thread_t reader_thread;
+ netdata_thread_create(&reader_thread, "APPS_READER", NETDATA_THREAD_OPTION_DONT_LOG, reader_main, NULL);
+ netdata_mutex_lock(&mutex);
+
+ usec_t step = update_every * USEC_PER_SEC;
+ global_iterations_counter = 1;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ for(; !apps_plugin_exit ; global_iterations_counter++) {
+ netdata_mutex_unlock(&mutex);
+
+#ifdef NETDATA_PROFILING
+#warning "compiling for profiling"
+ static int profiling_count=0;
+ profiling_count++;
+ if(unlikely(profiling_count > 2000)) exit(0);
+ usec_t dt = update_every * USEC_PER_SEC;
+#else
+ usec_t dt = heartbeat_next(&hb, step);
+#endif
+ netdata_mutex_lock(&mutex);
+
+ struct pollfd pollfd = { .fd = fileno(stdout), .events = POLLERR };
+ if (unlikely(poll(&pollfd, 1, 0) < 0)) {
+ netdata_mutex_unlock(&mutex);
+ netdata_thread_cancel(reader_thread);
+ fatal("Cannot check if a pipe is available");
+ }
+ if (unlikely(pollfd.revents & POLLERR)) {
+ netdata_mutex_unlock(&mutex);
+ netdata_thread_cancel(reader_thread);
+ fatal("Received error on read pipe.");
+ }
+
+ if(global_iterations_counter % 10 == 0)
+ get_MemTotal();
+
+ if(!collect_data_for_all_processes()) {
+ error("Cannot collect /proc data for running processes. Disabling apps.plugin...");
+ printf("DISABLE\n");
+ netdata_mutex_unlock(&mutex);
+ netdata_thread_cancel(reader_thread);
+ exit(1);
+ }
+
+ currentmaxfds = 0;
+ calculate_netdata_statistics();
+ normalize_utilization(apps_groups_root_target);
+
+ if(send_resource_usage)
+ send_resource_usage_to_netdata(dt);
+
+#ifndef __FreeBSD__
+ send_proc_states_count(dt);
+#endif
+
+ // this is smart enough to show only newly added apps, when needed
+ send_charts_updates_to_netdata(apps_groups_root_target, "apps", "Apps");
+ if(likely(enable_users_charts))
+ send_charts_updates_to_netdata(users_root_target, "users", "Users");
+
+ if(likely(enable_groups_charts))
+ send_charts_updates_to_netdata(groups_root_target, "groups", "User Groups");
+
+ send_collected_data_to_netdata(apps_groups_root_target, "apps", dt);
+
+ if(likely(enable_users_charts))
+ send_collected_data_to_netdata(users_root_target, "users", dt);
+
+ if(likely(enable_groups_charts))
+ send_collected_data_to_netdata(groups_root_target, "groups", dt);
+
+ fflush(stdout);
+
+ show_guest_time_old = show_guest_time;
+
+ debug_log("done Loop No %zu", global_iterations_counter);
+ }
+}
diff --git a/collectors/cgroups.plugin/Makefile.am b/collectors/cgroups.plugin/Makefile.am
new file mode 100644
index 0000000..354b9fb
--- /dev/null
+++ b/collectors/cgroups.plugin/Makefile.am
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_plugins_SCRIPTS = \
+ cgroup-name.sh \
+ cgroup-network-helper.sh \
+ $(NULL)
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/cgroups.plugin/README.md b/collectors/cgroups.plugin/README.md
new file mode 100644
index 0000000..d0f822e
--- /dev/null
+++ b/collectors/cgroups.plugin/README.md
@@ -0,0 +1,308 @@
+<!--
+title: "cgroups.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/cgroups.plugin/README.md
+-->
+
+# cgroups.plugin
+
+You can monitor containers and virtual machines using **cgroups**.
+
+cgroups (or control groups), are a Linux kernel feature that provides accounting and resource usage limiting for
+processes. When cgroups are bundled with namespaces (i.e. isolation), they form what we usually call **containers**.
+
+cgroups are hierarchical, meaning that cgroups can contain child cgroups, which can contain more cgroups, etc. All
+accounting is reported (and resource usage limits are applied) also in a hierarchical way.
+
+To visualize cgroup metrics Netdata provides configuration for cherry picking the cgroups of interest. By default (
+without any configuration) Netdata should pick **systemd services**, all kinds of **containers** (lxc, docker, etc)
+and **virtual machines** spawn by managers that register them with cgroups (qemu, libvirt, etc).
+
+## Configuring Netdata for cgroups
+
+In general, no additional settings are required. Netdata discovers all available cgroups on the host system and
+collects their metrics.
+
+### how Netdata finds the available cgroups
+
+Linux exposes resource usage reporting and provides dynamic configuration for cgroups, using virtual files (usually)
+under `/sys/fs/cgroup`. Netdata reads `/proc/self/mountinfo` to detect the exact mount point of cgroups. Netdata also
+allows manual configuration of this mount point, using these settings:
+
+```text
+[plugin:cgroups]
+ check for new cgroups every = 10
+ path to /sys/fs/cgroup/cpuacct = /sys/fs/cgroup/cpuacct
+ path to /sys/fs/cgroup/blkio = /sys/fs/cgroup/blkio
+ path to /sys/fs/cgroup/memory = /sys/fs/cgroup/memory
+ path to /sys/fs/cgroup/devices = /sys/fs/cgroup/devices
+```
+
+Netdata rescans these directories for added or removed cgroups every `check for new cgroups every` seconds.
+
+### hierarchical search for cgroups
+
+Since cgroups are hierarchical, for each of the directories shown above, Netdata walks through the subdirectories
+recursively searching for cgroups (each subdirectory is another cgroup).
+
+To provide a sane default for this setting, Netdata uses the following pattern list (patterns starting with `!` give a
+negative match and their order is important: the first matching a path will be used):
+
+```text
+[plugin:cgroups]
+ search for cgroups in subpaths matching = !*/init.scope !*-qemu !/init.scope !/system !/systemd !/user !/user.slice *
+```
+
+So, we disable checking for **child cgroups** in systemd internal
+cgroups ([systemd services are monitored by Netdata](#monitoring-systemd-services)), user cgroups (normally used for
+desktop and remote user sessions), qemu virtual machines (child cgroups of virtual machines) and `init.scope`. All
+others are enabled.
+
+### unified cgroups (cgroups v2) support
+
+Netdata automatically detects cgroups version. If detection fails Netdata assumes v1.
+To switch to v2 manually add:
+
+```text
+[plugin:cgroups]
+ use unified cgroups = yes
+ path to unified cgroups = /sys/fs/cgroup
+```
+
+Unified cgroups use same name pattern matching as v1 cgroups. `cgroup_enable_systemd_services_detailed_memory` is
+currently unsupported when using unified cgroups.
+
+### enabled cgroups
+
+To provide a sane default, Netdata uses the
+following [pattern list](https://learn.netdata.cloud/docs/agent/libnetdata/simple_pattern):
+
+- checks the pattern against the path of the cgroup
+
+ ```text
+ [plugin:cgroups]
+ enable by default cgroups matching = !*/init.scope *.scope !*/vcpu* !*/emulator !*.mount !*.partition !*.service !*.slice !*.swap !*.user !/ !/docker !/libvirt !/lxc !/lxc/*/ns !/lxc/*/ns/* !/machine !/qemu !/system !/systemd !/user *
+ ```
+
+- checks the pattern against the name of the cgroup (as you see it on the dashboard)
+
+ ```text
+ [plugin:cgroups]
+ enable by default cgroups names matching = *
+ ```
+
+Renaming is configured with the following options:
+
+```text
+[plugin:cgroups]
+ run script to rename cgroups matching = *.scope *docker* *lxc* *qemu* !/ !*.mount !*.partition !*.service !*.slice !*.swap !*.user *
+ script to get cgroup names = /usr/libexec/netdata/plugins.d/cgroup-name.sh
+```
+
+The whole point for the additional pattern list, is to limit the number of times the script will be called. Without this
+pattern list, the script might be called thousands of times, depending on the number of cgroups available in the system.
+
+The above pattern list is matched against the path of the cgroup. For matched cgroups, Netdata calls the
+script [cgroup-name.sh](https://raw.githubusercontent.com/netdata/netdata/master/collectors/cgroups.plugin/cgroup-name.sh)
+to get its name. This script queries `docker`, `kubectl`, `podman`, or applies heuristics to find give a name for the
+cgroup.
+
+#### Note on Podman container names
+
+Podman's security model is a lot more restrictive than Docker's, so Netdata will not be able to detect container names
+out of the box unless they were started by the same user as Netdata itself.
+
+If Podman is used in "rootful" mode, it's also possible to use `podman system service` to grant Netdata access to
+container names. To do this, ensure `podman system service` is running and Netdata has access
+to `/run/podman/podman.sock` (the default permissions as specified by upstream are `0600`, with owner `root`, so you
+will have to adjust the configuration).
+
+[docker-socket-proxy](https://github.com/Tecnativa/docker-socket-proxy) can also be used to give Netdata restricted
+access to the socket. Note that `PODMAN_HOST` in Netdata's environment should be set to the proxy's URL in this case.
+
+### charts with zero metrics
+
+By default, Netdata will enable monitoring metrics only when they are not zero. If they are constantly zero they are
+ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be
+automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). Set `yes` for a
+chart instead of `auto` to enable it permanently. For example:
+
+```text
+[plugin:cgroups]
+ enable memory (used mem including cache) = yes
+```
+
+You can also set the `enable zero metrics` option to `yes` in the `[global]` section which enables charts with zero
+metrics for all internal Netdata plugins.
+
+### alarms
+
+CPU and memory limits are watched and used to rise alarms. Memory usage for every cgroup is checked against `ram`
+and `ram+swap` limits. CPU usage for every cgroup is checked against `cpuset.cpus` and `cpu.cfs_period_us` + `cpu.cfs_quota_us` pair assigned for the cgroup. Configuration for the alarms is available in `health.d/cgroups.conf`
+file.
+
+## Monitoring systemd services
+
+Netdata monitors **systemd services**. Example:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/21964372/20cd7b84-db53-11e6-98a2-b9c986b082c0.png)
+
+Support per distribution:
+
+| system | charts shown | `/sys/fs/cgroup` tree | comments |
+|:----------------:|:------------:|:------------------------------------:|:--------------------------|
+| Arch Linux | YES | | |
+| Gentoo | NO | | can be enabled, see below |
+| Ubuntu 16.04 LTS | YES | | |
+| Ubuntu 16.10 | YES | [here](http://pastebin.com/PiWbQEXy) | |
+| Fedora 25 | YES | [here](http://pastebin.com/ax0373wF) | |
+| Debian 8 | NO | | can be enabled, see below |
+| AMI | NO | [here](http://pastebin.com/FrxmptjL) | not a systemd system |
+| CentOS 7.3.1611 | NO | [here](http://pastebin.com/SpzgezAg) | can be enabled, see below |
+
+### Monitored systemd service metrics
+
+- CPU utilization
+- Used memory
+- RSS memory
+- Mapped memory
+- Cache memory
+- Writeback memory
+- Memory minor page faults
+- Memory major page faults
+- Memory charging activity
+- Memory uncharging activity
+- Memory limit failures
+- Swap memory used
+- Disk read bandwidth
+- Disk write bandwidth
+- Disk read operations
+- Disk write operations
+- Throttle disk read bandwidth
+- Throttle disk write bandwidth
+- Throttle disk read operations
+- Throttle disk write operations
+- Queued disk read operations
+- Queued disk write operations
+- Merged disk read operations
+- Merged disk write operations
+
+### how to enable cgroup accounting on systemd systems that is by default disabled
+
+You can verify there is no accounting enabled, by running `systemd-cgtop`. The program will show only resources for
+cgroup `/`, but all services will show nothing.
+
+To enable cgroup accounting, execute this:
+
+```sh
+sed -e 's|^#Default\(.*\)Accounting=.*$|Default\1Accounting=yes|g' /etc/systemd/system.conf >/tmp/system.conf
+```
+
+To see the changes it made, run this:
+
+```sh
+# diff /etc/systemd/system.conf /tmp/system.conf
+40,44c40,44
+< #DefaultCPUAccounting=no
+< #DefaultIOAccounting=no
+< #DefaultBlockIOAccounting=no
+< #DefaultMemoryAccounting=no
+< #DefaultTasksAccounting=yes
+---
+> DefaultCPUAccounting=yes
+> DefaultIOAccounting=yes
+> DefaultBlockIOAccounting=yes
+> DefaultMemoryAccounting=yes
+> DefaultTasksAccounting=yes
+```
+
+If you are happy with the changes, run:
+
+```sh
+# copy the file to the right location
+sudo cp /tmp/system.conf /etc/systemd/system.conf
+
+# restart systemd to take it into account
+sudo systemctl daemon-reexec
+```
+
+(`systemctl daemon-reload` does not reload the configuration of the server - so you have to
+execute `systemctl daemon-reexec`).
+
+Now, when you run `systemd-cgtop`, services will start reporting usage (if it does not, restart any service to wake it up). Refresh your Netdata dashboard, and you will have the charts too.
+
+In case memory accounting is missing, you will need to enable it at your kernel, by appending the following kernel boot
+options and rebooting:
+
+```sh
+cgroup_enable=memory swapaccount=1
+```
+
+You can add the above, directly at the `linux` line in your `/boot/grub/grub.cfg` or appending them to
+the `GRUB_CMDLINE_LINUX` in `/etc/default/grub` (in which case you will have to run `update-grub` before rebooting). On
+DigitalOcean debian images you may have to set it at `/etc/default/grub.d/50-cloudimg-settings.cfg`.
+
+Which systemd services are monitored by Netdata is determined by the following pattern list:
+
+```text
+[plugin:cgroups]
+ cgroups to match as systemd services = !/system.slice/*/*.service /system.slice/*.service
+```
+
+- - -
+
+## Monitoring ephemeral containers
+
+Netdata monitors containers automatically when it is installed at the host, or when it is installed in a container that
+has access to the `/proc` and `/sys` filesystems of the host.
+
+Netdata prior to v1.6 had 2 issues when such containers were monitored:
+
+1. network interface alarms where triggering when containers were stopped
+
+2. charts were never cleaned up, so after some time dozens of containers were showing up on the dashboard, and they were
+ occupying memory.
+
+### the current Netdata
+
+network interfaces and cgroups (containers) are now self-cleaned.
+
+So, when a network interface or container stops, Netdata might log a few errors in error.log complaining about files it
+cannot find, but immediately:
+
+1. it will detect this is a removed container or network interface
+2. it will freeze/pause all alarms for them
+3. it will mark their charts as obsolete
+4. obsolete charts are not be offered on new dashboard sessions (so hit F5 and the charts are gone)
+5. existing dashboard sessions will continue to see them, but of course they will not refresh
+6. obsolete charts will be removed from memory, 1 hour after the last user viewed them (configurable
+ with `[global].cleanup obsolete charts after seconds = 3600` (at `netdata.conf`).
+7. when obsolete charts are removed from memory they are also deleted from disk (configurable
+ with `[global].delete obsolete charts files = yes`)
+
+### Monitored container metrics
+
+- CPU usage
+- CPU usage within the limits
+- CPU usage per core
+- Memory usage
+- Writeback memory
+- Memory activity
+- Memory page faults
+- Used memory
+- Used RAM within the limits
+- Memory utilization
+- Memory limit failures
+- I/O bandwidth (all disks)
+- Serviced I/O operations (all disks)
+- Throttle I/O bandwidth (all disks)
+- Throttle serviced I/O operations (all disks)
+- Queued I/O operations (all disks)
+- Merged I/O operations (all disks)
+- CPU pressure
+- Memory pressure
+- Memory full pressure
+- I/O pressure
+- I/O full pressure
+
+Network interfaces are monitored by means of
+the [proc plugin](/collectors/proc.plugin/README.md#monitored-network-interface-metrics).
diff --git a/collectors/cgroups.plugin/cgroup-name.sh b/collectors/cgroups.plugin/cgroup-name.sh
new file mode 100755
index 0000000..55b02ac
--- /dev/null
+++ b/collectors/cgroups.plugin/cgroup-name.sh
@@ -0,0 +1,597 @@
+#!/usr/bin/env bash
+#shellcheck disable=SC2001
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Script to find a better name for cgroups
+#
+
+export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
+export LC_ALL=C
+
+# -----------------------------------------------------------------------------
+
+PROGRAM_NAME="$(basename "${0}")"
+
+logdate() {
+ date "+%Y-%m-%d %H:%M:%S"
+}
+
+log() {
+ local status="${1}"
+ shift
+
+ echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
+
+}
+
+warning() {
+ log WARNING "${@}"
+}
+
+error() {
+ log ERROR "${@}"
+}
+
+info() {
+ log INFO "${@}"
+}
+
+fatal() {
+ log FATAL "${@}"
+ exit 1
+}
+
+function parse_docker_like_inspect_output() {
+ local output="${1}"
+ eval "$(grep -E "^(NOMAD_NAMESPACE|NOMAD_JOB_NAME|NOMAD_TASK_NAME|NOMAD_SHORT_ALLOC_ID|CONT_NAME)=" <<<"$output")"
+ if [ -n "$NOMAD_NAMESPACE" ] && [ -n "$NOMAD_JOB_NAME" ] && [ -n "$NOMAD_TASK_NAME" ] && [ -n "$NOMAD_SHORT_ALLOC_ID" ]; then
+ echo "${NOMAD_NAMESPACE}-${NOMAD_JOB_NAME}-${NOMAD_TASK_NAME}-${NOMAD_SHORT_ALLOC_ID}"
+ else
+ echo "${CONT_NAME}" | sed 's|^/||'
+ fi
+}
+
+function docker_like_get_name_command() {
+ local command="${1}"
+ local id="${2}"
+ info "Running command: ${command} inspect --format='{{range .Config.Env}}{{println .}}{{end}}CONT_NAME={{ .Name}}' \"${id}\""
+ if OUTPUT="$(${command} inspect --format='{{range .Config.Env}}{{println .}}{{end}}CONT_NAME={{ .Name}}' "${id}")" &&
+ [ -n "$OUTPUT" ]; then
+ NAME="$(parse_docker_like_inspect_output "$OUTPUT")"
+ fi
+ return 0
+}
+
+function docker_like_get_name_api() {
+ local host_var="${1}"
+ local host="${!host_var}"
+ local path="/containers/${2}/json"
+ if [ -z "${host}" ]; then
+ warning "No ${host_var} is set"
+ return 1
+ fi
+ if ! command -v jq >/dev/null 2>&1; then
+ warning "Can't find jq command line tool. jq is required for netdata to retrieve container name using ${host} API, falling back to docker ps"
+ return 1
+ fi
+ if [ -S "${host}" ]; then
+ info "Running API command: curl --unix-socket \"${host}\" http://localhost${path}"
+ JSON=$(curl -sS --unix-socket "${host}" "http://localhost${path}")
+ else
+ info "Running API command: curl \"${host}${path}\""
+ JSON=$(curl -sS "${host}${path}")
+ fi
+ if OUTPUT=$(echo "${JSON}" | jq -r '.Config.Env[],"CONT_NAME=\(.Name)"') && [ -n "$OUTPUT" ]; then
+ NAME="$(parse_docker_like_inspect_output "$OUTPUT")"
+ fi
+ return 0
+}
+
+# get_lbl_val returns the value for the label with the given name.
+# Returns "null" string if the label doesn't exist.
+# Expected labels format: 'name="value",...'.
+function get_lbl_val() {
+ local labels want_name
+ labels="${1}"
+ want_name="${2}"
+
+ IFS=, read -ra labels <<< "$labels"
+
+ local lname lval
+ for l in "${labels[@]}"; do
+ IFS="=" read -r lname lval <<< "$l"
+ if [ "$want_name" = "$lname" ] && [ -n "$lval" ]; then
+ echo "${lval:1:-1}" # trim "
+ return 0
+ fi
+ done
+
+ echo "null"
+ return 1
+}
+
+function add_lbl_prefix() {
+ local orig_labels prefix
+ orig_labels="${1}"
+ prefix="${2}"
+
+ IFS=, read -ra labels <<< "$orig_labels"
+
+ local new_labels
+ for l in "${labels[@]}"; do
+ new_labels+="${prefix}${l},"
+ done
+
+ echo "${new_labels:0:-1}" # trim last ','
+}
+
+function k8s_is_pause_container() {
+ local cgroup_path="${1}"
+
+ local file
+ if [ -d "${NETDATA_HOST_PREFIX}/sys/fs/cgroup/cpuacct" ]; then
+ file="${NETDATA_HOST_PREFIX}/sys/fs/cgroup/cpuacct/$cgroup_path/cgroup.procs"
+ else
+ file="${NETDATA_HOST_PREFIX}/sys/fs/cgroup/$cgroup_path/cgroup.procs"
+ fi
+
+ [ ! -f "$file" ] && return 1
+
+ local procs
+ IFS= read -rd' ' procs 2>/dev/null <"$file"
+ #shellcheck disable=SC2206
+ procs=($procs)
+
+ [ "${#procs[@]}" -ne 1 ] && return 1
+
+ IFS= read -r comm 2>/dev/null <"/proc/${procs[0]}/comm"
+
+ [ "$comm" == "pause" ]
+ return
+}
+
+function k8s_gcp_get_cluster_name() {
+ local header url id loc name
+ header="Metadata-Flavor: Google"
+ url="http://metadata/computeMetadata/v1"
+ if id=$(curl --fail -s -m 3 --noproxy "*" -H "$header" "$url/project/project-id") &&
+ loc=$(curl --fail -s -m 3 --noproxy "*" -H "$header" "$url/instance/attributes/cluster-location") &&
+ name=$(curl --fail -s -m 3 --noproxy "*" -H "$header" "$url/instance/attributes/cluster-name") &&
+ [ -n "$id" ] && [ -n "$loc" ] && [ -n "$name" ]; then
+ echo "gke_${id}_${loc}_${name}"
+ return 0
+ fi
+ return 1
+}
+
+# k8s_get_kubepod_name resolves */kubepods/* cgroup name.
+# pod level cgroup name format: 'pod_<namespace>_<pod_name>'
+# container level cgroup name format: 'cntr_<namespace>_<pod_name>_<container_name>'
+function k8s_get_kubepod_name() {
+ # GKE /sys/fs/cgroup/*/ (cri=docker, cgroups=v1):
+ # |-- kubepods
+ # | |-- burstable
+ # | | |-- pod98cee708-023b-11eb-933d-42010a800193
+ # | | | |-- 922161c98e6ea450bf665226cdc64ca2aa3e889934c2cff0aec4325f8f78ac03
+ # | `-- pode314bbac-d577-11ea-a171-42010a80013b
+ # | |-- 7d505356b04507de7b710016d540b2759483ed5f9136bb01a80872b08f771930
+ #
+ # GKE /sys/fs/cgroup/*/ (cri=containerd, cgroups=v1):
+ # |-- kubepods.slice
+ # | |-- kubepods-besteffort.slice
+ # | | |-- kubepods-besteffort-pode1465238_4518_4c21_832f_fd9f87033dad.slice
+ # | | | |-- cri-containerd-66be9b2efdf4d85288c319b8c1a2f50d2439b5617e36f45d9d0d0be1381113be.scope
+ # | `-- kubepods-pod91f5b561_369f_4103_8015_66391059996a.slice
+ # | |-- cri-containerd-24c53b774a586f06abc058619b47f71d9d869ac50c92898adbd199106fd0aaeb.scope
+ #
+ # GKE /sys/fs/cgroup/*/ (cri=crio, cgroups=v1):
+ # |-- kubepods.slice
+ # | |-- kubepods-besteffort.slice
+ # | | |-- kubepods-besteffort-podad412dfe_3589_4056_965a_592356172968.slice
+ # | | | |-- crio-77b019312fd9825828b70214b2c94da69c30621af2a7ee06f8beace4bc9439e5.scope
+ #
+ # Minikube (v1.8.2) /sys/fs/cgroup/*/ (cri=docker, cgroups=v1):
+ # |-- kubepods.slice
+ # | |-- kubepods-besteffort.slice
+ # | | |-- kubepods-besteffort-pod10fb5647_c724_400c_b9cc_0e6eae3110e7.slice
+ # | | | |-- docker-36e5eb5056dfdf6dbb75c0c44a1ecf23217fe2c50d606209d8130fcbb19fb5a7.scope
+ #
+ # kind v0.14.0
+ # |-- kubelet.slice
+ # | |-- kubelet-kubepods.slice
+ # | | |-- kubelet-kubepods-besteffort.slice
+ # | | | |-- kubelet-kubepods-besteffort-pod7881ed9e_c63e_4425_b5e0_ac55a08ae939.slice
+ # | | | | |-- cri-containerd-00c7939458bffc416bb03451526e9fde13301d6654cfeadf5b4964a7fb5be1a9.scope
+ #
+ # NOTE: cgroups plugin
+ # - uses '_' to join dir names (so it is <parent>_<child>_<child>_...)
+ # - replaces '.' with '-'
+
+ local fn="${FUNCNAME[0]}"
+ local cgroup_path="${1}"
+ local id="${2}"
+
+ if [[ ! $id =~ ^.*kubepods.* ]]; then
+ warning "${fn}: '${id}' is not kubepod cgroup."
+ return 1
+ fi
+
+ local clean_id="$id"
+ clean_id=${clean_id//.slice/}
+ clean_id=${clean_id//.scope/}
+
+ local name pod_uid cntr_id
+ if [[ $clean_id == "kubepods" ]]; then
+ name="$clean_id"
+ elif [[ $clean_id =~ .+(besteffort|burstable|guaranteed)$ ]]; then
+ # kubepods_<QOS_CLASS>
+ # kubepods_kubepods-<QOS_CLASS>
+ name=${clean_id//-/_}
+ name=${name/#kubepods_kubepods/kubepods}
+ elif [[ $clean_id =~ .+pod[a-f0-9_-]+_(docker|crio|cri-containerd)-([a-f0-9]+)$ ]]; then
+ # ...pod<POD_UID>_(docker|crio|cri-containerd)-<CONTAINER_ID> (POD_UID w/ "_")
+ cntr_id=${BASH_REMATCH[2]}
+ elif [[ $clean_id =~ .+pod[a-f0-9-]+_([a-f0-9]+)$ ]]; then
+ # ...pod<POD_UID>_<CONTAINER_ID>
+ cntr_id=${BASH_REMATCH[1]}
+ elif [[ $clean_id =~ .+pod([a-f0-9_-]+)$ ]]; then
+ # ...pod<POD_UID> (POD_UID w/ and w/o "_")
+ pod_uid=${BASH_REMATCH[1]}
+ pod_uid=${pod_uid//_/-}
+ fi
+
+ if [ -n "$name" ]; then
+ echo "$name"
+ return 0
+ fi
+
+ if [ -z "$pod_uid" ] && [ -z "$cntr_id" ]; then
+ warning "${fn}: can't extract pod_uid or container_id from the cgroup '$id'."
+ return 3
+ fi
+
+ [ -n "$pod_uid" ] && info "${fn}: cgroup '$id' is a pod(uid:$pod_uid)"
+ [ -n "$cntr_id" ] && info "${fn}: cgroup '$id' is a container(id:$cntr_id)"
+
+ if [ -n "$cntr_id" ] && k8s_is_pause_container "$cgroup_path"; then
+ return 3
+ fi
+
+ if ! command -v jq > /dev/null 2>&1; then
+ warning "${fn}: 'jq' command not available."
+ return 1
+ fi
+
+ local tmp_kube_cluster_name="${TMPDIR:-"/tmp"}/netdata-cgroups-k8s-cluster-name"
+ local tmp_kube_system_ns_uid_file="${TMPDIR:-"/tmp"}/netdata-cgroups-kubesystem-uid"
+ local tmp_kube_containers_file="${TMPDIR:-"/tmp"}/netdata-cgroups-containers"
+
+ local kube_cluster_name
+ local kube_system_uid
+ local labels
+
+ if [ -n "$cntr_id" ] &&
+ [ -f "$tmp_kube_cluster_name" ] &&
+ [ -f "$tmp_kube_system_ns_uid_file" ] &&
+ [ -f "$tmp_kube_containers_file" ] &&
+ labels=$(grep "$cntr_id" "$tmp_kube_containers_file" 2>/dev/null); then
+ IFS= read -r kube_system_uid 2>/dev/null <"$tmp_kube_system_ns_uid_file"
+ IFS= read -r kube_cluster_name 2>/dev/null <"$tmp_kube_cluster_name"
+ else
+ IFS= read -r kube_system_uid 2>/dev/null <"$tmp_kube_system_ns_uid_file"
+ IFS= read -r kube_cluster_name 2>/dev/null <"$tmp_kube_cluster_name"
+ [ -z "$kube_cluster_name" ] && ! kube_cluster_name=$(k8s_gcp_get_cluster_name) && kube_cluster_name="unknown"
+
+ local kube_system_ns
+ local pods
+
+ if [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_PORT_443_TCP_PORT}" ]; then
+ local token header host url
+ token="$(</var/run/secrets/kubernetes.io/serviceaccount/token)"
+ header="Authorization: Bearer $token"
+ host="$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT"
+
+ if [ -z "$kube_system_uid" ]; then
+ url="https://$host/api/v1/namespaces/kube-system"
+ # FIX: check HTTP response code
+ if ! kube_system_ns=$(curl --fail -sSk -H "$header" "$url" 2>&1); then
+ warning "${fn}: error on curl '${url}': ${kube_system_ns}."
+ fi
+ fi
+
+ url="https://$host/api/v1/pods"
+ [ -n "$MY_NODE_NAME" ] && url+="?fieldSelector=spec.nodeName==$MY_NODE_NAME"
+ # FIX: check HTTP response code
+ if ! pods=$(curl --fail -sSk -H "$header" "$url" 2>&1); then
+ warning "${fn}: error on curl '${url}': ${pods}."
+ return 1
+ fi
+ elif ps -C kubelet >/dev/null 2>&1 && command -v kubectl >/dev/null 2>&1; then
+ if [ -z "$kube_system_uid" ]; then
+ if ! kube_system_ns=$(kubectl --kubeconfig="$KUBE_CONFIG" get namespaces kube-system -o json 2>&1); then
+ warning "${fn}: error on 'kubectl': ${kube_system_ns}."
+ fi
+ fi
+
+ [[ -z ${KUBE_CONFIG+x} ]] && KUBE_CONFIG="/etc/kubernetes/admin.conf"
+ if ! pods=$(kubectl --kubeconfig="$KUBE_CONFIG" get pods --all-namespaces -o json 2>&1); then
+ warning "${fn}: error on 'kubectl': ${pods}."
+ return 1
+ fi
+ else
+ warning "${fn}: not inside the k8s cluster and 'kubectl' command not available."
+ return 1
+ fi
+
+ if [ -n "$kube_system_ns" ] && ! kube_system_uid=$(jq -r '.metadata.uid' <<<"$kube_system_ns" 2>&1); then
+ warning "${fn}: error on 'jq' parse kube_system_ns: ${kube_system_uid}."
+ fi
+
+ local jq_filter
+ jq_filter+='.items[] | "'
+ jq_filter+='namespace=\"\(.metadata.namespace)\",'
+ jq_filter+='pod_name=\"\(.metadata.name)\",'
+ jq_filter+='pod_uid=\"\(.metadata.uid)\",'
+ #jq_filter+='\(.metadata.labels | to_entries | map("pod_label_"+.key+"=\""+.value+"\"") | join(",") | if length > 0 then .+"," else . end)'
+ jq_filter+='\((.metadata.ownerReferences[]? | select(.controller==true) | "controller_kind=\""+.kind+"\",controller_name=\""+.name+"\",") // "")'
+ jq_filter+='node_name=\"\(.spec.nodeName)\",'
+ jq_filter+='" + '
+ jq_filter+='(.status.containerStatuses[]? | "'
+ jq_filter+='container_name=\"\(.name)\",'
+ jq_filter+='container_id=\"\(.containerID)\"'
+ jq_filter+='") | '
+ jq_filter+='sub("(docker|cri-o|containerd)://";"")' # containerID: docker://a346da9bc0e3eaba6b295f64ac16e02f2190db2cef570835706a9e7a36e2c722
+
+ local containers
+ if ! containers=$(jq -r "${jq_filter}" <<<"$pods" 2>&1); then
+ warning "${fn}: error on 'jq' parse pods: ${containers}."
+ return 1
+ fi
+
+ [ -n "$kube_cluster_name" ] && echo "$kube_cluster_name" >"$tmp_kube_cluster_name" 2>/dev/null
+ [ -n "$kube_system_ns" ] && [ -n "$kube_system_uid" ] && echo "$kube_system_uid" >"$tmp_kube_system_ns_uid_file" 2>/dev/null
+ echo "$containers" >"$tmp_kube_containers_file" 2>/dev/null
+ fi
+
+ local qos_class
+ if [[ $clean_id =~ .+(besteffort|burstable) ]]; then
+ qos_class="${BASH_REMATCH[1]}"
+ else
+ qos_class="guaranteed"
+ fi
+
+ # available labels:
+ # namespace, pod_name, pod_uid, container_name, container_id, node_name
+ if [ -n "$cntr_id" ]; then
+ if [ -n "$labels" ] || labels=$(grep "$cntr_id" <<< "$containers" 2> /dev/null); then
+ labels+=',kind="container"'
+ labels+=",qos_class=\"$qos_class\""
+ [ -n "$kube_system_uid" ] && [ "$kube_system_uid" != "null" ] && labels+=",cluster_id=\"$kube_system_uid\""
+ [ -n "$kube_cluster_name" ] && [ "$kube_cluster_name" != "unknown" ] && labels+=",cluster_name=\"$kube_cluster_name\""
+ name="cntr"
+ name+="_$(get_lbl_val "$labels" namespace)"
+ name+="_$(get_lbl_val "$labels" pod_name)"
+ name+="_$(get_lbl_val "$labels" container_name)"
+ labels=$(add_lbl_prefix "$labels" "k8s_")
+ name+=" $labels"
+ else
+ return 2
+ fi
+ elif [ -n "$pod_uid" ]; then
+ if labels=$(grep "$pod_uid" -m 1 <<< "$containers" 2> /dev/null); then
+ labels="${labels%%,container_*}"
+ labels+=',kind="pod"'
+ labels+=",qos_class=\"$qos_class\""
+ [ -n "$kube_system_uid" ] && [ "$kube_system_uid" != "null" ] && labels+=",cluster_id=\"$kube_system_uid\""
+ [ -n "$kube_cluster_name" ] && [ "$kube_cluster_name" != "unknown" ] && labels+=",cluster_name=\"$kube_cluster_name\""
+ name="pod"
+ name+="_$(get_lbl_val "$labels" namespace)"
+ name+="_$(get_lbl_val "$labels" pod_name)"
+ labels=$(add_lbl_prefix "$labels" "k8s_")
+ name+=" $labels"
+ else
+ return 2
+ fi
+ fi
+
+ # jq filter nonexistent field and nonexistent label value is 'null'
+ if [[ $name =~ _null(_|$) ]]; then
+ warning "${fn}: invalid name: $name (cgroup '$id')"
+ return 1
+ fi
+
+ echo "$name"
+ [ -n "$name" ]
+ return
+}
+
+function k8s_get_name() {
+ local fn="${FUNCNAME[0]}"
+ local cgroup_path="${1}"
+ local id="${2}"
+
+ NAME=$(k8s_get_kubepod_name "$cgroup_path" "$id")
+
+ case "$?" in
+ 0)
+ NAME="k8s_${NAME}"
+
+ local name labels
+ name=${NAME%% *}
+ labels=${NAME#* }
+ if [ "$name" != "$labels" ]; then
+ info "${fn}: cgroup '${id}' has chart name '${name}', labels '${labels}"
+ else
+ info "${fn}: cgroup '${id}' has chart name '${NAME}'"
+ fi
+ EXIT_CODE=$EXIT_SUCCESS
+ ;;
+ 1)
+ NAME="k8s_${id}"
+ warning "${fn}: cannot find the name of cgroup with id '${id}'. Setting name to ${NAME} and enabling it."
+ EXIT_CODE=$EXIT_SUCCESS
+ ;;
+ 2)
+ NAME="k8s_${id}"
+ warning "${fn}: cannot find the name of cgroup with id '${id}'. Setting name to ${NAME} and asking for retry."
+ EXIT_CODE=$EXIT_RETRY
+ ;;
+ *)
+ NAME="k8s_${id}"
+ warning "${fn}: cannot find the name of cgroup with id '${id}'. Setting name to ${NAME} and disabling it."
+ EXIT_CODE=$EXIT_DISABLE
+ ;;
+ esac
+}
+
+function docker_get_name() {
+ local id="${1}"
+ # See https://github.com/netdata/netdata/pull/13523 for details
+ if command -v snap >/dev/null 2>&1 && snap list docker >/dev/null 2>&1; then
+ docker_like_get_name_api DOCKER_HOST "${id}"
+ elif hash docker 2> /dev/null; then
+ docker_like_get_name_command docker "${id}"
+ else
+ docker_like_get_name_api DOCKER_HOST "${id}" || docker_like_get_name_command podman "${id}"
+ fi
+ if [ -z "${NAME}" ]; then
+ warning "cannot find the name of docker container '${id}'"
+ EXIT_CODE=$EXIT_RETRY
+ NAME="${id:0:12}"
+ else
+ info "docker container '${id}' is named '${NAME}'"
+ fi
+}
+
+function docker_validate_id() {
+ local id="${1}"
+ if [ -n "${id}" ] && { [ ${#id} -eq 64 ] || [ ${#id} -eq 12 ]; }; then
+ docker_get_name "${id}"
+ else
+ error "a docker id cannot be extracted from docker cgroup '${CGROUP}'."
+ fi
+}
+
+function podman_get_name() {
+ local id="${1}"
+
+ # for Podman, prefer using the API if we can, as netdata will not normally have access
+ # to other users' containers, so they will not be visible when running `podman ps`
+ docker_like_get_name_api PODMAN_HOST "${id}" || docker_like_get_name_command podman "${id}"
+
+ if [ -z "${NAME}" ]; then
+ warning "cannot find the name of podman container '${id}'"
+ EXIT_CODE=$EXIT_RETRY
+ NAME="${id:0:12}"
+ else
+ info "podman container '${id}' is named '${NAME}'"
+ fi
+}
+
+function podman_validate_id() {
+ local id="${1}"
+ if [ -n "${id}" ] && [ ${#id} -eq 64 ]; then
+ podman_get_name "${id}"
+ else
+ error "a podman id cannot be extracted from docker cgroup '${CGROUP}'."
+ fi
+}
+
+# -----------------------------------------------------------------------------
+
+DOCKER_HOST="${DOCKER_HOST:=/var/run/docker.sock}"
+PODMAN_HOST="${PODMAN_HOST:=/run/podman/podman.sock}"
+CGROUP_PATH="${1}" # the path as it is (e.g. '/docker/efcf4c409')
+CGROUP="${2}" # the modified path (e.g. 'docker_efcf4c409')
+EXIT_SUCCESS=0
+EXIT_RETRY=2
+EXIT_DISABLE=3
+EXIT_CODE=$EXIT_SUCCESS
+NAME=
+
+# -----------------------------------------------------------------------------
+
+if [ -z "${CGROUP}" ]; then
+ fatal "called without a cgroup name. Nothing to do."
+fi
+
+if [ -z "${NAME}" ]; then
+ if [[ ${CGROUP} =~ ^.*kubepods.* ]]; then
+ k8s_get_name "${CGROUP_PATH}" "${CGROUP}"
+ fi
+fi
+
+if [ -z "${NAME}" ]; then
+ if [[ ${CGROUP} =~ ^.*docker[-_/\.][a-fA-F0-9]+[-_\.]?.*$ ]]; then
+ # docker containers
+ #shellcheck disable=SC1117
+ DOCKERID="$(echo "${CGROUP}" | sed "s|^.*docker[-_/]\([a-fA-F0-9]\+\)[-_\.]\?.*$|\1|")"
+ docker_validate_id "${DOCKERID}"
+ elif [[ ${CGROUP} =~ ^.*ecs[-_/\.][a-fA-F0-9]+[-_\.]?.*$ ]]; then
+ # ECS
+ #shellcheck disable=SC1117
+ DOCKERID="$(echo "${CGROUP}" | sed "s|^.*ecs[-_/].*[-_/]\([a-fA-F0-9]\+\)[-_\.]\?.*$|\1|")"
+ docker_validate_id "${DOCKERID}"
+ elif [[ ${CGROUP} =~ system.slice_containerd.service_cpuset_[a-fA-F0-9]+[-_\.]?.*$ ]]; then
+ # docker containers under containerd
+ #shellcheck disable=SC1117
+ DOCKERID="$(echo "${CGROUP}" | sed "s|^.*ystem.slice_containerd.service_cpuset_\([a-fA-F0-9]\+\)[-_\.]\?.*$|\1|")"
+ docker_validate_id "${DOCKERID}"
+ elif [[ ${CGROUP} =~ ^.*libpod-[a-fA-F0-9]+.*$ ]]; then
+ # Podman
+ PODMANID="$(echo "${CGROUP}" | sed "s|^.*libpod-\([a-fA-F0-9]\+\).*$|\1|")"
+ podman_validate_id "${PODMANID}"
+
+ elif [[ ${CGROUP} =~ machine.slice[_/].*\.service ]]; then
+ # systemd-nspawn
+ NAME="$(echo "${CGROUP}" | sed 's/.*machine.slice[_\/]\(.*\)\.service/\1/g')"
+
+ elif [[ ${CGROUP} =~ machine.slice_machine.*-lxc ]]; then
+ # libvirtd / lxc containers
+ # machine.slice machine-lxc/x2d969/x2dhubud0xians01.scope => lxc/hubud0xians01
+ # machine.slice_machine-lxc/x2d969/x2dhubud0xians01.scope/libvirt_init.scope => lxc/hubud0xians01/libvirt_init
+ NAME="lxc/$(echo "${CGROUP}" | sed 's/machine.slice_machine.*-lxc//; s/[\/_]x2d[[:digit:]]*//; s/[\/_]x2d//g; s/\.scope//g')"
+ elif [[ ${CGROUP} =~ machine.slice_machine.*-qemu ]]; then
+ # libvirtd / qemu virtual machines
+ # machine.slice_machine-qemu_x2d1_x2dopnsense.scope => qemu_opnsense
+ NAME="qemu_$(echo "${CGROUP}" | sed 's/machine.slice_machine.*-qemu//; s/[\/_]x2d[[:digit:]]*//; s/[\/_]x2d//g; s/\.scope//g')"
+
+ elif [[ ${CGROUP} =~ machine_.*\.libvirt-qemu ]]; then
+ # libvirtd / qemu virtual machines
+ NAME="qemu_$(echo "${CGROUP}" | sed 's/^machine_//; s/\.libvirt-qemu$//; s/-/_/;')"
+
+ elif [[ ${CGROUP} =~ qemu.slice_([0-9]+).scope && -d /etc/pve ]]; then
+ # Proxmox VMs
+
+ FILENAME="/etc/pve/qemu-server/${BASH_REMATCH[1]}.conf"
+ if [[ -f $FILENAME && -r $FILENAME ]]; then
+ NAME="qemu_$(grep -e '^name: ' "/etc/pve/qemu-server/${BASH_REMATCH[1]}.conf" | head -1 | sed -rn 's|\s*name\s*:\s*(.*)?$|\1|p')"
+ else
+ error "proxmox config file missing ${FILENAME} or netdata does not have read access. Please ensure netdata is a member of www-data group."
+ fi
+ elif [[ ${CGROUP} =~ lxc_([0-9]+) && -d /etc/pve ]]; then
+ # Proxmox Containers (LXC)
+
+ FILENAME="/etc/pve/lxc/${BASH_REMATCH[1]}.conf"
+ if [[ -f ${FILENAME} && -r ${FILENAME} ]]; then
+ NAME=$(grep -e '^hostname: ' "/etc/pve/lxc/${BASH_REMATCH[1]}.conf" | head -1 | sed -rn 's|\s*hostname\s*:\s*(.*)?$|\1|p')
+ else
+ error "proxmox config file missing ${FILENAME} or netdata does not have read access. Please ensure netdata is a member of www-data group."
+ fi
+ elif [[ ${CGROUP} =~ lxc.payload.* ]]; then
+ # LXC 4.0
+ NAME="$(echo "${CGROUP}" | sed 's/lxc\.payload\.\(.*\)/\1/g')"
+ fi
+
+ [ -z "${NAME}" ] && NAME="${CGROUP}"
+ [ ${#NAME} -gt 100 ] && NAME="${NAME:0:100}"
+fi
+
+info "cgroup '${CGROUP}' is called '${NAME}'"
+echo "${NAME}"
+
+exit ${EXIT_CODE}
diff --git a/collectors/cgroups.plugin/cgroup-network-helper.sh b/collectors/cgroups.plugin/cgroup-network-helper.sh
new file mode 100755
index 0000000..783332f
--- /dev/null
+++ b/collectors/cgroups.plugin/cgroup-network-helper.sh
@@ -0,0 +1,302 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1117
+
+# cgroup-network-helper.sh
+# detect container and virtual machine interfaces
+#
+# (C) 2017 Costa Tsaousis
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This script is called as root (by cgroup-network), with either a pid, or a cgroup path.
+# It tries to find all the network interfaces that belong to the same cgroup.
+#
+# It supports several method for this detection:
+#
+# 1. cgroup-network (the binary father of this script) detects veth network interfaces,
+# by examining iflink and ifindex IDs and switching namespaces
+# (it also detects the interface name as it is used by the container).
+#
+# 2. this script, uses /proc/PID/fdinfo to find tun/tap network interfaces.
+#
+# 3. this script, calls virsh to find libvirt network interfaces.
+#
+
+# -----------------------------------------------------------------------------
+
+# the system path is cleared by cgroup-network
+# shellcheck source=/dev/null
+[ -f /etc/profile ] && source /etc/profile
+
+export LC_ALL=C
+
+PROGRAM_NAME="$(basename "${0}")"
+
+logdate() {
+ date "+%Y-%m-%d %H:%M:%S"
+}
+
+log() {
+ local status="${1}"
+ shift
+
+ echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
+
+}
+
+warning() {
+ log WARNING "${@}"
+}
+
+error() {
+ log ERROR "${@}"
+}
+
+info() {
+ log INFO "${@}"
+}
+
+fatal() {
+ log FATAL "${@}"
+ exit 1
+}
+
+debug=${NETDATA_CGROUP_NETWORK_HELPER_DEBUG=0}
+debug() {
+ [ "${debug}" = "1" ] && log DEBUG "${@}"
+}
+
+# -----------------------------------------------------------------------------
+# check for BASH v4+ (required for associative arrays)
+
+[ $(( BASH_VERSINFO[0] )) -lt 4 ] && \
+ fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
+
+# -----------------------------------------------------------------------------
+# parse the arguments
+
+pid=
+cgroup=
+while [ -n "${1}" ]
+do
+ case "${1}" in
+ --cgroup) cgroup="${2}"; shift 1;;
+ --pid|-p) pid="${2}"; shift 1;;
+ --debug|debug) debug=1;;
+ *) fatal "Cannot understand argument '${1}'";;
+ esac
+
+ shift
+done
+
+if [ -z "${pid}" ] && [ -z "${cgroup}" ]
+then
+ fatal "Either --pid or --cgroup is required"
+fi
+
+# -----------------------------------------------------------------------------
+
+set_source() {
+ [ ${debug} -eq 1 ] && echo "SRC ${*}"
+}
+
+
+# -----------------------------------------------------------------------------
+# veth interfaces via cgroup
+
+# cgroup-network can detect veth interfaces by itself (written in C).
+# If you seek for a shell version of what it does, check this:
+# https://github.com/netdata/netdata/issues/474#issuecomment-317866709
+
+
+# -----------------------------------------------------------------------------
+# tun/tap interfaces via /proc/PID/fdinfo
+
+# find any tun/tap devices linked to a pid
+proc_pid_fdinfo_iff() {
+ local p="${1}" # the pid
+
+ debug "Searching for tun/tap interfaces for pid ${p}..."
+ set_source "fdinfo"
+ grep "^iff:.*" "${NETDATA_HOST_PREFIX}/proc/${p}/fdinfo"/* 2>/dev/null | cut -f 2
+}
+
+find_tun_tap_interfaces_for_cgroup() {
+ local c="${1}" # the cgroup path
+ [ -d "${c}/emulator" ] && c="${c}/emulator" # check for 'emulator' subdirectory
+ c="${c}/cgroup.procs" # make full path
+
+ # for each pid of the cgroup
+ # find any tun/tap devices linked to the pid
+ if [ -f "${c}" ]
+ then
+ local p
+ for p in $(< "${c}" )
+ do
+ proc_pid_fdinfo_iff "${p}"
+ done
+ else
+ debug "Cannot find file '${c}', not searching for tun/tap interfaces."
+ fi
+}
+
+
+# -----------------------------------------------------------------------------
+# virsh domain network interfaces
+
+virsh_cgroup_to_domain_name() {
+ local c="${1}" # the cgroup path
+
+ debug "extracting a possible virsh domain from cgroup ${c}..."
+
+ # extract for the cgroup path
+ sed -n -e "s|.*/machine-qemu\\\\x2d[0-9]\+\\\\x2d\(.*\)\.scope$|\1|p" \
+ -e "s|.*/machine/qemu-[0-9]\+-\(.*\)\.libvirt-qemu$|\1|p" \
+ -e "s|.*/machine/\(.*\)\.libvirt-qemu$|\1|p" \
+ <<EOF
+${c}
+EOF
+}
+
+virsh_find_all_interfaces_for_cgroup() {
+ local c="${1}" # the cgroup path
+
+ # the virsh command
+ local virsh
+ # shellcheck disable=SC2230
+ virsh="$(which virsh 2>/dev/null || command -v virsh 2>/dev/null)"
+
+ if [ -n "${virsh}" ]
+ then
+ local d
+ d="$(virsh_cgroup_to_domain_name "${c}")"
+ # convert hex to character
+ # e.g.: vm01\x2dweb => vm01-web (https://github.com/netdata/netdata/issues/11088#issuecomment-832618149)
+ d="$(printf '%b' "${d}")"
+
+ if [ -n "${d}" ]
+ then
+ debug "running: virsh domiflist ${d}; to find the network interfaces"
+
+ # 'virsh -r domiflist <domain>' example output
+ # Interface Type Source Model MAC
+ #--------------------------------------------------------------
+ # vnet3 bridge br0 virtio 52:54:00:xx:xx:xx
+ # vnet4 network default virtio 52:54:00:yy:yy:yy
+
+ # match only 'network' interfaces from virsh output
+ set_source "virsh"
+ "${virsh}" -r domiflist "${d}" |\
+ sed -n \
+ -e "s|^[[:space:]]\?\([^[:space:]]\+\)[[:space:]]\+network[[:space:]]\+\([^[:space:]]\+\)[[:space:]]\+[^[:space:]]\+[[:space:]]\+[^[:space:]]\+$|\1 \1_\2|p" \
+ -e "s|^[[:space:]]\?\([^[:space:]]\+\)[[:space:]]\+bridge[[:space:]]\+\([^[:space:]]\+\)[[:space:]]\+[^[:space:]]\+[[:space:]]\+[^[:space:]]\+$|\1 \1_\2|p"
+ else
+ debug "no virsh domain extracted from cgroup ${c}"
+ fi
+ else
+ debug "virsh command is not available"
+ fi
+}
+
+# -----------------------------------------------------------------------------
+# netnsid detected interfaces
+
+netnsid_find_all_interfaces_for_pid() {
+ local pid="${1}"
+ [ -z "${pid}" ] && return 1
+
+ local nsid
+ nsid=$(lsns -t net -p "${pid}" -o NETNSID -nr 2>/dev/null)
+ if [ -z "${nsid}" ] || [ "${nsid}" = "unassigned" ]; then
+ return 1
+ fi
+
+ set_source "netnsid"
+ ip link show |\
+ grep -B 1 -E " link-netnsid ${nsid}($| )" |\
+ sed -n -e "s|^[[:space:]]*[0-9]\+:[[:space:]]\+\([A-Za-z0-9_]\+\)\(@[A-Za-z0-9_]\+\)*:[[:space:]].*$|\1|p"
+}
+
+netnsid_find_all_interfaces_for_cgroup() {
+ local c="${1}" # the cgroup path
+
+ if [ -f "${c}/cgroup.procs" ]; then
+ netnsid_find_all_interfaces_for_pid "$(head -n 1 "${c}/cgroup.procs" 2>/dev/null)"
+ else
+ debug "Cannot find file '${c}/cgroup.procs', not searching for netnsid interfaces."
+ fi
+}
+
+# -----------------------------------------------------------------------------
+
+find_all_interfaces_of_pid_or_cgroup() {
+ local p="${1}" c="${2}" # the pid and the cgroup path
+
+ if [ -n "${pid}" ]
+ then
+ # we have been called with a pid
+
+ proc_pid_fdinfo_iff "${p}"
+ netnsid_find_all_interfaces_for_pid "${p}"
+
+ elif [ -n "${c}" ]
+ then
+ # we have been called with a cgroup
+
+ info "searching for network interfaces of cgroup '${c}'"
+
+ find_tun_tap_interfaces_for_cgroup "${c}"
+ virsh_find_all_interfaces_for_cgroup "${c}"
+ netnsid_find_all_interfaces_for_cgroup "${c}"
+
+ else
+
+ error "Either a pid or a cgroup path is needed"
+ return 1
+
+ fi
+
+ return 0
+}
+
+# -----------------------------------------------------------------------------
+
+# an associative array to store the interfaces
+# the index is the interface name as seen by the host
+# the value is the interface name as seen by the guest / container
+declare -A devs=()
+
+# store all interfaces found in the associative array
+# this will also give the unique devices, as seen by the host
+last_src=
+# shellcheck disable=SC2162
+while read host_device guest_device
+do
+ [ -z "${host_device}" ] && continue
+
+ [ "${host_device}" = "SRC" ] && last_src="${guest_device}" && continue
+
+ # the default guest_device is the host_device
+ [ -z "${guest_device}" ] && guest_device="${host_device}"
+
+ # when we run in debug, show the source
+ debug "Found host device '${host_device}', guest device '${guest_device}', detected via '${last_src}'"
+
+ if [ -z "${devs[${host_device}]}" ] || [ "${devs[${host_device}]}" = "${host_device}" ]; then
+ devs[${host_device}]="${guest_device}"
+ fi
+
+done < <( find_all_interfaces_of_pid_or_cgroup "${pid}" "${cgroup}" )
+
+# print the interfaces found, in the format netdata expects them
+found=0
+for x in "${!devs[@]}"
+do
+ found=$((found + 1))
+ echo "${x} ${devs[${x}]}"
+done
+
+debug "found ${found} network interfaces for pid '${pid}', cgroup '${cgroup}', run as ${USER}, ${UID}"
+
+# let netdata know if we found any
+[ ${found} -eq 0 ] && exit 1
+exit 0
diff --git a/collectors/cgroups.plugin/cgroup-network.c b/collectors/cgroups.plugin/cgroup-network.c
new file mode 100644
index 0000000..0b66ea4
--- /dev/null
+++ b/collectors/cgroups.plugin/cgroup-network.c
@@ -0,0 +1,723 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata/libnetdata.h"
+#include "libnetdata/required_dummies.h"
+
+#ifdef HAVE_SETNS
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE /* See feature_test_macros(7) */
+#endif
+#include <sched.h>
+#endif
+
+char environment_variable2[FILENAME_MAX + 50] = "";
+char *environment[] = {
+ "PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin",
+ environment_variable2,
+ NULL
+};
+
+struct iface {
+ const char *device;
+ uint32_t hash;
+
+ unsigned int ifindex;
+ unsigned int iflink;
+
+ struct iface *next;
+};
+
+unsigned int calc_num_ifaces(struct iface *root) {
+ unsigned int num = 0;
+ for (struct iface *h = root; h; h = h->next) {
+ num++;
+ }
+ return num;
+}
+
+unsigned int read_iface_iflink(const char *prefix, const char *iface) {
+ if(!prefix) prefix = "";
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/sys/class/net/%s/iflink", prefix, iface);
+
+ unsigned long long iflink = 0;
+ int ret = read_single_number_file(filename, &iflink);
+ if(ret) error("Cannot read '%s'.", filename);
+
+ return (unsigned int)iflink;
+}
+
+unsigned int read_iface_ifindex(const char *prefix, const char *iface) {
+ if(!prefix) prefix = "";
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/sys/class/net/%s/ifindex", prefix, iface);
+
+ unsigned long long ifindex = 0;
+ int ret = read_single_number_file(filename, &ifindex);
+ if(ret) error("Cannot read '%s'.", filename);
+
+ return (unsigned int)ifindex;
+}
+
+struct iface *read_proc_net_dev(const char *scope __maybe_unused, const char *prefix) {
+ if(!prefix) prefix = "";
+
+ procfile *ff = NULL;
+ char filename[FILENAME_MAX + 1];
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", prefix, (*prefix)?"/proc/1/net/dev":"/proc/net/dev");
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("parsing '%s'", filename);
+#endif
+
+ ff = procfile_open(filename, " \t,:|", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) {
+ error("Cannot open file '%s'", filename);
+ return NULL;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ error("Cannot read file '%s'", filename);
+ return NULL;
+ }
+
+ size_t lines = procfile_lines(ff), l;
+ struct iface *root = NULL;
+ for(l = 2; l < lines ;l++) {
+ if (unlikely(procfile_linewords(ff, l) < 1)) continue;
+
+ struct iface *t = callocz(1, sizeof(struct iface));
+ t->device = strdupz(procfile_lineword(ff, l, 0));
+ t->hash = simple_hash(t->device);
+ t->ifindex = read_iface_ifindex(prefix, t->device);
+ t->iflink = read_iface_iflink(prefix, t->device);
+ t->next = root;
+ root = t;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("added %s interface '%s', ifindex %u, iflink %u", scope, t->device, t->ifindex, t->iflink);
+#endif
+ }
+
+ procfile_close(ff);
+
+ return root;
+}
+
+void free_iface(struct iface *iface) {
+ freez((void *)iface->device);
+ freez(iface);
+}
+
+void free_host_ifaces(struct iface *iface) {
+ while(iface) {
+ struct iface *t = iface->next;
+ free_iface(iface);
+ iface = t;
+ }
+}
+
+int iface_is_eligible(struct iface *iface) {
+ if(iface->iflink != iface->ifindex)
+ return 1;
+
+ return 0;
+}
+
+int eligible_ifaces(struct iface *root) {
+ int eligible = 0;
+
+ struct iface *t;
+ for(t = root; t ; t = t->next)
+ if(iface_is_eligible(t))
+ eligible++;
+
+ return eligible;
+}
+
+static void continue_as_child(void) {
+ pid_t child = fork();
+ int status;
+ pid_t ret;
+
+ if (child < 0)
+ error("fork() failed");
+
+ /* Only the child returns */
+ if (child == 0)
+ return;
+
+ for (;;) {
+ ret = waitpid(child, &status, WUNTRACED);
+ if ((ret == child) && (WIFSTOPPED(status))) {
+ /* The child suspended so suspend us as well */
+ kill(getpid(), SIGSTOP);
+ kill(child, SIGCONT);
+ } else {
+ break;
+ }
+ }
+
+ /* Return the child's exit code if possible */
+ if (WIFEXITED(status)) {
+ exit(WEXITSTATUS(status));
+ } else if (WIFSIGNALED(status)) {
+ kill(getpid(), WTERMSIG(status));
+ }
+
+ exit(EXIT_FAILURE);
+}
+
+int proc_pid_fd(const char *prefix, const char *ns, pid_t pid) {
+ if(!prefix) prefix = "";
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/%d/%s", prefix, (int)pid, ns);
+ int fd = open(filename, O_RDONLY);
+
+ if(fd == -1)
+ error("Cannot open proc_pid_fd() file '%s'", filename);
+
+ return fd;
+}
+
+static struct ns {
+ int nstype;
+ int fd;
+ int status;
+ const char *name;
+ const char *path;
+} all_ns[] = {
+ // { .nstype = CLONE_NEWUSER, .fd = -1, .status = -1, .name = "user", .path = "ns/user" },
+ // { .nstype = CLONE_NEWCGROUP, .fd = -1, .status = -1, .name = "cgroup", .path = "ns/cgroup" },
+ // { .nstype = CLONE_NEWIPC, .fd = -1, .status = -1, .name = "ipc", .path = "ns/ipc" },
+ // { .nstype = CLONE_NEWUTS, .fd = -1, .status = -1, .name = "uts", .path = "ns/uts" },
+ { .nstype = CLONE_NEWNET, .fd = -1, .status = -1, .name = "network", .path = "ns/net" },
+ { .nstype = CLONE_NEWPID, .fd = -1, .status = -1, .name = "pid", .path = "ns/pid" },
+ { .nstype = CLONE_NEWNS, .fd = -1, .status = -1, .name = "mount", .path = "ns/mnt" },
+
+ // terminator
+ { .nstype = 0, .fd = -1, .status = -1, .name = NULL, .path = NULL }
+};
+
+int switch_namespace(const char *prefix, pid_t pid) {
+
+#ifdef HAVE_SETNS
+
+ int i;
+ for(i = 0; all_ns[i].name ; i++)
+ all_ns[i].fd = proc_pid_fd(prefix, all_ns[i].path, pid);
+
+ int root_fd = proc_pid_fd(prefix, "root", pid);
+ int cwd_fd = proc_pid_fd(prefix, "cwd", pid);
+
+ setgroups(0, NULL);
+
+ // 2 passes - found it at nsenter source code
+ // this is related CLONE_NEWUSER functionality
+
+ // This code cannot switch user namespace (it can all the other namespaces)
+ // Fortunately, we don't need to switch user namespaces.
+
+ int pass;
+ for(pass = 0; pass < 2 ;pass++) {
+ for(i = 0; all_ns[i].name ; i++) {
+ if (all_ns[i].fd != -1 && all_ns[i].status == -1) {
+ if(setns(all_ns[i].fd, all_ns[i].nstype) == -1) {
+ if(pass == 1) {
+ all_ns[i].status = 0;
+ error("Cannot switch to %s namespace of pid %d", all_ns[i].name, (int) pid);
+ }
+ }
+ else
+ all_ns[i].status = 1;
+ }
+ }
+ }
+
+ setgroups(0, NULL);
+
+ if(root_fd != -1) {
+ if(fchdir(root_fd) < 0)
+ error("Cannot fchdir() to pid %d root directory", (int)pid);
+
+ if(chroot(".") < 0)
+ error("Cannot chroot() to pid %d root directory", (int)pid);
+
+ close(root_fd);
+ }
+
+ if(cwd_fd != -1) {
+ if(fchdir(cwd_fd) < 0)
+ error("Cannot fchdir() to pid %d current working directory", (int)pid);
+
+ close(cwd_fd);
+ }
+
+ int do_fork = 0;
+ for(i = 0; all_ns[i].name ; i++)
+ if(all_ns[i].fd != -1) {
+
+ // CLONE_NEWPID requires a fork() to become effective
+ if(all_ns[i].nstype == CLONE_NEWPID && all_ns[i].status)
+ do_fork = 1;
+
+ close(all_ns[i].fd);
+ }
+
+ if(do_fork)
+ continue_as_child();
+
+ return 0;
+
+#else
+
+ errno = ENOSYS;
+ error("setns() is missing on this system.");
+ return 1;
+
+#endif
+}
+
+pid_t read_pid_from_cgroup_file(const char *filename) {
+ int fd = open(filename, procfile_open_flags);
+ if(fd == -1) {
+ error("Cannot open pid_from_cgroup() file '%s'.", filename);
+ return 0;
+ }
+
+ FILE *fp = fdopen(fd, "r");
+ if(!fp) {
+ error("Cannot upgrade fd to fp for file '%s'.", filename);
+ return 0;
+ }
+
+ char buffer[100 + 1];
+ pid_t pid = 0;
+ char *s;
+ while((s = fgets(buffer, 100, fp))) {
+ buffer[100] = '\0';
+ pid = atoi(s);
+ if(pid > 0) break;
+ }
+
+ fclose(fp);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(pid > 0) info("found pid %d on file '%s'", pid, filename);
+#endif
+
+ return pid;
+}
+
+pid_t read_pid_from_cgroup_files(const char *path) {
+ char filename[FILENAME_MAX + 1];
+
+ snprintfz(filename, FILENAME_MAX, "%s/cgroup.procs", path);
+ pid_t pid = read_pid_from_cgroup_file(filename);
+ if(pid > 0) return pid;
+
+ snprintfz(filename, FILENAME_MAX, "%s/tasks", path);
+ return read_pid_from_cgroup_file(filename);
+}
+
+pid_t read_pid_from_cgroup(const char *path) {
+ pid_t pid = read_pid_from_cgroup_files(path);
+ if (pid > 0) return pid;
+
+ DIR *dir = opendir(path);
+ if (!dir) {
+ error("cannot read directory '%s'", path);
+ return 0;
+ }
+
+ struct dirent *de = NULL;
+ while ((de = readdir(dir))) {
+ if (de->d_type == DT_DIR
+ && (
+ (de->d_name[0] == '.' && de->d_name[1] == '\0')
+ || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ ))
+ continue;
+
+ if (de->d_type == DT_DIR) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/%s", path, de->d_name);
+ pid = read_pid_from_cgroup(filename);
+ if(pid > 0) break;
+ }
+ }
+ closedir(dir);
+ return pid;
+}
+
+// ----------------------------------------------------------------------------
+// send the result to netdata
+
+struct found_device {
+ const char *host_device;
+ const char *guest_device;
+
+ uint32_t host_device_hash;
+
+ struct found_device *next;
+} *detected_devices = NULL;
+
+void add_device(const char *host, const char *guest) {
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("adding device with host '%s', guest '%s'", host, guest);
+#endif
+
+ uint32_t hash = simple_hash(host);
+
+ if(guest && (!*guest || strcmp(host, guest) == 0))
+ guest = NULL;
+
+ struct found_device *f;
+ for(f = detected_devices; f ; f = f->next) {
+ if(f->host_device_hash == hash && !strcmp(host, f->host_device)) {
+
+ if(guest && (!f->guest_device || !strcmp(f->host_device, f->guest_device))) {
+ if(f->guest_device) freez((void *)f->guest_device);
+ f->guest_device = strdupz(guest);
+ }
+
+ return;
+ }
+ }
+
+ f = mallocz(sizeof(struct found_device));
+ f->host_device = strdupz(host);
+ f->host_device_hash = hash;
+ f->guest_device = (guest)?strdupz(guest):NULL;
+ f->next = detected_devices;
+ detected_devices = f;
+}
+
+int send_devices(void) {
+ int found = 0;
+
+ struct found_device *f;
+ for(f = detected_devices; f ; f = f->next) {
+ found++;
+ printf("%s %s\n", f->host_device, (f->guest_device)?f->guest_device:f->host_device);
+ }
+
+ return found;
+}
+
+// ----------------------------------------------------------------------------
+// this function should be called only **ONCE**
+// also it has to be the **LAST** to be called
+// since it switches namespaces, so after this call, everything is different!
+
+void detect_veth_interfaces(pid_t pid) {
+ struct iface *cgroup = NULL;
+ struct iface *host, *h, *c;
+
+ host = read_proc_net_dev("host", netdata_configured_host_prefix);
+ if(!host) {
+ errno = 0;
+ error("cannot read host interface list.");
+ goto cleanup;
+ }
+
+ if(!eligible_ifaces(host)) {
+ errno = 0;
+ info("there are no double-linked host interfaces available.");
+ goto cleanup;
+ }
+
+ if(switch_namespace(netdata_configured_host_prefix, pid)) {
+ errno = 0;
+ error("cannot switch to the namespace of pid %u", (unsigned int) pid);
+ goto cleanup;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("switched to namespaces of pid %d", pid);
+#endif
+
+ cgroup = read_proc_net_dev("cgroup", NULL);
+ if(!cgroup) {
+ errno = 0;
+ error("cannot read cgroup interface list.");
+ goto cleanup;
+ }
+
+ if(!eligible_ifaces(cgroup)) {
+ errno = 0;
+ error("there are not double-linked cgroup interfaces available.");
+ goto cleanup;
+ }
+
+ unsigned int host_dev_num = calc_num_ifaces(host);
+ unsigned int cgroup_dev_num = calc_num_ifaces(cgroup);
+ // host ifaces == guest ifaces => we are still in the host namespace
+ // and we can't really identify which ifaces belong to the cgroup (e.g. Proxmox VM).
+ if (host_dev_num == cgroup_dev_num) {
+ unsigned int m = 0;
+ for (h = host; h; h = h->next) {
+ for (c = cgroup; c; c = c->next) {
+ if (h->ifindex == c->ifindex && h->iflink == c->iflink) {
+ m++;
+ break;
+ }
+ }
+ }
+ if (host_dev_num == m) {
+ goto cleanup;
+ }
+ }
+
+ for(h = host; h ; h = h->next) {
+ if(iface_is_eligible(h)) {
+ for (c = cgroup; c; c = c->next) {
+ if(iface_is_eligible(c) && h->ifindex == c->iflink && h->iflink == c->ifindex) {
+ add_device(h->device, c->device);
+ }
+ }
+ }
+ }
+
+cleanup:
+ free_host_ifaces(cgroup);
+ free_host_ifaces(host);
+}
+
+// ----------------------------------------------------------------------------
+// call the external helper
+
+#define CGROUP_NETWORK_INTERFACE_MAX_LINE 2048
+void call_the_helper(pid_t pid, const char *cgroup) {
+ if(setresuid(0, 0, 0) == -1)
+ error("setresuid(0, 0, 0) failed.");
+
+ char command[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1];
+ if(cgroup)
+ snprintfz(command, CGROUP_NETWORK_INTERFACE_MAX_LINE, "exec " PLUGINS_DIR "/cgroup-network-helper.sh --cgroup '%s'", cgroup);
+ else
+ snprintfz(command, CGROUP_NETWORK_INTERFACE_MAX_LINE, "exec " PLUGINS_DIR "/cgroup-network-helper.sh --pid %d", pid);
+
+ info("running: %s", command);
+
+ pid_t cgroup_pid;
+ FILE *fp_child_input, *fp_child_output;
+
+ if(cgroup) {
+ (void)netdata_popen_raw_default_flags(&cgroup_pid, environment, &fp_child_input, &fp_child_output, PLUGINS_DIR "/cgroup-network-helper.sh", "--cgroup", cgroup);
+ }
+ else {
+ char buffer[100];
+ snprintfz(buffer, sizeof(buffer) - 1, "%d", pid);
+ (void)netdata_popen_raw_default_flags(&cgroup_pid, environment, &fp_child_input, &fp_child_output, PLUGINS_DIR "/cgroup-network-helper.sh", "--pid", buffer);
+ }
+
+ if(fp_child_output) {
+ char buffer[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1];
+ char *s;
+ while((s = fgets(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, fp_child_output))) {
+ trim(s);
+
+ if(*s && *s != '\n') {
+ char *t = s;
+ while(*t && *t != ' ') t++;
+ if(*t == ' ') {
+ *t = '\0';
+ t++;
+ }
+
+ if(!*s || !*t) continue;
+ add_device(s, t);
+ }
+ }
+
+ netdata_pclose(fp_child_input, fp_child_output, cgroup_pid);
+ }
+ else
+ error("cannot execute cgroup-network helper script: %s", command);
+}
+
+int is_valid_path_symbol(char c) {
+ switch(c) {
+ case '/': // path separators
+ case '\\': // needed for virsh domains \x2d1\x2dname
+ case ' ': // space
+ case '-': // hyphen
+ case '_': // underscore
+ case '.': // dot
+ case ',': // comma
+ return 1;
+
+ default:
+ return 0;
+ }
+}
+
+// we will pass this path a shell script running as root
+// so, we need to make sure the path will be valid
+// and will not include anything that could allow
+// the caller use shell expansion for gaining escalated
+// privileges.
+int verify_path(const char *path) {
+ struct stat sb;
+
+ char c;
+ const char *s = path;
+ while((c = *s++)) {
+ if(!( isalnum(c) || is_valid_path_symbol(c) )) {
+ error("invalid character in path '%s'", path);
+ return -1;
+ }
+ }
+
+ if(strstr(path, "\\") && !strstr(path, "\\x")) {
+ error("invalid escape sequence in path '%s'", path);
+ return 1;
+ }
+
+ if(strstr(path, "/../")) {
+ error("invalid parent path sequence detected in '%s'", path);
+ return 1;
+ }
+
+ if(path[0] != '/') {
+ error("only absolute path names are supported - invalid path '%s'", path);
+ return -1;
+ }
+
+ if (stat(path, &sb) == -1) {
+ error("cannot stat() path '%s'", path);
+ return -1;
+ }
+
+ if((sb.st_mode & S_IFMT) != S_IFDIR) {
+ error("path '%s' is not a directory", path);
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+char *fix_path_variable(void) {
+ const char *path = getenv("PATH");
+ if(!path || !*path) return 0;
+
+ char *p = strdupz(path);
+ char *safe_path = callocz(1, strlen(p) + strlen("PATH=") + 1);
+ strcpy(safe_path, "PATH=");
+
+ int added = 0;
+ char *ptr = p;
+ while(ptr && *ptr) {
+ char *s = strsep(&ptr, ":");
+ if(s && *s) {
+ if(verify_path(s) == -1) {
+ error("the PATH variable includes an invalid path '%s' - removed it.", s);
+ }
+ else {
+ info("the PATH variable includes a valid path '%s'.", s);
+ if(added) strcat(safe_path, ":");
+ strcat(safe_path, s);
+ added++;
+ }
+ }
+ }
+
+ info("unsafe PATH: '%s'.", path);
+ info(" safe PATH: '%s'.", safe_path);
+
+ freez(p);
+ return safe_path;
+}
+*/
+
+// ----------------------------------------------------------------------------
+// main
+
+void usage(void) {
+ fprintf(stderr, "%s [ -p PID | --pid PID | --cgroup /path/to/cgroup ]\n", program_name);
+ exit(1);
+}
+
+int main(int argc, char **argv) {
+ pid_t pid = 0;
+
+ program_name = argv[0];
+ program_version = VERSION;
+ error_log_syslog = 0;
+
+ // since cgroup-network runs as root, prevent it from opening symbolic links
+ procfile_open_flags = O_RDONLY|O_NOFOLLOW;
+
+ // ------------------------------------------------------------------------
+ // make sure NETDATA_HOST_PREFIX is safe
+
+ netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
+ if(verify_netdata_host_prefix() == -1) exit(1);
+
+ if(netdata_configured_host_prefix[0] != '\0' && verify_path(netdata_configured_host_prefix) == -1)
+ fatal("invalid NETDATA_HOST_PREFIX '%s'", netdata_configured_host_prefix);
+
+ // ------------------------------------------------------------------------
+ // build a safe environment for our script
+
+ // the first environment variable is a fixed PATH=
+ snprintfz(environment_variable2, sizeof(environment_variable2) - 1, "NETDATA_HOST_PREFIX=%s", netdata_configured_host_prefix);
+
+ // ------------------------------------------------------------------------
+
+ if(argc == 2 && (!strcmp(argv[1], "version") || !strcmp(argv[1], "-version") || !strcmp(argv[1], "--version") || !strcmp(argv[1], "-v") || !strcmp(argv[1], "-V"))) {
+ fprintf(stderr, "cgroup-network %s\n", VERSION);
+ exit(0);
+ }
+
+ if(argc != 3)
+ usage();
+
+ int arg = 1;
+ int helper = 1;
+ if (getenv("KUBERNETES_SERVICE_HOST") != NULL && getenv("KUBERNETES_SERVICE_PORT") != NULL)
+ helper = 0;
+
+ if(!strcmp(argv[arg], "-p") || !strcmp(argv[arg], "--pid")) {
+ pid = atoi(argv[arg+1]);
+
+ if(pid <= 0) {
+ errno = 0;
+ error("Invalid pid %d given", (int) pid);
+ return 2;
+ }
+
+ if(helper) call_the_helper(pid, NULL);
+ }
+ else if(!strcmp(argv[arg], "--cgroup")) {
+ char *cgroup = argv[arg+1];
+ if(verify_path(cgroup) == -1) {
+ error("cgroup '%s' does not exist or is not valid.", cgroup);
+ return 1;
+ }
+
+ pid = read_pid_from_cgroup(cgroup);
+ if(helper) call_the_helper(pid, cgroup);
+
+ if(pid <= 0 && !detected_devices) {
+ errno = 0;
+ error("Cannot find a cgroup PID from cgroup '%s'", cgroup);
+ }
+ }
+ else
+ usage();
+
+ if(pid > 0)
+ detect_veth_interfaces(pid);
+
+ int found = send_devices();
+ if(found <= 0) return 1;
+ return 0;
+}
diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c
new file mode 100644
index 0000000..8f75482
--- /dev/null
+++ b/collectors/cgroups.plugin/sys_fs_cgroup.c
@@ -0,0 +1,4887 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "sys_fs_cgroup.h"
+
+#define PLUGIN_CGROUPS_NAME "cgroups.plugin"
+#define PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME "systemd"
+#define PLUGIN_CGROUPS_MODULE_CGROUPS_NAME "/sys/fs/cgroup"
+
+#ifdef NETDATA_INTERNAL_CHECKS
+#define CGROUP_PROCFILE_FLAG PROCFILE_FLAG_DEFAULT
+#else
+#define CGROUP_PROCFILE_FLAG PROCFILE_FLAG_NO_ERROR_ON_FILE_IO
+#endif
+
+// main cgroups thread worker jobs
+#define WORKER_CGROUPS_LOCK 0
+#define WORKER_CGROUPS_READ 1
+#define WORKER_CGROUPS_CHART 2
+
+// discovery cgroup thread worker jobs
+#define WORKER_DISCOVERY_INIT 0
+#define WORKER_DISCOVERY_FIND 1
+#define WORKER_DISCOVERY_PROCESS 2
+#define WORKER_DISCOVERY_PROCESS_RENAME 3
+#define WORKER_DISCOVERY_PROCESS_NETWORK 4
+#define WORKER_DISCOVERY_PROCESS_FIRST_TIME 5
+#define WORKER_DISCOVERY_UPDATE 6
+#define WORKER_DISCOVERY_CLEANUP 7
+#define WORKER_DISCOVERY_COPY 8
+#define WORKER_DISCOVERY_SHARE 9
+#define WORKER_DISCOVERY_LOCK 10
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 11
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 11
+#endif
+
+// ----------------------------------------------------------------------------
+// cgroup globals
+
+static char cgroup_chart_id_prefix[] = "cgroup_";
+
+static int is_inside_k8s = 0;
+
+static long system_page_size = 4096; // system will be queried via sysconf() in configuration()
+
+static int cgroup_enable_cpuacct_stat = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_cpuacct_usage = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_cpuacct_cpu_throttling = CONFIG_BOOLEAN_YES;
+static int cgroup_enable_cpuacct_cpu_shares = CONFIG_BOOLEAN_NO;
+static int cgroup_enable_memory = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_detailed_memory = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_memory_failcnt = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_swap = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_blkio_io = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_blkio_ops = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_blkio_throttle_io = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_blkio_throttle_ops = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_blkio_merged_ops = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_blkio_queued_ops = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_pressure_cpu = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_pressure_io_some = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_pressure_io_full = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_pressure_memory_some = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_pressure_memory_full = CONFIG_BOOLEAN_AUTO;
+
+static int cgroup_enable_systemd_services = CONFIG_BOOLEAN_YES;
+static int cgroup_enable_systemd_services_detailed_memory = CONFIG_BOOLEAN_NO;
+static int cgroup_used_memory = CONFIG_BOOLEAN_YES;
+
+static int cgroup_use_unified_cgroups = CONFIG_BOOLEAN_NO;
+static int cgroup_unified_exist = CONFIG_BOOLEAN_AUTO;
+
+static int cgroup_search_in_devices = 1;
+
+static int cgroup_check_for_new_every = 10;
+static int cgroup_update_every = 1;
+static int cgroup_containers_chart_priority = NETDATA_CHART_PRIO_CGROUPS_CONTAINERS;
+
+static int cgroup_recheck_zero_blkio_every_iterations = 10;
+static int cgroup_recheck_zero_mem_failcnt_every_iterations = 10;
+static int cgroup_recheck_zero_mem_detailed_every_iterations = 10;
+
+static char *cgroup_cpuacct_base = NULL;
+static char *cgroup_cpuset_base = NULL;
+static char *cgroup_blkio_base = NULL;
+static char *cgroup_memory_base = NULL;
+static char *cgroup_devices_base = NULL;
+static char *cgroup_unified_base = NULL;
+
+static int cgroup_root_count = 0;
+static int cgroup_root_max = 1000;
+static int cgroup_max_depth = 0;
+
+static SIMPLE_PATTERN *enabled_cgroup_paths = NULL;
+static SIMPLE_PATTERN *enabled_cgroup_names = NULL;
+static SIMPLE_PATTERN *search_cgroup_paths = NULL;
+static SIMPLE_PATTERN *enabled_cgroup_renames = NULL;
+static SIMPLE_PATTERN *systemd_services_cgroups = NULL;
+
+static SIMPLE_PATTERN *entrypoint_parent_process_comm = NULL;
+
+static char *cgroups_rename_script = NULL;
+static char *cgroups_network_interface_script = NULL;
+
+static int cgroups_check = 0;
+
+static uint32_t Read_hash = 0;
+static uint32_t Write_hash = 0;
+static uint32_t user_hash = 0;
+static uint32_t system_hash = 0;
+static uint32_t user_usec_hash = 0;
+static uint32_t system_usec_hash = 0;
+static uint32_t nr_periods_hash = 0;
+static uint32_t nr_throttled_hash = 0;
+static uint32_t throttled_time_hash = 0;
+static uint32_t throttled_usec_hash = 0;
+
+enum cgroups_type { CGROUPS_AUTODETECT_FAIL, CGROUPS_V1, CGROUPS_V2 };
+
+enum cgroups_systemd_setting {
+ SYSTEMD_CGROUP_ERR,
+ SYSTEMD_CGROUP_LEGACY,
+ SYSTEMD_CGROUP_HYBRID,
+ SYSTEMD_CGROUP_UNIFIED
+};
+
+struct cgroups_systemd_config_setting {
+ char *name;
+ enum cgroups_systemd_setting setting;
+};
+
+static struct cgroups_systemd_config_setting cgroups_systemd_options[] = {
+ { .name = "legacy", .setting = SYSTEMD_CGROUP_LEGACY },
+ { .name = "hybrid", .setting = SYSTEMD_CGROUP_HYBRID },
+ { .name = "unified", .setting = SYSTEMD_CGROUP_UNIFIED },
+ { .name = NULL, .setting = SYSTEMD_CGROUP_ERR },
+};
+
+// Shared memory with information from detected cgroups
+netdata_ebpf_cgroup_shm_t shm_cgroup_ebpf = {NULL, NULL};
+static int shm_fd_cgroup_ebpf = -1;
+sem_t *shm_mutex_cgroup_ebpf = SEM_FAILED;
+
+/* on Fed systemd is not in PATH for some reason */
+#define SYSTEMD_CMD_RHEL "/usr/lib/systemd/systemd --version"
+#define SYSTEMD_HIERARCHY_STRING "default-hierarchy="
+
+#define MAXSIZE_PROC_CMDLINE 4096
+static enum cgroups_systemd_setting cgroups_detect_systemd(const char *exec)
+{
+ pid_t command_pid;
+ enum cgroups_systemd_setting retval = SYSTEMD_CGROUP_ERR;
+ char buf[MAXSIZE_PROC_CMDLINE];
+ char *begin, *end;
+
+ FILE *fp_child_input;
+ FILE *fp_child_output = netdata_popen(exec, &command_pid, &fp_child_input);
+
+ if (!fp_child_output)
+ return retval;
+
+ fd_set rfds;
+ struct timeval timeout;
+ int fd = fileno(fp_child_output);
+ int ret = -1;
+
+ FD_ZERO(&rfds);
+ FD_SET(fd, &rfds);
+ timeout.tv_sec = 3;
+ timeout.tv_usec = 0;
+
+ if (fd != -1) {
+ ret = select(fd + 1, &rfds, NULL, NULL, &timeout);
+ }
+
+ if (ret == -1) {
+ error("Failed to get the output of \"%s\"", exec);
+ } else if (ret == 0) {
+ info("Cannot get the output of \"%s\" within %"PRId64" seconds", exec, (int64_t)timeout.tv_sec);
+ } else {
+ while (fgets(buf, MAXSIZE_PROC_CMDLINE, fp_child_output) != NULL) {
+ if ((begin = strstr(buf, SYSTEMD_HIERARCHY_STRING))) {
+ end = begin = begin + strlen(SYSTEMD_HIERARCHY_STRING);
+ if (!*begin)
+ break;
+ while (isalpha(*end))
+ end++;
+ *end = 0;
+ for (int i = 0; cgroups_systemd_options[i].name; i++) {
+ if (!strcmp(begin, cgroups_systemd_options[i].name)) {
+ retval = cgroups_systemd_options[i].setting;
+ break;
+ }
+ }
+ break;
+ }
+ }
+ }
+
+ if (netdata_pclose(fp_child_input, fp_child_output, command_pid))
+ return SYSTEMD_CGROUP_ERR;
+
+ return retval;
+}
+
+static enum cgroups_type cgroups_try_detect_version()
+{
+ pid_t command_pid;
+ char buf[MAXSIZE_PROC_CMDLINE];
+ enum cgroups_systemd_setting systemd_setting;
+ int cgroups2_available = 0;
+
+ // 1. check if cgroups2 available on system at all
+ FILE *fp_child_input;
+ FILE *fp_child_output = netdata_popen("grep cgroup /proc/filesystems", &command_pid, &fp_child_input);
+ if (!fp_child_output) {
+ error("popen failed");
+ return CGROUPS_AUTODETECT_FAIL;
+ }
+ while (fgets(buf, MAXSIZE_PROC_CMDLINE, fp_child_output) != NULL) {
+ if (strstr(buf, "cgroup2")) {
+ cgroups2_available = 1;
+ break;
+ }
+ }
+ if(netdata_pclose(fp_child_input, fp_child_output, command_pid))
+ return CGROUPS_AUTODETECT_FAIL;
+
+ if(!cgroups2_available)
+ return CGROUPS_V1;
+
+#if defined CGROUP2_SUPER_MAGIC
+ // 2. check filesystem type for the default mountpoint
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/fs/cgroup");
+ struct statfs fsinfo;
+ if (!statfs(filename, &fsinfo)) {
+ if (fsinfo.f_type == CGROUP2_SUPER_MAGIC)
+ return CGROUPS_V2;
+ }
+#endif
+
+ // 3. check systemd compiletime setting
+ if ((systemd_setting = cgroups_detect_systemd("systemd --version")) == SYSTEMD_CGROUP_ERR)
+ systemd_setting = cgroups_detect_systemd(SYSTEMD_CMD_RHEL);
+
+ if(systemd_setting == SYSTEMD_CGROUP_ERR)
+ return CGROUPS_AUTODETECT_FAIL;
+
+ if(systemd_setting == SYSTEMD_CGROUP_LEGACY || systemd_setting == SYSTEMD_CGROUP_HYBRID) {
+ // currently we prefer V1 if HYBRID is set as it seems to be more feature complete
+ // in the future we might want to continue here if SYSTEMD_CGROUP_HYBRID
+ // and go ahead with V2
+ return CGROUPS_V1;
+ }
+
+ // 4. if we are unified as on Fedora (default cgroups2 only mode)
+ // check kernel command line flag that can override that setting
+ FILE *fp = fopen("/proc/cmdline", "r");
+ if (!fp) {
+ error("Error reading kernel boot commandline parameters");
+ return CGROUPS_AUTODETECT_FAIL;
+ }
+
+ if (!fgets(buf, MAXSIZE_PROC_CMDLINE, fp)) {
+ error("couldn't read all cmdline params into buffer");
+ fclose(fp);
+ return CGROUPS_AUTODETECT_FAIL;
+ }
+
+ fclose(fp);
+
+ if (strstr(buf, "systemd.unified_cgroup_hierarchy=0")) {
+ info("cgroups v2 (unified cgroups) is available but are disabled on this system.");
+ return CGROUPS_V1;
+ }
+ return CGROUPS_V2;
+}
+
+void set_cgroup_base_path(char *filename, char *path) {
+ if (strncmp(netdata_configured_host_prefix, path, strlen(netdata_configured_host_prefix)) == 0) {
+ snprintfz(filename, FILENAME_MAX, "%s", path);
+ } else {
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, path);
+ }
+}
+
+void read_cgroup_plugin_configuration() {
+ system_page_size = sysconf(_SC_PAGESIZE);
+
+ Read_hash = simple_hash("Read");
+ Write_hash = simple_hash("Write");
+ user_hash = simple_hash("user");
+ system_hash = simple_hash("system");
+ user_usec_hash = simple_hash("user_usec");
+ system_usec_hash = simple_hash("system_usec");
+ nr_periods_hash = simple_hash("nr_periods");
+ nr_throttled_hash = simple_hash("nr_throttled");
+ throttled_time_hash = simple_hash("throttled_time");
+ throttled_usec_hash = simple_hash("throttled_usec");
+
+ cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", localhost->rrd_update_every);
+ if(cgroup_update_every < localhost->rrd_update_every)
+ cgroup_update_every = localhost->rrd_update_every;
+
+ cgroup_check_for_new_every = (int)config_get_number("plugin:cgroups", "check for new cgroups every", (long long)cgroup_check_for_new_every * (long long)cgroup_update_every);
+ if(cgroup_check_for_new_every < cgroup_update_every)
+ cgroup_check_for_new_every = cgroup_update_every;
+
+ cgroup_use_unified_cgroups = config_get_boolean_ondemand("plugin:cgroups", "use unified cgroups", CONFIG_BOOLEAN_AUTO);
+ if(cgroup_use_unified_cgroups == CONFIG_BOOLEAN_AUTO)
+ cgroup_use_unified_cgroups = (cgroups_try_detect_version() == CGROUPS_V2);
+
+ info("use unified cgroups %s", cgroup_use_unified_cgroups ? "true" : "false");
+
+ cgroup_containers_chart_priority = (int)config_get_number("plugin:cgroups", "containers priority", cgroup_containers_chart_priority);
+ if(cgroup_containers_chart_priority < 1)
+ cgroup_containers_chart_priority = NETDATA_CHART_PRIO_CGROUPS_CONTAINERS;
+
+ cgroup_enable_cpuacct_stat = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct stat (total CPU)", cgroup_enable_cpuacct_stat);
+ cgroup_enable_cpuacct_usage = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct usage (per core CPU)", cgroup_enable_cpuacct_usage);
+ cgroup_enable_cpuacct_cpu_throttling = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct cpu throttling", cgroup_enable_cpuacct_cpu_throttling);
+ cgroup_enable_cpuacct_cpu_shares = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct cpu shares", cgroup_enable_cpuacct_cpu_shares);
+
+ cgroup_enable_memory = config_get_boolean_ondemand("plugin:cgroups", "enable memory", cgroup_enable_memory);
+ cgroup_enable_detailed_memory = config_get_boolean_ondemand("plugin:cgroups", "enable detailed memory", cgroup_enable_detailed_memory);
+ cgroup_enable_memory_failcnt = config_get_boolean_ondemand("plugin:cgroups", "enable memory limits fail count", cgroup_enable_memory_failcnt);
+ cgroup_enable_swap = config_get_boolean_ondemand("plugin:cgroups", "enable swap memory", cgroup_enable_swap);
+
+ cgroup_enable_blkio_io = config_get_boolean_ondemand("plugin:cgroups", "enable blkio bandwidth", cgroup_enable_blkio_io);
+ cgroup_enable_blkio_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio operations", cgroup_enable_blkio_ops);
+ cgroup_enable_blkio_throttle_io = config_get_boolean_ondemand("plugin:cgroups", "enable blkio throttle bandwidth", cgroup_enable_blkio_throttle_io);
+ cgroup_enable_blkio_throttle_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio throttle operations", cgroup_enable_blkio_throttle_ops);
+ cgroup_enable_blkio_queued_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio queued operations", cgroup_enable_blkio_queued_ops);
+ cgroup_enable_blkio_merged_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio merged operations", cgroup_enable_blkio_merged_ops);
+
+ cgroup_enable_pressure_cpu = config_get_boolean_ondemand("plugin:cgroups", "enable cpu pressure", cgroup_enable_pressure_cpu);
+ cgroup_enable_pressure_io_some = config_get_boolean_ondemand("plugin:cgroups", "enable io some pressure", cgroup_enable_pressure_io_some);
+ cgroup_enable_pressure_io_full = config_get_boolean_ondemand("plugin:cgroups", "enable io full pressure", cgroup_enable_pressure_io_full);
+ cgroup_enable_pressure_memory_some = config_get_boolean_ondemand("plugin:cgroups", "enable memory some pressure", cgroup_enable_pressure_memory_some);
+ cgroup_enable_pressure_memory_full = config_get_boolean_ondemand("plugin:cgroups", "enable memory full pressure", cgroup_enable_pressure_memory_full);
+
+ cgroup_recheck_zero_blkio_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero blkio every iterations", cgroup_recheck_zero_blkio_every_iterations);
+ cgroup_recheck_zero_mem_failcnt_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero memory failcnt every iterations", cgroup_recheck_zero_mem_failcnt_every_iterations);
+ cgroup_recheck_zero_mem_detailed_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero detailed memory every iterations", cgroup_recheck_zero_mem_detailed_every_iterations);
+
+ cgroup_enable_systemd_services = config_get_boolean("plugin:cgroups", "enable systemd services", cgroup_enable_systemd_services);
+ cgroup_enable_systemd_services_detailed_memory = config_get_boolean("plugin:cgroups", "enable systemd services detailed memory", cgroup_enable_systemd_services_detailed_memory);
+ cgroup_used_memory = config_get_boolean("plugin:cgroups", "report used memory", cgroup_used_memory);
+
+ char filename[FILENAME_MAX + 1], *s;
+ struct mountinfo *mi, *root = mountinfo_read(0);
+ if(!cgroup_use_unified_cgroups) {
+ // cgroup v1 does not have pressure metrics
+ cgroup_enable_pressure_cpu =
+ cgroup_enable_pressure_io_some =
+ cgroup_enable_pressure_io_full =
+ cgroup_enable_pressure_memory_some =
+ cgroup_enable_pressure_memory_full = CONFIG_BOOLEAN_NO;
+
+ mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "cpuacct");
+ if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "cpuacct");
+ if(!mi) {
+ error("CGROUP: cannot find cpuacct mountinfo. Assuming default: /sys/fs/cgroup/cpuacct");
+ s = "/sys/fs/cgroup/cpuacct";
+ }
+ else s = mi->mount_point;
+ set_cgroup_base_path(filename, s);
+ cgroup_cpuacct_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/cpuacct", filename);
+
+ mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "cpuset");
+ if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "cpuset");
+ if(!mi) {
+ error("CGROUP: cannot find cpuset mountinfo. Assuming default: /sys/fs/cgroup/cpuset");
+ s = "/sys/fs/cgroup/cpuset";
+ }
+ else s = mi->mount_point;
+ set_cgroup_base_path(filename, s);
+ cgroup_cpuset_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/cpuset", filename);
+
+ mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "blkio");
+ if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "blkio");
+ if(!mi) {
+ error("CGROUP: cannot find blkio mountinfo. Assuming default: /sys/fs/cgroup/blkio");
+ s = "/sys/fs/cgroup/blkio";
+ }
+ else s = mi->mount_point;
+ set_cgroup_base_path(filename, s);
+ cgroup_blkio_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/blkio", filename);
+
+ mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "memory");
+ if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "memory");
+ if(!mi) {
+ error("CGROUP: cannot find memory mountinfo. Assuming default: /sys/fs/cgroup/memory");
+ s = "/sys/fs/cgroup/memory";
+ }
+ else s = mi->mount_point;
+ set_cgroup_base_path(filename, s);
+ cgroup_memory_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/memory", filename);
+
+ mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "devices");
+ if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "devices");
+ if(!mi) {
+ error("CGROUP: cannot find devices mountinfo. Assuming default: /sys/fs/cgroup/devices");
+ s = "/sys/fs/cgroup/devices";
+ }
+ else s = mi->mount_point;
+ set_cgroup_base_path(filename, s);
+ cgroup_devices_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/devices", filename);
+ }
+ else {
+ //cgroup_enable_cpuacct_stat =
+ cgroup_enable_cpuacct_usage =
+ //cgroup_enable_memory =
+ //cgroup_enable_detailed_memory =
+ cgroup_enable_memory_failcnt =
+ //cgroup_enable_swap =
+ //cgroup_enable_blkio_io =
+ //cgroup_enable_blkio_ops =
+ cgroup_enable_blkio_throttle_io =
+ cgroup_enable_blkio_throttle_ops =
+ cgroup_enable_blkio_merged_ops =
+ cgroup_enable_blkio_queued_ops = CONFIG_BOOLEAN_NO;
+ cgroup_search_in_devices = 0;
+ cgroup_enable_systemd_services_detailed_memory = CONFIG_BOOLEAN_NO;
+ cgroup_used_memory = CONFIG_BOOLEAN_NO; //unified cgroups use different values
+
+ //TODO: can there be more than 1 cgroup2 mount point?
+ mi = mountinfo_find_by_filesystem_super_option(root, "cgroup2", "rw"); //there is no cgroup2 specific super option - for now use 'rw' option
+ if(mi) debug(D_CGROUP, "found unified cgroup root using super options, with path: '%s'", mi->mount_point);
+ if(!mi) {
+ mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup2", "cgroup");
+ if(mi) debug(D_CGROUP, "found unified cgroup root using mountsource info, with path: '%s'", mi->mount_point);
+ }
+ if(!mi) {
+ error("CGROUP: cannot find cgroup2 mountinfo. Assuming default: /sys/fs/cgroup");
+ s = "/sys/fs/cgroup";
+ }
+ else s = mi->mount_point;
+ set_cgroup_base_path(filename, s);
+ cgroup_unified_base = config_get("plugin:cgroups", "path to unified cgroups", filename);
+ debug(D_CGROUP, "using cgroup root: '%s'", cgroup_unified_base);
+ }
+
+ cgroup_root_max = (int)config_get_number("plugin:cgroups", "max cgroups to allow", cgroup_root_max);
+ cgroup_max_depth = (int)config_get_number("plugin:cgroups", "max cgroups depth to monitor", cgroup_max_depth);
+
+ enabled_cgroup_paths = simple_pattern_create(
+ config_get("plugin:cgroups", "enable by default cgroups matching",
+ // ----------------------------------------------------------------
+
+ " !*/init.scope " // ignore init.scope
+ " !/system.slice/run-*.scope " // ignore system.slice/run-XXXX.scope
+ " *.scope " // we need all other *.scope for sure
+
+ // ----------------------------------------------------------------
+
+ " /machine.slice/*.service " // #3367 systemd-nspawn
+
+ // ----------------------------------------------------------------
+
+ " */kubepods/pod*/* " // k8s containers
+ " */kubepods/*/pod*/* " // k8s containers
+ " */*-kubepods-pod*/* " // k8s containers
+ " */*-kubepods-*-pod*/* " // k8s containers
+ " !*kubepods* !*kubelet* " // all other k8s cgroups
+
+ // ----------------------------------------------------------------
+
+ " !*/vcpu* " // libvirtd adds these sub-cgroups
+ " !*/emulator " // libvirtd adds these sub-cgroups
+ " !*.mount "
+ " !*.partition "
+ " !*.service "
+ " !*.socket "
+ " !*.slice "
+ " !*.swap "
+ " !*.user "
+ " !/ "
+ " !/docker "
+ " !*/libvirt "
+ " !/lxc "
+ " !/lxc/*/* " // #1397 #2649
+ " !/lxc.monitor* "
+ " !/lxc.pivot "
+ " !/lxc.payload "
+ " !/machine "
+ " !/qemu "
+ " !/system "
+ " !/systemd "
+ " !/user "
+ " * " // enable anything else
+ ), NULL, SIMPLE_PATTERN_EXACT);
+
+ enabled_cgroup_names = simple_pattern_create(
+ config_get("plugin:cgroups", "enable by default cgroups names matching",
+ " * "
+ ), NULL, SIMPLE_PATTERN_EXACT);
+
+ search_cgroup_paths = simple_pattern_create(
+ config_get("plugin:cgroups", "search for cgroups in subpaths matching",
+ " !*/init.scope " // ignore init.scope
+ " !*-qemu " // #345
+ " !*.libvirt-qemu " // #3010
+ " !/init.scope "
+ " !/system "
+ " !/systemd "
+ " !/user "
+ " !/user.slice "
+ " !/lxc/*/* " // #2161 #2649
+ " !/lxc.monitor "
+ " !/lxc.payload/*/* "
+ " !/lxc.payload.* "
+ " * "
+ ), NULL, SIMPLE_PATTERN_EXACT);
+
+ snprintfz(filename, FILENAME_MAX, "%s/cgroup-name.sh", netdata_configured_primary_plugins_dir);
+ cgroups_rename_script = config_get("plugin:cgroups", "script to get cgroup names", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s/cgroup-network", netdata_configured_primary_plugins_dir);
+ cgroups_network_interface_script = config_get("plugin:cgroups", "script to get cgroup network interfaces", filename);
+
+ enabled_cgroup_renames = simple_pattern_create(
+ config_get("plugin:cgroups", "run script to rename cgroups matching",
+ " !/ "
+ " !*.mount "
+ " !*.socket "
+ " !*.partition "
+ " /machine.slice/*.service " // #3367 systemd-nspawn
+ " !*.service "
+ " !*.slice "
+ " !*.swap "
+ " !*.user "
+ " !init.scope "
+ " !*.scope/vcpu* " // libvirtd adds these sub-cgroups
+ " !*.scope/emulator " // libvirtd adds these sub-cgroups
+ " *.scope "
+ " *docker* "
+ " *lxc* "
+ " *qemu* "
+ " */kubepods/pod*/* " // k8s containers
+ " */kubepods/*/pod*/* " // k8s containers
+ " */*-kubepods-pod*/* " // k8s containers
+ " */*-kubepods-*-pod*/* " // k8s containers
+ " !*kubepods* !*kubelet* " // all other k8s cgroups
+ " *.libvirt-qemu " // #3010
+ " * "
+ ), NULL, SIMPLE_PATTERN_EXACT);
+
+ if(cgroup_enable_systemd_services) {
+ systemd_services_cgroups = simple_pattern_create(
+ config_get("plugin:cgroups", "cgroups to match as systemd services",
+ " !/system.slice/*/*.service "
+ " /system.slice/*.service "
+ ), NULL, SIMPLE_PATTERN_EXACT);
+ }
+
+ mountinfo_free_all(root);
+}
+
+void netdata_cgroup_ebpf_set_values(size_t length)
+{
+ sem_wait(shm_mutex_cgroup_ebpf);
+
+ shm_cgroup_ebpf.header->cgroup_max = cgroup_root_max;
+ shm_cgroup_ebpf.header->systemd_enabled = cgroup_enable_systemd_services |
+ cgroup_enable_systemd_services_detailed_memory |
+ cgroup_used_memory;
+ shm_cgroup_ebpf.header->body_length = length;
+
+ sem_post(shm_mutex_cgroup_ebpf);
+}
+
+void netdata_cgroup_ebpf_initialize_shm()
+{
+ shm_fd_cgroup_ebpf = shm_open(NETDATA_SHARED_MEMORY_EBPF_CGROUP_NAME, O_CREAT | O_RDWR, 0660);
+ if (shm_fd_cgroup_ebpf < 0) {
+ error("Cannot initialize shared memory used by cgroup and eBPF, integration won't happen.");
+ return;
+ }
+
+ size_t length = sizeof(netdata_ebpf_cgroup_shm_header_t) + cgroup_root_max * sizeof(netdata_ebpf_cgroup_shm_body_t);
+ if (ftruncate(shm_fd_cgroup_ebpf, length)) {
+ error("Cannot set size for shared memory.");
+ goto end_init_shm;
+ }
+
+ shm_cgroup_ebpf.header = (netdata_ebpf_cgroup_shm_header_t *) mmap(NULL, length,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ shm_fd_cgroup_ebpf, 0);
+
+ if (!shm_cgroup_ebpf.header) {
+ error("Cannot map shared memory used between cgroup and eBPF, integration won't happen");
+ goto end_init_shm;
+ }
+ shm_cgroup_ebpf.body = (netdata_ebpf_cgroup_shm_body_t *) ((char *)shm_cgroup_ebpf.header +
+ sizeof(netdata_ebpf_cgroup_shm_header_t));
+
+ shm_mutex_cgroup_ebpf = sem_open(NETDATA_NAMED_SEMAPHORE_EBPF_CGROUP_NAME, O_CREAT,
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, 1);
+
+ if (shm_mutex_cgroup_ebpf != SEM_FAILED) {
+ netdata_cgroup_ebpf_set_values(length);
+ return;
+ }
+
+ error("Cannot create semaphore, integration between eBPF and cgroup won't happen");
+ munmap(shm_cgroup_ebpf.header, length);
+
+end_init_shm:
+ close(shm_fd_cgroup_ebpf);
+ shm_fd_cgroup_ebpf = -1;
+ shm_unlink(NETDATA_SHARED_MEMORY_EBPF_CGROUP_NAME);
+}
+
+// ----------------------------------------------------------------------------
+// cgroup objects
+
+struct blkio {
+ int updated;
+ int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+ int delay_counter;
+
+ char *filename;
+
+ unsigned long long Read;
+ unsigned long long Write;
+/*
+ unsigned long long Sync;
+ unsigned long long Async;
+ unsigned long long Total;
+*/
+};
+
+// https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
+struct memory {
+ ARL_BASE *arl_base;
+ ARL_ENTRY *arl_dirty;
+ ARL_ENTRY *arl_swap;
+
+ int updated_detailed;
+ int updated_usage_in_bytes;
+ int updated_msw_usage_in_bytes;
+ int updated_failcnt;
+
+ int enabled_detailed; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+ int enabled_usage_in_bytes; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+ int enabled_msw_usage_in_bytes; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+ int enabled_failcnt; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+
+ int delay_counter_detailed;
+ int delay_counter_failcnt;
+
+ char *filename_detailed;
+ char *filename_usage_in_bytes;
+ char *filename_msw_usage_in_bytes;
+ char *filename_failcnt;
+
+ int detailed_has_dirty;
+ int detailed_has_swap;
+
+ // detailed metrics
+/*
+ unsigned long long cache;
+ unsigned long long rss;
+ unsigned long long rss_huge;
+ unsigned long long mapped_file;
+ unsigned long long writeback;
+ unsigned long long dirty;
+ unsigned long long swap;
+ unsigned long long pgpgin;
+ unsigned long long pgpgout;
+ unsigned long long pgfault;
+ unsigned long long pgmajfault;
+ unsigned long long inactive_anon;
+ unsigned long long active_anon;
+ unsigned long long inactive_file;
+ unsigned long long active_file;
+ unsigned long long unevictable;
+ unsigned long long hierarchical_memory_limit;
+*/
+ //unified cgroups metrics
+ unsigned long long anon;
+ unsigned long long kernel_stack;
+ unsigned long long slab;
+ unsigned long long sock;
+ unsigned long long shmem;
+ unsigned long long anon_thp;
+ //unsigned long long file_writeback;
+ //unsigned long long file_dirty;
+ //unsigned long long file;
+
+ unsigned long long total_cache;
+ unsigned long long total_rss;
+ unsigned long long total_rss_huge;
+ unsigned long long total_mapped_file;
+ unsigned long long total_writeback;
+ unsigned long long total_dirty;
+ unsigned long long total_swap;
+ unsigned long long total_pgpgin;
+ unsigned long long total_pgpgout;
+ unsigned long long total_pgfault;
+ unsigned long long total_pgmajfault;
+/*
+ unsigned long long total_inactive_anon;
+ unsigned long long total_active_anon;
+*/
+
+ unsigned long long total_inactive_file;
+
+/*
+ unsigned long long total_active_file;
+ unsigned long long total_unevictable;
+*/
+
+ // single file metrics
+ unsigned long long usage_in_bytes;
+ unsigned long long msw_usage_in_bytes;
+ unsigned long long failcnt;
+};
+
+// https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
+struct cpuacct_stat {
+ int updated;
+ int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+
+ char *filename;
+
+ unsigned long long user; // v1, v2(user_usec)
+ unsigned long long system; // v1, v2(system_usec)
+};
+
+// https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
+struct cpuacct_usage {
+ int updated;
+ int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+
+ char *filename;
+
+ unsigned int cpus;
+ unsigned long long *cpu_percpu;
+};
+
+// represents cpuacct/cpu.stat, for v2 'cpuacct_stat' is used for 'user_usec', 'system_usec'
+struct cpuacct_cpu_throttling {
+ int updated;
+ int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+
+ char *filename;
+
+ unsigned long long nr_periods;
+ unsigned long long nr_throttled;
+ unsigned long long throttled_time;
+
+ unsigned long long nr_throttled_perc;
+};
+
+// https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/sec-cpu#sect-cfs
+// https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/managing_monitoring_and_updating_the_kernel/using-cgroups-v2-to-control-distribution-of-cpu-time-for-applications_managing-monitoring-and-updating-the-kernel#proc_controlling-distribution-of-cpu-time-for-applications-by-adjusting-cpu-weight_using-cgroups-v2-to-control-distribution-of-cpu-time-for-applications
+struct cpuacct_cpu_shares {
+ int updated;
+ int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+
+ char *filename;
+
+ unsigned long long shares;
+};
+
+struct cgroup_network_interface {
+ const char *host_device;
+ const char *container_device;
+ struct cgroup_network_interface *next;
+};
+
+enum cgroups_container_orchestrator {
+ CGROUPS_ORCHESTRATOR_UNSET,
+ CGROUPS_ORCHESTRATOR_UNKNOWN,
+ CGROUPS_ORCHESTRATOR_K8S
+};
+
+// *** WARNING *** The fields are not thread safe. Take care of safe usage.
+struct cgroup {
+ uint32_t options;
+
+ int first_time_seen; // first time seen by the discoverer
+ int processed; // the discoverer is done processing a cgroup (resolved name, set 'enabled' option)
+
+ char available; // found in the filesystem
+ char enabled; // enabled in the config
+
+ char pending_renames;
+ char *intermediate_id; // TODO: remove it when the renaming script is fixed
+
+ char *id;
+ uint32_t hash;
+
+ char *chart_id;
+ uint32_t hash_chart;
+
+ char *chart_title;
+
+ DICTIONARY *chart_labels;
+
+ int container_orchestrator;
+
+ struct cpuacct_stat cpuacct_stat;
+ struct cpuacct_usage cpuacct_usage;
+ struct cpuacct_cpu_throttling cpuacct_cpu_throttling;
+ struct cpuacct_cpu_shares cpuacct_cpu_shares;
+
+ struct memory memory;
+
+ struct blkio io_service_bytes; // bytes
+ struct blkio io_serviced; // operations
+
+ struct blkio throttle_io_service_bytes; // bytes
+ struct blkio throttle_io_serviced; // operations
+
+ struct blkio io_merged; // operations
+ struct blkio io_queued; // operations
+
+ struct cgroup_network_interface *interfaces;
+
+ struct pressure cpu_pressure;
+ struct pressure io_pressure;
+ struct pressure memory_pressure;
+
+ // per cgroup charts
+ RRDSET *st_cpu;
+ RRDSET *st_cpu_limit;
+ RRDSET *st_cpu_per_core;
+ RRDSET *st_cpu_nr_throttled;
+ RRDSET *st_cpu_throttled_time;
+ RRDSET *st_cpu_shares;
+
+ RRDSET *st_mem;
+ RRDSET *st_mem_utilization;
+ RRDSET *st_writeback;
+ RRDSET *st_mem_activity;
+ RRDSET *st_pgfaults;
+ RRDSET *st_mem_usage;
+ RRDSET *st_mem_usage_limit;
+ RRDSET *st_mem_failcnt;
+
+ RRDSET *st_io;
+ RRDSET *st_serviced_ops;
+ RRDSET *st_throttle_io;
+ RRDSET *st_throttle_serviced_ops;
+ RRDSET *st_queued_ops;
+ RRDSET *st_merged_ops;
+
+ // per cgroup chart variables
+ char *filename_cpuset_cpus;
+ unsigned long long cpuset_cpus;
+
+ char *filename_cpu_cfs_period;
+ unsigned long long cpu_cfs_period;
+
+ char *filename_cpu_cfs_quota;
+ unsigned long long cpu_cfs_quota;
+
+ const RRDSETVAR_ACQUIRED *chart_var_cpu_limit;
+ NETDATA_DOUBLE prev_cpu_usage;
+
+ char *filename_memory_limit;
+ unsigned long long memory_limit;
+ const RRDSETVAR_ACQUIRED *chart_var_memory_limit;
+
+ char *filename_memoryswap_limit;
+ unsigned long long memoryswap_limit;
+ const RRDSETVAR_ACQUIRED *chart_var_memoryswap_limit;
+
+ // services
+ RRDDIM *rd_cpu;
+ RRDDIM *rd_mem_usage;
+ RRDDIM *rd_mem_failcnt;
+ RRDDIM *rd_swap_usage;
+
+ RRDDIM *rd_mem_detailed_cache;
+ RRDDIM *rd_mem_detailed_rss;
+ RRDDIM *rd_mem_detailed_mapped;
+ RRDDIM *rd_mem_detailed_writeback;
+ RRDDIM *rd_mem_detailed_pgpgin;
+ RRDDIM *rd_mem_detailed_pgpgout;
+ RRDDIM *rd_mem_detailed_pgfault;
+ RRDDIM *rd_mem_detailed_pgmajfault;
+
+ RRDDIM *rd_io_service_bytes_read;
+ RRDDIM *rd_io_serviced_read;
+ RRDDIM *rd_throttle_io_read;
+ RRDDIM *rd_throttle_io_serviced_read;
+ RRDDIM *rd_io_queued_read;
+ RRDDIM *rd_io_merged_read;
+
+ RRDDIM *rd_io_service_bytes_write;
+ RRDDIM *rd_io_serviced_write;
+ RRDDIM *rd_throttle_io_write;
+ RRDDIM *rd_throttle_io_serviced_write;
+ RRDDIM *rd_io_queued_write;
+ RRDDIM *rd_io_merged_write;
+
+ struct cgroup *next;
+ struct cgroup *discovered_next;
+
+} *cgroup_root = NULL;
+
+uv_mutex_t cgroup_root_mutex;
+
+struct cgroup *discovered_cgroup_root = NULL;
+
+struct discovery_thread {
+ uv_thread_t thread;
+ uv_mutex_t mutex;
+ uv_cond_t cond_var;
+ int start_discovery;
+ int exited;
+} discovery_thread;
+
+// ---------------------------------------------------------------------------------------------
+
+static inline int matches_enabled_cgroup_paths(char *id) {
+ return simple_pattern_matches(enabled_cgroup_paths, id);
+}
+
+static inline int matches_enabled_cgroup_names(char *name) {
+ return simple_pattern_matches(enabled_cgroup_names, name);
+}
+
+static inline int matches_enabled_cgroup_renames(char *id) {
+ return simple_pattern_matches(enabled_cgroup_renames, id);
+}
+
+static inline int matches_systemd_services_cgroups(char *id) {
+ return simple_pattern_matches(systemd_services_cgroups, id);
+}
+
+static inline int matches_search_cgroup_paths(const char *dir) {
+ return simple_pattern_matches(search_cgroup_paths, dir);
+}
+
+static inline int matches_entrypoint_parent_process_comm(const char *comm) {
+ return simple_pattern_matches(entrypoint_parent_process_comm, comm);
+}
+
+static inline int is_cgroup_systemd_service(struct cgroup *cg) {
+ return (cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE);
+}
+
+// ---------------------------------------------------------------------------------------------
+static int k8s_is_kubepod(struct cgroup *cg) {
+ return cg->container_orchestrator == CGROUPS_ORCHESTRATOR_K8S;
+}
+
+static int k8s_is_container(const char *id) {
+ // examples:
+ // https://github.com/netdata/netdata/blob/0fc101679dcd12f1cb8acdd07bb4c85d8e553e53/collectors/cgroups.plugin/cgroup-name.sh#L121-L147
+ const char *p = id;
+ const char *pp = NULL;
+ int i = 0;
+ size_t l = 3; // pod
+ while ((p = strstr(p, "pod"))) {
+ i++;
+ p += l;
+ pp = p;
+ }
+ return !(i < 2 || !pp || !(pp = strchr(pp, '/')) || !pp++ || !*pp);
+}
+
+#define TASK_COMM_LEN 16
+
+static int k8s_get_container_first_proc_comm(const char *id, char *comm) {
+ if (!k8s_is_container(id)) {
+ return 1;
+ }
+
+ static procfile *ff = NULL;
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/%s/cgroup.procs", cgroup_cpuacct_base, id);
+
+ ff = procfile_reopen(ff, filename, NULL, CGROUP_PROCFILE_FLAG);
+ if (unlikely(!ff)) {
+ debug(D_CGROUP, "CGROUP: k8s_is_pause_container(): cannot open file '%s'.", filename);
+ return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if (unlikely(!ff)) {
+ debug(D_CGROUP, "CGROUP: k8s_is_pause_container(): cannot read file '%s'.", filename);
+ return 1;
+ }
+
+ unsigned long lines = procfile_lines(ff);
+ if (likely(lines < 2)) {
+ return 1;
+ }
+
+ char *pid = procfile_lineword(ff, 0, 0);
+ if (!pid || !*pid) {
+ return 1;
+ }
+
+ snprintfz(filename, FILENAME_MAX, "%s/proc/%s/comm", netdata_configured_host_prefix, pid);
+
+ ff = procfile_reopen(ff, filename, NULL, PROCFILE_FLAG_DEFAULT);
+ if (unlikely(!ff)) {
+ debug(D_CGROUP, "CGROUP: k8s_is_pause_container(): cannot open file '%s'.", filename);
+ return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if (unlikely(!ff)) {
+ debug(D_CGROUP, "CGROUP: k8s_is_pause_container(): cannot read file '%s'.", filename);
+ return 1;
+ }
+
+ lines = procfile_lines(ff);
+ if (unlikely(lines != 2)) {
+ return 1;
+ }
+
+ char *proc_comm = procfile_lineword(ff, 0, 0);
+ if (!proc_comm || !*proc_comm) {
+ return 1;
+ }
+
+ strncpyz(comm, proc_comm, TASK_COMM_LEN);
+ return 0;
+}
+
+// ---------------------------------------------------------------------------------------------
+
+static unsigned long long calc_delta(unsigned long long curr, unsigned long long prev) {
+ if (prev > curr) {
+ return 0;
+ }
+ return curr - prev;
+}
+
+static unsigned long long calc_percentage(unsigned long long value, unsigned long long total) {
+ if (total == 0) {
+ return 0;
+ }
+ return (NETDATA_DOUBLE)value / (NETDATA_DOUBLE)total * 100;
+}
+
+static int calc_cgroup_depth(const char *id) {
+ int depth = 0;
+ const char *s;
+ for (s = id; *s; s++) {
+ depth += unlikely(*s == '/');
+ }
+ return depth;
+}
+
+// ----------------------------------------------------------------------------
+// read values from /sys
+
+static inline void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) {
+ static procfile *ff = NULL;
+
+ if(likely(cp->filename)) {
+ ff = procfile_reopen(ff, cp->filename, NULL, CGROUP_PROCFILE_FLAG);
+ if(unlikely(!ff)) {
+ cp->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ cp->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ unsigned long i, lines = procfile_lines(ff);
+
+ if(unlikely(lines < 1)) {
+ error("CGROUP: file '%s' should have 1+ lines.", cp->filename);
+ cp->updated = 0;
+ return;
+ }
+
+ for(i = 0; i < lines ; i++) {
+ char *s = procfile_lineword(ff, i, 0);
+ uint32_t hash = simple_hash(s);
+
+ if(unlikely(hash == user_hash && !strcmp(s, "user")))
+ cp->user = str2ull(procfile_lineword(ff, i, 1));
+
+ else if(unlikely(hash == system_hash && !strcmp(s, "system")))
+ cp->system = str2ull(procfile_lineword(ff, i, 1));
+ }
+
+ cp->updated = 1;
+
+ if(unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO &&
+ (cp->user || cp->system || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ cp->enabled = CONFIG_BOOLEAN_YES;
+ }
+}
+
+static inline void cgroup_read_cpuacct_cpu_stat(struct cpuacct_cpu_throttling *cp) {
+ if (unlikely(!cp->filename)) {
+ return;
+ }
+
+ static procfile *ff = NULL;
+ ff = procfile_reopen(ff, cp->filename, NULL, CGROUP_PROCFILE_FLAG);
+ if (unlikely(!ff)) {
+ cp->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ ff = procfile_readall(ff);
+ if (unlikely(!ff)) {
+ cp->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ unsigned long lines = procfile_lines(ff);
+ if (unlikely(lines < 3)) {
+ error("CGROUP: file '%s' should have 3 lines.", cp->filename);
+ cp->updated = 0;
+ return;
+ }
+
+ unsigned long long nr_periods_last = cp->nr_periods;
+ unsigned long long nr_throttled_last = cp->nr_throttled;
+
+ for (unsigned long i = 0; i < lines; i++) {
+ char *s = procfile_lineword(ff, i, 0);
+ uint32_t hash = simple_hash(s);
+
+ if (unlikely(hash == nr_periods_hash && !strcmp(s, "nr_periods"))) {
+ cp->nr_periods = str2ull(procfile_lineword(ff, i, 1));
+ } else if (unlikely(hash == nr_throttled_hash && !strcmp(s, "nr_throttled"))) {
+ cp->nr_throttled = str2ull(procfile_lineword(ff, i, 1));
+ } else if (unlikely(hash == throttled_time_hash && !strcmp(s, "throttled_time"))) {
+ cp->throttled_time = str2ull(procfile_lineword(ff, i, 1));
+ }
+ }
+ cp->nr_throttled_perc =
+ calc_percentage(calc_delta(cp->nr_throttled, nr_throttled_last), calc_delta(cp->nr_periods, nr_periods_last));
+
+ cp->updated = 1;
+
+ if (unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO)) {
+ if (likely(
+ cp->nr_periods || cp->nr_throttled || cp->throttled_time ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
+ cp->enabled = CONFIG_BOOLEAN_YES;
+ }
+ }
+}
+
+static inline void cgroup2_read_cpuacct_cpu_stat(struct cpuacct_stat *cp, struct cpuacct_cpu_throttling *cpt) {
+ static procfile *ff = NULL;
+ if (unlikely(!cp->filename)) {
+ return;
+ }
+
+ ff = procfile_reopen(ff, cp->filename, NULL, CGROUP_PROCFILE_FLAG);
+ if (unlikely(!ff)) {
+ cp->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ ff = procfile_readall(ff);
+ if (unlikely(!ff)) {
+ cp->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ unsigned long lines = procfile_lines(ff);
+
+ if (unlikely(lines < 3)) {
+ error("CGROUP: file '%s' should have at least 3 lines.", cp->filename);
+ cp->updated = 0;
+ return;
+ }
+
+ unsigned long long nr_periods_last = cpt->nr_periods;
+ unsigned long long nr_throttled_last = cpt->nr_throttled;
+
+ for (unsigned long i = 0; i < lines; i++) {
+ char *s = procfile_lineword(ff, i, 0);
+ uint32_t hash = simple_hash(s);
+
+ if (unlikely(hash == user_usec_hash && !strcmp(s, "user_usec"))) {
+ cp->user = str2ull(procfile_lineword(ff, i, 1));
+ } else if (unlikely(hash == system_usec_hash && !strcmp(s, "system_usec"))) {
+ cp->system = str2ull(procfile_lineword(ff, i, 1));
+ } else if (unlikely(hash == nr_periods_hash && !strcmp(s, "nr_periods"))) {
+ cpt->nr_periods = str2ull(procfile_lineword(ff, i, 1));
+ } else if (unlikely(hash == nr_throttled_hash && !strcmp(s, "nr_throttled"))) {
+ cpt->nr_throttled = str2ull(procfile_lineword(ff, i, 1));
+ } else if (unlikely(hash == throttled_usec_hash && !strcmp(s, "throttled_usec"))) {
+ cpt->throttled_time = str2ull(procfile_lineword(ff, i, 1)) * 1000; // usec -> ns
+ }
+ }
+ cpt->nr_throttled_perc =
+ calc_percentage(calc_delta(cpt->nr_throttled, nr_throttled_last), calc_delta(cpt->nr_periods, nr_periods_last));
+
+ cp->updated = 1;
+ cpt->updated = 1;
+
+ if (unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO)) {
+ if (likely(cp->user || cp->system || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
+ cp->enabled = CONFIG_BOOLEAN_YES;
+ }
+ }
+ if (unlikely(cpt->enabled == CONFIG_BOOLEAN_AUTO)) {
+ if (likely(
+ cpt->nr_periods || cpt->nr_throttled || cpt->throttled_time ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
+ cpt->enabled = CONFIG_BOOLEAN_YES;
+ }
+ }
+}
+
+static inline void cgroup_read_cpuacct_cpu_shares(struct cpuacct_cpu_shares *cp) {
+ if (unlikely(!cp->filename)) {
+ return;
+ }
+
+ if (unlikely(read_single_number_file(cp->filename, &cp->shares))) {
+ cp->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ cp->updated = 1;
+ if (unlikely((cp->enabled == CONFIG_BOOLEAN_AUTO)) &&
+ (cp->shares || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
+ cp->enabled = CONFIG_BOOLEAN_YES;
+ }
+}
+
+static inline void cgroup_read_cpuacct_usage(struct cpuacct_usage *ca) {
+ static procfile *ff = NULL;
+
+ if(likely(ca->filename)) {
+ ff = procfile_reopen(ff, ca->filename, NULL, CGROUP_PROCFILE_FLAG);
+ if(unlikely(!ff)) {
+ ca->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ ca->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ if(unlikely(procfile_lines(ff) < 1)) {
+ error("CGROUP: file '%s' should have 1+ lines but has %zu.", ca->filename, procfile_lines(ff));
+ ca->updated = 0;
+ return;
+ }
+
+ unsigned long i = procfile_linewords(ff, 0);
+ if(unlikely(i == 0)) {
+ ca->updated = 0;
+ return;
+ }
+
+ // we may have 1 more CPU reported
+ while(i > 0) {
+ char *s = procfile_lineword(ff, 0, i - 1);
+ if(!*s) i--;
+ else break;
+ }
+
+ if(unlikely(i != ca->cpus)) {
+ freez(ca->cpu_percpu);
+ ca->cpu_percpu = mallocz(sizeof(unsigned long long) * i);
+ ca->cpus = (unsigned int)i;
+ }
+
+ unsigned long long total = 0;
+ for(i = 0; i < ca->cpus ;i++) {
+ unsigned long long n = str2ull(procfile_lineword(ff, 0, i));
+ ca->cpu_percpu[i] = n;
+ total += n;
+ }
+
+ ca->updated = 1;
+
+ if(unlikely(ca->enabled == CONFIG_BOOLEAN_AUTO &&
+ (total || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ ca->enabled = CONFIG_BOOLEAN_YES;
+ }
+}
+
+static inline void cgroup_read_blkio(struct blkio *io) {
+ if(unlikely(io->enabled == CONFIG_BOOLEAN_AUTO && io->delay_counter > 0)) {
+ io->delay_counter--;
+ return;
+ }
+
+ if(likely(io->filename)) {
+ static procfile *ff = NULL;
+
+ ff = procfile_reopen(ff, io->filename, NULL, CGROUP_PROCFILE_FLAG);
+ if(unlikely(!ff)) {
+ io->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ io->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ unsigned long i, lines = procfile_lines(ff);
+
+ if(unlikely(lines < 1)) {
+ error("CGROUP: file '%s' should have 1+ lines.", io->filename);
+ io->updated = 0;
+ return;
+ }
+
+ io->Read = 0;
+ io->Write = 0;
+/*
+ io->Sync = 0;
+ io->Async = 0;
+ io->Total = 0;
+*/
+
+ for(i = 0; i < lines ; i++) {
+ char *s = procfile_lineword(ff, i, 1);
+ uint32_t hash = simple_hash(s);
+
+ if(unlikely(hash == Read_hash && !strcmp(s, "Read")))
+ io->Read += str2ull(procfile_lineword(ff, i, 2));
+
+ else if(unlikely(hash == Write_hash && !strcmp(s, "Write")))
+ io->Write += str2ull(procfile_lineword(ff, i, 2));
+
+/*
+ else if(unlikely(hash == Sync_hash && !strcmp(s, "Sync")))
+ io->Sync += str2ull(procfile_lineword(ff, i, 2));
+
+ else if(unlikely(hash == Async_hash && !strcmp(s, "Async")))
+ io->Async += str2ull(procfile_lineword(ff, i, 2));
+
+ else if(unlikely(hash == Total_hash && !strcmp(s, "Total")))
+ io->Total += str2ull(procfile_lineword(ff, i, 2));
+*/
+ }
+
+ io->updated = 1;
+
+ if(unlikely(io->enabled == CONFIG_BOOLEAN_AUTO)) {
+ if(unlikely(io->Read || io->Write || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))
+ io->enabled = CONFIG_BOOLEAN_YES;
+ else
+ io->delay_counter = cgroup_recheck_zero_blkio_every_iterations;
+ }
+ }
+}
+
+static inline void cgroup2_read_blkio(struct blkio *io, unsigned int word_offset) {
+ if(unlikely(io->enabled == CONFIG_BOOLEAN_AUTO && io->delay_counter > 0)) {
+ io->delay_counter--;
+ return;
+ }
+
+ if(likely(io->filename)) {
+ static procfile *ff = NULL;
+
+ ff = procfile_reopen(ff, io->filename, NULL, CGROUP_PROCFILE_FLAG);
+ if(unlikely(!ff)) {
+ io->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ io->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ unsigned long i, lines = procfile_lines(ff);
+
+ if (unlikely(lines < 1)) {
+ error("CGROUP: file '%s' should have 1+ lines.", io->filename);
+ io->updated = 0;
+ return;
+ }
+
+ io->Read = 0;
+ io->Write = 0;
+
+ for (i = 0; i < lines; i++) {
+ io->Read += str2ull(procfile_lineword(ff, i, 2 + word_offset));
+ io->Write += str2ull(procfile_lineword(ff, i, 4 + word_offset));
+ }
+
+ io->updated = 1;
+
+ if(unlikely(io->enabled == CONFIG_BOOLEAN_AUTO)) {
+ if(unlikely(io->Read || io->Write || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))
+ io->enabled = CONFIG_BOOLEAN_YES;
+ else
+ io->delay_counter = cgroup_recheck_zero_blkio_every_iterations;
+ }
+ }
+}
+
+static inline void cgroup2_read_pressure(struct pressure *res) {
+ static procfile *ff = NULL;
+
+ if (likely(res->filename)) {
+ ff = procfile_reopen(ff, res->filename, " =", CGROUP_PROCFILE_FLAG);
+ if (unlikely(!ff)) {
+ res->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ ff = procfile_readall(ff);
+ if (unlikely(!ff)) {
+ res->updated = 0;
+ cgroups_check = 1;
+ return;
+ }
+
+ size_t lines = procfile_lines(ff);
+ if (lines < 1) {
+ error("CGROUP: file '%s' should have 1+ lines.", res->filename);
+ res->updated = 0;
+ return;
+ }
+
+ res->some.share_time.value10 = strtod(procfile_lineword(ff, 0, 2), NULL);
+ res->some.share_time.value60 = strtod(procfile_lineword(ff, 0, 4), NULL);
+ res->some.share_time.value300 = strtod(procfile_lineword(ff, 0, 6), NULL);
+ res->some.total_time.value_total = str2ull(procfile_lineword(ff, 0, 8)) / 1000; // us->ms
+
+ if (lines > 2) {
+ res->full.share_time.value10 = strtod(procfile_lineword(ff, 1, 2), NULL);
+ res->full.share_time.value60 = strtod(procfile_lineword(ff, 1, 4), NULL);
+ res->full.share_time.value300 = strtod(procfile_lineword(ff, 1, 6), NULL);
+ res->full.total_time.value_total = str2ull(procfile_lineword(ff, 0, 8)) / 1000; // us->ms
+ }
+
+ res->updated = 1;
+
+ if (unlikely(res->some.enabled == CONFIG_BOOLEAN_AUTO)) {
+ res->some.enabled = CONFIG_BOOLEAN_YES;
+ if (lines > 2) {
+ res->full.enabled = CONFIG_BOOLEAN_YES;
+ } else {
+ res->full.enabled = CONFIG_BOOLEAN_NO;
+ }
+ }
+ }
+}
+
+static inline void cgroup_read_memory(struct memory *mem, char parent_cg_is_unified) {
+ static procfile *ff = NULL;
+
+ // read detailed ram usage
+ if(likely(mem->filename_detailed)) {
+ if(unlikely(mem->enabled_detailed == CONFIG_BOOLEAN_AUTO && mem->delay_counter_detailed > 0)) {
+ mem->delay_counter_detailed--;
+ goto memory_next;
+ }
+
+ ff = procfile_reopen(ff, mem->filename_detailed, NULL, CGROUP_PROCFILE_FLAG);
+ if(unlikely(!ff)) {
+ mem->updated_detailed = 0;
+ cgroups_check = 1;
+ goto memory_next;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ mem->updated_detailed = 0;
+ cgroups_check = 1;
+ goto memory_next;
+ }
+
+ unsigned long i, lines = procfile_lines(ff);
+
+ if(unlikely(lines < 1)) {
+ error("CGROUP: file '%s' should have 1+ lines.", mem->filename_detailed);
+ mem->updated_detailed = 0;
+ goto memory_next;
+ }
+
+
+ if(unlikely(!mem->arl_base)) {
+ if(parent_cg_is_unified == 0){
+ mem->arl_base = arl_create("cgroup/memory", NULL, 60);
+
+ arl_expect(mem->arl_base, "total_cache", &mem->total_cache);
+ arl_expect(mem->arl_base, "total_rss", &mem->total_rss);
+ arl_expect(mem->arl_base, "total_rss_huge", &mem->total_rss_huge);
+ arl_expect(mem->arl_base, "total_mapped_file", &mem->total_mapped_file);
+ arl_expect(mem->arl_base, "total_writeback", &mem->total_writeback);
+ mem->arl_dirty = arl_expect(mem->arl_base, "total_dirty", &mem->total_dirty);
+ mem->arl_swap = arl_expect(mem->arl_base, "total_swap", &mem->total_swap);
+ arl_expect(mem->arl_base, "total_pgpgin", &mem->total_pgpgin);
+ arl_expect(mem->arl_base, "total_pgpgout", &mem->total_pgpgout);
+ arl_expect(mem->arl_base, "total_pgfault", &mem->total_pgfault);
+ arl_expect(mem->arl_base, "total_pgmajfault", &mem->total_pgmajfault);
+ arl_expect(mem->arl_base, "total_inactive_file", &mem->total_inactive_file);
+ } else {
+ mem->arl_base = arl_create("cgroup/memory", NULL, 60);
+
+ arl_expect(mem->arl_base, "anon", &mem->anon);
+ arl_expect(mem->arl_base, "kernel_stack", &mem->kernel_stack);
+ arl_expect(mem->arl_base, "slab", &mem->slab);
+ arl_expect(mem->arl_base, "sock", &mem->sock);
+ arl_expect(mem->arl_base, "anon_thp", &mem->anon_thp);
+ arl_expect(mem->arl_base, "file", &mem->total_mapped_file);
+ arl_expect(mem->arl_base, "file_writeback", &mem->total_writeback);
+ mem->arl_dirty = arl_expect(mem->arl_base, "file_dirty", &mem->total_dirty);
+ arl_expect(mem->arl_base, "pgfault", &mem->total_pgfault);
+ arl_expect(mem->arl_base, "pgmajfault", &mem->total_pgmajfault);
+ arl_expect(mem->arl_base, "inactive_file", &mem->total_inactive_file);
+ }
+ }
+
+ arl_begin(mem->arl_base);
+
+ for(i = 0; i < lines ; i++) {
+ if(arl_check(mem->arl_base,
+ procfile_lineword(ff, i, 0),
+ procfile_lineword(ff, i, 1))) break;
+ }
+
+ if(unlikely(mem->arl_dirty->flags & ARL_ENTRY_FLAG_FOUND))
+ mem->detailed_has_dirty = 1;
+
+ if(unlikely(parent_cg_is_unified == 0 && mem->arl_swap->flags & ARL_ENTRY_FLAG_FOUND))
+ mem->detailed_has_swap = 1;
+
+ // fprintf(stderr, "READ: '%s', cache: %llu, rss: %llu, rss_huge: %llu, mapped_file: %llu, writeback: %llu, dirty: %llu, swap: %llu, pgpgin: %llu, pgpgout: %llu, pgfault: %llu, pgmajfault: %llu, inactive_anon: %llu, active_anon: %llu, inactive_file: %llu, active_file: %llu, unevictable: %llu, hierarchical_memory_limit: %llu, total_cache: %llu, total_rss: %llu, total_rss_huge: %llu, total_mapped_file: %llu, total_writeback: %llu, total_dirty: %llu, total_swap: %llu, total_pgpgin: %llu, total_pgpgout: %llu, total_pgfault: %llu, total_pgmajfault: %llu, total_inactive_anon: %llu, total_active_anon: %llu, total_inactive_file: %llu, total_active_file: %llu, total_unevictable: %llu\n", mem->filename, mem->cache, mem->rss, mem->rss_huge, mem->mapped_file, mem->writeback, mem->dirty, mem->swap, mem->pgpgin, mem->pgpgout, mem->pgfault, mem->pgmajfault, mem->inactive_anon, mem->active_anon, mem->inactive_file, mem->active_file, mem->unevictable, mem->hierarchical_memory_limit, mem->total_cache, mem->total_rss, mem->total_rss_huge, mem->total_mapped_file, mem->total_writeback, mem->total_dirty, mem->total_swap, mem->total_pgpgin, mem->total_pgpgout, mem->total_pgfault, mem->total_pgmajfault, mem->total_inactive_anon, mem->total_active_anon, mem->total_inactive_file, mem->total_active_file, mem->total_unevictable);
+
+ mem->updated_detailed = 1;
+
+ if(unlikely(mem->enabled_detailed == CONFIG_BOOLEAN_AUTO)) {
+ if(( (!parent_cg_is_unified) && ( mem->total_cache || mem->total_dirty || mem->total_rss || mem->total_rss_huge || mem->total_mapped_file || mem->total_writeback
+ || mem->total_swap || mem->total_pgpgin || mem->total_pgpgout || mem->total_pgfault || mem->total_pgmajfault || mem->total_inactive_file))
+ || (parent_cg_is_unified && ( mem->anon || mem->total_dirty || mem->kernel_stack || mem->slab || mem->sock || mem->total_writeback
+ || mem->anon_thp || mem->total_pgfault || mem->total_pgmajfault || mem->total_inactive_file))
+ || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)
+ mem->enabled_detailed = CONFIG_BOOLEAN_YES;
+ else
+ mem->delay_counter_detailed = cgroup_recheck_zero_mem_detailed_every_iterations;
+ }
+ }
+
+memory_next:
+
+ // read usage_in_bytes
+ if(likely(mem->filename_usage_in_bytes)) {
+ mem->updated_usage_in_bytes = !read_single_number_file(mem->filename_usage_in_bytes, &mem->usage_in_bytes);
+ if(unlikely(mem->updated_usage_in_bytes && mem->enabled_usage_in_bytes == CONFIG_BOOLEAN_AUTO &&
+ (mem->usage_in_bytes || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ mem->enabled_usage_in_bytes = CONFIG_BOOLEAN_YES;
+ }
+
+ if (likely(mem->updated_usage_in_bytes && mem->updated_detailed)) {
+ mem->usage_in_bytes =
+ (mem->usage_in_bytes > mem->total_inactive_file) ? (mem->usage_in_bytes - mem->total_inactive_file) : 0;
+ }
+
+ // read msw_usage_in_bytes
+ if(likely(mem->filename_msw_usage_in_bytes)) {
+ mem->updated_msw_usage_in_bytes = !read_single_number_file(mem->filename_msw_usage_in_bytes, &mem->msw_usage_in_bytes);
+ if(unlikely(mem->updated_msw_usage_in_bytes && mem->enabled_msw_usage_in_bytes == CONFIG_BOOLEAN_AUTO &&
+ (mem->msw_usage_in_bytes || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ mem->enabled_msw_usage_in_bytes = CONFIG_BOOLEAN_YES;
+ }
+
+ // read failcnt
+ if(likely(mem->filename_failcnt)) {
+ if(unlikely(mem->enabled_failcnt == CONFIG_BOOLEAN_AUTO && mem->delay_counter_failcnt > 0)) {
+ mem->updated_failcnt = 0;
+ mem->delay_counter_failcnt--;
+ }
+ else {
+ mem->updated_failcnt = !read_single_number_file(mem->filename_failcnt, &mem->failcnt);
+ if(unlikely(mem->updated_failcnt && mem->enabled_failcnt == CONFIG_BOOLEAN_AUTO)) {
+ if(unlikely(mem->failcnt || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))
+ mem->enabled_failcnt = CONFIG_BOOLEAN_YES;
+ else
+ mem->delay_counter_failcnt = cgroup_recheck_zero_mem_failcnt_every_iterations;
+ }
+ }
+ }
+}
+
+static inline void read_cgroup(struct cgroup *cg) {
+ debug(D_CGROUP, "reading metrics for cgroups '%s'", cg->id);
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ cgroup_read_cpuacct_stat(&cg->cpuacct_stat);
+ cgroup_read_cpuacct_usage(&cg->cpuacct_usage);
+ cgroup_read_cpuacct_cpu_stat(&cg->cpuacct_cpu_throttling);
+ cgroup_read_cpuacct_cpu_shares(&cg->cpuacct_cpu_shares);
+ cgroup_read_memory(&cg->memory, 0);
+ cgroup_read_blkio(&cg->io_service_bytes);
+ cgroup_read_blkio(&cg->io_serviced);
+ cgroup_read_blkio(&cg->throttle_io_service_bytes);
+ cgroup_read_blkio(&cg->throttle_io_serviced);
+ cgroup_read_blkio(&cg->io_merged);
+ cgroup_read_blkio(&cg->io_queued);
+ }
+ else {
+ //TODO: io_service_bytes and io_serviced use same file merge into 1 function
+ cgroup2_read_blkio(&cg->io_service_bytes, 0);
+ cgroup2_read_blkio(&cg->io_serviced, 4);
+ cgroup2_read_cpuacct_cpu_stat(&cg->cpuacct_stat, &cg->cpuacct_cpu_throttling);
+ cgroup_read_cpuacct_cpu_shares(&cg->cpuacct_cpu_shares);
+ cgroup2_read_pressure(&cg->cpu_pressure);
+ cgroup2_read_pressure(&cg->io_pressure);
+ cgroup2_read_pressure(&cg->memory_pressure);
+ cgroup_read_memory(&cg->memory, 1);
+ }
+}
+
+static inline void read_all_discovered_cgroups(struct cgroup *root) {
+ debug(D_CGROUP, "reading metrics for all cgroups");
+
+ struct cgroup *cg;
+ for (cg = root; cg; cg = cg->next) {
+ if (cg->enabled && !cg->pending_renames) {
+ read_cgroup(cg);
+ }
+ }
+}
+
+// ----------------------------------------------------------------------------
+// cgroup network interfaces
+
+#define CGROUP_NETWORK_INTERFACE_MAX_LINE 2048
+static inline void read_cgroup_network_interfaces(struct cgroup *cg) {
+ debug(D_CGROUP, "looking for the network interfaces of cgroup '%s' with chart id '%s' and title '%s'", cg->id, cg->chart_id, cg->chart_title);
+
+ pid_t cgroup_pid;
+ char cgroup_identifier[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1];
+
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ snprintfz(cgroup_identifier, CGROUP_NETWORK_INTERFACE_MAX_LINE, "%s%s", cgroup_cpuacct_base, cg->id);
+ }
+ else {
+ snprintfz(cgroup_identifier, CGROUP_NETWORK_INTERFACE_MAX_LINE, "%s%s", cgroup_unified_base, cg->id);
+ }
+
+ debug(D_CGROUP, "executing cgroup_identifier %s --cgroup '%s' for cgroup '%s'", cgroups_network_interface_script, cgroup_identifier, cg->id);
+ FILE *fp_child_input, *fp_child_output;
+ (void)netdata_popen_raw_default_flags_and_environment(&cgroup_pid, &fp_child_input, &fp_child_output, cgroups_network_interface_script, "--cgroup", cgroup_identifier);
+ if(!fp_child_output) {
+ error("CGROUP: cannot popen(%s --cgroup \"%s\", \"r\").", cgroups_network_interface_script, cgroup_identifier);
+ return;
+ }
+
+ char *s;
+ char buffer[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1];
+ while((s = fgets(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, fp_child_output))) {
+ trim(s);
+
+ if(*s && *s != '\n') {
+ char *t = s;
+ while(*t && *t != ' ') t++;
+ if(*t == ' ') {
+ *t = '\0';
+ t++;
+ }
+
+ if(!*s) {
+ error("CGROUP: empty host interface returned by script");
+ continue;
+ }
+
+ if(!*t) {
+ error("CGROUP: empty guest interface returned by script");
+ continue;
+ }
+
+ struct cgroup_network_interface *i = callocz(1, sizeof(struct cgroup_network_interface));
+ i->host_device = strdupz(s);
+ i->container_device = strdupz(t);
+ i->next = cg->interfaces;
+ cg->interfaces = i;
+
+ info("CGROUP: cgroup '%s' has network interface '%s' as '%s'", cg->id, i->host_device, i->container_device);
+
+ // register a device rename to proc_net_dev.c
+ netdev_rename_device_add(
+ i->host_device, i->container_device, cg->chart_id, cg->chart_labels, k8s_is_kubepod(cg) ? "k8s." : "");
+ }
+ }
+
+ netdata_pclose(fp_child_input, fp_child_output, cgroup_pid);
+ // debug(D_CGROUP, "closed cgroup_identifier for cgroup '%s'", cg->id);
+}
+
+static inline void free_cgroup_network_interfaces(struct cgroup *cg) {
+ while(cg->interfaces) {
+ struct cgroup_network_interface *i = cg->interfaces;
+ cg->interfaces = i->next;
+
+ // delete the registration of proc_net_dev rename
+ netdev_rename_device_del(i->host_device);
+
+ freez((void *)i->host_device);
+ freez((void *)i->container_device);
+ freez((void *)i);
+ }
+}
+
+// ----------------------------------------------------------------------------
+// add/remove/find cgroup objects
+
+#define CGROUP_CHARTID_LINE_MAX 1024
+
+static inline char *cgroup_title_strdupz(const char *s) {
+ if(!s || !*s) s = "/";
+
+ if(*s == '/' && s[1] != '\0') s++;
+
+ char *r = strdupz(s);
+ netdata_fix_chart_name(r);
+
+ return r;
+}
+
+static inline char *cgroup_chart_id_strdupz(const char *s) {
+ if(!s || !*s) s = "/";
+
+ if(*s == '/' && s[1] != '\0') s++;
+
+ char *r = strdupz(s);
+ netdata_fix_chart_id(r);
+
+ return r;
+}
+
+// TODO: move the code to cgroup_chart_id_strdupz() when the renaming script is fixed
+static inline void substitute_dots_in_id(char *s) {
+ // dots are used to distinguish chart type and id in streaming, so we should replace them
+ for (char *d = s; *d; d++) {
+ if (*d == '.')
+ *d = '-';
+ }
+}
+
+// ----------------------------------------------------------------------------
+// parse k8s labels
+
+char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data) {
+ // the first word, up to the first space is the name
+ char *name = mystrsep(&data, " ");
+
+ // the rest are key=value pairs separated by comma
+ while(data) {
+ char *pair = mystrsep(&data, ",");
+ rrdlabels_add_pair(labels, pair, RRDLABEL_SRC_AUTO| RRDLABEL_SRC_K8S);
+ }
+
+ return name;
+}
+
+// ----------------------------------------------------------------------------
+
+static inline void free_pressure(struct pressure *res) {
+ if (res->some.share_time.st) rrdset_is_obsolete(res->some.share_time.st);
+ if (res->some.total_time.st) rrdset_is_obsolete(res->some.total_time.st);
+ if (res->full.share_time.st) rrdset_is_obsolete(res->full.share_time.st);
+ if (res->full.total_time.st) rrdset_is_obsolete(res->full.total_time.st);
+ freez(res->filename);
+}
+
+static inline void cgroup_free(struct cgroup *cg) {
+ debug(D_CGROUP, "Removing cgroup '%s' with chart id '%s' (was %s and %s)", cg->id, cg->chart_id, (cg->enabled)?"enabled":"disabled", (cg->available)?"available":"not available");
+
+ if(cg->st_cpu) rrdset_is_obsolete(cg->st_cpu);
+ if(cg->st_cpu_limit) rrdset_is_obsolete(cg->st_cpu_limit);
+ if(cg->st_cpu_per_core) rrdset_is_obsolete(cg->st_cpu_per_core);
+ if(cg->st_cpu_nr_throttled) rrdset_is_obsolete(cg->st_cpu_nr_throttled);
+ if(cg->st_cpu_throttled_time) rrdset_is_obsolete(cg->st_cpu_throttled_time);
+ if(cg->st_cpu_shares) rrdset_is_obsolete(cg->st_cpu_shares);
+ if(cg->st_mem) rrdset_is_obsolete(cg->st_mem);
+ if(cg->st_writeback) rrdset_is_obsolete(cg->st_writeback);
+ if(cg->st_mem_activity) rrdset_is_obsolete(cg->st_mem_activity);
+ if(cg->st_pgfaults) rrdset_is_obsolete(cg->st_pgfaults);
+ if(cg->st_mem_usage) rrdset_is_obsolete(cg->st_mem_usage);
+ if(cg->st_mem_usage_limit) rrdset_is_obsolete(cg->st_mem_usage_limit);
+ if(cg->st_mem_utilization) rrdset_is_obsolete(cg->st_mem_utilization);
+ if(cg->st_mem_failcnt) rrdset_is_obsolete(cg->st_mem_failcnt);
+ if(cg->st_io) rrdset_is_obsolete(cg->st_io);
+ if(cg->st_serviced_ops) rrdset_is_obsolete(cg->st_serviced_ops);
+ if(cg->st_throttle_io) rrdset_is_obsolete(cg->st_throttle_io);
+ if(cg->st_throttle_serviced_ops) rrdset_is_obsolete(cg->st_throttle_serviced_ops);
+ if(cg->st_queued_ops) rrdset_is_obsolete(cg->st_queued_ops);
+ if(cg->st_merged_ops) rrdset_is_obsolete(cg->st_merged_ops);
+
+ freez(cg->filename_cpuset_cpus);
+ freez(cg->filename_cpu_cfs_period);
+ freez(cg->filename_cpu_cfs_quota);
+ freez(cg->filename_memory_limit);
+ freez(cg->filename_memoryswap_limit);
+
+ free_cgroup_network_interfaces(cg);
+
+ freez(cg->cpuacct_usage.cpu_percpu);
+
+ freez(cg->cpuacct_stat.filename);
+ freez(cg->cpuacct_usage.filename);
+ freez(cg->cpuacct_cpu_throttling.filename);
+ freez(cg->cpuacct_cpu_shares.filename);
+
+ arl_free(cg->memory.arl_base);
+ freez(cg->memory.filename_detailed);
+ freez(cg->memory.filename_failcnt);
+ freez(cg->memory.filename_usage_in_bytes);
+ freez(cg->memory.filename_msw_usage_in_bytes);
+
+ freez(cg->io_service_bytes.filename);
+ freez(cg->io_serviced.filename);
+
+ freez(cg->throttle_io_service_bytes.filename);
+ freez(cg->throttle_io_serviced.filename);
+
+ freez(cg->io_merged.filename);
+ freez(cg->io_queued.filename);
+
+ free_pressure(&cg->cpu_pressure);
+ free_pressure(&cg->io_pressure);
+ free_pressure(&cg->memory_pressure);
+
+ freez(cg->id);
+ freez(cg->intermediate_id);
+ freez(cg->chart_id);
+ freez(cg->chart_title);
+
+ rrdlabels_destroy(cg->chart_labels);
+
+ freez(cg);
+
+ cgroup_root_count--;
+}
+
+// ----------------------------------------------------------------------------
+
+static inline void discovery_rename_cgroup(struct cgroup *cg) {
+ if (!cg->pending_renames) {
+ return;
+ }
+ cg->pending_renames--;
+
+ debug(D_CGROUP, "looking for the name of cgroup '%s' with chart id '%s' and title '%s'", cg->id, cg->chart_id, cg->chart_title);
+ debug(D_CGROUP, "executing command %s \"%s\" for cgroup '%s'", cgroups_rename_script, cg->intermediate_id, cg->chart_id);
+ pid_t cgroup_pid;
+
+ FILE *fp_child_input, *fp_child_output;
+ (void)netdata_popen_raw_default_flags_and_environment(&cgroup_pid, &fp_child_input, &fp_child_output, cgroups_rename_script, cg->id, cg->intermediate_id);
+ if (!fp_child_output) {
+ error("CGROUP: cannot popen(%s \"%s\", \"r\").", cgroups_rename_script, cg->intermediate_id);
+ cg->pending_renames = 0;
+ cg->processed = 1;
+ return;
+ }
+
+ char buffer[CGROUP_CHARTID_LINE_MAX + 1];
+ char *new_name = fgets(buffer, CGROUP_CHARTID_LINE_MAX, fp_child_output);
+ int exit_code = netdata_pclose(fp_child_input, fp_child_output, cgroup_pid);
+
+ switch (exit_code) {
+ case 0:
+ cg->pending_renames = 0;
+ break;
+
+ case 3:
+ cg->pending_renames = 0;
+ cg->processed = 1;
+ break;
+ }
+
+ if(cg->pending_renames || cg->processed) return;
+ if(!new_name || !*new_name || *new_name == '\n') return;
+ if(!(new_name = trim(new_name))) return;
+
+ char *name = new_name;
+ if (!strncmp(new_name, "k8s_", 4)) {
+ if(!cg->chart_labels) cg->chart_labels = rrdlabels_create();
+
+ // read the new labels and remove the obsolete ones
+ rrdlabels_unmark_all(cg->chart_labels);
+ name = k8s_parse_resolved_name_and_labels(cg->chart_labels, new_name);
+ rrdlabels_remove_all_unmarked(cg->chart_labels);
+ }
+
+ freez(cg->chart_title);
+ cg->chart_title = cgroup_title_strdupz(name);
+
+ freez(cg->chart_id);
+ cg->chart_id = cgroup_chart_id_strdupz(name);
+
+ substitute_dots_in_id(cg->chart_id);
+ cg->hash_chart = simple_hash(cg->chart_id);
+}
+
+static void is_cgroup_procs_exist(netdata_ebpf_cgroup_shm_body_t *out, char *id) {
+ struct stat buf;
+
+ snprintfz(out->path, FILENAME_MAX, "%s%s/cgroup.procs", cgroup_cpuset_base, id);
+ if (likely(stat(out->path, &buf) == 0)) {
+ return;
+ }
+
+ snprintfz(out->path, FILENAME_MAX, "%s%s/cgroup.procs", cgroup_blkio_base, id);
+ if (likely(stat(out->path, &buf) == 0)) {
+ return;
+ }
+
+ snprintfz(out->path, FILENAME_MAX, "%s%s/cgroup.procs", cgroup_memory_base, id);
+ if (likely(stat(out->path, &buf) == 0)) {
+ return;
+ }
+
+ snprintfz(out->path, FILENAME_MAX, "%s%s/cgroup.procs", cgroup_devices_base, id);
+ if (likely(stat(out->path, &buf) == 0)) {
+ return;
+ }
+
+ out->path[0] = '\0';
+ out->enabled = 0;
+}
+
+static inline void convert_cgroup_to_systemd_service(struct cgroup *cg) {
+ char buffer[CGROUP_CHARTID_LINE_MAX];
+ cg->options |= CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE;
+ strncpyz(buffer, cg->id, CGROUP_CHARTID_LINE_MAX);
+ char *s = buffer;
+
+ // skip to the last slash
+ size_t len = strlen(s);
+ while (len--) {
+ if (unlikely(s[len] == '/')) {
+ break;
+ }
+ }
+ if (len) {
+ s = &s[len + 1];
+ }
+
+ // remove extension
+ len = strlen(s);
+ while (len--) {
+ if (unlikely(s[len] == '.')) {
+ break;
+ }
+ }
+ if (len) {
+ s[len] = '\0';
+ }
+
+ freez(cg->chart_title);
+ cg->chart_title = cgroup_title_strdupz(s);
+}
+
+static inline struct cgroup *discovery_cgroup_add(const char *id) {
+ debug(D_CGROUP, "adding to list, cgroup with id '%s'", id);
+
+ struct cgroup *cg = callocz(1, sizeof(struct cgroup));
+ cg->id = strdupz(id);
+ cg->hash = simple_hash(cg->id);
+ cg->chart_title = cgroup_title_strdupz(id);
+ cg->intermediate_id = cgroup_chart_id_strdupz(id);
+ cg->chart_id = cgroup_chart_id_strdupz(id);
+ substitute_dots_in_id(cg->chart_id);
+ cg->hash_chart = simple_hash(cg->chart_id);
+ if (cgroup_use_unified_cgroups) {
+ cg->options |= CGROUP_OPTIONS_IS_UNIFIED;
+ }
+
+ if (!discovered_cgroup_root)
+ discovered_cgroup_root = cg;
+ else {
+ struct cgroup *t;
+ for (t = discovered_cgroup_root; t->discovered_next; t = t->discovered_next) {
+ }
+ t->discovered_next = cg;
+ }
+
+ return cg;
+}
+
+static inline struct cgroup *discovery_cgroup_find(const char *id) {
+ debug(D_CGROUP, "searching for cgroup '%s'", id);
+
+ uint32_t hash = simple_hash(id);
+
+ struct cgroup *cg;
+ for(cg = discovered_cgroup_root; cg ; cg = cg->discovered_next) {
+ if(hash == cg->hash && strcmp(id, cg->id) == 0)
+ break;
+ }
+
+ debug(D_CGROUP, "cgroup '%s' %s in memory", id, (cg)?"found":"not found");
+ return cg;
+}
+
+static inline void discovery_find_cgroup_in_dir_callback(const char *dir) {
+ if (!dir || !*dir) {
+ dir = "/";
+ }
+ debug(D_CGROUP, "examining cgroup dir '%s'", dir);
+
+ struct cgroup *cg = discovery_cgroup_find(dir);
+ if (cg) {
+ cg->available = 1;
+ return;
+ }
+
+ if (cgroup_root_count >= cgroup_root_max) {
+ info("CGROUP: maximum number of cgroups reached (%d). Not adding cgroup '%s'", cgroup_root_count, dir);
+ return;
+ }
+
+ if (cgroup_max_depth > 0) {
+ int depth = calc_cgroup_depth(dir);
+ if (depth > cgroup_max_depth) {
+ info("CGROUP: '%s' is too deep (%d, while max is %d)", dir, depth, cgroup_max_depth);
+ return;
+ }
+ }
+
+ cg = discovery_cgroup_add(dir);
+ cg->available = 1;
+ cg->first_time_seen = 1;
+ cgroup_root_count++;
+}
+
+static inline int discovery_find_dir_in_subdirs(const char *base, const char *this, void (*callback)(const char *)) {
+ if(!this) this = base;
+ debug(D_CGROUP, "searching for directories in '%s' (base '%s')", this?this:"", base);
+
+ size_t dirlen = strlen(this), baselen = strlen(base);
+
+ int ret = -1;
+ int enabled = -1;
+
+ const char *relative_path = &this[baselen];
+ if(!*relative_path) relative_path = "/";
+
+ DIR *dir = opendir(this);
+ if(!dir) {
+ error("CGROUP: cannot read directory '%s'", base);
+ return ret;
+ }
+ ret = 1;
+
+ callback(relative_path);
+
+ struct dirent *de = NULL;
+ while((de = readdir(dir))) {
+ if(de->d_type == DT_DIR
+ && (
+ (de->d_name[0] == '.' && de->d_name[1] == '\0')
+ || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ ))
+ continue;
+
+ if(de->d_type == DT_DIR) {
+ if(enabled == -1) {
+ const char *r = relative_path;
+ if(*r == '\0') r = "/";
+
+ // do not decent in directories we are not interested
+ enabled = matches_search_cgroup_paths(r);
+ }
+
+ if(enabled) {
+ char *s = mallocz(dirlen + strlen(de->d_name) + 2);
+ strcpy(s, this);
+ strcat(s, "/");
+ strcat(s, de->d_name);
+ int ret2 = discovery_find_dir_in_subdirs(base, s, callback);
+ if(ret2 > 0) ret += ret2;
+ freez(s);
+ }
+ }
+ }
+
+ closedir(dir);
+ return ret;
+}
+
+static inline void discovery_mark_all_cgroups_as_unavailable() {
+ debug(D_CGROUP, "marking all cgroups as not available");
+ struct cgroup *cg;
+ for (cg = discovered_cgroup_root; cg; cg = cg->discovered_next) {
+ cg->available = 0;
+ }
+}
+
+static inline void discovery_update_filenames() {
+ struct cgroup *cg;
+ struct stat buf;
+ for(cg = discovered_cgroup_root; cg ; cg = cg->discovered_next) {
+ if(unlikely(!cg->available || !cg->enabled || cg->pending_renames))
+ continue;
+
+ debug(D_CGROUP, "checking paths for cgroup '%s'", cg->id);
+
+ // check for newly added cgroups
+ // and update the filenames they read
+ char filename[FILENAME_MAX + 1];
+ if(!cgroup_use_unified_cgroups) {
+ if(unlikely(cgroup_enable_cpuacct_stat && !cg->cpuacct_stat.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpuacct.stat", cgroup_cpuacct_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->cpuacct_stat.filename = strdupz(filename);
+ cg->cpuacct_stat.enabled = cgroup_enable_cpuacct_stat;
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpuset.cpus", cgroup_cpuset_base, cg->id);
+ cg->filename_cpuset_cpus = strdupz(filename);
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpu.cfs_period_us", cgroup_cpuacct_base, cg->id);
+ cg->filename_cpu_cfs_period = strdupz(filename);
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpu.cfs_quota_us", cgroup_cpuacct_base, cg->id);
+ cg->filename_cpu_cfs_quota = strdupz(filename);
+ debug(D_CGROUP, "cpuacct.stat filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_stat.filename);
+ }
+ else
+ debug(D_CGROUP, "cpuacct.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely(cgroup_enable_cpuacct_usage && !cg->cpuacct_usage.filename && !is_cgroup_systemd_service(cg))) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpuacct.usage_percpu", cgroup_cpuacct_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->cpuacct_usage.filename = strdupz(filename);
+ cg->cpuacct_usage.enabled = cgroup_enable_cpuacct_usage;
+ debug(D_CGROUP, "cpuacct.usage_percpu filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_usage.filename);
+ }
+ else
+ debug(D_CGROUP, "cpuacct.usage_percpu file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ if(unlikely(cgroup_enable_cpuacct_cpu_throttling && !cg->cpuacct_cpu_throttling.filename && !is_cgroup_systemd_service(cg))) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpu.stat", cgroup_cpuacct_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->cpuacct_cpu_throttling.filename = strdupz(filename);
+ cg->cpuacct_cpu_throttling.enabled = cgroup_enable_cpuacct_cpu_throttling;
+ debug(D_CGROUP, "cpu.stat filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_cpu_throttling.filename);
+ }
+ else
+ debug(D_CGROUP, "cpu.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ if (unlikely(
+ cgroup_enable_cpuacct_cpu_shares && !cg->cpuacct_cpu_shares.filename &&
+ !is_cgroup_systemd_service(cg))) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpu.shares", cgroup_cpuacct_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->cpuacct_cpu_shares.filename = strdupz(filename);
+ cg->cpuacct_cpu_shares.enabled = cgroup_enable_cpuacct_cpu_shares;
+ debug(
+ D_CGROUP, "cpu.shares filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_cpu_shares.filename);
+ } else
+ debug(D_CGROUP, "cpu.shares file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory) && !cg->memory.filename_detailed && (cgroup_used_memory || cgroup_enable_systemd_services_detailed_memory || !is_cgroup_systemd_service(cg)))) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.stat", cgroup_memory_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->memory.filename_detailed = strdupz(filename);
+ cg->memory.enabled_detailed = (cgroup_enable_detailed_memory == CONFIG_BOOLEAN_YES)?CONFIG_BOOLEAN_YES:CONFIG_BOOLEAN_AUTO;
+ debug(D_CGROUP, "memory.stat filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_detailed);
+ }
+ else
+ debug(D_CGROUP, "memory.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely(cgroup_enable_memory && !cg->memory.filename_usage_in_bytes)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.usage_in_bytes", cgroup_memory_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->memory.filename_usage_in_bytes = strdupz(filename);
+ cg->memory.enabled_usage_in_bytes = cgroup_enable_memory;
+ debug(D_CGROUP, "memory.usage_in_bytes filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_usage_in_bytes);
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.limit_in_bytes", cgroup_memory_base, cg->id);
+ cg->filename_memory_limit = strdupz(filename);
+ }
+ else
+ debug(D_CGROUP, "memory.usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely(cgroup_enable_swap && !cg->memory.filename_msw_usage_in_bytes)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.memsw.usage_in_bytes", cgroup_memory_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->memory.filename_msw_usage_in_bytes = strdupz(filename);
+ cg->memory.enabled_msw_usage_in_bytes = cgroup_enable_swap;
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.memsw.limit_in_bytes", cgroup_memory_base, cg->id);
+ cg->filename_memoryswap_limit = strdupz(filename);
+ debug(D_CGROUP, "memory.msw_usage_in_bytes filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_msw_usage_in_bytes);
+ }
+ else
+ debug(D_CGROUP, "memory.msw_usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely(cgroup_enable_memory_failcnt && !cg->memory.filename_failcnt)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.failcnt", cgroup_memory_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->memory.filename_failcnt = strdupz(filename);
+ cg->memory.enabled_failcnt = cgroup_enable_memory_failcnt;
+ debug(D_CGROUP, "memory.failcnt filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_failcnt);
+ }
+ else
+ debug(D_CGROUP, "memory.failcnt file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely(cgroup_enable_blkio_io && !cg->io_service_bytes.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_service_bytes_recursive", cgroup_blkio_base, cg->id);
+ if (unlikely(stat(filename, &buf) != -1)) {
+ cg->io_service_bytes.filename = strdupz(filename);
+ cg->io_service_bytes.enabled = cgroup_enable_blkio_io;
+ debug(D_CGROUP, "blkio.io_service_bytes_recursive filename for cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename);
+ } else {
+ debug(D_CGROUP, "blkio.io_service_bytes_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_service_bytes", cgroup_blkio_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->io_service_bytes.filename = strdupz(filename);
+ cg->io_service_bytes.enabled = cgroup_enable_blkio_io;
+ debug(D_CGROUP, "blkio.io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename);
+ } else {
+ debug(D_CGROUP, "blkio.io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ }
+ }
+
+ if (unlikely(cgroup_enable_blkio_ops && !cg->io_serviced.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_serviced_recursive", cgroup_blkio_base, cg->id);
+ if (unlikely(stat(filename, &buf) != -1)) {
+ cg->io_serviced.filename = strdupz(filename);
+ cg->io_serviced.enabled = cgroup_enable_blkio_ops;
+ debug(D_CGROUP, "blkio.io_serviced_recursive filename for cgroup '%s': '%s'", cg->id, cg->io_serviced.filename);
+ } else {
+ debug(D_CGROUP, "blkio.io_serviced_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_serviced", cgroup_blkio_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->io_serviced.filename = strdupz(filename);
+ cg->io_serviced.enabled = cgroup_enable_blkio_ops;
+ debug(D_CGROUP, "blkio.io_serviced filename for cgroup '%s': '%s'", cg->id, cg->io_serviced.filename);
+ } else {
+ debug(D_CGROUP, "blkio.io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ }
+ }
+
+ if (unlikely(cgroup_enable_blkio_throttle_io && !cg->throttle_io_service_bytes.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_service_bytes_recursive", cgroup_blkio_base, cg->id);
+ if (unlikely(stat(filename, &buf) != -1)) {
+ cg->throttle_io_service_bytes.filename = strdupz(filename);
+ cg->throttle_io_service_bytes.enabled = cgroup_enable_blkio_throttle_io;
+ debug(D_CGROUP,"blkio.throttle.io_service_bytes_recursive filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_service_bytes.filename);
+ } else {
+ debug(D_CGROUP, "blkio.throttle.io_service_bytes_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ snprintfz(
+ filename, FILENAME_MAX, "%s%s/blkio.throttle.io_service_bytes", cgroup_blkio_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->throttle_io_service_bytes.filename = strdupz(filename);
+ cg->throttle_io_service_bytes.enabled = cgroup_enable_blkio_throttle_io;
+ debug(D_CGROUP, "blkio.throttle.io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_service_bytes.filename);
+ } else {
+ debug(D_CGROUP, "blkio.throttle.io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ }
+ }
+
+ if (unlikely(cgroup_enable_blkio_throttle_ops && !cg->throttle_io_serviced.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_serviced_recursive", cgroup_blkio_base, cg->id);
+ if (unlikely(stat(filename, &buf) != -1)) {
+ cg->throttle_io_serviced.filename = strdupz(filename);
+ cg->throttle_io_serviced.enabled = cgroup_enable_blkio_throttle_ops;
+ debug(D_CGROUP, "blkio.throttle.io_serviced_recursive filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_serviced.filename);
+ } else {
+ debug(D_CGROUP, "blkio.throttle.io_serviced_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_serviced", cgroup_blkio_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->throttle_io_serviced.filename = strdupz(filename);
+ cg->throttle_io_serviced.enabled = cgroup_enable_blkio_throttle_ops;
+ debug(D_CGROUP, "blkio.throttle.io_serviced filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_serviced.filename);
+ } else {
+ debug(D_CGROUP, "blkio.throttle.io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ }
+ }
+
+ if (unlikely(cgroup_enable_blkio_merged_ops && !cg->io_merged.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_merged_recursive", cgroup_blkio_base, cg->id);
+ if (unlikely(stat(filename, &buf) != -1)) {
+ cg->io_merged.filename = strdupz(filename);
+ cg->io_merged.enabled = cgroup_enable_blkio_merged_ops;
+ debug(D_CGROUP, "blkio.io_merged_recursive filename for cgroup '%s': '%s'", cg->id, cg->io_merged.filename);
+ } else {
+ debug(D_CGROUP, "blkio.io_merged_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_merged", cgroup_blkio_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->io_merged.filename = strdupz(filename);
+ cg->io_merged.enabled = cgroup_enable_blkio_merged_ops;
+ debug(D_CGROUP, "blkio.io_merged filename for cgroup '%s': '%s'", cg->id, cg->io_merged.filename);
+ } else {
+ debug(D_CGROUP, "blkio.io_merged file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ }
+ }
+
+ if (unlikely(cgroup_enable_blkio_queued_ops && !cg->io_queued.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_queued_recursive", cgroup_blkio_base, cg->id);
+ if (unlikely(stat(filename, &buf) != -1)) {
+ cg->io_queued.filename = strdupz(filename);
+ cg->io_queued.enabled = cgroup_enable_blkio_queued_ops;
+ debug(D_CGROUP, "blkio.io_queued_recursive filename for cgroup '%s': '%s'", cg->id, cg->io_queued.filename);
+ } else {
+ debug(D_CGROUP, "blkio.io_queued_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_queued", cgroup_blkio_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->io_queued.filename = strdupz(filename);
+ cg->io_queued.enabled = cgroup_enable_blkio_queued_ops;
+ debug(D_CGROUP, "blkio.io_queued filename for cgroup '%s': '%s'", cg->id, cg->io_queued.filename);
+ } else {
+ debug(D_CGROUP, "blkio.io_queued file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ }
+ }
+ }
+ else if(likely(cgroup_unified_exist)) {
+ if(unlikely(cgroup_enable_blkio_io && !cg->io_service_bytes.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/io.stat", cgroup_unified_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->io_service_bytes.filename = strdupz(filename);
+ cg->io_service_bytes.enabled = cgroup_enable_blkio_io;
+ debug(D_CGROUP, "io.stat filename for unified cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename);
+ } else
+ debug(D_CGROUP, "io.stat file for unified cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ if (unlikely(cgroup_enable_blkio_ops && !cg->io_serviced.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/io.stat", cgroup_unified_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->io_serviced.filename = strdupz(filename);
+ cg->io_serviced.enabled = cgroup_enable_blkio_ops;
+ debug(D_CGROUP, "io.stat filename for unified cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename);
+ } else
+ debug(D_CGROUP, "io.stat file for unified cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ if (unlikely(
+ (cgroup_enable_cpuacct_stat || cgroup_enable_cpuacct_cpu_throttling) &&
+ !cg->cpuacct_stat.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpu.stat", cgroup_unified_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->cpuacct_stat.filename = strdupz(filename);
+ cg->cpuacct_stat.enabled = cgroup_enable_cpuacct_stat;
+ cg->cpuacct_cpu_throttling.enabled = cgroup_enable_cpuacct_cpu_throttling;
+ cg->filename_cpuset_cpus = NULL;
+ cg->filename_cpu_cfs_period = NULL;
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpu.max", cgroup_unified_base, cg->id);
+ cg->filename_cpu_cfs_quota = strdupz(filename);
+ debug(D_CGROUP, "cpu.stat filename for unified cgroup '%s': '%s'", cg->id, cg->cpuacct_stat.filename);
+ }
+ else
+ debug(D_CGROUP, "cpu.stat file for unified cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+ if (unlikely(cgroup_enable_cpuacct_cpu_shares && !cg->cpuacct_cpu_shares.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpu.weight", cgroup_unified_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->cpuacct_cpu_shares.filename = strdupz(filename);
+ cg->cpuacct_cpu_shares.enabled = cgroup_enable_cpuacct_cpu_shares;
+ debug(D_CGROUP, "cpu.weight filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_cpu_shares.filename);
+ } else
+ debug(D_CGROUP, "cpu.weight file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory) && !cg->memory.filename_detailed && (cgroup_used_memory || cgroup_enable_systemd_services_detailed_memory || !is_cgroup_systemd_service(cg)))) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.stat", cgroup_unified_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->memory.filename_detailed = strdupz(filename);
+ cg->memory.enabled_detailed = (cgroup_enable_detailed_memory == CONFIG_BOOLEAN_YES)?CONFIG_BOOLEAN_YES:CONFIG_BOOLEAN_AUTO;
+ debug(D_CGROUP, "memory.stat filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_detailed);
+ }
+ else
+ debug(D_CGROUP, "memory.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely(cgroup_enable_memory && !cg->memory.filename_usage_in_bytes)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.current", cgroup_unified_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->memory.filename_usage_in_bytes = strdupz(filename);
+ cg->memory.enabled_usage_in_bytes = cgroup_enable_memory;
+ debug(D_CGROUP, "memory.current filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_usage_in_bytes);
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.max", cgroup_unified_base, cg->id);
+ cg->filename_memory_limit = strdupz(filename);
+ }
+ else
+ debug(D_CGROUP, "memory.current file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if(unlikely(cgroup_enable_swap && !cg->memory.filename_msw_usage_in_bytes)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.swap.current", cgroup_unified_base, cg->id);
+ if(likely(stat(filename, &buf) != -1)) {
+ cg->memory.filename_msw_usage_in_bytes = strdupz(filename);
+ cg->memory.enabled_msw_usage_in_bytes = cgroup_enable_swap;
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.swap.max", cgroup_unified_base, cg->id);
+ cg->filename_memoryswap_limit = strdupz(filename);
+ debug(D_CGROUP, "memory.swap.current filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_msw_usage_in_bytes);
+ }
+ else
+ debug(D_CGROUP, "memory.swap file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+ }
+
+ if (unlikely(cgroup_enable_pressure_cpu && !cg->cpu_pressure.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/cpu.pressure", cgroup_unified_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->cpu_pressure.filename = strdupz(filename);
+ cg->cpu_pressure.some.enabled = cgroup_enable_pressure_cpu;
+ cg->cpu_pressure.full.enabled = CONFIG_BOOLEAN_NO;
+ debug(D_CGROUP, "cpu.pressure filename for cgroup '%s': '%s'", cg->id, cg->cpu_pressure.filename);
+ } else {
+ debug(D_CGROUP, "cpu.pressure file for cgroup '%s': '%s' does not exist", cg->id, filename);
+ }
+ }
+
+ if (unlikely((cgroup_enable_pressure_io_some || cgroup_enable_pressure_io_full) && !cg->io_pressure.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/io.pressure", cgroup_unified_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->io_pressure.filename = strdupz(filename);
+ cg->io_pressure.some.enabled = cgroup_enable_pressure_io_some;
+ cg->io_pressure.full.enabled = cgroup_enable_pressure_io_full;
+ debug(D_CGROUP, "io.pressure filename for cgroup '%s': '%s'", cg->id, cg->io_pressure.filename);
+ } else {
+ debug(D_CGROUP, "io.pressure file for cgroup '%s': '%s' does not exist", cg->id, filename);
+ }
+ }
+
+ if (unlikely((cgroup_enable_pressure_memory_some || cgroup_enable_pressure_memory_full) && !cg->memory_pressure.filename)) {
+ snprintfz(filename, FILENAME_MAX, "%s%s/memory.pressure", cgroup_unified_base, cg->id);
+ if (likely(stat(filename, &buf) != -1)) {
+ cg->memory_pressure.filename = strdupz(filename);
+ cg->memory_pressure.some.enabled = cgroup_enable_pressure_memory_some;
+ cg->memory_pressure.full.enabled = cgroup_enable_pressure_memory_full;
+ debug(D_CGROUP, "memory.pressure filename for cgroup '%s': '%s'", cg->id, cg->memory_pressure.filename);
+ } else {
+ debug(D_CGROUP, "memory.pressure file for cgroup '%s': '%s' does not exist", cg->id, filename);
+ }
+ }
+ }
+ }
+}
+
+static inline void discovery_cleanup_all_cgroups() {
+ struct cgroup *cg = discovered_cgroup_root, *last = NULL;
+
+ for(; cg ;) {
+ if(!cg->available) {
+ // enable the first duplicate cgroup
+ {
+ struct cgroup *t;
+ for(t = discovered_cgroup_root; t ; t = t->discovered_next) {
+ if(t != cg && t->available && !t->enabled && t->options & CGROUP_OPTIONS_DISABLED_DUPLICATE && t->hash_chart == cg->hash_chart && !strcmp(t->chart_id, cg->chart_id)) {
+ debug(D_CGROUP, "Enabling duplicate of cgroup '%s' with id '%s', because the original with id '%s' stopped.", t->chart_id, t->id, cg->id);
+ t->enabled = 1;
+ t->options &= ~CGROUP_OPTIONS_DISABLED_DUPLICATE;
+ break;
+ }
+ }
+ }
+
+ if(!last)
+ discovered_cgroup_root = cg->discovered_next;
+ else
+ last->discovered_next = cg->discovered_next;
+
+ cgroup_free(cg);
+
+ if(!last)
+ cg = discovered_cgroup_root;
+ else
+ cg = last->discovered_next;
+ }
+ else {
+ last = cg;
+ cg = cg->discovered_next;
+ }
+ }
+}
+
+static inline void discovery_copy_discovered_cgroups_to_reader() {
+ debug(D_CGROUP, "copy discovered cgroups to the main group list");
+
+ struct cgroup *cg;
+
+ for (cg = discovered_cgroup_root; cg; cg = cg->discovered_next) {
+ cg->next = cg->discovered_next;
+ }
+
+ cgroup_root = discovered_cgroup_root;
+}
+
+static inline void discovery_share_cgroups_with_ebpf() {
+ struct cgroup *cg;
+ int count;
+ struct stat buf;
+
+ if (shm_mutex_cgroup_ebpf == SEM_FAILED) {
+ return;
+ }
+ sem_wait(shm_mutex_cgroup_ebpf);
+
+ for (cg = cgroup_root, count = 0; cg; cg = cg->next, count++) {
+ netdata_ebpf_cgroup_shm_body_t *ptr = &shm_cgroup_ebpf.body[count];
+ char *prefix = (is_cgroup_systemd_service(cg)) ? "" : "cgroup_";
+ snprintfz(ptr->name, CGROUP_EBPF_NAME_SHARED_LENGTH - 1, "%s%s", prefix, cg->chart_title);
+ ptr->hash = simple_hash(ptr->name);
+ ptr->options = cg->options;
+ ptr->enabled = cg->enabled;
+ if (cgroup_use_unified_cgroups) {
+ snprintfz(ptr->path, FILENAME_MAX, "%s%s/cgroup.procs", cgroup_unified_base, cg->id);
+ if (likely(stat(ptr->path, &buf) == -1)) {
+ ptr->path[0] = '\0';
+ ptr->enabled = 0;
+ }
+ } else {
+ is_cgroup_procs_exist(ptr, cg->id);
+ }
+
+ debug(D_CGROUP, "cgroup shared: NAME=%s, ENABLED=%d", ptr->name, ptr->enabled);
+ }
+
+ shm_cgroup_ebpf.header->cgroup_root_count = count;
+ sem_post(shm_mutex_cgroup_ebpf);
+}
+
+static inline void discovery_find_all_cgroups_v1() {
+ if (cgroup_enable_cpuacct_stat || cgroup_enable_cpuacct_usage) {
+ if (discovery_find_dir_in_subdirs(cgroup_cpuacct_base, NULL, discovery_find_cgroup_in_dir_callback) == -1) {
+ cgroup_enable_cpuacct_stat = cgroup_enable_cpuacct_usage = CONFIG_BOOLEAN_NO;
+ error("CGROUP: disabled cpu statistics.");
+ }
+ }
+
+ if (cgroup_enable_blkio_io || cgroup_enable_blkio_ops || cgroup_enable_blkio_throttle_io ||
+ cgroup_enable_blkio_throttle_ops || cgroup_enable_blkio_merged_ops || cgroup_enable_blkio_queued_ops) {
+ if (discovery_find_dir_in_subdirs(cgroup_blkio_base, NULL, discovery_find_cgroup_in_dir_callback) == -1) {
+ cgroup_enable_blkio_io = cgroup_enable_blkio_ops = cgroup_enable_blkio_throttle_io =
+ cgroup_enable_blkio_throttle_ops = cgroup_enable_blkio_merged_ops = cgroup_enable_blkio_queued_ops =
+ CONFIG_BOOLEAN_NO;
+ error("CGROUP: disabled blkio statistics.");
+ }
+ }
+
+ if (cgroup_enable_memory || cgroup_enable_detailed_memory || cgroup_enable_swap || cgroup_enable_memory_failcnt) {
+ if (discovery_find_dir_in_subdirs(cgroup_memory_base, NULL, discovery_find_cgroup_in_dir_callback) == -1) {
+ cgroup_enable_memory = cgroup_enable_detailed_memory = cgroup_enable_swap = cgroup_enable_memory_failcnt =
+ CONFIG_BOOLEAN_NO;
+ error("CGROUP: disabled memory statistics.");
+ }
+ }
+
+ if (cgroup_search_in_devices) {
+ if (discovery_find_dir_in_subdirs(cgroup_devices_base, NULL, discovery_find_cgroup_in_dir_callback) == -1) {
+ cgroup_search_in_devices = 0;
+ error("CGROUP: disabled devices statistics.");
+ }
+ }
+}
+
+static inline void discovery_find_all_cgroups_v2() {
+ if (discovery_find_dir_in_subdirs(cgroup_unified_base, NULL, discovery_find_cgroup_in_dir_callback) == -1) {
+ cgroup_unified_exist = CONFIG_BOOLEAN_NO;
+ error("CGROUP: disabled unified cgroups statistics.");
+ }
+}
+
+static int is_digits_only(const char *s) {
+ do {
+ if (!isdigit(*s++)) {
+ return 0;
+ }
+ } while (*s);
+
+ return 1;
+}
+
+static inline void discovery_process_first_time_seen_cgroup(struct cgroup *cg) {
+ if (!cg->first_time_seen) {
+ return;
+ }
+ cg->first_time_seen = 0;
+
+ char comm[TASK_COMM_LEN];
+
+ if (cg->container_orchestrator == CGROUPS_ORCHESTRATOR_UNSET) {
+ if (strstr(cg->id, "kubepods")) {
+ cg->container_orchestrator = CGROUPS_ORCHESTRATOR_K8S;
+ } else {
+ cg->container_orchestrator = CGROUPS_ORCHESTRATOR_UNKNOWN;
+ }
+ }
+
+ if (is_inside_k8s && !k8s_get_container_first_proc_comm(cg->id, comm)) {
+ // container initialization may take some time when CPU % is high
+ // seen on GKE: comm is '6' before 'runc:[2:INIT]' (dunno if it could be another number)
+ if (is_digits_only(comm) || matches_entrypoint_parent_process_comm(comm)) {
+ cg->first_time_seen = 1;
+ return;
+ }
+ if (!strcmp(comm, "pause")) {
+ // a container that holds the network namespace for the pod
+ // we don't need to collect its metrics
+ cg->processed = 1;
+ return;
+ }
+ }
+
+ if (cgroup_enable_systemd_services && matches_systemd_services_cgroups(cg->id)) {
+ debug(D_CGROUP, "cgroup '%s' (name '%s') matches 'cgroups to match as systemd services'", cg->id, cg->chart_title);
+ convert_cgroup_to_systemd_service(cg);
+ return;
+ }
+
+ if (matches_enabled_cgroup_renames(cg->id)) {
+ debug(D_CGROUP, "cgroup '%s' (name '%s') matches 'run script to rename cgroups matching', will try to rename it", cg->id, cg->chart_title);
+ if (is_inside_k8s && k8s_is_container(cg->id)) {
+ // it may take up to a minute for the K8s API to return data for the container
+ // tested on AWS K8s cluster with 100% CPU utilization
+ cg->pending_renames = 9; // 1.5 minute
+ } else {
+ cg->pending_renames = 2;
+ }
+ }
+}
+
+static int discovery_is_cgroup_duplicate(struct cgroup *cg) {
+ // https://github.com/netdata/netdata/issues/797#issuecomment-241248884
+ struct cgroup *c;
+ for (c = discovered_cgroup_root; c; c = c->discovered_next) {
+ if (c != cg && c->enabled && c->hash_chart == cg->hash_chart && !strcmp(c->chart_id, cg->chart_id)) {
+ error("CGROUP: chart id '%s' already exists with id '%s' and is enabled and available. Disabling cgroup with id '%s'.", cg->chart_id, c->id, cg->id);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+static inline void discovery_process_cgroup(struct cgroup *cg) {
+ if (!cg) {
+ debug(D_CGROUP, "discovery_process_cgroup() received NULL");
+ return;
+ }
+ if (!cg->available || cg->processed) {
+ return;
+ }
+
+ if (cg->first_time_seen) {
+ worker_is_busy(WORKER_DISCOVERY_PROCESS_FIRST_TIME);
+ discovery_process_first_time_seen_cgroup(cg);
+ if (unlikely(cg->first_time_seen || cg->processed)) {
+ return;
+ }
+ }
+
+ if (cg->pending_renames) {
+ worker_is_busy(WORKER_DISCOVERY_PROCESS_RENAME);
+ discovery_rename_cgroup(cg);
+ if (unlikely(cg->pending_renames || cg->processed)) {
+ return;
+ }
+ }
+
+ cg->processed = 1;
+
+ if ((strlen(cg->chart_id) + strlen(cgroup_chart_id_prefix)) >= RRD_ID_LENGTH_MAX) {
+ info("cgroup '%s' (chart id '%s') disabled because chart_id exceeds the limit (RRD_ID_LENGTH_MAX)", cg->id, cg->chart_id);
+ return;
+ }
+
+ if (is_cgroup_systemd_service(cg)) {
+ cg->enabled = 1;
+ return;
+ }
+
+ if (!(cg->enabled = matches_enabled_cgroup_names(cg->chart_title))) {
+ debug(D_CGROUP, "cgroup '%s' (name '%s') disabled by 'enable by default cgroups names matching'", cg->id, cg->chart_title);
+ return;
+ }
+
+ if (!(cg->enabled = matches_enabled_cgroup_paths(cg->id))) {
+ debug(D_CGROUP, "cgroup '%s' (name '%s') disabled by 'enable by default cgroups matching'", cg->id, cg->chart_title);
+ return;
+ }
+
+ if (discovery_is_cgroup_duplicate(cg)) {
+ cg->enabled = 0;
+ cg->options |= CGROUP_OPTIONS_DISABLED_DUPLICATE;
+ return;
+ }
+
+ worker_is_busy(WORKER_DISCOVERY_PROCESS_NETWORK);
+ read_cgroup_network_interfaces(cg);
+}
+
+static inline void discovery_find_all_cgroups() {
+ debug(D_CGROUP, "searching for cgroups");
+
+ worker_is_busy(WORKER_DISCOVERY_INIT);
+ discovery_mark_all_cgroups_as_unavailable();
+
+ worker_is_busy(WORKER_DISCOVERY_FIND);
+ if (!cgroup_use_unified_cgroups) {
+ discovery_find_all_cgroups_v1();
+ } else {
+ discovery_find_all_cgroups_v2();
+ }
+
+ struct cgroup *cg;
+ for (cg = discovered_cgroup_root; cg; cg = cg->discovered_next) {
+ worker_is_busy(WORKER_DISCOVERY_PROCESS);
+ discovery_process_cgroup(cg);
+ }
+
+ worker_is_busy(WORKER_DISCOVERY_UPDATE);
+ discovery_update_filenames();
+
+ worker_is_busy(WORKER_DISCOVERY_LOCK);
+ uv_mutex_lock(&cgroup_root_mutex);
+
+ worker_is_busy(WORKER_DISCOVERY_CLEANUP);
+ discovery_cleanup_all_cgroups();
+
+ worker_is_busy(WORKER_DISCOVERY_COPY);
+ discovery_copy_discovered_cgroups_to_reader();
+
+ uv_mutex_unlock(&cgroup_root_mutex);
+
+ worker_is_busy(WORKER_DISCOVERY_SHARE);
+ discovery_share_cgroups_with_ebpf();
+
+ debug(D_CGROUP, "done searching for cgroups");
+}
+
+void cgroup_discovery_worker(void *ptr)
+{
+ UNUSED(ptr);
+
+ worker_register("CGROUPSDISC");
+ worker_register_job_name(WORKER_DISCOVERY_INIT, "init");
+ worker_register_job_name(WORKER_DISCOVERY_FIND, "find");
+ worker_register_job_name(WORKER_DISCOVERY_PROCESS, "process");
+ worker_register_job_name(WORKER_DISCOVERY_PROCESS_RENAME, "rename");
+ worker_register_job_name(WORKER_DISCOVERY_PROCESS_NETWORK, "network");
+ worker_register_job_name(WORKER_DISCOVERY_PROCESS_FIRST_TIME, "new");
+ worker_register_job_name(WORKER_DISCOVERY_UPDATE, "update");
+ worker_register_job_name(WORKER_DISCOVERY_CLEANUP, "cleanup");
+ worker_register_job_name(WORKER_DISCOVERY_COPY, "copy");
+ worker_register_job_name(WORKER_DISCOVERY_SHARE, "share");
+ worker_register_job_name(WORKER_DISCOVERY_LOCK, "lock");
+
+ entrypoint_parent_process_comm = simple_pattern_create(
+ " runc:[* " // http://terenceli.github.io/%E6%8A%80%E6%9C%AF/2021/12/28/runc-internals-3)
+ " exe ", // https://github.com/falcosecurity/falco/blob/9d41b0a151b83693929d3a9c84f7c5c85d070d3a/rules/falco_rules.yaml#L1961
+ NULL,
+ SIMPLE_PATTERN_EXACT);
+
+ while (!netdata_exit) {
+ worker_is_idle();
+
+ uv_mutex_lock(&discovery_thread.mutex);
+ while (!discovery_thread.start_discovery)
+ uv_cond_wait(&discovery_thread.cond_var, &discovery_thread.mutex);
+ discovery_thread.start_discovery = 0;
+ uv_mutex_unlock(&discovery_thread.mutex);
+
+ if (unlikely(netdata_exit))
+ break;
+
+ discovery_find_all_cgroups();
+ }
+
+ discovery_thread.exited = 1;
+ worker_unregister();
+}
+
+// ----------------------------------------------------------------------------
+// generate charts
+
+#define CHART_TITLE_MAX 300
+
+void update_systemd_services_charts(
+ int update_every
+ , int do_cpu
+ , int do_mem_usage
+ , int do_mem_detailed
+ , int do_mem_failcnt
+ , int do_swap_usage
+ , int do_io
+ , int do_io_ops
+ , int do_throttle_io
+ , int do_throttle_ops
+ , int do_queued_ops
+ , int do_merged_ops
+) {
+ static RRDSET
+ *st_cpu = NULL,
+ *st_mem_usage = NULL,
+ *st_mem_failcnt = NULL,
+ *st_swap_usage = NULL,
+
+ *st_mem_detailed_cache = NULL,
+ *st_mem_detailed_rss = NULL,
+ *st_mem_detailed_mapped = NULL,
+ *st_mem_detailed_writeback = NULL,
+ *st_mem_detailed_pgfault = NULL,
+ *st_mem_detailed_pgmajfault = NULL,
+ *st_mem_detailed_pgpgin = NULL,
+ *st_mem_detailed_pgpgout = NULL,
+
+ *st_io_read = NULL,
+ *st_io_serviced_read = NULL,
+ *st_throttle_io_read = NULL,
+ *st_throttle_ops_read = NULL,
+ *st_queued_ops_read = NULL,
+ *st_merged_ops_read = NULL,
+
+ *st_io_write = NULL,
+ *st_io_serviced_write = NULL,
+ *st_throttle_io_write = NULL,
+ *st_throttle_ops_write = NULL,
+ *st_queued_ops_write = NULL,
+ *st_merged_ops_write = NULL;
+
+ // create the charts
+
+ if (unlikely(do_cpu && !st_cpu)) {
+ char title[CHART_TITLE_MAX + 1];
+ snprintfz(title, CHART_TITLE_MAX, "Systemd Services CPU utilization (100%% = 1 core)");
+
+ st_cpu = rrdset_create_localhost(
+ "services"
+ , "cpu"
+ , NULL
+ , "cpu"
+ , "services.cpu"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if (unlikely(do_mem_usage && !st_mem_usage)) {
+ st_mem_usage = rrdset_create_localhost(
+ "services"
+ , "mem_usage"
+ , NULL
+ , "mem"
+ , "services.mem_usage"
+ , "Systemd Services Used Memory"
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 10
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(likely(do_mem_detailed)) {
+ if(unlikely(!st_mem_detailed_rss)) {
+ st_mem_detailed_rss = rrdset_create_localhost(
+ "services"
+ , "mem_rss"
+ , NULL
+ , "mem"
+ , "services.mem_rss"
+ , "Systemd Services RSS Memory"
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 20
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_mem_detailed_mapped)) {
+ st_mem_detailed_mapped = rrdset_create_localhost(
+ "services"
+ , "mem_mapped"
+ , NULL
+ , "mem"
+ , "services.mem_mapped"
+ , "Systemd Services Mapped Memory"
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 30
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_mem_detailed_cache)) {
+ st_mem_detailed_cache = rrdset_create_localhost(
+ "services"
+ , "mem_cache"
+ , NULL
+ , "mem"
+ , "services.mem_cache"
+ , "Systemd Services Cache Memory"
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 40
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_mem_detailed_writeback)) {
+ st_mem_detailed_writeback = rrdset_create_localhost(
+ "services"
+ , "mem_writeback"
+ , NULL
+ , "mem"
+ , "services.mem_writeback"
+ , "Systemd Services Writeback Memory"
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 50
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ }
+
+ if(unlikely(!st_mem_detailed_pgfault)) {
+ st_mem_detailed_pgfault = rrdset_create_localhost(
+ "services"
+ , "mem_pgfault"
+ , NULL
+ , "mem"
+ , "services.mem_pgfault"
+ , "Systemd Services Memory Minor Page Faults"
+ , "MiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 60
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_mem_detailed_pgmajfault)) {
+ st_mem_detailed_pgmajfault = rrdset_create_localhost(
+ "services"
+ , "mem_pgmajfault"
+ , NULL
+ , "mem"
+ , "services.mem_pgmajfault"
+ , "Systemd Services Memory Major Page Faults"
+ , "MiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 70
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_mem_detailed_pgpgin)) {
+ st_mem_detailed_pgpgin = rrdset_create_localhost(
+ "services"
+ , "mem_pgpgin"
+ , NULL
+ , "mem"
+ , "services.mem_pgpgin"
+ , "Systemd Services Memory Charging Activity"
+ , "MiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 80
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ }
+
+ if(unlikely(!st_mem_detailed_pgpgout)) {
+ st_mem_detailed_pgpgout = rrdset_create_localhost(
+ "services"
+ , "mem_pgpgout"
+ , NULL
+ , "mem"
+ , "services.mem_pgpgout"
+ , "Systemd Services Memory Uncharging Activity"
+ , "MiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 90
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+ }
+
+ if(unlikely(do_mem_failcnt && !st_mem_failcnt)) {
+ st_mem_failcnt = rrdset_create_localhost(
+ "services"
+ , "mem_failcnt"
+ , NULL
+ , "mem"
+ , "services.mem_failcnt"
+ , "Systemd Services Memory Limit Failures"
+ , "failures"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 110
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if (do_swap_usage && !st_swap_usage) {
+ st_swap_usage = rrdset_create_localhost(
+ "services"
+ , "swap_usage"
+ , NULL
+ , "swap"
+ , "services.swap_usage"
+ , "Systemd Services Swap Memory Used"
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 100
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(likely(do_io)) {
+ if(unlikely(!st_io_read)) {
+ st_io_read = rrdset_create_localhost(
+ "services"
+ , "io_read"
+ , NULL
+ , "disk"
+ , "services.io_read"
+ , "Systemd Services Disk Read Bandwidth"
+ , "KiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 120
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_io_write)) {
+ st_io_write = rrdset_create_localhost(
+ "services"
+ , "io_write"
+ , NULL
+ , "disk"
+ , "services.io_write"
+ , "Systemd Services Disk Write Bandwidth"
+ , "KiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 130
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+ }
+
+ if(likely(do_io_ops)) {
+ if(unlikely(!st_io_serviced_read)) {
+ st_io_serviced_read = rrdset_create_localhost(
+ "services"
+ , "io_ops_read"
+ , NULL
+ , "disk"
+ , "services.io_ops_read"
+ , "Systemd Services Disk Read Operations"
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 140
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_io_serviced_write)) {
+ st_io_serviced_write = rrdset_create_localhost(
+ "services"
+ , "io_ops_write"
+ , NULL
+ , "disk"
+ , "services.io_ops_write"
+ , "Systemd Services Disk Write Operations"
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 150
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+ }
+
+ if(likely(do_throttle_io)) {
+ if(unlikely(!st_throttle_io_read)) {
+
+ st_throttle_io_read = rrdset_create_localhost(
+ "services"
+ , "throttle_io_read"
+ , NULL
+ , "disk"
+ , "services.throttle_io_read"
+ , "Systemd Services Throttle Disk Read Bandwidth"
+ , "KiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 160
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ }
+
+ if(unlikely(!st_throttle_io_write)) {
+ st_throttle_io_write = rrdset_create_localhost(
+ "services"
+ , "throttle_io_write"
+ , NULL
+ , "disk"
+ , "services.throttle_io_write"
+ , "Systemd Services Throttle Disk Write Bandwidth"
+ , "KiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 170
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+ }
+
+ if(likely(do_throttle_ops)) {
+ if(unlikely(!st_throttle_ops_read)) {
+ st_throttle_ops_read = rrdset_create_localhost(
+ "services"
+ , "throttle_io_ops_read"
+ , NULL
+ , "disk"
+ , "services.throttle_io_ops_read"
+ , "Systemd Services Throttle Disk Read Operations"
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 180
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_throttle_ops_write)) {
+ st_throttle_ops_write = rrdset_create_localhost(
+ "services"
+ , "throttle_io_ops_write"
+ , NULL
+ , "disk"
+ , "services.throttle_io_ops_write"
+ , "Systemd Services Throttle Disk Write Operations"
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 190
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+ }
+
+ if(likely(do_queued_ops)) {
+ if(unlikely(!st_queued_ops_read)) {
+ st_queued_ops_read = rrdset_create_localhost(
+ "services"
+ , "queued_io_ops_read"
+ , NULL
+ , "disk"
+ , "services.queued_io_ops_read"
+ , "Systemd Services Queued Disk Read Operations"
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 200
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_queued_ops_write)) {
+
+ st_queued_ops_write = rrdset_create_localhost(
+ "services"
+ , "queued_io_ops_write"
+ , NULL
+ , "disk"
+ , "services.queued_io_ops_write"
+ , "Systemd Services Queued Disk Write Operations"
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 210
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+ }
+
+ if(likely(do_merged_ops)) {
+ if(unlikely(!st_merged_ops_read)) {
+ st_merged_ops_read = rrdset_create_localhost(
+ "services"
+ , "merged_io_ops_read"
+ , NULL
+ , "disk"
+ , "services.merged_io_ops_read"
+ , "Systemd Services Merged Disk Read Operations"
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 220
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ if(unlikely(!st_merged_ops_write)) {
+ st_merged_ops_write = rrdset_create_localhost(
+ "services"
+ , "merged_io_ops_write"
+ , NULL
+ , "disk"
+ , "services.merged_io_ops_write"
+ , "Systemd Services Merged Disk Write Operations"
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
+ , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 230
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+ }
+
+ // update the values
+ struct cgroup *cg;
+ for(cg = cgroup_root; cg ; cg = cg->next) {
+ if(unlikely(!cg->enabled || cg->pending_renames || !is_cgroup_systemd_service(cg)))
+ continue;
+
+ if(likely(do_cpu && cg->cpuacct_stat.updated)) {
+ if(unlikely(!cg->rd_cpu)){
+
+
+ if (!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ cg->rd_cpu = rrddim_add(st_cpu, cg->chart_id, cg->chart_title, 100, system_hz, RRD_ALGORITHM_INCREMENTAL);
+ } else {
+ cg->rd_cpu = rrddim_add(st_cpu, cg->chart_id, cg->chart_title, 100, 1000000, RRD_ALGORITHM_INCREMENTAL);
+ }
+ }
+
+ rrddim_set_by_pointer(st_cpu, cg->rd_cpu, cg->cpuacct_stat.user + cg->cpuacct_stat.system);
+ }
+
+ if(likely(do_mem_usage && cg->memory.updated_usage_in_bytes)) {
+ if(unlikely(!cg->rd_mem_usage))
+ cg->rd_mem_usage = rrddim_add(st_mem_usage, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(st_mem_usage, cg->rd_mem_usage, cg->memory.usage_in_bytes);
+ }
+
+ if(likely(do_mem_detailed && cg->memory.updated_detailed)) {
+ if(unlikely(!cg->rd_mem_detailed_rss))
+ cg->rd_mem_detailed_rss = rrddim_add(st_mem_detailed_rss, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(st_mem_detailed_rss, cg->rd_mem_detailed_rss, cg->memory.total_rss);
+
+ if(unlikely(!cg->rd_mem_detailed_mapped))
+ cg->rd_mem_detailed_mapped = rrddim_add(st_mem_detailed_mapped, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(st_mem_detailed_mapped, cg->rd_mem_detailed_mapped, cg->memory.total_mapped_file);
+
+ if(unlikely(!cg->rd_mem_detailed_cache))
+ cg->rd_mem_detailed_cache = rrddim_add(st_mem_detailed_cache, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(st_mem_detailed_cache, cg->rd_mem_detailed_cache, cg->memory.total_cache);
+
+ if(unlikely(!cg->rd_mem_detailed_writeback))
+ cg->rd_mem_detailed_writeback = rrddim_add(st_mem_detailed_writeback, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(st_mem_detailed_writeback, cg->rd_mem_detailed_writeback, cg->memory.total_writeback);
+
+ if(unlikely(!cg->rd_mem_detailed_pgfault))
+ cg->rd_mem_detailed_pgfault = rrddim_add(st_mem_detailed_pgfault, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_mem_detailed_pgfault, cg->rd_mem_detailed_pgfault, cg->memory.total_pgfault);
+
+ if(unlikely(!cg->rd_mem_detailed_pgmajfault))
+ cg->rd_mem_detailed_pgmajfault = rrddim_add(st_mem_detailed_pgmajfault, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_mem_detailed_pgmajfault, cg->rd_mem_detailed_pgmajfault, cg->memory.total_pgmajfault);
+
+ if(unlikely(!cg->rd_mem_detailed_pgpgin))
+ cg->rd_mem_detailed_pgpgin = rrddim_add(st_mem_detailed_pgpgin, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_mem_detailed_pgpgin, cg->rd_mem_detailed_pgpgin, cg->memory.total_pgpgin);
+
+ if(unlikely(!cg->rd_mem_detailed_pgpgout))
+ cg->rd_mem_detailed_pgpgout = rrddim_add(st_mem_detailed_pgpgout, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_mem_detailed_pgpgout, cg->rd_mem_detailed_pgpgout, cg->memory.total_pgpgout);
+ }
+
+ if(likely(do_mem_failcnt && cg->memory.updated_failcnt)) {
+ if(unlikely(!cg->rd_mem_failcnt))
+ cg->rd_mem_failcnt = rrddim_add(st_mem_failcnt, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_mem_failcnt, cg->rd_mem_failcnt, cg->memory.failcnt);
+ }
+
+ if(likely(do_swap_usage && cg->memory.updated_msw_usage_in_bytes)) {
+ if(unlikely(!cg->rd_swap_usage))
+ cg->rd_swap_usage = rrddim_add(st_swap_usage, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ rrddim_set_by_pointer(
+ st_swap_usage,
+ cg->rd_swap_usage,
+ cg->memory.msw_usage_in_bytes > (cg->memory.usage_in_bytes + cg->memory.total_inactive_file) ?
+ cg->memory.msw_usage_in_bytes - (cg->memory.usage_in_bytes + cg->memory.total_inactive_file) : 0);
+ } else {
+ rrddim_set_by_pointer(st_swap_usage, cg->rd_swap_usage, cg->memory.msw_usage_in_bytes);
+ }
+ }
+
+ if(likely(do_io && cg->io_service_bytes.updated)) {
+ if(unlikely(!cg->rd_io_service_bytes_read))
+ cg->rd_io_service_bytes_read = rrddim_add(st_io_read, cg->chart_id, cg->chart_title, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_io_read, cg->rd_io_service_bytes_read, cg->io_service_bytes.Read);
+
+ if(unlikely(!cg->rd_io_service_bytes_write))
+ cg->rd_io_service_bytes_write = rrddim_add(st_io_write, cg->chart_id, cg->chart_title, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_io_write, cg->rd_io_service_bytes_write, cg->io_service_bytes.Write);
+ }
+
+ if(likely(do_io_ops && cg->io_serviced.updated)) {
+ if(unlikely(!cg->rd_io_serviced_read))
+ cg->rd_io_serviced_read = rrddim_add(st_io_serviced_read, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_io_serviced_read, cg->rd_io_serviced_read, cg->io_serviced.Read);
+
+ if(unlikely(!cg->rd_io_serviced_write))
+ cg->rd_io_serviced_write = rrddim_add(st_io_serviced_write, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_io_serviced_write, cg->rd_io_serviced_write, cg->io_serviced.Write);
+ }
+
+ if(likely(do_throttle_io && cg->throttle_io_service_bytes.updated)) {
+ if(unlikely(!cg->rd_throttle_io_read))
+ cg->rd_throttle_io_read = rrddim_add(st_throttle_io_read, cg->chart_id, cg->chart_title, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_throttle_io_read, cg->rd_throttle_io_read, cg->throttle_io_service_bytes.Read);
+
+ if(unlikely(!cg->rd_throttle_io_write))
+ cg->rd_throttle_io_write = rrddim_add(st_throttle_io_write, cg->chart_id, cg->chart_title, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_throttle_io_write, cg->rd_throttle_io_write, cg->throttle_io_service_bytes.Write);
+ }
+
+ if(likely(do_throttle_ops && cg->throttle_io_serviced.updated)) {
+ if(unlikely(!cg->rd_throttle_io_serviced_read))
+ cg->rd_throttle_io_serviced_read = rrddim_add(st_throttle_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_throttle_ops_read, cg->rd_throttle_io_serviced_read, cg->throttle_io_serviced.Read);
+
+ if(unlikely(!cg->rd_throttle_io_serviced_write))
+ cg->rd_throttle_io_serviced_write = rrddim_add(st_throttle_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_throttle_ops_write, cg->rd_throttle_io_serviced_write, cg->throttle_io_serviced.Write);
+ }
+
+ if(likely(do_queued_ops && cg->io_queued.updated)) {
+ if(unlikely(!cg->rd_io_queued_read))
+ cg->rd_io_queued_read = rrddim_add(st_queued_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_queued_ops_read, cg->rd_io_queued_read, cg->io_queued.Read);
+
+ if(unlikely(!cg->rd_io_queued_write))
+ cg->rd_io_queued_write = rrddim_add(st_queued_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_queued_ops_write, cg->rd_io_queued_write, cg->io_queued.Write);
+ }
+
+ if(likely(do_merged_ops && cg->io_merged.updated)) {
+ if(unlikely(!cg->rd_io_merged_read))
+ cg->rd_io_merged_read = rrddim_add(st_merged_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_merged_ops_read, cg->rd_io_merged_read, cg->io_merged.Read);
+
+ if(unlikely(!cg->rd_io_merged_write))
+ cg->rd_io_merged_write = rrddim_add(st_merged_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_merged_ops_write, cg->rd_io_merged_write, cg->io_merged.Write);
+ }
+ }
+
+ // complete the iteration
+ if(likely(do_cpu))
+ rrdset_done(st_cpu);
+
+ if(likely(do_mem_usage))
+ rrdset_done(st_mem_usage);
+
+ if(unlikely(do_mem_detailed)) {
+ rrdset_done(st_mem_detailed_cache);
+ rrdset_done(st_mem_detailed_rss);
+ rrdset_done(st_mem_detailed_mapped);
+ rrdset_done(st_mem_detailed_writeback);
+ rrdset_done(st_mem_detailed_pgfault);
+ rrdset_done(st_mem_detailed_pgmajfault);
+ rrdset_done(st_mem_detailed_pgpgin);
+ rrdset_done(st_mem_detailed_pgpgout);
+ }
+
+ if(likely(do_mem_failcnt))
+ rrdset_done(st_mem_failcnt);
+
+ if(likely(do_swap_usage))
+ rrdset_done(st_swap_usage);
+
+ if(likely(do_io)) {
+ rrdset_done(st_io_read);
+ rrdset_done(st_io_write);
+ }
+
+ if(likely(do_io_ops)) {
+ rrdset_done(st_io_serviced_read);
+ rrdset_done(st_io_serviced_write);
+ }
+
+ if(likely(do_throttle_io)) {
+ rrdset_done(st_throttle_io_read);
+ rrdset_done(st_throttle_io_write);
+ }
+
+ if(likely(do_throttle_ops)) {
+ rrdset_done(st_throttle_ops_read);
+ rrdset_done(st_throttle_ops_write);
+ }
+
+ if(likely(do_queued_ops)) {
+ rrdset_done(st_queued_ops_read);
+ rrdset_done(st_queued_ops_write);
+ }
+
+ if(likely(do_merged_ops)) {
+ rrdset_done(st_merged_ops_read);
+ rrdset_done(st_merged_ops_write);
+ }
+}
+
+static inline char *cgroup_chart_type(char *buffer, const char *id, size_t len) {
+ if(buffer[0]) return buffer;
+
+ if(id[0] == '\0' || (id[0] == '/' && id[1] == '\0'))
+ strncpy(buffer, "cgroup_root", len);
+ else
+ snprintfz(buffer, len, "%s%s", cgroup_chart_id_prefix, id);
+
+ netdata_fix_chart_id(buffer);
+ return buffer;
+}
+
+static inline unsigned long long cpuset_str2ull(char **s) {
+ unsigned long long n = 0;
+ char c;
+ for(c = **s; c >= '0' && c <= '9' ; c = *(++*s)) {
+ n *= 10;
+ n += c - '0';
+ }
+ return n;
+}
+
+static inline void update_cpu_limits(char **filename, unsigned long long *value, struct cgroup *cg) {
+ if(*filename) {
+ int ret = -1;
+
+ if(value == &cg->cpuset_cpus) {
+ static char *buf = NULL;
+ static size_t buf_size = 0;
+
+ if(!buf) {
+ buf_size = 100U + 6 * get_system_cpus(); // taken from kernel/cgroup/cpuset.c
+ buf = mallocz(buf_size + 1);
+ }
+
+ ret = read_file(*filename, buf, buf_size);
+
+ if(!ret) {
+ char *s = buf;
+ unsigned long long ncpus = 0;
+
+ // parse the cpuset string and calculate the number of cpus the cgroup is allowed to use
+ while(*s) {
+ unsigned long long n = cpuset_str2ull(&s);
+ ncpus++;
+ if(*s == ',') {
+ s++;
+ continue;
+ }
+ if(*s == '-') {
+ s++;
+ unsigned long long m = cpuset_str2ull(&s);
+ ncpus += m - n; // calculate the number of cpus in the region
+ }
+ s++;
+ }
+
+ if(likely(ncpus)) *value = ncpus;
+ }
+ }
+ else if(value == &cg->cpu_cfs_period) {
+ ret = read_single_number_file(*filename, value);
+ }
+ else if(value == &cg->cpu_cfs_quota) {
+ ret = read_single_number_file(*filename, value);
+ }
+ else ret = -1;
+
+ if(ret) {
+ error("Cannot refresh cgroup %s cpu limit by reading '%s'. Will not update its limit anymore.", cg->id, *filename);
+ freez(*filename);
+ *filename = NULL;
+ }
+ }
+}
+
+static inline void update_cpu_limits2(struct cgroup *cg) {
+ if(cg->filename_cpu_cfs_quota){
+ static procfile *ff = NULL;
+
+ ff = procfile_reopen(ff, cg->filename_cpu_cfs_quota, NULL, CGROUP_PROCFILE_FLAG);
+ if(unlikely(!ff)) {
+ goto cpu_limits2_err;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ goto cpu_limits2_err;
+ }
+
+ unsigned long lines = procfile_lines(ff);
+
+ if (unlikely(lines < 1)) {
+ error("CGROUP: file '%s' should have 1 lines.", cg->filename_cpu_cfs_quota);
+ return;
+ }
+
+ cg->cpu_cfs_period = str2ull(procfile_lineword(ff, 0, 1));
+ cg->cpuset_cpus = get_system_cpus();
+
+ char *s = "max\n\0";
+ if(strcmp(s, procfile_lineword(ff, 0, 0)) == 0){
+ cg->cpu_cfs_quota = cg->cpu_cfs_period * cg->cpuset_cpus;
+ } else {
+ cg->cpu_cfs_quota = str2ull(procfile_lineword(ff, 0, 0));
+ }
+ debug(D_CGROUP, "CPU limits values: %llu %llu %llu", cg->cpu_cfs_period, cg->cpuset_cpus, cg->cpu_cfs_quota);
+ return;
+
+cpu_limits2_err:
+ error("Cannot refresh cgroup %s cpu limit by reading '%s'. Will not update its limit anymore.", cg->id, cg->filename_cpu_cfs_quota);
+ freez(cg->filename_cpu_cfs_quota);
+ cg->filename_cpu_cfs_quota = NULL;
+
+ }
+}
+
+static inline int update_memory_limits(char **filename, const RRDSETVAR_ACQUIRED **chart_var, unsigned long long *value, const char *chart_var_name, struct cgroup *cg) {
+ if(*filename) {
+ if(unlikely(!*chart_var)) {
+ *chart_var = rrdsetvar_custom_chart_variable_add_and_acquire(cg->st_mem_usage, chart_var_name);
+ if(!*chart_var) {
+ error("Cannot create cgroup %s chart variable '%s'. Will not update its limit anymore.", cg->id, chart_var_name);
+ freez(*filename);
+ *filename = NULL;
+ }
+ }
+
+ if(*filename && *chart_var) {
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ if(read_single_number_file(*filename, value)) {
+ error("Cannot refresh cgroup %s memory limit by reading '%s'. Will not update its limit anymore.", cg->id, *filename);
+ freez(*filename);
+ *filename = NULL;
+ }
+ else {
+ rrdsetvar_custom_chart_variable_set(cg->st_mem_usage, *chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
+ return 1;
+ }
+ } else {
+ char buffer[30 + 1];
+ int ret = read_file(*filename, buffer, 30);
+ if(ret) {
+ error("Cannot refresh cgroup %s memory limit by reading '%s'. Will not update its limit anymore.", cg->id, *filename);
+ freez(*filename);
+ *filename = NULL;
+ return 0;
+ }
+ char *s = "max\n\0";
+ if(strcmp(s, buffer) == 0){
+ *value = UINT64_MAX;
+ rrdsetvar_custom_chart_variable_set(cg->st_mem_usage, *chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
+ return 1;
+ }
+ *value = str2ull(buffer);
+ rrdsetvar_custom_chart_variable_set(cg->st_mem_usage, *chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
+void update_cgroup_charts(int update_every) {
+ debug(D_CGROUP, "updating cgroups charts");
+
+ char type[RRD_ID_LENGTH_MAX + 1];
+ char title[CHART_TITLE_MAX + 1];
+
+ int services_do_cpu = 0,
+ services_do_mem_usage = 0,
+ services_do_mem_detailed = 0,
+ services_do_mem_failcnt = 0,
+ services_do_swap_usage = 0,
+ services_do_io = 0,
+ services_do_io_ops = 0,
+ services_do_throttle_io = 0,
+ services_do_throttle_ops = 0,
+ services_do_queued_ops = 0,
+ services_do_merged_ops = 0;
+
+ struct cgroup *cg;
+ for(cg = cgroup_root; cg ; cg = cg->next) {
+ if(unlikely(!cg->enabled || cg->pending_renames))
+ continue;
+
+ if(likely(cgroup_enable_systemd_services && is_cgroup_systemd_service(cg))) {
+ if(cg->cpuacct_stat.updated && cg->cpuacct_stat.enabled == CONFIG_BOOLEAN_YES) services_do_cpu++;
+
+ if(cgroup_enable_systemd_services_detailed_memory && cg->memory.updated_detailed && cg->memory.enabled_detailed) services_do_mem_detailed++;
+ if(cg->memory.updated_usage_in_bytes && cg->memory.enabled_usage_in_bytes == CONFIG_BOOLEAN_YES) services_do_mem_usage++;
+ if(cg->memory.updated_failcnt && cg->memory.enabled_failcnt == CONFIG_BOOLEAN_YES) services_do_mem_failcnt++;
+ if(cg->memory.updated_msw_usage_in_bytes && cg->memory.enabled_msw_usage_in_bytes == CONFIG_BOOLEAN_YES) services_do_swap_usage++;
+
+ if(cg->io_service_bytes.updated && cg->io_service_bytes.enabled == CONFIG_BOOLEAN_YES) services_do_io++;
+ if(cg->io_serviced.updated && cg->io_serviced.enabled == CONFIG_BOOLEAN_YES) services_do_io_ops++;
+ if(cg->throttle_io_service_bytes.updated && cg->throttle_io_service_bytes.enabled == CONFIG_BOOLEAN_YES) services_do_throttle_io++;
+ if(cg->throttle_io_serviced.updated && cg->throttle_io_serviced.enabled == CONFIG_BOOLEAN_YES) services_do_throttle_ops++;
+ if(cg->io_queued.updated && cg->io_queued.enabled == CONFIG_BOOLEAN_YES) services_do_queued_ops++;
+ if(cg->io_merged.updated && cg->io_merged.enabled == CONFIG_BOOLEAN_YES) services_do_merged_ops++;
+ continue;
+ }
+
+ type[0] = '\0';
+
+ if(likely(cg->cpuacct_stat.updated && cg->cpuacct_stat.enabled == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_cpu)) {
+ snprintfz(title, CHART_TITLE_MAX, "CPU Usage (100%% = 1 core)");
+
+ cg->st_cpu = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "cpu"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.cpu" : "cgroup.cpu"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_update_rrdlabels(cg->st_cpu, cg->chart_labels);
+
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ rrddim_add(cg->st_cpu, "user", NULL, 100, system_hz, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_cpu, "system", NULL, 100, system_hz, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else {
+ rrddim_add(cg->st_cpu, "user", NULL, 100, 1000000, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_cpu, "system", NULL, 100, 1000000, RRD_ALGORITHM_INCREMENTAL);
+ }
+ }
+
+ rrddim_set(cg->st_cpu, "user", cg->cpuacct_stat.user);
+ rrddim_set(cg->st_cpu, "system", cg->cpuacct_stat.system);
+ rrdset_done(cg->st_cpu);
+
+ if(likely(cg->filename_cpuset_cpus || cg->filename_cpu_cfs_period || cg->filename_cpu_cfs_quota)) {
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ update_cpu_limits(&cg->filename_cpuset_cpus, &cg->cpuset_cpus, cg);
+ update_cpu_limits(&cg->filename_cpu_cfs_period, &cg->cpu_cfs_period, cg);
+ update_cpu_limits(&cg->filename_cpu_cfs_quota, &cg->cpu_cfs_quota, cg);
+ } else {
+ update_cpu_limits2(cg);
+ }
+
+ if(unlikely(!cg->chart_var_cpu_limit)) {
+ cg->chart_var_cpu_limit = rrdsetvar_custom_chart_variable_add_and_acquire(cg->st_cpu, "cpu_limit");
+ if(!cg->chart_var_cpu_limit) {
+ error("Cannot create cgroup %s chart variable 'cpu_limit'. Will not update its limit anymore.", cg->id);
+ if(cg->filename_cpuset_cpus) freez(cg->filename_cpuset_cpus);
+ cg->filename_cpuset_cpus = NULL;
+ if(cg->filename_cpu_cfs_period) freez(cg->filename_cpu_cfs_period);
+ cg->filename_cpu_cfs_period = NULL;
+ if(cg->filename_cpu_cfs_quota) freez(cg->filename_cpu_cfs_quota);
+ cg->filename_cpu_cfs_quota = NULL;
+ }
+ }
+ else {
+ NETDATA_DOUBLE value = 0, quota = 0;
+
+ if(likely( ((!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) && (cg->filename_cpuset_cpus || (cg->filename_cpu_cfs_period && cg->filename_cpu_cfs_quota)))
+ || ((cg->options & CGROUP_OPTIONS_IS_UNIFIED) && cg->filename_cpu_cfs_quota))) {
+ if(unlikely(cg->cpu_cfs_quota > 0))
+ quota = (NETDATA_DOUBLE)cg->cpu_cfs_quota / (NETDATA_DOUBLE)cg->cpu_cfs_period;
+
+ if(unlikely(quota > 0 && quota < cg->cpuset_cpus))
+ value = quota * 100;
+ else
+ value = (NETDATA_DOUBLE)cg->cpuset_cpus * 100;
+ }
+ if(likely(value)) {
+ if(unlikely(!cg->st_cpu_limit)) {
+ snprintfz(title, CHART_TITLE_MAX, "CPU Usage within the limits");
+
+ cg->st_cpu_limit = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "cpu_limit"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.cpu_limit" : "cgroup.cpu_limit"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority - 1
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_cpu_limit, cg->chart_labels);
+
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED))
+ rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE);
+ else
+ rrddim_add(cg->st_cpu_limit, "used", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE);
+ cg->prev_cpu_usage = (NETDATA_DOUBLE)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
+ }
+
+ NETDATA_DOUBLE cpu_usage = 0;
+ cpu_usage = (NETDATA_DOUBLE)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
+ NETDATA_DOUBLE cpu_used = 100 * (cpu_usage - cg->prev_cpu_usage) / (value * update_every);
+
+ rrdset_isnot_obsolete(cg->st_cpu_limit);
+
+ rrddim_set(cg->st_cpu_limit, "used", (cpu_used > 0)?cpu_used:0);
+
+ cg->prev_cpu_usage = cpu_usage;
+
+ rrdsetvar_custom_chart_variable_set(cg->st_cpu, cg->chart_var_cpu_limit, value);
+ rrdset_done(cg->st_cpu_limit);
+ }
+ else {
+ if(unlikely(cg->st_cpu_limit)) {
+ rrdset_is_obsolete(cg->st_cpu_limit);
+ cg->st_cpu_limit = NULL;
+ }
+ rrdsetvar_custom_chart_variable_set(cg->st_cpu, cg->chart_var_cpu_limit, NAN);
+ }
+ }
+ }
+ }
+
+ if (likely(cg->cpuacct_cpu_throttling.updated && cg->cpuacct_cpu_throttling.enabled == CONFIG_BOOLEAN_YES)) {
+ if (unlikely(!cg->st_cpu_nr_throttled)) {
+ snprintfz(title, CHART_TITLE_MAX, "CPU Throttled Runnable Periods");
+
+ cg->st_cpu_nr_throttled = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "throttled"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.throttled" : "cgroup.throttled"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 10
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_cpu_nr_throttled, cg->chart_labels);
+ rrddim_add(cg->st_cpu_nr_throttled, "throttled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ } else {
+ rrddim_set(cg->st_cpu_nr_throttled, "throttled", cg->cpuacct_cpu_throttling.nr_throttled_perc);
+ rrdset_done(cg->st_cpu_nr_throttled);
+ }
+
+ if (unlikely(!cg->st_cpu_throttled_time)) {
+ snprintfz(title, CHART_TITLE_MAX, "CPU Throttled Time Duration");
+
+ cg->st_cpu_throttled_time = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "throttled_duration"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.throttled_duration" : "cgroup.throttled_duration"
+ , title
+ , "ms"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 15
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_cpu_throttled_time, cg->chart_labels);
+ rrddim_add(cg->st_cpu_throttled_time, "duration", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL);
+ } else {
+ rrddim_set(cg->st_cpu_throttled_time, "duration", cg->cpuacct_cpu_throttling.throttled_time);
+ rrdset_done(cg->st_cpu_throttled_time);
+ }
+ }
+
+ if (likely(cg->cpuacct_cpu_shares.updated && cg->cpuacct_cpu_shares.enabled == CONFIG_BOOLEAN_YES)) {
+ if (unlikely(!cg->st_cpu_shares)) {
+ snprintfz(title, CHART_TITLE_MAX, "CPU Time Relative Share");
+
+ cg->st_cpu_shares = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "cpu_shares"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.cpu_shares" : "cgroup.cpu_shares"
+ , title
+ , "shares"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 20
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_cpu_shares, cg->chart_labels);
+ rrddim_add(cg->st_cpu_shares, "shares", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ } else {
+ rrddim_set(cg->st_cpu_shares, "shares", cg->cpuacct_cpu_shares.shares);
+ rrdset_done(cg->st_cpu_shares);
+ }
+ }
+
+ if(likely(cg->cpuacct_usage.updated && cg->cpuacct_usage.enabled == CONFIG_BOOLEAN_YES)) {
+ char id[RRD_ID_LENGTH_MAX + 1];
+ unsigned int i;
+
+ if(unlikely(!cg->st_cpu_per_core)) {
+ snprintfz(title, CHART_TITLE_MAX, "CPU Usage (100%% = 1 core) Per Core");
+
+ cg->st_cpu_per_core = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "cpu_per_core"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.cpu_per_core" : "cgroup.cpu_per_core"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 100
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_update_rrdlabels(cg->st_cpu_per_core, cg->chart_labels);
+
+ for(i = 0; i < cg->cpuacct_usage.cpus; i++) {
+ snprintfz(id, RRD_ID_LENGTH_MAX, "cpu%u", i);
+ rrddim_add(cg->st_cpu_per_core, id, NULL, 100, 1000000000, RRD_ALGORITHM_INCREMENTAL);
+ }
+ }
+
+ for(i = 0; i < cg->cpuacct_usage.cpus ;i++) {
+ snprintfz(id, RRD_ID_LENGTH_MAX, "cpu%u", i);
+ rrddim_set(cg->st_cpu_per_core, id, cg->cpuacct_usage.cpu_percpu[i]);
+ }
+ rrdset_done(cg->st_cpu_per_core);
+ }
+
+ if(likely(cg->memory.updated_detailed && cg->memory.enabled_detailed == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_mem)) {
+ snprintfz(title, CHART_TITLE_MAX, "Memory Usage");
+
+ cg->st_mem = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "mem"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.mem" : "cgroup.mem"
+ , title
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 220
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_update_rrdlabels(cg->st_mem, cg->chart_labels);
+
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ rrddim_add(cg->st_mem, "cache", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem, "rss", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ if(cg->memory.detailed_has_swap)
+ rrddim_add(cg->st_mem, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_add(cg->st_mem, "rss_huge", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem, "mapped_file", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ } else {
+ rrddim_add(cg->st_mem, "anon", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem, "kernel_stack", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem, "slab", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem, "sock", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem, "anon_thp", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem, "file", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+ }
+
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ rrddim_set(cg->st_mem, "cache", cg->memory.total_cache);
+ rrddim_set(cg->st_mem, "rss", (cg->memory.total_rss > cg->memory.total_rss_huge)?(cg->memory.total_rss - cg->memory.total_rss_huge):0);
+
+ if(cg->memory.detailed_has_swap)
+ rrddim_set(cg->st_mem, "swap", cg->memory.total_swap);
+
+ rrddim_set(cg->st_mem, "rss_huge", cg->memory.total_rss_huge);
+ rrddim_set(cg->st_mem, "mapped_file", cg->memory.total_mapped_file);
+ } else {
+ rrddim_set(cg->st_mem, "anon", cg->memory.anon);
+ rrddim_set(cg->st_mem, "kernel_stack", cg->memory.kernel_stack);
+ rrddim_set(cg->st_mem, "slab", cg->memory.slab);
+ rrddim_set(cg->st_mem, "sock", cg->memory.sock);
+ rrddim_set(cg->st_mem, "anon_thp", cg->memory.anon_thp);
+ rrddim_set(cg->st_mem, "file", cg->memory.total_mapped_file);
+ }
+ rrdset_done(cg->st_mem);
+
+ if(unlikely(!cg->st_writeback)) {
+ snprintfz(title, CHART_TITLE_MAX, "Writeback Memory");
+
+ cg->st_writeback = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "writeback"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.writeback" : "cgroup.writeback"
+ , title
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 300
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_update_rrdlabels(cg->st_writeback, cg->chart_labels);
+
+ if(cg->memory.detailed_has_dirty)
+ rrddim_add(cg->st_writeback, "dirty", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_add(cg->st_writeback, "writeback", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if(cg->memory.detailed_has_dirty)
+ rrddim_set(cg->st_writeback, "dirty", cg->memory.total_dirty);
+
+ rrddim_set(cg->st_writeback, "writeback", cg->memory.total_writeback);
+ rrdset_done(cg->st_writeback);
+
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ if(unlikely(!cg->st_mem_activity)) {
+ snprintfz(title, CHART_TITLE_MAX, "Memory Activity");
+
+ cg->st_mem_activity = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "mem_activity"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.mem_activity" : "cgroup.mem_activity"
+ , title
+ , "MiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 400
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_mem_activity, cg->chart_labels);
+
+ rrddim_add(cg->st_mem_activity, "pgpgin", "in", system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_mem_activity, "pgpgout", "out", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(cg->st_mem_activity, "pgpgin", cg->memory.total_pgpgin);
+ rrddim_set(cg->st_mem_activity, "pgpgout", cg->memory.total_pgpgout);
+ rrdset_done(cg->st_mem_activity);
+ }
+
+ if(unlikely(!cg->st_pgfaults)) {
+ snprintfz(title, CHART_TITLE_MAX, "Memory Page Faults");
+
+ cg->st_pgfaults = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "pgfaults"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.pgfaults" : "cgroup.pgfaults"
+ , title
+ , "MiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 500
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_pgfaults, cg->chart_labels);
+
+ rrddim_add(cg->st_pgfaults, "pgfault", NULL, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_pgfaults, "pgmajfault", "swap", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(cg->st_pgfaults, "pgfault", cg->memory.total_pgfault);
+ rrddim_set(cg->st_pgfaults, "pgmajfault", cg->memory.total_pgmajfault);
+ rrdset_done(cg->st_pgfaults);
+ }
+
+ if(likely(cg->memory.updated_usage_in_bytes && cg->memory.enabled_usage_in_bytes == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_mem_usage)) {
+ snprintfz(title, CHART_TITLE_MAX, "Used Memory");
+
+ cg->st_mem_usage = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "mem_usage"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.mem_usage" : "cgroup.mem_usage"
+ , title
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 210
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_update_rrdlabels(cg->st_mem_usage, cg->chart_labels);
+
+ rrddim_add(cg->st_mem_usage, "ram", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem_usage, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(cg->st_mem_usage, "ram", cg->memory.usage_in_bytes);
+ if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
+ rrddim_set(
+ cg->st_mem_usage,
+ "swap",
+ cg->memory.msw_usage_in_bytes > (cg->memory.usage_in_bytes + cg->memory.total_inactive_file) ?
+ cg->memory.msw_usage_in_bytes - (cg->memory.usage_in_bytes + cg->memory.total_inactive_file) : 0);
+ } else {
+ rrddim_set(cg->st_mem_usage, "swap", cg->memory.msw_usage_in_bytes);
+ }
+ rrdset_done(cg->st_mem_usage);
+
+ if (likely(update_memory_limits(&cg->filename_memory_limit, &cg->chart_var_memory_limit, &cg->memory_limit, "memory_limit", cg))) {
+ static unsigned long long ram_total = 0;
+
+ if(unlikely(!ram_total)) {
+ procfile *ff = NULL;
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/meminfo");
+ ff = procfile_open(config_get("plugin:cgroups", "meminfo filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+
+ if(likely(ff))
+ ff = procfile_readall(ff);
+ if(likely(ff && procfile_lines(ff) && !strncmp(procfile_word(ff, 0), "MemTotal", 8)))
+ ram_total = str2ull(procfile_word(ff, 1)) * 1024;
+ else {
+ error("Cannot read file %s. Will not update cgroup %s RAM limit anymore.", filename, cg->id);
+ freez(cg->filename_memory_limit);
+ cg->filename_memory_limit = NULL;
+ }
+
+ procfile_close(ff);
+ }
+
+ if(likely(ram_total)) {
+ unsigned long long memory_limit = ram_total;
+
+ if(unlikely(cg->memory_limit < ram_total))
+ memory_limit = cg->memory_limit;
+
+ if(unlikely(!cg->st_mem_usage_limit)) {
+ snprintfz(title, CHART_TITLE_MAX, "Used RAM within the limits");
+
+ cg->st_mem_usage_limit = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "mem_usage_limit"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.mem_usage_limit": "cgroup.mem_usage_limit"
+ , title
+ , "MiB"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 200
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_update_rrdlabels(cg->st_mem_usage_limit, cg->chart_labels);
+
+ rrddim_add(cg->st_mem_usage_limit, "available", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_mem_usage_limit, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrdset_isnot_obsolete(cg->st_mem_usage_limit);
+
+ rrddim_set(cg->st_mem_usage_limit, "available", memory_limit - cg->memory.usage_in_bytes);
+ rrddim_set(cg->st_mem_usage_limit, "used", cg->memory.usage_in_bytes);
+ rrdset_done(cg->st_mem_usage_limit);
+
+ if (unlikely(!cg->st_mem_utilization)) {
+ snprintfz(title, CHART_TITLE_MAX, "Memory Utilization");
+
+ cg->st_mem_utilization = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "mem_utilization"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.mem_utilization" : "cgroup.mem_utilization"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 199
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_update_rrdlabels(cg->st_mem_utilization, cg->chart_labels);
+
+ rrddim_add(cg->st_mem_utilization, "utilization", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if (memory_limit) {
+ rrdset_isnot_obsolete(cg->st_mem_utilization);
+
+ rrddim_set(
+ cg->st_mem_utilization, "utilization", cg->memory.usage_in_bytes * 100 / memory_limit);
+ rrdset_done(cg->st_mem_utilization);
+ }
+ }
+ }
+ else {
+ if(unlikely(cg->st_mem_usage_limit)) {
+ rrdset_is_obsolete(cg->st_mem_usage_limit);
+ cg->st_mem_usage_limit = NULL;
+ }
+
+ if(unlikely(cg->st_mem_utilization)) {
+ rrdset_is_obsolete(cg->st_mem_utilization);
+ cg->st_mem_utilization = NULL;
+ }
+ }
+
+ update_memory_limits(&cg->filename_memoryswap_limit, &cg->chart_var_memoryswap_limit, &cg->memoryswap_limit, "memory_and_swap_limit", cg);
+ }
+
+ if(likely(cg->memory.updated_failcnt && cg->memory.enabled_failcnt == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_mem_failcnt)) {
+ snprintfz(title, CHART_TITLE_MAX, "Memory Limit Failures");
+
+ cg->st_mem_failcnt = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "mem_failcnt"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.mem_failcnt" : "cgroup.mem_failcnt"
+ , title
+ , "count"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 250
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_mem_failcnt, cg->chart_labels);
+
+ rrddim_add(cg->st_mem_failcnt, "failures", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(cg->st_mem_failcnt, "failures", cg->memory.failcnt);
+ rrdset_done(cg->st_mem_failcnt);
+ }
+
+ if(likely(cg->io_service_bytes.updated && cg->io_service_bytes.enabled == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_io)) {
+ snprintfz(title, CHART_TITLE_MAX, "I/O Bandwidth (all disks)");
+
+ cg->st_io = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "io"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.io" : "cgroup.io"
+ , title
+ , "KiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 1200
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_update_rrdlabels(cg->st_io, cg->chart_labels);
+
+ rrddim_add(cg->st_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(cg->st_io, "read", cg->io_service_bytes.Read);
+ rrddim_set(cg->st_io, "write", cg->io_service_bytes.Write);
+ rrdset_done(cg->st_io);
+ }
+
+ if(likely(cg->io_serviced.updated && cg->io_serviced.enabled == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_serviced_ops)) {
+ snprintfz(title, CHART_TITLE_MAX, "Serviced I/O Operations (all disks)");
+
+ cg->st_serviced_ops = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "serviced_ops"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.serviced_ops" : "cgroup.serviced_ops"
+ , title
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 1200
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_serviced_ops, cg->chart_labels);
+
+ rrddim_add(cg->st_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(cg->st_serviced_ops, "read", cg->io_serviced.Read);
+ rrddim_set(cg->st_serviced_ops, "write", cg->io_serviced.Write);
+ rrdset_done(cg->st_serviced_ops);
+ }
+
+ if(likely(cg->throttle_io_service_bytes.updated && cg->throttle_io_service_bytes.enabled == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_throttle_io)) {
+ snprintfz(title, CHART_TITLE_MAX, "Throttle I/O Bandwidth (all disks)");
+
+ cg->st_throttle_io = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "throttle_io"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.throttle_io" : "cgroup.throttle_io"
+ , title
+ , "KiB/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 1200
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_update_rrdlabels(cg->st_throttle_io, cg->chart_labels);
+
+ rrddim_add(cg->st_throttle_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_throttle_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(cg->st_throttle_io, "read", cg->throttle_io_service_bytes.Read);
+ rrddim_set(cg->st_throttle_io, "write", cg->throttle_io_service_bytes.Write);
+ rrdset_done(cg->st_throttle_io);
+ }
+
+ if(likely(cg->throttle_io_serviced.updated && cg->throttle_io_serviced.enabled == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_throttle_serviced_ops)) {
+ snprintfz(title, CHART_TITLE_MAX, "Throttle Serviced I/O Operations (all disks)");
+
+ cg->st_throttle_serviced_ops = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "throttle_serviced_ops"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.throttle_serviced_ops" : "cgroup.throttle_serviced_ops"
+ , title
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 1200
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_throttle_serviced_ops, cg->chart_labels);
+
+ rrddim_add(cg->st_throttle_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_throttle_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(cg->st_throttle_serviced_ops, "read", cg->throttle_io_serviced.Read);
+ rrddim_set(cg->st_throttle_serviced_ops, "write", cg->throttle_io_serviced.Write);
+ rrdset_done(cg->st_throttle_serviced_ops);
+ }
+
+ if(likely(cg->io_queued.updated && cg->io_queued.enabled == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_queued_ops)) {
+ snprintfz(title, CHART_TITLE_MAX, "Queued I/O Operations (all disks)");
+
+ cg->st_queued_ops = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "queued_ops"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.queued_ops" : "cgroup.queued_ops"
+ , title
+ , "operations"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2000
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_queued_ops, cg->chart_labels);
+
+ rrddim_add(cg->st_queued_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(cg->st_queued_ops, "write", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(cg->st_queued_ops, "read", cg->io_queued.Read);
+ rrddim_set(cg->st_queued_ops, "write", cg->io_queued.Write);
+ rrdset_done(cg->st_queued_ops);
+ }
+
+ if(likely(cg->io_merged.updated && cg->io_merged.enabled == CONFIG_BOOLEAN_YES)) {
+ if(unlikely(!cg->st_merged_ops)) {
+ snprintfz(title, CHART_TITLE_MAX, "Merged I/O Operations (all disks)");
+
+ cg->st_merged_ops = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "merged_ops"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.merged_ops" : "cgroup.merged_ops"
+ , title
+ , "operations/s"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2100
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(cg->st_merged_ops, cg->chart_labels);
+
+ rrddim_add(cg->st_merged_ops, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(cg->st_merged_ops, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(cg->st_merged_ops, "read", cg->io_merged.Read);
+ rrddim_set(cg->st_merged_ops, "write", cg->io_merged.Write);
+ rrdset_done(cg->st_merged_ops);
+ }
+
+ if (cg->options & CGROUP_OPTIONS_IS_UNIFIED) {
+ struct pressure *res = &cg->cpu_pressure;
+
+ if (likely(res->updated && res->some.enabled)) {
+ struct pressure_charts *pcs;
+ pcs = &res->some;
+
+ if (unlikely(!pcs->share_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "CPU some pressure");
+ chart = pcs->share_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "cpu_some_pressure"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.cpu_some_pressure" : "cgroup.cpu_some_pressure"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2200
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
+ pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if (unlikely(!pcs->total_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "CPU some pressure stall time");
+ chart = pcs->total_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "cpu_some_pressure_stall_time"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.cpu_some_pressure_stall_time" : "cgroup.cpu_some_pressure_stall_time"
+ , title
+ , "ms"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2220
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
+ pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ update_pressure_charts(pcs);
+ }
+ if (likely(res->updated && res->full.enabled)) {
+ struct pressure_charts *pcs;
+ pcs = &res->full;
+
+ if (unlikely(!pcs->share_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "CPU full pressure");
+ chart = pcs->share_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "cpu_full_pressure"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.cpu_full_pressure" : "cgroup.cpu_full_pressure"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2240
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
+ pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if (unlikely(!pcs->total_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "CPU full pressure stall time");
+ chart = pcs->total_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "cpu_full_pressure_stall_time"
+ , NULL
+ , "cpu"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.cpu_full_pressure_stall_time" : "cgroup.cpu_full_pressure_stall_time"
+ , title
+ , "ms"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2260
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
+ pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ update_pressure_charts(pcs);
+ }
+
+ res = &cg->memory_pressure;
+
+ if (likely(res->updated && res->some.enabled)) {
+ struct pressure_charts *pcs;
+ pcs = &res->some;
+
+ if (unlikely(!pcs->share_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "Memory some pressure");
+ chart = pcs->share_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "mem_some_pressure"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.memory_some_pressure" : "cgroup.memory_some_pressure"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2300
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
+ pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if (unlikely(!pcs->total_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "Memory some pressure stall time");
+ chart = pcs->total_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "memory_some_pressure_stall_time"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.memory_some_pressure_stall_time" : "cgroup.memory_some_pressure_stall_time"
+ , title
+ , "ms"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2320
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
+ pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ update_pressure_charts(pcs);
+ }
+
+ if (likely(res->updated && res->full.enabled)) {
+ struct pressure_charts *pcs;
+ pcs = &res->full;
+
+ if (unlikely(!pcs->share_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "Memory full pressure");
+
+ chart = pcs->share_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "mem_full_pressure"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.memory_full_pressure" : "cgroup.memory_full_pressure"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2340
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
+ pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if (unlikely(!pcs->total_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "Memory full pressure stall time");
+ chart = pcs->total_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "memory_full_pressure_stall_time"
+ , NULL
+ , "mem"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.memory_full_pressure_stall_time" : "cgroup.memory_full_pressure_stall_time"
+ , title
+ , "ms"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2360
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
+ pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ update_pressure_charts(pcs);
+ }
+
+ res = &cg->io_pressure;
+
+ if (likely(res->updated && res->some.enabled)) {
+ struct pressure_charts *pcs;
+ pcs = &res->some;
+
+ if (unlikely(!pcs->share_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "I/O some pressure");
+ chart = pcs->share_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "io_some_pressure"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.io_some_pressure" : "cgroup.io_some_pressure"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2400
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
+ pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if (unlikely(!pcs->total_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "I/O some pressure stall time");
+ chart = pcs->total_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "io_some_pressure_stall_time"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.io_some_pressure_stall_time" : "cgroup.io_some_pressure_stall_time"
+ , title
+ , "ms"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2420
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
+ pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ update_pressure_charts(pcs);
+ }
+
+ if (likely(res->updated && res->full.enabled)) {
+ struct pressure_charts *pcs;
+ pcs = &res->full;
+
+ if (unlikely(!pcs->share_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "I/O full pressure");
+ chart = pcs->share_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "io_full_pressure"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.io_full_pressure" : "cgroup.io_full_pressure"
+ , title
+ , "percentage"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2440
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
+ pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if (unlikely(!pcs->total_time.st)) {
+ RRDSET *chart;
+ snprintfz(title, CHART_TITLE_MAX, "I/O full pressure stall time");
+ chart = pcs->total_time.st = rrdset_create_localhost(
+ cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+ , "io_full_pressure_stall_time"
+ , NULL
+ , "disk"
+ , k8s_is_kubepod(cg) ? "k8s.cgroup.io_full_pressure_stall_time" : "cgroup.io_full_pressure_stall_time"
+ , title
+ , "ms"
+ , PLUGIN_CGROUPS_NAME
+ , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+ , cgroup_containers_chart_priority + 2460
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
+ pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ update_pressure_charts(pcs);
+ }
+ }
+ }
+
+ if(likely(cgroup_enable_systemd_services))
+ update_systemd_services_charts(update_every, services_do_cpu, services_do_mem_usage, services_do_mem_detailed
+ , services_do_mem_failcnt, services_do_swap_usage, services_do_io
+ , services_do_io_ops, services_do_throttle_io, services_do_throttle_ops
+ , services_do_queued_ops, services_do_merged_ops
+ );
+
+ debug(D_CGROUP, "done updating cgroups charts");
+}
+
+// ----------------------------------------------------------------------------
+// cgroups main
+
+static void cgroup_main_cleanup(void *ptr) {
+ worker_unregister();
+
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ info("cleaning up...");
+
+ usec_t max = 2 * USEC_PER_SEC, step = 50000;
+
+ if (!discovery_thread.exited) {
+ info("stopping discovery thread worker");
+ uv_mutex_lock(&discovery_thread.mutex);
+ discovery_thread.start_discovery = 1;
+ uv_cond_signal(&discovery_thread.cond_var);
+ uv_mutex_unlock(&discovery_thread.mutex);
+ }
+
+ info("waiting for discovery thread to finish...");
+
+ while (!discovery_thread.exited && max > 0) {
+ max -= step;
+ sleep_usec(step);
+ }
+
+ if (shm_mutex_cgroup_ebpf != SEM_FAILED) {
+ sem_close(shm_mutex_cgroup_ebpf);
+ }
+
+ if (shm_cgroup_ebpf.header) {
+ munmap(shm_cgroup_ebpf.header, shm_cgroup_ebpf.header->body_length);
+ }
+
+ if (shm_fd_cgroup_ebpf > 0) {
+ close(shm_fd_cgroup_ebpf);
+ }
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+void *cgroups_main(void *ptr) {
+ worker_register("CGROUPS");
+ worker_register_job_name(WORKER_CGROUPS_LOCK, "lock");
+ worker_register_job_name(WORKER_CGROUPS_READ, "read");
+ worker_register_job_name(WORKER_CGROUPS_CHART, "chart");
+
+ netdata_thread_cleanup_push(cgroup_main_cleanup, ptr);
+
+ if (getenv("KUBERNETES_SERVICE_HOST") != NULL && getenv("KUBERNETES_SERVICE_PORT") != NULL) {
+ is_inside_k8s = 1;
+ cgroup_enable_cpuacct_cpu_shares = CONFIG_BOOLEAN_YES;
+ }
+
+ read_cgroup_plugin_configuration();
+ netdata_cgroup_ebpf_initialize_shm();
+
+ if (uv_mutex_init(&cgroup_root_mutex)) {
+ error("CGROUP: cannot initialize mutex for the main cgroup list");
+ goto exit;
+ }
+
+ // dispatch a discovery worker thread
+ discovery_thread.start_discovery = 0;
+ discovery_thread.exited = 0;
+
+ if (uv_mutex_init(&discovery_thread.mutex)) {
+ error("CGROUP: cannot initialize mutex for discovery thread");
+ goto exit;
+ }
+ if (uv_cond_init(&discovery_thread.cond_var)) {
+ error("CGROUP: cannot initialize conditional variable for discovery thread");
+ goto exit;
+ }
+
+ int error = uv_thread_create(&discovery_thread.thread, cgroup_discovery_worker, NULL);
+ if (error) {
+ error("CGROUP: cannot create thread worker. uv_thread_create(): %s", uv_strerror(error));
+ goto exit;
+ }
+ uv_thread_set_name_np(discovery_thread.thread, "PLUGIN[cgroups]");
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = cgroup_update_every * USEC_PER_SEC;
+ usec_t find_every = cgroup_check_for_new_every * USEC_PER_SEC, find_dt = 0;
+
+ while(!netdata_exit) {
+ worker_is_idle();
+
+ usec_t hb_dt = heartbeat_next(&hb, step);
+ if(unlikely(netdata_exit)) break;
+
+ find_dt += hb_dt;
+ if (unlikely(find_dt >= find_every || (!is_inside_k8s && cgroups_check))) {
+ uv_cond_signal(&discovery_thread.cond_var);
+ discovery_thread.start_discovery = 1;
+ find_dt = 0;
+ cgroups_check = 0;
+ }
+
+ worker_is_busy(WORKER_CGROUPS_LOCK);
+ uv_mutex_lock(&cgroup_root_mutex);
+
+ worker_is_busy(WORKER_CGROUPS_READ);
+ read_all_discovered_cgroups(cgroup_root);
+
+ worker_is_busy(WORKER_CGROUPS_CHART);
+ update_cgroup_charts(cgroup_update_every);
+
+ worker_is_idle();
+ uv_mutex_unlock(&cgroup_root_mutex);
+ }
+
+exit:
+ worker_unregister();
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.h b/collectors/cgroups.plugin/sys_fs_cgroup.h
new file mode 100644
index 0000000..d1adf8a
--- /dev/null
+++ b/collectors/cgroups.plugin/sys_fs_cgroup.h
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_SYS_FS_CGROUP_H
+#define NETDATA_SYS_FS_CGROUP_H 1
+
+#include "daemon/common.h"
+
+#define CGROUP_OPTIONS_DISABLED_DUPLICATE 0x00000001
+#define CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE 0x00000002
+#define CGROUP_OPTIONS_IS_UNIFIED 0x00000004
+
+typedef struct netdata_ebpf_cgroup_shm_header {
+ int cgroup_root_count;
+ int cgroup_max;
+ int systemd_enabled;
+ int __pad;
+ size_t body_length;
+} netdata_ebpf_cgroup_shm_header_t;
+
+#define CGROUP_EBPF_NAME_SHARED_LENGTH 256
+
+typedef struct netdata_ebpf_cgroup_shm_body {
+ // Considering what is exposed in this link https://en.wikipedia.org/wiki/Comparison_of_file_systems#Limits
+ // this length is enough to store what we want.
+ char name[CGROUP_EBPF_NAME_SHARED_LENGTH];
+ uint32_t hash;
+ uint32_t options;
+ int enabled;
+ char path[FILENAME_MAX + 1];
+} netdata_ebpf_cgroup_shm_body_t;
+
+typedef struct netdata_ebpf_cgroup_shm {
+ netdata_ebpf_cgroup_shm_header_t *header;
+ netdata_ebpf_cgroup_shm_body_t *body;
+} netdata_ebpf_cgroup_shm_t;
+
+#define NETDATA_SHARED_MEMORY_EBPF_CGROUP_NAME "netdata_shm_cgroup_ebpf"
+#define NETDATA_NAMED_SEMAPHORE_EBPF_CGROUP_NAME "/netdata_sem_cgroup_ebpf"
+
+#include "../proc.plugin/plugin_proc.h"
+
+char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data);
+
+#endif //NETDATA_SYS_FS_CGROUP_H
diff --git a/collectors/cgroups.plugin/tests/test_cgroups_plugin.c b/collectors/cgroups.plugin/tests/test_cgroups_plugin.c
new file mode 100644
index 0000000..25939a9
--- /dev/null
+++ b/collectors/cgroups.plugin/tests/test_cgroups_plugin.c
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "test_cgroups_plugin.h"
+#include "libnetdata/required_dummies.h"
+
+RRDHOST *localhost;
+int netdata_zero_metrics_enabled = 1;
+struct config netdata_config;
+char *netdata_configured_primary_plugins_dir = NULL;
+
+struct k8s_test_data {
+ char *data;
+ char *name;
+ char *key[3];
+ char *value[3];
+
+ const char *result_key[3];
+ const char *result_value[3];
+ int result_ls[3];
+ int i;
+};
+
+static int read_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data)
+{
+ struct k8s_test_data *test_data = (struct k8s_test_data *)data;
+
+ test_data->result_key[test_data->i] = name;
+ test_data->result_value[test_data->i] = value;
+ test_data->result_ls[test_data->i] = ls;
+
+ test_data->i++;
+
+ return 1;
+}
+
+static void test_k8s_parse_resolved_name(void **state)
+{
+ UNUSED(state);
+
+ DICTIONARY *labels = rrdlabels_create();
+
+ struct k8s_test_data test_data[] = {
+ // One label
+ { .data = "name label1=\"value1\"",
+ .name = "name",
+ .key[0] = "label1", .value[0] = "value1" },
+
+ // Three labels
+ { .data = "name label1=\"value1\",label2=\"value2\",label3=\"value3\"",
+ .name = "name",
+ .key[0] = "label1", .value[0] = "value1",
+ .key[1] = "label2", .value[1] = "value2",
+ .key[2] = "label3", .value[2] = "value3" },
+
+ // Comma at the end of the data string
+ { .data = "name label1=\"value1\",",
+ .name = "name",
+ .key[0] = "label1", .value[0] = "value1" },
+
+ // Equals sign in the value
+ // { .data = "name label1=\"value=1\"",
+ // .name = "name",
+ // .key[0] = "label1", .value[0] = "value=1" },
+
+ // Double quotation mark in the value
+ // { .data = "name label1=\"value\"1\"",
+ // .name = "name",
+ // .key[0] = "label1", .value[0] = "value" },
+
+ // Escaped double quotation mark in the value
+ // { .data = "name label1=\"value\\\"1\"",
+ // .name = "name",
+ // .key[0] = "label1", .value[0] = "value\\\"1" },
+
+ // Equals sign in the key
+ // { .data = "name label=1=\"value1\"",
+ // .name = "name",
+ // .key[0] = "label", .value[0] = "1=\"value1\"" },
+
+ // Skipped value
+ // { .data = "name label1=,label2=\"value2\"",
+ // .name = "name",
+ // .key[0] = "label2", .value[0] = "value2" },
+
+ // A pair of equals signs
+ { .data = "name= =",
+ .name = "name=" },
+
+ // A pair of commas
+ { .data = "name, ,",
+ .name = "name," },
+
+ { .data = NULL }
+ };
+
+ for (int i = 0; test_data[i].data != NULL; i++) {
+ char *data = strdup(test_data[i].data);
+
+ char *name = k8s_parse_resolved_name_and_labels(labels, data);
+
+ assert_string_equal(name, test_data[i].name);
+
+ rrdlabels_walkthrough_read(labels, read_label_callback, &test_data[i]);
+
+ for (int l = 0; l < 3 && test_data[i].key[l] != NULL; l++) {
+ char *key = test_data[i].key[l];
+ char *value = test_data[i].value[l];
+
+ const char *result_key = test_data[i].result_key[l];
+ const char *result_value = test_data[i].result_value[l];
+ int ls = test_data[i].result_ls[l];
+
+ assert_string_equal(key, result_key);
+ assert_string_equal(value, result_value);
+ assert_int_equal(RRDLABEL_SRC_AUTO | RRDLABEL_SRC_K8S, ls);
+ }
+
+ free(data);
+ }
+}
+
+int main(void)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test(test_k8s_parse_resolved_name),
+ };
+
+ int test_res = cmocka_run_group_tests_name("test_k8s_parse_resolved_name", tests, NULL, NULL);
+
+ return test_res;
+}
diff --git a/collectors/cgroups.plugin/tests/test_cgroups_plugin.h b/collectors/cgroups.plugin/tests/test_cgroups_plugin.h
new file mode 100644
index 0000000..3d68e92
--- /dev/null
+++ b/collectors/cgroups.plugin/tests/test_cgroups_plugin.h
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef TEST_CGROUPS_PLUGIN_H
+#define TEST_CGROUPS_PLUGIN_H 1
+
+#include "libnetdata/libnetdata.h"
+
+#include "../sys_fs_cgroup.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <stdint.h>
+#include <cmocka.h>
+
+#endif /* TEST_CGROUPS_PLUGIN_H */
diff --git a/collectors/cgroups.plugin/tests/test_doubles.c b/collectors/cgroups.plugin/tests/test_doubles.c
new file mode 100644
index 0000000..498f649
--- /dev/null
+++ b/collectors/cgroups.plugin/tests/test_doubles.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "test_cgroups_plugin.h"
+
+void rrdset_is_obsolete(RRDSET *st)
+{
+ UNUSED(st);
+}
+
+void rrdset_isnot_obsolete(RRDSET *st)
+{
+ UNUSED(st);
+}
+
+struct mountinfo *mountinfo_read(int do_statvfs)
+{
+ UNUSED(do_statvfs);
+
+ return NULL;
+}
+
+struct mountinfo *
+mountinfo_find_by_filesystem_mount_source(struct mountinfo *root, const char *filesystem, const char *mount_source)
+{
+ UNUSED(root);
+ UNUSED(filesystem);
+ UNUSED(mount_source);
+
+ return NULL;
+}
+
+struct mountinfo *
+mountinfo_find_by_filesystem_super_option(struct mountinfo *root, const char *filesystem, const char *super_options)
+{
+ UNUSED(root);
+ UNUSED(filesystem);
+ UNUSED(super_options);
+
+ return NULL;
+}
+
+void mountinfo_free_all(struct mountinfo *mi)
+{
+ UNUSED(mi);
+}
+
+RRDSET *rrdset_create_custom(
+ RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context,
+ const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every,
+ RRDSET_TYPE chart_type, RRD_MEMORY_MODE memory_mode, long history_entries)
+{
+ UNUSED(host);
+ UNUSED(type);
+ UNUSED(id);
+ UNUSED(name);
+ UNUSED(family);
+ UNUSED(context);
+ UNUSED(title);
+ UNUSED(units);
+ UNUSED(plugin);
+ UNUSED(module);
+ UNUSED(priority);
+ UNUSED(update_every);
+ UNUSED(chart_type);
+ UNUSED(memory_mode);
+ UNUSED(history_entries);
+
+ return NULL;
+}
+
+RRDDIM *rrddim_add_custom(
+ RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor,
+ RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode)
+{
+ UNUSED(st);
+ UNUSED(id);
+ UNUSED(name);
+ UNUSED(multiplier);
+ UNUSED(divisor);
+ UNUSED(algorithm);
+ UNUSED(memory_mode);
+
+ return NULL;
+}
+
+collected_number rrddim_set(RRDSET *st, const char *id, collected_number value)
+{
+ UNUSED(st);
+ UNUSED(id);
+ UNUSED(value);
+
+ return 0;
+}
+
+collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value)
+{
+ UNUSED(st);
+ UNUSED(rd);
+ UNUSED(value);
+
+ return 0;
+}
+
+const RRDSETVAR_ACQUIRED *rrdsetvar_custom_chart_variable_add_and_acquire(RRDSET *st, const char *name)
+{
+ UNUSED(st);
+ UNUSED(name);
+
+ return NULL;
+}
+
+void rrdsetvar_custom_chart_variable_set(RRDSET *st, const RRDSETVAR_ACQUIRED *rsa, NETDATA_DOUBLE value)
+{
+ UNUSED(st);
+ UNUSED(rsa);
+ UNUSED(value);
+}
+
+void rrdset_next_usec(RRDSET *st, usec_t microseconds)
+{
+ UNUSED(st);
+ UNUSED(microseconds);
+}
+
+void rrdset_done(RRDSET *st)
+{
+ UNUSED(st);
+}
+
+void update_pressure_charts(struct pressure_charts *charts)
+{
+ UNUSED(charts);
+}
+
+void netdev_rename_device_add(
+ const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels, const char *ctx_prefix)
+{
+ UNUSED(host_device);
+ UNUSED(container_device);
+ UNUSED(container_name);
+ UNUSED(labels);
+ UNUSED(ctx_prefix);
+}
+
+void netdev_rename_device_del(const char *host_device)
+{
+ UNUSED(host_device);
+}
+
+void rrdcalc_update_rrdlabels(RRDSET *st) {
+ (void)st;
+}
+
+void db_execute(const char *cmd)
+{
+ UNUSED(cmd);
+}
diff --git a/collectors/charts.d.plugin/Makefile.am b/collectors/charts.d.plugin/Makefile.am
new file mode 100644
index 0000000..03c7f0a
--- /dev/null
+++ b/collectors/charts.d.plugin/Makefile.am
@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+CLEANFILES = \
+ charts.d.plugin \
+ $(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+dist_libconfig_DATA = \
+ charts.d.conf \
+ $(NULL)
+
+dist_plugins_SCRIPTS = \
+ charts.d.dryrun-helper.sh \
+ charts.d.plugin \
+ loopsleepms.sh.inc \
+ $(NULL)
+
+dist_noinst_DATA = \
+ charts.d.plugin.in \
+ README.md \
+ $(NULL)
+
+dist_charts_SCRIPTS = \
+ $(NULL)
+
+dist_charts_DATA = \
+ $(NULL)
+
+userchartsconfigdir=$(configdir)/charts.d
+dist_userchartsconfig_DATA = \
+ $(NULL)
+
+# Explicitly install directories to avoid permission issues due to umask
+install-exec-local:
+ $(INSTALL) -d $(DESTDIR)$(userchartsconfigdir)
+
+chartsconfigdir=$(libconfigdir)/charts.d
+dist_chartsconfig_DATA = \
+ $(NULL)
+
+include ap/Makefile.inc
+include apcupsd/Makefile.inc
+include example/Makefile.inc
+include libreswan/Makefile.inc
+include nut/Makefile.inc
+include opensips/Makefile.inc
+include sensors/Makefile.inc
diff --git a/collectors/charts.d.plugin/README.md b/collectors/charts.d.plugin/README.md
new file mode 100644
index 0000000..06f4af1
--- /dev/null
+++ b/collectors/charts.d.plugin/README.md
@@ -0,0 +1,198 @@
+<!--
+title: "charts.d.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/charts.d.plugin/README.md
+-->
+
+# charts.d.plugin
+
+`charts.d.plugin` is a Netdata external plugin. It is an **orchestrator** for data collection modules written in `BASH` v4+.
+
+1. It runs as an independent process `ps fax` shows it
+2. It is started and stopped automatically by Netdata
+3. It communicates with Netdata via a unidirectional pipe (sending data to the `netdata` daemon)
+4. Supports any number of data collection **modules**
+
+`charts.d.plugin` has been designed so that the actual script that will do data collection will be permanently in
+memory, collecting data with as little overheads as possible
+(i.e. initialize once, repeatedly collect values with minimal overhead).
+
+`charts.d.plugin` looks for scripts in `/usr/lib/netdata/charts.d`.
+The scripts should have the filename suffix: `.chart.sh`.
+
+## Configuration
+
+`charts.d.plugin` itself can be configured using the configuration file `/etc/netdata/charts.d.conf`
+(to edit it on your system run `/etc/netdata/edit-config charts.d.conf`). This file is also a BASH script.
+
+In this file, you can place statements like this:
+
+```
+enable_all_charts="yes"
+X="yes"
+Y="no"
+```
+
+where `X` and `Y` are the names of individual charts.d collector scripts.
+When set to `yes`, charts.d will evaluate the collector script (see below).
+When set to `no`, charts.d will ignore the collector script.
+
+The variable `enable_all_charts` sets the default enable/disable state for all charts.
+
+## A charts.d module
+
+A `charts.d.plugin` module is a BASH script defining a few functions.
+
+For a module called `X`, the following criteria must be met:
+
+1. The module script must be called `X.chart.sh` and placed in `/usr/libexec/netdata/charts.d`.
+
+2. If the module needs a configuration, it should be called `X.conf` and placed in `/etc/netdata/charts.d`.
+ The configuration file `X.conf` is also a BASH script itself.
+ To edit the default files supplied by Netdata, run `/etc/netdata/edit-config charts.d/X.conf`,
+ where `X` is the name of the module.
+
+3. All functions and global variables defined in the script and its configuration, must begin with `X_`.
+
+4. The following functions must be defined:
+
+ - `X_check()` - returns 0 or 1 depending on whether the module is able to run or not
+ (following the standard Linux command line return codes: 0 = OK, the collector can operate and 1 = FAILED,
+ the collector cannot be used).
+
+ - `X_create()` - creates the Netdata charts, following the standard Netdata plugin guides as described in
+ **[External Plugins](/collectors/plugins.d/README.md)** (commands `CHART` and `DIMENSION`).
+ The return value does matter: 0 = OK, 1 = FAILED.
+
+ - `X_update()` - collects the values for the defined charts, following the standard Netdata plugin guides
+ as described in **[External Plugins](/collectors/plugins.d/README.md)** (commands `BEGIN`, `SET`, `END`).
+ The return value also matters: 0 = OK, 1 = FAILED.
+
+5. The following global variables are available to be set:
+ - `X_update_every` - is the data collection frequency for the module script, in seconds.
+
+The module script may use more functions or variables. But all of them must begin with `X_`.
+
+The standard Netdata plugin variables are also available (check **[External Plugins](/collectors/plugins.d/README.md)**).
+
+### X_check()
+
+The purpose of the BASH function `X_check()` is to check if the module can collect data (or check its config).
+
+For example, if the module is about monitoring a local mysql database, the `X_check()` function may attempt to
+connect to a local mysql database to find out if it can read the values it needs.
+
+`X_check()` is run only once for the lifetime of the module.
+
+### X_create()
+
+The purpose of the BASH function `X_create()` is to create the charts and dimensions using the standard Netdata
+plugin guides (**[External Plugins](/collectors/plugins.d/README.md)**).
+
+`X_create()` will be called just once and only after `X_check()` was successful.
+You can however call it yourself when there is need for it (for example to add a new dimension to an existing chart).
+
+A non-zero return value will disable the collector.
+
+### X_update()
+
+`X_update()` will be called repeatedly every `X_update_every` seconds, to collect new values and send them to Netdata,
+following the Netdata plugin guides (**[External Plugins](/collectors/plugins.d/README.md)**).
+
+The function will be called with one parameter: microseconds since the last time it was run. This value should be
+appended to the `BEGIN` statement of every chart updated by the collector script.
+
+A non-zero return value will disable the collector.
+
+### Useful functions charts.d provides
+
+Module scripts can use the following charts.d functions:
+
+#### require_cmd command
+
+`require_cmd()` will check if a command is available in the running system.
+
+For example, your `X_check()` function may use it like this:
+
+```sh
+mysql_check() {
+ require_cmd mysql || return 1
+ return 0
+}
+```
+
+Using the above, if the command `mysql` is not available in the system, the `mysql` module will be disabled.
+
+#### fixid "string"
+
+`fixid()` will get a string and return a properly formatted id for a chart or dimension.
+
+This is an expensive function that should not be used in `X_update()`.
+You can keep the generated id in a BASH associative array to have the values availables in `X_update()`, like this:
+
+```sh
+declare -A X_ids=()
+X_create() {
+ local name="a very bad name for id"
+
+ X_ids[$name]="$(fixid "$name")"
+}
+
+X_update() {
+ local microseconds="$1"
+
+ ...
+ local name="a very bad name for id"
+ ...
+
+ echo "BEGIN ${X_ids[$name]} $microseconds"
+ ...
+}
+```
+
+### Debugging your collectors
+
+You can run `charts.d.plugin` by hand with something like this:
+
+```sh
+# become user netdata
+sudo su -s /bin/sh netdata
+
+# run the plugin in debug mode
+/usr/libexec/netdata/plugins.d/charts.d.plugin debug 1 X Y Z
+```
+
+Charts.d will run in `debug` mode, with an update frequency of `1`, evaluating only the collector scripts
+`X`, `Y` and `Z`. You can define zero or more module scripts. If none is defined, charts.d will evaluate all
+module scripts available.
+
+Keep in mind that if your configs are not in `/etc/netdata`, you should do the following before running
+`charts.d.plugin`:
+
+```sh
+export NETDATA_USER_CONFIG_DIR="/path/to/etc/netdata"
+```
+
+Also, remember that Netdata runs `chart.d.plugin` as user `netdata` (or any other user the `netdata` process is configured to run as).
+
+## Running multiple instances of charts.d.plugin
+
+`charts.d.plugin` will call the `X_update()` function one after another. This means that a delay in collector `X`
+will also delay the collection of `Y` and `Z`.
+
+You can have multiple `charts.d.plugin` running to overcome this problem.
+
+This is what you need to do:
+
+1. Decide a new name for the new charts.d instance: example `charts2.d`.
+
+2. Create/edit the files `/etc/netdata/charts.d.conf` and `/etc/netdata/charts2.d.conf` and enable / disable the
+ module you want each to run. Remember to set `enable_all_charts="no"` to both of them, and enable the individual
+ modules for each.
+
+3. link `/usr/libexec/netdata/plugins.d/charts.d.plugin` to `/usr/libexec/netdata/plugins.d/charts2.d.plugin`.
+ Netdata will spawn a new charts.d process.
+
+Execute the above in this order, since Netdata will (by default) attempt to start new plugins soon after they are
+created in `/usr/libexec/netdata/plugins.d/`.
+
+
diff --git a/collectors/charts.d.plugin/ap/Makefile.inc b/collectors/charts.d.plugin/ap/Makefile.inc
new file mode 100644
index 0000000..a2dd375
--- /dev/null
+++ b/collectors/charts.d.plugin/ap/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_charts_DATA += ap/ap.chart.sh
+dist_chartsconfig_DATA += ap/ap.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += ap/README.md ap/Makefile.inc
+
diff --git a/collectors/charts.d.plugin/ap/README.md b/collectors/charts.d.plugin/ap/README.md
new file mode 100644
index 0000000..a7953a5
--- /dev/null
+++ b/collectors/charts.d.plugin/ap/README.md
@@ -0,0 +1,99 @@
+<!--
+title: "Access point monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/charts.d.plugin/ap/README.md
+sidebar_label: "Access points"
+-->
+
+# Access point monitoring with Netdata
+
+The `ap` collector visualizes data related to access points.
+
+## Example Netdata charts
+
+![image](https://cloud.githubusercontent.com/assets/2662304/12377654/9f566e88-bd2d-11e5-855a-e0ba96b8fd98.png)
+
+## How it works
+
+It does the following:
+
+1. Runs `iw dev` searching for interfaces that have `type AP`.
+
+ From the same output it collects the SSIDs each AP supports by looking for lines `ssid NAME`.
+
+ Example:
+
+```sh
+# iw dev
+phy#0
+ Interface wlan0
+ ifindex 3
+ wdev 0x1
+ addr 7c:dd:90:77:34:2a
+ ssid TSAOUSIS
+ type AP
+ channel 7 (2442 MHz), width: 20 MHz, center1: 2442 MHz
+```
+
+2. For each interface found, it runs `iw INTERFACE station dump`.
+
+ From the output is collects:
+
+ - rx/tx bytes
+ - rx/tx packets
+ - tx retries
+ - tx failed
+ - signal strength
+ - rx/tx bitrate
+ - expected throughput
+
+ Example:
+
+```sh
+# iw wlan0 station dump
+Station 40:b8:37:5a:ed:5e (on wlan0)
+ inactive time: 910 ms
+ rx bytes: 15588897
+ rx packets: 127772
+ tx bytes: 52257763
+ tx packets: 95802
+ tx retries: 2162
+ tx failed: 28
+ signal: -43 dBm
+ signal avg: -43 dBm
+ tx bitrate: 65.0 MBit/s MCS 7
+ rx bitrate: 1.0 MBit/s
+ expected throughput: 32.125Mbps
+ authorized: yes
+ authenticated: yes
+ preamble: long
+ WMM/WME: yes
+ MFP: no
+ TDLS peer: no
+```
+
+3. For each interface found, it creates 6 charts:
+
+ - Number of Connected clients
+ - Bandwidth for all clients
+ - Packets for all clients
+ - Transmit Issues for all clients
+ - Average Signal among all clients
+ - Average Bitrate (including average expected throughput) among all clients
+
+## Configuration
+
+Edit the `charts.d/ap.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config charts.d/ap.conf
+```
+
+You can only set `ap_update_every=NUMBER` to change the data collection frequency.
+
+## Auto-detection
+
+The plugin is able to auto-detect if you are running access points on your linux box.
+
+
diff --git a/collectors/charts.d.plugin/ap/ap.chart.sh b/collectors/charts.d.plugin/ap/ap.chart.sh
new file mode 100644
index 0000000..80c9dc6
--- /dev/null
+++ b/collectors/charts.d.plugin/ap/ap.chart.sh
@@ -0,0 +1,179 @@
+# shellcheck shell=bash
+# no need for shebang - this file is loaded from charts.d.plugin
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
+#
+
+# _update_every is a special variable - it holds the number of seconds
+# between the calls of the _update() function
+ap_update_every=
+ap_priority=6900
+
+declare -A ap_devs=()
+
+# _check is called once, to find out if this chart should be enabled or not
+ap_check() {
+ require_cmd iw || return 1
+ local ev
+ ev=$(run iw dev | awk '
+ BEGIN {
+ i = "";
+ ssid = "";
+ ap = 0;
+ }
+ /^[ \t]+Interface / {
+ if( ap == 1 ) {
+ print "ap_devs[" i "]=\"" ssid "\""
+ }
+
+ i = $2;
+ ssid = "";
+ ap = 0;
+ }
+ /^[ \t]+ssid / { ssid = $2; }
+ /^[ \t]+type AP$/ { ap = 1; }
+ END {
+ if( ap == 1 ) {
+ print "ap_devs[" i "]=\"" ssid "\""
+ }
+ }
+ ')
+ eval "${ev}"
+
+ # this should return:
+ # - 0 to enable the chart
+ # - 1 to disable the chart
+
+ [ ${#ap_devs[@]} -gt 0 ] && return 0
+ error "no devices found in AP mode, with 'iw dev'"
+ return 1
+}
+
+# _create is called once, to create the charts
+ap_create() {
+ local ssid dev
+
+ for dev in "${!ap_devs[@]}"; do
+ ssid="${ap_devs[${dev}]}"
+
+ # create the chart with 3 dimensions
+ cat << EOF
+CHART ap_clients.${dev} '' "Connected clients to ${ssid} on ${dev}" "clients" ${dev} ap.clients line $((ap_priority + 1)) $ap_update_every '' '' 'ap'
+DIMENSION clients '' absolute 1 1
+
+CHART ap_bandwidth.${dev} '' "Bandwidth for ${ssid} on ${dev}" "kilobits/s" ${dev} ap.net area $((ap_priority + 2)) $ap_update_every '' '' 'ap'
+DIMENSION received '' incremental 8 1024
+DIMENSION sent '' incremental -8 1024
+
+CHART ap_packets.${dev} '' "Packets for ${ssid} on ${dev}" "packets/s" ${dev} ap.packets line $((ap_priority + 3)) $ap_update_every '' '' 'ap'
+DIMENSION received '' incremental 1 1
+DIMENSION sent '' incremental -1 1
+
+CHART ap_issues.${dev} '' "Transmit Issues for ${ssid} on ${dev}" "issues/s" ${dev} ap.issues line $((ap_priority + 4)) $ap_update_every '' '' 'ap'
+DIMENSION retries 'tx retries' incremental 1 1
+DIMENSION failures 'tx failures' incremental -1 1
+
+CHART ap_signal.${dev} '' "Average Signal for ${ssid} on ${dev}" "dBm" ${dev} ap.signal line $((ap_priority + 5)) $ap_update_every '' '' 'ap'
+DIMENSION signal 'average signal' absolute 1 1000
+
+CHART ap_bitrate.${dev} '' "Bitrate for ${ssid} on ${dev}" "Mbps" ${dev} ap.bitrate line $((ap_priority + 6)) $ap_update_every '' '' 'ap'
+DIMENSION receive '' absolute 1 1000
+DIMENSION transmit '' absolute -1 1000
+DIMENSION expected 'expected throughput' absolute 1 1000
+EOF
+ done
+
+ return 0
+}
+
+# _update is called continuously, to collect the values
+ap_update() {
+ # the first argument to this function is the microseconds since last update
+ # pass this parameter to the BEGIN statement (see below).
+
+ # do all the work to collect / calculate the values
+ # for each dimension
+ # remember: KEEP IT SIMPLE AND SHORT
+
+ for dev in "${!ap_devs[@]}"; do
+ echo
+ echo "DEVICE ${dev}"
+ iw "${dev}" station dump
+ done | awk '
+ function zero_data() {
+ dev = "";
+ c = 0;
+ rb = 0;
+ tb = 0;
+ rp = 0;
+ tp = 0;
+ tr = 0;
+ tf = 0;
+ tt = 0;
+ rt = 0;
+ s = 0;
+ g = 0;
+ e = 0;
+ }
+ function print_device() {
+ if(dev != "" && length(dev) > 0) {
+ print "BEGIN ap_clients." dev;
+ print "SET clients = " c;
+ print "END";
+ print "BEGIN ap_bandwidth." dev;
+ print "SET received = " rb;
+ print "SET sent = " tb;
+ print "END";
+ print "BEGIN ap_packets." dev;
+ print "SET received = " rp;
+ print "SET sent = " tp;
+ print "END";
+ print "BEGIN ap_issues." dev;
+ print "SET retries = " tr;
+ print "SET failures = " tf;
+ print "END";
+
+ if( c == 0 ) c = 1;
+ print "BEGIN ap_signal." dev;
+ print "SET signal = " int(s / c);
+ print "END";
+ print "BEGIN ap_bitrate." dev;
+ print "SET receive = " int(rt / c);
+ print "SET transmit = " int(tt / c);
+ print "SET expected = " int(e / c);
+ print "END";
+ }
+ zero_data();
+ }
+ BEGIN {
+ zero_data();
+ }
+ /^DEVICE / {
+ print_device();
+ dev = $2;
+ }
+ /^Station/ { c++; }
+ /^[ \t]+rx bytes:/ { rb += $3; }
+ /^[ \t]+tx bytes:/ { tb += $3; }
+ /^[ \t]+rx packets:/ { rp += $3; }
+ /^[ \t]+tx packets:/ { tp += $3; }
+ /^[ \t]+tx retries:/ { tr += $3; }
+ /^[ \t]+tx failed:/ { tf += $3; }
+ /^[ \t]+signal:/ { x = $2; s += x * 1000; }
+ /^[ \t]+rx bitrate:/ { x = $3; rt += x * 1000; }
+ /^[ \t]+tx bitrate:/ { x = $3; tt += x * 1000; }
+ /^[ \t]+expected throughput:(.*)Mbps/ {
+ x=$3;
+ sub(/Mbps/, "", x);
+ e += x * 1000;
+ }
+ END {
+ print_device();
+ }
+ '
+
+ return 0
+}
diff --git a/collectors/charts.d.plugin/ap/ap.conf b/collectors/charts.d.plugin/ap/ap.conf
new file mode 100644
index 0000000..38fc157
--- /dev/null
+++ b/collectors/charts.d.plugin/ap/ap.conf
@@ -0,0 +1,23 @@
+# no need for shebang - this file is loaded from charts.d.plugin
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2018 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+
+# nothing fancy to configure.
+# this module will run
+# iw dev - to find wireless devices in AP mode
+# iw ${dev} station dump - to get connected clients
+# based on the above, it generates several charts
+
+# the data collection frequency
+# if unset, will inherit the netdata update frequency
+#ap_update_every=
+
+# the charts priority on the dashboard
+#ap_priority=6900
+
+# the number of retries to do in case of failure
+# before disabling the module
+#ap_retries=10
diff --git a/collectors/charts.d.plugin/apcupsd/Makefile.inc b/collectors/charts.d.plugin/apcupsd/Makefile.inc
new file mode 100644
index 0000000..19cb9ca
--- /dev/null
+++ b/collectors/charts.d.plugin/apcupsd/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_charts_DATA += apcupsd/apcupsd.chart.sh
+dist_chartsconfig_DATA += apcupsd/apcupsd.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += apcupsd/README.md apcupsd/Makefile.inc
+
diff --git a/collectors/charts.d.plugin/apcupsd/README.md b/collectors/charts.d.plugin/apcupsd/README.md
new file mode 100644
index 0000000..f1aebf9
--- /dev/null
+++ b/collectors/charts.d.plugin/apcupsd/README.md
@@ -0,0 +1,21 @@
+<!--
+title: "APC UPS monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/charts.d.plugin/apcupsd/README.md
+sidebar_label: "APC UPS"
+-->
+
+# APC UPS monitoring with Netdata
+
+Monitors different APC UPS models and retrieves status information using `apcaccess` tool.
+
+## Configuration
+
+Edit the `charts.d/apcupsd.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config charts.d/apcupsd.conf
+```
+
+
diff --git a/collectors/charts.d.plugin/apcupsd/apcupsd.chart.sh b/collectors/charts.d.plugin/apcupsd/apcupsd.chart.sh
new file mode 100644
index 0000000..ef9a905
--- /dev/null
+++ b/collectors/charts.d.plugin/apcupsd/apcupsd.chart.sh
@@ -0,0 +1,223 @@
+# shellcheck shell=bash
+# no need for shebang - this file is loaded from charts.d.plugin
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
+#
+
+apcupsd_ip=
+apcupsd_port=
+
+declare -A apcupsd_sources=(
+ ["local"]="127.0.0.1:3551"
+)
+
+# how frequently to collect UPS data
+apcupsd_update_every=10
+
+apcupsd_timeout=3
+
+# the priority of apcupsd related to other charts
+apcupsd_priority=90000
+
+apcupsd_get() {
+ run -t $apcupsd_timeout apcaccess status "$1"
+}
+
+is_ups_alive() {
+ local status
+ status="$(apcupsd_get "$1" | sed -e 's/STATUS.*: //' -e 't' -e 'd')"
+ case "$status" in
+ "" | "COMMLOST" | "SHUTTING DOWN") return 1 ;;
+ *) return 0 ;;
+ esac
+}
+
+apcupsd_check() {
+
+ # this should return:
+ # - 0 to enable the chart
+ # - 1 to disable the chart
+
+ require_cmd apcaccess || return 1
+
+ # backwards compatibility
+ if [ "${apcupsd_ip}:${apcupsd_port}" != ":" ]; then
+ apcupsd_sources["local"]="${apcupsd_ip}:${apcupsd_port}"
+ fi
+
+ local host working=0 failed=0
+ for host in "${!apcupsd_sources[@]}"; do
+ apcupsd_get "${apcupsd_sources[${host}]}" > /dev/null
+ # shellcheck disable=2181
+ if [ $? -ne 0 ]; then
+ error "cannot get information for apcupsd server ${host} on ${apcupsd_sources[${host}]}."
+ failed=$((failed + 1))
+ else
+ if ! is_ups_alive ${apcupsd_sources[${host}]}; then
+ error "APC UPS ${host} on ${apcupsd_sources[${host}]} is not online."
+ failed=$((failed + 1))
+ else
+ working=$((working + 1))
+ fi
+ fi
+ done
+
+ if [ ${working} -eq 0 ]; then
+ error "No APC UPSes found available."
+ return 1
+ fi
+
+ return 0
+}
+
+apcupsd_create() {
+ local host
+ for host in "${!apcupsd_sources[@]}"; do
+ # create the charts
+ cat << EOF
+CHART apcupsd_${host}.charge '' "UPS Charge" "percentage" ups apcupsd.charge area $((apcupsd_priority + 2)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION battery_charge charge absolute 1 100
+
+CHART apcupsd_${host}.battery_voltage '' "UPS Battery Voltage" "Volts" ups apcupsd.battery.voltage line $((apcupsd_priority + 4)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION battery_voltage voltage absolute 1 100
+DIMENSION battery_voltage_nominal nominal absolute 1 100
+
+CHART apcupsd_${host}.input_voltage '' "UPS Input Voltage" "Volts" input apcupsd.input.voltage line $((apcupsd_priority + 5)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION input_voltage voltage absolute 1 100
+DIMENSION input_voltage_min min absolute 1 100
+DIMENSION input_voltage_max max absolute 1 100
+
+CHART apcupsd_${host}.input_frequency '' "UPS Input Frequency" "Hz" input apcupsd.input.frequency line $((apcupsd_priority + 6)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION input_frequency frequency absolute 1 100
+
+CHART apcupsd_${host}.output_voltage '' "UPS Output Voltage" "Volts" output apcupsd.output.voltage line $((apcupsd_priority + 7)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION output_voltage voltage absolute 1 100
+DIMENSION output_voltage_nominal nominal absolute 1 100
+
+CHART apcupsd_${host}.load '' "UPS Load" "percentage" ups apcupsd.load area $((apcupsd_priority)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION load load absolute 1 100
+
+CHART apcupsd_${host}.load_usage '' "UPS Load Usage" "Watts" ups apcupsd.load_usage area $((apcupsd_priority + 1)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION load_usage load absolute 1 100
+
+CHART apcupsd_${host}.temp '' "UPS Temperature" "Celsius" ups apcupsd.temperature line $((apcupsd_priority + 8)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION temp temp absolute 1 100
+
+CHART apcupsd_${host}.time '' "UPS Time Remaining" "Minutes" ups apcupsd.time area $((apcupsd_priority + 3)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION time time absolute 1 100
+
+CHART apcupsd_${host}.online '' "UPS ONLINE flag" "boolean" ups apcupsd.online line $((apcupsd_priority + 9)) $apcupsd_update_every '' '' 'apcupsd'
+DIMENSION online online absolute 0 1
+
+EOF
+ done
+ return 0
+}
+
+apcupsd_update() {
+ # the first argument to this function is the microseconds since last update
+ # pass this parameter to the BEGIN statement (see below).
+
+ # do all the work to collect / calculate the values
+ # for each dimension
+ # remember: KEEP IT SIMPLE AND SHORT
+
+ local host working=0 failed=0
+ for host in "${!apcupsd_sources[@]}"; do
+ apcupsd_get "${apcupsd_sources[${host}]}" | awk "
+
+BEGIN {
+ battery_charge = 0;
+ battery_voltage = 0;
+ battery_voltage_nominal = 0;
+ input_voltage = 0;
+ input_voltage_min = 0;
+ input_voltage_max = 0;
+ input_frequency = 0;
+ output_voltage = 0;
+ output_voltage_nominal = 0;
+ load = 0;
+ temp = 0;
+ time = 0;
+ nompower = 0;
+ load_usage = 0;
+}
+/^BCHARGE.*/ { battery_charge = \$3 * 100 };
+/^BATTV.*/ { battery_voltage = \$3 * 100 };
+/^NOMBATTV.*/ { battery_voltage_nominal = \$3 * 100 };
+/^LINEV.*/ { input_voltage = \$3 * 100 };
+/^MINLINEV.*/ { input_voltage_min = \$3 * 100 };
+/^MAXLINEV.*/ { input_voltage_max = \$3 * 100 };
+/^LINEFREQ.*/ { input_frequency = \$3 * 100 };
+/^OUTPUTV.*/ { output_voltage = \$3 * 100 };
+/^NOMOUTV.*/ { output_voltage_nominal = \$3 * 100 };
+/^LOADPCT.*/ { load = \$3 * 100 };
+/^ITEMP.*/ { temp = \$3 * 100 };
+/^NOMPOWER.*/ { nompower = \$3 };
+/^TIMELEFT.*/ { time = \$3 * 100 };
+/^STATUS.*/ { online=(\$3 != \"COMMLOST\" && !(\$3 == \"SHUTTING\" && \$4 == \"DOWN\"))?1:0 };
+END {
+ { load_usage = nompower * load / 100 };
+
+ print \"BEGIN apcupsd_${host}.online $1\";
+ print \"SET online = \" online;
+ print \"END\"
+
+ if (online == 1) {
+ print \"BEGIN apcupsd_${host}.charge $1\";
+ print \"SET battery_charge = \" battery_charge;
+ print \"END\"
+
+ print \"BEGIN apcupsd_${host}.battery_voltage $1\";
+ print \"SET battery_voltage = \" battery_voltage;
+ print \"SET battery_voltage_nominal = \" battery_voltage_nominal;
+ print \"END\"
+
+ print \"BEGIN apcupsd_${host}.input_voltage $1\";
+ print \"SET input_voltage = \" input_voltage;
+ print \"SET input_voltage_min = \" input_voltage_min;
+ print \"SET input_voltage_max = \" input_voltage_max;
+ print \"END\"
+
+ print \"BEGIN apcupsd_${host}.input_frequency $1\";
+ print \"SET input_frequency = \" input_frequency;
+ print \"END\"
+
+ print \"BEGIN apcupsd_${host}.output_voltage $1\";
+ print \"SET output_voltage = \" output_voltage;
+ print \"SET output_voltage_nominal = \" output_voltage_nominal;
+ print \"END\"
+
+ print \"BEGIN apcupsd_${host}.load $1\";
+ print \"SET load = \" load;
+ print \"END\"
+
+ print \"BEGIN apcupsd_${host}.load_usage $1\";
+ print \"SET load_usage = \" load_usage;
+ print \"END\"
+
+ print \"BEGIN apcupsd_${host}.temp $1\";
+ print \"SET temp = \" temp;
+ print \"END\"
+
+ print \"BEGIN apcupsd_${host}.time $1\";
+ print \"SET time = \" time;
+ print \"END\"
+ }
+}"
+ # shellcheck disable=SC2181
+ if [ $? -ne 0 ]; then
+ failed=$((failed + 1))
+ error "failed to get values for APC UPS ${host} on ${apcupsd_sources[${host}]}" && return 1
+ else
+ working=$((working + 1))
+ fi
+ done
+
+ [ $working -eq 0 ] && error "failed to get values from all APC UPSes" && return 1
+
+ return 0
+}
diff --git a/collectors/charts.d.plugin/apcupsd/apcupsd.conf b/collectors/charts.d.plugin/apcupsd/apcupsd.conf
new file mode 100644
index 0000000..679c0d6
--- /dev/null
+++ b/collectors/charts.d.plugin/apcupsd/apcupsd.conf
@@ -0,0 +1,25 @@
+# no need for shebang - this file is loaded from charts.d.plugin
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2018 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+
+# add all your APC UPSes in this array - uncomment it too
+#declare -A apcupsd_sources=(
+# ["local"]="127.0.0.1:3551"
+#)
+
+# how long to wait for apcupsd to respond
+#apcupsd_timeout=3
+
+# the data collection frequency
+# if unset, will inherit the netdata update frequency
+#apcupsd_update_every=10
+
+# the charts priority on the dashboard
+#apcupsd_priority=90000
+
+# the number of retries to do in case of failure
+# before disabling the module
+#apcupsd_retries=10
diff --git a/collectors/charts.d.plugin/charts.d.conf b/collectors/charts.d.plugin/charts.d.conf
new file mode 100644
index 0000000..2d32f73
--- /dev/null
+++ b/collectors/charts.d.plugin/charts.d.conf
@@ -0,0 +1,48 @@
+# This is the configuration for charts.d.plugin
+
+# Each of its collectors can read configuration either from this file
+# or a NAME.conf file (where NAME is the collector name).
+# The collector specific file has higher precedence.
+
+# This file is a shell script too.
+
+# -----------------------------------------------------------------------------
+
+# number of seconds to run without restart
+# after this time, charts.d.plugin will exit
+# netdata will restart it, but a small gap
+# will appear in the charts.d.plugin charts.
+#restart_timeout=$[3600 * 4]
+
+# when making iterations, charts.d can loop more frequently
+# to prevent plugins missing iterations.
+# this is a percentage relative to update_every to align its
+# iterations.
+# The minimum is 10%, the maximum 100%.
+# So, if update_every is 1 second and time_divisor is 50,
+# charts.d will iterate every 500ms.
+# Charts will be called to collect data only if the time
+# passed since the last time the collected data is equal or
+# above their update_every.
+#time_divisor=50
+
+# -----------------------------------------------------------------------------
+
+# the default enable/disable for all charts.d collectors
+# the default is "yes"
+# enable_all_charts="yes"
+
+# BY DEFAULT ENABLED MODULES
+# ap=yes
+# apcupsd=yes
+# libreswan=yes
+# nut=yes
+# opensips=yes
+
+# -----------------------------------------------------------------------------
+# THESE NEED TO BE SET TO "force" TO BE ENABLED
+
+# Nothing useful.
+# Just an example charts.d plugin you can use as a template.
+# example=force
+# sensors=force
diff --git a/collectors/charts.d.plugin/charts.d.dryrun-helper.sh b/collectors/charts.d.plugin/charts.d.dryrun-helper.sh
new file mode 100755
index 0000000..91af2c5
--- /dev/null
+++ b/collectors/charts.d.plugin/charts.d.dryrun-helper.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# shellcheck disable=SC2181
+
+# will stop the script for any error
+set -e
+
+me="$0"
+name="$1"
+chart="$2"
+conf="$3"
+
+can_diff=1
+
+tmp1="$(mktemp)"
+tmp2="$(mktemp)"
+
+myset() {
+ set | grep -v "^_=" | grep -v "^PIPESTATUS=" | grep -v "^BASH_LINENO="
+}
+
+# save 2 'set'
+myset >"$tmp1"
+myset >"$tmp2"
+
+# make sure they don't differ
+diff "$tmp1" "$tmp2" >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+ # they differ, we cannot do the check
+ echo >&2 "$me: cannot check with diff."
+ can_diff=0
+fi
+
+# do it again, now including the script
+myset >"$tmp1"
+
+# include the plugin and its config
+if [ -f "$conf" ]; then
+ # shellcheck source=/dev/null
+ . "$conf"
+ if [ $? -ne 0 ]; then
+ echo >&2 "$me: cannot load config file $conf"
+ rm "$tmp1" "$tmp2"
+ exit 1
+ fi
+fi
+
+# shellcheck source=/dev/null
+. "$chart"
+if [ $? -ne 0 ]; then
+ echo >&2 "$me: cannot load chart file $chart"
+ rm "$tmp1" "$tmp2"
+ exit 1
+fi
+
+# remove all variables starting with the plugin name
+myset | grep -v "^$name" >"$tmp2"
+
+if [ $can_diff -eq 1 ]; then
+ # check if they are different
+ # make sure they don't differ
+ diff "$tmp1" "$tmp2" >&2
+ if [ $? -ne 0 ]; then
+ # they differ
+ rm "$tmp1" "$tmp2"
+ exit 1
+ fi
+fi
+
+rm "$tmp1" "$tmp2"
+exit 0
diff --git a/collectors/charts.d.plugin/charts.d.plugin.in b/collectors/charts.d.plugin/charts.d.plugin.in
new file mode 100755
index 0000000..9187fc2
--- /dev/null
+++ b/collectors/charts.d.plugin/charts.d.plugin.in
@@ -0,0 +1,732 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+#
+# charts.d.plugin allows easy development of BASH plugins
+#
+# if you need to run parallel charts.d processes, link this file to a different name
+# in the same directory, with a .plugin suffix and netdata will start both of them,
+# each will have a different config file and modules configuration directory.
+#
+
+export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+
+PROGRAM_FILE="$0"
+PROGRAM_NAME="$(basename $0)"
+PROGRAM_NAME="${PROGRAM_NAME/.plugin/}"
+MODULE_NAME="main"
+
+# -----------------------------------------------------------------------------
+# create temp dir
+
+debug=0
+TMP_DIR=
+chartsd_cleanup() {
+ trap '' EXIT QUIT HUP INT TERM
+
+ if [ ! -z "$TMP_DIR" -a -d "$TMP_DIR" ]; then
+ [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: cleaning up temporary directory $TMP_DIR ..."
+ rm -rf "$TMP_DIR"
+ fi
+ exit 0
+}
+trap chartsd_cleanup EXIT QUIT HUP INT TERM
+
+if [ $UID = "0" ]; then
+ TMP_DIR="$(mktemp -d /var/run/netdata-${PROGRAM_NAME}-XXXXXXXXXX)"
+else
+ TMP_DIR="$(mktemp -d /tmp/.netdata-${PROGRAM_NAME}-XXXXXXXXXX)"
+fi
+
+logdate() {
+ date "+%Y-%m-%d %H:%M:%S"
+}
+
+log() {
+ local status="${1}"
+ shift
+
+ echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${MODULE_NAME}: ${*}"
+
+}
+
+warning() {
+ log WARNING "${@}"
+}
+
+error() {
+ log ERROR "${@}"
+}
+
+info() {
+ log INFO "${@}"
+}
+
+fatal() {
+ log FATAL "${@}"
+ echo "DISABLE"
+ exit 1
+}
+
+debug() {
+ [ $debug -eq 1 ] && log DEBUG "${@}"
+}
+
+# -----------------------------------------------------------------------------
+# check a few commands
+
+require_cmd() {
+ local x=$(which "${1}" 2>/dev/null || command -v "${1}" 2>/dev/null)
+ if [ -z "${x}" -o ! -x "${x}" ]; then
+ warning "command '${1}' is not found in ${PATH}."
+ eval "${1^^}_CMD=\"\""
+ return 1
+ fi
+
+ eval "${1^^}_CMD=\"${x}\""
+ return 0
+}
+
+require_cmd date || exit 1
+require_cmd sed || exit 1
+require_cmd basename || exit 1
+require_cmd dirname || exit 1
+require_cmd cat || exit 1
+require_cmd grep || exit 1
+require_cmd egrep || exit 1
+require_cmd mktemp || exit 1
+require_cmd awk || exit 1
+require_cmd timeout || exit 1
+require_cmd curl || exit 1
+
+# -----------------------------------------------------------------------------
+
+[ $((BASH_VERSINFO[0])) -lt 4 ] && fatal "BASH version 4 or later is required, but found version: ${BASH_VERSION}. Please upgrade."
+
+info "started from '$PROGRAM_FILE' with options: $*"
+
+# -----------------------------------------------------------------------------
+# internal defaults
+# netdata exposes a few environment variables for us
+
+[ -z "${NETDATA_PLUGINS_DIR}" ] && NETDATA_PLUGINS_DIR="$(dirname "${0}")"
+[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
+[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
+
+pluginsd="${NETDATA_PLUGINS_DIR}"
+stockconfd="${NETDATA_STOCK_CONFIG_DIR}/${PROGRAM_NAME}"
+userconfd="${NETDATA_USER_CONFIG_DIR}/${PROGRAM_NAME}"
+olduserconfd="${NETDATA_USER_CONFIG_DIR}"
+chartsd="$pluginsd/../charts.d"
+
+minimum_update_frequency="${NETDATA_UPDATE_EVERY-1}"
+update_every=${minimum_update_frequency} # this will be overwritten by the command line
+
+# work around for non BASH shells
+charts_create="_create"
+charts_update="_update"
+charts_check="_check"
+charts_underscore="_"
+
+# when making iterations, charts.d can loop more frequently
+# to prevent plugins missing iterations.
+# this is a percentage relative to update_every to align its
+# iterations.
+# The minimum is 10%, the maximum 100%.
+# So, if update_every is 1 second and time_divisor is 50,
+# charts.d will iterate every 500ms.
+# Charts will be called to collect data only if the time
+# passed since the last time the collected data is equal or
+# above their update_every.
+time_divisor=50
+
+# number of seconds to run without restart
+# after this time, charts.d.plugin will exit
+# netdata will restart it
+restart_timeout=$((3600 * 4))
+
+# check if the charts.d plugins are using global variables
+# they should not.
+# It does not currently support BASH v4 arrays, so it is
+# disabled
+dryrunner=0
+
+# check for timeout command
+check_for_timeout=1
+
+# the default enable/disable value for all charts
+enable_all_charts="yes"
+
+# -----------------------------------------------------------------------------
+# parse parameters
+
+check=0
+chart_only=
+while [ ! -z "$1" ]; do
+ if [ "$1" = "check" ]; then
+ check=1
+ shift
+ continue
+ fi
+
+ if [ "$1" = "debug" -o "$1" = "all" ]; then
+ debug=1
+ shift
+ continue
+ fi
+
+ if [ -f "$chartsd/$1.chart.sh" ]; then
+ debug=1
+ chart_only="$(echo $1.chart.sh | sed "s/\.chart\.sh$//g")"
+ shift
+ continue
+ fi
+
+ if [ -f "$chartsd/$1" ]; then
+ debug=1
+ chart_only="$(echo $1 | sed "s/\.chart\.sh$//g")"
+ shift
+ continue
+ fi
+
+ # number check
+ n="$1"
+ x=$((n))
+ if [ "$x" = "$n" ]; then
+ shift
+ update_every=$x
+ [ $update_every -lt $minimum_update_frequency ] && update_every=$minimum_update_frequency
+ continue
+ fi
+
+ fatal "Cannot understand parameter $1. Aborting."
+done
+
+# -----------------------------------------------------------------------------
+# loop control
+
+# default sleep function
+LOOPSLEEPMS_HIGHRES=0
+now_ms=
+current_time_ms_default() {
+ now_ms="$(date +'%s')000"
+}
+current_time_ms="current_time_ms_default"
+current_time_ms_accuracy=1
+mysleep="sleep"
+
+# if found and included, this file overwrites loopsleepms()
+# and current_time_ms() with a high resolution timer function
+# for precise looping.
+source "$pluginsd/loopsleepms.sh.inc"
+[ $? -ne 0 ] && error "Failed to load '$pluginsd/loopsleepms.sh.inc'."
+
+# -----------------------------------------------------------------------------
+# load my configuration
+
+for myconfig in "${NETDATA_STOCK_CONFIG_DIR}/${PROGRAM_NAME}.conf" "${NETDATA_USER_CONFIG_DIR}/${PROGRAM_NAME}.conf"; do
+ if [ -f "$myconfig" ]; then
+ source "$myconfig"
+ if [ $? -ne 0 ]; then
+ error "Config file '$myconfig' loaded with errors."
+ else
+ info "Configuration file '$myconfig' loaded."
+ fi
+ else
+ warning "Configuration file '$myconfig' not found."
+ fi
+done
+
+# make sure time_divisor is right
+time_divisor=$((time_divisor))
+[ $time_divisor -lt 10 ] && time_divisor=10
+[ $time_divisor -gt 100 ] && time_divisor=100
+
+# we check for the timeout command, after we load our
+# configuration, so that the user may overwrite the
+# timeout command we use, providing a function that
+# can emulate the timeout command we need:
+# > timeout SECONDS command ...
+if [ $check_for_timeout -eq 1 ]; then
+ require_cmd timeout || exit 1
+fi
+
+# -----------------------------------------------------------------------------
+# internal checks
+
+# netdata passes the requested update frequency as the first argument
+update_every=$((update_every + 1 - 1)) # makes sure it is a number
+test $update_every -eq 0 && update_every=1 # if it is zero, make it 1
+
+# check the charts.d directory
+[ ! -d "$chartsd" ] && fatal "cannot find charts directory '$chartsd'"
+
+# -----------------------------------------------------------------------------
+# library functions
+
+fixid() {
+ echo "$*" |
+ tr -c "[A-Z][a-z][0-9]" "_" |
+ sed -e "s|^_\+||g" -e "s|_\+$||g" -e "s|_\+|_|g" |
+ tr "[A-Z]" "[a-z]"
+}
+
+isvarset() {
+ [ -n "$1" ] && [ "$1" != "unknown" ] && [ "$1" != "none" ]
+ return $?
+}
+
+getosid() {
+ if isvarset "${NETDATA_CONTAINER_OS_ID}"; then
+ echo "${NETDATA_CONTAINER_OS_ID}"
+ else
+ echo "${NETDATA_SYSTEM_OS_ID}"
+ fi
+}
+
+run() {
+ local ret pid="${BASHPID}" t
+
+ if [ "z${1}" = "z-t" -a "${2}" != "0" ]; then
+ t="${2}"
+ shift 2
+ timeout "${t}" "${@}" 2>"${TMP_DIR}/run.${pid}"
+ ret=$?
+ else
+ "${@}" 2>"${TMP_DIR}/run.${pid}"
+ ret=$?
+ fi
+
+ if [ ${ret} -ne 0 ]; then
+ {
+ printf "$(logdate): ${PROGRAM_NAME}: ${status}: ${MODULE_NAME}: command '"
+ printf "%q " "${@}"
+ printf "' failed with code ${ret}:\n --- BEGIN TRACE ---\n"
+ cat "${TMP_DIR}/run.${pid}"
+ printf " --- END TRACE ---\n"
+ } >&2
+ fi
+ rm -f "${TMP_DIR}/run.${pid}"
+
+ return ${ret}
+}
+
+# convert any floating point number
+# to integer, give a multiplier
+# the result is stored in ${FLOAT2INT_RESULT}
+# so that no fork is necessary
+# the multiplier must be a power of 10
+float2int() {
+ local f m="$2" a b l v=($1)
+ f=${v[0]}
+
+ # the length of the multiplier - 1
+ l=$((${#m} - 1))
+
+ # check if the number is in scientific notation
+ if [[ ${f} =~ ^[[:space:]]*(-)?[0-9.]+(e|E)(\+|-)[0-9]+ ]]; then
+ # convert it to decimal
+ # unfortunately, this fork cannot be avoided
+ # if you know of a way to avoid it, please let me know
+ f=$(printf "%0.${l}f" ${f})
+ fi
+
+ # split the floating point number
+ # in integer (a) and decimal (b)
+ a=${f/.*/}
+ b=${f/*./}
+
+ # if the integer part is missing
+ # set it to zero
+ [ -z "${a}" ] && a="0"
+
+ # strip leading zeros from the integer part
+ # base 10 conversion
+ a=$((10#$a))
+
+ # check the length of the decimal part
+ # against the length of the multiplier
+ if [ ${#b} -gt ${l} ]; then
+ # too many digits - take the most significant
+ b=${b:0:l}
+
+ elif [ ${#b} -lt ${l} ]; then
+ # too few digits - pad with zero on the right
+ local z="00000000000000000000000" r=$((l - ${#b}))
+ b="${b}${z:0:r}"
+ fi
+
+ # strip leading zeros from the decimal part
+ # base 10 conversion
+ b=$((10#$b))
+
+ # store the result
+ FLOAT2INT_RESULT=$(((a * m) + b))
+}
+
+# -----------------------------------------------------------------------------
+# charts check functions
+
+all_charts() {
+ cd "$chartsd"
+ [ $? -ne 0 ] && error "cannot cd to $chartsd" && return 1
+
+ ls *.chart.sh | sed "s/\.chart\.sh$//g"
+}
+
+declare -A charts_enable_keyword=(
+ ['apache']="force"
+ ['cpu_apps']="force"
+ ['cpufreq']="force"
+ ['example']="force"
+ ['exim']="force"
+ ['hddtemp']="force"
+ ['load_average']="force"
+ ['mem_apps']="force"
+ ['mysql']="force"
+ ['nginx']="force"
+ ['phpfpm']="force"
+ ['postfix']="force"
+ ['sensors']="force"
+ ['squid']="force"
+ ['tomcat']="force"
+)
+
+declare -A obsolete_charts=(
+ ['apache']="python.d.plugin module"
+ ['cpu_apps']="apps.plugin"
+ ['cpufreq']="proc plugin"
+ ['exim']="python.d.plugin module"
+ ['hddtemp']="python.d.plugin module"
+ ['load_average']="proc plugin"
+ ['mem_apps']="proc plugin"
+ ['mysql']="python.d.plugin module"
+ ['nginx']="python.d.plugin module"
+ ['phpfpm']="python.d.plugin module"
+ ['postfix']="python.d.plugin module"
+ ['squid']="python.d.plugin module"
+ ['tomcat']="python.d.plugin module"
+)
+
+all_enabled_charts() {
+ local charts enabled required
+
+ # find all enabled charts
+ for chart in $(all_charts); do
+ MODULE_NAME="${chart}"
+
+ if [ -n "${obsolete_charts["$MODULE_NAME"]}" ]; then
+ debug "is replaced by ${obsolete_charts["$MODULE_NAME"]}, skipping it."
+ continue
+ fi
+
+ eval "enabled=\$$chart"
+ if [ -z "${enabled}" ]; then
+ enabled="${enable_all_charts}"
+ fi
+
+ required="${charts_enable_keyword[${chart}]}"
+ [ -z "${required}" ] && required="yes"
+
+ if [ ! "${enabled}" = "${required}" ]; then
+ info "is disabled. Add a line with $chart=$required in '${NETDATA_USER_CONFIG_DIR}/${PROGRAM_NAME}.conf' to enable it (or remove the line that disables it)."
+ else
+ debug "is enabled for auto-detection."
+ local charts="$charts $chart"
+ fi
+ done
+ MODULE_NAME="main"
+
+ local charts2=
+ for chart in $charts; do
+ MODULE_NAME="${chart}"
+
+ # check the enabled charts
+ local check="$(cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_check()")"
+ if [ -z "$check" ]; then
+ error "module '$chart' does not seem to have a $chart$charts_check() function. Disabling it."
+ continue
+ fi
+
+ local create="$(cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_create()")"
+ if [ -z "$create" ]; then
+ error "module '$chart' does not seem to have a $chart$charts_create() function. Disabling it."
+ continue
+ fi
+
+ local update="$(cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_update()")"
+ if [ -z "$update" ]; then
+ error "module '$chart' does not seem to have a $chart$charts_update() function. Disabling it."
+ continue
+ fi
+
+ # check its config
+ #if [ -f "$userconfd/$chart.conf" ]
+ #then
+ # if [ ! -z "$( cat "$userconfd/$chart.conf" | sed "s/^ \+//g" | grep -v "^$" | grep -v "^#" | grep -v "^$chart$charts_underscore" )" ]
+ # then
+ # error "module's $chart config $userconfd/$chart.conf should only have lines starting with $chart$charts_underscore . Disabling it."
+ # continue
+ # fi
+ #fi
+
+ #if [ $dryrunner -eq 1 ]
+ # then
+ # "$pluginsd/charts.d.dryrun-helper.sh" "$chart" "$chartsd/$chart.chart.sh" "$userconfd/$chart.conf" >/dev/null
+ # if [ $? -ne 0 ]
+ # then
+ # error "module's $chart did not pass the dry run check. This means it uses global variables not starting with $chart. Disabling it."
+ # continue
+ # fi
+ #fi
+
+ local charts2="$charts2 $chart"
+ done
+ MODULE_NAME="main"
+
+ echo $charts2
+ debug "enabled charts: $charts2"
+}
+
+# -----------------------------------------------------------------------------
+# load the charts
+
+suffix_retries="_retries"
+suffix_update_every="_update_every"
+active_charts=
+for chart in $(all_enabled_charts); do
+ MODULE_NAME="${chart}"
+
+ debug "loading module: '$chartsd/$chart.chart.sh'"
+
+ source "$chartsd/$chart.chart.sh"
+ [ $? -ne 0 ] && warning "Module '$chartsd/$chart.chart.sh' loaded with errors."
+
+ # first load the stock config
+ if [ -f "$stockconfd/$chart.conf" ]; then
+ debug "loading module configuration: '$stockconfd/$chart.conf'"
+ source "$stockconfd/$chart.conf"
+ [ $? -ne 0 ] && warning "Config file '$stockconfd/$chart.conf' loaded with errors."
+ else
+ debug "not found module configuration: '$stockconfd/$chart.conf'"
+ fi
+
+ # then load the user config (it overwrites the stock)
+ if [ -f "$userconfd/$chart.conf" ]; then
+ debug "loading module configuration: '$userconfd/$chart.conf'"
+ source "$userconfd/$chart.conf"
+ [ $? -ne 0 ] && warning "Config file '$userconfd/$chart.conf' loaded with errors."
+ else
+ debug "not found module configuration: '$userconfd/$chart.conf'"
+
+ if [ -f "$olduserconfd/$chart.conf" ]; then
+ # support for very old netdata that had the charts.d module configs in /etc/netdata
+ info "loading module configuration from obsolete location: '$olduserconfd/$chart.conf'"
+ source "$olduserconfd/$chart.conf"
+ [ $? -ne 0 ] && warning "Config file '$olduserconfd/$chart.conf' loaded with errors."
+ fi
+ fi
+
+ eval "dt=\$$chart$suffix_update_every"
+ dt=$((dt + 1 - 1)) # make sure it is a number
+ if [ $dt -lt $update_every ]; then
+ eval "$chart$suffix_update_every=$update_every"
+ fi
+
+ $chart$charts_check
+ if [ $? -eq 0 ]; then
+ debug "module '$chart' activated"
+ active_charts="$active_charts $chart"
+ else
+ error "module's '$chart' check() function reports failure."
+ fi
+done
+MODULE_NAME="main"
+debug "activated modules: $active_charts"
+
+# -----------------------------------------------------------------------------
+# check overwrites
+
+# enable work time reporting
+debug_time=
+test $debug -eq 1 && debug_time=tellwork
+
+# if we only need a specific chart, remove all the others
+if [ ! -z "${chart_only}" ]; then
+ debug "requested to run only for: '${chart_only}'"
+ check_charts=
+ for chart in $active_charts; do
+ if [ "$chart" = "$chart_only" ]; then
+ check_charts="$chart"
+ break
+ fi
+ done
+ active_charts="$check_charts"
+fi
+debug "activated charts: $active_charts"
+
+# stop if we just need a pre-check
+if [ $check -eq 1 ]; then
+ info "CHECK RESULT"
+ info "Will run the charts: $active_charts"
+ exit 0
+fi
+
+# -----------------------------------------------------------------------------
+
+cd "${TMP_DIR}" || exit 1
+
+# -----------------------------------------------------------------------------
+# create charts
+
+run_charts=
+for chart in $active_charts; do
+ MODULE_NAME="${chart}"
+
+ debug "calling '$chart$charts_create()'..."
+ $chart$charts_create
+ if [ $? -eq 0 ]; then
+ run_charts="$run_charts $chart"
+ debug "'$chart' initialized."
+ else
+ error "module's '$chart' function '$chart$charts_create()' reports failure."
+ fi
+done
+MODULE_NAME="main"
+debug "run_charts='$run_charts'"
+
+# -----------------------------------------------------------------------------
+# update dimensions
+
+[ -z "$run_charts" ] && fatal "No charts to collect data from."
+
+keepalive() {
+ if [ ! -t 1 ] && ! printf "\n"; then
+ chartsd_cleanup
+ fi
+}
+
+declare -A charts_last_update=() charts_update_every=() charts_retries=() charts_next_update=() charts_run_counter=() charts_serial_failures=()
+global_update() {
+ local exit_at \
+ c=0 dt ret last_ms exec_start_ms exec_end_ms \
+ chart now_charts=() next_charts=($run_charts) \
+ next_ms x seconds millis
+
+ # return the current time in ms in $now_ms
+ ${current_time_ms}
+
+ exit_at=$((now_ms + (restart_timeout * 1000)))
+
+ for chart in $run_charts; do
+ eval "charts_update_every[$chart]=\$$chart$suffix_update_every"
+ test -z "${charts_update_every[$chart]}" && charts_update_every[$chart]=$update_every
+
+ eval "charts_retries[$chart]=\$$chart$suffix_retries"
+ test -z "${charts_retries[$chart]}" && charts_retries[$chart]=10
+
+ charts_last_update[$chart]=$((now_ms - (now_ms % (charts_update_every[$chart] * 1000))))
+ charts_next_update[$chart]=$((charts_last_update[$chart] + (charts_update_every[$chart] * 1000)))
+ charts_run_counter[$chart]=0
+ charts_serial_failures[$chart]=0
+
+ echo "CHART netdata.plugin_chartsd_$chart '' 'Execution time for $chart plugin' 'milliseconds / run' charts.d netdata.plugin_charts area 145000 ${charts_update_every[$chart]} '' '' '$chart'"
+ echo "DIMENSION run_time 'run time' absolute 1 1"
+ done
+
+ # the main loop
+ while [ "${#next_charts[@]}" -gt 0 ]; do
+ keepalive
+
+ c=$((c + 1))
+ now_charts=("${next_charts[@]}")
+ next_charts=()
+
+ # return the current time in ms in $now_ms
+ ${current_time_ms}
+
+ for chart in "${now_charts[@]}"; do
+ MODULE_NAME="${chart}"
+
+ if [ ${now_ms} -ge ${charts_next_update[$chart]} ]; then
+ last_ms=${charts_last_update[$chart]}
+ dt=$((now_ms - last_ms))
+
+ charts_last_update[$chart]=${now_ms}
+
+ while [ ${charts_next_update[$chart]} -lt ${now_ms} ]; do
+ charts_next_update[$chart]=$((charts_next_update[$chart] + (charts_update_every[$chart] * 1000)))
+ done
+
+ # the first call should not give a duration
+ # so that netdata calibrates to current time
+ dt=$((dt * 1000))
+ charts_run_counter[$chart]=$((charts_run_counter[$chart] + 1))
+ if [ ${charts_run_counter[$chart]} -eq 1 ]; then
+ dt=
+ fi
+
+ exec_start_ms=$now_ms
+ $chart$charts_update $dt
+ ret=$?
+
+ # return the current time in ms in $now_ms
+ ${current_time_ms}
+ exec_end_ms=$now_ms
+
+ echo "BEGIN netdata.plugin_chartsd_$chart $dt"
+ echo "SET run_time = $((exec_end_ms - exec_start_ms))"
+ echo "END"
+
+ if [ $ret -eq 0 ]; then
+ charts_serial_failures[$chart]=0
+ next_charts+=($chart)
+ else
+ charts_serial_failures[$chart]=$((charts_serial_failures[$chart] + 1))
+
+ if [ ${charts_serial_failures[$chart]} -gt ${charts_retries[$chart]} ]; then
+ error "module's '$chart' update() function reported failure ${charts_serial_failures[$chart]} times. Disabling it."
+ else
+ error "module's '$chart' update() function reports failure. Will keep trying for a while."
+ next_charts+=($chart)
+ fi
+ fi
+ else
+ next_charts+=($chart)
+ fi
+ done
+ MODULE_NAME="${chart}"
+
+ # wait the time you are required to
+ next_ms=$((now_ms + (update_every * 1000 * 100)))
+ for x in "${charts_next_update[@]}"; do [ ${x} -lt ${next_ms} ] && next_ms=${x}; done
+ next_ms=$((next_ms - now_ms))
+
+ if [ ${LOOPSLEEPMS_HIGHRES} -eq 1 -a ${next_ms} -gt 0 ]; then
+ next_ms=$((next_ms + current_time_ms_accuracy))
+ seconds=$((next_ms / 1000))
+ millis=$((next_ms % 1000))
+ if [ ${millis} -lt 10 ]; then
+ millis="00${millis}"
+ elif [ ${millis} -lt 100 ]; then
+ millis="0${millis}"
+ fi
+
+ debug "sleeping for ${seconds}.${millis} seconds."
+ ${mysleep} ${seconds}.${millis}
+ else
+ debug "sleeping for ${update_every} seconds."
+ ${mysleep} $update_every
+ fi
+
+ test ${now_ms} -ge ${exit_at} && exit 0
+ done
+
+ fatal "nothing left to do, exiting..."
+}
+
+global_update
diff --git a/collectors/charts.d.plugin/example/Makefile.inc b/collectors/charts.d.plugin/example/Makefile.inc
new file mode 100644
index 0000000..e6838fb
--- /dev/null
+++ b/collectors/charts.d.plugin/example/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_charts_DATA += example/example.chart.sh
+dist_chartsconfig_DATA += example/example.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += example/README.md example/Makefile.inc
+
diff --git a/collectors/charts.d.plugin/example/README.md b/collectors/charts.d.plugin/example/README.md
new file mode 100644
index 0000000..77446b2
--- /dev/null
+++ b/collectors/charts.d.plugin/example/README.md
@@ -0,0 +1,10 @@
+<!--
+title: "Example"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/charts.d.plugin/example/README.md
+-->
+
+# Example
+
+If you want to understand how charts.d data collector functions, check out the [charts.d example](https://raw.githubusercontent.com/netdata/netdata/master/collectors/charts.d.plugin/example/example.chart.sh).
+
+
diff --git a/collectors/charts.d.plugin/example/example.chart.sh b/collectors/charts.d.plugin/example/example.chart.sh
new file mode 100644
index 0000000..6bbbcf1
--- /dev/null
+++ b/collectors/charts.d.plugin/example/example.chart.sh
@@ -0,0 +1,123 @@
+# shellcheck shell=bash
+# no need for shebang - this file is loaded from charts.d.plugin
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
+#
+
+# if this chart is called X.chart.sh, then all functions and global variables
+# must start with X_
+
+# _update_every is a special variable - it holds the number of seconds
+# between the calls of the _update() function
+example_update_every=
+
+# the priority is used to sort the charts on the dashboard
+# 1 = the first chart
+example_priority=150000
+
+# to enable this chart, you have to set this to 12345
+# (just a demonstration for something that needs to be checked)
+example_magic_number=
+
+# global variables to store our collected data
+# remember: they need to start with the module name example_
+example_value1=
+example_value2=
+example_value3=
+example_value4=
+example_last=0
+example_count=0
+
+example_get() {
+ # do all the work to collect / calculate the values
+ # for each dimension
+ #
+ # Remember:
+ # 1. KEEP IT SIMPLE AND SHORT
+ # 2. AVOID FORKS (avoid piping commands)
+ # 3. AVOID CALLING TOO MANY EXTERNAL PROGRAMS
+ # 4. USE LOCAL VARIABLES (global variables may overlap with other modules)
+
+ example_value1=$RANDOM
+ example_value2=$RANDOM
+ example_value3=$RANDOM
+ example_value4=$((8192 + (RANDOM * 16383 / 32767)))
+
+ if [ $example_count -gt 0 ]; then
+ example_count=$((example_count - 1))
+
+ [ $example_last -gt 16383 ] && example_value4=$((example_last + (RANDOM * ((32767 - example_last) / 2) / 32767)))
+ [ $example_last -le 16383 ] && example_value4=$((example_last - (RANDOM * (example_last / 2) / 32767)))
+ else
+ example_count=$((1 + (RANDOM * 5 / 32767)))
+
+ if [ $example_last -gt 16383 ] && [ $example_value4 -gt 16383 ]; then
+ example_value4=$((example_value4 - 16383))
+ fi
+ if [ $example_last -le 16383 ] && [ $example_value4 -lt 16383 ]; then
+ example_value4=$((example_value4 + 16383))
+ fi
+ fi
+ example_last=$example_value4
+
+ # this should return:
+ # - 0 to send the data to netdata
+ # - 1 to report a failure to collect the data
+
+ return 0
+}
+
+# _check is called once, to find out if this chart should be enabled or not
+example_check() {
+ # this should return:
+ # - 0 to enable the chart
+ # - 1 to disable the chart
+
+ # check something
+ [ "${example_magic_number}" != "12345" ] && error "manual configuration required: you have to set example_magic_number=$example_magic_number in example.conf to start example chart." && return 1
+
+ # check that we can collect data
+ example_get || return 1
+
+ return 0
+}
+
+# _create is called once, to create the charts
+example_create() {
+ # create the chart with 3 dimensions
+ cat << EOF
+CHART example.random '' "Random Numbers Stacked Chart" "% of random numbers" random random stacked $((example_priority)) $example_update_every '' '' 'example'
+DIMENSION random1 '' percentage-of-absolute-row 1 1
+DIMENSION random2 '' percentage-of-absolute-row 1 1
+DIMENSION random3 '' percentage-of-absolute-row 1 1
+CHART example.random2 '' "A random number" "random number" random random area $((example_priority + 1)) $example_update_every '' '' 'example'
+DIMENSION random '' absolute 1 1
+EOF
+
+ return 0
+}
+
+# _update is called continuously, to collect the values
+example_update() {
+ # the first argument to this function is the microseconds since last update
+ # pass this parameter to the BEGIN statement (see below).
+
+ example_get || return 1
+
+ # write the result of the work.
+ cat << VALUESEOF
+BEGIN example.random $1
+SET random1 = $example_value1
+SET random2 = $example_value2
+SET random3 = $example_value3
+END
+BEGIN example.random2 $1
+SET random = $example_value4
+END
+VALUESEOF
+
+ return 0
+}
diff --git a/collectors/charts.d.plugin/example/example.conf b/collectors/charts.d.plugin/example/example.conf
new file mode 100644
index 0000000..6232ca5
--- /dev/null
+++ b/collectors/charts.d.plugin/example/example.conf
@@ -0,0 +1,21 @@
+# no need for shebang - this file is loaded from charts.d.plugin
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2018 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+
+# to enable this chart, you have to set this to 12345
+# (just a demonstration for something that needs to be checked)
+#example_magic_number=12345
+
+# the data collection frequency
+# if unset, will inherit the netdata update frequency
+#example_update_every=
+
+# the charts priority on the dashboard
+#example_priority=150000
+
+# the number of retries to do in case of failure
+# before disabling the module
+#example_retries=10
diff --git a/collectors/charts.d.plugin/libreswan/Makefile.inc b/collectors/charts.d.plugin/libreswan/Makefile.inc
new file mode 100644
index 0000000..af767d0
--- /dev/null
+++ b/collectors/charts.d.plugin/libreswan/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_charts_DATA += libreswan/libreswan.chart.sh
+dist_chartsconfig_DATA += libreswan/libreswan.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += libreswan/README.md libreswan/Makefile.inc
+
diff --git a/collectors/charts.d.plugin/libreswan/README.md b/collectors/charts.d.plugin/libreswan/README.md
new file mode 100644
index 0000000..41c4e24
--- /dev/null
+++ b/collectors/charts.d.plugin/libreswan/README.md
@@ -0,0 +1,56 @@
+<!--
+title: "Libreswan IPSec tunnel monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/charts.d.plugin/libreswan/README.md
+sidebar_label: "Libreswan IPSec tunnels"
+-->
+
+# Libreswan IPSec tunnel monitoring with Netdata
+
+Collects bytes-in, bytes-out and uptime for all established libreswan IPSEC tunnels.
+
+The following charts are created, **per tunnel**:
+
+1. **Uptime**
+
+- the uptime of the tunnel
+
+2. **Traffic**
+
+- bytes in
+- bytes out
+
+## Configuration
+
+Edit the `charts.d/libreswan.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config charts.d/libreswan.conf
+```
+
+The plugin executes 2 commands to collect all the information it needs:
+
+```sh
+ipsec whack --status
+ipsec whack --trafficstatus
+```
+
+The first command is used to extract the currently established tunnels, their IDs and their names.
+The second command is used to extract the current uptime and traffic.
+
+Most probably user `netdata` will not be able to query libreswan, so the `ipsec` commands will be denied.
+The plugin attempts to run `ipsec` as `sudo ipsec ...`, to get access to libreswan statistics.
+
+To allow user `netdata` execute `sudo ipsec ...`, create the file `/etc/sudoers.d/netdata` with this content:
+
+```
+netdata ALL = (root) NOPASSWD: /sbin/ipsec whack --status
+netdata ALL = (root) NOPASSWD: /sbin/ipsec whack --trafficstatus
+```
+
+Make sure the path `/sbin/ipsec` matches your setup (execute `which ipsec` to find the right path).
+
+---
+
+
diff --git a/collectors/charts.d.plugin/libreswan/libreswan.chart.sh b/collectors/charts.d.plugin/libreswan/libreswan.chart.sh
new file mode 100644
index 0000000..d526f7a
--- /dev/null
+++ b/collectors/charts.d.plugin/libreswan/libreswan.chart.sh
@@ -0,0 +1,187 @@
+# shellcheck shell=bash disable=SC1117
+# no need for shebang - this file is loaded from charts.d.plugin
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2018 Costa Tsaousis <costa@tsaousis.gr>
+#
+
+# _update_every is a special variable - it holds the number of seconds
+# between the calls of the _update() function
+libreswan_update_every=1
+
+# the priority is used to sort the charts on the dashboard
+# 1 = the first chart
+libreswan_priority=90000
+
+# set to 1, to run ipsec with sudo
+libreswan_sudo=1
+
+# global variables to store our collected data
+
+# [TUNNELID] = TUNNELNAME
+# here we track the *latest* established tunnels
+# as detected by: ipsec whack --status
+declare -A libreswan_connected_tunnels=()
+
+# [TUNNELID] = VALUE
+# here we track values of all established tunnels (not only the latest)
+# as detected by: ipsec whack --trafficstatus
+declare -A libreswan_traffic_in=()
+declare -A libreswan_traffic_out=()
+declare -A libreswan_established_add_time=()
+
+# [TUNNELNAME] = CHARTID
+# here we remember CHARTIDs of all tunnels
+# we need this to avoid converting tunnel names to chart IDs on every iteration
+declare -A libreswan_tunnel_charts=()
+
+is_able_sudo_ipsec() {
+ if ! sudo -n -l "${IPSEC_CMD}" whack --status > /dev/null 2>&1; then
+ return 1
+ fi
+ if ! sudo -n -l "${IPSEC_CMD}" whack --trafficstatus > /dev/null 2>&1; then
+ return 1
+ fi
+ return 0
+}
+
+# run the ipsec command
+libreswan_ipsec() {
+ if [ ${libreswan_sudo} -ne 0 ]; then
+ sudo -n "${IPSEC_CMD}" "${@}"
+ return $?
+ else
+ "${IPSEC_CMD}" "${@}"
+ return $?
+ fi
+}
+
+# fetch latest values - fill the arrays
+libreswan_get() {
+ # do all the work to collect / calculate the values
+ # for each dimension
+
+ # empty the variables
+ libreswan_traffic_in=()
+ libreswan_traffic_out=()
+ libreswan_established_add_time=()
+ libreswan_connected_tunnels=()
+
+ # convert the ipsec command output to a shell script
+ # and source it to get the values
+ # shellcheck disable=SC1090
+ source <(
+ {
+ libreswan_ipsec whack --status
+ libreswan_ipsec whack --trafficstatus
+ } | sed -n \
+ -e "s|[0-9]\+ #\([0-9]\+\): \"\(.*\)\".*IPsec SA established.*newest IPSEC.*|libreswan_connected_tunnels[\"\1\"]=\"\2\"|p" \
+ -e "s|[0-9]\+ #\([0-9]\+\): \"\(.*\)\",\{0,1\}.* add_time=\([0-9]\+\),.* inBytes=\([0-9]\+\),.* outBytes=\([0-9]\+\).*|libreswan_traffic_in[\"\1\"]=\"\4\"; libreswan_traffic_out[\"\1\"]=\"\5\"; libreswan_established_add_time[\"\1\"]=\"\3\";|p"
+ ) || return 1
+
+ # check we got some data
+ [ ${#libreswan_connected_tunnels[@]} -eq 0 ] && return 1
+
+ return 0
+}
+
+# _check is called once, to find out if this chart should be enabled or not
+libreswan_check() {
+ # this should return:
+ # - 0 to enable the chart
+ # - 1 to disable the chart
+
+ require_cmd ipsec || return 1
+
+ # make sure it is libreswan
+ # shellcheck disable=SC2143
+ if [ -z "$(ipsec --version | grep -i libreswan)" ]; then
+ error "ipsec command is not Libreswan. Disabling Libreswan plugin."
+ return 1
+ fi
+
+ if [ ${libreswan_sudo} -ne 0 ] && ! is_able_sudo_ipsec; then
+ error "not enough permissions to execute ipsec with sudo. Disabling Libreswan plugin."
+ return 1
+ fi
+
+ # check that we can collect data
+ libreswan_get || return 1
+
+ return 0
+}
+
+# create the charts for an ipsec tunnel
+libreswan_create_one() {
+ local n="${1}" name
+
+ name="${libreswan_connected_tunnels[${n}]}"
+
+ [ -n "${libreswan_tunnel_charts[${name}]}" ] && return 0
+
+ libreswan_tunnel_charts[${name}]="$(fixid "${name}")"
+
+ cat << EOF
+CHART libreswan.${libreswan_tunnel_charts[${name}]}_net '${name}_net' "LibreSWAN Tunnel ${name} Traffic" "kilobits/s" "${name}" libreswan.net area $((libreswan_priority)) $libreswan_update_every '' '' 'libreswan'
+DIMENSION in '' incremental 8 1000
+DIMENSION out '' incremental -8 1000
+CHART libreswan.${libreswan_tunnel_charts[${name}]}_uptime '${name}_uptime' "LibreSWAN Tunnel ${name} Uptime" "seconds" "${name}" libreswan.uptime line $((libreswan_priority + 1)) $libreswan_update_every '' '' 'libreswan'
+DIMENSION uptime '' absolute 1 1
+EOF
+
+ return 0
+
+}
+
+# _create is called once, to create the charts
+libreswan_create() {
+ local n
+ for n in "${!libreswan_connected_tunnels[@]}"; do
+ libreswan_create_one "${n}"
+ done
+ return 0
+}
+
+libreswan_now=$(date +%s)
+
+# send the values to netdata for an ipsec tunnel
+libreswan_update_one() {
+ local n="${1}" microseconds="${2}" name id uptime
+
+ name="${libreswan_connected_tunnels[${n}]}"
+ id="${libreswan_tunnel_charts[${name}]}"
+
+ [ -z "${id}" ] && libreswan_create_one "${name}"
+
+ uptime=$((libreswan_now - libreswan_established_add_time[${n}]))
+ [ ${uptime} -lt 0 ] && uptime=0
+
+ # write the result of the work.
+ cat << VALUESEOF
+BEGIN libreswan.${id}_net ${microseconds}
+SET in = ${libreswan_traffic_in[${n}]}
+SET out = ${libreswan_traffic_out[${n}]}
+END
+BEGIN libreswan.${id}_uptime ${microseconds}
+SET uptime = ${uptime}
+END
+VALUESEOF
+}
+
+# _update is called continuously, to collect the values
+libreswan_update() {
+ # the first argument to this function is the microseconds since last update
+ # pass this parameter to the BEGIN statement (see below).
+
+ libreswan_get || return 1
+ libreswan_now=$(date +%s)
+
+ local n
+ for n in "${!libreswan_connected_tunnels[@]}"; do
+ libreswan_update_one "${n}" "${@}"
+ done
+
+ return 0
+}
diff --git a/collectors/charts.d.plugin/libreswan/libreswan.conf b/collectors/charts.d.plugin/libreswan/libreswan.conf
new file mode 100644
index 0000000..9b3ee77
--- /dev/null
+++ b/collectors/charts.d.plugin/libreswan/libreswan.conf
@@ -0,0 +1,29 @@
+# no need for shebang - this file is loaded from charts.d.plugin
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2018 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+#
+
+# the data collection frequency
+# if unset, will inherit the netdata update frequency
+#libreswan_update_every=1
+
+# the charts priority on the dashboard
+#libreswan_priority=90000
+
+# the number of retries to do in case of failure
+# before disabling the module
+#libreswan_retries=10
+
+# set to 1, to run ipsec with sudo (the default)
+# set to 0, to run ipsec without sudo
+#libreswan_sudo=1
+
+# TO ALLOW NETDATA RUN ipsec AS ROOT
+# CREATE THE FILE: /etc/sudoers.d/netdata
+# WITH THESE 2 LINES (uncommented of course):
+#
+# netdata ALL = (root) NOPASSWD: /sbin/ipsec whack --status
+# netdata ALL = (root) NOPASSWD: /sbin/ipsec whack --trafficstatus
diff --git a/collectors/charts.d.plugin/loopsleepms.sh.inc b/collectors/charts.d.plugin/loopsleepms.sh.inc
new file mode 100644
index 0000000..5608b8d
--- /dev/null
+++ b/collectors/charts.d.plugin/loopsleepms.sh.inc
@@ -0,0 +1,227 @@
+# no need for shebang - this file is included from other scripts
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+LOOPSLEEP_DATE="$(which date 2>/dev/null || command -v date 2>/dev/null)"
+if [ -z "$LOOPSLEEP_DATE" ]; then
+ echo >&2 "$0: ERROR: Cannot find the command 'date' in the system path."
+ exit 1
+fi
+
+# -----------------------------------------------------------------------------
+# use the date command as a high resolution timer
+
+# macOS 'date' doesn't support '%N' precision
+# echo $(/bin/date +"%N") is "N"
+if [ "$($LOOPSLEEP_DATE +"%N")" = "N" ]; then
+ LOOPSLEEP_DATE_FORMAT="%s * 1000"
+else
+ LOOPSLEEP_DATE_FORMAT="%s * 1000 + 10#%-N / 1000000"
+fi
+
+now_ms=
+LOOPSLEEPMS_HIGHRES=1
+test "$($LOOPSLEEP_DATE +%N)" = "%N" && LOOPSLEEPMS_HIGHRES=0
+test -z "$($LOOPSLEEP_DATE +%N)" && LOOPSLEEPMS_HIGHRES=0
+current_time_ms_from_date() {
+ if [ $LOOPSLEEPMS_HIGHRES -eq 0 ]; then
+ now_ms="$($LOOPSLEEP_DATE +'%s')000"
+ else
+ now_ms="$(($($LOOPSLEEP_DATE +"$LOOPSLEEP_DATE_FORMAT")))"
+ fi
+}
+
+# -----------------------------------------------------------------------------
+# use /proc/uptime as a high resolution timer
+
+current_time_ms_from_date
+current_time_ms_from_uptime_started="${now_ms}"
+current_time_ms_from_uptime_last="${now_ms}"
+current_time_ms_from_uptime_first=0
+current_time_ms_from_uptime() {
+ local up rest arr=() n
+
+ read up rest </proc/uptime
+ if [ $? -ne 0 ]; then
+ echo >&2 "$0: Cannot read /proc/uptime - falling back to current_time_ms_from_date()."
+ current_time_ms="current_time_ms_from_date"
+ current_time_ms_from_date
+ current_time_ms_accuracy=1
+ return
+ fi
+
+ arr=(${up//./ })
+
+ if [ ${#arr[1]} -lt 1 ]; then
+ n="${arr[0]}000"
+ elif [ ${#arr[1]} -lt 2 ]; then
+ n="${arr[0]}${arr[1]}00"
+ elif [ ${#arr[1]} -lt 3 ]; then
+ n="${arr[0]}${arr[1]}0"
+ else
+ n="${arr[0]}${arr[1]}"
+ fi
+
+ now_ms=$((current_time_ms_from_uptime_started - current_time_ms_from_uptime_first + n))
+
+ if [ "${now_ms}" -lt "${current_time_ms_from_uptime_last}" ]; then
+ echo >&2 "$0: Cannot use current_time_ms_from_uptime() - new time ${now_ms} is older than the last ${current_time_ms_from_uptime_last} - falling back to current_time_ms_from_date()."
+ current_time_ms="current_time_ms_from_date"
+ current_time_ms_from_date
+ current_time_ms_accuracy=1
+ fi
+
+ current_time_ms_from_uptime_last="${now_ms}"
+}
+current_time_ms_from_uptime
+current_time_ms_from_uptime_first="$((now_ms - current_time_ms_from_uptime_started))"
+current_time_ms_from_uptime_last="${current_time_ms_from_uptime_first}"
+current_time_ms="current_time_ms_from_uptime"
+current_time_ms_accuracy=10
+if [ "${current_time_ms_from_uptime_first}" -eq 0 ]; then
+ echo >&2 "$0: Invalid setup for current_time_ms_from_uptime() - falling back to current_time_ms_from_date()."
+ current_time_ms="current_time_ms_from_date"
+ current_time_ms_accuracy=1
+fi
+
+# -----------------------------------------------------------------------------
+# use read with timeout for sleep
+
+mysleep=""
+
+mysleep_fifo="${NETDATA_CACHE_DIR-/tmp}/.netdata_bash_sleep_timer_fifo"
+[ -f "${mysleep_fifo}" ] && rm "${mysleep_fifo}"
+[ ! -p "${mysleep_fifo}" ] && mkfifo "${mysleep_fifo}"
+[ -p "${mysleep_fifo}" ] && mysleep="mysleep_read"
+
+mysleep_read() {
+ read -t "${1}" <>"${mysleep_fifo}"
+ ret=$?
+ if [ $ret -le 128 ]; then
+ echo >&2 "$0: Cannot use read for sleeping (return code ${ret})."
+ mysleep="sleep"
+ ${mysleep} "${1}"
+ fi
+}
+
+# -----------------------------------------------------------------------------
+# use bash loadable module for sleep
+
+mysleep_builtin() {
+ builtin sleep "${1}"
+ ret=$?
+ if [ $ret -ne 0 ]; then
+ echo >&2 "$0: Cannot use builtin sleep for sleeping (return code ${ret})."
+ mysleep="sleep"
+ ${mysleep} "${1}"
+ fi
+}
+
+if [ -z "${mysleep}" -a "$((BASH_VERSINFO[0] + 0))" -ge 3 -a "${NETDATA_BASH_LOADABLES}" != "DISABLE" ]; then
+ # enable modules only for bash version 3+
+
+ for bash_modules_path in ${BASH_LOADABLES_PATH//:/ } "$(pkg-config bash --variable=loadablesdir 2>/dev/null)" "/usr/lib/bash" "/lib/bash" "/lib64/bash" "/usr/local/lib/bash" "/usr/local/lib64/bash"; do
+ [ -z "${bash_modules_path}" -o ! -d "${bash_modules_path}" ] && continue
+
+ # check for sleep
+ for bash_module_sleep in "sleep" "sleep.so"; do
+ if [ -f "${bash_modules_path}/${bash_module_sleep}" ]; then
+ if enable -f "${bash_modules_path}/${bash_module_sleep}" sleep 2>/dev/null; then
+ mysleep="mysleep_builtin"
+ # echo >&2 "$0: Using bash loadable ${bash_modules_path}/${bash_module_sleep} for sleep"
+ break
+ fi
+ fi
+
+ done
+
+ [ ! -z "${mysleep}" ] && break
+ done
+fi
+
+# -----------------------------------------------------------------------------
+# fallback to external sleep
+
+[ -z "${mysleep}" ] && mysleep="sleep"
+
+# -----------------------------------------------------------------------------
+# this function is used to sleep a fraction of a second
+# it calculates the difference between every time is called
+# and tries to align the sleep time to give you exactly the
+# loop you need.
+
+LOOPSLEEPMS_LASTRUN=0
+LOOPSLEEPMS_NEXTRUN=0
+LOOPSLEEPMS_LASTSLEEP=0
+LOOPSLEEPMS_LASTWORK=0
+
+loopsleepms() {
+ local tellwork=0 t="${1}" div s m now mstosleep
+
+ if [ "${t}" = "tellwork" ]; then
+ tellwork=1
+ shift
+ t="${1}"
+ fi
+
+ # $t = the time in seconds to wait
+
+ # if high resolution is not supported
+ # just sleep the time requested, in seconds
+ if [ ${LOOPSLEEPMS_HIGHRES} -eq 0 ]; then
+ sleep ${t}
+ return
+ fi
+
+ # get the current time, in ms in ${now_ms}
+ ${current_time_ms}
+
+ # calculate ms since last run
+ [ ${LOOPSLEEPMS_LASTRUN} -gt 0 ] &&
+ LOOPSLEEPMS_LASTWORK=$((now_ms - LOOPSLEEPMS_LASTRUN - LOOPSLEEPMS_LASTSLEEP + current_time_ms_accuracy))
+ # echo "# last loop's work took $LOOPSLEEPMS_LASTWORK ms"
+
+ # remember this run
+ LOOPSLEEPMS_LASTRUN=${now_ms}
+
+ # calculate the next run
+ LOOPSLEEPMS_NEXTRUN=$(((now_ms - (now_ms % (t * 1000))) + (t * 1000)))
+
+ # calculate ms to sleep
+ mstosleep=$((LOOPSLEEPMS_NEXTRUN - now_ms + current_time_ms_accuracy))
+ # echo "# mstosleep is $mstosleep ms"
+
+ # if we are too slow, sleep some time
+ test ${mstosleep} -lt 200 && mstosleep=200
+
+ s=$((mstosleep / 1000))
+ m=$((mstosleep - (s * 1000)))
+ [ "${m}" -lt 100 ] && m="0${m}"
+ [ "${m}" -lt 10 ] && m="0${m}"
+
+ test $tellwork -eq 1 && echo >&2 " >>> PERFORMANCE >>> WORK TOOK ${LOOPSLEEPMS_LASTWORK} ms ( $((LOOPSLEEPMS_LASTWORK * 100 / 1000)).$((LOOPSLEEPMS_LASTWORK % 10))% cpu ) >>> SLEEPING ${mstosleep} ms"
+
+ # echo "# sleeping ${s}.${m}"
+ # echo
+ ${mysleep} ${s}.${m}
+
+ # keep the values we need
+ # for our next run
+ LOOPSLEEPMS_LASTSLEEP=$mstosleep
+}
+
+# test it
+#while [ 1 ]
+#do
+# r=$(( (RANDOM * 2000 / 32767) ))
+# s=$((r / 1000))
+# m=$((r - (s * 1000)))
+# [ "${m}" -lt 100 ] && m="0${m}"
+# [ "${m}" -lt 10 ] && m="0${m}"
+# echo "${r} = ${s}.${m}"
+#
+# # the work
+# ${mysleep} ${s}.${m}
+#
+# # the alignment loop
+# loopsleepms tellwork 1
+#done
diff --git a/collectors/charts.d.plugin/nut/Makefile.inc b/collectors/charts.d.plugin/nut/Makefile.inc
new file mode 100644
index 0000000..4fb4714
--- /dev/null
+++ b/collectors/charts.d.plugin/nut/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_charts_DATA += nut/nut.chart.sh
+dist_chartsconfig_DATA += nut/nut.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += nut/README.md nut/Makefile.inc
+
diff --git a/collectors/charts.d.plugin/nut/README.md b/collectors/charts.d.plugin/nut/README.md
new file mode 100644
index 0000000..69d7622
--- /dev/null
+++ b/collectors/charts.d.plugin/nut/README.md
@@ -0,0 +1,74 @@
+<!--
+title: "UPS/PDU monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/charts.d.plugin/nut/README.md
+sidebar_label: "UPS/PDU"
+-->
+
+# UPS/PDU monitoring with Netdata
+
+Collects UPS data for all power devices configured in the system.
+
+The following charts will be created:
+
+1. **UPS Charge**
+
+- percentage changed
+
+2. **UPS Battery Voltage**
+
+- current voltage
+- high voltage
+- low voltage
+- nominal voltage
+
+3. **UPS Input Voltage**
+
+- current voltage
+- fault voltage
+- nominal voltage
+
+4. **UPS Input Current**
+
+- nominal current
+
+5. **UPS Input Frequency**
+
+- current frequency
+- nominal frequency
+
+6. **UPS Output Voltage**
+
+- current voltage
+
+7. **UPS Load**
+
+- current load
+
+8. **UPS Temperature**
+
+- current temperature
+
+## Configuration
+
+Edit the `charts.d/nut.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config charts.d/nut.conf
+```
+
+This is the internal default for `charts.d/nut.conf`
+
+```sh
+# a space separated list of UPS names
+# if empty, the list returned by 'upsc -l' will be used
+nut_ups=
+
+# how frequently to collect UPS data
+nut_update_every=2
+```
+
+---
+
+
diff --git a/collectors/charts.d.plugin/nut/nut.chart.sh b/collectors/charts.d.plugin/nut/nut.chart.sh
new file mode 100644
index 0000000..2f7e3f3
--- /dev/null
+++ b/collectors/charts.d.plugin/nut/nut.chart.sh
@@ -0,0 +1,232 @@
+# shellcheck shell=bash
+# no need for shebang - this file is loaded from charts.d.plugin
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2016-2017 Costa Tsaousis <costa@tsaousis.gr>
+#
+
+# a space separated list of UPS names
+# if empty, the list returned by 'upsc -l' will be used
+nut_ups=
+
+# how frequently to collect UPS data
+nut_update_every=2
+
+# how much time in seconds, to wait for nut to respond
+nut_timeout=2
+
+# set this to 1, to enable another chart showing the number
+# of UPS clients connected to upsd
+nut_clients_chart=0
+
+# the priority of nut related to other charts
+nut_priority=90000
+
+declare -A nut_ids=()
+declare -A nut_names=()
+
+nut_get_all() {
+ run -t $nut_timeout upsc -l
+}
+
+nut_get() {
+ run -t $nut_timeout upsc "$1"
+
+ if [ "${nut_clients_chart}" -eq "1" ]; then
+ printf "ups.connected_clients: "
+ run -t $nut_timeout upsc -c "$1" | wc -l
+ fi
+}
+
+nut_check() {
+
+ # this should return:
+ # - 0 to enable the chart
+ # - 1 to disable the chart
+
+ local x
+
+ require_cmd upsc || return 1
+
+ [ -z "$nut_ups" ] && nut_ups="$(nut_get_all)"
+
+ for x in $nut_ups; do
+ nut_get "$x" > /dev/null
+ # shellcheck disable=SC2181
+ if [ $? -eq 0 ]; then
+ if [ -n "${nut_names[${x}]}" ]; then
+ nut_ids[$x]="$(fixid "${nut_names[${x}]}")"
+ else
+ nut_ids[$x]="$(fixid "$x")"
+ fi
+ continue
+ fi
+ error "cannot get information for NUT UPS '$x'."
+ done
+
+ if [ ${#nut_ids[@]} -eq 0 ]; then
+ # shellcheck disable=SC2154
+ error "Cannot find UPSes - please set nut_ups='ups_name' in $confd/nut.conf"
+ return 1
+ fi
+
+ return 0
+}
+
+nut_create() {
+ # create the charts
+ local x
+
+ for x in "${nut_ids[@]}"; do
+ cat << EOF
+CHART nut_$x.charge '' "UPS Charge" "percentage" ups nut.charge area $((nut_priority + 1)) $nut_update_every
+DIMENSION battery_charge charge absolute 1 100
+
+CHART nut_$x.runtime '' "UPS Runtime" "seconds" ups nut.runtime area $((nut_priority + 2)) $nut_update_every
+DIMENSION battery_runtime runtime absolute 1 100
+
+CHART nut_$x.battery_voltage '' "UPS Battery Voltage" "Volts" ups nut.battery.voltage line $((nut_priority + 3)) $nut_update_every
+DIMENSION battery_voltage voltage absolute 1 100
+DIMENSION battery_voltage_high high absolute 1 100
+DIMENSION battery_voltage_low low absolute 1 100
+DIMENSION battery_voltage_nominal nominal absolute 1 100
+
+CHART nut_$x.input_voltage '' "UPS Input Voltage" "Volts" input nut.input.voltage line $((nut_priority + 4)) $nut_update_every
+DIMENSION input_voltage voltage absolute 1 100
+DIMENSION input_voltage_fault fault absolute 1 100
+DIMENSION input_voltage_nominal nominal absolute 1 100
+
+CHART nut_$x.input_current '' "UPS Input Current" "Ampere" input nut.input.current line $((nut_priority + 5)) $nut_update_every
+DIMENSION input_current_nominal nominal absolute 1 100
+
+CHART nut_$x.input_frequency '' "UPS Input Frequency" "Hz" input nut.input.frequency line $((nut_priority + 6)) $nut_update_every
+DIMENSION input_frequency frequency absolute 1 100
+DIMENSION input_frequency_nominal nominal absolute 1 100
+
+CHART nut_$x.output_voltage '' "UPS Output Voltage" "Volts" output nut.output.voltage line $((nut_priority + 7)) $nut_update_every
+DIMENSION output_voltage voltage absolute 1 100
+
+CHART nut_$x.load '' "UPS Load" "percentage" ups nut.load area $((nut_priority)) $nut_update_every
+DIMENSION load load absolute 1 100
+
+CHART nut_$x.temp '' "UPS Temperature" "temperature" ups nut.temperature line $((nut_priority + 8)) $nut_update_every
+DIMENSION temp temp absolute 1 100
+EOF
+
+ if [ "${nut_clients_chart}" = "1" ]; then
+ cat << EOF2
+CHART nut_$x.clients '' "UPS Connected Clients" "clients" ups nut.clients area $((nut_priority + 9)) $nut_update_every
+DIMENSION clients '' absolute 1 1
+EOF2
+ fi
+
+ done
+
+ return 0
+}
+
+nut_update() {
+ # the first argument to this function is the microseconds since last update
+ # pass this parameter to the BEGIN statement (see below).
+
+ # do all the work to collect / calculate the values
+ # for each dimension
+ # remember: KEEP IT SIMPLE AND SHORT
+
+ local i x
+ for i in "${!nut_ids[@]}"; do
+ x="${nut_ids[$i]}"
+ nut_get "$i" | awk "
+BEGIN {
+ battery_charge = 0;
+ battery_runtime = 0;
+ battery_voltage = 0;
+ battery_voltage_high = 0;
+ battery_voltage_low = 0;
+ battery_voltage_nominal = 0;
+ input_voltage = 0;
+ input_voltage_fault = 0;
+ input_voltage_nominal = 0;
+ input_current_nominal = 0;
+ input_frequency = 0;
+ input_frequency_nominal = 0;
+ output_voltage = 0;
+ load = 0;
+ temp = 0;
+ client = 0;
+ do_clients = ${nut_clients_chart};
+}
+/^battery.charge: .*/ { battery_charge = \$2 * 100 };
+/^battery.runtime: .*/ { battery_runtime = \$2 * 100 };
+/^battery.voltage: .*/ { battery_voltage = \$2 * 100 };
+/^battery.voltage.high: .*/ { battery_voltage_high = \$2 * 100 };
+/^battery.voltage.low: .*/ { battery_voltage_low = \$2 * 100 };
+/^battery.voltage.nominal: .*/ { battery_voltage_nominal = \$2 * 100 };
+/^input.voltage: .*/ { input_voltage = \$2 * 100 };
+/^input.voltage.fault: .*/ { input_voltage_fault = \$2 * 100 };
+/^input.voltage.nominal: .*/ { input_voltage_nominal = \$2 * 100 };
+/^input.current.nominal: .*/ { input_current_nominal = \$2 * 100 };
+/^input.frequency: .*/ { input_frequency = \$2 * 100 };
+/^input.frequency.nominal: .*/ { input_frequency_nominal = \$2 * 100 };
+/^output.voltage: .*/ { output_voltage = \$2 * 100 };
+/^ups.load: .*/ { load = \$2 * 100 };
+/^ups.temperature: .*/ { temp = \$2 * 100 };
+/^ups.connected_clients: .*/ { clients = \$2 };
+END {
+ print \"BEGIN nut_$x.charge $1\";
+ print \"SET battery_charge = \" battery_charge;
+ print \"END\"
+
+ print \"BEGIN nut_$x.runtime $1\";
+ print \"SET battery_runtime = \" battery_runtime;
+ print \"END\"
+
+ print \"BEGIN nut_$x.battery_voltage $1\";
+ print \"SET battery_voltage = \" battery_voltage;
+ print \"SET battery_voltage_high = \" battery_voltage_high;
+ print \"SET battery_voltage_low = \" battery_voltage_low;
+ print \"SET battery_voltage_nominal = \" battery_voltage_nominal;
+ print \"END\"
+
+ print \"BEGIN nut_$x.input_voltage $1\";
+ print \"SET input_voltage = \" input_voltage;
+ print \"SET input_voltage_fault = \" input_voltage_fault;
+ print \"SET input_voltage_nominal = \" input_voltage_nominal;
+ print \"END\"
+
+ print \"BEGIN nut_$x.input_current $1\";
+ print \"SET input_current_nominal = \" input_current_nominal;
+ print \"END\"
+
+ print \"BEGIN nut_$x.input_frequency $1\";
+ print \"SET input_frequency = \" input_frequency;
+ print \"SET input_frequency_nominal = \" input_frequency_nominal;
+ print \"END\"
+
+ print \"BEGIN nut_$x.output_voltage $1\";
+ print \"SET output_voltage = \" output_voltage;
+ print \"END\"
+
+ print \"BEGIN nut_$x.load $1\";
+ print \"SET load = \" load;
+ print \"END\"
+
+ print \"BEGIN nut_$x.temp $1\";
+ print \"SET temp = \" temp;
+ print \"END\"
+
+ if(do_clients) {
+ print \"BEGIN nut_$x.clients $1\";
+ print \"SET clients = \" clients;
+ print \"END\"
+ }
+}"
+ # shellcheck disable=2181
+ [ $? -ne 0 ] && unset "nut_ids[$i]" && error "failed to get values for '$i', disabling it."
+ done
+
+ [ ${#nut_ids[@]} -eq 0 ] && error "no UPSes left active." && return 1
+ return 0
+}
diff --git a/collectors/charts.d.plugin/nut/nut.conf b/collectors/charts.d.plugin/nut/nut.conf
new file mode 100644
index 0000000..b95ad90
--- /dev/null
+++ b/collectors/charts.d.plugin/nut/nut.conf
@@ -0,0 +1,33 @@
+# no need for shebang - this file is loaded from charts.d.plugin
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2018 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+
+# a space separated list of UPS names
+# if empty, the list returned by 'upsc -l' will be used
+#nut_ups=
+
+# each line represents an alias for one UPS
+# if empty, the FQDN will be used
+#nut_names["FQDN1"]="alias"
+#nut_names["FQDN2"]="alias"
+
+# how much time in seconds, to wait for nut to respond
+#nut_timeout=2
+
+# set this to 1, to enable another chart showing the number
+# of UPS clients connected to upsd
+#nut_clients_chart=1
+
+# the data collection frequency
+# if unset, will inherit the netdata update frequency
+#nut_update_every=2
+
+# the charts priority on the dashboard
+#nut_priority=90000
+
+# the number of retries to do in case of failure
+# before disabling the module
+#nut_retries=10
diff --git a/collectors/charts.d.plugin/opensips/Makefile.inc b/collectors/charts.d.plugin/opensips/Makefile.inc
new file mode 100644
index 0000000..a7b5d3a
--- /dev/null
+++ b/collectors/charts.d.plugin/opensips/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_charts_DATA += opensips/opensips.chart.sh
+dist_chartsconfig_DATA += opensips/opensips.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += opensips/README.md opensips/Makefile.inc
+
diff --git a/collectors/charts.d.plugin/opensips/README.md b/collectors/charts.d.plugin/opensips/README.md
new file mode 100644
index 0000000..b08d192
--- /dev/null
+++ b/collectors/charts.d.plugin/opensips/README.md
@@ -0,0 +1,19 @@
+<!--
+title: "OpenSIPS monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/charts.d.plugin/opensips/README.md
+sidebar_label: "OpenSIPS"
+-->
+
+# OpenSIPS monitoring with Netdata
+
+## Configuration
+
+Edit the `charts.d/opensips.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config charts.d/opensips.conf
+```
+
+
diff --git a/collectors/charts.d.plugin/opensips/opensips.chart.sh b/collectors/charts.d.plugin/opensips/opensips.chart.sh
new file mode 100644
index 0000000..02401fd
--- /dev/null
+++ b/collectors/charts.d.plugin/opensips/opensips.chart.sh
@@ -0,0 +1,325 @@
+# shellcheck shell=bash disable=SC1117,SC2154,SC2086
+# no need for shebang - this file is loaded from charts.d.plugin
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
+#
+
+opensips_opts="fifo get_statistics all"
+opensips_cmd=
+opensips_update_every=5
+opensips_timeout=2
+opensips_priority=80000
+
+opensips_get_stats() {
+ run -t $opensips_timeout "$opensips_cmd" $opensips_opts |
+ grep "^\(core\|dialog\|net\|registrar\|shmem\|siptrace\|sl\|tm\|uri\|usrloc\):[a-zA-Z0-9_-]\+[[:space:]]*[=:]\+[[:space:]]*[0-9]\+[[:space:]]*$" |
+ sed \
+ -e "s|[[:space:]]*[=:]\+[[:space:]]*\([0-9]\+\)[[:space:]]*$|=\1|g" \
+ -e "s|[[:space:]:-]\+|_|g" \
+ -e "s|^|opensips_|g"
+
+ local ret=$?
+ [ $ret -ne 0 ] && echo "opensips_command_failed=1"
+ return $ret
+}
+
+opensips_check() {
+ # if the user did not provide an opensips_cmd
+ # try to find it in the system
+ if [ -z "$opensips_cmd" ]; then
+ require_cmd opensipsctl || return 1
+ opensips_cmd="$OPENSIPSCTL_CMD"
+ fi
+
+ # check once if the command works
+ local x
+ x="$(opensips_get_stats | grep "^opensips_core_")"
+ # shellcheck disable=SC2181
+ if [ ! $? -eq 0 ] || [ -z "$x" ]; then
+ error "cannot get global status. Please set opensips_opts='options' whatever needed to get connected to opensips server, in $confd/opensips.conf"
+ return 1
+ fi
+
+ return 0
+}
+
+opensips_create() {
+ # create the charts
+ cat << EOF
+CHART opensips.dialogs_active '' "OpenSIPS Active Dialogs" "dialogs" dialogs '' area $((opensips_priority + 1)) $opensips_update_every '' '' 'opensips'
+DIMENSION dialog_active_dialogs active absolute 1 1
+DIMENSION dialog_early_dialogs early absolute -1 1
+
+CHART opensips.users '' "OpenSIPS Users" "users" users '' line $((opensips_priority + 2)) $opensips_update_every '' '' 'opensips'
+DIMENSION usrloc_registered_users registered absolute 1 1
+DIMENSION usrloc_location_users location absolute 1 1
+DIMENSION usrloc_location_contacts contacts absolute 1 1
+DIMENSION usrloc_location_expires expires incremental -1 1
+
+CHART opensips.registrar '' "OpenSIPS Registrar" "registrations/s" registrar '' line $((opensips_priority + 3)) $opensips_update_every '' '' 'opensips'
+DIMENSION registrar_accepted_regs accepted incremental 1 1
+DIMENSION registrar_rejected_regs rejected incremental -1 1
+
+CHART opensips.transactions '' "OpenSIPS Transactions" "transactions/s" transactions '' line $((opensips_priority + 4)) $opensips_update_every '' '' 'opensips'
+DIMENSION tm_UAS_transactions UAS incremental 1 1
+DIMENSION tm_UAC_transactions UAC incremental -1 1
+
+CHART opensips.core_rcv '' "OpenSIPS Core Receives" "queries/s" core '' line $((opensips_priority + 5)) $opensips_update_every '' '' 'opensips'
+DIMENSION core_rcv_requests requests incremental 1 1
+DIMENSION core_rcv_replies replies incremental -1 1
+
+CHART opensips.core_fwd '' "OpenSIPS Core Forwards" "queries/s" core '' line $((opensips_priority + 6)) $opensips_update_every '' '' 'opensips'
+DIMENSION core_fwd_requests requests incremental 1 1
+DIMENSION core_fwd_replies replies incremental -1 1
+
+CHART opensips.core_drop '' "OpenSIPS Core Drops" "queries/s" core '' line $((opensips_priority + 7)) $opensips_update_every '' '' 'opensips'
+DIMENSION core_drop_requests requests incremental 1 1
+DIMENSION core_drop_replies replies incremental -1 1
+
+CHART opensips.core_err '' "OpenSIPS Core Errors" "queries/s" core '' line $((opensips_priority + 8)) $opensips_update_every '' '' 'opensips'
+DIMENSION core_err_requests requests incremental 1 1
+DIMENSION core_err_replies replies incremental -1 1
+
+CHART opensips.core_bad '' "OpenSIPS Core Bad" "queries/s" core '' line $((opensips_priority + 9)) $opensips_update_every '' '' 'opensips'
+DIMENSION core_bad_URIs_rcvd bad_URIs_rcvd incremental 1 1
+DIMENSION core_unsupported_methods unsupported_methods incremental 1 1
+DIMENSION core_bad_msg_hdr bad_msg_hdr incremental 1 1
+
+CHART opensips.tm_replies '' "OpenSIPS TM Replies" "replies/s" transactions '' line $((opensips_priority + 10)) $opensips_update_every '' '' 'opensips'
+DIMENSION tm_received_replies received incremental 1 1
+DIMENSION tm_relayed_replies relayed incremental 1 1
+DIMENSION tm_local_replies local incremental 1 1
+
+CHART opensips.transactions_status '' "OpenSIPS Transactions Status" "transactions/s" transactions '' line $((opensips_priority + 11)) $opensips_update_every '' '' 'opensips'
+DIMENSION tm_2xx_transactions 2xx incremental 1 1
+DIMENSION tm_3xx_transactions 3xx incremental 1 1
+DIMENSION tm_4xx_transactions 4xx incremental 1 1
+DIMENSION tm_5xx_transactions 5xx incremental 1 1
+DIMENSION tm_6xx_transactions 6xx incremental 1 1
+
+CHART opensips.transactions_inuse '' "OpenSIPS InUse Transactions" "transactions" transactions '' line $((opensips_priority + 12)) $opensips_update_every '' '' 'opensips'
+DIMENSION tm_inuse_transactions inuse absolute 1 1
+
+CHART opensips.sl_replies '' "OpenSIPS SL Replies" "replies/s" core '' line $((opensips_priority + 13)) $opensips_update_every '' '' 'opensips'
+DIMENSION sl_1xx_replies 1xx incremental 1 1
+DIMENSION sl_2xx_replies 2xx incremental 1 1
+DIMENSION sl_3xx_replies 3xx incremental 1 1
+DIMENSION sl_4xx_replies 4xx incremental 1 1
+DIMENSION sl_5xx_replies 5xx incremental 1 1
+DIMENSION sl_6xx_replies 6xx incremental 1 1
+DIMENSION sl_sent_replies sent incremental 1 1
+DIMENSION sl_sent_err_replies error incremental 1 1
+DIMENSION sl_received_ACKs ACKed incremental 1 1
+
+CHART opensips.dialogs '' "OpenSIPS Dialogs" "dialogs/s" dialogs '' line $((opensips_priority + 14)) $opensips_update_every '' '' 'opensips'
+DIMENSION dialog_processed_dialogs processed incremental 1 1
+DIMENSION dialog_expired_dialogs expired incremental 1 1
+DIMENSION dialog_failed_dialogs failed incremental -1 1
+
+CHART opensips.net_waiting '' "OpenSIPS Network Waiting" "kilobytes" net '' line $((opensips_priority + 15)) $opensips_update_every '' '' 'opensips'
+DIMENSION net_waiting_udp UDP absolute 1 1024
+DIMENSION net_waiting_tcp TCP absolute 1 1024
+
+CHART opensips.uri_checks '' "OpenSIPS URI Checks" "checks / sec" uri '' line $((opensips_priority + 16)) $opensips_update_every '' '' 'opensips'
+DIMENSION uri_positive_checks positive incremental 1 1
+DIMENSION uri_negative_checks negative incremental -1 1
+
+CHART opensips.traces '' "OpenSIPS Traces" "traces / sec" traces '' line $((opensips_priority + 17)) $opensips_update_every '' '' 'opensips'
+DIMENSION siptrace_traced_requests requests incremental 1 1
+DIMENSION siptrace_traced_replies replies incremental -1 1
+
+CHART opensips.shmem '' "OpenSIPS Shared Memory" "kilobytes" mem '' line $((opensips_priority + 18)) $opensips_update_every '' '' 'opensips'
+DIMENSION shmem_total_size total absolute 1 1024
+DIMENSION shmem_used_size used absolute 1 1024
+DIMENSION shmem_real_used_size real_used absolute 1 1024
+DIMENSION shmem_max_used_size max_used absolute 1 1024
+DIMENSION shmem_free_size free absolute 1 1024
+
+CHART opensips.shmem_fragments '' "OpenSIPS Shared Memory Fragmentation" "fragments" mem '' line $((opensips_priority + 19)) $opensips_update_every '' '' 'opensips'
+DIMENSION shmem_fragments fragments absolute 1 1
+EOF
+
+ return 0
+}
+
+opensips_update() {
+ # the first argument to this function is the microseconds since last update
+ # pass this parameter to the BEGIN statement (see below).
+
+ # do all the work to collect / calculate the values
+ # for each dimension
+
+ # 1. get the counters page from opensips
+ # 2. sed to remove spaces; replace . with _; remove spaces around =; prepend each line with: local opensips_
+ # 3. egrep lines starting with:
+ # local opensips_client_http_ then one or more of these a-z 0-9 _ then = and one of more of 0-9
+ # local opensips_server_all_ then one or more of these a-z 0-9 _ then = and one of more of 0-9
+ # 4. then execute this as a script with the eval
+ # be very careful with eval:
+ # prepare the script and always grep at the end the lines that are useful, so that
+ # even if something goes wrong, no other code can be executed
+
+ unset \
+ opensips_dialog_active_dialogs \
+ opensips_dialog_early_dialogs \
+ opensips_usrloc_registered_users \
+ opensips_usrloc_location_users \
+ opensips_usrloc_location_contacts \
+ opensips_usrloc_location_expires \
+ opensips_registrar_accepted_regs \
+ opensips_registrar_rejected_regs \
+ opensips_tm_UAS_transactions \
+ opensips_tm_UAC_transactions \
+ opensips_core_rcv_requests \
+ opensips_core_rcv_replies \
+ opensips_core_fwd_requests \
+ opensips_core_fwd_replies \
+ opensips_core_drop_requests \
+ opensips_core_drop_replies \
+ opensips_core_err_requests \
+ opensips_core_err_replies \
+ opensips_core_bad_URIs_rcvd \
+ opensips_core_unsupported_methods \
+ opensips_core_bad_msg_hdr \
+ opensips_tm_received_replies \
+ opensips_tm_relayed_replies \
+ opensips_tm_local_replies \
+ opensips_tm_2xx_transactions \
+ opensips_tm_3xx_transactions \
+ opensips_tm_4xx_transactions \
+ opensips_tm_5xx_transactions \
+ opensips_tm_6xx_transactions \
+ opensips_tm_inuse_transactions \
+ opensips_sl_1xx_replies \
+ opensips_sl_2xx_replies \
+ opensips_sl_3xx_replies \
+ opensips_sl_4xx_replies \
+ opensips_sl_5xx_replies \
+ opensips_sl_6xx_replies \
+ opensips_sl_sent_replies \
+ opensips_sl_sent_err_replies \
+ opensips_sl_received_ACKs \
+ opensips_dialog_processed_dialogs \
+ opensips_dialog_expired_dialogs \
+ opensips_dialog_failed_dialogs \
+ opensips_net_waiting_udp \
+ opensips_net_waiting_tcp \
+ opensips_uri_positive_checks \
+ opensips_uri_negative_checks \
+ opensips_siptrace_traced_requests \
+ opensips_siptrace_traced_replies \
+ opensips_shmem_total_size \
+ opensips_shmem_used_size \
+ opensips_shmem_real_used_size \
+ opensips_shmem_max_used_size \
+ opensips_shmem_free_size \
+ opensips_shmem_fragments
+
+ opensips_command_failed=0
+ eval "local $(opensips_get_stats)"
+ # shellcheck disable=SC2181
+ [ $? -ne 0 ] && return 1
+
+ [ $opensips_command_failed -eq 1 ] && error "failed to get values, disabling." && return 1
+
+ # write the result of the work.
+ cat << VALUESEOF
+BEGIN opensips.dialogs_active $1
+SET dialog_active_dialogs = $opensips_dialog_active_dialogs
+SET dialog_early_dialogs = $opensips_dialog_early_dialogs
+END
+BEGIN opensips.users $1
+SET usrloc_registered_users = $opensips_usrloc_registered_users
+SET usrloc_location_users = $opensips_usrloc_location_users
+SET usrloc_location_contacts = $opensips_usrloc_location_contacts
+SET usrloc_location_expires = $opensips_usrloc_location_expires
+END
+BEGIN opensips.registrar $1
+SET registrar_accepted_regs = $opensips_registrar_accepted_regs
+SET registrar_rejected_regs = $opensips_registrar_rejected_regs
+END
+BEGIN opensips.transactions $1
+SET tm_UAS_transactions = $opensips_tm_UAS_transactions
+SET tm_UAC_transactions = $opensips_tm_UAC_transactions
+END
+BEGIN opensips.core_rcv $1
+SET core_rcv_requests = $opensips_core_rcv_requests
+SET core_rcv_replies = $opensips_core_rcv_replies
+END
+BEGIN opensips.core_fwd $1
+SET core_fwd_requests = $opensips_core_fwd_requests
+SET core_fwd_replies = $opensips_core_fwd_replies
+END
+BEGIN opensips.core_drop $1
+SET core_drop_requests = $opensips_core_drop_requests
+SET core_drop_replies = $opensips_core_drop_replies
+END
+BEGIN opensips.core_err $1
+SET core_err_requests = $opensips_core_err_requests
+SET core_err_replies = $opensips_core_err_replies
+END
+BEGIN opensips.core_bad $1
+SET core_bad_URIs_rcvd = $opensips_core_bad_URIs_rcvd
+SET core_unsupported_methods = $opensips_core_unsupported_methods
+SET core_bad_msg_hdr = $opensips_core_bad_msg_hdr
+END
+BEGIN opensips.tm_replies $1
+SET tm_received_replies = $opensips_tm_received_replies
+SET tm_relayed_replies = $opensips_tm_relayed_replies
+SET tm_local_replies = $opensips_tm_local_replies
+END
+BEGIN opensips.transactions_status $1
+SET tm_2xx_transactions = $opensips_tm_2xx_transactions
+SET tm_3xx_transactions = $opensips_tm_3xx_transactions
+SET tm_4xx_transactions = $opensips_tm_4xx_transactions
+SET tm_5xx_transactions = $opensips_tm_5xx_transactions
+SET tm_6xx_transactions = $opensips_tm_6xx_transactions
+END
+BEGIN opensips.transactions_inuse $1
+SET tm_inuse_transactions = $opensips_tm_inuse_transactions
+END
+BEGIN opensips.sl_replies $1
+SET sl_1xx_replies = $opensips_sl_1xx_replies
+SET sl_2xx_replies = $opensips_sl_2xx_replies
+SET sl_3xx_replies = $opensips_sl_3xx_replies
+SET sl_4xx_replies = $opensips_sl_4xx_replies
+SET sl_5xx_replies = $opensips_sl_5xx_replies
+SET sl_6xx_replies = $opensips_sl_6xx_replies
+SET sl_sent_replies = $opensips_sl_sent_replies
+SET sl_sent_err_replies = $opensips_sl_sent_err_replies
+SET sl_received_ACKs = $opensips_sl_received_ACKs
+END
+BEGIN opensips.dialogs $1
+SET dialog_processed_dialogs = $opensips_dialog_processed_dialogs
+SET dialog_expired_dialogs = $opensips_dialog_expired_dialogs
+SET dialog_failed_dialogs = $opensips_dialog_failed_dialogs
+END
+BEGIN opensips.net_waiting $1
+SET net_waiting_udp = $opensips_net_waiting_udp
+SET net_waiting_tcp = $opensips_net_waiting_tcp
+END
+BEGIN opensips.uri_checks $1
+SET uri_positive_checks = $opensips_uri_positive_checks
+SET uri_negative_checks = $opensips_uri_negative_checks
+END
+BEGIN opensips.traces $1
+SET siptrace_traced_requests = $opensips_siptrace_traced_requests
+SET siptrace_traced_replies = $opensips_siptrace_traced_replies
+END
+BEGIN opensips.shmem $1
+SET shmem_total_size = $opensips_shmem_total_size
+SET shmem_used_size = $opensips_shmem_used_size
+SET shmem_real_used_size = $opensips_shmem_real_used_size
+SET shmem_max_used_size = $opensips_shmem_max_used_size
+SET shmem_free_size = $opensips_shmem_free_size
+END
+BEGIN opensips.shmem_fragments $1
+SET shmem_fragments = $opensips_shmem_fragments
+END
+VALUESEOF
+
+ return 0
+}
diff --git a/collectors/charts.d.plugin/opensips/opensips.conf b/collectors/charts.d.plugin/opensips/opensips.conf
new file mode 100644
index 0000000..e25111d
--- /dev/null
+++ b/collectors/charts.d.plugin/opensips/opensips.conf
@@ -0,0 +1,21 @@
+# no need for shebang - this file is loaded from charts.d.plugin
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2018 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+
+#opensips_opts="fifo get_statistics all"
+#opensips_cmd=
+#opensips_timeout=2
+
+# the data collection frequency
+# if unset, will inherit the netdata update frequency
+#opensips_update_every=5
+
+# the charts priority on the dashboard
+#opensips_priority=80000
+
+# the number of retries to do in case of failure
+# before disabling the module
+#opensips_retries=10
diff --git a/collectors/charts.d.plugin/sensors/Makefile.inc b/collectors/charts.d.plugin/sensors/Makefile.inc
new file mode 100644
index 0000000..f466a1b
--- /dev/null
+++ b/collectors/charts.d.plugin/sensors/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_charts_DATA += sensors/sensors.chart.sh
+dist_chartsconfig_DATA += sensors/sensors.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += sensors/README.md sensors/Makefile.inc
+
diff --git a/collectors/charts.d.plugin/sensors/README.md b/collectors/charts.d.plugin/sensors/README.md
new file mode 100644
index 0000000..1b98b1a
--- /dev/null
+++ b/collectors/charts.d.plugin/sensors/README.md
@@ -0,0 +1,79 @@
+<!--
+title: "Linux machine sensors monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/charts.d.plugin/sensors/README.md
+-->
+
+# Linux machine sensors monitoring with Netdata
+
+Use this collector when `lm-sensors` doesn't work on your device (e.g. for RPi temperatures).
+For all other cases use the [Python collector](/collectors/python.d.plugin/sensors), which supports multiple
+jobs, is more efficient and performs calculations on top of the kernel provided values.
+
+This plugin will provide charts for all configured system sensors, by reading sensors directly from the kernel.
+The values graphed are the raw hardware values of the sensors.
+
+The plugin will create Netdata charts for:
+
+1. **Temperature**
+2. **Voltage**
+3. **Current**
+4. **Power**
+5. **Fans Speed**
+6. **Energy**
+7. **Humidity**
+
+One chart for every sensor chip found and each of the above will be created.
+
+## Enable the collector
+
+The `sensors` collector is disabled by default. To enable it, edit the `charts.d.conf` file using `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config charts.d.conf
+```
+
+It also needs to be set to "force" to be enabled:
+
+```shell
+# example=force
+sensors=force
+```
+
+## Configuration
+
+Edit the `charts.d/sensors.conf` configuration file using `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config charts.d/sensors.conf
+```
+
+This is the internal default for `charts.d/sensors.conf`
+
+```sh
+# the directory the kernel keeps sensor data
+sensors_sys_dir="${NETDATA_HOST_PREFIX}/sys/devices"
+
+# how deep in the tree to check for sensor data
+sensors_sys_depth=10
+
+# if set to 1, the script will overwrite internal
+# script functions with code generated ones
+# leave to 1, is faster
+sensors_source_update=1
+
+# how frequently to collect sensor data
+# the default is to collect it at every iteration of charts.d
+sensors_update_every=
+
+# array of sensors which are excluded
+# the default is to include all
+sensors_excluded=()
+```
+
+---
+
+
diff --git a/collectors/charts.d.plugin/sensors/sensors.chart.sh b/collectors/charts.d.plugin/sensors/sensors.chart.sh
new file mode 100644
index 0000000..0527e1e
--- /dev/null
+++ b/collectors/charts.d.plugin/sensors/sensors.chart.sh
@@ -0,0 +1,250 @@
+# shellcheck shell=bash
+# no need for shebang - this file is loaded from charts.d.plugin
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
+#
+
+# sensors docs
+# https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface
+
+# if this chart is called X.chart.sh, then all functions and global variables
+# must start with X_
+
+# the directory the kernel keeps sensor data
+sensors_sys_dir="${NETDATA_HOST_PREFIX}/sys/devices"
+
+# how deep in the tree to check for sensor data
+sensors_sys_depth=10
+
+# if set to 1, the script will overwrite internal
+# script functions with code generated ones
+# leave to 1, is faster
+sensors_source_update=1
+
+# how frequently to collect sensor data
+# the default is to collect it at every iteration of charts.d
+sensors_update_every=
+
+sensors_priority=90000
+
+declare -A sensors_excluded=()
+
+sensors_find_all_files() {
+ find "$1" -maxdepth $sensors_sys_depth -name \*_input -o -name temp 2>/dev/null
+}
+
+sensors_find_all_dirs() {
+ # shellcheck disable=SC2162
+ sensors_find_all_files "$1" | while read; do
+ dirname "$REPLY"
+ done | sort -u
+}
+
+# _check is called once, to find out if this chart should be enabled or not
+sensors_check() {
+
+ # this should return:
+ # - 0 to enable the chart
+ # - 1 to disable the chart
+
+ [ -z "$(sensors_find_all_files "$sensors_sys_dir")" ] && error "no sensors found in '$sensors_sys_dir'." && return 1
+ return 0
+}
+
+sensors_check_files() {
+ # we only need sensors that report a non-zero value
+ # also remove not needed sensors
+
+ local f v excluded
+ for f in "$@"; do
+ [ ! -f "$f" ] && continue
+ for ex in "${sensors_excluded[@]}"; do
+ [[ $f =~ .*$ex$ ]] && excluded='1' && break
+ done
+
+ [ "$excluded" != "1" ] && v="$(cat "$f")" || v=0
+ v=$((v + 1 - 1))
+ [ $v -ne 0 ] && echo "$f" && continue
+ excluded=
+
+ error "$f gives zero values"
+ done
+}
+
+sensors_check_temp_type() {
+ # valid temp types are 1 to 6
+ # disabled sensors have the value 0
+
+ local f t v
+ for f in "$@"; do
+ # shellcheck disable=SC2001
+ t=$(echo "$f" | sed "s|_input$|_type|g")
+ [ "$f" = "$t" ] && echo "$f" && continue
+ [ ! -f "$t" ] && echo "$f" && continue
+
+ v="$(cat "$t")"
+ v=$((v + 1 - 1))
+ [ $v -ne 0 ] && echo "$f" && continue
+
+ error "$f is disabled"
+ done
+}
+
+# _create is called once, to create the charts
+sensors_create() {
+ local path dir name x file lfile labelname device subsystem id type mode files multiplier divisor
+
+ # we create a script with the source of the
+ # sensors_update() function
+ # - the highest speed we can achieve -
+ [ $sensors_source_update -eq 1 ] && echo >"$TMP_DIR/sensors.sh" "sensors_update() {"
+
+ for path in $(sensors_find_all_dirs "$sensors_sys_dir" | sort -u); do
+ dir=$(basename "$path")
+ device=
+ subsystem=
+ id=
+ type=
+ name=
+
+ [ -h "$path/device" ] && device=$(readlink -f "$path/device")
+ [ ! -z "$device" ] && device=$(basename "$device")
+ [ -z "$device" ] && device="$dir"
+
+ [ -h "$path/subsystem" ] && subsystem=$(readlink -f "$path/subsystem")
+ [ ! -z "$subsystem" ] && subsystem=$(basename "$subsystem")
+ [ -z "$subsystem" ] && subsystem="$dir"
+
+ [ -f "$path/name" ] && name=$(cat "$path/name")
+ [ -z "$name" ] && name="$dir"
+
+ [ -f "$path/type" ] && type=$(cat "$path/type")
+ [ -z "$type" ] && type="$dir"
+
+ id="$(fixid "$device.$subsystem.$dir")"
+
+ debug "path='$path', dir='$dir', device='$device', subsystem='$subsystem', id='$id', name='$name'"
+
+ for mode in temperature voltage fans power current energy humidity; do
+ files=
+ multiplier=1
+ divisor=1
+ algorithm="absolute"
+
+ case $mode in
+ temperature)
+ files="$(
+ ls "$path"/temp*_input 2>/dev/null
+ ls "$path/temp" 2>/dev/null
+ )"
+ files="$(sensors_check_files "$files")"
+ files="$(sensors_check_temp_type "$files")"
+ [ -z "$files" ] && continue
+ echo "CHART 'sensors.temp_${id}_${name}' '' 'Temperature' 'Celsius' 'temperature' 'sensors.temp' line $((sensors_priority + 1)) $sensors_update_every '' '' 'sensors'"
+ echo >>"$TMP_DIR/sensors.sh" "echo \"BEGIN 'sensors.temp_${id}_${name}' \$1\""
+ divisor=1000
+ ;;
+
+ voltage)
+ files="$(ls "$path"/in*_input 2>/dev/null)"
+ files="$(sensors_check_files "$files")"
+ [ -z "$files" ] && continue
+ echo "CHART 'sensors.volt_${id}_${name}' '' 'Voltage' 'Volts' 'voltage' 'sensors.volt' line $((sensors_priority + 2)) $sensors_update_every '' '' 'sensors'"
+ echo >>"$TMP_DIR/sensors.sh" "echo \"BEGIN 'sensors.volt_${id}_${name}' \$1\""
+ divisor=1000
+ ;;
+
+ current)
+ files="$(ls "$path"/curr*_input 2>/dev/null)"
+ files="$(sensors_check_files "$files")"
+ [ -z "$files" ] && continue
+ echo "CHART 'sensors.curr_${id}_${name}' '' 'Current' 'Ampere' 'current' 'sensors.curr' line $((sensors_priority + 3)) $sensors_update_every '' '' 'sensors'"
+ echo >>"$TMP_DIR/sensors.sh" "echo \"BEGIN 'sensors.curr_${id}_${name}' \$1\""
+ divisor=1000
+ ;;
+
+ power)
+ files="$(ls "$path"/power*_input 2>/dev/null)"
+ files="$(sensors_check_files "$files")"
+ [ -z "$files" ] && continue
+ echo "CHART 'sensors.power_${id}_${name}' '' 'Power' 'Watt' 'power' 'sensors.power' line $((sensors_priority + 4)) $sensors_update_every '' '' 'sensors'"
+ echo >>"$TMP_DIR/sensors.sh" "echo \"BEGIN 'sensors.power_${id}_${name}' \$1\""
+ divisor=1000000
+ ;;
+
+ fans)
+ files="$(ls "$path"/fan*_input 2>/dev/null)"
+ files="$(sensors_check_files "$files")"
+ [ -z "$files" ] && continue
+ echo "CHART 'sensors.fan_${id}_${name}' '' 'Fans Speed' 'Rotations / Minute' 'fans' 'sensors.fans' line $((sensors_priority + 5)) $sensors_update_every '' '' 'sensors'"
+ echo >>"$TMP_DIR/sensors.sh" "echo \"BEGIN 'sensors.fan_${id}_${name}' \$1\""
+ ;;
+
+ energy)
+ files="$(ls "$path"/energy*_input 2>/dev/null)"
+ files="$(sensors_check_files "$files")"
+ [ -z "$files" ] && continue
+ echo "CHART 'sensors.energy_${id}_${name}' '' 'Energy' 'Joule' 'energy' 'sensors.energy' areastack $((sensors_priority + 6)) $sensors_update_every '' '' 'sensors'"
+ echo >>"$TMP_DIR/sensors.sh" "echo \"BEGIN 'sensors.energy_${id}_${name}' \$1\""
+ algorithm="incremental"
+ divisor=1000000
+ ;;
+
+ humidity)
+ files="$(ls "$path"/humidity*_input 2>/dev/null)"
+ files="$(sensors_check_files "$files")"
+ [ -z "$files" ] && continue
+ echo "CHART 'sensors.humidity_${id}_${name}' '' 'Humidity' 'Percent' 'humidity' 'sensors.humidity' line $((sensors_priority + 7)) $sensors_update_every '' '' 'sensors'"
+ echo >>"$TMP_DIR/sensors.sh" "echo \"BEGIN 'sensors.humidity_${id}_${name}' \$1\""
+ divisor=1000
+ ;;
+
+ *)
+ continue
+ ;;
+ esac
+
+ for x in $files; do
+ file="$x"
+ fid="$(fixid "$file")"
+ lfile="$(basename "$file" | sed "s|_input$|_label|g")"
+ labelname="$(basename "$file" | sed "s|_input$||g")"
+
+ if [ ! "$path/$lfile" = "$file" ] && [ -f "$path/$lfile" ]; then
+ labelname="$(cat "$path/$lfile")"
+ fi
+
+ echo "DIMENSION $fid '$labelname' $algorithm $multiplier $divisor"
+ echo >>"$TMP_DIR/sensors.sh" "echo \"SET $fid = \"\$(< $file )"
+ done
+
+ echo >>"$TMP_DIR/sensors.sh" "echo END"
+ done
+ done
+
+ [ $sensors_source_update -eq 1 ] && echo >>"$TMP_DIR/sensors.sh" "}"
+
+ # ok, load the function sensors_update() we created
+ # shellcheck source=/dev/null
+ [ $sensors_source_update -eq 1 ] && . "$TMP_DIR/sensors.sh"
+
+ return 0
+}
+
+# _update is called continuously, to collect the values
+sensors_update() {
+ # the first argument to this function is the microseconds since last update
+ # pass this parameter to the BEGIN statement (see below).
+
+ # do all the work to collect / calculate the values
+ # for each dimension
+ # remember: KEEP IT SIMPLE AND SHORT
+
+ # shellcheck source=/dev/null
+ [ $sensors_source_update -eq 0 ] && . "$TMP_DIR/sensors.sh" "$1"
+
+ return 0
+}
diff --git a/collectors/charts.d.plugin/sensors/sensors.conf b/collectors/charts.d.plugin/sensors/sensors.conf
new file mode 100644
index 0000000..bcb2880
--- /dev/null
+++ b/collectors/charts.d.plugin/sensors/sensors.conf
@@ -0,0 +1,32 @@
+# no need for shebang - this file is loaded from charts.d.plugin
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2018 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+
+# THIS PLUGIN IS DEPRECATED
+# USE THE PYTHON.D ONE
+
+# the directory the kernel keeps sensor data
+#sensors_sys_dir="/sys/devices"
+
+# how deep in the tree to check for sensor data
+#sensors_sys_depth=10
+
+# if set to 1, the script will overwrite internal
+# script functions with code generated ones
+# leave to 1, is faster
+#sensors_source_update=1
+
+# the data collection frequency
+# if unset, will inherit the netdata update frequency
+#sensors_update_every=
+
+# the charts priority on the dashboard
+#sensors_priority=90000
+
+# the number of retries to do in case of failure
+# before disabling the module
+#sensors_retries=10
+
diff --git a/collectors/cups.plugin/Makefile.am b/collectors/cups.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/cups.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/cups.plugin/README.md b/collectors/cups.plugin/README.md
new file mode 100644
index 0000000..f3b2a28
--- /dev/null
+++ b/collectors/cups.plugin/README.md
@@ -0,0 +1,64 @@
+<!--
+title: "cups.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/cups.plugin/README.md
+-->
+
+# cups.plugin
+
+`cups.plugin` collects Common Unix Printing System (CUPS) metrics.
+
+## Prerequisites
+
+This plugin needs a running local CUPS daemon (`cupsd`). This plugin does not need any configuration. Supports cups since version 1.7.
+
+If you installed Netdata using our native packages, you will have to additionaly install `netdata-plugin-cups` to use this plugin for data collection. It is not installed by default due to the large number of dependencies it requires.
+
+## Charts
+
+`cups.plugin` provides one common section `destinations` and one section per destination.
+
+> Destinations in CUPS represent individual printers or classes (collections or pools) of printers (<https://www.cups.org/doc/cupspm.html#working-with-destinations>)
+
+The section `server` provides these charts:
+
+1. **destinations by state**
+
+ - idle
+ - printing
+ - stopped
+
+2. **destinations by options**
+
+ - total
+ - accepting jobs
+ - shared
+
+3. **total job number by status**
+
+ - pending
+ - processing
+ - held
+
+4. **total job size by status**
+
+ - pending
+ - processing
+ - held
+
+For each destination the plugin provides these charts:
+
+1. **job number by status**
+
+ - pending
+ - held
+ - processing
+
+2. **job size by status**
+
+ - pending
+ - held
+ - processing
+
+At the moment only job status pending, processing, and held are reported because we do not have a method to collect stopped, canceled, aborted and completed jobs which scales.
+
+
diff --git a/collectors/cups.plugin/cups_plugin.c b/collectors/cups.plugin/cups_plugin.c
new file mode 100644
index 0000000..9a200c3
--- /dev/null
+++ b/collectors/cups.plugin/cups_plugin.c
@@ -0,0 +1,439 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+/*
+ * netdata cups.plugin
+ * (C) Copyright 2017-2018 Simon Nagl <simon.nagl@gmx.de>
+ * Released under GPL v3+
+ */
+
+#include "libnetdata/libnetdata.h"
+#include "libnetdata/required_dummies.h"
+
+#include <cups/cups.h>
+#include <limits.h>
+
+// Variables
+
+static int debug = 0;
+
+static int netdata_update_every = 1;
+static int netdata_priority = 100004;
+
+http_t *http; // connection to the cups daemon
+
+/*
+ * Used to aggregate job metrics for a destination (and all destinations).
+ */
+struct job_metrics {
+ int is_collected; // flag if this was collected in the current cycle
+
+ int num_pending;
+ int num_processing;
+ int num_held;
+
+ int size_pending; // in kilobyte
+ int size_processing; // in kilobyte
+ int size_held; // in kilobyte
+};
+DICTIONARY *dict_dest_job_metrics = NULL;
+struct job_metrics global_job_metrics;
+
+int num_dest_total;
+int num_dest_accepting_jobs;
+int num_dest_shared;
+
+int num_dest_idle;
+int num_dest_printing;
+int num_dest_stopped;
+
+void print_help() {
+ fprintf(stderr,
+ "\n"
+ "netdata cups.plugin %s\n"
+ "\n"
+ "Copyright (C) 2017-2018 Simon Nagl <simon.nagl@gmx.de>\n"
+ "Released under GNU General Public License v3+.\n"
+ "All rights reserved.\n"
+ "\n"
+ "This program is a data collector plugin for netdata.\n"
+ "\n"
+ "SYNOPSIS: cups.plugin [-d][-h][-v] COLLECTION_FREQUENCY\n"
+ "\n"
+ "Options:"
+ "\n"
+ " COLLECTION_FREQUENCY data collection frequency in seconds\n"
+ "\n"
+ " -d enable verbose output\n"
+ " default: disabled\n"
+ "\n"
+ " -v print version and exit\n"
+ "\n"
+ " -h print this message and exit\n"
+ "\n",
+ VERSION);
+}
+
+void parse_command_line(int argc, char **argv) {
+ int i;
+ int freq = 0;
+ int update_every_found = 0;
+ for (i = 1; i < argc; i++) {
+ if (isdigit(*argv[i]) && !update_every_found) {
+ int n = str2i(argv[i]);
+ if (n > 0 && n < 86400) {
+ freq = n;
+ continue;
+ }
+ } else if (strcmp("-v", argv[i]) == 0) {
+ printf("cups.plugin %s\n", VERSION);
+ exit(0);
+ } else if (strcmp("-d", argv[i]) == 0) {
+ debug = 1;
+ continue;
+ } else if (strcmp("-h", argv[i]) == 0) {
+ print_help();
+ exit(0);
+ }
+
+ print_help();
+ exit(1);
+ }
+
+ if (freq >= netdata_update_every) {
+ netdata_update_every = freq;
+ } else if (freq) {
+ error("update frequency %d seconds is too small for CUPS. Using %d.", freq, netdata_update_every);
+ }
+}
+
+/*
+ * 'cupsGetIntegerOption()' - Get an integer option value.
+ *
+ * INT_MIN is returned when the option does not exist, is not an integer, or
+ * exceeds the range of values for the "int" type.
+ *
+ * @since CUPS 2.2.4/macOS 10.13@
+ */
+
+int /* O - Option value or @code INT_MIN@ */
+getIntegerOption(
+ const char *name, /* I - Name of option */
+ int num_options, /* I - Number of options */
+ cups_option_t *options) /* I - Options */
+{
+ const char *value = cupsGetOption(name, num_options, options);
+ /* String value of option */
+ char *ptr; /* Pointer into string value */
+ long intvalue; /* Integer value */
+
+
+ if (!value || !*value)
+ return (INT_MIN);
+
+ intvalue = strtol(value, &ptr, 10);
+ if (intvalue < INT_MIN || intvalue > INT_MAX || *ptr)
+ return (INT_MIN);
+
+ return ((int)intvalue);
+}
+
+static int reset_job_metrics(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data __maybe_unused) {
+ struct job_metrics *jm = (struct job_metrics *)entry;
+
+ jm->is_collected = 0;
+ jm->num_held = 0;
+ jm->num_pending = 0;
+ jm->num_processing = 0;
+ jm->size_held = 0;
+ jm->size_pending = 0;
+ jm->size_processing = 0;
+
+ return 0;
+}
+
+struct job_metrics *get_job_metrics(char *dest) {
+ struct job_metrics *jm = dictionary_get(dict_dest_job_metrics, dest);
+
+ if (unlikely(!jm)) {
+ struct job_metrics new_job_metrics;
+ reset_job_metrics(NULL, &new_job_metrics, NULL);
+ jm = dictionary_set(dict_dest_job_metrics, dest, &new_job_metrics, sizeof(struct job_metrics));
+
+ printf("CHART cups.job_num_%s '' 'Active jobs of %s' jobs '%s' cups.job_num stacked %i %i\n", dest, dest, dest, netdata_priority++, netdata_update_every);
+ printf("DIMENSION pending '' absolute 1 1\n");
+ printf("DIMENSION held '' absolute 1 1\n");
+ printf("DIMENSION processing '' absolute 1 1\n");
+
+ printf("CHART cups.job_size_%s '' 'Active jobs size of %s' KB '%s' cups.job_size stacked %i %i\n", dest, dest, dest, netdata_priority++, netdata_update_every);
+ printf("DIMENSION pending '' absolute 1 1\n");
+ printf("DIMENSION held '' absolute 1 1\n");
+ printf("DIMENSION processing '' absolute 1 1\n");
+ };
+ return jm;
+}
+
+int collect_job_metrics(const DICTIONARY_ITEM *item, void *entry, void *data __maybe_unused) {
+ const char *name = dictionary_acquired_item_name(item);
+
+ struct job_metrics *jm = (struct job_metrics *)entry;
+
+ if (jm->is_collected) {
+ printf(
+ "BEGIN cups.job_num_%s\n"
+ "SET pending = %d\n"
+ "SET held = %d\n"
+ "SET processing = %d\n"
+ "END\n",
+ name, jm->num_pending, jm->num_held, jm->num_processing);
+ printf(
+ "BEGIN cups.job_size_%s\n"
+ "SET pending = %d\n"
+ "SET held = %d\n"
+ "SET processing = %d\n"
+ "END\n",
+ name, jm->size_pending, jm->size_held, jm->size_processing);
+ } else {
+ printf("CHART cups.job_num_%s '' 'Active jobs of %s' jobs '%s' cups.job_num stacked 1 %i 'obsolete'\n", name, name, name, netdata_update_every);
+ printf("DIMENSION pending '' absolute 1 1\n");
+ printf("DIMENSION held '' absolute 1 1\n");
+ printf("DIMENSION processing '' absolute 1 1\n");
+
+ printf("CHART cups.job_size_%s '' 'Active jobs size of %s' KB '%s' cups.job_size stacked 1 %i 'obsolete'\n", name, name, name, netdata_update_every);
+ printf("DIMENSION pending '' absolute 1 1\n");
+ printf("DIMENSION held '' absolute 1 1\n");
+ printf("DIMENSION processing '' absolute 1 1\n");
+ dictionary_del(dict_dest_job_metrics, name);
+ }
+
+ return 0;
+}
+
+void reset_metrics() {
+ num_dest_total = 0;
+ num_dest_accepting_jobs = 0;
+ num_dest_shared = 0;
+
+ num_dest_idle = 0;
+ num_dest_printing = 0;
+ num_dest_stopped = 0;
+
+ reset_job_metrics(NULL, &global_job_metrics, NULL);
+ dictionary_walkthrough_write(dict_dest_job_metrics, reset_job_metrics, NULL);
+}
+
+int main(int argc, char **argv) {
+ clocks_init();
+
+ // ------------------------------------------------------------------------
+ // initialization of netdata plugin
+
+ program_name = "cups.plugin";
+
+ // disable syslog
+ error_log_syslog = 0;
+
+ // set errors flood protection to 100 logs per hour
+ error_log_errors_per_period = 100;
+ error_log_throttle_period = 3600;
+
+ parse_command_line(argc, argv);
+
+ errno = 0;
+
+ dict_dest_job_metrics = dictionary_create(DICT_OPTION_SINGLE_THREADED);
+
+ // ------------------------------------------------------------------------
+ // the main loop
+
+ if (debug)
+ fprintf(stderr, "starting data collection\n");
+
+ time_t started_t = now_monotonic_sec();
+ size_t iteration = 0;
+ usec_t step = netdata_update_every * USEC_PER_SEC;
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ for (iteration = 0; 1; iteration++)
+ {
+ heartbeat_next(&hb, step);
+
+ if (unlikely(netdata_exit))
+ {
+ break;
+ }
+
+ reset_metrics();
+
+ cups_dest_t *dests;
+ num_dest_total = cupsGetDests2(http, &dests);
+
+ if(unlikely(num_dest_total == 0)) {
+ // reconnect to cups to check if the server is down.
+ httpClose(http);
+ http = httpConnect2(cupsServer(), ippPort(), NULL, AF_UNSPEC, cupsEncryption(), 0, netdata_update_every * 1000, NULL);
+ if(http == NULL) {
+ error("cups daemon is not running. Exiting!");
+ exit(1);
+ }
+ }
+
+ cups_dest_t *curr_dest = dests;
+ int counter = 0;
+ while (counter < num_dest_total) {
+ if (counter != 0) {
+ curr_dest++;
+ }
+ counter++;
+
+ const char *printer_uri_supported = cupsGetOption("printer-uri-supported", curr_dest->num_options, curr_dest->options);
+ if (!printer_uri_supported) {
+ if(debug)
+ fprintf(stderr, "destination %s discovered, but not yet setup as a local printer", curr_dest->name);
+ continue;
+ }
+
+ const char *printer_is_accepting_jobs = cupsGetOption("printer-is-accepting-jobs", curr_dest->num_options, curr_dest->options);
+ if (printer_is_accepting_jobs && !strcmp(printer_is_accepting_jobs, "true")) {
+ num_dest_accepting_jobs++;
+ }
+
+ const char *printer_is_shared = cupsGetOption("printer-is-shared", curr_dest->num_options, curr_dest->options);
+ if (printer_is_shared && !strcmp(printer_is_shared, "true")) {
+ num_dest_shared++;
+ }
+
+ int printer_state = getIntegerOption("printer-state", curr_dest->num_options, curr_dest->options);
+ switch (printer_state) {
+ case 3:
+ num_dest_idle++;
+ break;
+ case 4:
+ num_dest_printing++;
+ break;
+ case 5:
+ num_dest_stopped++;
+ break;
+ case INT_MIN:
+ if(debug)
+ fprintf(stderr, "printer state is missing for destination %s", curr_dest->name);
+ break;
+ default:
+ error("Unknown printer state (%d) found.", printer_state);
+ break;
+ }
+
+ /*
+ * flag job metrics to print values.
+ * This is needed to report also destinations with zero active jobs.
+ */
+ struct job_metrics *jm = get_job_metrics(curr_dest->name);
+ jm->is_collected = 1;
+ }
+ cupsFreeDests(num_dest_total, dests);
+
+ if (unlikely(netdata_exit))
+ break;
+
+ cups_job_t *jobs, *curr_job;
+ int num_jobs = cupsGetJobs2(http, &jobs, NULL, 0, CUPS_WHICHJOBS_ACTIVE);
+ int i;
+ for (i = num_jobs, curr_job = jobs; i > 0; i--, curr_job++) {
+ struct job_metrics *jm = get_job_metrics(curr_job->dest);
+ jm->is_collected = 1;
+
+ switch (curr_job->state) {
+ case IPP_JOB_PENDING:
+ jm->num_pending++;
+ jm->size_pending += curr_job->size;
+ global_job_metrics.num_pending++;
+ global_job_metrics.size_pending += curr_job->size;
+ break;
+ case IPP_JOB_HELD:
+ jm->num_held++;
+ jm->size_held += curr_job->size;
+ global_job_metrics.num_held++;
+ global_job_metrics.size_held += curr_job->size;
+ break;
+ case IPP_JOB_PROCESSING:
+ jm->num_processing++;
+ jm->size_processing += curr_job->size;
+ global_job_metrics.num_processing++;
+ global_job_metrics.size_processing += curr_job->size;
+ break;
+ default:
+ error("Unsupported job state (%u) found.", curr_job->state);
+ break;
+ }
+ }
+ cupsFreeJobs(num_jobs, jobs);
+
+ dictionary_walkthrough_write(dict_dest_job_metrics, collect_job_metrics, NULL);
+
+ static int cups_printer_by_option_created = 0;
+ if (unlikely(!cups_printer_by_option_created))
+ {
+ cups_printer_by_option_created = 1;
+ printf("CHART cups.dest_state '' 'Destinations by state' dests overview cups.dests_state stacked 100000 %i\n", netdata_update_every);
+ printf("DIMENSION idle '' absolute 1 1\n");
+ printf("DIMENSION printing '' absolute 1 1\n");
+ printf("DIMENSION stopped '' absolute 1 1\n");
+
+ printf("CHART cups.dest_option '' 'Destinations by option' dests overview cups.dests_option line 100001 %i\n", netdata_update_every);
+ printf("DIMENSION total '' absolute 1 1\n");
+ printf("DIMENSION acceptingjobs '' absolute 1 1\n");
+ printf("DIMENSION shared '' absolute 1 1\n");
+
+ printf("CHART cups.job_num '' 'Active jobs' jobs overview cups.job_num stacked 100002 %i\n", netdata_update_every);
+ printf("DIMENSION pending '' absolute 1 1\n");
+ printf("DIMENSION held '' absolute 1 1\n");
+ printf("DIMENSION processing '' absolute 1 1\n");
+
+ printf("CHART cups.job_size '' 'Active jobs size' KB overview cups.job_size stacked 100003 %i\n", netdata_update_every);
+ printf("DIMENSION pending '' absolute 1 1\n");
+ printf("DIMENSION held '' absolute 1 1\n");
+ printf("DIMENSION processing '' absolute 1 1\n");
+ }
+
+ printf(
+ "BEGIN cups.dest_state\n"
+ "SET idle = %d\n"
+ "SET printing = %d\n"
+ "SET stopped = %d\n"
+ "END\n",
+ num_dest_idle, num_dest_printing, num_dest_stopped);
+ printf(
+ "BEGIN cups.dest_option\n"
+ "SET total = %d\n"
+ "SET acceptingjobs = %d\n"
+ "SET shared = %d\n"
+ "END\n",
+ num_dest_total, num_dest_accepting_jobs, num_dest_shared);
+ printf(
+ "BEGIN cups.job_num\n"
+ "SET pending = %d\n"
+ "SET held = %d\n"
+ "SET processing = %d\n"
+ "END\n",
+ global_job_metrics.num_pending, global_job_metrics.num_held, global_job_metrics.num_processing);
+ printf(
+ "BEGIN cups.job_size\n"
+ "SET pending = %d\n"
+ "SET held = %d\n"
+ "SET processing = %d\n"
+ "END\n",
+ global_job_metrics.size_pending, global_job_metrics.size_held, global_job_metrics.size_processing);
+
+ fflush(stdout);
+
+ if (unlikely(netdata_exit))
+ break;
+
+ // restart check (14400 seconds)
+ if (!now_monotonic_sec() - started_t > 14400)
+ break;
+ }
+
+ httpClose(http);
+ info("CUPS process exiting");
+}
diff --git a/collectors/diskspace.plugin/Makefile.am b/collectors/diskspace.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/diskspace.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/diskspace.plugin/README.md b/collectors/diskspace.plugin/README.md
new file mode 100644
index 0000000..c037a0b
--- /dev/null
+++ b/collectors/diskspace.plugin/README.md
@@ -0,0 +1,43 @@
+<!--
+title: "diskspace.plugin"
+description: "Monitor the disk usage space of mounted disks in real-time with the Netdata Agent, plus preconfigured alarms for disks at risk of filling up."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/diskspace.plugin/README.md
+-->
+
+# diskspace.plugin
+
+This plugin monitors the disk space usage of mounted disks, under Linux. The plugin requires Netdata to have execute/search permissions on the mount point itself, as well as each component of the absolute path to the mount point.
+
+Two charts are available for every mount:
+
+- Disk Space Usage
+- Disk Files (inodes) Usage
+
+## configuration
+
+Simple patterns can be used to exclude mounts from showed statistics based on path or filesystem. By default read-only mounts are not displayed. To display them `yes` should be set for a chart instead of `auto`.
+
+By default, Netdata will enable monitoring metrics only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). Set `yes` for a chart instead of `auto` to enable it permanently. You can also set the `enable zero metrics` option to `yes` in the `[global]` section which enables charts with zero metrics for all internal Netdata plugins.
+
+```
+[plugin:proc:diskspace]
+ # remove charts of unmounted disks = yes
+ # update every = 1
+ # check for new mount points every = 15
+ # exclude space metrics on paths = /proc/* /sys/* /var/run/user/* /run/user/* /snap/* /var/lib/docker/*
+ # exclude space metrics on filesystems = *gvfs *gluster* *s3fs *ipfs *davfs2 *httpfs *sshfs *gdfs *moosefs fusectl autofs
+ # space usage for all disks = auto
+ # inodes usage for all disks = auto
+```
+
+Charts can be enabled/disabled for every mount separately:
+
+```
+[plugin:proc:diskspace:/]
+ # space usage = auto
+ # inodes usage = auto
+```
+
+> for disks performance monitoring, see the `proc` plugin, [here](/collectors/proc.plugin/README.md#monitoring-disks)
+
+
diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c
new file mode 100644
index 0000000..e806a33
--- /dev/null
+++ b/collectors/diskspace.plugin/plugin_diskspace.c
@@ -0,0 +1,690 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../proc.plugin/plugin_proc.h"
+
+#define PLUGIN_DISKSPACE_NAME "diskspace.plugin"
+#define THREAD_DISKSPACE_SLOW_NAME "PLUGIN[diskspace slow]"
+
+#define DEFAULT_EXCLUDED_PATHS "/proc/* /sys/* /var/run/user/* /run/user/* /snap/* /var/lib/docker/*"
+#define DEFAULT_EXCLUDED_FILESYSTEMS "*gvfs *gluster* *s3fs *ipfs *davfs2 *httpfs *sshfs *gdfs *moosefs fusectl autofs"
+#define CONFIG_SECTION_DISKSPACE "plugin:proc:diskspace"
+
+#define MAX_STAT_USEC 10000LU
+#define SLOW_UPDATE_EVERY 5
+
+static netdata_thread_t *diskspace_slow_thread = NULL;
+
+static struct mountinfo *disk_mountinfo_root = NULL;
+static int check_for_new_mountpoints_every = 15;
+static int cleanup_mount_points = 1;
+
+static inline void mountinfo_reload(int force) {
+ static time_t last_loaded = 0;
+ time_t now = now_realtime_sec();
+
+ if(force || now - last_loaded >= check_for_new_mountpoints_every) {
+ // mountinfo_free_all() can be called with NULL disk_mountinfo_root
+ mountinfo_free_all(disk_mountinfo_root);
+
+ // re-read mountinfo in case something changed
+ disk_mountinfo_root = mountinfo_read(0);
+
+ last_loaded = now;
+ }
+}
+
+// Data to be stored in DICTIONARY dict_mountpoints used by do_disk_space_stats().
+// This DICTIONARY is used to lookup the settings of the mount point on each iteration.
+struct mount_point_metadata {
+ int do_space;
+ int do_inodes;
+ int shown_error;
+ int updated;
+ int slow;
+
+ DICTIONARY *chart_labels;
+
+ size_t collected; // the number of times this has been collected
+
+ RRDSET *st_space;
+ RRDDIM *rd_space_used;
+ RRDDIM *rd_space_avail;
+ RRDDIM *rd_space_reserved;
+
+ RRDSET *st_inodes;
+ RRDDIM *rd_inodes_used;
+ RRDDIM *rd_inodes_avail;
+ RRDDIM *rd_inodes_reserved;
+};
+
+static DICTIONARY *dict_mountpoints = NULL;
+
+#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); (st) = NULL; } } while(st)
+
+int mount_point_cleanup(const char *name, void *entry, int slow) {
+ (void)name;
+
+ struct mount_point_metadata *mp = (struct mount_point_metadata *)entry;
+ if(!mp) return 0;
+
+ if (slow != mp->slow)
+ return 0;
+
+ if(likely(mp->updated)) {
+ mp->updated = 0;
+ return 0;
+ }
+
+ if(likely(cleanup_mount_points && mp->collected)) {
+ mp->collected = 0;
+ mp->updated = 0;
+ mp->shown_error = 0;
+
+ mp->rd_space_avail = NULL;
+ mp->rd_space_used = NULL;
+ mp->rd_space_reserved = NULL;
+
+ mp->rd_inodes_avail = NULL;
+ mp->rd_inodes_used = NULL;
+ mp->rd_inodes_reserved = NULL;
+
+ rrdset_obsolete_and_pointer_null(mp->st_space);
+ rrdset_obsolete_and_pointer_null(mp->st_inodes);
+ }
+
+ return 0;
+}
+
+int mount_point_cleanup_cb(const DICTIONARY_ITEM *item, void *entry, void *data __maybe_unused) {
+ const char *name = dictionary_acquired_item_name(item);
+
+ return mount_point_cleanup(name, (struct mount_point_metadata *)entry, 0);
+}
+
+// a copy of basic mountinfo fields
+struct basic_mountinfo {
+ char *persistent_id;
+ char *root;
+ char *mount_point;
+ char *filesystem;
+
+ struct basic_mountinfo *next;
+};
+
+static struct basic_mountinfo *slow_mountinfo_tmp_root = NULL;
+static netdata_mutex_t slow_mountinfo_mutex;
+
+static struct basic_mountinfo *basic_mountinfo_create_and_copy(struct mountinfo* mi)
+{
+ struct basic_mountinfo *bmi = callocz(1, sizeof(struct basic_mountinfo));
+
+ if (mi) {
+ bmi->persistent_id = strdupz(mi->persistent_id);
+ bmi->root = strdupz(mi->root);
+ bmi->mount_point = strdupz(mi->mount_point);
+ bmi->filesystem = strdupz(mi->filesystem);
+ }
+
+ return bmi;
+}
+
+static void add_basic_mountinfo(struct basic_mountinfo **root, struct mountinfo *mi)
+{
+ if (!root)
+ return;
+
+ struct basic_mountinfo *bmi = basic_mountinfo_create_and_copy(mi);
+
+ bmi->next = *root;
+ *root = bmi;
+};
+
+static void free_basic_mountinfo(struct basic_mountinfo *bmi)
+{
+ if (bmi) {
+ freez(bmi->persistent_id);
+ freez(bmi->root);
+ freez(bmi->mount_point);
+ freez(bmi->filesystem);
+
+ freez(bmi);
+ }
+};
+
+static void free_basic_mountinfo_list(struct basic_mountinfo *root)
+{
+ struct basic_mountinfo *bmi = root, *next;
+
+ while (bmi) {
+ next = bmi->next;
+ free_basic_mountinfo(bmi);
+ bmi = next;
+ }
+}
+
+static void calculate_values_and_show_charts(
+ struct basic_mountinfo *mi,
+ struct mount_point_metadata *m,
+ struct statvfs *buff_statvfs,
+ int update_every)
+{
+ const char *family = mi->mount_point;
+ const char *disk = mi->persistent_id;
+
+ // logic found at get_fs_usage() in coreutils
+ unsigned long bsize = (buff_statvfs->f_frsize) ? buff_statvfs->f_frsize : buff_statvfs->f_bsize;
+
+ fsblkcnt_t bavail = buff_statvfs->f_bavail;
+ fsblkcnt_t btotal = buff_statvfs->f_blocks;
+ fsblkcnt_t bavail_root = buff_statvfs->f_bfree;
+ fsblkcnt_t breserved_root = bavail_root - bavail;
+ fsblkcnt_t bused = likely(btotal >= bavail_root) ? btotal - bavail_root : bavail_root - btotal;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(btotal != bavail + breserved_root + bused))
+ error("DISKSPACE: disk block statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)btotal, (unsigned long long)bavail, (unsigned long long)breserved_root, (unsigned long long)bused);
+#endif
+
+ // --------------------------------------------------------------------------
+
+ fsfilcnt_t favail = buff_statvfs->f_favail;
+ fsfilcnt_t ftotal = buff_statvfs->f_files;
+ fsfilcnt_t favail_root = buff_statvfs->f_ffree;
+ fsfilcnt_t freserved_root = favail_root - favail;
+ fsfilcnt_t fused = ftotal - favail_root;
+
+ if(m->do_inodes == CONFIG_BOOLEAN_AUTO && favail == (fsfilcnt_t)-1) {
+ // this file system does not support inodes reporting
+ // eg. cephfs
+ m->do_inodes = CONFIG_BOOLEAN_NO;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(btotal != bavail + breserved_root + bused))
+ error("DISKSPACE: disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused);
+#endif
+
+ int rendered = 0;
+
+ if(m->do_space == CONFIG_BOOLEAN_YES || (m->do_space == CONFIG_BOOLEAN_AUTO &&
+ (bavail || breserved_root || bused ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if(unlikely(!m->st_space) || m->st_space->update_every != update_every) {
+ m->do_space = CONFIG_BOOLEAN_YES;
+ m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
+ if(unlikely(!m->st_space || m->st_space->update_every != update_every)) {
+ char title[4096 + 1];
+ snprintfz(title, 4096, "Disk Space Usage");
+ m->st_space = rrdset_create_localhost(
+ "disk_space"
+ , disk
+ , NULL
+ , family
+ , "disk.space"
+ , title
+ , "GiB"
+ , PLUGIN_DISKSPACE_NAME
+ , NULL
+ , NETDATA_CHART_PRIO_DISKSPACE_SPACE
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ rrdset_update_rrdlabels(m->st_space, m->chart_labels);
+
+ m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL, (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_space_used = rrddim_add(m->st_space, "used", NULL, (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root", (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number)bavail);
+ rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number)bused);
+ rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number)breserved_root);
+ rrdset_done(m->st_space);
+
+ rendered++;
+ }
+
+ if(m->do_inodes == CONFIG_BOOLEAN_YES || (m->do_inodes == CONFIG_BOOLEAN_AUTO &&
+ (favail || freserved_root || fused ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if(unlikely(!m->st_inodes) || m->st_inodes->update_every != update_every) {
+ m->do_inodes = CONFIG_BOOLEAN_YES;
+ m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
+ if(unlikely(!m->st_inodes) || m->st_inodes->update_every != update_every) {
+ char title[4096 + 1];
+ snprintfz(title, 4096, "Disk Files (inodes) Usage");
+ m->st_inodes = rrdset_create_localhost(
+ "disk_inodes"
+ , disk
+ , NULL
+ , family
+ , "disk.inodes"
+ , title
+ , "inodes"
+ , PLUGIN_DISKSPACE_NAME
+ , NULL
+ , NETDATA_CHART_PRIO_DISKSPACE_INODES
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ rrdset_update_rrdlabels(m->st_inodes, m->chart_labels);
+
+ m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_inodes_reserved = rrddim_add(m->st_inodes, "reserved_for_root", "reserved for root", 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number)favail);
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number)fused);
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_reserved, (collected_number)freserved_root);
+ rrdset_done(m->st_inodes);
+
+ rendered++;
+ }
+
+ if(likely(rendered))
+ m->collected++;
+}
+
+static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
+ const char *disk = mi->persistent_id;
+
+ static SIMPLE_PATTERN *excluded_mountpoints = NULL;
+ static SIMPLE_PATTERN *excluded_filesystems = NULL;
+
+ usec_t slow_timeout = MAX_STAT_USEC * update_every;
+
+ int do_space, do_inodes;
+
+ if(unlikely(!dict_mountpoints)) {
+ SIMPLE_PREFIX_MODE mode = SIMPLE_PATTERN_EXACT;
+
+ if(config_move("plugin:proc:/proc/diskstats", "exclude space metrics on paths", CONFIG_SECTION_DISKSPACE, "exclude space metrics on paths") != -1) {
+ // old configuration, enable backwards compatibility
+ mode = SIMPLE_PATTERN_PREFIX;
+ }
+
+ excluded_mountpoints = simple_pattern_create(
+ config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on paths", DEFAULT_EXCLUDED_PATHS)
+ , NULL
+ , mode
+ );
+
+ excluded_filesystems = simple_pattern_create(
+ config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on filesystems", DEFAULT_EXCLUDED_FILESYSTEMS)
+ , NULL
+ , SIMPLE_PATTERN_EXACT
+ );
+
+ dict_mountpoints = dictionary_create(DICT_OPTION_NONE);
+ }
+
+ struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point);
+ if(unlikely(!m)) {
+ int slow = 0;
+ char var_name[4096 + 1];
+ snprintfz(var_name, 4096, "plugin:proc:diskspace:%s", mi->mount_point);
+
+ int def_space = config_get_boolean_ondemand(CONFIG_SECTION_DISKSPACE, "space usage for all disks", CONFIG_BOOLEAN_AUTO);
+ int def_inodes = config_get_boolean_ondemand(CONFIG_SECTION_DISKSPACE, "inodes usage for all disks", CONFIG_BOOLEAN_AUTO);
+
+ if(unlikely(simple_pattern_matches(excluded_mountpoints, mi->mount_point))) {
+ def_space = CONFIG_BOOLEAN_NO;
+ def_inodes = CONFIG_BOOLEAN_NO;
+ }
+
+ if(unlikely(simple_pattern_matches(excluded_filesystems, mi->filesystem))) {
+ def_space = CONFIG_BOOLEAN_NO;
+ def_inodes = CONFIG_BOOLEAN_NO;
+ }
+
+ // check if the mount point is a directory #2407
+ // but only when it is enabled by default #4491
+ if(def_space != CONFIG_BOOLEAN_NO || def_inodes != CONFIG_BOOLEAN_NO) {
+ usec_t start_time = now_monotonic_high_precision_usec();
+ struct stat bs;
+
+ if(stat(mi->mount_point, &bs) == -1) {
+ error("DISKSPACE: Cannot stat() mount point '%s' (disk '%s', filesystem '%s', root '%s')."
+ , mi->mount_point
+ , disk
+ , mi->filesystem?mi->filesystem:""
+ , mi->root?mi->root:""
+ );
+ def_space = CONFIG_BOOLEAN_NO;
+ def_inodes = CONFIG_BOOLEAN_NO;
+ }
+ else {
+ if((bs.st_mode & S_IFMT) != S_IFDIR) {
+ error("DISKSPACE: Mount point '%s' (disk '%s', filesystem '%s', root '%s') is not a directory."
+ , mi->mount_point
+ , disk
+ , mi->filesystem?mi->filesystem:""
+ , mi->root?mi->root:""
+ );
+ def_space = CONFIG_BOOLEAN_NO;
+ def_inodes = CONFIG_BOOLEAN_NO;
+ }
+ }
+
+ if ((now_monotonic_high_precision_usec() - start_time) > slow_timeout)
+ slow = 1;
+ }
+
+ do_space = config_get_boolean_ondemand(var_name, "space usage", def_space);
+ do_inodes = config_get_boolean_ondemand(var_name, "inodes usage", def_inodes);
+
+ struct mount_point_metadata mp = {
+ .do_space = do_space,
+ .do_inodes = do_inodes,
+ .shown_error = 0,
+ .updated = 0,
+ .slow = 0,
+
+ .collected = 0,
+
+ .st_space = NULL,
+ .rd_space_avail = NULL,
+ .rd_space_used = NULL,
+ .rd_space_reserved = NULL,
+
+ .st_inodes = NULL,
+ .rd_inodes_avail = NULL,
+ .rd_inodes_used = NULL,
+ .rd_inodes_reserved = NULL
+ };
+
+ mp.chart_labels = rrdlabels_create();
+ rrdlabels_add(mp.chart_labels, "mount_point", mi->mount_point, RRDLABEL_SRC_AUTO);
+ rrdlabels_add(mp.chart_labels, "filesystem", mi->filesystem, RRDLABEL_SRC_AUTO);
+ rrdlabels_add(mp.chart_labels, "mount_root", mi->root, RRDLABEL_SRC_AUTO);
+
+ m = dictionary_set(dict_mountpoints, mi->mount_point, &mp, sizeof(struct mount_point_metadata));
+
+ m->slow = slow;
+ }
+
+ if (m->slow) {
+ add_basic_mountinfo(&slow_mountinfo_tmp_root, mi);
+ return;
+ }
+
+ m->updated = 1;
+
+ if(unlikely(m->do_space == CONFIG_BOOLEAN_NO && m->do_inodes == CONFIG_BOOLEAN_NO))
+ return;
+
+ if (unlikely(
+ mi->flags & MOUNTINFO_READONLY &&
+ !(mi->flags & MOUNTINFO_IS_IN_SYSD_PROTECTED_LIST) &&
+ !m->collected &&
+ m->do_space != CONFIG_BOOLEAN_YES &&
+ m->do_inodes != CONFIG_BOOLEAN_YES))
+ return;
+
+ usec_t start_time = now_monotonic_high_precision_usec();
+ struct statvfs buff_statvfs;
+
+ if (statvfs(mi->mount_point, &buff_statvfs) < 0) {
+ if(!m->shown_error) {
+ error("DISKSPACE: failed to statvfs() mount point '%s' (disk '%s', filesystem '%s', root '%s')"
+ , mi->mount_point
+ , disk
+ , mi->filesystem?mi->filesystem:""
+ , mi->root?mi->root:""
+ );
+ m->shown_error = 1;
+ }
+ return;
+ }
+
+ if ((now_monotonic_high_precision_usec() - start_time) > slow_timeout)
+ m->slow = 1;
+
+ m->shown_error = 0;
+
+ struct basic_mountinfo bmi;
+ bmi.mount_point = mi->mount_point;
+ bmi.persistent_id = mi->persistent_id;
+ bmi.filesystem = mi->filesystem;
+ bmi.root = mi->root;
+
+ calculate_values_and_show_charts(&bmi, m, &buff_statvfs, update_every);
+}
+
+static inline void do_slow_disk_space_stats(struct basic_mountinfo *mi, int update_every) {
+ struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point);
+
+ m->updated = 1;
+
+ struct statvfs buff_statvfs;
+ if (statvfs(mi->mount_point, &buff_statvfs) < 0) {
+ if(!m->shown_error) {
+ error("DISKSPACE: failed to statvfs() mount point '%s' (disk '%s', filesystem '%s', root '%s')"
+ , mi->mount_point
+ , mi->persistent_id
+ , mi->filesystem?mi->filesystem:""
+ , mi->root?mi->root:""
+ );
+ m->shown_error = 1;
+ }
+ return;
+ }
+ m->shown_error = 0;
+
+ calculate_values_and_show_charts(mi, m, &buff_statvfs, update_every);
+}
+
+static void diskspace_slow_worker_cleanup(void *ptr)
+{
+ UNUSED(ptr);
+
+ info("cleaning up...");
+
+ worker_unregister();
+}
+
+#define WORKER_JOB_SLOW_MOUNTPOINT 0
+#define WORKER_JOB_SLOW_CLEANUP 1
+
+struct slow_worker_data {
+ netdata_thread_t *slow_thread;
+ int update_every;
+};
+
+void *diskspace_slow_worker(void *ptr)
+{
+ struct slow_worker_data *data = (struct slow_worker_data *)ptr;
+
+ worker_register("DISKSPACE_SLOW");
+ worker_register_job_name(WORKER_JOB_SLOW_MOUNTPOINT, "mountpoint");
+ worker_register_job_name(WORKER_JOB_SLOW_CLEANUP, "cleanup");
+
+ struct basic_mountinfo *slow_mountinfo_root = NULL;
+
+ int slow_update_every = data->update_every > SLOW_UPDATE_EVERY ? data->update_every : SLOW_UPDATE_EVERY;
+
+ netdata_thread_cleanup_push(diskspace_slow_worker_cleanup, data->slow_thread);
+
+ usec_t step = slow_update_every * USEC_PER_SEC;
+ usec_t real_step = USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ while(!netdata_exit) {
+ worker_is_idle();
+ heartbeat_next(&hb, USEC_PER_SEC);
+
+ if (real_step < step) {
+ real_step += USEC_PER_SEC;
+ continue;
+ }
+ real_step = USEC_PER_SEC;
+
+ usec_t start_time = now_monotonic_high_precision_usec();
+
+ if (!dict_mountpoints)
+ continue;
+
+ if(unlikely(netdata_exit)) break;
+
+ // --------------------------------------------------------------------------
+ // disk space metrics
+
+ worker_is_busy(WORKER_JOB_SLOW_MOUNTPOINT);
+
+ netdata_mutex_lock(&slow_mountinfo_mutex);
+ free_basic_mountinfo_list(slow_mountinfo_root);
+ slow_mountinfo_root = slow_mountinfo_tmp_root;
+ slow_mountinfo_tmp_root = NULL;
+ netdata_mutex_unlock(&slow_mountinfo_mutex);
+
+ struct basic_mountinfo *bmi;
+ for(bmi = slow_mountinfo_root; bmi; bmi = bmi->next) {
+ do_slow_disk_space_stats(bmi, slow_update_every);
+
+ if(unlikely(netdata_exit)) break;
+ }
+
+ if(unlikely(netdata_exit)) break;
+
+ worker_is_busy(WORKER_JOB_SLOW_CLEANUP);
+
+ for(bmi = slow_mountinfo_root; bmi; bmi = bmi->next) {
+ struct mount_point_metadata *m = dictionary_get(dict_mountpoints, bmi->mount_point);
+
+ if (m)
+ mount_point_cleanup(bmi->mount_point, m, 1);
+ }
+
+ usec_t dt = now_monotonic_high_precision_usec() - start_time;
+ if (dt > step) {
+ slow_update_every = (dt / USEC_PER_SEC) * 3 / 2;
+ if (slow_update_every % SLOW_UPDATE_EVERY)
+ slow_update_every += SLOW_UPDATE_EVERY - slow_update_every % SLOW_UPDATE_EVERY;
+ step = slow_update_every * USEC_PER_SEC;
+ }
+ }
+
+ netdata_thread_cleanup_pop(1);
+
+ free_basic_mountinfo_list(slow_mountinfo_root);
+
+ return NULL;
+}
+
+static void diskspace_main_cleanup(void *ptr) {
+ rrd_collector_finished();
+ worker_unregister();
+
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ info("cleaning up...");
+
+ if (diskspace_slow_thread) {
+ netdata_thread_join(*diskspace_slow_thread, NULL);
+ freez(diskspace_slow_thread);
+ }
+
+ free_basic_mountinfo_list(slow_mountinfo_tmp_root);
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+#define WORKER_JOB_MOUNTINFO 0
+#define WORKER_JOB_MOUNTPOINT 1
+#define WORKER_JOB_CLEANUP 2
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
+#endif
+
+void *diskspace_main(void *ptr) {
+ worker_register("DISKSPACE");
+ worker_register_job_name(WORKER_JOB_MOUNTINFO, "mountinfo");
+ worker_register_job_name(WORKER_JOB_MOUNTPOINT, "mountpoint");
+ worker_register_job_name(WORKER_JOB_CLEANUP, "cleanup");
+
+ rrd_collector_started();
+
+ netdata_thread_cleanup_push(diskspace_main_cleanup, ptr);
+
+ cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points);
+
+ int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", localhost->rrd_update_every);
+ if(update_every < localhost->rrd_update_every)
+ update_every = localhost->rrd_update_every;
+
+ check_for_new_mountpoints_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "check for new mount points every", check_for_new_mountpoints_every);
+ if(check_for_new_mountpoints_every < update_every)
+ check_for_new_mountpoints_every = update_every;
+
+ netdata_mutex_init(&slow_mountinfo_mutex);
+
+ diskspace_slow_thread = mallocz(sizeof(netdata_thread_t));
+
+ struct slow_worker_data slow_worker_data = {.slow_thread = diskspace_slow_thread, .update_every = update_every};
+
+ netdata_thread_create(
+ diskspace_slow_thread,
+ THREAD_DISKSPACE_SLOW_NAME,
+ NETDATA_THREAD_OPTION_JOINABLE,
+ diskspace_slow_worker,
+ &slow_worker_data);
+
+ usec_t step = update_every * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ while(!netdata_exit) {
+ worker_is_idle();
+ /* usec_t hb_dt = */ heartbeat_next(&hb, step);
+
+ if(unlikely(netdata_exit)) break;
+
+ // --------------------------------------------------------------------------
+ // this is smart enough not to reload it every time
+
+ worker_is_busy(WORKER_JOB_MOUNTINFO);
+ mountinfo_reload(0);
+
+ // --------------------------------------------------------------------------
+ // disk space metrics
+
+ netdata_mutex_lock(&slow_mountinfo_mutex);
+ free_basic_mountinfo_list(slow_mountinfo_tmp_root);
+ slow_mountinfo_tmp_root = NULL;
+
+ struct mountinfo *mi;
+ for(mi = disk_mountinfo_root; mi; mi = mi->next) {
+ if(unlikely(mi->flags & (MOUNTINFO_IS_DUMMY | MOUNTINFO_IS_BIND)))
+ continue;
+
+ // exclude mounts made by ProtectHome and ProtectSystem systemd hardening options
+ // https://github.com/netdata/netdata/issues/11498#issuecomment-950982878
+ if(mi->flags & MOUNTINFO_READONLY && mi->flags & MOUNTINFO_IS_IN_SYSD_PROTECTED_LIST && !strcmp(mi->root, mi->mount_point))
+ continue;
+
+ worker_is_busy(WORKER_JOB_MOUNTPOINT);
+ do_disk_space_stats(mi, update_every);
+ if(unlikely(netdata_exit)) break;
+ }
+ netdata_mutex_unlock(&slow_mountinfo_mutex);
+
+ if(unlikely(netdata_exit)) break;
+
+ if(dict_mountpoints) {
+ worker_is_busy(WORKER_JOB_CLEANUP);
+ dictionary_walkthrough_read(dict_mountpoints, mount_point_cleanup_cb, NULL);
+ }
+
+ }
+ worker_unregister();
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/Makefile.am b/collectors/ebpf.plugin/Makefile.am
new file mode 100644
index 0000000..2d5f92a
--- /dev/null
+++ b/collectors/ebpf.plugin/Makefile.am
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+userebpfconfigdir=$(configdir)/ebpf.d
+
+# Explicitly install directories to avoid permission issues due to umask
+install-exec-local:
+ $(INSTALL) -d $(DESTDIR)$(userebpfconfigdir)
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
+
+ebpfconfigdir=$(libconfigdir)/ebpf.d
+dist_libconfig_DATA = \
+ ebpf.d.conf \
+ $(NULL)
+
+dist_ebpfconfig_DATA = \
+ ebpf.d/ebpf_kernel_reject_list.txt \
+ ebpf.d/cachestat.conf \
+ ebpf.d/dcstat.conf \
+ ebpf.d/disk.conf \
+ ebpf.d/fd.conf \
+ ebpf.d/filesystem.conf \
+ ebpf.d/hardirq.conf \
+ ebpf.d/mdflush.conf \
+ ebpf.d/mount.conf \
+ ebpf.d/network.conf \
+ ebpf.d/oomkill.conf \
+ ebpf.d/process.conf \
+ ebpf.d/shm.conf \
+ ebpf.d/softirq.conf \
+ ebpf.d/sync.conf \
+ ebpf.d/swap.conf \
+ ebpf.d/vfs.conf \
+ $(NULL)
diff --git a/collectors/ebpf.plugin/README.md b/collectors/ebpf.plugin/README.md
new file mode 100644
index 0000000..7762ed3
--- /dev/null
+++ b/collectors/ebpf.plugin/README.md
@@ -0,0 +1,970 @@
+<!--
+title: "eBPF monitoring with Netdata"
+description: "Use Netdata's extended Berkeley Packet Filter (eBPF) collector to monitor kernel-level metrics about your
+complex applications with per-second granularity."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/ebpf.plugin/README.md
+sidebar_label: "eBPF"
+-->
+
+# eBPF monitoring with Netdata
+
+The Netdata Agent provides many [eBPF](https://ebpf.io/what-is-ebpf/) programs to help you troubleshoot and debug how applications interact with the Linux kernel. The `ebpf.plugin` uses [tracepoints, trampoline, and2 kprobes](#how-netdata-collects-data-using-probes-and-tracepoints) to collect a wide array of high value data about the host that would otherwise be impossible to capture.
+
+> ❗ eBPF monitoring only works on Linux systems and with specific Linux kernels, including all kernels newer than `4.11.0`, and all kernels on CentOS 7.6 or later. For kernels older than `4.11.0`, improved support is in active development.
+
+This document provides comprehensive details about the `ebpf.plugin`.
+For hands-on configuration and troubleshooting tips see our [tutorial on troubleshooting apps with eBPF metrics](/docs/guides/troubleshoot/monitor-debug-applications-ebpf.md).
+
+<figure>
+ <img src="https://user-images.githubusercontent.com/1153921/74746434-ad6a1e00-5222-11ea-858a-a7882617ae02.png" alt="An example of VFS charts, made possible by the eBPF collector plugin" />
+ <figcaption>An example of virtual file system (VFS) charts made possible by the eBPF collector plugin.</figcaption>
+</figure>
+
+## How Netdata collects data using probes and tracepoints
+
+Netdata uses the following features from the Linux kernel to run eBPF programs:
+
+- Tracepoints are hooks to call specific functions. Tracepoints are more stable than `kprobes` and are preferred when
+ both options are available.
+- Trampolines are bridges between kernel functions, and BPF programs. Netdata uses them by default whenever available.
+- Kprobes and return probes (`kretprobe`): Probes can insert virtually into any kernel instruction. When eBPF runs in `entry` mode, it attaches only `kprobes` for internal functions monitoring calls and some arguments every time a function is called. The user can also change configuration to use [`return`](#global-configuration-options) mode, and this will allow users to monitor return from these functions and detect possible failures.
+
+In each case, wherever a normal kprobe, kretprobe, or tracepoint would have run its hook function, an eBPF program is run instead, performing various collection logic before letting the kernel continue its normal control flow.
+
+There are more methods to trigger eBPF programs, such as uprobes, but currently are not supported.
+
+## Configuring ebpf.plugin
+
+The eBPF collector is installed and enabled by default on most new installations of the Agent.
+If your Agent is v1.22 or older, you may to enable the collector yourself.
+
+### Enable the eBPF collector
+
+To enable or disable the entire eBPF collector:
+
+1. Navigate to the [Netdata config directory](/docs/configure/nodes.md#the-netdata-config-directory).
+ ```bash
+ cd /etc/netdata
+ ```
+
+2. Use the [`edit-config`](/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) script to edit `netdata.conf`.
+
+ ```bash
+ ./edit-config netdata.conf
+ ```
+
+3. Enable the collector by scrolling down to the `[plugins]` section. Uncomment the line `ebpf` (not
+ `ebpf_process`) and set it to `yes`.
+
+ ```conf
+ [plugins]
+ ebpf = yes
+ ```
+
+### Configure the eBPF collector
+
+You can configure the eBPF collector's behavior to fine-tune which metrics you receive and [optimize performance]\(#performance opimization).
+
+To edit the `ebpf.d.conf`:
+
+1. Navigate to the [Netdata config directory](/docs/configure/nodes.md#the-netdata-config-directory).
+ ```bash
+ cd /etc/netdata
+ ```
+2. Use the [`edit-config`](/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) script to edit [`ebpf.d.conf`](https://github.com/netdata/netdata/blob/master/collectors/ebpf.plugin/ebpf.d.conf).
+
+ ```bash
+ ./edit-config ebpf.d.conf
+ ```
+
+ You can now edit the behavior of the eBPF collector. The following sections describe each configuration option in detail.
+
+### `[global]` configuration options
+
+The `[global]` section defines settings for the whole eBPF collector.
+
+#### eBPF load mode
+
+The collector uses two different eBPF programs. These programs rely on the same functions inside the kernel, but they
+monitor, process, and display different kinds of information.
+
+By default, this plugin uses the `entry` mode. Changing this mode can create significant overhead on your operating
+system, but also offer valuable information if you are developing or debugging software. The `ebpf load mode` option
+accepts the following values:
+
+- `entry`: This is the default mode. In this mode, the eBPF collector only monitors calls for the functions described in
+ the sections above, and does not show charts related to errors.
+- `return`: In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates new
+ charts for the return of these functions, such as errors. Monitoring function returns can help in debugging software,
+ such as failing to close file descriptors or creating zombie processes.
+- `update every`: Number of seconds used for eBPF to send data for Netdata.
+- `pid table size`: Defines the maximum number of PIDs stored inside the application hash table.
+
+#### Integration with `apps.plugin`
+
+The eBPF collector also creates charts for each running application through an integration with the
+[`apps.plugin`](/collectors/apps.plugin/README.md). This integration helps you understand how specific applications
+interact with the Linux kernel.
+
+If you want to _disable_ the integration with `apps.plugin` along with the above charts, change the setting `apps` to
+`no`.
+
+```conf
+[global]
+ apps = yes
+```
+
+When the integration is enabled, eBPF collector allocates memory for each process running. The total allocated memory
+has direct relationship with the kernel version. When the eBPF plugin is running on kernels newer than `4.15`, it uses
+per-cpu maps to speed up the update of hash tables. This also implies storing data for the same PID for each processor
+it runs.
+
+#### Integration with `cgroups.plugin`
+
+The eBPF collector also creates charts for each cgroup through an integration with the
+[`cgroups.plugin`](/collectors/cgroups.plugin/README.md). This integration helps you understand how a specific cgroup
+interacts with the Linux kernel.
+
+The integration with `cgroups.plugin` is disabled by default to avoid creating overhead on your system. If you want to
+_enable_ the integration with `cgroups.plugin`, change the `cgroups` setting to `yes`.
+
+```conf
+[global]
+ cgroups = yes
+```
+
+If you do not need to monitor specific metrics for your `cgroups`, you can enable `cgroups` inside
+`ebpf.d.conf`, and then disable the plugin for a specific `thread` by following the steps in the
+[Configuration](#configuring-ebpfplugin) section.
+
+#### Collect PID
+
+When one of the previous integrations is enabled, `ebpf.plugin` will use Process Identifier (`PID`) to identify the
+process group for which it needs to plot data.
+
+There are different ways to collect PID, and you can select the way `ebpf.plugin` collects data with the following
+values:
+
+- `real parent`: This is the default mode. Collection will aggregate data for the real parent, the thread that creates
+ child threads.
+- `parent`: Parent and real parent are the same when a process starts, but this value can be changed during run time.
+- `all`: This option will store all PIDs that run on the host. Note, this method can be expensive for the host,
+ because more memory needs to be allocated and parsed.
+
+The threads that have integration with other collectors have an internal clean up wherein they attach either a
+`trampoline` or a `kprobe` to `release_task` internal function. To avoid `overload` on this function, `ebpf.plugin`
+will only enable these threads integrated with other collectors when the kernel is compiled with
+`CONFIG_DEBUG_INFO_BTF`, unless you enable them manually.
+
+#### Integration Dashboard Elements
+
+When an integration is enabled, your dashboard will also show the following cgroups and apps charts using low-level
+Linux metrics:
+
+> Note: The parenthetical accompanying each bulleted item provides the chart name.
+
+- mem
+ - Number of processes killed due out of memory. (`oomkills`)
+- process
+ - Number of processes created with `do_fork`. (`process_create`)
+ - Number of threads created with `do_fork` or `clone (2)`, depending on your system's kernel
+ version. (`thread_create`)
+ - Number of times that a process called `do_exit`. (`task_exit`)
+ - Number of times that a process called `release_task`. (`task_close`)
+ - Number of times that an error happened to create thread or process. (`task_error`)
+- swap
+ - Number of calls to `swap_readpage`. (`swap_read_call`)
+ - Number of calls to `swap_writepage`. (`swap_write_call`)
+- network
+ - Number of outbound connections using TCP/IPv4. (`outbound_conn_ipv4`)
+ - Number of outbound connections using TCP/IPv6. (`outbound_conn_ipv6`)
+ - Number of bytes sent. (`total_bandwidth_sent`)
+ - Number of bytes received. (`total_bandwidth_recv`)
+ - Number of calls to `tcp_sendmsg`. (`bandwidth_tcp_send`)
+ - Number of calls to `tcp_cleanup_rbuf`. (`bandwidth_tcp_recv`)
+ - Number of calls to `tcp_retransmit_skb`. (`bandwidth_tcp_retransmit`)
+ - Number of calls to `udp_sendmsg`. (`bandwidth_udp_send`)
+ - Number of calls to `udp_recvmsg`. (`bandwidth_udp_recv`)
+- file access
+ - Number of calls to open files. (`file_open`)
+ - Number of calls to open files that returned errors. (`open_error`)
+ - Number of files closed. (`file_closed`)
+ - Number of calls to close files that returned errors. (`file_error_closed`)
+- vfs
+ - Number of calls to `vfs_unlink`. (`file_deleted`)
+ - Number of calls to `vfs_write`. (`vfs_write_call`)
+ - Number of calls to write a file that returned errors. (`vfs_write_error`)
+ - Number of calls to `vfs_read`. (`vfs_read_call`)
+ - - Number of calls to read a file that returned errors. (`vfs_read_error`)
+ - Number of bytes written with `vfs_write`. (`vfs_write_bytes`)
+ - Number of bytes read with `vfs_read`. (`vfs_read_bytes`)
+ - Number of calls to `vfs_fsync`. (`vfs_fsync`)
+ - Number of calls to sync file that returned errors. (`vfs_fsync_error`)
+ - Number of calls to `vfs_open`. (`vfs_open`)
+ - Number of calls to open file that returned errors. (`vfs_open_error`)
+ - Number of calls to `vfs_create`. (`vfs_create`)
+ - Number of calls to open file that returned errors. (`vfs_create_error`)
+- page cache
+ - Ratio of pages accessed. (`cachestat_ratio`)
+ - Number of modified pages ("dirty"). (`cachestat_dirties`)
+ - Number of accessed pages. (`cachestat_hits`)
+ - Number of pages brought from disk. (`cachestat_misses`)
+- directory cache
+ - Ratio of files available in directory cache. (`dc_hit_ratio`)
+ - Number of files accessed. (`dc_reference`)
+ - Number of files accessed that were not in cache. (`dc_not_cache`)
+ - Number of files not found. (`dc_not_found`)
+- ipc shm
+ - Number of calls to `shm_get`. (`shmget_call`)
+ - Number of calls to `shm_at`. (`shmat_call`)
+ - Number of calls to `shm_dt`. (`shmdt_call`)
+ - Number of calls to `shm_ctl`. (`shmctl_call`)
+
+### `[ebpf programs]` configuration options
+
+The eBPF collector enables and runs the following eBPF programs by default:
+
+- `fd` : This eBPF program creates charts that show information about calls to open files.
+- `mount`: This eBPF program creates charts that show calls to syscalls mount(2) and umount(2).
+- `shm`: This eBPF program creates charts that show calls to syscalls shmget(2), shmat(2), shmdt(2) and shmctl(2).
+- `sync`: Monitor calls to syscalls sync(2), fsync(2), fdatasync(2), syncfs(2), msync(2), and sync_file_range(2).
+- `network viewer`: This eBPF program creates charts with information about `TCP` and `UDP` functions, including the
+ bandwidth consumed by each.
+- `vfs`: This eBPF program creates charts that show information about VFS (Virtual File System) functions.
+- `process`: This eBPF program creates charts that show information about process life. When in `return` mode, it also
+ creates charts showing errors when these operations are executed.
+- `hardirq`: This eBPF program creates charts that show information about time spent servicing individual hardware
+ interrupt requests (hard IRQs).
+- `softirq`: This eBPF program creates charts that show information about time spent servicing individual software
+ interrupt requests (soft IRQs).
+- `oomkill`: This eBPF program creates a chart that shows OOM kills for all applications recognized via
+ the `apps.plugin` integration. Note that this program will show application charts regardless of whether apps
+ integration is turned on or off.
+
+You can also enable the following eBPF programs:
+
+- `cachestat`: Netdata's eBPF data collector creates charts about the memory page cache. When the integration with
+ [`apps.plugin`](/collectors/apps.plugin/README.md) is enabled, this collector creates charts for the whole host _and_
+ for each application.
+- `dcstat` : This eBPF program creates charts that show information about file access using directory cache. It appends
+ `kprobes` for `lookup_fast()` and `d_lookup()` to identify if files are inside directory cache, outside and files are
+ not found.
+- `disk` : This eBPF program creates charts that show information about disk latency independent of filesystem.
+- `filesystem` : This eBPF program creates charts that show information about some filesystem latency.
+- `swap` : This eBPF program creates charts that show information about swap access.
+- `mdflush`: This eBPF program creates charts that show information about
+ multi-device software flushes.
+
+### Configuring eBPF threads
+
+You can configure each thread of the eBPF data collector. This allows you to overwrite global options defined in `/etc/netdata/ebpf.d.conf` and configure specific options for each thread.
+
+To configure an eBPF thread:
+
+1. Navigate to the [Netdata config directory](/docs/configure/nodes.md#the-netdata-config-directory).
+ ```bash
+ cd /etc/netdata
+ ```
+2. Use the [`edit-config`](/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) script to edit a thread configuration file. The following configuration files are available:
+
+ - `network.conf`: Configuration for the [`network` thread](#network-configuration). This config file overwrites the global options and also
+ lets you specify which network the eBPF collector monitors.
+ - `process.conf`: Configuration for the [`process` thread](#sync-configuration).
+ - `cachestat.conf`: Configuration for the `cachestat` thread(#filesystem-configuration).
+ - `dcstat.conf`: Configuration for the `dcstat` thread.
+ - `disk.conf`: Configuration for the `disk` thread.
+ - `fd.conf`: Configuration for the `file descriptor` thread.
+ - `filesystem.conf`: Configuration for the `filesystem` thread.
+ - `hardirq.conf`: Configuration for the `hardirq` thread.
+ - `softirq.conf`: Configuration for the `softirq` thread.
+ - `sync.conf`: Configuration for the `sync` thread.
+ - `vfs.conf`: Configuration for the `vfs` thread.
+
+ ```bash
+ ./edit-config FILE.conf
+ ```
+
+### Network configuration
+
+The network configuration has specific options to configure which network(s) the eBPF collector monitors. These options
+are divided in the following sections:
+
+#### `[network connections]`
+
+You can configure the information shown on `outbound` and `inbound` charts with the settings in this section.
+
+```conf
+[network connections]
+ maximum dimensions = 500
+ resolve hostname ips = no
+ ports = 1-1024 !145 !domain
+ hostnames = !example.com
+ ips = !127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7
+```
+
+When you define a `ports` setting, Netdata will collect network metrics for that specific port. For example, if you
+write `ports = 19999`, Netdata will collect only connections for itself. The `hostnames` setting accepts
+[simple patterns](/libnetdata/simple_pattern/README.md). The `ports`, and `ips` settings accept negation (`!`) to deny
+specific values or asterisk alone to define all values.
+
+In the above example, Netdata will collect metrics for all ports between 1 and 443, with the exception of 53 (domain)
+and 145.
+
+The following options are available:
+
+- `ports`: Define the destination ports for Netdata to monitor.
+- `hostnames`: The list of hostnames that can be resolved to an IP address.
+- `ips`: The IP or range of IPs that you want to monitor. You can use IPv4 or IPv6 addresses, use dashes to define a
+ range of IPs, or use CIDR values. By default, only data for private IP addresses is collected, but this can
+ be changed with the `ips` setting.
+
+By default, Netdata displays up to 500 dimensions on network connection charts. If there are more possible dimensions,
+they will be bundled into the `other` dimension. You can increase the number of shown dimensions by changing
+the `maximum dimensions` setting.
+
+The dimensions for the traffic charts are created using the destination IPs of the sockets by default. This can be
+changed setting `resolve hostname ips = yes` and restarting Netdata, after this Netdata will create dimensions using
+the `hostnames` every time that is possible to resolve IPs to their hostnames.
+
+#### `[service name]`
+
+Netdata uses the list of services in `/etc/services` to plot network connection charts. If this file does not contain
+the name for a particular service you use in your infrastructure, you will need to add it to the `[service name]`
+section.
+
+For example, Netdata's default port (`19999`) is not listed in `/etc/services`. To associate that port with the Netdata
+service in network connection charts, and thus see the name of the service instead of its port, define it:
+
+```conf
+[service name]
+ 19999 = Netdata
+```
+
+### Sync configuration
+
+The sync configuration has specific options to disable monitoring for syscalls. All syscalls are monitored by default.
+
+```conf
+[syscalls]
+ sync = yes
+ msync = yes
+ fsync = yes
+ fdatasync = yes
+ syncfs = yes
+ sync_file_range = yes
+```
+
+### Filesystem configuration
+
+The filesystem configuration has specific options to disable monitoring for filesystems; by default, all filesystems are
+monitored.
+
+```conf
+[filesystem]
+ btrfsdist = yes
+ ext4dist = yes
+ nfsdist = yes
+ xfsdist = yes
+ zfsdist = yes
+```
+
+The ebpf program `nfsdist` monitors only `nfs` mount points.
+
+## Troubleshooting
+
+If the eBPF collector does not work, you can troubleshoot it by running the `ebpf.plugin` command and investigating its
+output.
+
+```bash
+cd /usr/libexec/netdata/plugins.d/
+sudo su -s /bin/bash ./ebpf.plugin
+```
+
+You can also use `grep` to search the Agent's `error.log` for messages related to eBPF monitoring.
+
+```bash
+grep -i ebpf /var/log/netdata/error.log
+```
+
+### Confirm kernel compatibility
+
+The eBPF collector only works on Linux systems and with specific Linux kernels. We support all kernels more recent than
+`4.11.0`, and all kernels on CentOS 7.6 or later.
+
+You can run our helper script to determine whether your system can support eBPF monitoring. If it returns no output, your system is ready to compile and run the eBPF collector.
+
+```bash
+curl -sSL https://raw.githubusercontent.com/netdata/kernel-collector/master/tools/check-kernel-config.sh | sudo bash
+```
+
+
+If you see a warning about a missing kernel
+configuration (`KPROBES KPROBES_ON_FTRACE HAVE_KPROBES BPF BPF_SYSCALL BPF_JIT`), you will need to recompile your kernel
+to support this configuration. The process of recompiling Linux kernels varies based on your distribution and version.
+Read the documentation for your system's distribution to learn more about the specific workflow for recompiling the
+kernel, ensuring that you set all the necessary
+
+- [Ubuntu](https://wiki.ubuntu.com/Kernel/BuildYourOwnKernel)
+- [Debian](https://kernel-team.pages.debian.net/kernel-handbook/ch-common-tasks.html#s-common-official)
+- [Fedora](https://fedoraproject.org/wiki/Building_a_custom_kernel)
+- [CentOS](https://wiki.centos.org/HowTos/Custom_Kernel)
+- [Arch Linux](https://wiki.archlinux.org/index.php/Kernel/Traditional_compilation)
+- [Slackware](https://docs.slackware.com/howtos:slackware_admin:kernelbuilding)
+
+### Mount `debugfs` and `tracefs`
+
+The eBPF collector also requires both the `tracefs` and `debugfs` filesystems. Try mounting the `tracefs` and `debugfs`
+filesystems using the commands below:
+
+```bash
+sudo mount -t debugfs nodev /sys/kernel/debug
+sudo mount -t tracefs nodev /sys/kernel/tracing
+```
+
+If they are already mounted, you will see an error. You can also configure your system's `/etc/fstab` configuration to
+mount these filesystems on startup. More information can be found in
+the [ftrace documentation](https://www.kernel.org/doc/Documentation/trace/ftrace.txt).
+
+## Charts
+
+The eBPF collector creates charts on different menus, like System Overview, Memory, MD arrays, Disks, Filesystem,
+Mount Points, Networking Stack, systemd Services, and Applications.
+
+The collector stores the actual value inside of its process, but charts only show the difference between the values
+collected in the previous and current seconds.
+
+### System overview
+
+Not all charts within the System Overview menu are enabled by default. Charts that rely on `kprobes` are disabled by default because they add around 100ns overhead for each function call. This is a small number from a human's perspective, but the functions are called many times and create an impact
+on host. See the [configuration](#configuring-ebpfplugin) section for details about how to enable them.
+
+#### Processes
+
+Internally, the Linux kernel treats both processes and threads as `tasks`. To create a thread, the kernel offers a few
+system calls: `fork(2)`, `vfork(2)`, and `clone(2)`. To generate this chart, the eBPF
+collector uses the following `tracepoints` and `kprobe`:
+
+- `sched/sched_process_fork`: Tracepoint called after a call for `fork (2)`, `vfork (2)` and `clone (2)`.
+- `sched/sched_process_exec`: Tracepoint called after a exec-family syscall.
+- `kprobe/kernel_clone`: This is the main [`fork()`](https://elixir.bootlin.com/linux/v5.10/source/kernel/fork.c#L2415)
+ routine since kernel `5.10.0` was released.
+- `kprobe/_do_fork`: Like `kernel_clone`, but this was the main function between kernels `4.2.0` and `5.9.16`
+- `kprobe/do_fork`: This was the main function before kernel `4.2.0`.
+
+#### Process Exit
+
+Ending a task requires two steps. The first is a call to the internal function `do_exit`, which notifies the operating
+system that the task is finishing its work. The second step is to release the kernel information with the internal
+function `release_task`. The difference between the two dimensions can help you discover
+[zombie processes](https://en.wikipedia.org/wiki/Zombie_process). To get the metrics, the collector uses:
+
+- `sched/sched_process_exit`: Tracepoint called after a task exits.
+- `kprobe/release_task`: This function is called when a process exits, as the kernel still needs to remove the process
+ descriptor.
+
+#### Task error
+
+The functions responsible for ending tasks do not return values, so this chart contains information about failures on
+process and thread creation only.
+
+#### Swap
+
+Inside the swap submenu the eBPF plugin creates the chart `swapcalls`; this chart is displaying when processes are
+calling functions [`swap_readpage` and `swap_writepage`](https://hzliu123.github.io/linux-kernel/Page%20Cache%20in%20Linux%202.6.pdf),
+which are functions responsible for doing IO in swap memory. To collect the exact moment that an access to swap happens,
+the collector attaches `kprobes` for cited functions.
+
+#### Soft IRQ
+
+The following `tracepoints` are used to measure time usage for soft IRQs:
+
+- [`irq/softirq_entry`](https://www.kernel.org/doc/html/latest/core-api/tracepoint.html#c.trace_softirq_entry): Called
+ before softirq handler
+- [`irq/softirq_exit`](https://www.kernel.org/doc/html/latest/core-api/tracepoint.html#c.trace_softirq_exit): Called when
+ softirq handler returns.
+
+#### Hard IRQ
+
+The following tracepoints are used to measure the latency of servicing a
+hardware interrupt request (hard IRQ).
+
+- [`irq/irq_handler_entry`](https://www.kernel.org/doc/html/latest/core-api/tracepoint.html#c.trace_irq_handler_entry):
+ Called immediately before the IRQ action handler.
+- [`irq/irq_handler_exit`](https://www.kernel.org/doc/html/latest/core-api/tracepoint.html#c.trace_irq_handler_exit):
+ Called immediately after the IRQ action handler returns.
+- `irq_vectors`: These are traces from `irq_handler_entry` and
+ `irq_handler_exit` when an IRQ is handled. The following elements from vector
+ are triggered:
+ - `irq_vectors/local_timer_entry`
+ - `irq_vectors/local_timer_exit`
+ - `irq_vectors/reschedule_entry`
+ - `irq_vectors/reschedule_exit`
+ - `irq_vectors/call_function_entry`
+ - `irq_vectors/call_function_exit`
+ - `irq_vectors/call_function_single_entry`
+ - `irq_vectors/call_function_single_xit`
+ - `irq_vectors/irq_work_entry`
+ - `irq_vectors/irq_work_exit`
+ - `irq_vectors/error_apic_entry`
+ - `irq_vectors/error_apic_exit`
+ - `irq_vectors/thermal_apic_entry`
+ - `irq_vectors/thermal_apic_exit`
+ - `irq_vectors/threshold_apic_entry`
+ - `irq_vectors/threshold_apic_exit`
+ - `irq_vectors/deferred_error_entry`
+ - `irq_vectors/deferred_error_exit`
+ - `irq_vectors/spurious_apic_entry`
+ - `irq_vectors/spurious_apic_exit`
+ - `irq_vectors/x86_platform_ipi_entry`
+ - `irq_vectors/x86_platform_ipi_exit`
+
+#### IPC shared memory
+
+To monitor shared memory system call counts, Netdata attaches tracing in the following functions:
+
+- `shmget`: Runs when [`shmget`](https://man7.org/linux/man-pages/man2/shmget.2.html) is called.
+- `shmat`: Runs when [`shmat`](https://man7.org/linux/man-pages/man2/shmat.2.html) is called.
+- `shmdt`: Runs when [`shmdt`](https://man7.org/linux/man-pages/man2/shmat.2.html) is called.
+- `shmctl`: Runs when [`shmctl`](https://man7.org/linux/man-pages/man2/shmctl.2.html) is called.
+
+### Memory
+
+In the memory submenu the eBPF plugin creates two submenus **page cache** and **synchronization** with the following
+organization:
+
+- Page Cache
+ - Page cache ratio
+ - Dirty pages
+ - Page cache hits
+ - Page cache misses
+- Synchronization
+ - File sync
+ - Memory map sync
+ - File system sync
+ - File range sync
+
+#### Page cache hits
+
+When the processor needs to read or write a location in main memory, it checks for a corresponding entry in the page cache.
+ If the entry is there, a page cache hit has occurred and the read is from the cache.
+
+A page cache hit is when the page cache is successfully accessed with a read operation. We do not count pages that were
+added relatively recently.
+
+#### Dirty pages
+
+A "dirty page" is a page in the page cache that was modified after being created. Since non-dirty pages in the page cache
+ have identical copies in secondary storage (e.g. hard disk drive or solid-state drive), discarding and reusing their space
+ is much quicker than paging out application memory, and is often preferred over flushing the dirty pages into secondary storage
+ and reusing their space.
+
+On `cachestat_dirties` Netdata demonstrates the number of pages that were modified. This chart shows the number of calls
+to the function `mark_buffer_dirty`.
+
+#### Page cache ratio
+
+When the processor needs to read or write in a specific memory address, it checks for a corresponding entry in the page cache.
+If the processor hits a page cache (`page cache hit`), it reads the entry from the cache. If there is no entry (`page cache miss`),
+ the kernel allocates a new entry and copies data from the disk. Netdata calculates the percentage of accessed files that are cached on
+ memory. The ratio is calculated counting the accessed cached pages
+ (without counting [dirty pages](#dirty-pages) and pages added because of read misses) divided by total access without dirty pages.
+
+> \_\_**\_\_\_\_**<ins>Number of accessed cached pages</ins>\***\*\_\_\*\***<br/>
+> Number of total accessed pages - dirty pages - missed pages
+
+The chart `cachestat_ratio` shows how processes are accessing page cache. In a normal scenario, we expect values around
+100%, which means that the majority of the work on the machine is processed in memory. To calculate the ratio, Netdata
+attaches `kprobes` for kernel functions:
+
+- `add_to_page_cache_lru`: Page addition.
+- `mark_page_accessed`: Access to cache.
+- `account_page_dirtied`: Dirty (modified) pages.
+- `mark_buffer_dirty`: Writes to page cache.
+
+#### Page cache misses
+
+A page cache miss means that a page was not inside memory when the process tried to access it. This chart shows the
+result of the difference for calls between functions `add_to_page_cache_lru` and `account_page_dirtied`.
+
+#### File sync
+
+This chart shows calls to synchronization methods, [`fsync(2)`](https://man7.org/linux/man-pages/man2/fdatasync.2.html)
+and [`fdatasync(2)`](https://man7.org/linux/man-pages/man2/fdatasync.2.html), to transfer all modified page caches
+for the files on disk devices. These calls block until the disk reports that the transfer has been completed. They flush
+data for specific file descriptors.
+
+#### Memory map sync
+
+The chart shows calls to [`msync(2)`](https://man7.org/linux/man-pages/man2/msync.2.html) syscalls. This syscall flushes
+changes to a file that was mapped into memory using [`mmap(2)`](https://man7.org/linux/man-pages/man2/mmap.2.html).
+
+#### File system sync
+
+This chart monitors calls demonstrating commits from filesystem caches to disk. Netdata attaches `tracing` for
+[`sync(2)`](https://man7.org/linux/man-pages/man2/sync.2.html), and [`syncfs(2)`](https://man7.org/linux/man-pages/man2/sync.2.html).
+
+#### File range sync
+
+This chart shows calls to [`sync_file_range(2)`](https://man7.org/linux/man-pages/man2/sync_file_range.2.html) which
+synchronizes file segments with disk.
+
+> Note: This is the most dangerous syscall to synchronize data, according to its manual.
+
+### Multiple Device (MD) arrays
+
+The eBPF plugin shows multi-device flushes happening in real time. This can be used to explain some spikes happening
+in [disk latency](#disk) charts.
+
+By default, MD flush is disabled. To enable it, configure your
+`/etc/netdata/ebpf.d.conf` file as:
+
+```conf
+[global]
+ mdflush = yes
+```
+
+#### MD flush
+
+To collect data related to Linux multi-device (MD) flushing, the following kprobe is used:
+
+- `kprobe/md_flush_request`: called whenever a request for flushing multi-device data is made.
+
+### Disk
+
+The eBPF plugin also shows a chart in the Disk section when the `disk` thread is enabled.
+
+#### Disk Latency
+
+This will create the chart `disk_latency_io` for each disk on the host. The following tracepoints are used:
+
+- [`block/block_rq_issue`](https://www.kernel.org/doc/html/latest/core-api/tracepoint.html#c.trace_block_rq_issue):
+ IO request operation to a device drive.
+- [`block/block_rq_complete`](https://www.kernel.org/doc/html/latest/core-api/tracepoint.html#c.trace_block_rq_complete):
+ IO operation completed by device.
+
+Disk Latency is the single most important metric to focus on when it comes to storage performance, under most circumstances.
+For hard drives, an average latency somewhere between 10 to 20 ms can be considered acceptable. For SSD (Solid State Drives),
+in most cases, workloads experience less than 1 ms latency numbers, but workloads should never reach higher than 3 ms.
+The dimensions refer to time intervals.
+
+### Filesystem
+
+This group has charts demonstrating how applications interact with the Linux kernel to open and close file descriptors.
+It also brings latency charts for several different filesystems.
+
+#### Latency Algorithm
+
+We calculate the difference between the calling and return times, spanning disk I/O, file system operations (lock, I/O),
+run queue latency and all events related to the monitored action.
+
+#### ext4
+
+To measure the latency of executing some actions in an
+[ext4](https://elixir.bootlin.com/linux/latest/source/fs/ext4) filesystem, the
+collector needs to attach `kprobes` and `kretprobes` for each of the following
+functions:
+
+- `ext4_file_read_iter`: Function used to measure read latency.
+- `ext4_file_write_iter`: Function used to measure write latency.
+- `ext4_file_open`: Function used to measure open latency.
+- `ext4_sync_file`: Function used to measure sync latency.
+
+#### ZFS
+
+To measure the latency of executing some actions in a zfs filesystem, the
+collector needs to attach `kprobes` and `kretprobes` for each of the following
+functions:
+
+- `zpl_iter_read`: Function used to measure read latency.
+- `zpl_iter_write`: Function used to measure write latency.
+- `zpl_open`: Function used to measure open latency.
+- `zpl_fsync`: Function used to measure sync latency.
+
+#### XFS
+
+To measure the latency of executing some actions in an
+[xfs](https://elixir.bootlin.com/linux/latest/source/fs/xfs) filesystem, the
+collector needs to attach `kprobes` and `kretprobes` for each of the following
+functions:
+
+- `xfs_file_read_iter`: Function used to measure read latency.
+- `xfs_file_write_iter`: Function used to measure write latency.
+- `xfs_file_open`: Function used to measure open latency.
+- `xfs_file_fsync`: Function used to measure sync latency.
+
+#### NFS
+
+To measure the latency of executing some actions in an
+[nfs](https://elixir.bootlin.com/linux/latest/source/fs/nfs) filesystem, the
+collector needs to attach `kprobes` and `kretprobes` for each of the following
+functions:
+
+- `nfs_file_read`: Function used to measure read latency.
+- `nfs_file_write`: Function used to measure write latency.
+- `nfs_file_open`: Functions used to measure open latency.
+- `nfs4_file_open`: Functions used to measure open latency for NFS v4.
+- `nfs_getattr`: Function used to measure sync latency.
+
+#### btrfs
+
+To measure the latency of executing some actions in a [btrfs](https://elixir.bootlin.com/linux/latest/source/fs/btrfs/file.c)
+filesystem, the collector needs to attach `kprobes` and `kretprobes` for each of the following functions:
+
+> Note: We are listing two functions used to measure `read` latency, but we use either `btrfs_file_read_iter` or
+> `generic_file_read_iter`, depending on kernel version.
+
+- `btrfs_file_read_iter`: Function used to measure read latency since kernel `5.10.0`.
+- `generic_file_read_iter`: Like `btrfs_file_read_iter`, but this function was used before kernel `5.10.0`.
+- `btrfs_file_write_iter`: Function used to write data.
+- `btrfs_file_open`: Function used to open files.
+- `btrfs_sync_file`: Function used to synchronize data to filesystem.
+
+#### File descriptor
+
+To give metrics related to `open` and `close` events, instead of attaching kprobes for each syscall used to do these
+events, the collector attaches `kprobes` for the common function used for syscalls:
+
+- [`do_sys_open`](https://0xax.gitbooks.io/linux-insides/content/SysCall/linux-syscall-5.html): Internal function used to
+ open files.
+- [`do_sys_openat2`](https://elixir.bootlin.com/linux/v5.6/source/fs/open.c#L1162):
+ Function called from `do_sys_open` since version `5.6.0`.
+- [`close_fd`](https://www.mail-archive.com/linux-kernel@vger.kernel.org/msg2271761.html): Function used to close file
+ descriptor since kernel `5.11.0`.
+- `__close_fd`: Function used to close files before version `5.11.0`.
+
+#### File error
+
+This chart shows the number of times some software tried and failed to open or close a file descriptor.
+
+#### VFS
+
+The Linux Virtual File System (VFS) is an abstraction layer on top of a
+concrete filesystem like the ones listed in the parent section, e.g. `ext4`.
+
+In this section we list the mechanism by which we gather VFS data, and what
+charts are consequently created.
+
+##### VFS eBPF Hooks
+
+To measure the latency and total quantity of executing some VFS-level
+functions, ebpf.plugin needs to attach kprobes and kretprobes for each of the
+following functions:
+
+- `vfs_write`: Function used monitoring the number of successful & failed
+ filesystem write calls, as well as the total number of written bytes.
+- `vfs_writev`: Same function as `vfs_write` but for vector writes (i.e. a
+ single write operation using a group of buffers rather than 1).
+- `vfs_read`: Function used for monitoring the number of successful & failed
+ filesystem read calls, as well as the total number of read bytes.
+- `vfs_readv` Same function as `vfs_read` but for vector reads (i.e. a single
+ read operation using a group of buffers rather than 1).
+- `vfs_unlink`: Function used for monitoring the number of successful & failed
+ filesystem unlink calls.
+- `vfs_fsync`: Function used for monitoring the number of successful & failed
+ filesystem fsync calls.
+- `vfs_open`: Function used for monitoring the number of successful & failed
+ filesystem open calls.
+- `vfs_create`: Function used for monitoring the number of successful & failed
+ filesystem create calls.
+
+##### VFS Deleted objects
+
+This chart monitors calls to `vfs_unlink`. This function is responsible for removing objects from the file system.
+
+##### VFS IO
+
+This chart shows the number of calls to the functions `vfs_read` and `vfs_write`.
+
+##### VFS IO bytes
+
+This chart also monitors `vfs_read` and `vfs_write` but, instead of the number of calls, it shows the total amount of
+bytes read and written with these functions.
+
+The Agent displays the number of bytes written as negative because they are moving down to disk.
+
+##### VFS IO errors
+
+The Agent counts and shows the number of instances where a running program experiences a read or write error.
+
+##### VFS Create
+
+This chart shows the number of calls to `vfs_create`. This function is responsible for creating files.
+
+##### VFS Synchronization
+
+This chart shows the number of calls to `vfs_fsync`. This function is responsible for calling `fsync(2)` or
+`fdatasync(2)` on a file. You can see more details in the Synchronization section.
+
+##### VFS Open
+
+This chart shows the number of calls to `vfs_open`. This function is responsible for opening files.
+
+#### Directory Cache
+
+Metrics for directory cache are collected using kprobe for `lookup_fast`, because we are interested in the number of
+times this function is accessed. On the other hand, for `d_lookup` we are not only interested in the number of times it
+is accessed, but also in possible errors, so we need to attach a `kretprobe`. For this reason, the following is used:
+
+- [`lookup_fast`](https://lwn.net/Articles/649115/): Called to look at data inside the directory cache.
+- [`d_lookup`](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/dcache.c?id=052b398a43a7de8c68c13e7fa05d6b3d16ce6801#n2223):
+ Called when the desired file is not inside the directory cache.
+
+##### Directory Cache Interpretation
+
+When directory cache is showing 100% that means that every accessed file was present in the directory cache.
+If files are not present in the directory cache, they are either not present in the file system or the files were not
+accessed before.
+
+### Mount Points
+
+The following `tracing` are used to collect `mount` & `unmount` call counts:
+
+- [`mount`](https://man7.org/linux/man-pages/man2/mount.2.html): mount filesystem on host.
+- [`umount`](https://man7.org/linux/man-pages/man2/umount.2.html): umount filesystem on host.
+
+### Networking Stack
+
+Netdata monitors socket bandwidth attaching `tracing` for internal functions.
+
+#### TCP outbound connections
+
+This chart demonstrates calls to `tcp_v4_connection` and `tcp_v6_connection` that start connections for IPV4 and IPV6, respectively.
+
+#### TCP inbound connections
+
+This chart demonstrates TCP and UDP connections that the host receives.
+To collect this information, netdata attaches a tracing to `inet_csk_accept`.
+
+#### TCP bandwidth functions
+
+This chart demonstrates calls to functions `tcp_sendmsg`, `tcp_cleanup_rbuf`, and `tcp_close`; these functions are used
+to send & receive data and to close connections when `TCP` protocol is used.
+
+#### TCP bandwidth
+
+This chart demonstrates calls to functions:
+
+- `tcp_sendmsg`: Function responsible to send data for a specified destination.
+- `tcp_cleanup_rbuf`: We use this function instead of `tcp_recvmsg`, because the last one misses `tcp_read_sock` traffic
+ and we would also need to add more `tracing` to get the socket and package size.
+- `tcp_close`: Function responsible to close connection.
+
+#### TCP retransmit
+
+This chart demonstrates calls to function `tcp_retransmit` that is responsible for executing TCP retransmission when the
+receiver did not return the packet during the expected time.
+
+#### UDP functions
+
+This chart demonstrates calls to functions `udp_sendmsg` and `udp_recvmsg`, which are responsible for sending &
+receiving data for connections when the `UDP` protocol is used.
+
+#### UDP bandwidth
+
+Like the previous chart, this one also monitors `udp_sendmsg` and `udp_recvmsg`, but instead of showing the number of
+calls, it monitors the number of bytes sent and received.
+
+### Apps
+
+#### OOM Killing
+
+These are tracepoints related to [OOM](https://en.wikipedia.org/wiki/Out_of_memory) killing processes.
+
+- `oom/mark_victim`: Monitors when an oomkill event happens.
+
+## Known issues
+
+### Performance opimization
+
+eBPF monitoring is complex and produces a large volume of metrics. We've discovered scenarios where the eBPF plugin
+significantly increases kernel memory usage by several hundred MB.
+
+If your node is experiencing high memory usage and there is no obvious culprit to be found in the `apps.mem` chart,
+consider testing for high kernel memory usage by [disabling eBPF monitoring](#configuring-ebpfplugin). Next,
+[restart Netdata](/docs/configure/start-stop-restart.md) with `sudo systemctl restart netdata` to see if system memory
+usage (see the `system.ram` chart) has dropped significantly.
+
+Beginning with `v1.31`, kernel memory usage is configurable via the [`pid table size` setting](#ebpf-load-mode)
+in `ebpf.conf`.
+
+### SELinux
+
+When [SELinux](https://www.redhat.com/en/topics/linux/what-is-selinux) is enabled, it may prevent `ebpf.plugin` from
+starting correctly. Check the Agent's `error.log` file for errors like the ones below:
+
+```bash
+2020-06-14 15:32:08: ebpf.plugin ERROR : EBPF PROCESS : Cannot load program: /usr/libexec/netdata/plugins.d/pnetdata_ebpf_process.3.10.0.o (errno 13, Permission denied)
+2020-06-14 15:32:19: netdata ERROR : PLUGINSD[ebpf] : read failed: end of file (errno 9, Bad file descriptor)
+```
+
+You can also check for errors related to `ebpf.plugin` inside `/var/log/audit/audit.log`:
+
+```bash
+type=AVC msg=audit(1586260134.952:97): avc: denied { map_create } for pid=1387 comm="ebpf.pl" scontext=system_u:system_r:unconfined_service_t:s0 tcontext=system_u:system_r:unconfined_service_t:s0 tclass=bpf permissive=0
+type=SYSCALL msg=audit(1586260134.952:97): arch=c000003e syscall=321 success=no exit=-13 a0=0 a1=7ffe6b36f000 a2=70 a3=0 items=0 ppid=1135 pid=1387 auid=4294967295 uid=994 gid=990 euid=0 suid=0 fsuid=0 egid=990 sgid=990 fsgid=990 tty=(none) ses=4294967295 comm="ebpf_proc
+ess.pl" exe="/usr/libexec/netdata/plugins.d/ebpf.plugin" subj=system_u:system_r:unconfined_service_t:s0 key=(null)
+```
+
+If you see similar errors, you will have to adjust SELinux's policies to enable the eBPF collector.
+
+#### Creation of bpf policies
+
+To enable `ebpf.plugin` to run on a distribution with SELinux enabled, it will be necessary to take the following
+actions.
+
+First, stop the Netdata Agent.
+
+```bash
+# systemctl stop netdata
+```
+
+Next, create a policy with the `audit.log` file you examined earlier.
+
+```bash
+# grep ebpf.plugin /var/log/audit/audit.log | audit2allow -M netdata_ebpf
+```
+
+This will create two new files: `netdata_ebpf.te` and `netdata_ebpf.mod`.
+
+Edit the `netdata_ebpf.te` file to change the options `class` and `allow`. You should have the following at the end of
+the `netdata_ebpf.te` file.
+
+```conf
+module netdata_ebpf 1.0;
+require {
+ type unconfined_service_t;
+ class bpf { map_create map_read map_write prog_load prog_run };
+}
+#============= unconfined_service_t ==============
+allow unconfined_service_t self:bpf { map_create map_read map_write prog_load prog_run };
+```
+
+Then compile your `netdata_ebpf.te` file with the following commands to create a binary that loads the new policies:
+
+```bash
+# checkmodule -M -m -o netdata_ebpf.mod netdata_ebpf.te
+# semodule_package -o netdata_ebpf.pp -m netdata_ebpf.mod
+```
+
+Finally, you can load the new policy and start the Netdata agent again:
+
+```bash
+# semodule -i netdata_ebpf.pp
+# systemctl start netdata
+```
+
+### Linux kernel lockdown
+
+Beginning with [version 5.4](https://www.zdnet.com/article/linux-to-get-kernel-lockdown-feature/), the Linux kernel has
+a feature called "lockdown," which may affect `ebpf.plugin` depending how the kernel was compiled. The following table
+shows how the lockdown module impacts `ebpf.plugin` based on the selected options:
+
+| Enforcing kernel lockdown | Enable lockdown LSM early in init | Default lockdown mode | Can `ebpf.plugin` run with this? |
+| :------------------------ | :-------------------------------- | :-------------------- | :------------------------------- |
+| YES | NO | NO | YES |
+| YES | Yes | None | YES |
+| YES | Yes | Integrity | YES |
+| YES | Yes | Confidentiality | NO |
+
+If you or your distribution compiled the kernel with the last combination, your system cannot load shared libraries
+required to run `ebpf.plugin`.
diff --git a/collectors/ebpf.plugin/ebpf.c b/collectors/ebpf.plugin/ebpf.c
new file mode 100644
index 0000000..00b53a5
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.c
@@ -0,0 +1,2249 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <ifaddrs.h>
+
+#include "ebpf.h"
+#include "ebpf_socket.h"
+#include "libnetdata/required_dummies.h"
+
+/*****************************************************************
+ *
+ * GLOBAL VARIABLES
+ *
+ *****************************************************************/
+
+char *ebpf_plugin_dir = PLUGINS_DIR;
+static char *ebpf_configured_log_dir = LOG_DIR;
+
+char *ebpf_algorithms[] = {"absolute", "incremental"};
+struct config collector_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+int running_on_kernel = 0;
+int ebpf_nprocs;
+int isrh = 0;
+int main_thread_id = 0;
+
+pthread_mutex_t lock;
+pthread_mutex_t ebpf_exit_cleanup;
+pthread_mutex_t collect_data_mutex;
+pthread_cond_t collect_data_cond_var;
+
+ebpf_module_t ebpf_modules[] = {
+ { .thread_name = "process", .config_name = "process", .enabled = 0, .start_routine = ebpf_process_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_process_create_apps_charts, .maps = NULL,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &process_config,
+ .config_file = NETDATA_PROCESS_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_10 |
+ NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "socket", .config_name = "socket", .enabled = 0, .start_routine = ebpf_socket_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_socket_create_apps_charts, .maps = NULL,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &socket_config,
+ .config_file = NETDATA_NETWORK_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = socket_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "cachestat", .config_name = "cachestat", .enabled = 0, .start_routine = ebpf_cachestat_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_cachestat_create_apps_charts, .maps = cachestat_maps,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &cachestat_config,
+ .config_file = NETDATA_CACHESTAT_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18|
+ NETDATA_V5_4 | NETDATA_V5_14 | NETDATA_V5_15 | NETDATA_V5_16,
+ .load = EBPF_LOAD_LEGACY, .targets = cachestat_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "sync", .config_name = "sync", .enabled = 0, .start_routine = ebpf_sync_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &sync_config,
+ .config_file = NETDATA_SYNC_CONFIG_FILE,
+ // All syscalls have the same kernels
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = sync_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "dc", .config_name = "dc", .enabled = 0, .start_routine = ebpf_dcstat_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_dcstat_create_apps_charts, .maps = dcstat_maps,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &dcstat_config,
+ .config_file = NETDATA_DIRECTORY_DCSTAT_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = dc_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "swap", .config_name = "swap", .enabled = 0, .start_routine = ebpf_swap_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_swap_create_apps_charts, .maps = NULL,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &swap_config,
+ .config_file = NETDATA_DIRECTORY_SWAP_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = swap_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "vfs", .config_name = "vfs", .enabled = 0, .start_routine = ebpf_vfs_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_vfs_create_apps_charts, .maps = NULL,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &vfs_config,
+ .config_file = NETDATA_DIRECTORY_VFS_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = vfs_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "filesystem", .config_name = "filesystem", .enabled = 0, .start_routine = ebpf_filesystem_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &fs_config,
+ .config_file = NETDATA_FILESYSTEM_CONFIG_FILE,
+ //We are setting kernels as zero, because we load eBPF programs according the kernel running.
+ .kernels = 0, .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL },
+ { .thread_name = "disk", .config_name = "disk", .enabled = 0, .start_routine = ebpf_disk_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &disk_config,
+ .config_file = NETDATA_DISK_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "mount", .config_name = "mount", .enabled = 0, .start_routine = ebpf_mount_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &mount_config,
+ .config_file = NETDATA_MOUNT_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = mount_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "fd", .config_name = "fd", .enabled = 0, .start_routine = ebpf_fd_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_fd_create_apps_charts, .maps = NULL,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &fd_config,
+ .config_file = NETDATA_FD_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_11 |
+ NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = fd_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "hardirq", .config_name = "hardirq", .enabled = 0, .start_routine = ebpf_hardirq_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &hardirq_config,
+ .config_file = NETDATA_HARDIRQ_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "softirq", .config_name = "softirq", .enabled = 0, .start_routine = ebpf_softirq_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &softirq_config,
+ .config_file = NETDATA_SOFTIRQ_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "oomkill", .config_name = "oomkill", .enabled = 0, .start_routine = ebpf_oomkill_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_oomkill_create_apps_charts, .maps = NULL,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &oomkill_config,
+ .config_file = NETDATA_OOMKILL_CONFIG_FILE,
+ .kernels = NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "shm", .config_name = "shm", .enabled = 0, .start_routine = ebpf_shm_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = ebpf_shm_create_apps_charts, .maps = NULL,
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &shm_config,
+ .config_file = NETDATA_DIRECTORY_SHM_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = shm_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = "mdflush", .config_name = "mdflush", .enabled = 0, .start_routine = ebpf_mdflush_thread,
+ .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
+ .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
+ .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &mdflush_config,
+ .config_file = NETDATA_DIRECTORY_MDFLUSH_CONFIG_FILE,
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
+ .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
+ { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_every = EBPF_DEFAULT_UPDATE_EVERY,
+ .global_charts = 0, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET,
+ .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL,
+ .pid_map_size = 0, .names = NULL, .cfg = NULL, .config_name = NULL, .kernels = 0, .load = EBPF_LOAD_LEGACY,
+ .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
+};
+
+struct netdata_static_thread ebpf_threads[] = {
+ {
+ .name = "EBPF PROCESS",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF SOCKET",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF CACHESTAT",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF SYNC",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF DCSTAT",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF SWAP",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF VFS",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF FILESYSTEM",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF DISK",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF MOUNT",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF FD",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF HARDIRQ",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF SOFTIRQ",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF OOMKILL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF SHM",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = "EBPF MDFLUSH",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+ {
+ .name = NULL,
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 0,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+ },
+};
+
+ebpf_filesystem_partitions_t localfs[] =
+ {{.filesystem = "ext4",
+ .optional_filesystem = NULL,
+ .family = "ext4",
+ .objects = NULL,
+ .probe_links = NULL,
+ .flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION,
+ .enabled = CONFIG_BOOLEAN_YES,
+ .addresses = {.function = NULL, .addr = 0},
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4},
+ {.filesystem = "xfs",
+ .optional_filesystem = NULL,
+ .family = "xfs",
+ .objects = NULL,
+ .probe_links = NULL,
+ .flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION,
+ .enabled = CONFIG_BOOLEAN_YES,
+ .addresses = {.function = NULL, .addr = 0},
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4},
+ {.filesystem = "nfs",
+ .optional_filesystem = "nfs4",
+ .family = "nfs",
+ .objects = NULL,
+ .probe_links = NULL,
+ .flags = NETDATA_FILESYSTEM_ATTR_CHARTS,
+ .enabled = CONFIG_BOOLEAN_YES,
+ .addresses = {.function = NULL, .addr = 0},
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4},
+ {.filesystem = "zfs",
+ .optional_filesystem = NULL,
+ .family = "zfs",
+ .objects = NULL,
+ .probe_links = NULL,
+ .flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION,
+ .enabled = CONFIG_BOOLEAN_YES,
+ .addresses = {.function = NULL, .addr = 0},
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4},
+ {.filesystem = "btrfs",
+ .optional_filesystem = NULL,
+ .family = "btrfs",
+ .objects = NULL,
+ .probe_links = NULL,
+ .flags = NETDATA_FILESYSTEM_FILL_ADDRESS_TABLE,
+ .enabled = CONFIG_BOOLEAN_YES,
+ .addresses = {.function = "btrfs_file_operations", .addr = 0},
+ .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_10},
+ {.filesystem = NULL,
+ .optional_filesystem = NULL,
+ .family = NULL,
+ .objects = NULL,
+ .probe_links = NULL,
+ .flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION,
+ .enabled = CONFIG_BOOLEAN_YES,
+ .addresses = {.function = NULL, .addr = 0},
+ .kernels = 0}};
+
+ebpf_sync_syscalls_t local_syscalls[] = {
+ {.syscall = NETDATA_SYSCALLS_SYNC, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL,
+#ifdef LIBBPF_MAJOR_VERSION
+ .sync_obj = NULL
+#endif
+ },
+ {.syscall = NETDATA_SYSCALLS_SYNCFS, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL,
+#ifdef LIBBPF_MAJOR_VERSION
+ .sync_obj = NULL
+#endif
+ },
+ {.syscall = NETDATA_SYSCALLS_MSYNC, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL,
+#ifdef LIBBPF_MAJOR_VERSION
+ .sync_obj = NULL
+#endif
+ },
+ {.syscall = NETDATA_SYSCALLS_FSYNC, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL,
+#ifdef LIBBPF_MAJOR_VERSION
+ .sync_obj = NULL
+#endif
+ },
+ {.syscall = NETDATA_SYSCALLS_FDATASYNC, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL,
+#ifdef LIBBPF_MAJOR_VERSION
+ .sync_obj = NULL
+#endif
+ },
+ {.syscall = NETDATA_SYSCALLS_SYNC_FILE_RANGE, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL,
+#ifdef LIBBPF_MAJOR_VERSION
+ .sync_obj = NULL
+#endif
+ },
+ {.syscall = NULL, .enabled = CONFIG_BOOLEAN_NO, .objects = NULL, .probe_links = NULL,
+#ifdef LIBBPF_MAJOR_VERSION
+ .sync_obj = NULL
+#endif
+ }
+};
+
+
+// Link with apps.plugin
+ebpf_process_stat_t *global_process_stat = NULL;
+
+// Link with cgroup.plugin
+netdata_ebpf_cgroup_shm_t shm_ebpf_cgroup = {NULL, NULL};
+int shm_fd_ebpf_cgroup = -1;
+sem_t *shm_sem_ebpf_cgroup = SEM_FAILED;
+pthread_mutex_t mutex_cgroup_shm;
+
+//Network viewer
+ebpf_network_viewer_options_t network_viewer_opt;
+
+// Statistic
+ebpf_plugin_stats_t plugin_statistics = {.core = 0, .legacy = 0, .running = 0, .threads = 0, .tracepoints = 0,
+ .probes = 0, .retprobes = 0, .trampolines = 0};
+
+#ifdef LIBBPF_MAJOR_VERSION
+struct btf *default_btf = NULL;
+#else
+void *default_btf = NULL;
+#endif
+char *btf_path = NULL;
+
+/*****************************************************************
+ *
+ * FUNCTIONS USED TO CLEAN MEMORY AND OPERATE SYSTEM FILES
+ *
+ *****************************************************************/
+
+/**
+ * Close the collector gracefully
+ */
+static void ebpf_exit()
+{
+#ifdef LIBBPF_MAJOR_VERSION
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (default_btf) {
+ btf__free(default_btf);
+ default_btf = NULL;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+#endif
+
+ char filename[FILENAME_MAX + 1];
+ ebpf_pid_file(filename, FILENAME_MAX);
+ if (unlink(filename))
+ error("Cannot remove PID file %s", filename);
+
+ exit(0);
+}
+
+/**
+ * Unload loegacy code
+ *
+ * @param objects objects loaded from eBPF programs
+ * @param probe_links links from loader
+ */
+static void ebpf_unload_legacy_code(struct bpf_object *objects, struct bpf_link **probe_links)
+{
+ if (!probe_links || !objects)
+ return;
+
+ struct bpf_program *prog;
+ size_t j = 0 ;
+ bpf_object__for_each_program(prog, objects) {
+ bpf_link__destroy(probe_links[j]);
+ j++;
+ }
+ freez(probe_links);
+ if (objects)
+ bpf_object__close(objects);
+}
+
+int ebpf_exit_plugin = 0;
+/**
+ * Close the collector gracefully
+ *
+ * @param sig is the signal number used to close the collector
+ */
+static void ebpf_stop_threads(int sig)
+{
+ UNUSED(sig);
+ static int only_one = 0;
+
+ int i;
+ // Child thread should be closed by itself.
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (main_thread_id != gettid() || only_one) {
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ only_one = 1;
+ for (i = 0; ebpf_threads[i].name != NULL; i++) {
+ if (ebpf_threads[i].enabled != NETDATA_THREAD_EBPF_STOPPED)
+ netdata_thread_cancel(*ebpf_threads[i].thread);
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ ebpf_exit_plugin = 1;
+ usec_t max = 3 * USEC_PER_SEC, step = 100000;
+ while (i && max) {
+ max -= step;
+ sleep_usec(step);
+ i = 0;
+ int j;
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ for (j = 0; ebpf_threads[j].name != NULL; j++) {
+ if (ebpf_threads[j].enabled != NETDATA_THREAD_EBPF_STOPPED)
+ i++;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ }
+
+ //Unload threads(except sync and filesystem)
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ for (i = 0; ebpf_threads[i].name != NULL; i++) {
+ if (ebpf_threads[i].enabled == NETDATA_THREAD_EBPF_STOPPED && i != EBPF_MODULE_FILESYSTEM_IDX &&
+ i != EBPF_MODULE_SYNC_IDX)
+ ebpf_unload_legacy_code(ebpf_modules[i].objects, ebpf_modules[i].probe_links);
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ //Unload filesystem
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (ebpf_threads[EBPF_MODULE_FILESYSTEM_IDX].enabled == NETDATA_THREAD_EBPF_STOPPED) {
+ for (i = 0; localfs[i].filesystem != NULL; i++) {
+ ebpf_unload_legacy_code(localfs[i].objects, localfs[i].probe_links);
+ }
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ //Unload Sync
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (ebpf_threads[EBPF_MODULE_SYNC_IDX].enabled == NETDATA_THREAD_EBPF_STOPPED) {
+ for (i = 0; local_syscalls[i].syscall != NULL; i++) {
+ ebpf_unload_legacy_code(local_syscalls[i].objects, local_syscalls[i].probe_links);
+ }
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ ebpf_exit();
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CREATE CHARTS
+ *
+ *****************************************************************/
+
+/**
+ * Get a value from a structure.
+ *
+ * @param basis it is the first address of the structure
+ * @param offset it is the offset of the data you want to access.
+ * @return
+ */
+collected_number get_value_from_structure(char *basis, size_t offset)
+{
+ collected_number *value = (collected_number *)(basis + offset);
+
+ collected_number ret = (collected_number)llabs(*value);
+ // this reset is necessary to avoid keep a constant value while processing is not executing a task
+ *value = 0;
+
+ return ret;
+}
+
+/**
+ * Write begin command on standard output
+ *
+ * @param family the chart family name
+ * @param name the chart name
+ */
+void write_begin_chart(char *family, char *name)
+{
+ printf("BEGIN %s.%s\n", family, name);
+}
+
+/**
+ * Write END command on stdout.
+ */
+inline void write_end_chart()
+{
+ printf("END\n");
+}
+
+/**
+ * Write set command on standard output
+ *
+ * @param dim the dimension name
+ * @param value the value for the dimension
+ */
+void write_chart_dimension(char *dim, long long value)
+{
+ printf("SET %s = %lld\n", dim, value);
+}
+
+/**
+ * Call the necessary functions to create a chart.
+ *
+ * @param name the chart name
+ * @param family the chart family
+ * @param move the pointer with the values that will be published
+ * @param end the number of values that will be written on standard output
+ *
+ * @return It returns a variable that maps the charts that did not have zero values.
+ */
+void write_count_chart(char *name, char *family, netdata_publish_syscall_t *move, uint32_t end)
+{
+ write_begin_chart(family, name);
+
+ uint32_t i = 0;
+ while (move && i < end) {
+ write_chart_dimension(move->name, move->ncall);
+
+ move = move->next;
+ i++;
+ }
+
+ write_end_chart();
+}
+
+/**
+ * Call the necessary functions to create a chart.
+ *
+ * @param name the chart name
+ * @param family the chart family
+ * @param move the pointer with the values that will be published
+ * @param end the number of values that will be written on standard output
+ */
+void write_err_chart(char *name, char *family, netdata_publish_syscall_t *move, int end)
+{
+ write_begin_chart(family, name);
+
+ int i = 0;
+ while (move && i < end) {
+ write_chart_dimension(move->name, move->nerr);
+
+ move = move->next;
+ i++;
+ }
+
+ write_end_chart();
+}
+
+/**
+ * Write charts
+ *
+ * Write the current information to publish the charts.
+ *
+ * @param family chart family
+ * @param chart chart id
+ * @param dim dimension name
+ * @param v1 value.
+ */
+void ebpf_one_dimension_write_charts(char *family, char *chart, char *dim, long long v1)
+{
+ write_begin_chart(family, chart);
+
+ write_chart_dimension(dim, v1);
+
+ write_end_chart();
+}
+
+/**
+ * Call the necessary functions to create a chart.
+ *
+ * @param chart the chart name
+ * @param family the chart family
+ * @param dwrite the dimension name
+ * @param vwrite the value for previous dimension
+ * @param dread the dimension name
+ * @param vread the value for previous dimension
+ *
+ * @return It returns a variable that maps the charts that did not have zero values.
+ */
+void write_io_chart(char *chart, char *family, char *dwrite, long long vwrite, char *dread, long long vread)
+{
+ write_begin_chart(family, chart);
+
+ write_chart_dimension(dwrite, vwrite);
+ write_chart_dimension(dread, vread);
+
+ write_end_chart();
+}
+
+/**
+ * Write chart cmd on standard output
+ *
+ * @param type chart type
+ * @param id chart id
+ * @param title chart title
+ * @param units units label
+ * @param family group name used to attach the chart on dashboard
+ * @param charttype chart type
+ * @param context chart context
+ * @param order chart order
+ * @param update_every update interval used by plugin
+ * @param module chart module name, this is the eBPF thread.
+ */
+void ebpf_write_chart_cmd(char *type, char *id, char *title, char *units, char *family,
+ char *charttype, char *context, int order, int update_every, char *module)
+{
+ printf("CHART %s.%s '' '%s' '%s' '%s' '%s' '%s' %d %d '' 'ebpf.plugin' '%s'\n",
+ type,
+ id,
+ title,
+ units,
+ (family)?family:"",
+ (context)?context:"",
+ (charttype)?charttype:"",
+ order,
+ update_every,
+ module);
+}
+
+/**
+ * Write chart cmd on standard output
+ *
+ * @param type chart type
+ * @param id chart id
+ * @param title chart title
+ * @param units units label
+ * @param family group name used to attach the chart on dashboard
+ * @param charttype chart type
+ * @param context chart context
+ * @param order chart order
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+void ebpf_write_chart_obsolete(char *type, char *id, char *title, char *units, char *family,
+ char *charttype, char *context, int order, int update_every)
+{
+ printf("CHART %s.%s '' '%s' '%s' '%s' '%s' '%s' %d %d 'obsolete'\n",
+ type,
+ id,
+ title,
+ units,
+ (family)?family:"",
+ (context)?context:"",
+ (charttype)?charttype:"",
+ order,
+ update_every);
+}
+
+/**
+ * Write the dimension command on standard output
+ *
+ * @param name the dimension name
+ * @param id the dimension id
+ * @param algo the dimension algorithm
+ */
+void ebpf_write_global_dimension(char *name, char *id, char *algorithm)
+{
+ printf("DIMENSION %s %s %s 1 1\n", name, id, algorithm);
+}
+
+/**
+ * Call ebpf_write_global_dimension to create the dimensions for a specific chart
+ *
+ * @param ptr a pointer to a structure of the type netdata_publish_syscall_t
+ * @param end the number of dimensions for the structure ptr
+ */
+void ebpf_create_global_dimension(void *ptr, int end)
+{
+ netdata_publish_syscall_t *move = ptr;
+
+ int i = 0;
+ while (move && i < end) {
+ ebpf_write_global_dimension(move->name, move->dimension, move->algorithm);
+
+ move = move->next;
+ i++;
+ }
+}
+
+/**
+ * Call write_chart_cmd to create the charts
+ *
+ * @param type chart type
+ * @param id chart id
+ * @param title chart title
+ * @param units axis label
+ * @param family group name used to attach the chart on dashboard
+ * @param context chart context
+ * @param charttype chart type
+ * @param order order number of the specified chart
+ * @param ncd a pointer to a function called to create dimensions
+ * @param move a pointer for a structure that has the dimensions
+ * @param end number of dimensions for the chart created
+ * @param update_every update interval used with chart.
+ * @param module chart module name, this is the eBPF thread.
+ */
+void ebpf_create_chart(char *type,
+ char *id,
+ char *title,
+ char *units,
+ char *family,
+ char *context,
+ char *charttype,
+ int order,
+ void (*ncd)(void *, int),
+ void *move,
+ int end,
+ int update_every,
+ char *module)
+{
+ ebpf_write_chart_cmd(type, id, title, units, family, charttype, context, order, update_every, module);
+
+ if (ncd) {
+ ncd(move, end);
+ }
+}
+
+/**
+ * Create charts on apps submenu
+ *
+ * @param id the chart id
+ * @param title the value displayed on vertical axis.
+ * @param units the value displayed on vertical axis.
+ * @param family Submenu that the chart will be attached on dashboard.
+ * @param charttype chart type
+ * @param order the chart order
+ * @param algorithm the algorithm used by dimension
+ * @param root structure used to create the dimensions.
+ * @param update_every update interval used by plugin
+ * @param module chart module name, this is the eBPF thread.
+ */
+void ebpf_create_charts_on_apps(char *id, char *title, char *units, char *family, char *charttype, int order,
+ char *algorithm, struct target *root, int update_every, char *module)
+{
+ struct target *w;
+ ebpf_write_chart_cmd(NETDATA_APPS_FAMILY, id, title, units, family, charttype, NULL, order,
+ update_every, module);
+
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed))
+ fprintf(stdout, "DIMENSION %s '' %s 1 1\n", w->name, algorithm);
+ }
+}
+
+/**
+ * Call the necessary functions to create a name.
+ *
+ * @param family family name
+ * @param name chart name
+ * @param hist0 histogram values
+ * @param dimensions dimension values.
+ * @param end number of bins that will be sent to Netdata.
+ *
+ * @return It returns a variable that maps the charts that did not have zero values.
+ */
+void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, char **dimensions, uint32_t end)
+{
+ write_begin_chart(family, name);
+
+ uint32_t i;
+ for (i = 0; i < end; i++) {
+ write_chart_dimension(dimensions[i], (long long) hist[i]);
+ }
+
+ write_end_chart();
+
+ fflush(stdout);
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO DEFINE OPTIONS
+ *
+ *****************************************************************/
+
+/**
+ * Define labels used to generate charts
+ *
+ * @param is structure with information about number of calls made for a function.
+ * @param pio structure used to generate charts.
+ * @param dim a pointer for the dimensions name
+ * @param name a pointer for the tensor with the name of the functions.
+ * @param algorithm a vector with the algorithms used to make the charts
+ * @param end the number of elements in the previous 4 arguments.
+ */
+void ebpf_global_labels(netdata_syscall_stat_t *is, netdata_publish_syscall_t *pio, char **dim,
+ char **name, int *algorithm, int end)
+{
+ int i;
+
+ netdata_syscall_stat_t *prev = NULL;
+ netdata_publish_syscall_t *publish_prev = NULL;
+ for (i = 0; i < end; i++) {
+ if (prev) {
+ prev->next = &is[i];
+ }
+ prev = &is[i];
+
+ pio[i].dimension = dim[i];
+ pio[i].name = name[i];
+ pio[i].algorithm = strdupz(ebpf_algorithms[algorithm[i]]);
+ if (publish_prev) {
+ publish_prev->next = &pio[i];
+ }
+ publish_prev = &pio[i];
+ }
+}
+
+/**
+ * Define thread mode for all ebpf program.
+ *
+ * @param lmode the mode that will be used for them.
+ */
+static inline void ebpf_set_thread_mode(netdata_run_mode_t lmode)
+{
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_modules[i].mode = lmode;
+ }
+}
+
+/**
+ * Enable specific charts selected by user.
+ *
+ * @param em the structure that will be changed
+ * @param disable_apps the status about the apps charts.
+ * @param disable_cgroup the status about the cgroups charts.
+ */
+static inline void ebpf_enable_specific_chart(struct ebpf_module *em, int disable_apps, int disable_cgroup)
+{
+ em->enabled = CONFIG_BOOLEAN_YES;
+
+ // oomkill stores data inside apps submenu, so it always need to have apps_enabled for plugin to create
+ // its chart, without this comparison eBPF.plugin will try to store invalid data when apps is disabled.
+ if (!disable_apps || !strcmp(em->thread_name, "oomkill")) {
+ em->apps_charts = NETDATA_EBPF_APPS_FLAG_YES;
+ }
+
+ if (!disable_cgroup) {
+ em->cgroup_charts = CONFIG_BOOLEAN_YES;
+ }
+
+ em->global_charts = CONFIG_BOOLEAN_YES;
+}
+
+/**
+ * Enable all charts
+ *
+ * @param apps what is the current status of apps
+ * @param cgroups what is the current status of cgroups
+ */
+static inline void ebpf_enable_all_charts(int apps, int cgroups)
+{
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_enable_specific_chart(&ebpf_modules[i], apps, cgroups);
+ }
+}
+
+/**
+ * Disable all Global charts
+ *
+ * Disable charts
+ */
+static inline void disable_all_global_charts()
+{
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_modules[i].enabled = 0;
+ ebpf_modules[i].global_charts = 0;
+ }
+}
+
+
+/**
+ * Enable the specified chart group
+ *
+ * @param idx the index of ebpf_modules that I am enabling
+ * @param disable_apps should I keep apps charts?
+ */
+static inline void ebpf_enable_chart(int idx, int disable_apps, int disable_cgroup)
+{
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ if (i == idx) {
+ ebpf_enable_specific_chart(&ebpf_modules[i], disable_apps, disable_cgroup);
+ break;
+ }
+ }
+}
+
+/**
+ * Disable APPs
+ *
+ * Disable charts for apps loading only global charts.
+ */
+static inline void ebpf_disable_apps()
+{
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_modules[i].apps_charts = NETDATA_EBPF_APPS_FLAG_NO;
+ }
+}
+
+/**
+ * Disable Cgroups
+ *
+ * Disable charts for apps loading only global charts.
+ */
+static inline void ebpf_disable_cgroups()
+{
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_modules[i].cgroup_charts = 0;
+ }
+}
+
+/**
+ * Update Disabled Plugins
+ *
+ * This function calls ebpf_update_stats to update statistics for collector.
+ *
+ * @param em a pointer to `struct ebpf_module`
+ */
+void ebpf_update_disabled_plugin_stats(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&lock);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+}
+
+/**
+ * Print help on standard error for user knows how to use the collector.
+ */
+void ebpf_print_help()
+{
+ const time_t t = time(NULL);
+ struct tm ct;
+ struct tm *test = localtime_r(&t, &ct);
+ int year;
+ if (test)
+ year = ct.tm_year;
+ else
+ year = 0;
+
+ fprintf(stderr,
+ "\n"
+ " Netdata ebpf.plugin %s\n"
+ " Copyright (C) 2016-%d Costa Tsaousis <costa@tsaousis.gr>\n"
+ " Released under GNU General Public License v3 or later.\n"
+ " All rights reserved.\n"
+ "\n"
+ " This eBPF.plugin is a data collector plugin for netdata.\n"
+ "\n"
+ " This plugin only accepts long options with one or two dashes. The available command line options are:\n"
+ "\n"
+ " SECONDS Set the data collection frequency.\n"
+ "\n"
+ " [-]-help Show this help.\n"
+ "\n"
+ " [-]-version Show software version.\n"
+ "\n"
+ " [-]-global Disable charts per application and cgroup.\n"
+ "\n"
+ " [-]-all Enable all chart groups (global, apps, and cgroup), unless -g is also given.\n"
+ "\n"
+ " [-]-cachestat Enable charts related to process run time.\n"
+ "\n"
+ " [-]-dcstat Enable charts related to directory cache.\n"
+ "\n"
+ " [-]-disk Enable charts related to disk monitoring.\n"
+ "\n"
+ " [-]-filesystem Enable chart related to filesystem run time.\n"
+ "\n"
+ " [-]-hardirq Enable chart related to hard IRQ latency.\n"
+ "\n"
+ " [-]-mdflush Enable charts related to multi-device flush.\n"
+ "\n"
+ " [-]-mount Enable charts related to mount monitoring.\n"
+ "\n"
+ " [-]-net Enable network viewer charts.\n"
+ "\n"
+ " [-]-oomkill Enable chart related to OOM kill tracking.\n"
+ "\n"
+ " [-]-process Enable charts related to process run time.\n"
+ "\n"
+ " [-]-return Run the collector in return mode.\n"
+ "\n"
+ " [-]-shm Enable chart related to shared memory tracking.\n"
+ "\n"
+ " [-]-softirq Enable chart related to soft IRQ latency.\n"
+ "\n"
+ " [-]-sync Enable chart related to sync run time.\n"
+ "\n"
+ " [-]-swap Enable chart related to swap run time.\n"
+ "\n"
+ " [-]-vfs Enable chart related to vfs run time.\n"
+ "\n"
+ " [-]-legacy Load legacy eBPF programs.\n"
+ "\n"
+ " [-]-core Use CO-RE when available(Working in progress).\n"
+ "\n",
+ VERSION,
+ (year >= 116) ? year + 1900 : 2020);
+}
+
+/*****************************************************************
+ *
+ * TRACEPOINT MANAGEMENT FUNCTIONS
+ *
+ *****************************************************************/
+
+/**
+ * Enable a tracepoint.
+ *
+ * @return 0 on success, -1 on error.
+ */
+int ebpf_enable_tracepoint(ebpf_tracepoint_t *tp)
+{
+ int test = ebpf_is_tracepoint_enabled(tp->class, tp->event);
+
+ // err?
+ if (test == -1) {
+ return -1;
+ }
+ // disabled?
+ else if (test == 0) {
+ // enable it then.
+ if (ebpf_enable_tracing_values(tp->class, tp->event)) {
+ return -1;
+ }
+ }
+
+ // enabled now or already was.
+ tp->enabled = true;
+
+ return 0;
+}
+
+/**
+ * Disable a tracepoint if it's enabled.
+ *
+ * @return 0 on success, -1 on error.
+ */
+int ebpf_disable_tracepoint(ebpf_tracepoint_t *tp)
+{
+ int test = ebpf_is_tracepoint_enabled(tp->class, tp->event);
+
+ // err?
+ if (test == -1) {
+ return -1;
+ }
+ // enabled?
+ else if (test == 1) {
+ // disable it then.
+ if (ebpf_disable_tracing_values(tp->class, tp->event)) {
+ return -1;
+ }
+ }
+
+ // disable now or already was.
+ tp->enabled = false;
+
+ return 0;
+}
+
+/**
+ * Enable multiple tracepoints on a list of tracepoints which end when the
+ * class is NULL.
+ *
+ * @return the number of successful enables.
+ */
+uint32_t ebpf_enable_tracepoints(ebpf_tracepoint_t *tps)
+{
+ uint32_t cnt = 0;
+ for (int i = 0; tps[i].class != NULL; i++) {
+ if (ebpf_enable_tracepoint(&tps[i]) == -1) {
+ infoerr("failed to enable tracepoint %s:%s",
+ tps[i].class, tps[i].event);
+ }
+ else {
+ cnt += 1;
+ }
+ }
+ return cnt;
+}
+
+/*****************************************************************
+ *
+ * AUXILIARY FUNCTIONS USED DURING INITIALIZATION
+ *
+ *****************************************************************/
+
+/**
+ * Read Local Ports
+ *
+ * Parse /proc/net/{tcp,udp} and get the ports Linux is listening.
+ *
+ * @param filename the proc file to parse.
+ * @param proto is the magic number associated to the protocol file we are reading.
+ */
+static void read_local_ports(char *filename, uint8_t proto)
+{
+ procfile *ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if (!ff)
+ return;
+
+ ff = procfile_readall(ff);
+ if (!ff)
+ return;
+
+ size_t lines = procfile_lines(ff), l;
+ netdata_passive_connection_t values = {.counter = 0, .tgid = 0, .pid = 0};
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ // This is header or end of file
+ if (unlikely(words < 14))
+ continue;
+
+ // https://elixir.bootlin.com/linux/v5.7.8/source/include/net/tcp_states.h
+ // 0A = TCP_LISTEN
+ if (strcmp("0A", procfile_lineword(ff, l, 5)))
+ continue;
+
+ // Read local port
+ uint16_t port = (uint16_t)strtol(procfile_lineword(ff, l, 2), NULL, 16);
+ update_listen_table(htons(port), proto, &values);
+ }
+
+ procfile_close(ff);
+}
+
+/**
+ * Read Local addresseses
+ *
+ * Read the local address from the interfaces.
+ */
+static void read_local_addresses()
+{
+ struct ifaddrs *ifaddr, *ifa;
+ if (getifaddrs(&ifaddr) == -1) {
+ error("Cannot get the local IP addresses, it is no possible to do separation between inbound and outbound connections");
+ return;
+ }
+
+ char *notext = { "No text representation" };
+ for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr == NULL)
+ continue;
+
+ if ((ifa->ifa_addr->sa_family != AF_INET) && (ifa->ifa_addr->sa_family != AF_INET6))
+ continue;
+
+ ebpf_network_viewer_ip_list_t *w = callocz(1, sizeof(ebpf_network_viewer_ip_list_t));
+
+ int family = ifa->ifa_addr->sa_family;
+ w->ver = (uint8_t) family;
+ char text[INET6_ADDRSTRLEN];
+ if (family == AF_INET) {
+ struct sockaddr_in *in = (struct sockaddr_in*) ifa->ifa_addr;
+
+ w->first.addr32[0] = in->sin_addr.s_addr;
+ w->last.addr32[0] = in->sin_addr.s_addr;
+
+ if (inet_ntop(AF_INET, w->first.addr8, text, INET_ADDRSTRLEN)) {
+ w->value = strdupz(text);
+ w->hash = simple_hash(text);
+ } else {
+ w->value = strdupz(notext);
+ w->hash = simple_hash(notext);
+ }
+ } else {
+ struct sockaddr_in6 *in6 = (struct sockaddr_in6*) ifa->ifa_addr;
+
+ memcpy(w->first.addr8, (void *)&in6->sin6_addr, sizeof(struct in6_addr));
+ memcpy(w->last.addr8, (void *)&in6->sin6_addr, sizeof(struct in6_addr));
+
+ if (inet_ntop(AF_INET6, w->first.addr8, text, INET_ADDRSTRLEN)) {
+ w->value = strdupz(text);
+ w->hash = simple_hash(text);
+ } else {
+ w->value = strdupz(notext);
+ w->hash = simple_hash(notext);
+ }
+ }
+
+ fill_ip_list((family == AF_INET)?&network_viewer_opt.ipv4_local_ip:&network_viewer_opt.ipv6_local_ip,
+ w,
+ "selector");
+ }
+
+ freeifaddrs(ifaddr);
+}
+
+/**
+ * Start Pthread Variable
+ *
+ * This function starts all pthread variables.
+ *
+ * @return It returns 0 on success and -1.
+ */
+int ebpf_start_pthread_variables()
+{
+ pthread_mutex_init(&lock, NULL);
+ pthread_mutex_init(&ebpf_exit_cleanup, NULL);
+ pthread_mutex_init(&collect_data_mutex, NULL);
+
+ if (pthread_cond_init(&collect_data_cond_var, NULL)) {
+ error("Cannot start conditional variable to control Apps charts.");
+ return -1;
+ }
+
+ return 0;
+}
+
+/**
+ * Am I collecting PIDs?
+ *
+ * Test if eBPF plugin needs to collect PID information.
+ *
+ * @return It returns 1 if at least one thread needs to collect the data, or zero otherwise.
+ */
+static inline uint32_t ebpf_am_i_collect_pids()
+{
+ uint32_t ret = 0;
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ret |= ebpf_modules[i].cgroup_charts | (ebpf_modules[i].apps_charts & NETDATA_EBPF_APPS_FLAG_YES);
+ }
+
+ return ret;
+}
+
+/**
+ * Allocate the vectors used for all threads.
+ */
+static void ebpf_allocate_common_vectors()
+{
+ if (unlikely(!ebpf_am_i_collect_pids())) {
+ return;
+ }
+
+ all_pids = callocz((size_t)pid_max, sizeof(struct pid_stat *));
+ global_process_stat = callocz((size_t)ebpf_nprocs, sizeof(ebpf_process_stat_t));
+}
+
+/**
+ * Define how to load the ebpf programs
+ *
+ * @param ptr the option given by users
+ */
+static inline void how_to_load(char *ptr)
+{
+ if (!strcasecmp(ptr, EBPF_CFG_LOAD_MODE_RETURN))
+ ebpf_set_thread_mode(MODE_RETURN);
+ else if (!strcasecmp(ptr, EBPF_CFG_LOAD_MODE_DEFAULT))
+ ebpf_set_thread_mode(MODE_ENTRY);
+ else
+ error("the option %s for \"ebpf load mode\" is not a valid option.", ptr);
+}
+
+/**
+ * Update interval
+ *
+ * Update default interval with value from user
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_update_interval(int update_every)
+{
+ int i;
+ int value = (int) appconfig_get_number(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_UPDATE_EVERY,
+ update_every);
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_modules[i].update_every = value;
+ }
+}
+
+/**
+ * Update PID table size
+ *
+ * Update default size with value from user
+ */
+static void ebpf_update_table_size()
+{
+ int i;
+ uint32_t value = (uint32_t) appconfig_get_number(&collector_config, EBPF_GLOBAL_SECTION,
+ EBPF_CFG_PID_SIZE, ND_EBPF_DEFAULT_PID_SIZE);
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_modules[i].pid_map_size = value;
+ }
+}
+
+/**
+ * Set Load mode
+ *
+ * @param origin specify the configuration file loaded
+ */
+static inline void ebpf_set_load_mode(netdata_ebpf_load_mode_t load, netdata_ebpf_load_mode_t origin)
+{
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_modules[i].load &= ~NETDATA_EBPF_LOAD_METHODS;
+ ebpf_modules[i].load |= load | origin ;
+ }
+}
+
+/**
+ * Update mode
+ *
+ * @param str value read from configuration file.
+ * @param origin specify the configuration file loaded
+ */
+static inline void epbf_update_load_mode(char *str, netdata_ebpf_load_mode_t origin)
+{
+ netdata_ebpf_load_mode_t load = epbf_convert_string_to_load_mode(str);
+
+ ebpf_set_load_mode(load, origin);
+}
+
+/**
+ * Read collector values
+ *
+ * @param disable_apps variable to store information related to apps.
+ * @param disable_cgroups variable to store information related to cgroups.
+ * @param update_every value to overwrite the update frequency set by the server.
+ * @param origin specify the configuration file loaded
+ */
+static void read_collector_values(int *disable_apps, int *disable_cgroups,
+ int update_every, netdata_ebpf_load_mode_t origin)
+{
+ // Read global section
+ char *value;
+ if (appconfig_exists(&collector_config, EBPF_GLOBAL_SECTION, "load")) // Backward compatibility
+ value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, "load",
+ EBPF_CFG_LOAD_MODE_DEFAULT);
+ else
+ value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_LOAD_MODE,
+ EBPF_CFG_LOAD_MODE_DEFAULT);
+
+ how_to_load(value);
+
+ btf_path = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_PROGRAM_PATH,
+ EBPF_DEFAULT_BTF_PATH);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ default_btf = ebpf_load_btf_file(btf_path, EBPF_DEFAULT_BTF_FILE);
+#endif
+
+ value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_TYPE_FORMAT, EBPF_CFG_DEFAULT_PROGRAM);
+
+ epbf_update_load_mode(value, origin);
+
+ ebpf_update_interval(update_every);
+
+ ebpf_update_table_size();
+
+ // This is kept to keep compatibility
+ uint32_t enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, "disable apps",
+ CONFIG_BOOLEAN_NO);
+ if (!enabled) {
+ // Apps is a positive sentence, so we need to invert the values to disable apps.
+ enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_APPLICATION,
+ CONFIG_BOOLEAN_YES);
+ enabled = (enabled == CONFIG_BOOLEAN_NO)?CONFIG_BOOLEAN_YES:CONFIG_BOOLEAN_NO;
+ }
+ *disable_apps = (int)enabled;
+
+ // Cgroup is a positive sentence, so we need to invert the values to disable apps.
+ // We are using the same pattern for cgroup and apps
+ enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_CGROUP, CONFIG_BOOLEAN_NO);
+ *disable_cgroups = (enabled == CONFIG_BOOLEAN_NO)?CONFIG_BOOLEAN_YES:CONFIG_BOOLEAN_NO;
+
+ // Read ebpf programs section
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION,
+ ebpf_modules[EBPF_MODULE_PROCESS_IDX].config_name, CONFIG_BOOLEAN_YES);
+ int started = 0;
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_PROCESS_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ // This is kept to keep compatibility
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "network viewer",
+ CONFIG_BOOLEAN_NO);
+ if (!enabled)
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION,
+ ebpf_modules[EBPF_MODULE_SOCKET_IDX].config_name,
+ CONFIG_BOOLEAN_NO);
+
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_SOCKET_IDX, *disable_apps, *disable_cgroups);
+ // Read network viewer section if network viewer is enabled
+ // This is kept here to keep backward compatibility
+ parse_network_viewer_section(&collector_config);
+ parse_service_name_section(&collector_config);
+ started++;
+ }
+
+ // This is kept to keep compatibility
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "network connection monitoring",
+ CONFIG_BOOLEAN_NO);
+ if (!enabled)
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "network connections",
+ CONFIG_BOOLEAN_NO);
+ ebpf_modules[EBPF_MODULE_SOCKET_IDX].optional = (int)enabled;
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "cachestat",
+ CONFIG_BOOLEAN_NO);
+
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_CACHESTAT_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "sync",
+ CONFIG_BOOLEAN_YES);
+
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_SYNC_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "dcstat",
+ CONFIG_BOOLEAN_NO);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_DCSTAT_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "swap",
+ CONFIG_BOOLEAN_NO);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_SWAP_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "vfs",
+ CONFIG_BOOLEAN_NO);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_VFS_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "filesystem",
+ CONFIG_BOOLEAN_NO);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_FILESYSTEM_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "disk",
+ CONFIG_BOOLEAN_NO);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_DISK_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "mount",
+ CONFIG_BOOLEAN_YES);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_MOUNT_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "fd",
+ CONFIG_BOOLEAN_YES);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_FD_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "hardirq",
+ CONFIG_BOOLEAN_YES);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_HARDIRQ_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "softirq",
+ CONFIG_BOOLEAN_YES);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_SOFTIRQ_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "oomkill",
+ CONFIG_BOOLEAN_YES);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_OOMKILL_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "shm",
+ CONFIG_BOOLEAN_YES);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_SHM_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "mdflush",
+ CONFIG_BOOLEAN_NO);
+ if (enabled) {
+ ebpf_enable_chart(EBPF_MODULE_MDFLUSH_IDX, *disable_apps, *disable_cgroups);
+ started++;
+ }
+
+ if (!started){
+ ebpf_enable_all_charts(*disable_apps, *disable_cgroups);
+ // Read network viewer section
+ // This is kept here to keep backward compatibility
+ parse_network_viewer_section(&collector_config);
+ parse_service_name_section(&collector_config);
+ }
+}
+
+/**
+ * Load collector config
+ *
+ * @param path the path where the file ebpf.conf is stored.
+ * @param disable_apps variable to store the information about apps plugin status.
+ * @param disable_cgroups variable to store the information about cgroups plugin status.
+ * @param update_every value to overwrite the update frequency set by the server.
+ *
+ * @return 0 on success and -1 otherwise.
+ */
+static int load_collector_config(char *path, int *disable_apps, int *disable_cgroups, int update_every)
+{
+ char lpath[4096];
+ netdata_ebpf_load_mode_t origin;
+
+ snprintf(lpath, 4095, "%s/%s", path, NETDATA_EBPF_CONFIG_FILE);
+ if (!appconfig_load(&collector_config, lpath, 0, NULL)) {
+ snprintf(lpath, 4095, "%s/%s", path, NETDATA_EBPF_OLD_CONFIG_FILE);
+ if (!appconfig_load(&collector_config, lpath, 0, NULL)) {
+ return -1;
+ }
+ origin = EBPF_LOADED_FROM_STOCK;
+ } else
+ origin = EBPF_LOADED_FROM_USER;
+
+ read_collector_values(disable_apps, disable_cgroups, update_every, origin);
+
+ return 0;
+}
+
+/**
+ * Set global variables reading environment variables
+ */
+void set_global_variables()
+{
+ // Get environment variables
+ ebpf_plugin_dir = getenv("NETDATA_PLUGINS_DIR");
+ if (!ebpf_plugin_dir)
+ ebpf_plugin_dir = PLUGINS_DIR;
+
+ ebpf_user_config_dir = getenv("NETDATA_USER_CONFIG_DIR");
+ if (!ebpf_user_config_dir)
+ ebpf_user_config_dir = CONFIG_DIR;
+
+ ebpf_stock_config_dir = getenv("NETDATA_STOCK_CONFIG_DIR");
+ if (!ebpf_stock_config_dir)
+ ebpf_stock_config_dir = LIBCONFIG_DIR;
+
+ ebpf_configured_log_dir = getenv("NETDATA_LOG_DIR");
+ if (!ebpf_configured_log_dir)
+ ebpf_configured_log_dir = LOG_DIR;
+
+ ebpf_nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN);
+ if (ebpf_nprocs > NETDATA_MAX_PROCESSOR) {
+ ebpf_nprocs = NETDATA_MAX_PROCESSOR;
+ }
+
+ isrh = get_redhat_release();
+ pid_max = get_system_pid_max();
+ running_on_kernel = ebpf_get_kernel_version();
+}
+
+/**
+ * Load collector config
+ */
+static inline void ebpf_load_thread_config()
+{
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_update_module(&ebpf_modules[i], default_btf, running_on_kernel, isrh);
+ }
+}
+
+/**
+ * Parse arguments given from user.
+ *
+ * @param argc the number of arguments
+ * @param argv the pointer to the arguments
+ */
+static void ebpf_parse_args(int argc, char **argv)
+{
+ int disable_apps = 0;
+ int disable_cgroups = 1;
+ int freq = 0;
+ int option_index = 0;
+ uint64_t select_threads = 0;
+ static struct option long_options[] = {
+ {"process", no_argument, 0, 0 },
+ {"net", no_argument, 0, 0 },
+ {"cachestat", no_argument, 0, 0 },
+ {"sync", no_argument, 0, 0 },
+ {"dcstat", no_argument, 0, 0 },
+ {"swap", no_argument, 0, 0 },
+ {"vfs", no_argument, 0, 0 },
+ {"filesystem", no_argument, 0, 0 },
+ {"disk", no_argument, 0, 0 },
+ {"mount", no_argument, 0, 0 },
+ {"filedescriptor", no_argument, 0, 0 },
+ {"hardirq", no_argument, 0, 0 },
+ {"softirq", no_argument, 0, 0 },
+ {"oomkill", no_argument, 0, 0 },
+ {"shm", no_argument, 0, 0 },
+ {"mdflush", no_argument, 0, 0 },
+ /* INSERT NEW THREADS BEFORE THIS COMMENT TO KEEP COMPATIBILITY WITH enum ebpf_module_indexes */
+ {"all", no_argument, 0, 0 },
+ {"version", no_argument, 0, 0 },
+ {"help", no_argument, 0, 0 },
+ {"global", no_argument, 0, 0 },
+ {"return", no_argument, 0, 0 },
+ {"legacy", no_argument, 0, 0 },
+ {"core", no_argument, 0, 0 },
+ {0, 0, 0, 0}
+ };
+
+ memset(&network_viewer_opt, 0, sizeof(network_viewer_opt));
+ network_viewer_opt.max_dim = NETDATA_NV_CAP_VALUE;
+
+ if (argc > 1) {
+ int n = (int)str2l(argv[1]);
+ if (n > 0) {
+ freq = n;
+ }
+ }
+
+ if (!freq)
+ freq = EBPF_DEFAULT_UPDATE_EVERY;
+
+ if (load_collector_config(ebpf_user_config_dir, &disable_apps, &disable_cgroups, freq)) {
+ info(
+ "Does not have a configuration file inside `%s/ebpf.d.conf. It will try to load stock file.",
+ ebpf_user_config_dir);
+ if (load_collector_config(ebpf_stock_config_dir, &disable_apps, &disable_cgroups, freq)) {
+ info("Does not have a stock file. It is starting with default options.");
+ }
+ }
+
+ ebpf_load_thread_config();
+
+ while (1) {
+ int c = getopt_long_only(argc, argv, "", long_options, &option_index);
+ if (c == -1)
+ break;
+
+ switch (option_index) {
+ case EBPF_MODULE_PROCESS_IDX: {
+ select_threads |= 1<<EBPF_MODULE_PROCESS_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"PROCESS\" charts, because it was started with the option \"[-]-process\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_SOCKET_IDX: {
+ select_threads |= 1<<EBPF_MODULE_SOCKET_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"NET\" charts, because it was started with the option \"[-]-net\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_CACHESTAT_IDX: {
+ select_threads |= 1<<EBPF_MODULE_CACHESTAT_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"CACHESTAT\" charts, because it was started with the option \"[-]-cachestat\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_SYNC_IDX: {
+ select_threads |= 1<<EBPF_MODULE_SYNC_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"SYNC\" chart, because it was started with the option \"[-]-sync\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_DCSTAT_IDX: {
+ select_threads |= 1<<EBPF_MODULE_DCSTAT_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"DCSTAT\" charts, because it was started with the option \"[-]-dcstat\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_SWAP_IDX: {
+ select_threads |= 1<<EBPF_MODULE_SWAP_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"SWAP\" chart, because it was started with the option \"[-]-swap\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_VFS_IDX: {
+ select_threads |= 1<<EBPF_MODULE_VFS_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"VFS\" chart, because it was started with the option \"[-]-vfs\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_FILESYSTEM_IDX: {
+ select_threads |= 1<<EBPF_MODULE_FILESYSTEM_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"FILESYSTEM\" chart, because it was started with the option \"[-]-filesystem\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_DISK_IDX: {
+ select_threads |= 1<<EBPF_MODULE_DISK_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"DISK\" chart, because it was started with the option \"[-]-disk\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_MOUNT_IDX: {
+ select_threads |= 1<<EBPF_MODULE_MOUNT_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"MOUNT\" chart, because it was started with the option \"[-]-mount\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_FD_IDX: {
+ select_threads |= 1<<EBPF_MODULE_FD_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"FILEDESCRIPTOR\" chart, because it was started with the option \"[-]-filedescriptor\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_HARDIRQ_IDX: {
+ select_threads |= 1<<EBPF_MODULE_HARDIRQ_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"HARDIRQ\" chart, because it was started with the option \"[-]-hardirq\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_SOFTIRQ_IDX: {
+ select_threads |= 1<<EBPF_MODULE_SOFTIRQ_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"SOFTIRQ\" chart, because it was started with the option \"[-]-softirq\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_OOMKILL_IDX: {
+ select_threads |= 1<<EBPF_MODULE_OOMKILL_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"OOMKILL\" chart, because it was started with the option \"[-]-oomkill\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_SHM_IDX: {
+ select_threads |= 1<<EBPF_MODULE_SHM_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"SHM\" chart, because it was started with the option \"[-]-shm\".");
+#endif
+ break;
+ }
+ case EBPF_MODULE_MDFLUSH_IDX: {
+ select_threads |= 1<<EBPF_MODULE_MDFLUSH_IDX;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF enabling \"MDFLUSH\" chart, because it was started with the option \"[-]-mdflush\".");
+#endif
+ break;
+ }
+ case EBPF_OPTION_ALL_CHARTS: {
+ disable_apps = 0;
+ disable_cgroups = 0;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF running with all chart groups, because it was started with the option \"[-]-all\".");
+#endif
+ break;
+ }
+ case EBPF_OPTION_VERSION: {
+ printf("ebpf.plugin %s\n", VERSION);
+ exit(0);
+ }
+ case EBPF_OPTION_HELP: {
+ ebpf_print_help();
+ exit(0);
+ }
+ case EBPF_OPTION_GLOBAL_CHART: {
+ disable_apps = 1;
+ disable_cgroups = 1;
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF running with global chart group, because it was started with the option \"[-]-global\".");
+#endif
+ break;
+ }
+ case EBPF_OPTION_RETURN_MODE: {
+ ebpf_set_thread_mode(MODE_RETURN);
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF running in \"RETURN\" mode, because it was started with the option \"[-]-return\".");
+#endif
+ break;
+ }
+ case EBPF_OPTION_LEGACY: {
+ ebpf_set_load_mode(EBPF_LOAD_LEGACY, EBPF_LOADED_FROM_USER);
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF running with \"LEGACY\" code, because it was started with the option \"[-]-legacy\".");
+#endif
+ break;
+ }
+ case EBPF_OPTION_CORE: {
+ ebpf_set_load_mode(EBPF_LOAD_CORE, EBPF_LOADED_FROM_USER);
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("EBPF running with \"CO-RE\" code, because it was started with the option \"[-]-core\".");
+#endif
+ break;
+ }
+ default: {
+ break;
+ }
+ }
+ }
+
+ if (disable_apps || disable_cgroups) {
+ if (disable_apps)
+ ebpf_disable_apps();
+
+ if (disable_cgroups)
+ ebpf_disable_cgroups();
+ }
+
+ if (select_threads) {
+ disable_all_global_charts();
+ uint64_t idx;
+ for (idx = 0; idx < EBPF_OPTION_ALL_CHARTS; idx++) {
+ if (select_threads & 1<<idx)
+ ebpf_enable_specific_chart(&ebpf_modules[idx], disable_apps, disable_cgroups);
+ }
+ }
+
+ // Load apps_groups.conf
+ if (ebpf_read_apps_groups_conf(
+ &apps_groups_default_target, &apps_groups_root_target, ebpf_user_config_dir, "groups")) {
+ info("Cannot read process groups configuration file '%s/apps_groups.conf'. Will try '%s/apps_groups.conf'",
+ ebpf_user_config_dir, ebpf_stock_config_dir);
+ if (ebpf_read_apps_groups_conf(
+ &apps_groups_default_target, &apps_groups_root_target, ebpf_stock_config_dir, "groups")) {
+ error("Cannot read process groups '%s/apps_groups.conf'. There are no internal defaults. Failing.",
+ ebpf_stock_config_dir);
+ ebpf_exit();
+ }
+ } else
+ info("Loaded config file '%s/apps_groups.conf'", ebpf_user_config_dir);
+}
+
+/*****************************************************************
+ *
+ * COLLECTOR ENTRY POINT
+ *
+ *****************************************************************/
+
+/**
+ * Update PID file
+ *
+ * Update the content of PID file
+ *
+ * @param filename is the full name of the file.
+ * @param pid that identifies the process
+ */
+static void ebpf_update_pid_file(char *filename, pid_t pid)
+{
+ FILE *fp = fopen(filename, "w");
+ if (!fp)
+ return;
+
+ fprintf(fp, "%d", pid);
+ fclose(fp);
+}
+
+/**
+ * Get Process Name
+ *
+ * Get process name from /proc/PID/status
+ *
+ * @param pid that identifies the process
+ */
+static char *ebpf_get_process_name(pid_t pid)
+{
+ char *name = NULL;
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "/proc/%d/status", pid);
+
+ procfile *ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) {
+ error("Cannot open %s", filename);
+ return name;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return name;
+
+ unsigned long i, lines = procfile_lines(ff);
+ for(i = 0; i < lines ; i++) {
+ char *cmp = procfile_lineword(ff, i, 0);
+ if (!strcmp(cmp, "Name:")) {
+ name = strdupz(procfile_lineword(ff, i, 1));
+ break;
+ }
+ }
+
+ procfile_close(ff);
+
+ return name;
+}
+
+/**
+ * Read Previous PID
+ *
+ * @param filename is the full name of the file.
+ *
+ * @return It returns the PID used during previous execution on success or 0 otherwise
+ */
+static pid_t ebpf_read_previous_pid(char *filename)
+{
+ FILE *fp = fopen(filename, "r");
+ if (!fp)
+ return 0;
+
+ char buffer[64];
+ size_t length = fread(buffer, sizeof(*buffer), 63, fp);
+ pid_t old_pid = 0;
+ if (length) {
+ if (length > 63)
+ length = 63;
+
+ buffer[length] = '\0';
+ old_pid = (pid_t)str2uint32_t(buffer);
+ }
+ fclose(fp);
+
+ return old_pid;
+}
+
+/**
+ * Kill previous process
+ *
+ * Kill previous process whether it was not closed.
+ *
+ * @param filename is the full name of the file.
+ * @param pid that identifies the process
+ */
+static void ebpf_kill_previous_process(char *filename, pid_t pid)
+{
+ pid_t old_pid = ebpf_read_previous_pid(filename);
+ if (!old_pid)
+ return;
+
+ // Process is not running
+ char *prev_name = ebpf_get_process_name(old_pid);
+ if (!prev_name)
+ return;
+
+ char *current_name = ebpf_get_process_name(pid);
+
+ if (!strcmp(prev_name, current_name))
+ kill(old_pid, SIGKILL);
+
+ freez(prev_name);
+ freez(current_name);
+
+ // wait few microseconds before start new plugin
+ sleep_usec(USEC_PER_MS * 300);
+}
+
+/**
+ * PID file
+ *
+ * Write the filename for PID inside the given vector.
+ *
+ * @param filename vector where we will store the name.
+ * @param length number of bytes available in filename vector
+ */
+void ebpf_pid_file(char *filename, size_t length)
+{
+ snprintfz(filename, length, "%s%s/ebpf.d/ebpf.pid", netdata_configured_host_prefix, ebpf_plugin_dir);
+}
+
+/**
+ * Manage PID
+ *
+ * This function kills another instance of eBPF whether it is necessary and update the file content.
+ *
+ * @param pid that identifies the process
+ */
+static void ebpf_manage_pid(pid_t pid)
+{
+ char filename[FILENAME_MAX + 1];
+ ebpf_pid_file(filename, FILENAME_MAX);
+
+ ebpf_kill_previous_process(filename, pid);
+ ebpf_update_pid_file(filename, pid);
+}
+
+/**
+ * Set start routine
+ *
+ * Set static routine before threads to be created.
+ */
+ static void ebpf_set_static_routine()
+ {
+ int i;
+ for (i = 0; ebpf_modules[i].thread_name; i++) {
+ ebpf_threads[i].start_routine = ebpf_modules[i].start_routine;
+ }
+ }
+
+/**
+ * Entry point
+ *
+ * @param argc the number of arguments
+ * @param argv the pointer to the arguments
+ *
+ * @return it returns 0 on success and another integer otherwise
+ */
+int main(int argc, char **argv)
+{
+ clocks_init();
+ main_thread_id = gettid();
+
+ set_global_variables();
+ ebpf_parse_args(argc, argv);
+ ebpf_manage_pid(getpid());
+
+ if (!has_condition_to_run(running_on_kernel)) {
+ error("The current collector cannot run on this kernel.");
+ return 2;
+ }
+
+ if (!am_i_running_as_root()) {
+ error(
+ "ebpf.plugin should either run as root (now running with uid %u, euid %u) or have special capabilities..",
+ (unsigned int)getuid(), (unsigned int)geteuid());
+ return 3;
+ }
+
+ // set name
+ program_name = "ebpf.plugin";
+
+ // disable syslog
+ error_log_syslog = 0;
+
+ // set errors flood protection to 100 logs per hour
+ error_log_errors_per_period = 100;
+ error_log_throttle_period = 3600;
+
+ struct rlimit r = { RLIM_INFINITY, RLIM_INFINITY };
+ if (setrlimit(RLIMIT_MEMLOCK, &r)) {
+ error("Setrlimit(RLIMIT_MEMLOCK)");
+ return 4;
+ }
+
+ signal(SIGINT, ebpf_stop_threads);
+ signal(SIGQUIT, ebpf_stop_threads);
+ signal(SIGTERM, ebpf_stop_threads);
+ signal(SIGPIPE, ebpf_stop_threads);
+
+ if (ebpf_start_pthread_variables()) {
+ error("Cannot start mutex to control overall charts.");
+ ebpf_exit();
+ }
+
+ netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
+ if(verify_netdata_host_prefix() == -1) ebpf_exit(6);
+
+ ebpf_allocate_common_vectors();
+
+#ifdef LIBBPF_MAJOR_VERSION
+ libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
+#endif
+
+ read_local_addresses();
+ read_local_ports("/proc/net/tcp", IPPROTO_TCP);
+ read_local_ports("/proc/net/tcp6", IPPROTO_TCP);
+ read_local_ports("/proc/net/udp", IPPROTO_UDP);
+ read_local_ports("/proc/net/udp6", IPPROTO_UDP);
+
+ ebpf_set_static_routine();
+
+ int i;
+ for (i = 0; ebpf_threads[i].name != NULL; i++) {
+ struct netdata_static_thread *st = &ebpf_threads[i];
+
+ ebpf_module_t *em = &ebpf_modules[i];
+ em->thread = st;
+ // We always initialize process, because it is responsible to take care of apps integration
+ if (em->enabled || !i) {
+ st->thread = mallocz(sizeof(netdata_thread_t));
+ em->thread_id = i;
+ st->enabled = NETDATA_THREAD_EBPF_RUNNING;
+ netdata_thread_create(st->thread, st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, em);
+ } else {
+ st->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ }
+ }
+
+ usec_t step = EBPF_DEFAULT_UPDATE_EVERY * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ //Plugin will be killed when it receives a signal
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ }
+
+ return 0;
+}
diff --git a/collectors/ebpf.plugin/ebpf.d.conf b/collectors/ebpf.plugin/ebpf.d.conf
new file mode 100644
index 0000000..cf5c740
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d.conf
@@ -0,0 +1,69 @@
+#
+# Global options
+#
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change the setting
+# `apps` and `cgroups` to 'no'.
+#
+# The `update every` option defines the number of seconds used to read data from kernel and send to netdata
+#
+# The `pid table size` defines the maximum number of PIDs stored in the application hash tables.
+#
+[global]
+ ebpf load mode = entry
+ apps = yes
+ cgroups = no
+ update every = 5
+ pid table size = 32768
+ btf path = /sys/kernel/btf/
+
+#
+# eBPF Programs
+#
+# The eBPF collector has the following eBPF programs:
+#
+# `cachestat` : Make charts for kernel functions related to page cache.
+# `dcstat` : Make charts for kernel functions related to directory cache.
+# `disk` : Monitor I/O latencies for disks
+# `fd` : This eBPF program creates charts that show information about file manipulation.
+# `filesystem`: Monitor calls for functions used to manipulate specific filesystems
+# `hardirq` : Monitor latency of serving hardware interrupt requests (hard IRQs).
+# `mdflush` : Monitors flush counts for multi-devices.
+# `mount` : Monitor calls for syscalls mount and umount
+# `oomkill` : This eBPF program creates a chart that shows which process got OOM killed and when.
+# `process` : This eBPF program creates charts that show information about process life.
+# `shm` : Monitor calls for syscalls shmget, shmat, shmdt and shmctl.
+# `socket` : This eBPF program creates charts with information about `TCP` and `UDP` functions, including the
+# bandwidth consumed by each.
+# `softirq` : Monitor latency of serving software interrupt requests (soft IRQs).
+# `sync` : Monitor calls for syscall sync(2).
+# `swap` : Monitor calls for internal swap functions.
+# `vfs` : This eBPF program creates charts that show information about process VFS IO, VFS file manipulation and
+# files removed.
+#
+# When plugin detects that system has support to BTF, it enables integration with apps.plugin.
+#
+[ebpf programs]
+ cachestat = no
+ dcstat = no
+ disk = no
+ fd = yes
+ filesystem = no
+ hardirq = yes
+ mdflush = no
+ mount = yes
+ oomkill = yes
+ process = yes
+ shm = no
+ socket = yes
+ softirq = yes
+ sync = yes
+ swap = no
+ vfs = yes
+ network connections = no
diff --git a/collectors/ebpf.plugin/ebpf.d/cachestat.conf b/collectors/ebpf.plugin/ebpf.d/cachestat.conf
new file mode 100644
index 0000000..52466be
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/cachestat.conf
@@ -0,0 +1,36 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# The `pid table size` defines the maximum number of PIDs stored inside the application hash table.
+#
+# The `ebpf type format` option accepts the following values :
+# `auto` : The eBPF collector will investigate hardware and select between the two next options.
+# `legacy`: The eBPF collector will load the legacy code. Note: This has a bigger overload.
+# `co-re` : The eBPF collector will use latest tracing method. Note: This is not available on all platforms.
+#
+# The `ebpf co-re tracing` option accepts the following values:
+# `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host.
+# `probe` : This is the same as legacy code.
+#
+# The `collect pid` option defines the PID stored inside hash tables and accepts the following options:
+# `real parent`: Only stores real parent inside PID
+# `parent` : Only stores parent PID.
+# `all` : Stores all PIDs used by software. This is the most expensive option.
+#
+# Uncomment lines to define specific options for thread.
+[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+# pid table size = 32768
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
+ collect pid = real parent
diff --git a/collectors/ebpf.plugin/ebpf.d/dcstat.conf b/collectors/ebpf.plugin/ebpf.d/dcstat.conf
new file mode 100644
index 0000000..8aed8f7
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/dcstat.conf
@@ -0,0 +1,34 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# The `ebpf type format` option accepts the following values :
+# `auto` : The eBPF collector will investigate hardware and select between the two next options.
+# `legacy`: The eBPF collector will load the legacy code. Note: This has a bigger overload.
+# `co-re` : The eBPF collector will use latest tracing method. Note: This is not available on all platforms.
+#
+# The `ebpf co-re tracing` option accepts the following values:
+# `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host.
+# `probe` : This is the same as legacy code.
+#
+# The `collect pid` option defines the PID stored inside hash tables and accepts the following options:
+# `real parent`: Only stores real parent inside PID
+# `parent` : Only stores parent PID.
+# `all` : Stores all PIDs used by software. This is the most expensive option.
+#
+# Uncomment lines to define specific options for thread.
+[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+# pid table size = 32768
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
+ collect pid = real parent
diff --git a/collectors/ebpf.plugin/ebpf.d/disk.conf b/collectors/ebpf.plugin/ebpf.d/disk.conf
new file mode 100644
index 0000000..4adf88e
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/disk.conf
@@ -0,0 +1,9 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+#[global]
+# ebpf load mode = entry
+# update every = 10
+
diff --git a/collectors/ebpf.plugin/ebpf.d/ebpf_kernel_reject_list.txt b/collectors/ebpf.plugin/ebpf.d/ebpf_kernel_reject_list.txt
new file mode 100644
index 0000000..539bf35
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/ebpf_kernel_reject_list.txt
@@ -0,0 +1 @@
+Ubuntu 4.18.0
diff --git a/collectors/ebpf.plugin/ebpf.d/fd.conf b/collectors/ebpf.plugin/ebpf.d/fd.conf
new file mode 100644
index 0000000..8333520
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/fd.conf
@@ -0,0 +1,21 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# The `pid table size` defines the maximum number of PIDs stored inside the hash table.
+#
+# Uncomment lines to define specific options for thread.
+[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+# pid table size = 32768
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
diff --git a/collectors/ebpf.plugin/ebpf.d/filesystem.conf b/collectors/ebpf.plugin/ebpf.d/filesystem.conf
new file mode 100644
index 0000000..c5eb01e
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/filesystem.conf
@@ -0,0 +1,20 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps plugin`.
+# If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to
+# 'no'.
+#
+#[global]
+# ebpf load mode = entry
+# update every = 10
+
+# All filesystems are named as 'NAMEdist' where NAME is the filesystem name while 'dist' is a reference for distribution.
+[filesystem]
+ btrfsdist = yes
+ ext4dist = yes
+ nfsdist = yes
+ xfsdist = yes
+ zfsdist = yes
diff --git a/collectors/ebpf.plugin/ebpf.d/hardirq.conf b/collectors/ebpf.plugin/ebpf.d/hardirq.conf
new file mode 100644
index 0000000..f2bae1d
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/hardirq.conf
@@ -0,0 +1,8 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+#[global]
+# ebpf load mode = entry
+# update every = 10
diff --git a/collectors/ebpf.plugin/ebpf.d/mdflush.conf b/collectors/ebpf.plugin/ebpf.d/mdflush.conf
new file mode 100644
index 0000000..e65e867
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/mdflush.conf
@@ -0,0 +1,7 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#[global]
+# ebpf load mode = entry
+# update every = 1
diff --git a/collectors/ebpf.plugin/ebpf.d/mount.conf b/collectors/ebpf.plugin/ebpf.d/mount.conf
new file mode 100644
index 0000000..fdd82f2
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/mount.conf
@@ -0,0 +1,19 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The `ebpf type format` option accepts the following values :
+# `auto` : The eBPF collector will investigate hardware and select between the two next options.
+# `legacy`: The eBPF collector will load the legacy code. Note: This has a bigger overload.
+# `co-re` : The eBPF collector will use latest tracing method. Note: This is not available on all platforms.
+#
+# The `ebpf co-re tracing` option accepts the following values:
+# `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host.
+# `tracepoint`: When available, the eBPF collector will use kernel tracepoint to monitor syscall.
+# `probe` : This is the same as legacy code.
+[global]
+# ebpf load mode = entry
+# update every = 1
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
diff --git a/collectors/ebpf.plugin/ebpf.d/network.conf b/collectors/ebpf.plugin/ebpf.d/network.conf
new file mode 100644
index 0000000..d939d8e
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/network.conf
@@ -0,0 +1,53 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# The following options change the hash table size:
+# `bandwidth table size`: Maximum number of connections monitored
+# `ipv4 connection table size`: Maximum number of IPV4 connections monitored
+# `ipv6 connection table size`: Maximum number of IPV6 connections monitored
+# `udp connection table size`: Maximum number of UDP connections monitored
+#
+# The `ebpf type format` option accepts the following values :
+# `auto` : The eBPF collector will investigate hardware and select between the two next options.
+# `legacy`: The eBPF collector will load the legacy code. Note: This has a bigger overload.
+# `co-re` : The eBPF collector will use latest tracing method. Note: This is not available on all platforms.
+#
+# The `ebpf co-re tracing` option accepts the following values:
+# `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host.
+# `tracepoint`: When available, the eBPF collector will use kernel tracepoint to monitor syscall.
+# `probe` : This is the same as legacy code.
+#
+[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+ bandwidth table size = 16384
+ ipv4 connection table size = 16384
+ ipv6 connection table size = 16384
+ udp connection table size = 4096
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
+
+#
+# Network Connection
+#
+# This is a feature with status WIP(Work in Progress)
+#
+[network connections]
+ maximum dimensions = 50
+ resolve hostnames = no
+ resolve service names = no
+ ports = *
+ ips = !127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 !::1/128
+ hostnames = *
+
+[service name]
+ 19999 = Netdata
diff --git a/collectors/ebpf.plugin/ebpf.d/oomkill.conf b/collectors/ebpf.plugin/ebpf.d/oomkill.conf
new file mode 100644
index 0000000..e65e867
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/oomkill.conf
@@ -0,0 +1,7 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#[global]
+# ebpf load mode = entry
+# update every = 1
diff --git a/collectors/ebpf.plugin/ebpf.d/process.conf b/collectors/ebpf.plugin/ebpf.d/process.conf
new file mode 100644
index 0000000..1da5f84
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/process.conf
@@ -0,0 +1,25 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# The `pid table size` defines the maximum number of PIDs stored inside the hash table.
+#
+# The `collect pid` option defines the PID stored inside hash tables and accepts the following options:
+# `real parent`: Only stores real parent inside PID
+# `parent` : Only stores parent PID.
+# `all` : Stores all PIDs used by software. This is the most expensive option.
+#
+# Uncomment lines to define specific options for thread.
+#[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+# pid table size = 32768
+# collect pid = real parent
diff --git a/collectors/ebpf.plugin/ebpf.d/shm.conf b/collectors/ebpf.plugin/ebpf.d/shm.conf
new file mode 100644
index 0000000..23ab96d
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/shm.conf
@@ -0,0 +1,36 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# The `ebpf type format` option accepts the following values :
+# `auto` : The eBPF collector will investigate hardware and select between the two next options.
+# `legacy`: The eBPF collector will load the legacy code. Note: This has a bigger overload.
+# `co-re` : The eBPF collector will use latest tracing method. Note: This is not available on all platforms.
+#
+# The `ebpf co-re tracing` option accepts the following values:
+# `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host.
+# `tracepoint`: When available, the eBPF collector will use kernel tracepoint to monitor syscall.
+# `probe` : This is the same as legacy code.
+#
+# Uncomment lines to define specific options for thread.
+[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+# pid table size = 32768
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
+
+# List of monitored syscalls
+[syscalls]
+ shmget = yes
+ shmat = yes
+ shmdt = yes
+ shmctl = yes
diff --git a/collectors/ebpf.plugin/ebpf.d/softirq.conf b/collectors/ebpf.plugin/ebpf.d/softirq.conf
new file mode 100644
index 0000000..f2bae1d
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/softirq.conf
@@ -0,0 +1,8 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+#[global]
+# ebpf load mode = entry
+# update every = 10
diff --git a/collectors/ebpf.plugin/ebpf.d/swap.conf b/collectors/ebpf.plugin/ebpf.d/swap.conf
new file mode 100644
index 0000000..3986ae4
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/swap.conf
@@ -0,0 +1,28 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# The `ebpf type format` option accepts the following values :
+# `auto` : The eBPF collector will investigate hardware and select between the two next options.
+# `legacy`: The eBPF collector will load the legacy code. Note: This has a bigger overload.
+# `co-re` : The eBPF collector will use latest tracing method. Note: This is not available on all platforms.
+#
+# The `ebpf co-re tracing` option accepts the following values:
+# `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host.
+# `probe` : This is the same as legacy code.
+#
+# Uncomment lines to define specific options for thread.
+[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+# pid table size = 32768
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
diff --git a/collectors/ebpf.plugin/ebpf.d/sync.conf b/collectors/ebpf.plugin/ebpf.d/sync.conf
new file mode 100644
index 0000000..ebec5d3
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/sync.conf
@@ -0,0 +1,36 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# The `ebpf type format` option accepts the following values :
+# `auto` : The eBPF collector will investigate hardware and select between the two next options.
+# `legacy`: The eBPF collector will load the legacy code. Note: This has a bigger overload.
+# `co-re` : The eBPF collector will use latest tracing method. Note: This is not available on all platforms.
+#
+# The `ebpf co-re tracing` option accepts the following values:
+# `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host.
+# `tracepoint`: When available, the eBPF collector will use kernel tracepoint to monitor syscall.
+# `probe` : This is the same as legacy code.
+#
+[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
+
+# List of monitored syscalls
+[syscalls]
+ sync = yes
+ msync = yes
+ fsync = yes
+ fdatasync = yes
+ syncfs = yes
+ sync_file_range = yes
diff --git a/collectors/ebpf.plugin/ebpf.d/vfs.conf b/collectors/ebpf.plugin/ebpf.d/vfs.conf
new file mode 100644
index 0000000..fa5d5b4
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.d/vfs.conf
@@ -0,0 +1,19 @@
+# The `ebpf load mode` option accepts the following values :
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
+# new charts for the return of these functions, such as errors.
+#
+# The eBPF collector also creates charts for each running application through an integration with the `apps.plugin`
+# or `cgroups.plugin`.
+# If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change
+# the setting `apps` and `cgroups` to 'no'.
+#
+# Uncomment lines to define specific options for thread.
+[global]
+# ebpf load mode = entry
+# apps = yes
+# cgroups = no
+# update every = 10
+# pid table size = 32768
+ ebpf type format = auto
+ ebpf co-re tracing = trampoline
diff --git a/collectors/ebpf.plugin/ebpf.h b/collectors/ebpf.plugin/ebpf.h
new file mode 100644
index 0000000..28b04ce
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf.h
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_COLLECTOR_EBPF_H
+#define NETDATA_COLLECTOR_EBPF_H 1
+
+#ifndef __FreeBSD__
+#include <linux/perf_event.h>
+#endif
+#include <stdint.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <dlfcn.h>
+
+#include <fcntl.h>
+#include <ctype.h>
+#include <dirent.h>
+
+// From libnetdata.h
+#include "libnetdata/threads/threads.h"
+#include "libnetdata/locks/locks.h"
+#include "libnetdata/avl/avl.h"
+#include "libnetdata/clocks/clocks.h"
+#include "libnetdata/config/appconfig.h"
+#include "libnetdata/ebpf/ebpf.h"
+#include "libnetdata/procfile/procfile.h"
+#include "collectors/cgroups.plugin/sys_fs_cgroup.h"
+#include "daemon/main.h"
+
+#include "ebpf_apps.h"
+#include "ebpf_cgroup.h"
+
+#define NETDATA_EBPF_OLD_CONFIG_FILE "ebpf.conf"
+#define NETDATA_EBPF_CONFIG_FILE "ebpf.d.conf"
+
+typedef struct netdata_syscall_stat {
+ unsigned long bytes; // total number of bytes
+ uint64_t call; // total number of calls
+ uint64_t ecall; // number of calls that returned error
+ struct netdata_syscall_stat *next; // Link list
+} netdata_syscall_stat_t;
+
+typedef uint64_t netdata_idx_t;
+
+typedef struct netdata_publish_syscall {
+ char *dimension;
+ char *name;
+ char *algorithm;
+ unsigned long nbyte;
+ unsigned long pbyte;
+ uint64_t ncall;
+ uint64_t pcall;
+ uint64_t nerr;
+ uint64_t perr;
+ struct netdata_publish_syscall *next;
+} netdata_publish_syscall_t;
+
+typedef struct netdata_publish_vfs_common {
+ long write;
+ long read;
+
+ long running;
+ long zombie;
+} netdata_publish_vfs_common_t;
+
+typedef struct netdata_error_report {
+ char comm[16];
+ __u32 pid;
+
+ int type;
+ int err;
+} netdata_error_report_t;
+
+extern ebpf_module_t ebpf_modules[];
+enum ebpf_main_index {
+ EBPF_MODULE_PROCESS_IDX,
+ EBPF_MODULE_SOCKET_IDX,
+ EBPF_MODULE_CACHESTAT_IDX,
+ EBPF_MODULE_SYNC_IDX,
+ EBPF_MODULE_DCSTAT_IDX,
+ EBPF_MODULE_SWAP_IDX,
+ EBPF_MODULE_VFS_IDX,
+ EBPF_MODULE_FILESYSTEM_IDX,
+ EBPF_MODULE_DISK_IDX,
+ EBPF_MODULE_MOUNT_IDX,
+ EBPF_MODULE_FD_IDX,
+ EBPF_MODULE_HARDIRQ_IDX,
+ EBPF_MODULE_SOFTIRQ_IDX,
+ EBPF_MODULE_OOMKILL_IDX,
+ EBPF_MODULE_SHM_IDX,
+ EBPF_MODULE_MDFLUSH_IDX,
+ /* THREADS MUST BE INCLUDED BEFORE THIS COMMENT */
+ EBPF_OPTION_ALL_CHARTS,
+ EBPF_OPTION_VERSION,
+ EBPF_OPTION_HELP,
+ EBPF_OPTION_GLOBAL_CHART,
+ EBPF_OPTION_RETURN_MODE,
+ EBPF_OPTION_LEGACY,
+ EBPF_OPTION_CORE
+};
+
+typedef struct ebpf_tracepoint {
+ bool enabled;
+ char *class;
+ char *event;
+} ebpf_tracepoint_t;
+
+enum ebpf_threads_status {
+ NETDATA_THREAD_EBPF_RUNNING,
+ NETDATA_THREAD_EBPF_STOPPING,
+ NETDATA_THREAD_EBPF_STOPPED
+};
+
+// Copied from musl header
+#ifndef offsetof
+#if __GNUC__ > 3
+#define offsetof(type, member) __builtin_offsetof(type, member)
+#else
+#define offsetof(type, member) ((size_t)((char *)&(((type *)0)->member) - (char *)0))
+#endif
+#endif
+
+// Chart definitions
+#define NETDATA_EBPF_FAMILY "ebpf"
+#define NETDATA_EBPF_IP_FAMILY "ip"
+#define NETDATA_FILESYSTEM_FAMILY "filesystem"
+#define NETDATA_EBPF_MOUNT_GLOBAL_FAMILY "mount_points"
+#define NETDATA_EBPF_CHART_TYPE_LINE "line"
+#define NETDATA_EBPF_CHART_TYPE_STACKED "stacked"
+#define NETDATA_EBPF_MEMORY_GROUP "mem"
+#define NETDATA_EBPF_SYSTEM_GROUP "system"
+#define NETDATA_SYSTEM_SWAP_SUBMENU "swap"
+#define NETDATA_SYSTEM_CGROUP_SWAP_SUBMENU "swap (eBPF)"
+#define NETDATA_SYSTEM_IPC_SHM_SUBMENU "ipc shared memory"
+#define NETDATA_MONITORING_FAMILY "netdata"
+
+// Statistics charts
+#define NETDATA_EBPF_THREADS "ebpf_threads"
+#define NETDATA_EBPF_LOAD_METHOD "ebpf_load_methods"
+
+// Log file
+#define NETDATA_DEVELOPER_LOG_FILE "developer.log"
+
+// Maximum number of processors monitored on perf events
+#define NETDATA_MAX_PROCESSOR 512
+
+// Kernel versions calculated with the formula:
+// R = MAJOR*65536 + MINOR*256 + PATCH
+#define NETDATA_KERNEL_V5_3 328448
+#define NETDATA_KERNEL_V4_15 265984
+
+#define EBPF_SYS_CLONE_IDX 11
+#define EBPF_MAX_MAPS 32
+
+#define EBPF_DEFAULT_UPDATE_EVERY 10
+
+enum ebpf_algorithms_list {
+ NETDATA_EBPF_ABSOLUTE_IDX,
+ NETDATA_EBPF_INCREMENTAL_IDX
+};
+
+// Threads
+void *ebpf_process_thread(void *ptr);
+void *ebpf_socket_thread(void *ptr);
+
+// Common variables
+extern pthread_mutex_t lock;
+extern pthread_mutex_t ebpf_exit_cleanup;
+extern int ebpf_nprocs;
+extern int running_on_kernel;
+extern int isrh;
+extern char *ebpf_plugin_dir;
+
+extern pthread_mutex_t collect_data_mutex;
+extern pthread_cond_t collect_data_cond_var;
+
+// Common functions
+void ebpf_global_labels(netdata_syscall_stat_t *is,
+ netdata_publish_syscall_t *pio,
+ char **dim,
+ char **name,
+ int *algorithm,
+ int end);
+
+void ebpf_write_chart_cmd(char *type,
+ char *id,
+ char *title,
+ char *units,
+ char *family,
+ char *charttype,
+ char *context,
+ int order,
+ int update_every,
+ char *module);
+
+void ebpf_write_global_dimension(char *name, char *id, char *algorithm);
+
+void ebpf_create_global_dimension(void *ptr, int end);
+
+void ebpf_create_chart(char *type,
+ char *id,
+ char *title,
+ char *units,
+ char *family,
+ char *context,
+ char *charttype,
+ int order,
+ void (*ncd)(void *, int),
+ void *move,
+ int end,
+ int update_every,
+ char *module);
+
+void write_begin_chart(char *family, char *name);
+
+void write_chart_dimension(char *dim, long long value);
+
+void write_count_chart(char *name, char *family, netdata_publish_syscall_t *move, uint32_t end);
+
+void write_err_chart(char *name, char *family, netdata_publish_syscall_t *move, int end);
+
+void write_io_chart(char *chart, char *family, char *dwrite, long long vwrite,
+ char *dread, long long vread);
+
+void ebpf_create_charts_on_apps(char *name,
+ char *title,
+ char *units,
+ char *family,
+ char *charttype,
+ int order,
+ char *algorithm,
+ struct target *root,
+ int update_every,
+ char *module);
+
+void write_end_chart();
+
+void ebpf_cleanup_publish_syscall(netdata_publish_syscall_t *nps);
+
+int ebpf_enable_tracepoint(ebpf_tracepoint_t *tp);
+int ebpf_disable_tracepoint(ebpf_tracepoint_t *tp);
+uint32_t ebpf_enable_tracepoints(ebpf_tracepoint_t *tps);
+
+void ebpf_pid_file(char *filename, size_t length);
+
+#define EBPF_PROGRAMS_SECTION "ebpf programs"
+
+#define EBPF_COMMON_DIMENSION_PERCENTAGE "%"
+#define EBPF_COMMON_DIMENSION_CALL "calls/s"
+#define EBPF_COMMON_DIMENSION_CONNECTIONS "connections/s"
+#define EBPF_COMMON_DIMENSION_BITS "kilobits/s"
+#define EBPF_COMMON_DIMENSION_BYTES "bytes/s"
+#define EBPF_COMMON_DIMENSION_DIFFERENCE "difference"
+#define EBPF_COMMON_DIMENSION_PACKETS "packets"
+#define EBPF_COMMON_DIMENSION_FILES "files"
+#define EBPF_COMMON_DIMENSION_MILLISECONDS "milliseconds"
+#define EBPF_COMMON_DIMENSION_KILLS "kills"
+
+// Common variables
+extern int debug_enabled;
+extern struct pid_stat *root_of_pids;
+extern ebpf_cgroup_target_t *ebpf_cgroup_pids;
+extern char *ebpf_algorithms[];
+extern struct config collector_config;
+extern ebpf_process_stat_t *global_process_stat;
+extern netdata_ebpf_cgroup_shm_t shm_ebpf_cgroup;
+extern int shm_fd_ebpf_cgroup;
+extern sem_t *shm_sem_ebpf_cgroup;
+extern pthread_mutex_t mutex_cgroup_shm;
+extern size_t all_pids_count;
+extern ebpf_plugin_stats_t plugin_statistics;
+#ifdef LIBBPF_MAJOR_VERSION
+extern struct btf *default_btf;
+#else
+extern void *default_btf;
+#endif
+
+// Socket functions and variables
+// Common functions
+void ebpf_process_create_apps_charts(struct ebpf_module *em, void *ptr);
+void ebpf_socket_create_apps_charts(struct ebpf_module *em, void *ptr);
+void ebpf_cachestat_create_apps_charts(struct ebpf_module *em, void *root);
+void ebpf_one_dimension_write_charts(char *family, char *chart, char *dim, long long v1);
+collected_number get_value_from_structure(char *basis, size_t offset);
+void ebpf_update_pid_table(ebpf_local_maps_t *pid, ebpf_module_t *em);
+void ebpf_write_chart_obsolete(char *type, char *id, char *title, char *units, char *family,
+ char *charttype, char *context, int order, int update_every);
+void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, char **dimensions, uint32_t end);
+void ebpf_update_disabled_plugin_stats(ebpf_module_t *em);
+extern ebpf_filesystem_partitions_t localfs[];
+extern ebpf_sync_syscalls_t local_syscalls[];
+extern int ebpf_exit_plugin;
+
+#define EBPF_MAX_SYNCHRONIZATION_TIME 300
+
+#endif /* NETDATA_COLLECTOR_EBPF_H */
diff --git a/collectors/ebpf.plugin/ebpf_apps.c b/collectors/ebpf.plugin/ebpf_apps.c
new file mode 100644
index 0000000..7519e06
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_apps.c
@@ -0,0 +1,1155 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_socket.h"
+#include "ebpf_apps.h"
+
+// ----------------------------------------------------------------------------
+// internal flags
+// handled in code (automatically set)
+
+static int proc_pid_cmdline_is_needed = 0; // 1 when we need to read /proc/cmdline
+
+/*****************************************************************
+ *
+ * FUNCTIONS USED TO READ HASH TABLES
+ *
+ *****************************************************************/
+
+/**
+ * Read statistic hash table.
+ *
+ * @param ep the output structure.
+ * @param fd the file descriptor mapped from kernel ring.
+ * @param pid the index used to select the data.
+ * @param bpf_map_lookup_elem a pointer for the function used to read data.
+ *
+ * @return It returns 0 when the data was copied and -1 otherwise
+ */
+int ebpf_read_hash_table(void *ep, int fd, uint32_t pid)
+{
+ if (!ep)
+ return -1;
+
+ if (!bpf_map_lookup_elem(fd, &pid, ep))
+ return 0;
+
+ return -1;
+}
+
+/**
+ * Read socket statistic
+ *
+ * Read information from kernel ring to user ring.
+ *
+ * @param ep the table with all process stats values.
+ * @param fd the file descriptor mapped from kernel
+ * @param ef a pointer for the functions mapped from dynamic library
+ * @param pids the list of pids associated to a target.
+ *
+ * @return
+ */
+size_t read_bandwidth_statistic_using_pid_on_target(ebpf_bandwidth_t **ep, int fd, struct pid_on_target *pids)
+{
+ size_t count = 0;
+ while (pids) {
+ uint32_t current_pid = pids->pid;
+ if (!ebpf_read_hash_table(ep[current_pid], fd, current_pid))
+ count++;
+
+ pids = pids->next;
+ }
+
+ return count;
+}
+
+/**
+ * Read bandwidth statistic using hash table
+ *
+ * @param out the output tensor that will receive the information.
+ * @param fd the file descriptor that has the data
+ * @param bpf_map_lookup_elem a pointer for the function to read the data
+ * @param bpf_map_get_next_key a pointer fo the function to read the index.
+ */
+size_t read_bandwidth_statistic_using_hash_table(ebpf_bandwidth_t **out, int fd)
+{
+ size_t count = 0;
+ uint32_t key = 0;
+ uint32_t next_key = 0;
+
+ while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
+ ebpf_bandwidth_t *eps = out[next_key];
+ if (!eps) {
+ eps = callocz(1, sizeof(ebpf_process_stat_t));
+ out[next_key] = eps;
+ }
+ ebpf_read_hash_table(eps, fd, next_key);
+ }
+
+ return count;
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS CALLED FROM COLLECTORS
+ *
+ *****************************************************************/
+
+/**
+ * Am I running as Root
+ *
+ * Verify the user that is running the collector.
+ *
+ * @return It returns 1 for root and 0 otherwise.
+ */
+int am_i_running_as_root()
+{
+ uid_t uid = getuid(), euid = geteuid();
+
+ if (uid == 0 || euid == 0) {
+ return 1;
+ }
+
+ return 0;
+}
+
+/**
+ * Reset the target values
+ *
+ * @param root the pointer to the chain that will be reset.
+ *
+ * @return it returns the number of structures that was reset.
+ */
+size_t zero_all_targets(struct target *root)
+{
+ struct target *w;
+ size_t count = 0;
+
+ for (w = root; w; w = w->next) {
+ count++;
+
+ if (unlikely(w->root_pid)) {
+ struct pid_on_target *pid_on_target = w->root_pid;
+
+ while (pid_on_target) {
+ struct pid_on_target *pid_on_target_to_free = pid_on_target;
+ pid_on_target = pid_on_target->next;
+ freez(pid_on_target_to_free);
+ }
+
+ w->root_pid = NULL;
+ }
+ }
+
+ return count;
+}
+
+/**
+ * Clean the allocated structures
+ *
+ * @param agrt the pointer to be cleaned.
+ */
+void clean_apps_groups_target(struct target *agrt)
+{
+ struct target *current_target;
+ while (agrt) {
+ current_target = agrt;
+ agrt = current_target->target;
+
+ freez(current_target);
+ }
+}
+
+/**
+ * Find or create a new target
+ * there are targets that are just aggregated to other target (the second argument)
+ *
+ * @param id
+ * @param target
+ * @param name
+ *
+ * @return It returns the target on success and NULL otherwise
+ */
+struct target *get_apps_groups_target(struct target **agrt, const char *id, struct target *target, const char *name)
+{
+ int tdebug = 0, thidden = target ? target->hidden : 0, ends_with = 0;
+ const char *nid = id;
+
+ // extract the options
+ while (nid[0] == '-' || nid[0] == '+' || nid[0] == '*') {
+ if (nid[0] == '-')
+ thidden = 1;
+ if (nid[0] == '+')
+ tdebug = 1;
+ if (nid[0] == '*')
+ ends_with = 1;
+ nid++;
+ }
+ uint32_t hash = simple_hash(id);
+
+ // find if it already exists
+ struct target *w, *last = *agrt;
+ for (w = *agrt; w; w = w->next) {
+ if (w->idhash == hash && strncmp(nid, w->id, MAX_NAME) == 0)
+ return w;
+
+ last = w;
+ }
+
+ // find an existing target
+ if (unlikely(!target)) {
+ while (*name == '-') {
+ if (*name == '-')
+ thidden = 1;
+ name++;
+ }
+
+ for (target = *agrt; target != NULL; target = target->next) {
+ if (!target->target && strcmp(name, target->name) == 0)
+ break;
+ }
+ }
+
+ if (target && target->target)
+ fatal(
+ "Internal Error: request to link process '%s' to target '%s' which is linked to target '%s'", id,
+ target->id, target->target->id);
+
+ w = callocz(1, sizeof(struct target));
+ strncpyz(w->id, nid, MAX_NAME);
+ w->idhash = simple_hash(w->id);
+
+ if (unlikely(!target))
+ // copy the name
+ strncpyz(w->name, name, MAX_NAME);
+ else
+ // copy the id
+ strncpyz(w->name, nid, MAX_NAME);
+
+ strncpyz(w->compare, nid, MAX_COMPARE_NAME);
+ size_t len = strlen(w->compare);
+ if (w->compare[len - 1] == '*') {
+ w->compare[len - 1] = '\0';
+ w->starts_with = 1;
+ }
+ w->ends_with = ends_with;
+
+ if (w->starts_with && w->ends_with)
+ proc_pid_cmdline_is_needed = 1;
+
+ w->comparehash = simple_hash(w->compare);
+ w->comparelen = strlen(w->compare);
+
+ w->hidden = thidden;
+#ifdef NETDATA_INTERNAL_CHECKS
+ w->debug_enabled = tdebug;
+#else
+ if (tdebug)
+ fprintf(stderr, "apps.plugin has been compiled without debugging\n");
+#endif
+ w->target = target;
+
+ // append it, to maintain the order in apps_groups.conf
+ if (last)
+ last->next = w;
+ else
+ *agrt = w;
+
+ return w;
+}
+
+/**
+ * Read the apps_groups.conf file
+ *
+ * @param agrt a pointer to apps_group_root_target
+ * @param path the directory to search apps_%s.conf
+ * @param file the word to complement the file name.
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+int ebpf_read_apps_groups_conf(struct target **agdt, struct target **agrt, const char *path, const char *file)
+{
+ char filename[FILENAME_MAX + 1];
+
+ snprintfz(filename, FILENAME_MAX, "%s/apps_%s.conf", path, file);
+
+ // ----------------------------------------
+
+ procfile *ff = procfile_open_no_log(filename, " :\t", PROCFILE_FLAG_DEFAULT);
+ if (!ff)
+ return -1;
+
+ procfile_set_quotes(ff, "'\"");
+
+ ff = procfile_readall(ff);
+ if (!ff)
+ return -1;
+
+ size_t line, lines = procfile_lines(ff);
+
+ for (line = 0; line < lines; line++) {
+ size_t word, words = procfile_linewords(ff, line);
+ if (!words)
+ continue;
+
+ char *name = procfile_lineword(ff, line, 0);
+ if (!name || !*name)
+ continue;
+
+ // find a possibly existing target
+ struct target *w = NULL;
+
+ // loop through all words, skipping the first one (the name)
+ for (word = 0; word < words; word++) {
+ char *s = procfile_lineword(ff, line, word);
+ if (!s || !*s)
+ continue;
+ if (*s == '#')
+ break;
+
+ // is this the first word? skip it
+ if (s == name)
+ continue;
+
+ // add this target
+ struct target *n = get_apps_groups_target(agrt, s, w, name);
+ if (!n) {
+ error("Cannot create target '%s' (line %zu, word %zu)", s, line, word);
+ continue;
+ }
+
+ // just some optimization
+ // to avoid searching for a target for each process
+ if (!w)
+ w = n->target ? n->target : n;
+ }
+ }
+
+ procfile_close(ff);
+
+ *agdt = get_apps_groups_target(agrt, "p+!o@w#e$i^r&7*5(-i)l-o_", NULL, "other"); // match nothing
+ if (!*agdt)
+ fatal("Cannot create default target");
+
+ struct target *ptr = *agdt;
+ if (ptr->target)
+ *agdt = ptr->target;
+
+ return 0;
+}
+
+// the minimum PID of the system
+// this is also the pid of the init process
+#define INIT_PID 1
+
+// ----------------------------------------------------------------------------
+// string lengths
+
+#define MAX_COMPARE_NAME 100
+#define MAX_NAME 100
+#define MAX_CMDLINE 16384
+
+struct pid_stat **all_pids = NULL; // to avoid allocations, we pre-allocate the
+ // the entire pid space.
+struct pid_stat *root_of_pids = NULL; // global list of all processes running
+
+size_t all_pids_count = 0; // the number of processes running
+
+struct target
+ *apps_groups_default_target = NULL, // the default target
+ *apps_groups_root_target = NULL, // apps_groups.conf defined
+ *users_root_target = NULL, // users
+ *groups_root_target = NULL; // user groups
+
+size_t apps_groups_targets_count = 0; // # of apps_groups.conf targets
+
+// ----------------------------------------------------------------------------
+// internal counters
+
+static size_t
+ // global_iterations_counter = 1,
+ calls_counter = 0,
+ // file_counter = 0,
+ // filenames_allocated_counter = 0,
+ // inodes_changed_counter = 0,
+ // links_changed_counter = 0,
+ targets_assignment_counter = 0;
+
+// ----------------------------------------------------------------------------
+// debugging
+
+// log each problem once per process
+// log flood protection flags (log_thrown)
+#define PID_LOG_IO 0x00000001
+#define PID_LOG_STATUS 0x00000002
+#define PID_LOG_CMDLINE 0x00000004
+#define PID_LOG_FDS 0x00000008
+#define PID_LOG_STAT 0x00000010
+
+int debug_enabled = 0;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+
+#define debug_log(fmt, args...) \
+ do { \
+ if (unlikely(debug_enabled)) \
+ debug_log_int(fmt, ##args); \
+ } while (0)
+
+#else
+
+static inline void debug_log_dummy(void)
+{
+}
+#define debug_log(fmt, args...) debug_log_dummy()
+
+#endif
+
+/**
+ * Managed log
+ *
+ * Store log information if it is necessary.
+ *
+ * @param p the pid stat structure
+ * @param log the log id
+ * @param status the return from a function.
+ *
+ * @return It returns the status value.
+ */
+static inline int managed_log(struct pid_stat *p, uint32_t log, int status)
+{
+ if (unlikely(!status)) {
+ // error("command failed log %u, errno %d", log, errno);
+
+ if (unlikely(debug_enabled || errno != ENOENT)) {
+ if (unlikely(debug_enabled || !(p->log_thrown & log))) {
+ p->log_thrown |= log;
+ switch (log) {
+ case PID_LOG_IO:
+ error(
+ "Cannot process %s/proc/%d/io (command '%s')", netdata_configured_host_prefix, p->pid,
+ p->comm);
+ break;
+
+ case PID_LOG_STATUS:
+ error(
+ "Cannot process %s/proc/%d/status (command '%s')", netdata_configured_host_prefix, p->pid,
+ p->comm);
+ break;
+
+ case PID_LOG_CMDLINE:
+ error(
+ "Cannot process %s/proc/%d/cmdline (command '%s')", netdata_configured_host_prefix, p->pid,
+ p->comm);
+ break;
+
+ case PID_LOG_FDS:
+ error(
+ "Cannot process entries in %s/proc/%d/fd (command '%s')", netdata_configured_host_prefix,
+ p->pid, p->comm);
+ break;
+
+ case PID_LOG_STAT:
+ break;
+
+ default:
+ error("unhandled error for pid %d, command '%s'", p->pid, p->comm);
+ break;
+ }
+ }
+ }
+ errno = 0;
+ } else if (unlikely(p->log_thrown & log)) {
+ // error("unsetting log %u on pid %d", log, p->pid);
+ p->log_thrown &= ~log;
+ }
+
+ return status;
+}
+
+/**
+ * Get PID entry
+ *
+ * Get or allocate the PID entry for the specified pid.
+ *
+ * @param pid the pid to search the data.
+ *
+ * @return It returns the pid entry structure
+ */
+static inline struct pid_stat *get_pid_entry(pid_t pid)
+{
+ if (unlikely(all_pids[pid]))
+ return all_pids[pid];
+
+ struct pid_stat *p = callocz(1, sizeof(struct pid_stat));
+
+ if (likely(root_of_pids))
+ root_of_pids->prev = p;
+
+ p->next = root_of_pids;
+ root_of_pids = p;
+
+ p->pid = pid;
+
+ all_pids[pid] = p;
+ all_pids_count++;
+
+ return p;
+}
+
+/**
+ * Assign the PID to a target.
+ *
+ * @param p the pid_stat structure to assign for a target.
+ */
+static inline void assign_target_to_pid(struct pid_stat *p)
+{
+ targets_assignment_counter++;
+
+ uint32_t hash = simple_hash(p->comm);
+ size_t pclen = strlen(p->comm);
+
+ struct target *w;
+ for (w = apps_groups_root_target; w; w = w->next) {
+ // if(debug_enabled || (p->target && p->target->debug_enabled)) debug_log_int("\t\tcomparing '%s' with '%s'", w->compare, p->comm);
+
+ // find it - 4 cases:
+ // 1. the target is not a pattern
+ // 2. the target has the prefix
+ // 3. the target has the suffix
+ // 4. the target is something inside cmdline
+
+ if (unlikely(
+ ((!w->starts_with && !w->ends_with && w->comparehash == hash && !strcmp(w->compare, p->comm)) ||
+ (w->starts_with && !w->ends_with && !strncmp(w->compare, p->comm, w->comparelen)) ||
+ (!w->starts_with && w->ends_with && pclen >= w->comparelen && !strcmp(w->compare, &p->comm[pclen - w->comparelen])) ||
+ (proc_pid_cmdline_is_needed && w->starts_with && w->ends_with && p->cmdline && strstr(p->cmdline, w->compare))))) {
+ if (w->target)
+ p->target = w->target;
+ else
+ p->target = w;
+
+ if (debug_enabled || (p->target && p->target->debug_enabled))
+ debug_log_int("%s linked to target %s", p->comm, p->target->name);
+
+ break;
+ }
+ }
+}
+
+// ----------------------------------------------------------------------------
+// update pids from proc
+
+/**
+ * Read cmd line from /proc/PID/cmdline
+ *
+ * @param p the pid_stat_structure.
+ *
+ * @return It returns 1 on success and 0 otherwise.
+ */
+static inline int read_proc_pid_cmdline(struct pid_stat *p)
+{
+ static char cmdline[MAX_CMDLINE + 1];
+
+ if (unlikely(!p->cmdline_filename)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/%d/cmdline", netdata_configured_host_prefix, p->pid);
+ p->cmdline_filename = strdupz(filename);
+ }
+
+ int fd = open(p->cmdline_filename, procfile_open_flags, 0666);
+ if (unlikely(fd == -1))
+ goto cleanup;
+
+ ssize_t i, bytes = read(fd, cmdline, MAX_CMDLINE);
+ close(fd);
+
+ if (unlikely(bytes < 0))
+ goto cleanup;
+
+ cmdline[bytes] = '\0';
+ for (i = 0; i < bytes; i++) {
+ if (unlikely(!cmdline[i]))
+ cmdline[i] = ' ';
+ }
+
+ if (p->cmdline)
+ freez(p->cmdline);
+ p->cmdline = strdupz(cmdline);
+
+ debug_log("Read file '%s' contents: %s", p->cmdline_filename, p->cmdline);
+
+ return 1;
+
+cleanup:
+ // copy the command to the command line
+ if (p->cmdline)
+ freez(p->cmdline);
+ p->cmdline = strdupz(p->comm);
+ return 0;
+}
+
+/**
+ * Read information from /proc/PID/stat and /proc/PID/cmdline
+ * Assign target to pid
+ *
+ * @param p the pid stat structure to store the data.
+ * @param ptr an useless argument.
+ */
+static inline int read_proc_pid_stat(struct pid_stat *p, void *ptr)
+{
+ UNUSED(ptr);
+
+ static procfile *ff = NULL;
+
+ if (unlikely(!p->stat_filename)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/%d/stat", netdata_configured_host_prefix, p->pid);
+ p->stat_filename = strdupz(filename);
+ }
+
+ int set_quotes = (!ff) ? 1 : 0;
+
+ struct stat statbuf;
+ if (stat(p->stat_filename, &statbuf))
+ return 0;
+
+ ff = procfile_reopen(ff, p->stat_filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
+ if (unlikely(!ff))
+ return 0;
+
+ if (unlikely(set_quotes))
+ procfile_set_open_close(ff, "(", ")");
+
+ ff = procfile_readall(ff);
+ if (unlikely(!ff))
+ return 0;
+
+ p->last_stat_collected_usec = p->stat_collected_usec;
+ p->stat_collected_usec = now_monotonic_usec();
+ calls_counter++;
+
+ char *comm = procfile_lineword(ff, 0, 1);
+ p->ppid = (int32_t)str2pid_t(procfile_lineword(ff, 0, 3));
+
+ if (strcmp(p->comm, comm) != 0) {
+ if (unlikely(debug_enabled)) {
+ if (p->comm[0])
+ debug_log("\tpid %d (%s) changed name to '%s'", p->pid, p->comm, comm);
+ else
+ debug_log("\tJust added %d (%s)", p->pid, comm);
+ }
+
+ strncpyz(p->comm, comm, MAX_COMPARE_NAME);
+
+ // /proc/<pid>/cmdline
+ if (likely(proc_pid_cmdline_is_needed))
+ managed_log(p, PID_LOG_CMDLINE, read_proc_pid_cmdline(p));
+
+ assign_target_to_pid(p);
+ }
+
+ if (unlikely(debug_enabled || (p->target && p->target->debug_enabled)))
+ debug_log_int(
+ "READ PROC/PID/STAT: %s/proc/%d/stat, process: '%s' on target '%s' (dt=%llu)",
+ netdata_configured_host_prefix, p->pid, p->comm, (p->target) ? p->target->name : "UNSET",
+ p->stat_collected_usec - p->last_stat_collected_usec);
+
+ return 1;
+}
+
+/**
+ * Collect data for PID
+ *
+ * @param pid the current pid that we are working
+ * @param ptr a NULL value
+ *
+ * @return It returns 1 on success and 0 otherwise
+ */
+static inline int collect_data_for_pid(pid_t pid, void *ptr)
+{
+ if (unlikely(pid < 0 || pid > pid_max)) {
+ error("Invalid pid %d read (expected %d to %d). Ignoring process.", pid, 0, pid_max);
+ return 0;
+ }
+
+ struct pid_stat *p = get_pid_entry(pid);
+ if (unlikely(!p || p->read))
+ return 0;
+ p->read = 1;
+
+ if (unlikely(!managed_log(p, PID_LOG_STAT, read_proc_pid_stat(p, ptr))))
+ // there is no reason to proceed if we cannot get its status
+ return 0;
+
+ // check its parent pid
+ if (unlikely(p->ppid < 0 || p->ppid > pid_max)) {
+ error("Pid %d (command '%s') states invalid parent pid %d. Using 0.", pid, p->comm, p->ppid);
+ p->ppid = 0;
+ }
+
+ // mark it as updated
+ p->updated = 1;
+ p->keep = 0;
+ p->keeploops = 0;
+
+ return 1;
+}
+
+/**
+ * Fill link list of parents with children PIDs
+ */
+static inline void link_all_processes_to_their_parents(void)
+{
+ struct pid_stat *p, *pp;
+
+ // link all children to their parents
+ // and update children count on parents
+ for (p = root_of_pids; p; p = p->next) {
+ // for each process found
+
+ p->sortlist = 0;
+ p->parent = NULL;
+
+ if (unlikely(!p->ppid)) {
+ p->parent = NULL;
+ continue;
+ }
+
+ pp = all_pids[p->ppid];
+ if (likely(pp)) {
+ p->parent = pp;
+ pp->children_count++;
+
+ if (unlikely(debug_enabled || (p->target && p->target->debug_enabled)))
+ debug_log_int(
+ "child %d (%s, %s) on target '%s' has parent %d (%s, %s).", p->pid, p->comm,
+ p->updated ? "running" : "exited", (p->target) ? p->target->name : "UNSET", pp->pid, pp->comm,
+ pp->updated ? "running" : "exited");
+ } else {
+ p->parent = NULL;
+ debug_log("pid %d %s states parent %d, but the later does not exist.", p->pid, p->comm, p->ppid);
+ }
+ }
+}
+
+/**
+ * Aggregate PIDs to targets.
+ */
+static void apply_apps_groups_targets_inheritance(void)
+{
+ struct pid_stat *p = NULL;
+
+ // children that do not have a target
+ // inherit their target from their parent
+ int found = 1, loops = 0;
+ while (found) {
+ if (unlikely(debug_enabled))
+ loops++;
+ found = 0;
+ for (p = root_of_pids; p; p = p->next) {
+ // if this process does not have a target
+ // and it has a parent
+ // and its parent has a target
+ // then, set the parent's target to this process
+ if (unlikely(!p->target && p->parent && p->parent->target)) {
+ p->target = p->parent->target;
+ found++;
+
+ if (debug_enabled || (p->target && p->target->debug_enabled))
+ debug_log_int(
+ "TARGET INHERITANCE: %s is inherited by %d (%s) from its parent %d (%s).", p->target->name,
+ p->pid, p->comm, p->parent->pid, p->parent->comm);
+ }
+ }
+ }
+
+ // find all the procs with 0 childs and merge them to their parents
+ // repeat, until nothing more can be done.
+ int sortlist = 1;
+ found = 1;
+ while (found) {
+ if (unlikely(debug_enabled))
+ loops++;
+ found = 0;
+
+ for (p = root_of_pids; p; p = p->next) {
+ if (unlikely(!p->sortlist && !p->children_count))
+ p->sortlist = sortlist++;
+
+ if (unlikely(
+ !p->children_count // if this process does not have any children
+ && !p->merged // and is not already merged
+ && p->parent // and has a parent
+ && p->parent->children_count // and its parent has children
+ // and the target of this process and its parent is the same,
+ // or the parent does not have a target
+ && (p->target == p->parent->target || !p->parent->target) &&
+ p->ppid != INIT_PID // and its parent is not init
+ )) {
+ // mark it as merged
+ p->parent->children_count--;
+ p->merged = 1;
+
+ // the parent inherits the child's target, if it does not have a target itself
+ if (unlikely(p->target && !p->parent->target)) {
+ p->parent->target = p->target;
+
+ if (debug_enabled || (p->target && p->target->debug_enabled))
+ debug_log_int(
+ "TARGET INHERITANCE: %s is inherited by %d (%s) from its child %d (%s).", p->target->name,
+ p->parent->pid, p->parent->comm, p->pid, p->comm);
+ }
+
+ found++;
+ }
+ }
+
+ debug_log("TARGET INHERITANCE: merged %d processes", found);
+ }
+
+ // init goes always to default target
+ if (all_pids[INIT_PID])
+ all_pids[INIT_PID]->target = apps_groups_default_target;
+
+ // pid 0 goes always to default target
+ if (all_pids[0])
+ all_pids[0]->target = apps_groups_default_target;
+
+ // give a default target on all top level processes
+ if (unlikely(debug_enabled))
+ loops++;
+ for (p = root_of_pids; p; p = p->next) {
+ // if the process is not merged itself
+ // then is is a top level process
+ if (unlikely(!p->merged && !p->target))
+ p->target = apps_groups_default_target;
+
+ // make sure all processes have a sortlist
+ if (unlikely(!p->sortlist))
+ p->sortlist = sortlist++;
+ }
+
+ if (all_pids[1])
+ all_pids[1]->sortlist = sortlist++;
+
+ // give a target to all merged child processes
+ found = 1;
+ while (found) {
+ if (unlikely(debug_enabled))
+ loops++;
+ found = 0;
+ for (p = root_of_pids; p; p = p->next) {
+ if (unlikely(!p->target && p->merged && p->parent && p->parent->target)) {
+ p->target = p->parent->target;
+ found++;
+
+ if (debug_enabled || (p->target && p->target->debug_enabled))
+ debug_log_int(
+ "TARGET INHERITANCE: %s is inherited by %d (%s) from its parent %d (%s) at phase 2.",
+ p->target->name, p->pid, p->comm, p->parent->pid, p->parent->comm);
+ }
+ }
+ }
+
+ debug_log("apply_apps_groups_targets_inheritance() made %d loops on the process tree", loops);
+}
+
+/**
+ * Update target timestamp.
+ *
+ * @param root the targets that will be updated.
+ */
+static inline void post_aggregate_targets(struct target *root)
+{
+ struct target *w;
+ for (w = root; w; w = w->next) {
+ if (w->collected_starttime) {
+ if (!w->starttime || w->collected_starttime < w->starttime) {
+ w->starttime = w->collected_starttime;
+ }
+ } else {
+ w->starttime = 0;
+ }
+ }
+}
+
+/**
+ * Remove PID from the link list.
+ *
+ * @param pid the PID that will be removed.
+ */
+static inline void del_pid_entry(pid_t pid)
+{
+ struct pid_stat *p = all_pids[pid];
+
+ if (unlikely(!p)) {
+ error("attempted to free pid %d that is not allocated.", pid);
+ return;
+ }
+
+ debug_log("process %d %s exited, deleting it.", pid, p->comm);
+
+ if (root_of_pids == p)
+ root_of_pids = p->next;
+
+ if (p->next)
+ p->next->prev = p->prev;
+ if (p->prev)
+ p->prev->next = p->next;
+
+ freez(p->stat_filename);
+ freez(p->status_filename);
+ freez(p->io_filename);
+ freez(p->cmdline_filename);
+ freez(p->cmdline);
+ freez(p);
+
+ all_pids[pid] = NULL;
+ all_pids_count--;
+}
+
+/**
+ * Get command string associated with a PID.
+ * This can only safely be used when holding the `collect_data_mutex` lock.
+ *
+ * @param pid the pid to search the data.
+ * @param n the maximum amount of bytes to copy into dest.
+ * if this is greater than the size of the command, it is clipped.
+ * @param dest the target memory buffer to write the command into.
+ * @return -1 if the PID hasn't been scraped yet, 0 otherwise.
+ */
+int get_pid_comm(pid_t pid, size_t n, char *dest)
+{
+ struct pid_stat *stat;
+
+ stat = all_pids[pid];
+ if (unlikely(stat == NULL)) {
+ return -1;
+ }
+
+ if (unlikely(n > sizeof(stat->comm))) {
+ n = sizeof(stat->comm);
+ }
+
+ strncpyz(dest, stat->comm, n);
+ return 0;
+}
+
+/**
+ * Cleanup variable from other threads
+ *
+ * @param pid current pid.
+ */
+void cleanup_variables_from_other_threads(uint32_t pid)
+{
+ // Clean socket structures
+ if (socket_bandwidth_curr) {
+ freez(socket_bandwidth_curr[pid]);
+ socket_bandwidth_curr[pid] = NULL;
+ }
+
+ // Clean cachestat structure
+ if (cachestat_pid) {
+ freez(cachestat_pid[pid]);
+ cachestat_pid[pid] = NULL;
+ }
+
+ // Clean directory cache structure
+ if (dcstat_pid) {
+ freez(dcstat_pid[pid]);
+ dcstat_pid[pid] = NULL;
+ }
+
+ // Clean swap structure
+ if (swap_pid) {
+ freez(swap_pid[pid]);
+ swap_pid[pid] = NULL;
+ }
+
+ // Clean vfs structure
+ if (vfs_pid) {
+ freez(vfs_pid[pid]);
+ vfs_pid[pid] = NULL;
+ }
+
+ // Clean fd structure
+ if (fd_pid) {
+ freez(fd_pid[pid]);
+ fd_pid[pid] = NULL;
+ }
+
+ // Clean shm structure
+ if (shm_pid) {
+ freez(shm_pid[pid]);
+ shm_pid[pid] = NULL;
+ }
+}
+
+/**
+ * Remove PIDs when they are not running more.
+ */
+void cleanup_exited_pids()
+{
+ struct pid_stat *p = NULL;
+
+ for (p = root_of_pids; p;) {
+ if (!p->updated && (!p->keep || p->keeploops > 0)) {
+ if (unlikely(debug_enabled && (p->keep || p->keeploops)))
+ debug_log(" > CLEANUP cannot keep exited process %d (%s) anymore - removing it.", p->pid, p->comm);
+
+ pid_t r = p->pid;
+ p = p->next;
+
+ // Clean process structure
+ freez(global_process_stats[r]);
+ global_process_stats[r] = NULL;
+
+ freez(current_apps_data[r]);
+ current_apps_data[r] = NULL;
+
+ cleanup_variables_from_other_threads(r);
+
+ del_pid_entry(r);
+ } else {
+ if (unlikely(p->keep))
+ p->keeploops++;
+ p->keep = 0;
+ p = p->next;
+ }
+ }
+}
+
+/**
+ * Read proc filesystem for the first time.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+static inline void read_proc_filesystem()
+{
+ char dirname[FILENAME_MAX + 1];
+
+ snprintfz(dirname, FILENAME_MAX, "%s/proc", netdata_configured_host_prefix);
+ DIR *dir = opendir(dirname);
+ if (!dir)
+ return;
+
+ struct dirent *de = NULL;
+
+ while ((de = readdir(dir))) {
+ char *endptr = de->d_name;
+
+ if (unlikely(de->d_type != DT_DIR || de->d_name[0] < '0' || de->d_name[0] > '9'))
+ continue;
+
+ pid_t pid = (pid_t)strtoul(de->d_name, &endptr, 10);
+
+ // make sure we read a valid number
+ if (unlikely(endptr == de->d_name || *endptr != '\0'))
+ continue;
+
+ collect_data_for_pid(pid, NULL);
+ }
+ closedir(dir);
+}
+
+/**
+ * Aggregated PID on target
+ *
+ * @param w the target output
+ * @param p the pid with information to update
+ * @param o never used
+ */
+static inline void aggregate_pid_on_target(struct target *w, struct pid_stat *p, struct target *o)
+{
+ UNUSED(o);
+
+ if (unlikely(!p->updated)) {
+ // the process is not running
+ return;
+ }
+
+ if (unlikely(!w)) {
+ error("pid %d %s was left without a target!", p->pid, p->comm);
+ return;
+ }
+
+ w->processes++;
+ struct pid_on_target *pid_on_target = mallocz(sizeof(struct pid_on_target));
+ pid_on_target->pid = p->pid;
+ pid_on_target->next = w->root_pid;
+ w->root_pid = pid_on_target;
+}
+
+/**
+ * Collect data for all process
+ *
+ * Read data from hash table and store it in appropriate vectors.
+ * It also creates the link between targets and PIDs.
+ *
+ * @param tbl_pid_stats_fd The mapped file descriptor for the hash table.
+ */
+void collect_data_for_all_processes(int tbl_pid_stats_fd)
+{
+ if (unlikely(!all_pids))
+ return;
+
+ struct pid_stat *pids = root_of_pids; // global list of all processes running
+ while (pids) {
+ if (pids->updated_twice) {
+ pids->read = 0; // mark it as not read, so that collect_data_for_pid() will read it
+ pids->updated = 0;
+ pids->merged = 0;
+ pids->children_count = 0;
+ pids->parent = NULL;
+ } else {
+ if (pids->updated)
+ pids->updated_twice = 1;
+ }
+
+ pids = pids->next;
+ }
+
+ read_proc_filesystem();
+
+ uint32_t key;
+ pids = root_of_pids; // global list of all processes running
+ // while (bpf_map_get_next_key(tbl_pid_stats_fd, &key, &next_key) == 0) {
+ while (pids) {
+ key = pids->pid;
+ ebpf_process_stat_t *w = global_process_stats[key];
+ if (!w) {
+ w = callocz(1, sizeof(ebpf_process_stat_t));
+ global_process_stats[key] = w;
+ }
+
+ if (bpf_map_lookup_elem(tbl_pid_stats_fd, &key, w)) {
+ // Clean Process structures
+ freez(w);
+ global_process_stats[key] = NULL;
+
+ freez(current_apps_data[key]);
+ current_apps_data[key] = NULL;
+
+ cleanup_variables_from_other_threads(key);
+
+ pids = pids->next;
+ continue;
+ }
+
+ pids = pids->next;
+ }
+
+ link_all_processes_to_their_parents();
+
+ apply_apps_groups_targets_inheritance();
+
+ apps_groups_targets_count = zero_all_targets(apps_groups_root_target);
+
+ // this has to be done, before the cleanup
+ // // concentrate everything on the targets
+ for (pids = root_of_pids; pids; pids = pids->next)
+ aggregate_pid_on_target(pids->target, pids, NULL);
+
+ post_aggregate_targets(apps_groups_root_target);
+}
diff --git a/collectors/ebpf.plugin/ebpf_apps.h b/collectors/ebpf.plugin/ebpf_apps.h
new file mode 100644
index 0000000..0bea912
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_apps.h
@@ -0,0 +1,441 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_APPS_H
+#define NETDATA_EBPF_APPS_H 1
+
+#include "libnetdata/threads/threads.h"
+#include "libnetdata/locks/locks.h"
+#include "libnetdata/avl/avl.h"
+#include "libnetdata/clocks/clocks.h"
+#include "libnetdata/config/appconfig.h"
+#include "libnetdata/ebpf/ebpf.h"
+
+#define NETDATA_APPS_FAMILY "apps"
+#define NETDATA_APPS_FILE_GROUP "file_access"
+#define NETDATA_APPS_FILE_CGROUP_GROUP "file_access (eBPF)"
+#define NETDATA_APPS_PROCESS_GROUP "process (eBPF)"
+#define NETDATA_APPS_NET_GROUP "net"
+#define NETDATA_APPS_IPC_SHM_GROUP "ipc shm (eBPF)"
+
+#include "ebpf_process.h"
+#include "ebpf_dcstat.h"
+#include "ebpf_disk.h"
+#include "ebpf_fd.h"
+#include "ebpf_filesystem.h"
+#include "ebpf_hardirq.h"
+#include "ebpf_cachestat.h"
+#include "ebpf_mdflush.h"
+#include "ebpf_mount.h"
+#include "ebpf_oomkill.h"
+#include "ebpf_shm.h"
+#include "ebpf_socket.h"
+#include "ebpf_softirq.h"
+#include "ebpf_sync.h"
+#include "ebpf_swap.h"
+#include "ebpf_vfs.h"
+
+#define MAX_COMPARE_NAME 100
+#define MAX_NAME 100
+
+// ----------------------------------------------------------------------------
+// process_pid_stat
+//
+// Fields read from the kernel ring for a specific PID
+//
+typedef struct process_pid_stat {
+ uint64_t pid_tgid; // Unique identifier
+ uint32_t pid; // process id
+
+ // Count number of calls done for specific function
+ uint32_t open_call;
+ uint32_t write_call;
+ uint32_t writev_call;
+ uint32_t read_call;
+ uint32_t readv_call;
+ uint32_t unlink_call;
+ uint32_t exit_call;
+ uint32_t release_call;
+ uint32_t fork_call;
+ uint32_t clone_call;
+ uint32_t close_call;
+
+ // Count number of bytes written or read
+ uint64_t write_bytes;
+ uint64_t writev_bytes;
+ uint64_t readv_bytes;
+ uint64_t read_bytes;
+
+ // Count number of errors for the specified function
+ uint32_t open_err;
+ uint32_t write_err;
+ uint32_t writev_err;
+ uint32_t read_err;
+ uint32_t readv_err;
+ uint32_t unlink_err;
+ uint32_t fork_err;
+ uint32_t clone_err;
+ uint32_t close_err;
+} process_pid_stat_t;
+
+// ----------------------------------------------------------------------------
+// socket_bandwidth
+//
+// Fields read from the kernel ring for a specific PID
+//
+typedef struct socket_bandwidth {
+ uint64_t first;
+ uint64_t ct;
+ uint64_t sent;
+ uint64_t received;
+ unsigned char removed;
+} socket_bandwidth_t;
+
+// ----------------------------------------------------------------------------
+// pid_stat
+//
+// structure to store data for each process running
+// see: man proc for the description of the fields
+
+struct pid_fd {
+ int fd;
+
+#ifndef __FreeBSD__
+ ino_t inode;
+ char *filename;
+ uint32_t link_hash;
+ size_t cache_iterations_counter;
+ size_t cache_iterations_reset;
+#endif
+};
+
+struct target {
+ char compare[MAX_COMPARE_NAME + 1];
+ uint32_t comparehash;
+ size_t comparelen;
+
+ char id[MAX_NAME + 1];
+ uint32_t idhash;
+
+ char name[MAX_NAME + 1];
+
+ uid_t uid;
+ gid_t gid;
+
+ // Changes made to simplify integration between apps and eBPF.
+ netdata_publish_cachestat_t cachestat;
+ netdata_publish_dcstat_t dcstat;
+ netdata_publish_swap_t swap;
+ netdata_publish_vfs_t vfs;
+ netdata_fd_stat_t fd;
+ netdata_publish_shm_t shm;
+
+ /* These variables are not necessary for eBPF collector
+ kernel_uint_t minflt;
+ kernel_uint_t cminflt;
+ kernel_uint_t majflt;
+ kernel_uint_t cmajflt;
+ kernel_uint_t utime;
+ kernel_uint_t stime;
+ kernel_uint_t gtime;
+ kernel_uint_t cutime;
+ kernel_uint_t cstime;
+ kernel_uint_t cgtime;
+ kernel_uint_t num_threads;
+ // kernel_uint_t rss;
+
+ kernel_uint_t status_vmsize;
+ kernel_uint_t status_vmrss;
+ kernel_uint_t status_vmshared;
+ kernel_uint_t status_rssfile;
+ kernel_uint_t status_rssshmem;
+ kernel_uint_t status_vmswap;
+
+ kernel_uint_t io_logical_bytes_read;
+ kernel_uint_t io_logical_bytes_written;
+ // kernel_uint_t io_read_calls;
+ // kernel_uint_t io_write_calls;
+ kernel_uint_t io_storage_bytes_read;
+ kernel_uint_t io_storage_bytes_written;
+ // kernel_uint_t io_cancelled_write_bytes;
+
+ int *target_fds;
+ int target_fds_size;
+
+ kernel_uint_t openfiles;
+ kernel_uint_t openpipes;
+ kernel_uint_t opensockets;
+ kernel_uint_t openinotifies;
+ kernel_uint_t openeventfds;
+ kernel_uint_t opentimerfds;
+ kernel_uint_t opensignalfds;
+ kernel_uint_t openeventpolls;
+ kernel_uint_t openother;
+ */
+
+ kernel_uint_t starttime;
+ kernel_uint_t collected_starttime;
+
+ /*
+ kernel_uint_t uptime_min;
+ kernel_uint_t uptime_sum;
+ kernel_uint_t uptime_max;
+ */
+
+ unsigned int processes; // how many processes have been merged to this
+ int exposed; // if set, we have sent this to netdata
+ int hidden; // if set, we set the hidden flag on the dimension
+ int debug_enabled;
+ int ends_with;
+ int starts_with; // if set, the compare string matches only the
+ // beginning of the command
+
+ struct pid_on_target *root_pid; // list of aggregated pids for target debugging
+
+ struct target *target; // the one that will be reported to netdata
+ struct target *next;
+};
+
+extern struct target *apps_groups_default_target;
+extern struct target *apps_groups_root_target;
+extern struct target *users_root_target;
+extern struct target *groups_root_target;
+
+struct pid_stat {
+ int32_t pid;
+ char comm[MAX_COMPARE_NAME + 1];
+ char *cmdline;
+
+ uint32_t log_thrown;
+
+ // char state;
+ int32_t ppid;
+
+ // int32_t pgrp;
+ // int32_t session;
+ // int32_t tty_nr;
+ // int32_t tpgid;
+ // uint64_t flags;
+
+ /*
+ // these are raw values collected
+ kernel_uint_t minflt_raw;
+ kernel_uint_t cminflt_raw;
+ kernel_uint_t majflt_raw;
+ kernel_uint_t cmajflt_raw;
+ kernel_uint_t utime_raw;
+ kernel_uint_t stime_raw;
+ kernel_uint_t gtime_raw; // guest_time
+ kernel_uint_t cutime_raw;
+ kernel_uint_t cstime_raw;
+ kernel_uint_t cgtime_raw; // cguest_time
+
+ // these are rates
+ kernel_uint_t minflt;
+ kernel_uint_t cminflt;
+ kernel_uint_t majflt;
+ kernel_uint_t cmajflt;
+ kernel_uint_t utime;
+ kernel_uint_t stime;
+ kernel_uint_t gtime;
+ kernel_uint_t cutime;
+ kernel_uint_t cstime;
+ kernel_uint_t cgtime;
+
+ // int64_t priority;
+ // int64_t nice;
+ int32_t num_threads;
+ // int64_t itrealvalue;
+ kernel_uint_t collected_starttime;
+ // kernel_uint_t vsize;
+ // kernel_uint_t rss;
+ // kernel_uint_t rsslim;
+ // kernel_uint_t starcode;
+ // kernel_uint_t endcode;
+ // kernel_uint_t startstack;
+ // kernel_uint_t kstkesp;
+ // kernel_uint_t kstkeip;
+ // uint64_t signal;
+ // uint64_t blocked;
+ // uint64_t sigignore;
+ // uint64_t sigcatch;
+ // uint64_t wchan;
+ // uint64_t nswap;
+ // uint64_t cnswap;
+ // int32_t exit_signal;
+ // int32_t processor;
+ // uint32_t rt_priority;
+ // uint32_t policy;
+ // kernel_uint_t delayacct_blkio_ticks;
+
+ uid_t uid;
+ gid_t gid;
+
+ kernel_uint_t status_vmsize;
+ kernel_uint_t status_vmrss;
+ kernel_uint_t status_vmshared;
+ kernel_uint_t status_rssfile;
+ kernel_uint_t status_rssshmem;
+ kernel_uint_t status_vmswap;
+#ifndef __FreeBSD__
+ ARL_BASE *status_arl;
+#endif
+
+ kernel_uint_t io_logical_bytes_read_raw;
+ kernel_uint_t io_logical_bytes_written_raw;
+ // kernel_uint_t io_read_calls_raw;
+ // kernel_uint_t io_write_calls_raw;
+ kernel_uint_t io_storage_bytes_read_raw;
+ kernel_uint_t io_storage_bytes_written_raw;
+ // kernel_uint_t io_cancelled_write_bytes_raw;
+
+ kernel_uint_t io_logical_bytes_read;
+ kernel_uint_t io_logical_bytes_written;
+ // kernel_uint_t io_read_calls;
+ // kernel_uint_t io_write_calls;
+ kernel_uint_t io_storage_bytes_read;
+ kernel_uint_t io_storage_bytes_written;
+ // kernel_uint_t io_cancelled_write_bytes;
+ */
+
+ struct pid_fd *fds; // array of fds it uses
+ size_t fds_size; // the size of the fds array
+
+ int children_count; // number of processes directly referencing this
+ unsigned char keep : 1; // 1 when we need to keep this process in memory even after it exited
+ int keeploops; // increases by 1 every time keep is 1 and updated 0
+ unsigned char updated : 1; // 1 when the process is currently running
+ unsigned char updated_twice : 1; // 1 when the process was running in the previous iteration
+ unsigned char merged : 1; // 1 when it has been merged to its parent
+ unsigned char read : 1; // 1 when we have already read this process for this iteration
+
+ int sortlist; // higher numbers = top on the process tree
+
+ // each process gets a unique number
+
+ struct target *target; // app_groups.conf targets
+ struct target *user_target; // uid based targets
+ struct target *group_target; // gid based targets
+
+ usec_t stat_collected_usec;
+ usec_t last_stat_collected_usec;
+
+ usec_t io_collected_usec;
+ usec_t last_io_collected_usec;
+
+ kernel_uint_t uptime;
+
+ char *fds_dirname; // the full directory name in /proc/PID/fd
+
+ char *stat_filename;
+ char *status_filename;
+ char *io_filename;
+ char *cmdline_filename;
+
+ struct pid_stat *parent;
+ struct pid_stat *prev;
+ struct pid_stat *next;
+};
+
+// ----------------------------------------------------------------------------
+// target
+//
+// target is the structure that processes are aggregated to be reported
+// to netdata.
+//
+// - Each entry in /etc/apps_groups.conf creates a target.
+// - Each user and group used by a process in the system, creates a target.
+struct pid_on_target {
+ int32_t pid;
+ struct pid_on_target *next;
+};
+
+// ----------------------------------------------------------------------------
+// Structures used to read information from kernel ring
+typedef struct ebpf_process_stat {
+ uint64_t pid_tgid;
+ uint32_t pid;
+
+ //Counter
+ uint32_t exit_call;
+ uint32_t release_call;
+ uint32_t create_process;
+ uint32_t create_thread;
+
+ //Counter
+ uint32_t task_err;
+
+ uint8_t removeme;
+} ebpf_process_stat_t;
+
+typedef struct ebpf_bandwidth {
+ uint32_t pid;
+
+ uint64_t first; // First timestamp
+ uint64_t ct; // Last timestamp
+ uint64_t bytes_sent; // Bytes sent
+ uint64_t bytes_received; // Bytes received
+ uint64_t call_tcp_sent; // Number of times tcp_sendmsg was called
+ uint64_t call_tcp_received; // Number of times tcp_cleanup_rbuf was called
+ uint64_t retransmit; // Number of times tcp_retransmit was called
+ uint64_t call_udp_sent; // Number of times udp_sendmsg was called
+ uint64_t call_udp_received; // Number of times udp_recvmsg was called
+ uint64_t close; // Number of times tcp_close was called
+ uint64_t drop; // THIS IS NOT USED FOR WHILE, we are in groom section
+ uint32_t tcp_v4_connection; // Number of times tcp_v4_connection was called.
+ uint32_t tcp_v6_connection; // Number of times tcp_v6_connection was called.
+} ebpf_bandwidth_t;
+
+/**
+ * Internal function used to write debug messages.
+ *
+ * @param fmt the format to create the message.
+ * @param ... the arguments to fill the format.
+ */
+static inline void debug_log_int(const char *fmt, ...)
+{
+ va_list args;
+
+ fprintf(stderr, "apps.plugin: ");
+ va_start(args, fmt);
+ vfprintf(stderr, fmt, args);
+ va_end(args);
+
+ fputc('\n', stderr);
+}
+
+// ----------------------------------------------------------------------------
+// Exported variabled and functions
+//
+extern struct pid_stat **all_pids;
+
+int ebpf_read_apps_groups_conf(struct target **apps_groups_default_target,
+ struct target **apps_groups_root_target,
+ const char *path,
+ const char *file);
+
+void clean_apps_groups_target(struct target *apps_groups_root_target);
+
+size_t zero_all_targets(struct target *root);
+
+int am_i_running_as_root();
+
+void cleanup_exited_pids();
+
+int ebpf_read_hash_table(void *ep, int fd, uint32_t pid);
+
+int get_pid_comm(pid_t pid, size_t n, char *dest);
+
+size_t read_processes_statistic_using_pid_on_target(ebpf_process_stat_t **ep,
+ int fd,
+ struct pid_on_target *pids);
+
+size_t read_bandwidth_statistic_using_pid_on_target(ebpf_bandwidth_t **ep, int fd, struct pid_on_target *pids);
+
+void collect_data_for_all_processes(int tbl_pid_stats_fd);
+
+extern ebpf_process_stat_t **global_process_stats;
+extern ebpf_process_publish_apps_t **current_apps_data;
+extern netdata_publish_cachestat_t **cachestat_pid;
+extern netdata_publish_dcstat_t **dcstat_pid;
+
+#endif /* NETDATA_EBPF_APPS_H */
diff --git a/collectors/ebpf.plugin/ebpf_cachestat.c b/collectors/ebpf.plugin/ebpf_cachestat.c
new file mode 100644
index 0000000..4c41064
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_cachestat.c
@@ -0,0 +1,1335 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_cachestat.h"
+
+netdata_publish_cachestat_t **cachestat_pid;
+
+static char *cachestat_counter_dimension_name[NETDATA_CACHESTAT_END] = { "ratio", "dirty", "hit",
+ "miss" };
+static netdata_syscall_stat_t cachestat_counter_aggregated_data[NETDATA_CACHESTAT_END];
+static netdata_publish_syscall_t cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_END];
+
+netdata_cachestat_pid_t *cachestat_vector = NULL;
+
+static netdata_idx_t cachestat_hash_values[NETDATA_CACHESTAT_END];
+static netdata_idx_t *cachestat_values = NULL;
+
+struct netdata_static_thread cachestat_threads = {.name = "CACHESTAT KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL};
+
+ebpf_local_maps_t cachestat_maps[] = {{.name = "cstat_global", .internal_input = NETDATA_CACHESTAT_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "cstat_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "cstat_ctrl", .internal_input = NETDATA_CONTROLLER_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
+
+struct config cachestat_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+netdata_ebpf_targets_t cachestat_targets[] = { {.name = "add_to_page_cache_lru", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "mark_page_accessed", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "mark_buffer_dirty", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/cachestat.skel.h" // BTF code
+
+static struct cachestat_bpf *bpf_obj = NULL;
+
+/**
+ * Disable probe
+ *
+ * Disable all probes to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects
+ */
+static void ebpf_cachestat_disable_probe(struct cachestat_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_add_to_page_cache_lru_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_mark_page_accessed_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_mark_buffer_dirty_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_release_task_kprobe, false);
+}
+
+/*
+ * Disable specific probe
+ *
+ * Disable probes according the kernel version
+ *
+ * @param obj is the main structure for bpf objects
+ */
+static void ebpf_cachestat_disable_specific_probe(struct cachestat_bpf *obj)
+{
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_16) {
+ bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_kprobe, false);
+ } else if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_15) {
+ bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_kprobe, false);
+ } else {
+ bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_kprobe, false);
+ }
+}
+
+/*
+ * Disable trampoline
+ *
+ * Disable all trampoline to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_cachestat_disable_trampoline(struct cachestat_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_add_to_page_cache_lru_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_mark_page_accessed_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_mark_buffer_dirty_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fentry, false);
+}
+
+/*
+ * Disable specific trampoline
+ *
+ * Disable trampoline according to kernel version.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_cachestat_disable_specific_trampoline(struct cachestat_bpf *obj)
+{
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_16) {
+ bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_fentry, false);
+ } else if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_15) {
+ bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_fentry, false);
+ } else {
+ bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_fentry, false);
+ }
+}
+
+/**
+ * Set trampoline target
+ *
+ * Set the targets we will monitor.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void netdata_set_trampoline_target(struct cachestat_bpf *obj)
+{
+ bpf_program__set_attach_target(obj->progs.netdata_add_to_page_cache_lru_fentry, 0,
+ cachestat_targets[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_mark_page_accessed_fentry, 0,
+ cachestat_targets[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED].name);
+
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_16) {
+ bpf_program__set_attach_target(obj->progs.netdata_folio_mark_dirty_fentry, 0,
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
+ } else if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_15) {
+ bpf_program__set_attach_target(obj->progs.netdata_set_page_dirty_fentry, 0,
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
+ } else {
+ bpf_program__set_attach_target(obj->progs.netdata_account_page_dirtied_fentry, 0,
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
+ }
+
+ bpf_program__set_attach_target(obj->progs.netdata_mark_buffer_dirty_fentry, 0,
+ cachestat_targets[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_release_task_fentry, 0,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+}
+
+/**
+ * Mount Attach Probe
+ *
+ * Attach probes to target
+ *
+ * @param obj is the main structure for bpf objects.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+static int ebpf_cachestat_attach_probe(struct cachestat_bpf *obj)
+{
+ obj->links.netdata_add_to_page_cache_lru_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_add_to_page_cache_lru_kprobe,
+ false,
+ cachestat_targets[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].name);
+ int ret = libbpf_get_error(obj->links.netdata_add_to_page_cache_lru_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_mark_page_accessed_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_mark_page_accessed_kprobe,
+ false,
+ cachestat_targets[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED].name);
+ ret = libbpf_get_error(obj->links.netdata_mark_page_accessed_kprobe);
+ if (ret)
+ return -1;
+
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_16) {
+ obj->links.netdata_folio_mark_dirty_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_folio_mark_dirty_kprobe,
+ false,
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
+ ret = libbpf_get_error(obj->links.netdata_folio_mark_dirty_kprobe);
+ } else if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_15) {
+ obj->links.netdata_set_page_dirty_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_set_page_dirty_kprobe,
+ false,
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
+ ret = libbpf_get_error(obj->links.netdata_set_page_dirty_kprobe);
+ } else {
+ obj->links.netdata_account_page_dirtied_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_account_page_dirtied_kprobe,
+ false,
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
+ ret = libbpf_get_error(obj->links.netdata_account_page_dirtied_kprobe);
+ }
+
+ if (ret)
+ return -1;
+
+ obj->links.netdata_mark_buffer_dirty_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_mark_buffer_dirty_kprobe,
+ false,
+ cachestat_targets[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY].name);
+ ret = libbpf_get_error(obj->links.netdata_mark_buffer_dirty_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_release_task_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_release_task_kprobe,
+ false,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+ ret = libbpf_get_error(obj->links.netdata_release_task_kprobe);
+ if (ret)
+ return -1;
+
+ return 0;
+}
+
+/**
+ * Adjust Map Size
+ *
+ * Resize maps according input from users.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ */
+static void ebpf_cachestat_adjust_map_size(struct cachestat_bpf *obj, ebpf_module_t *em)
+{
+ ebpf_update_map_size(obj->maps.cstat_pid, &cachestat_maps[NETDATA_CACHESTAT_PID_STATS],
+ em, bpf_map__name(obj->maps.cstat_pid));
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_cachestat_set_hash_tables(struct cachestat_bpf *obj)
+{
+ cachestat_maps[NETDATA_CACHESTAT_GLOBAL_STATS].map_fd = bpf_map__fd(obj->maps.cstat_global);
+ cachestat_maps[NETDATA_CACHESTAT_PID_STATS].map_fd = bpf_map__fd(obj->maps.cstat_pid);
+ cachestat_maps[NETDATA_CACHESTAT_CTRL].map_fd = bpf_map__fd(obj->maps.cstat_ctrl);
+}
+
+/**
+ * Disable Release Task
+ *
+ * Disable release task when apps is not enabled.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_cachestat_disable_release_task(struct cachestat_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_release_task_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fentry, false);
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_cachestat_load_and_attach(struct cachestat_bpf *obj, ebpf_module_t *em)
+{
+ netdata_ebpf_targets_t *mt = em->targets;
+ netdata_ebpf_program_loaded_t test = mt[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].mode;
+
+ if (test == EBPF_LOAD_TRAMPOLINE) {
+ ebpf_cachestat_disable_probe(obj);
+ ebpf_cachestat_disable_specific_trampoline(obj);
+
+ netdata_set_trampoline_target(obj);
+ } else {
+ ebpf_cachestat_disable_trampoline(obj);
+ ebpf_cachestat_disable_specific_probe(obj);
+ }
+
+ ebpf_cachestat_adjust_map_size(obj, em);
+
+ if (!em->apps_charts && !em->cgroup_charts)
+ ebpf_cachestat_disable_release_task(obj);
+
+ int ret = cachestat_bpf__load(obj);
+ if (ret) {
+ return ret;
+ }
+
+ ret = (test == EBPF_LOAD_TRAMPOLINE) ? cachestat_bpf__attach(obj) : ebpf_cachestat_attach_probe(obj);
+ if (!ret) {
+ ebpf_cachestat_set_hash_tables(obj);
+
+ ebpf_update_controller(cachestat_maps[NETDATA_CACHESTAT_CTRL].map_fd, em);
+ }
+
+ return ret;
+}
+#endif
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Cachestat Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_cachestat_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ ebpf_cleanup_publish_syscall(cachestat_counter_publish_aggregated);
+
+ freez(cachestat_vector);
+ freez(cachestat_values);
+ freez(cachestat_threads.thread);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ if (bpf_obj)
+ cachestat_bpf__destroy(bpf_obj);
+#endif
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Cachestat exit.
+ *
+ * Cancel child and exit.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_cachestat_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*cachestat_threads.thread);
+ ebpf_cachestat_free(em);
+}
+
+/**
+ * Cachestat cleanup
+ *
+ * Clean up allocated addresses.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_cachestat_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_cachestat_free(em);
+}
+
+/*****************************************************************
+ *
+ * COMMON FUNCTIONS
+ *
+ *****************************************************************/
+
+/**
+ * Update publish
+ *
+ * Update publish values before to write dimension.
+ *
+ * @param out structure that will receive data.
+ * @param mpa calls for mark_page_accessed during the last second.
+ * @param mbd calls for mark_buffer_dirty during the last second.
+ * @param apcl calls for add_to_page_cache_lru during the last second.
+ * @param apd calls for account_page_dirtied during the last second.
+ */
+void cachestat_update_publish(netdata_publish_cachestat_t *out, uint64_t mpa, uint64_t mbd,
+ uint64_t apcl, uint64_t apd)
+{
+ // Adapted algorithm from https://github.com/iovisor/bcc/blob/master/tools/cachestat.py#L126-L138
+ NETDATA_DOUBLE total = (NETDATA_DOUBLE) (((long long)mpa) - ((long long)mbd));
+ if (total < 0)
+ total = 0;
+
+ NETDATA_DOUBLE misses = (NETDATA_DOUBLE) ( ((long long) apcl) - ((long long) apd) );
+ if (misses < 0)
+ misses = 0;
+
+ // If hits are < 0, then its possible misses are overestimate due to possibly page cache read ahead adding
+ // more pages than needed. In this case just assume misses as total and reset hits.
+ NETDATA_DOUBLE hits = total - misses;
+ if (hits < 0 ) {
+ misses = total;
+ hits = 0;
+ }
+
+ NETDATA_DOUBLE ratio = (total > 0) ? hits/total : 1;
+
+ out->ratio = (long long )(ratio*100);
+ out->hit = (long long)hits;
+ out->miss = (long long)misses;
+}
+
+/**
+ * Save previous values
+ *
+ * Save values used this time.
+ *
+ * @param publish
+ */
+static void save_previous_values(netdata_publish_cachestat_t *publish) {
+ publish->prev.mark_page_accessed = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED];
+ publish->prev.account_page_dirtied = cachestat_hash_values[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED];
+ publish->prev.add_to_page_cache_lru = cachestat_hash_values[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU];
+ publish->prev.mark_buffer_dirty = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY];
+}
+
+/**
+ * Calculate statistics
+ *
+ * @param publish the structure where we will store the data.
+ */
+static void calculate_stats(netdata_publish_cachestat_t *publish) {
+ if (!publish->prev.mark_page_accessed) {
+ save_previous_values(publish);
+ return;
+ }
+
+ uint64_t mpa = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED] - publish->prev.mark_page_accessed;
+ uint64_t mbd = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY] - publish->prev.mark_buffer_dirty;
+ uint64_t apcl = cachestat_hash_values[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU] - publish->prev.add_to_page_cache_lru;
+ uint64_t apd = cachestat_hash_values[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED] - publish->prev.account_page_dirtied;
+
+ save_previous_values(publish);
+
+ // We are changing the original algorithm to have a smooth ratio.
+ cachestat_update_publish(publish, mpa, mbd, apcl, apd);
+}
+
+
+/*****************************************************************
+ *
+ * APPS
+ *
+ *****************************************************************/
+
+/**
+ * Apps Accumulator
+ *
+ * Sum all values read from kernel and store in the first address.
+ *
+ * @param out the vector with read values.
+ */
+static void cachestat_apps_accumulator(netdata_cachestat_pid_t *out)
+{
+ int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1;
+ netdata_cachestat_pid_t *total = &out[0];
+ for (i = 1; i < end; i++) {
+ netdata_cachestat_pid_t *w = &out[i];
+ total->account_page_dirtied += w->account_page_dirtied;
+ total->add_to_page_cache_lru += w->add_to_page_cache_lru;
+ total->mark_buffer_dirty += w->mark_buffer_dirty;
+ total->mark_page_accessed += w->mark_page_accessed;
+ }
+}
+
+/**
+ * Save Pid values
+ *
+ * Save the current values inside the structure
+ *
+ * @param out vector used to plot charts
+ * @param publish vector with values read from hash tables.
+ */
+static inline void cachestat_save_pid_values(netdata_publish_cachestat_t *out, netdata_cachestat_pid_t *publish)
+{
+ if (!out->current.mark_page_accessed) {
+ memcpy(&out->current, &publish[0], sizeof(netdata_cachestat_pid_t));
+ return;
+ }
+
+ memcpy(&out->prev, &out->current, sizeof(netdata_cachestat_pid_t));
+ memcpy(&out->current, &publish[0], sizeof(netdata_cachestat_pid_t));
+}
+
+/**
+ * Fill PID
+ *
+ * Fill PID structures
+ *
+ * @param current_pid pid that we are collecting data
+ * @param out values read from hash tables;
+ */
+static void cachestat_fill_pid(uint32_t current_pid, netdata_cachestat_pid_t *publish)
+{
+ netdata_publish_cachestat_t *curr = cachestat_pid[current_pid];
+ if (!curr) {
+ curr = callocz(1, sizeof(netdata_publish_cachestat_t));
+ cachestat_pid[current_pid] = curr;
+
+ cachestat_save_pid_values(curr, publish);
+ return;
+ }
+
+ cachestat_save_pid_values(curr, publish);
+}
+
+/**
+ * Read APPS table
+ *
+ * Read the apps table and store data inside the structure.
+ */
+static void read_apps_table()
+{
+ netdata_cachestat_pid_t *cv = cachestat_vector;
+ uint32_t key;
+ struct pid_stat *pids = root_of_pids;
+ int fd = cachestat_maps[NETDATA_CACHESTAT_PID_STATS].map_fd;
+ size_t length = sizeof(netdata_cachestat_pid_t)*ebpf_nprocs;
+ while (pids) {
+ key = pids->pid;
+
+ if (bpf_map_lookup_elem(fd, &key, cv)) {
+ pids = pids->next;
+ continue;
+ }
+
+ cachestat_apps_accumulator(cv);
+
+ cachestat_fill_pid(key, cv);
+
+ // We are cleaning to avoid passing data read from one process to other.
+ memset(cv, 0, length);
+
+ pids = pids->next;
+ }
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ */
+static void ebpf_update_cachestat_cgroup()
+{
+ netdata_cachestat_pid_t *cv = cachestat_vector;
+ int fd = cachestat_maps[NETDATA_CACHESTAT_PID_STATS].map_fd;
+ size_t length = sizeof(netdata_cachestat_pid_t) * ebpf_nprocs;
+
+ ebpf_cgroup_target_t *ect;
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ int pid = pids->pid;
+ netdata_cachestat_pid_t *out = &pids->cachestat;
+ if (likely(cachestat_pid) && cachestat_pid[pid]) {
+ netdata_publish_cachestat_t *in = cachestat_pid[pid];
+
+ memcpy(out, &in->current, sizeof(netdata_cachestat_pid_t));
+ } else {
+ memset(cv, 0, length);
+ if (bpf_map_lookup_elem(fd, &pid, cv)) {
+ continue;
+ }
+
+ cachestat_apps_accumulator(cv);
+
+ memcpy(out, cv, sizeof(netdata_cachestat_pid_t));
+ }
+ }
+ }
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Create apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+void ebpf_cachestat_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+ ebpf_create_charts_on_apps(NETDATA_CACHESTAT_HIT_RATIO_CHART,
+ "Hit ratio",
+ EBPF_COMMON_DIMENSION_PERCENTAGE,
+ NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 20090,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_charts_on_apps(NETDATA_CACHESTAT_DIRTY_CHART,
+ "Number of dirty pages",
+ EBPF_CACHESTAT_DIMENSION_PAGE,
+ NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20091,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_charts_on_apps(NETDATA_CACHESTAT_HIT_CHART,
+ "Number of accessed files",
+ EBPF_CACHESTAT_DIMENSION_HITS,
+ NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20092,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_charts_on_apps(NETDATA_CACHESTAT_MISSES_CHART,
+ "Files out of page cache",
+ EBPF_CACHESTAT_DIMENSION_MISSES,
+ NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20093,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/*****************************************************************
+ *
+ * MAIN LOOP
+ *
+ *****************************************************************/
+
+/**
+ * Read global counter
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_global_table()
+{
+ uint32_t idx;
+ netdata_idx_t *val = cachestat_hash_values;
+ netdata_idx_t *stored = cachestat_values;
+ int fd = cachestat_maps[NETDATA_CACHESTAT_GLOBAL_STATS].map_fd;
+
+ for (idx = NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU; idx < NETDATA_CACHESTAT_END; idx++) {
+ if (!bpf_map_lookup_elem(fd, &idx, stored)) {
+ int i;
+ int end = ebpf_nprocs;
+ netdata_idx_t total = 0;
+ for (i = 0; i < end; i++)
+ total += stored[i];
+
+ val[idx] = total;
+ }
+ }
+}
+
+/**
+ * Socket read hash
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data on very busy socket.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_cachestat_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_cachestat_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ usec_t step = NETDATA_LATENCY_CACHESTAT_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_global_table();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Send global
+ *
+ * Send global charts to Netdata
+ */
+static void cachestat_send_global(netdata_publish_cachestat_t *publish)
+{
+ calculate_stats(publish);
+
+ netdata_publish_syscall_t *ptr = cachestat_counter_publish_aggregated;
+ ebpf_one_dimension_write_charts(
+ NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_RATIO_CHART, ptr[NETDATA_CACHESTAT_IDX_RATIO].dimension,
+ publish->ratio);
+
+ ebpf_one_dimension_write_charts(
+ NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_DIRTY_CHART, ptr[NETDATA_CACHESTAT_IDX_DIRTY].dimension,
+ cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY]);
+
+ ebpf_one_dimension_write_charts(
+ NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_CHART, ptr[NETDATA_CACHESTAT_IDX_HIT].dimension, publish->hit);
+
+ ebpf_one_dimension_write_charts(
+ NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_MISSES_CHART, ptr[NETDATA_CACHESTAT_IDX_MISS].dimension,
+ publish->miss);
+}
+
+/**
+ * Cachestat sum PIDs
+ *
+ * Sum values for all PIDs associated to a group
+ *
+ * @param publish output structure.
+ * @param root structure with listed IPs
+ */
+void ebpf_cachestat_sum_pids(netdata_publish_cachestat_t *publish, struct pid_on_target *root)
+{
+ memcpy(&publish->prev, &publish->current,sizeof(publish->current));
+ memset(&publish->current, 0, sizeof(publish->current));
+
+ netdata_cachestat_pid_t *dst = &publish->current;
+ while (root) {
+ int32_t pid = root->pid;
+ netdata_publish_cachestat_t *w = cachestat_pid[pid];
+ if (w) {
+ netdata_cachestat_pid_t *src = &w->current;
+ dst->account_page_dirtied += src->account_page_dirtied;
+ dst->add_to_page_cache_lru += src->add_to_page_cache_lru;
+ dst->mark_buffer_dirty += src->mark_buffer_dirty;
+ dst->mark_page_accessed += src->mark_page_accessed;
+ }
+
+ root = root->next;
+ }
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param root the target list.
+*/
+void ebpf_cache_send_apps_data(struct target *root)
+{
+ struct target *w;
+ collected_number value;
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_HIT_RATIO_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ ebpf_cachestat_sum_pids(&w->cachestat, w->root_pid);
+ netdata_cachestat_pid_t *current = &w->cachestat.current;
+ netdata_cachestat_pid_t *prev = &w->cachestat.prev;
+
+ uint64_t mpa = current->mark_page_accessed - prev->mark_page_accessed;
+ uint64_t mbd = current->mark_buffer_dirty - prev->mark_buffer_dirty;
+ w->cachestat.dirty = mbd;
+ uint64_t apcl = current->add_to_page_cache_lru - prev->add_to_page_cache_lru;
+ uint64_t apd = current->account_page_dirtied - prev->account_page_dirtied;
+
+ cachestat_update_publish(&w->cachestat, mpa, mbd, apcl, apd);
+ value = (collected_number) w->cachestat.ratio;
+ // Here we are using different approach to have a chart more smooth
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_DIRTY_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = (collected_number) w->cachestat.dirty;
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_HIT_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = (collected_number) w->cachestat.hit;
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_MISSES_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = (collected_number) w->cachestat.miss;
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+}
+
+/**
+ * Cachestat sum PIDs
+ *
+ * Sum values for all PIDs associated to a group
+ *
+ * @param publish output structure.
+ * @param root structure with listed IPs
+ */
+void ebpf_cachestat_sum_cgroup_pids(netdata_publish_cachestat_t *publish, struct pid_on_target2 *root)
+{
+ memcpy(&publish->prev, &publish->current,sizeof(publish->current));
+ memset(&publish->current, 0, sizeof(publish->current));
+
+ netdata_cachestat_pid_t *dst = &publish->current;
+ while (root) {
+ netdata_cachestat_pid_t *src = &root->cachestat;
+
+ dst->account_page_dirtied += src->account_page_dirtied;
+ dst->add_to_page_cache_lru += src->add_to_page_cache_lru;
+ dst->mark_buffer_dirty += src->mark_buffer_dirty;
+ dst->mark_page_accessed += src->mark_page_accessed;
+
+ root = root->next;
+ }
+}
+
+/**
+ * Calc chart values
+ *
+ * Do necessary math to plot charts.
+ */
+void ebpf_cachestat_calc_chart_values()
+{
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ ebpf_cachestat_sum_cgroup_pids(&ect->publish_cachestat, ect->pids);
+
+ netdata_cachestat_pid_t *current = &ect->publish_cachestat.current;
+ netdata_cachestat_pid_t *prev = &ect->publish_cachestat.prev;
+
+ uint64_t mpa = current->mark_page_accessed - prev->mark_page_accessed;
+ uint64_t mbd = current->mark_buffer_dirty - prev->mark_buffer_dirty;
+ ect->publish_cachestat.dirty = mbd;
+ uint64_t apcl = current->add_to_page_cache_lru - prev->add_to_page_cache_lru;
+ uint64_t apd = current->account_page_dirtied - prev->account_page_dirtied;
+
+ cachestat_update_publish(&ect->publish_cachestat, mpa, mbd, apcl, apd);
+ }
+}
+
+/**
+ * Create Systemd cachestat Charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ **/
+static void ebpf_create_systemd_cachestat_charts(int update_every)
+{
+ ebpf_create_charts_on_systemd(NETDATA_CACHESTAT_HIT_RATIO_CHART,
+ "Hit ratio",
+ EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, 21100,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ NETDATA_SYSTEMD_CACHESTAT_HIT_RATIO_CONTEXT, NETDATA_EBPF_MODULE_NAME_CACHESTAT,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_CACHESTAT_DIRTY_CHART,
+ "Number of dirty pages",
+ EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, 21101,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ NETDATA_SYSTEMD_CACHESTAT_MODIFIED_CACHE_CONTEXT, NETDATA_EBPF_MODULE_NAME_CACHESTAT,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_CACHESTAT_HIT_CHART, "Number of accessed files",
+ EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, 21102,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ NETDATA_SYSTEMD_CACHESTAT_HIT_FILE_CONTEXT, NETDATA_EBPF_MODULE_NAME_CACHESTAT,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_CACHESTAT_MISSES_CHART, "Files out of page cache",
+ EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, 21103,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ NETDATA_SYSTEMD_CACHESTAT_MISS_FILES_CONTEXT, NETDATA_EBPF_MODULE_NAME_CACHESTAT,
+ update_every);
+}
+
+/**
+ * Send Cache Stat charts
+ *
+ * Send collected data to Netdata.
+ */
+static void ebpf_send_systemd_cachestat_charts()
+{
+ ebpf_cgroup_target_t *ect;
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_CACHESTAT_HIT_RATIO_CHART);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_cachestat.ratio);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_CACHESTAT_DIRTY_CHART);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_cachestat.dirty);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_CACHESTAT_HIT_CHART);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_cachestat.hit);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_CACHESTAT_MISSES_CHART);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_cachestat.miss);
+ }
+ }
+ write_end_chart();
+}
+
+/**
+ * Send Directory Cache charts
+ *
+ * Send collected data to Netdata.
+ */
+static void ebpf_send_specific_cachestat_data(char *type, netdata_publish_cachestat_t *npc)
+{
+ write_begin_chart(type, NETDATA_CACHESTAT_HIT_RATIO_CHART);
+ write_chart_dimension(cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_RATIO].name, (long long)npc->ratio);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_CACHESTAT_DIRTY_CHART);
+ write_chart_dimension(cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_DIRTY].name, (long long)npc->dirty);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_CACHESTAT_HIT_CHART);
+ write_chart_dimension(cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_HIT].name, (long long)npc->hit);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_CACHESTAT_MISSES_CHART);
+ write_chart_dimension(cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_MISS].name, (long long)npc->miss);
+ write_end_chart();
+}
+
+/**
+ * Create specific cache Stat charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_specific_cachestat_charts(char *type, int update_every)
+{
+ ebpf_create_chart(type, NETDATA_CACHESTAT_HIT_RATIO_CHART,
+ "Hit ratio",
+ EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_CACHESTAT_CGROUP_SUBMENU,
+ NETDATA_CGROUP_CACHESTAT_HIT_RATIO_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5200,
+ ebpf_create_global_dimension,
+ cachestat_counter_publish_aggregated, 1, update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_chart(type, NETDATA_CACHESTAT_DIRTY_CHART,
+ "Number of dirty pages",
+ EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_CGROUP_SUBMENU,
+ NETDATA_CGROUP_CACHESTAT_MODIFIED_CACHE_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5201,
+ ebpf_create_global_dimension,
+ &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_DIRTY], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_chart(type, NETDATA_CACHESTAT_HIT_CHART,
+ "Number of accessed files",
+ EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_CGROUP_SUBMENU,
+ NETDATA_CGROUP_CACHESTAT_HIT_FILES_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5202,
+ ebpf_create_global_dimension,
+ &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_HIT], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_chart(type, NETDATA_CACHESTAT_MISSES_CHART,
+ "Files out of page cache",
+ EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_CGROUP_SUBMENU,
+ NETDATA_CGROUP_CACHESTAT_MISS_FILES_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5203,
+ ebpf_create_global_dimension,
+ &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_MISS], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+}
+
+/**
+ * Obsolete specific cache stat charts
+ *
+ * Obsolete charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_obsolete_specific_cachestat_charts(char *type, int update_every)
+{
+ ebpf_write_chart_obsolete(type, NETDATA_CACHESTAT_HIT_RATIO_CHART,
+ "Hit ratio",
+ EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_CACHESTAT_HIT_RATIO_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5200, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_CACHESTAT_DIRTY_CHART,
+ "Number of dirty pages",
+ EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_CACHESTAT_MODIFIED_CACHE_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5201, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_CACHESTAT_HIT_CHART,
+ "Number of accessed files",
+ EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_CACHESTAT_HIT_FILES_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5202, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_CACHESTAT_MISSES_CHART,
+ "Files out of page cache",
+ EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_CACHESTAT_MISS_FILES_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5203, update_every);
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+*/
+void ebpf_cachestat_send_cgroup_data(int update_every)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+ ebpf_cachestat_calc_chart_values();
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_cachestat_charts(update_every);
+ }
+
+ ebpf_send_systemd_cachestat_charts();
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART) && ect->updated) {
+ ebpf_create_specific_cachestat_charts(ect->name, update_every);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART) {
+ if (ect->updated) {
+ ebpf_send_specific_cachestat_data(ect->name, &ect->publish_cachestat);
+ } else {
+ ebpf_obsolete_specific_cachestat_charts(ect->name, update_every);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+* Main loop for this collector.
+*/
+static void cachestat_collector(ebpf_module_t *em)
+{
+ cachestat_threads.thread = callocz(1, sizeof(netdata_thread_t));
+ cachestat_threads.start_routine = ebpf_cachestat_read_hash;
+
+ netdata_thread_create(cachestat_threads.thread, cachestat_threads.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_cachestat_read_hash, em);
+
+ netdata_publish_cachestat_t publish;
+ memset(&publish, 0, sizeof(publish));
+ int cgroups = em->cgroup_charts;
+ int update_every = em->update_every;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = update_every * USEC_PER_SEC;
+ //This will be cancelled by its parent
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ netdata_apps_integration_flags_t apps = em->apps_charts;
+ pthread_mutex_lock(&collect_data_mutex);
+ if (apps)
+ read_apps_table();
+
+ if (cgroups)
+ ebpf_update_cachestat_cgroup();
+
+ pthread_mutex_lock(&lock);
+
+ cachestat_send_global(&publish);
+
+ if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED)
+ ebpf_cache_send_apps_data(apps_groups_root_target);
+
+ if (cgroups)
+ ebpf_cachestat_send_cgroup_data(update_every);
+
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+}
+
+/*****************************************************************
+ *
+ * INITIALIZE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Create global charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param em a pointer to `struct ebpf_module`
+ */
+static void ebpf_create_memory_charts(ebpf_module_t *em)
+{
+ ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_RATIO_CHART,
+ "Hit ratio",
+ EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_CACHESTAT_SUBMENU,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21100,
+ ebpf_create_global_dimension,
+ cachestat_counter_publish_aggregated, 1, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_DIRTY_CHART,
+ "Number of dirty pages",
+ EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_SUBMENU,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21101,
+ ebpf_create_global_dimension,
+ &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_DIRTY], 1,
+ em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_CHART,
+ "Number of accessed files",
+ EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_SUBMENU,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21102,
+ ebpf_create_global_dimension,
+ &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_HIT], 1,
+ em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_MISSES_CHART,
+ "Files out of page cache",
+ EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_SUBMENU,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21103,
+ ebpf_create_global_dimension,
+ &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_MISS], 1,
+ em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
+
+ fflush(stdout);
+}
+
+/**
+ * Allocate vectors used with this thread.
+ *
+ * We are not testing the return, because callocz does this and shutdown the software
+ * case it was not possible to allocate.
+ *
+ * @param apps is apps enabled?
+ */
+static void ebpf_cachestat_allocate_global_vectors(int apps)
+{
+ if (apps)
+ cachestat_pid = callocz((size_t)pid_max, sizeof(netdata_publish_cachestat_t *));
+
+ cachestat_vector = callocz((size_t)ebpf_nprocs, sizeof(netdata_cachestat_pid_t));
+
+ cachestat_values = callocz((size_t)ebpf_nprocs, sizeof(netdata_idx_t));
+
+ memset(cachestat_hash_values, 0, NETDATA_CACHESTAT_END * sizeof(netdata_idx_t));
+ memset(cachestat_counter_aggregated_data, 0, NETDATA_CACHESTAT_END * sizeof(netdata_syscall_stat_t));
+ memset(cachestat_counter_publish_aggregated, 0, NETDATA_CACHESTAT_END * sizeof(netdata_publish_syscall_t));
+}
+
+/*****************************************************************
+ *
+ * MAIN THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Update Internal value
+ *
+ * Update values used during runtime.
+ */
+static void ebpf_cachestat_set_internal_value()
+{
+ static char *account_page[] = { "account_page_dirtied", "__set_page_dirty", "__folio_mark_dirty" };
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_16)
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name = account_page[NETDATA_CACHESTAT_FOLIO_DIRTY];
+ else if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_15)
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name = account_page[NETDATA_CACHESTAT_SET_PAGE_DIRTY];
+ else
+ cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name = account_page[NETDATA_CACHESTAT_ACCOUNT_PAGE_DIRTY];
+}
+
+/*
+ * Load BPF
+ *
+ * Load BPF files.
+ *
+ * @param em the structure with configuration
+ */
+static int ebpf_cachestat_load_bpf(ebpf_module_t *em)
+{
+ int ret = 0;
+ ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].mode);
+ if (em->load & EBPF_LOAD_LEGACY) {
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ ret = -1;
+ }
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ bpf_obj = cachestat_bpf__open();
+ if (!bpf_obj)
+ ret = -1;
+ else
+ ret = ebpf_cachestat_load_and_attach(bpf_obj, em);
+ }
+#endif
+
+ if (ret)
+ error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->thread_name);
+
+ return ret;
+}
+
+/**
+ * Cachestat thread
+ *
+ * Thread used to make cachestat thread
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always return NULL
+ */
+void *ebpf_cachestat_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_cachestat_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = cachestat_maps;
+
+ ebpf_update_pid_table(&cachestat_maps[NETDATA_CACHESTAT_PID_STATS], em);
+
+ ebpf_cachestat_set_internal_value();
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_cachestat_load_bpf(em)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endcachestat;
+ }
+
+ ebpf_cachestat_allocate_global_vectors(em->apps_charts);
+
+ int algorithms[NETDATA_CACHESTAT_END] = {
+ NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX
+ };
+
+ ebpf_global_labels(cachestat_counter_aggregated_data, cachestat_counter_publish_aggregated,
+ cachestat_counter_dimension_name, cachestat_counter_dimension_name,
+ algorithms, NETDATA_CACHESTAT_END);
+
+ pthread_mutex_lock(&lock);
+ ebpf_update_stats(&plugin_statistics, em);
+ ebpf_create_memory_charts(em);
+ pthread_mutex_unlock(&lock);
+
+ cachestat_collector(em);
+
+endcachestat:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_cachestat.h b/collectors/ebpf.plugin/ebpf_cachestat.h
new file mode 100644
index 0000000..07f0745
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_cachestat.h
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_CACHESTAT_H
+#define NETDATA_EBPF_CACHESTAT_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_CACHESTAT "cachestat"
+
+// charts
+#define NETDATA_CACHESTAT_HIT_RATIO_CHART "cachestat_ratio"
+#define NETDATA_CACHESTAT_DIRTY_CHART "cachestat_dirties"
+#define NETDATA_CACHESTAT_HIT_CHART "cachestat_hits"
+#define NETDATA_CACHESTAT_MISSES_CHART "cachestat_misses"
+
+#define NETDATA_CACHESTAT_SUBMENU "page_cache"
+#define NETDATA_CACHESTAT_CGROUP_SUBMENU "page cache (eBPF)"
+
+#define EBPF_CACHESTAT_DIMENSION_PAGE "pages/s"
+#define EBPF_CACHESTAT_DIMENSION_HITS "hits/s"
+#define EBPF_CACHESTAT_DIMENSION_MISSES "misses/s"
+
+#define NETDATA_LATENCY_CACHESTAT_SLEEP_MS 600000ULL
+
+// configuration file
+#define NETDATA_CACHESTAT_CONFIG_FILE "cachestat.conf"
+
+// Contexts
+#define NETDATA_CGROUP_CACHESTAT_HIT_RATIO_CONTEXT "cgroup.cachestat_ratio"
+#define NETDATA_CGROUP_CACHESTAT_MODIFIED_CACHE_CONTEXT "cgroup.cachestat_dirties"
+#define NETDATA_CGROUP_CACHESTAT_HIT_FILES_CONTEXT "cgroup.cachestat_hits"
+#define NETDATA_CGROUP_CACHESTAT_MISS_FILES_CONTEXT "cgroup.cachestat_misses"
+
+#define NETDATA_SYSTEMD_CACHESTAT_HIT_RATIO_CONTEXT "services.cachestat_ratio"
+#define NETDATA_SYSTEMD_CACHESTAT_MODIFIED_CACHE_CONTEXT "services.cachestat_dirties"
+#define NETDATA_SYSTEMD_CACHESTAT_HIT_FILE_CONTEXT "services.cachestat_hits"
+#define NETDATA_SYSTEMD_CACHESTAT_MISS_FILES_CONTEXT "services.cachestat_misses"
+
+// variables
+enum cachestat_counters {
+ NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU,
+ NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED,
+ NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED,
+ NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY,
+
+ NETDATA_CACHESTAT_END
+};
+
+enum cachestat_account_dirty_pages {
+ NETDATA_CACHESTAT_ACCOUNT_PAGE_DIRTY,
+ NETDATA_CACHESTAT_SET_PAGE_DIRTY,
+ NETDATA_CACHESTAT_FOLIO_DIRTY
+};
+
+enum cachestat_indexes {
+ NETDATA_CACHESTAT_IDX_RATIO,
+ NETDATA_CACHESTAT_IDX_DIRTY,
+ NETDATA_CACHESTAT_IDX_HIT,
+ NETDATA_CACHESTAT_IDX_MISS
+};
+
+enum cachestat_tables {
+ NETDATA_CACHESTAT_GLOBAL_STATS,
+ NETDATA_CACHESTAT_PID_STATS,
+ NETDATA_CACHESTAT_CTRL
+};
+
+typedef struct netdata_publish_cachestat_pid {
+ uint64_t add_to_page_cache_lru;
+ uint64_t mark_page_accessed;
+ uint64_t account_page_dirtied;
+ uint64_t mark_buffer_dirty;
+} netdata_cachestat_pid_t;
+
+typedef struct netdata_publish_cachestat {
+ long long ratio;
+ long long dirty;
+ long long hit;
+ long long miss;
+
+ netdata_cachestat_pid_t current;
+ netdata_cachestat_pid_t prev;
+} netdata_publish_cachestat_t;
+
+void *ebpf_cachestat_thread(void *ptr);
+
+extern struct config cachestat_config;
+extern netdata_ebpf_targets_t cachestat_targets[];
+extern ebpf_local_maps_t cachestat_maps[];
+
+#endif // NETDATA_EBPF_CACHESTAT_H
diff --git a/collectors/ebpf.plugin/ebpf_cgroup.c b/collectors/ebpf.plugin/ebpf_cgroup.c
new file mode 100644
index 0000000..42c0453
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_cgroup.c
@@ -0,0 +1,317 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <sys/resource.h>
+
+#include "ebpf.h"
+#include "ebpf_cgroup.h"
+
+ebpf_cgroup_target_t *ebpf_cgroup_pids = NULL;
+int send_cgroup_chart = 0;
+
+// --------------------------------------------------------------------------------------------------------------------
+// Map shared memory
+
+/**
+ * Map Shared Memory locally
+ *
+ * Map the shared memory for current process
+ *
+ * @param fd file descriptor returned after shm_open was called.
+ * @param length length of the shared memory
+ *
+ * @return It returns a pointer to the region mapped.
+ */
+static inline void *ebpf_cgroup_map_shm_locally(int fd, size_t length)
+{
+ void *value;
+
+ value = mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (!value) {
+ error("Cannot map shared memory used between eBPF and cgroup, integration between processes won't happen");
+ close(shm_fd_ebpf_cgroup);
+ shm_fd_ebpf_cgroup = -1;
+ shm_unlink(NETDATA_SHARED_MEMORY_EBPF_CGROUP_NAME);
+ }
+
+ return value;
+}
+
+/**
+ * Map cgroup shared memory
+ *
+ * Map cgroup shared memory from cgroup to plugin
+ */
+void ebpf_map_cgroup_shared_memory()
+{
+ static int limit_try = 0;
+ static time_t next_try = 0;
+
+ if (shm_ebpf_cgroup.header || limit_try > NETDATA_EBPF_CGROUP_MAX_TRIES)
+ return;
+
+ time_t curr_time = time(NULL);
+ if (curr_time < next_try)
+ return;
+
+ limit_try++;
+ next_try = curr_time + NETDATA_EBPF_CGROUP_NEXT_TRY_SEC;
+
+ shm_fd_ebpf_cgroup = shm_open(NETDATA_SHARED_MEMORY_EBPF_CGROUP_NAME, O_RDWR, 0660);
+ if (shm_fd_ebpf_cgroup < 0) {
+ if (limit_try == NETDATA_EBPF_CGROUP_MAX_TRIES)
+ error("Shared memory was not initialized, integration between processes won't happen.");
+
+ return;
+ }
+
+ // Map only header
+ shm_ebpf_cgroup.header = (netdata_ebpf_cgroup_shm_header_t *) ebpf_cgroup_map_shm_locally(shm_fd_ebpf_cgroup,
+ sizeof(netdata_ebpf_cgroup_shm_header_t));
+ if (!shm_ebpf_cgroup.header) {
+ limit_try = NETDATA_EBPF_CGROUP_MAX_TRIES + 1;
+ return;
+ }
+
+ size_t length = shm_ebpf_cgroup.header->body_length;
+
+ munmap(shm_ebpf_cgroup.header, sizeof(netdata_ebpf_cgroup_shm_header_t));
+
+ shm_ebpf_cgroup.header = (netdata_ebpf_cgroup_shm_header_t *)ebpf_cgroup_map_shm_locally(shm_fd_ebpf_cgroup, length);
+ if (!shm_ebpf_cgroup.header) {
+ limit_try = NETDATA_EBPF_CGROUP_MAX_TRIES + 1;
+ return;
+ }
+ shm_ebpf_cgroup.body = (netdata_ebpf_cgroup_shm_body_t *) ((char *)shm_ebpf_cgroup.header +
+ sizeof(netdata_ebpf_cgroup_shm_header_t));
+
+ shm_sem_ebpf_cgroup = sem_open(NETDATA_NAMED_SEMAPHORE_EBPF_CGROUP_NAME, O_CREAT, 0660, 1);
+
+ if (shm_sem_ebpf_cgroup == SEM_FAILED) {
+ error("Cannot create semaphore, integration between eBPF and cgroup won't happen");
+ munmap(shm_ebpf_cgroup.header, length);
+ shm_ebpf_cgroup.header = NULL;
+ close(shm_fd_ebpf_cgroup);
+ shm_fd_ebpf_cgroup = -1;
+ shm_unlink(NETDATA_SHARED_MEMORY_EBPF_CGROUP_NAME);
+ }
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// Close and Cleanup
+
+/**
+ * Clean Specific cgroup pid
+ *
+ * Clean all PIDs associated with cgroup.
+ *
+ * @param pt structure pid on target that will have your PRs removed
+ */
+static inline void ebpf_clean_specific_cgroup_pids(struct pid_on_target2 *pt)
+{
+ while (pt) {
+ struct pid_on_target2 *next_pid = pt->next;
+
+ freez(pt);
+ pt = next_pid;
+ }
+}
+
+/**
+ * Remove Cgroup Update Target Update List
+ *
+ * Remove from cgroup target and update the link list
+ */
+static void ebpf_remove_cgroup_target_update_list()
+{
+ ebpf_cgroup_target_t *next, *ect = ebpf_cgroup_pids;
+ ebpf_cgroup_target_t *prev = ebpf_cgroup_pids;
+ while (ect) {
+ next = ect->next;
+ if (!ect->updated) {
+ if (ect == ebpf_cgroup_pids) {
+ ebpf_cgroup_pids = next;
+ prev = next;
+ } else {
+ prev->next = next;
+ }
+
+ ebpf_clean_specific_cgroup_pids(ect->pids);
+ freez(ect);
+ } else {
+ prev = ect;
+ }
+
+ ect = next;
+ }
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// Fill variables
+
+/**
+ * Set Target Data
+ *
+ * Set local variable values according shared memory information.
+ *
+ * @param out local output variable.
+ * @param ptr input from shared memory.
+ */
+static inline void ebpf_cgroup_set_target_data(ebpf_cgroup_target_t *out, netdata_ebpf_cgroup_shm_body_t *ptr)
+{
+ out->hash = ptr->hash;
+ snprintfz(out->name, 255, "%s", ptr->name);
+ out->systemd = ptr->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE;
+ out->updated = 1;
+}
+
+/**
+ * Find or create
+ *
+ * Find the structure inside the link list or allocate and link when it is not present.
+ *
+ * @param ptr Input from shared memory.
+ *
+ * @return It returns a pointer for the structure associated with the input.
+ */
+static ebpf_cgroup_target_t * ebpf_cgroup_find_or_create(netdata_ebpf_cgroup_shm_body_t *ptr)
+{
+ ebpf_cgroup_target_t *ect, *prev;
+ for (ect = ebpf_cgroup_pids, prev = ebpf_cgroup_pids; ect; prev = ect, ect = ect->next) {
+ if (ect->hash == ptr->hash && !strcmp(ect->name, ptr->name)) {
+ ect->updated = 1;
+ return ect;
+ }
+ }
+
+ ebpf_cgroup_target_t *new_ect = callocz(1, sizeof(ebpf_cgroup_target_t));
+
+ ebpf_cgroup_set_target_data(new_ect, ptr);
+ if (!ebpf_cgroup_pids) {
+ ebpf_cgroup_pids = new_ect;
+ } else {
+ prev->next = new_ect;
+ }
+
+ return new_ect;
+}
+
+/**
+ * Update pid link list
+ *
+ * Update PIDs list associated with specific cgroup.
+ *
+ * @param ect cgroup structure where pids will be stored
+ * @param path file with PIDs associated to cgroup.
+ */
+static void ebpf_update_pid_link_list(ebpf_cgroup_target_t *ect, char *path)
+{
+ procfile *ff = procfile_open_no_log(path, " \t:", PROCFILE_FLAG_DEFAULT);
+ if (!ff)
+ return;
+
+ ff = procfile_readall(ff);
+ if (!ff)
+ return;
+
+ size_t lines = procfile_lines(ff), l;
+ for (l = 0; l < lines ;l++) {
+ int pid = (int)str2l(procfile_lineword(ff, l, 0));
+ if (pid) {
+ struct pid_on_target2 *pt, *prev;
+ for (pt = ect->pids, prev = ect->pids; pt; prev = pt, pt = pt->next) {
+ if (pt->pid == pid)
+ break;
+ }
+
+ if (!pt) {
+ struct pid_on_target2 *w = callocz(1, sizeof(struct pid_on_target2));
+ w->pid = pid;
+ if (!ect->pids)
+ ect->pids = w;
+ else
+ prev->next = w;
+ }
+ }
+ }
+
+ procfile_close(ff);
+}
+
+/**
+ * Set remove var
+ *
+ * Set variable remove. If this variable is not reset, the structure will be removed from link list.
+ */
+void ebpf_reset_updated_var()
+ {
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ ect->updated = 0;
+ }
+ }
+
+/**
+ * Parse cgroup shared memory
+ *
+ * This function is responsible to copy necessary data from shared memory to local memory.
+ */
+void ebpf_parse_cgroup_shm_data()
+{
+ static int previous = 0;
+ if (shm_ebpf_cgroup.header) {
+ sem_wait(shm_sem_ebpf_cgroup);
+ int i, end = shm_ebpf_cgroup.header->cgroup_root_count;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+
+ ebpf_remove_cgroup_target_update_list();
+
+ ebpf_reset_updated_var();
+
+ for (i = 0; i < end; i++) {
+ netdata_ebpf_cgroup_shm_body_t *ptr = &shm_ebpf_cgroup.body[i];
+ if (ptr->enabled) {
+ ebpf_cgroup_target_t *ect = ebpf_cgroup_find_or_create(ptr);
+ ebpf_update_pid_link_list(ect, ptr->path);
+ }
+ }
+ send_cgroup_chart = previous != shm_ebpf_cgroup.header->cgroup_root_count;
+ previous = shm_ebpf_cgroup.header->cgroup_root_count;
+#ifdef NETDATA_DEV_MODE
+ error("Updating cgroup %d (Previous: %d, Current: %d)", send_cgroup_chart, previous, shm_ebpf_cgroup.header->cgroup_root_count);
+#endif
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+
+ sem_post(shm_sem_ebpf_cgroup);
+ }
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// Create charts
+
+/**
+ * Create charts on systemd submenu
+ *
+ * @param id the chart id
+ * @param title the value displayed on vertical axis.
+ * @param units the value displayed on vertical axis.
+ * @param family Submenu that the chart will be attached on dashboard.
+ * @param charttype chart type
+ * @param order the chart order
+ * @param algorithm the algorithm used by dimension
+ * @param context add context for chart
+ * @param module chart module name, this is the eBPF thread.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+void ebpf_create_charts_on_systemd(char *id, char *title, char *units, char *family, char *charttype, int order,
+ char *algorithm, char *context, char *module, int update_every)
+{
+ ebpf_cgroup_target_t *w;
+ ebpf_write_chart_cmd(NETDATA_SERVICE_FAMILY, id, title, units, family, charttype, context,
+ order, update_every, module);
+
+ for (w = ebpf_cgroup_pids; w; w = w->next) {
+ if (unlikely(w->systemd) && unlikely(w->updated))
+ fprintf(stdout, "DIMENSION %s '' %s 1 1\n", w->name, algorithm);
+ }
+}
diff --git a/collectors/ebpf.plugin/ebpf_cgroup.h b/collectors/ebpf.plugin/ebpf_cgroup.h
new file mode 100644
index 0000000..19da7fc
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_cgroup.h
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_CGROUP_H
+#define NETDATA_EBPF_CGROUP_H 1
+
+#define NETDATA_EBPF_CGROUP_MAX_TRIES 3
+#define NETDATA_EBPF_CGROUP_NEXT_TRY_SEC 30
+
+#include "ebpf.h"
+#include "ebpf_apps.h"
+
+#define NETDATA_SERVICE_FAMILY "services"
+
+struct pid_on_target2 {
+ int32_t pid;
+ int updated;
+
+ netdata_publish_swap_t swap;
+ netdata_fd_stat_t fd;
+ netdata_publish_vfs_t vfs;
+ ebpf_process_stat_t ps;
+ netdata_dcstat_pid_t dc;
+ netdata_publish_shm_t shm;
+ ebpf_bandwidth_t socket;
+ netdata_cachestat_pid_t cachestat;
+
+ struct pid_on_target2 *next;
+};
+
+enum ebpf_cgroup_flags {
+ NETDATA_EBPF_CGROUP_HAS_PROCESS_CHART = 1,
+ NETDATA_EBPF_CGROUP_HAS_SWAP_CHART = 1<<2,
+ NETDATA_EBPF_CGROUP_HAS_SOCKET_CHART = 1<<3,
+ NETDATA_EBPF_CGROUP_HAS_FD_CHART = 1<<4,
+ NETDATA_EBPF_CGROUP_HAS_VFS_CHART = 1<<5,
+ NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART = 1<<6,
+ NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART = 1<<7,
+ NETDATA_EBPF_CGROUP_HAS_DC_CHART = 1<<8,
+ NETDATA_EBPF_CGROUP_HAS_SHM_CHART = 1<<9
+};
+
+typedef struct ebpf_cgroup_target {
+ char name[256]; // title
+ uint32_t hash;
+ uint32_t flags;
+ uint32_t systemd;
+ uint32_t updated;
+
+ netdata_publish_swap_t publish_systemd_swap;
+ netdata_fd_stat_t publish_systemd_fd;
+ netdata_publish_vfs_t publish_systemd_vfs;
+ ebpf_process_stat_t publish_systemd_ps;
+ netdata_publish_dcstat_t publish_dc;
+ int oomkill;
+ netdata_publish_shm_t publish_shm;
+ ebpf_socket_publish_apps_t publish_socket;
+ netdata_publish_cachestat_t publish_cachestat;
+
+ struct pid_on_target2 *pids;
+ struct ebpf_cgroup_target *next;
+} ebpf_cgroup_target_t;
+
+void ebpf_map_cgroup_shared_memory();
+void ebpf_parse_cgroup_shm_data();
+void ebpf_create_charts_on_systemd(char *id, char *title, char *units, char *family, char *charttype, int order,
+ char *algorithm, char *context, char *module, int update_every);
+extern int send_cgroup_chart;
+
+#endif /* NETDATA_EBPF_CGROUP_H */
diff --git a/collectors/ebpf.plugin/ebpf_dcstat.c b/collectors/ebpf.plugin/ebpf_dcstat.c
new file mode 100644
index 0000000..71169e1
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_dcstat.c
@@ -0,0 +1,1225 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_dcstat.h"
+
+static char *dcstat_counter_dimension_name[NETDATA_DCSTAT_IDX_END] = { "ratio", "reference", "slow", "miss" };
+static netdata_syscall_stat_t dcstat_counter_aggregated_data[NETDATA_DCSTAT_IDX_END];
+static netdata_publish_syscall_t dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_END];
+
+netdata_dcstat_pid_t *dcstat_vector = NULL;
+netdata_publish_dcstat_t **dcstat_pid = NULL;
+
+static netdata_idx_t dcstat_hash_values[NETDATA_DCSTAT_IDX_END];
+static netdata_idx_t *dcstat_values = NULL;
+
+struct config dcstat_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+struct netdata_static_thread dcstat_threads = {"DCSTAT KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL};
+
+ebpf_local_maps_t dcstat_maps[] = {{.name = "dcstat_global", .internal_input = NETDATA_DIRECTORY_CACHE_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "dcstat_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "dcstat_ctrl", .internal_input = NETDATA_CONTROLLER_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
+
+static ebpf_specify_name_t dc_optional_name[] = { {.program_name = "netdata_lookup_fast",
+ .function_to_attach = "lookup_fast",
+ .optional = NULL,
+ .retprobe = CONFIG_BOOLEAN_NO},
+ {.program_name = NULL}};
+
+netdata_ebpf_targets_t dc_targets[] = { {.name = "lookup_fast", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "d_lookup", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/dc.skel.h" // BTF code
+
+static struct dc_bpf *bpf_obj = NULL;
+
+/**
+ * Disable probe
+ *
+ * Disable all probes to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects
+ */
+static inline void ebpf_dc_disable_probes(struct dc_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_lookup_fast_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_d_lookup_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_dcstat_release_task_kprobe, false);
+}
+
+/*
+ * Disable trampoline
+ *
+ * Disable all trampoline to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void ebpf_dc_disable_trampoline(struct dc_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_lookup_fast_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_d_lookup_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_dcstat_release_task_fentry, false);
+}
+
+/**
+ * Set trampoline target
+ *
+ * Set the targets we will monitor.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_dc_set_trampoline_target(struct dc_bpf *obj)
+{
+ bpf_program__set_attach_target(obj->progs.netdata_lookup_fast_fentry, 0,
+ dc_targets[NETDATA_DC_TARGET_LOOKUP_FAST].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_d_lookup_fexit, 0,
+ dc_targets[NETDATA_DC_TARGET_D_LOOKUP].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_dcstat_release_task_fentry, 0,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+}
+
+/**
+ * Mount Attach Probe
+ *
+ * Attach probes to target
+ *
+ * @param obj is the main structure for bpf objects.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+static int ebpf_dc_attach_probes(struct dc_bpf *obj)
+{
+ obj->links.netdata_d_lookup_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_d_lookup_kretprobe,
+ true,
+ dc_targets[NETDATA_DC_TARGET_D_LOOKUP].name);
+ int ret = libbpf_get_error(obj->links.netdata_d_lookup_kretprobe);
+ if (ret)
+ return -1;
+
+ char *lookup_name = (dc_optional_name[NETDATA_DC_TARGET_LOOKUP_FAST].optional) ?
+ dc_optional_name[NETDATA_DC_TARGET_LOOKUP_FAST].optional :
+ dc_targets[NETDATA_DC_TARGET_LOOKUP_FAST].name ;
+
+ obj->links.netdata_lookup_fast_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_lookup_fast_kprobe,
+ false,
+ lookup_name);
+ ret = libbpf_get_error(obj->links.netdata_lookup_fast_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_dcstat_release_task_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_dcstat_release_task_kprobe,
+ false,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+ ret = libbpf_get_error(obj->links.netdata_dcstat_release_task_kprobe);
+ if (ret)
+ return -1;
+
+ return 0;
+}
+
+/**
+ * Adjust Map Size
+ *
+ * Resize maps according input from users.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ */
+static void ebpf_dc_adjust_map_size(struct dc_bpf *obj, ebpf_module_t *em)
+{
+ ebpf_update_map_size(obj->maps.dcstat_pid, &dcstat_maps[NETDATA_DCSTAT_PID_STATS],
+ em, bpf_map__name(obj->maps.dcstat_pid));
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_dc_set_hash_tables(struct dc_bpf *obj)
+{
+ dcstat_maps[NETDATA_DCSTAT_GLOBAL_STATS].map_fd = bpf_map__fd(obj->maps.dcstat_global);
+ dcstat_maps[NETDATA_DCSTAT_PID_STATS].map_fd = bpf_map__fd(obj->maps.dcstat_pid);
+ dcstat_maps[NETDATA_DCSTAT_CTRL].map_fd = bpf_map__fd(obj->maps.dcstat_ctrl);
+}
+
+/**
+ * Update Load
+ *
+ * For directory cache, some distributions change the function name, and we do not have condition to use
+ * TRAMPOLINE like other functions.
+ *
+ * @param em structure with configuration
+ *
+ * @return When then symbols were not modified, it returns TRAMPOLINE, else it returns RETPROBE.
+ */
+netdata_ebpf_program_loaded_t ebpf_dc_update_load(ebpf_module_t *em)
+{
+ if (!strcmp(dc_optional_name[NETDATA_DC_TARGET_LOOKUP_FAST].optional,
+ dc_optional_name[NETDATA_DC_TARGET_LOOKUP_FAST].function_to_attach))
+ return EBPF_LOAD_TRAMPOLINE;
+
+ if (em->targets[NETDATA_DC_TARGET_LOOKUP_FAST].mode != EBPF_LOAD_RETPROBE)
+ info("When your kernel was compiled the symbol %s was modified, instead to use `trampoline`, the plugin will use `probes`.",
+ dc_optional_name[NETDATA_DC_TARGET_LOOKUP_FAST].function_to_attach);
+
+ return EBPF_LOAD_RETPROBE;
+}
+
+/**
+ * Disable Release Task
+ *
+ * Disable release task when apps is not enabled.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_dc_disable_release_task(struct dc_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_dcstat_release_task_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_dcstat_release_task_fentry, false);
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_dc_load_and_attach(struct dc_bpf *obj, ebpf_module_t *em)
+{
+ netdata_ebpf_program_loaded_t test = ebpf_dc_update_load(em);
+ if (test == EBPF_LOAD_TRAMPOLINE) {
+ ebpf_dc_disable_probes(obj);
+
+ ebpf_dc_set_trampoline_target(obj);
+ } else {
+ ebpf_dc_disable_trampoline(obj);
+ }
+
+ ebpf_dc_adjust_map_size(obj, em);
+
+ if (!em->apps_charts && !em->cgroup_charts)
+ ebpf_dc_disable_release_task(obj);
+
+ int ret = dc_bpf__load(obj);
+ if (ret) {
+ return ret;
+ }
+
+ ret = (test == EBPF_LOAD_TRAMPOLINE) ? dc_bpf__attach(obj) : ebpf_dc_attach_probes(obj);
+ if (!ret) {
+ ebpf_dc_set_hash_tables(obj);
+
+ ebpf_update_controller(dcstat_maps[NETDATA_DCSTAT_CTRL].map_fd, em);
+ }
+
+ return ret;
+}
+#endif
+
+/*****************************************************************
+ *
+ * COMMON FUNCTIONS
+ *
+ *****************************************************************/
+
+/**
+ * Update publish
+ *
+ * Update publish values before to write dimension.
+ *
+ * @param out structure that will receive data.
+ * @param cache_access number of access to directory cache.
+ * @param not_found number of files not found on the file system
+ */
+void dcstat_update_publish(netdata_publish_dcstat_t *out, uint64_t cache_access, uint64_t not_found)
+{
+ NETDATA_DOUBLE successful_access = (NETDATA_DOUBLE) (((long long)cache_access) - ((long long)not_found));
+ NETDATA_DOUBLE ratio = (cache_access) ? successful_access/(NETDATA_DOUBLE)cache_access : 0;
+
+ out->ratio = (long long )(ratio*100);
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Clean names
+ *
+ * Clean the optional names allocated during startup.
+ */
+void ebpf_dcstat_clean_names()
+{
+ size_t i = 0;
+ while (dc_optional_name[i].program_name) {
+ freez(dc_optional_name[i].optional);
+ i++;
+ }
+}
+
+/**
+ * DCstat Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_dcstat_free(ebpf_module_t *em )
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ freez(dcstat_vector);
+ freez(dcstat_values);
+ freez(dcstat_threads.thread);
+
+ ebpf_cleanup_publish_syscall(dcstat_counter_publish_aggregated);
+
+ ebpf_dcstat_clean_names();
+
+#ifdef LIBBPF_MAJOR_VERSION
+ if (bpf_obj)
+ dc_bpf__destroy(bpf_obj);
+#endif
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * DCstat exit
+ *
+ * Cancel child and exit.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_dcstat_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*dcstat_threads.thread);
+ ebpf_dcstat_free(em);
+}
+
+/**
+ * Clean up the main thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_dcstat_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_dcstat_free(em);
+}
+
+/*****************************************************************
+ *
+ * APPS
+ *
+ *****************************************************************/
+
+/**
+ * Create apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+void ebpf_dcstat_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+ ebpf_create_charts_on_apps(NETDATA_DC_HIT_CHART,
+ "Percentage of files inside directory cache",
+ EBPF_COMMON_DIMENSION_PERCENTAGE,
+ NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 20100,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ ebpf_create_charts_on_apps(NETDATA_DC_REFERENCE_CHART,
+ "Count file access",
+ EBPF_COMMON_DIMENSION_FILES,
+ NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20101,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ ebpf_create_charts_on_apps(NETDATA_DC_REQUEST_NOT_CACHE_CHART,
+ "Files not present inside directory cache",
+ EBPF_COMMON_DIMENSION_FILES,
+ NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20102,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ ebpf_create_charts_on_apps(NETDATA_DC_REQUEST_NOT_FOUND_CHART,
+ "Files not found",
+ EBPF_COMMON_DIMENSION_FILES,
+ NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20103,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/*****************************************************************
+ *
+ * MAIN LOOP
+ *
+ *****************************************************************/
+
+/**
+ * Apps Accumulator
+ *
+ * Sum all values read from kernel and store in the first address.
+ *
+ * @param out the vector with read values.
+ */
+static void dcstat_apps_accumulator(netdata_dcstat_pid_t *out)
+{
+ int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1;
+ netdata_dcstat_pid_t *total = &out[0];
+ for (i = 1; i < end; i++) {
+ netdata_dcstat_pid_t *w = &out[i];
+ total->cache_access += w->cache_access;
+ total->file_system += w->file_system;
+ total->not_found += w->not_found;
+ }
+}
+
+/**
+ * Save PID values
+ *
+ * Save the current values inside the structure
+ *
+ * @param out vector used to plot charts
+ * @param publish vector with values read from hash tables.
+ */
+static inline void dcstat_save_pid_values(netdata_publish_dcstat_t *out, netdata_dcstat_pid_t *publish)
+{
+ memcpy(&out->curr, &publish[0], sizeof(netdata_dcstat_pid_t));
+}
+
+/**
+ * Fill PID
+ *
+ * Fill PID structures
+ *
+ * @param current_pid pid that we are collecting data
+ * @param out values read from hash tables;
+ */
+static void dcstat_fill_pid(uint32_t current_pid, netdata_dcstat_pid_t *publish)
+{
+ netdata_publish_dcstat_t *curr = dcstat_pid[current_pid];
+ if (!curr) {
+ curr = callocz(1, sizeof(netdata_publish_dcstat_t));
+ dcstat_pid[current_pid] = curr;
+ }
+
+ dcstat_save_pid_values(curr, publish);
+}
+
+/**
+ * Read APPS table
+ *
+ * Read the apps table and store data inside the structure.
+ */
+static void read_apps_table()
+{
+ netdata_dcstat_pid_t *cv = dcstat_vector;
+ uint32_t key;
+ struct pid_stat *pids = root_of_pids;
+ int fd = dcstat_maps[NETDATA_DCSTAT_PID_STATS].map_fd;
+ size_t length = sizeof(netdata_dcstat_pid_t)*ebpf_nprocs;
+ while (pids) {
+ key = pids->pid;
+
+ if (bpf_map_lookup_elem(fd, &key, cv)) {
+ pids = pids->next;
+ continue;
+ }
+
+ dcstat_apps_accumulator(cv);
+
+ dcstat_fill_pid(key, cv);
+
+ // We are cleaning to avoid passing data read from one process to other.
+ memset(cv, 0, length);
+
+ pids = pids->next;
+ }
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ */
+static void ebpf_update_dc_cgroup()
+{
+ netdata_dcstat_pid_t *cv = dcstat_vector;
+ int fd = dcstat_maps[NETDATA_DCSTAT_PID_STATS].map_fd;
+ size_t length = sizeof(netdata_dcstat_pid_t)*ebpf_nprocs;
+
+ ebpf_cgroup_target_t *ect;
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ int pid = pids->pid;
+ netdata_dcstat_pid_t *out = &pids->dc;
+ if (likely(dcstat_pid) && dcstat_pid[pid]) {
+ netdata_publish_dcstat_t *in = dcstat_pid[pid];
+
+ memcpy(out, &in->curr, sizeof(netdata_dcstat_pid_t));
+ } else {
+ memset(cv, 0, length);
+ if (bpf_map_lookup_elem(fd, &pid, cv)) {
+ continue;
+ }
+
+ dcstat_apps_accumulator(cv);
+
+ memcpy(out, cv, sizeof(netdata_dcstat_pid_t));
+ }
+ }
+ }
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Read global table
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_global_table()
+{
+ uint32_t idx;
+ netdata_idx_t *val = dcstat_hash_values;
+ netdata_idx_t *stored = dcstat_values;
+ int fd = dcstat_maps[NETDATA_DCSTAT_GLOBAL_STATS].map_fd;
+
+ for (idx = NETDATA_KEY_DC_REFERENCE; idx < NETDATA_DIRECTORY_CACHE_END; idx++) {
+ if (!bpf_map_lookup_elem(fd, &idx, stored)) {
+ int i;
+ int end = ebpf_nprocs;
+ netdata_idx_t total = 0;
+ for (i = 0; i < end; i++)
+ total += stored[i];
+
+ val[idx] = total;
+ }
+ }
+}
+
+/**
+ * DCstat read hash
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_dcstat_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_dcstat_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ usec_t step = NETDATA_LATENCY_DCSTAT_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_global_table();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Cachestat sum PIDs
+ *
+ * Sum values for all PIDs associated to a group
+ *
+ * @param publish output structure.
+ * @param root structure with listed IPs
+ */
+void ebpf_dcstat_sum_pids(netdata_publish_dcstat_t *publish, struct pid_on_target *root)
+{
+ memset(&publish->curr, 0, sizeof(netdata_dcstat_pid_t));
+ netdata_dcstat_pid_t *dst = &publish->curr;
+ while (root) {
+ int32_t pid = root->pid;
+ netdata_publish_dcstat_t *w = dcstat_pid[pid];
+ if (w) {
+ netdata_dcstat_pid_t *src = &w->curr;
+ dst->cache_access += src->cache_access;
+ dst->file_system += src->file_system;
+ dst->not_found += src->not_found;
+ }
+
+ root = root->next;
+ }
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param root the target list.
+*/
+void ebpf_dcache_send_apps_data(struct target *root)
+{
+ struct target *w;
+ collected_number value;
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_DC_HIT_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ ebpf_dcstat_sum_pids(&w->dcstat, w->root_pid);
+
+ uint64_t cache = w->dcstat.curr.cache_access;
+ uint64_t not_found = w->dcstat.curr.not_found;
+
+ dcstat_update_publish(&w->dcstat, cache, not_found);
+ value = (collected_number) w->dcstat.ratio;
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_DC_REFERENCE_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ if (w->dcstat.curr.cache_access < w->dcstat.prev.cache_access) {
+ w->dcstat.prev.cache_access = 0;
+ }
+
+ w->dcstat.cache_access = (long long)w->dcstat.curr.cache_access - (long long)w->dcstat.prev.cache_access;
+ value = (collected_number) w->dcstat.cache_access;
+ write_chart_dimension(w->name, value);
+ w->dcstat.prev.cache_access = w->dcstat.curr.cache_access;
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_DC_REQUEST_NOT_CACHE_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ if (w->dcstat.curr.file_system < w->dcstat.prev.file_system) {
+ w->dcstat.prev.file_system = 0;
+ }
+
+ value = (collected_number) (!w->dcstat.cache_access) ? 0 :
+ (long long )w->dcstat.curr.file_system - (long long)w->dcstat.prev.file_system;
+ write_chart_dimension(w->name, value);
+ w->dcstat.prev.file_system = w->dcstat.curr.file_system;
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_DC_REQUEST_NOT_FOUND_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ if (w->dcstat.curr.not_found < w->dcstat.prev.not_found) {
+ w->dcstat.prev.not_found = 0;
+ }
+ value = (collected_number) (!w->dcstat.cache_access) ? 0 :
+ (long long)w->dcstat.curr.not_found - (long long)w->dcstat.prev.not_found;
+ write_chart_dimension(w->name, value);
+ w->dcstat.prev.not_found = w->dcstat.curr.not_found;
+ }
+ }
+ write_end_chart();
+}
+
+/**
+ * Send global
+ *
+ * Send global charts to Netdata
+ */
+static void dcstat_send_global(netdata_publish_dcstat_t *publish)
+{
+ dcstat_update_publish(publish, dcstat_hash_values[NETDATA_KEY_DC_REFERENCE],
+ dcstat_hash_values[NETDATA_KEY_DC_MISS]);
+
+ netdata_publish_syscall_t *ptr = dcstat_counter_publish_aggregated;
+ netdata_idx_t value = dcstat_hash_values[NETDATA_KEY_DC_REFERENCE];
+ if (value != ptr[NETDATA_DCSTAT_IDX_REFERENCE].pcall) {
+ ptr[NETDATA_DCSTAT_IDX_REFERENCE].ncall = value - ptr[NETDATA_DCSTAT_IDX_REFERENCE].pcall;
+ ptr[NETDATA_DCSTAT_IDX_REFERENCE].pcall = value;
+
+ value = dcstat_hash_values[NETDATA_KEY_DC_SLOW];
+ ptr[NETDATA_DCSTAT_IDX_SLOW].ncall = value - ptr[NETDATA_DCSTAT_IDX_SLOW].pcall;
+ ptr[NETDATA_DCSTAT_IDX_SLOW].pcall = value;
+
+ value = dcstat_hash_values[NETDATA_KEY_DC_MISS];
+ ptr[NETDATA_DCSTAT_IDX_MISS].ncall = value - ptr[NETDATA_DCSTAT_IDX_MISS].pcall;
+ ptr[NETDATA_DCSTAT_IDX_MISS].pcall = value;
+ } else {
+ ptr[NETDATA_DCSTAT_IDX_REFERENCE].ncall = 0;
+ ptr[NETDATA_DCSTAT_IDX_SLOW].ncall = 0;
+ ptr[NETDATA_DCSTAT_IDX_MISS].ncall = 0;
+ }
+
+ ebpf_one_dimension_write_charts(NETDATA_FILESYSTEM_FAMILY, NETDATA_DC_HIT_CHART,
+ ptr[NETDATA_DCSTAT_IDX_RATIO].dimension, publish->ratio);
+
+ write_count_chart(
+ NETDATA_DC_REFERENCE_CHART, NETDATA_FILESYSTEM_FAMILY,
+ &dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_REFERENCE], 3);
+}
+
+/**
+ * Create specific directory cache charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_specific_dc_charts(char *type, int update_every)
+{
+ ebpf_create_chart(type, NETDATA_DC_HIT_CHART, "Percentage of files inside directory cache",
+ EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_CGROUP_DC_HIT_RATIO_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5700,
+ ebpf_create_global_dimension,
+ dcstat_counter_publish_aggregated, 1, update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ ebpf_create_chart(type, NETDATA_DC_REFERENCE_CHART, "Count file access",
+ EBPF_COMMON_DIMENSION_FILES, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_CGROUP_DC_REFERENCE_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5701,
+ ebpf_create_global_dimension,
+ &dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_REFERENCE], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ ebpf_create_chart(type, NETDATA_DC_REQUEST_NOT_CACHE_CHART,
+ "Files not present inside directory cache",
+ EBPF_COMMON_DIMENSION_FILES, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_CGROUP_DC_NOT_CACHE_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5702,
+ ebpf_create_global_dimension,
+ &dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_SLOW], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ ebpf_create_chart(type, NETDATA_DC_REQUEST_NOT_FOUND_CHART,
+ "Files not found",
+ EBPF_COMMON_DIMENSION_FILES, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_CGROUP_DC_NOT_FOUND_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5703,
+ ebpf_create_global_dimension,
+ &dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_MISS], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+}
+
+/**
+ * Obsolete specific directory cache charts
+ *
+ * Obsolete charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_obsolete_specific_dc_charts(char *type, int update_every)
+{
+ ebpf_write_chart_obsolete(type, NETDATA_DC_HIT_CHART,
+ "Percentage of files inside directory cache",
+ EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_DC_HIT_RATIO_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5700, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_DC_REFERENCE_CHART,
+ "Count file access",
+ EBPF_COMMON_DIMENSION_FILES, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_DC_REFERENCE_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5701, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_DC_REQUEST_NOT_CACHE_CHART,
+ "Files not present inside directory cache",
+ EBPF_COMMON_DIMENSION_FILES, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_DC_NOT_CACHE_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5702, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_DC_REQUEST_NOT_FOUND_CHART,
+ "Files not found",
+ EBPF_COMMON_DIMENSION_FILES, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_DC_NOT_FOUND_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5703, update_every);
+}
+
+/**
+ * Cachestat sum PIDs
+ *
+ * Sum values for all PIDs associated to a group
+ *
+ * @param publish output structure.
+ * @param root structure with listed IPs
+ */
+void ebpf_dc_sum_cgroup_pids(netdata_publish_dcstat_t *publish, struct pid_on_target2 *root)
+{
+ memset(&publish->curr, 0, sizeof(netdata_dcstat_pid_t));
+ netdata_dcstat_pid_t *dst = &publish->curr;
+ while (root) {
+ netdata_dcstat_pid_t *src = &root->dc;
+
+ dst->cache_access += src->cache_access;
+ dst->file_system += src->file_system;
+ dst->not_found += src->not_found;
+
+ root = root->next;
+ }
+}
+
+/**
+ * Calc chart values
+ *
+ * Do necessary math to plot charts.
+ */
+void ebpf_dc_calc_chart_values()
+{
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ ebpf_dc_sum_cgroup_pids(&ect->publish_dc, ect->pids);
+ uint64_t cache = ect->publish_dc.curr.cache_access;
+ uint64_t not_found = ect->publish_dc.curr.not_found;
+
+ dcstat_update_publish(&ect->publish_dc, cache, not_found);
+
+ ect->publish_dc.cache_access = (long long)ect->publish_dc.curr.cache_access -
+ (long long)ect->publish_dc.prev.cache_access;
+ ect->publish_dc.prev.cache_access = ect->publish_dc.curr.cache_access;
+
+ if (ect->publish_dc.curr.not_found < ect->publish_dc.prev.not_found) {
+ ect->publish_dc.prev.not_found = 0;
+ }
+ }
+}
+
+/**
+ * Create Systemd directory cache Charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ **/
+static void ebpf_create_systemd_dc_charts(int update_every)
+{
+ ebpf_create_charts_on_systemd(NETDATA_DC_HIT_CHART,
+ "Percentage of files inside directory cache",
+ EBPF_COMMON_DIMENSION_PERCENTAGE,
+ NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21200,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ NETDATA_SYSTEMD_DC_HIT_RATIO_CONTEXT, NETDATA_EBPF_MODULE_NAME_DCSTAT,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_DC_REFERENCE_CHART,
+ "Count file access",
+ EBPF_COMMON_DIMENSION_FILES,
+ NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21201,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ NETDATA_SYSTEMD_DC_REFERENCE_CONTEXT, NETDATA_EBPF_MODULE_NAME_DCSTAT,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_DC_REQUEST_NOT_CACHE_CHART,
+ "Files not present inside directory cache",
+ EBPF_COMMON_DIMENSION_FILES,
+ NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21202,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ NETDATA_SYSTEMD_DC_NOT_CACHE_CONTEXT, NETDATA_EBPF_MODULE_NAME_DCSTAT,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_DC_REQUEST_NOT_FOUND_CHART,
+ "Files not found",
+ EBPF_COMMON_DIMENSION_FILES,
+ NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21202,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ NETDATA_SYSTEMD_DC_NOT_FOUND_CONTEXT, NETDATA_EBPF_MODULE_NAME_DCSTAT,
+ update_every);
+}
+
+/**
+ * Send Directory Cache charts
+ *
+ * Send collected data to Netdata.
+ */
+static void ebpf_send_systemd_dc_charts()
+{
+ collected_number value;
+ ebpf_cgroup_target_t *ect;
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_DC_HIT_CHART);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long) ect->publish_dc.ratio);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_DC_REFERENCE_CHART);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long) ect->publish_dc.cache_access);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_DC_REQUEST_NOT_CACHE_CHART);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ value = (collected_number) (!ect->publish_dc.cache_access) ? 0 :
+ (long long )ect->publish_dc.curr.file_system - (long long)ect->publish_dc.prev.file_system;
+ ect->publish_dc.prev.file_system = ect->publish_dc.curr.file_system;
+
+ write_chart_dimension(ect->name, (long long) value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_DC_REQUEST_NOT_FOUND_CHART);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ value = (collected_number) (!ect->publish_dc.cache_access) ? 0 :
+ (long long)ect->publish_dc.curr.not_found - (long long)ect->publish_dc.prev.not_found;
+
+ ect->publish_dc.prev.not_found = ect->publish_dc.curr.not_found;
+
+ write_chart_dimension(ect->name, (long long) value);
+ }
+ }
+ write_end_chart();
+}
+
+/**
+ * Send Directory Cache charts
+ *
+ * Send collected data to Netdata.
+ *
+ */
+static void ebpf_send_specific_dc_data(char *type, netdata_publish_dcstat_t *pdc)
+{
+ collected_number value;
+ write_begin_chart(type, NETDATA_DC_HIT_CHART);
+ write_chart_dimension(dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_RATIO].name,
+ (long long) pdc->ratio);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_DC_REFERENCE_CHART);
+ write_chart_dimension(dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_REFERENCE].name,
+ (long long) pdc->cache_access);
+ write_end_chart();
+
+ value = (collected_number) (!pdc->cache_access) ? 0 :
+ (long long )pdc->curr.file_system - (long long)pdc->prev.file_system;
+ pdc->prev.file_system = pdc->curr.file_system;
+
+ write_begin_chart(type, NETDATA_DC_REQUEST_NOT_CACHE_CHART);
+ write_chart_dimension(dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_SLOW].name, (long long) value);
+ write_end_chart();
+
+ value = (collected_number) (!pdc->cache_access) ? 0 :
+ (long long)pdc->curr.not_found - (long long)pdc->prev.not_found;
+ pdc->prev.not_found = pdc->curr.not_found;
+
+ write_begin_chart(type, NETDATA_DC_REQUEST_NOT_FOUND_CHART);
+ write_chart_dimension(dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_MISS].name, (long long) value);
+ write_end_chart();
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+*/
+void ebpf_dc_send_cgroup_data(int update_every)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+ ebpf_dc_calc_chart_values();
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_dc_charts(update_every);
+ }
+
+ ebpf_send_systemd_dc_charts();
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_DC_CHART) && ect->updated) {
+ ebpf_create_specific_dc_charts(ect->name, update_every);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_DC_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_DC_CHART) {
+ if (ect->updated) {
+ ebpf_send_specific_dc_data(ect->name, &ect->publish_dc);
+ } else {
+ ebpf_obsolete_specific_dc_charts(ect->name, update_every);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_DC_CHART;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+* Main loop for this collector.
+*/
+static void dcstat_collector(ebpf_module_t *em)
+{
+ dcstat_threads.thread = mallocz(sizeof(netdata_thread_t));
+ dcstat_threads.start_routine = ebpf_dcstat_read_hash;
+
+ netdata_thread_create(dcstat_threads.thread, dcstat_threads.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_dcstat_read_hash, em);
+
+ netdata_publish_dcstat_t publish;
+ memset(&publish, 0, sizeof(publish));
+ int cgroups = em->cgroup_charts;
+ int update_every = em->update_every;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ netdata_apps_integration_flags_t apps = em->apps_charts;
+ pthread_mutex_lock(&collect_data_mutex);
+ if (apps)
+ read_apps_table();
+
+ if (cgroups)
+ ebpf_update_dc_cgroup();
+
+ pthread_mutex_lock(&lock);
+
+ dcstat_send_global(&publish);
+
+ if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED)
+ ebpf_dcache_send_apps_data(apps_groups_root_target);
+
+ if (cgroups)
+ ebpf_dc_send_cgroup_data(update_every);
+
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+}
+
+/*****************************************************************
+ *
+ * INITIALIZE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Create filesystem charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_filesystem_charts(int update_every)
+{
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY, NETDATA_DC_HIT_CHART,
+ "Percentage of files inside directory cache",
+ EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21200,
+ ebpf_create_global_dimension,
+ dcstat_counter_publish_aggregated, 1, update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY, NETDATA_DC_REFERENCE_CHART,
+ "Variables used to calculate hit ratio.",
+ EBPF_COMMON_DIMENSION_FILES, NETDATA_DIRECTORY_CACHE_SUBMENU,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21201,
+ ebpf_create_global_dimension,
+ &dcstat_counter_publish_aggregated[NETDATA_DCSTAT_IDX_REFERENCE], 3,
+ update_every, NETDATA_EBPF_MODULE_NAME_DCSTAT);
+
+ fflush(stdout);
+}
+
+/**
+ * Allocate vectors used with this thread.
+ *
+ * We are not testing the return, because callocz does this and shutdown the software
+ * case it was not possible to allocate.
+ *
+ * @param apps is apps enabled?
+ */
+static void ebpf_dcstat_allocate_global_vectors(int apps)
+{
+ if (apps)
+ dcstat_pid = callocz((size_t)pid_max, sizeof(netdata_publish_dcstat_t *));
+
+ dcstat_vector = callocz((size_t)ebpf_nprocs, sizeof(netdata_dcstat_pid_t));
+ dcstat_values = callocz((size_t)ebpf_nprocs, sizeof(netdata_idx_t));
+
+ memset(dcstat_counter_aggregated_data, 0, NETDATA_DCSTAT_IDX_END * sizeof(netdata_syscall_stat_t));
+ memset(dcstat_counter_publish_aggregated, 0, NETDATA_DCSTAT_IDX_END * sizeof(netdata_publish_syscall_t));
+}
+
+/*****************************************************************
+ *
+ * MAIN THREAD
+ *
+ *****************************************************************/
+
+/*
+ * Load BPF
+ *
+ * Load BPF files.
+ *
+ * @param em the structure with configuration
+ */
+static int ebpf_dcstat_load_bpf(ebpf_module_t *em)
+{
+ int ret = 0;
+ ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_DC_TARGET_LOOKUP_FAST].mode);
+ if (em->load & EBPF_LOAD_LEGACY) {
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ ret = -1;
+ }
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ bpf_obj = dc_bpf__open();
+ if (!bpf_obj)
+ ret = -1;
+ else
+ ret = ebpf_dc_load_and_attach(bpf_obj, em);
+ }
+#endif
+
+ if (ret)
+ error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->thread_name);
+
+ return ret;
+}
+
+/**
+ * Directory Cache thread
+ *
+ * Thread used to make dcstat thread
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always returns NULL
+ */
+void *ebpf_dcstat_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_dcstat_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = dcstat_maps;
+
+ ebpf_update_pid_table(&dcstat_maps[NETDATA_DCSTAT_PID_STATS], em);
+
+ ebpf_update_names(dc_optional_name, em);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_dcstat_load_bpf(em)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto enddcstat;
+ }
+
+ ebpf_dcstat_allocate_global_vectors(em->apps_charts);
+
+ int algorithms[NETDATA_DCSTAT_IDX_END] = {
+ NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX,
+ NETDATA_EBPF_ABSOLUTE_IDX
+ };
+
+ ebpf_global_labels(dcstat_counter_aggregated_data, dcstat_counter_publish_aggregated,
+ dcstat_counter_dimension_name, dcstat_counter_dimension_name,
+ algorithms, NETDATA_DCSTAT_IDX_END);
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_filesystem_charts(em->update_every);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ dcstat_collector(em);
+
+enddcstat:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_dcstat.h b/collectors/ebpf.plugin/ebpf_dcstat.h
new file mode 100644
index 0000000..d8687f9
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_dcstat.h
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_DCSTAT_H
+#define NETDATA_EBPF_DCSTAT_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_DCSTAT "dcstat"
+
+// charts
+#define NETDATA_DC_HIT_CHART "dc_hit_ratio"
+#define NETDATA_DC_REFERENCE_CHART "dc_reference"
+#define NETDATA_DC_REQUEST_NOT_CACHE_CHART "dc_not_cache"
+#define NETDATA_DC_REQUEST_NOT_FOUND_CHART "dc_not_found"
+
+#define NETDATA_DIRECTORY_CACHE_SUBMENU "directory cache (eBPF)"
+
+// configuration file
+#define NETDATA_DIRECTORY_DCSTAT_CONFIG_FILE "dcstat.conf"
+
+// Contexts
+#define NETDATA_CGROUP_DC_HIT_RATIO_CONTEXT "cgroup.dc_ratio"
+#define NETDATA_CGROUP_DC_REFERENCE_CONTEXT "cgroup.dc_reference"
+#define NETDATA_CGROUP_DC_NOT_CACHE_CONTEXT "cgroup.dc_not_cache"
+#define NETDATA_CGROUP_DC_NOT_FOUND_CONTEXT "cgroup.dc_not_found"
+
+#define NETDATA_SYSTEMD_DC_HIT_RATIO_CONTEXT "services.dc_ratio"
+#define NETDATA_SYSTEMD_DC_REFERENCE_CONTEXT "services.dc_reference"
+#define NETDATA_SYSTEMD_DC_NOT_CACHE_CONTEXT "services.dc_not_cache"
+#define NETDATA_SYSTEMD_DC_NOT_FOUND_CONTEXT "services.dc_not_found"
+
+#define NETDATA_LATENCY_DCSTAT_SLEEP_MS 700000ULL
+
+enum directory_cache_indexes {
+ NETDATA_DCSTAT_IDX_RATIO,
+ NETDATA_DCSTAT_IDX_REFERENCE,
+ NETDATA_DCSTAT_IDX_SLOW,
+ NETDATA_DCSTAT_IDX_MISS,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_DCSTAT_IDX_END
+};
+
+enum directory_cache_tables {
+ NETDATA_DCSTAT_GLOBAL_STATS,
+ NETDATA_DCSTAT_PID_STATS,
+ NETDATA_DCSTAT_CTRL
+};
+
+// variables
+enum directory_cache_counters {
+ NETDATA_KEY_DC_REFERENCE,
+ NETDATA_KEY_DC_SLOW,
+ NETDATA_KEY_DC_MISS,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_DIRECTORY_CACHE_END
+};
+
+enum directory_cache_targets {
+ NETDATA_DC_TARGET_LOOKUP_FAST,
+ NETDATA_DC_TARGET_D_LOOKUP
+};
+
+typedef struct netdata_publish_dcstat_pid {
+ uint64_t cache_access;
+ uint64_t file_system;
+ uint64_t not_found;
+} netdata_dcstat_pid_t;
+
+typedef struct netdata_publish_dcstat {
+ long long ratio;
+ long long cache_access;
+
+ netdata_dcstat_pid_t curr;
+ netdata_dcstat_pid_t prev;
+} netdata_publish_dcstat_t;
+
+void *ebpf_dcstat_thread(void *ptr);
+void ebpf_dcstat_create_apps_charts(struct ebpf_module *em, void *ptr);
+extern struct config dcstat_config;
+extern netdata_ebpf_targets_t dc_targets[];
+extern ebpf_local_maps_t dcstat_maps[];
+
+#endif // NETDATA_EBPF_DCSTAT_H
diff --git a/collectors/ebpf.plugin/ebpf_disk.c b/collectors/ebpf.plugin/ebpf_disk.c
new file mode 100644
index 0000000..a27bd81
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_disk.c
@@ -0,0 +1,865 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <sys/resource.h>
+#include <stdlib.h>
+
+#include "ebpf.h"
+#include "ebpf_disk.h"
+
+struct config disk_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+static ebpf_local_maps_t disk_maps[] = {{.name = "tbl_disk_iocall", .internal_input = NETDATA_DISK_HISTOGRAM_LENGTH,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
+static avl_tree_lock disk_tree;
+netdata_ebpf_disks_t *disk_list = NULL;
+
+char *tracepoint_block_type = { "block"} ;
+char *tracepoint_block_issue = { "block_rq_issue" };
+char *tracepoint_block_rq_complete = { "block_rq_complete" };
+
+static int was_block_issue_enabled = 0;
+static int was_block_rq_complete_enabled = 0;
+
+static char **dimensions = NULL;
+static netdata_syscall_stat_t disk_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS];
+static netdata_publish_syscall_t disk_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS];
+
+static netdata_idx_t *disk_hash_values = NULL;
+static struct netdata_static_thread disk_threads = {
+ .name = "DISK KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+ebpf_publish_disk_t *plot_disks = NULL;
+pthread_mutex_t plot_mutex;
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO MANIPULATE HARD DISKS
+ *
+ *****************************************************************/
+
+/**
+ * Parse start
+ *
+ * Parse start address of disk
+ *
+ * @param w structure where data is stored
+ * @param filename variable used to store value
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+static inline int ebpf_disk_parse_start(netdata_ebpf_disks_t *w, char *filename)
+{
+ char content[FILENAME_MAX + 1];
+ int fd = open(filename, O_RDONLY, 0);
+ if (fd < 0) {
+ return -1;
+ }
+
+ ssize_t file_length = read(fd, content, 4095);
+ if (file_length > 0) {
+ if (file_length > FILENAME_MAX)
+ file_length = FILENAME_MAX;
+
+ content[file_length] = '\0';
+ w->start = strtoul(content, NULL, 10);
+ }
+ close(fd);
+
+ return 0;
+}
+
+/**
+ * Parse uevent
+ *
+ * Parse uevent file
+ *
+ * @param w structure where data is stored
+ * @param filename variable used to store value
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+static inline int ebpf_parse_uevent(netdata_ebpf_disks_t *w, char *filename)
+{
+ char content[FILENAME_MAX + 1];
+ int fd = open(filename, O_RDONLY, 0);
+ if (fd < 0) {
+ return -1;
+ }
+
+ ssize_t file_length = read(fd, content, FILENAME_MAX);
+ if (file_length > 0) {
+ if (file_length > FILENAME_MAX)
+ file_length = FILENAME_MAX;
+
+ content[file_length] = '\0';
+
+ char *s = strstr(content, "PARTNAME=EFI");
+ if (s) {
+ w->main->boot_partition = w;
+ w->flags |= NETDATA_DISK_HAS_EFI;
+ w->boot_chart = strdupz("disk_bootsector");
+ }
+ }
+ close(fd);
+
+ return 0;
+}
+
+/**
+ * Parse Size
+ *
+ * @param w structure where data is stored
+ * @param filename variable used to store value
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+static inline int ebpf_parse_size(netdata_ebpf_disks_t *w, char *filename)
+{
+ char content[FILENAME_MAX + 1];
+ int fd = open(filename, O_RDONLY, 0);
+ if (fd < 0) {
+ return -1;
+ }
+
+ ssize_t file_length = read(fd, content, FILENAME_MAX);
+ if (file_length > 0) {
+ if (file_length > FILENAME_MAX)
+ file_length = FILENAME_MAX;
+
+ content[file_length] = '\0';
+ w->end = w->start + strtoul(content, NULL, 10) -1;
+ }
+ close(fd);
+
+ return 0;
+}
+
+/**
+ * Read Disk information
+ *
+ * Read disk information from /sys/block
+ *
+ * @param w structure where data is stored
+ * @param name disk name
+ */
+static void ebpf_read_disk_info(netdata_ebpf_disks_t *w, char *name)
+{
+ static netdata_ebpf_disks_t *main_disk = NULL;
+ static uint32_t key = 0;
+ char *path = { "/sys/block" };
+ char disk[NETDATA_DISK_NAME_LEN + 1];
+ char filename[FILENAME_MAX + 1];
+ snprintfz(disk, NETDATA_DISK_NAME_LEN, "%s", name);
+ size_t length = strlen(disk);
+ if (!length) {
+ return;
+ }
+
+ length--;
+ size_t curr = length;
+ while (isdigit((int)disk[length])) {
+ disk[length--] = '\0';
+ }
+
+ // We are looking for partition information, if it is a device we will ignore it.
+ if (curr == length) {
+ main_disk = w;
+ key = MKDEV(w->major, w->minor);
+ w->bootsector_key = key;
+ return;
+ }
+ w->bootsector_key = key;
+ w->main = main_disk;
+
+ snprintfz(filename, FILENAME_MAX, "%s/%s/%s/uevent", path, disk, name);
+ if (ebpf_parse_uevent(w, filename))
+ return;
+
+ snprintfz(filename, FILENAME_MAX, "%s/%s/%s/start", path, disk, name);
+ if (ebpf_disk_parse_start(w, filename))
+ return;
+
+ snprintfz(filename, FILENAME_MAX, "%s/%s/%s/size", path, disk, name);
+ ebpf_parse_size(w, filename);
+}
+
+/**
+ * New encode dev
+ *
+ * New encode algorithm extracted from https://elixir.bootlin.com/linux/v5.10.8/source/include/linux/kdev_t.h#L39
+ *
+ * @param major driver major number
+ * @param minor driver minor number
+ *
+ * @return
+ */
+static inline uint32_t netdata_new_encode_dev(uint32_t major, uint32_t minor) {
+ return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12);
+}
+
+/**
+ * Compare disks
+ *
+ * Compare major and minor values to add disks to tree.
+ *
+ * @param a pointer to netdata_ebpf_disks
+ * @param b pointer to netdata_ebpf_disks
+ *
+ * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b.
+*/
+static int ebpf_compare_disks(void *a, void *b)
+{
+ netdata_ebpf_disks_t *ptr1 = a;
+ netdata_ebpf_disks_t *ptr2 = b;
+
+ if (ptr1->dev > ptr2->dev)
+ return 1;
+ if (ptr1->dev < ptr2->dev)
+ return -1;
+
+ return 0;
+}
+
+/**
+ * Update listen table
+ *
+ * Update link list when it is necessary.
+ *
+ * @param name disk name
+ * @param major major disk identifier
+ * @param minor minor disk identifier
+ * @param current_time current timestamp
+ */
+static void update_disk_table(char *name, int major, int minor, time_t current_time)
+{
+ netdata_ebpf_disks_t find;
+ netdata_ebpf_disks_t *w;
+ size_t length;
+
+ uint32_t dev = netdata_new_encode_dev(major, minor);
+ find.dev = dev;
+ netdata_ebpf_disks_t *ret = (netdata_ebpf_disks_t *) avl_search_lock(&disk_tree, (avl_t *)&find);
+ if (ret) { // Disk is already present
+ ret->flags |= NETDATA_DISK_IS_HERE;
+ ret->last_update = current_time;
+ return;
+ }
+
+ netdata_ebpf_disks_t *update_next = disk_list;
+ if (likely(disk_list)) {
+ netdata_ebpf_disks_t *move = disk_list;
+ while (move) {
+ if (dev == move->dev)
+ return;
+
+ update_next = move;
+ move = move->next;
+ }
+
+ w = callocz(1, sizeof(netdata_ebpf_disks_t));
+ length = strlen(name);
+ if (length >= NETDATA_DISK_NAME_LEN)
+ length = NETDATA_DISK_NAME_LEN;
+
+ memcpy(w->family, name, length);
+ w->family[length] = '\0';
+ w->major = major;
+ w->minor = minor;
+ w->dev = netdata_new_encode_dev(major, minor);
+ update_next->next = w;
+ } else {
+ disk_list = callocz(1, sizeof(netdata_ebpf_disks_t));
+ length = strlen(name);
+ if (length >= NETDATA_DISK_NAME_LEN)
+ length = NETDATA_DISK_NAME_LEN;
+
+ memcpy(disk_list->family, name, length);
+ disk_list->family[length] = '\0';
+ disk_list->major = major;
+ disk_list->minor = minor;
+ disk_list->dev = netdata_new_encode_dev(major, minor);
+
+ w = disk_list;
+ }
+
+ ebpf_read_disk_info(w, name);
+
+ netdata_ebpf_disks_t *check;
+ check = (netdata_ebpf_disks_t *) avl_insert_lock(&disk_tree, (avl_t *)w);
+ if (check != w)
+ error("Internal error, cannot insert the AVL tree.");
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("The Latency is monitoring the hard disk %s (Major = %d, Minor = %d, Device = %u)", name, major, minor,w->dev);
+#endif
+
+ w->flags |= NETDATA_DISK_IS_HERE;
+}
+
+/**
+ * Read Local Disks
+ *
+ * Parse /proc/partitions to get block disks used to measure latency.
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+static int read_local_disks()
+{
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, NETDATA_EBPF_PROC_PARTITIONS);
+ procfile *ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if (!ff)
+ return -1;
+
+ ff = procfile_readall(ff);
+ if (!ff)
+ return -1;
+
+ size_t lines = procfile_lines(ff), l;
+ time_t current_time = now_realtime_sec();
+ for(l = 2; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ // This is header or end of file
+ if (unlikely(words < 4))
+ continue;
+
+ int major = (int)strtol(procfile_lineword(ff, l, 0), NULL, 10);
+ // The main goal of this thread is to measure block devices, so any block device with major number
+ // smaller than 7 according /proc/devices is not "important".
+ if (major > 7) {
+ int minor = (int)strtol(procfile_lineword(ff, l, 1), NULL, 10);
+ update_disk_table(procfile_lineword(ff, l, 3), major, minor, current_time);
+ }
+ }
+
+ procfile_close(ff);
+
+ return 0;
+}
+
+/**
+ * Update disks
+ *
+ * @param em main thread structure
+ */
+void ebpf_update_disks(ebpf_module_t *em)
+{
+ static time_t update_every = 0;
+ time_t curr = now_realtime_sec();
+ if (curr < update_every)
+ return;
+
+ update_every = curr + 5 * em->update_every;
+
+ (void)read_local_disks();
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Disk disable tracepoints
+ *
+ * Disable tracepoints when the plugin was responsible to enable it.
+ */
+static void ebpf_disk_disable_tracepoints()
+{
+ char *default_message = { "Cannot disable the tracepoint" };
+ if (!was_block_issue_enabled) {
+ if (ebpf_disable_tracing_values(tracepoint_block_type, tracepoint_block_issue))
+ error("%s %s/%s.", default_message, tracepoint_block_type, tracepoint_block_issue);
+ }
+
+ if (!was_block_rq_complete_enabled) {
+ if (ebpf_disable_tracing_values(tracepoint_block_type, tracepoint_block_rq_complete))
+ error("%s %s/%s.", default_message, tracepoint_block_type, tracepoint_block_rq_complete);
+ }
+}
+
+/**
+ * Cleanup plot disks
+ *
+ * Clean disk list
+ */
+static void ebpf_cleanup_plot_disks()
+{
+ ebpf_publish_disk_t *move = plot_disks, *next;
+ while (move) {
+ next = move->next;
+
+ freez(move);
+
+ move = next;
+ }
+}
+
+/**
+ * Cleanup Disk List
+ */
+static void ebpf_cleanup_disk_list()
+{
+ netdata_ebpf_disks_t *move = disk_list;
+ while (move) {
+ netdata_ebpf_disks_t *next = move->next;
+
+ freez(move->histogram.name);
+ freez(move->boot_chart);
+ freez(move);
+
+ move = next;
+ }
+}
+
+/**
+ * DISK Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_disk_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ ebpf_disk_disable_tracepoints();
+
+ if (dimensions)
+ ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+
+ freez(disk_hash_values);
+ freez(disk_threads.thread);
+ pthread_mutex_destroy(&plot_mutex);
+
+ ebpf_cleanup_plot_disks();
+ ebpf_cleanup_disk_list();
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Disk exit.
+ *
+ * Cancel child and exit.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_disk_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*disk_threads.thread);
+ ebpf_disk_free(em);
+}
+
+/**
+ * Disk Cleanup
+ *
+ * Clean up allocated memory.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_disk_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_disk_free(em);
+}
+
+/*****************************************************************
+ *
+ * MAIN LOOP
+ *
+ *****************************************************************/
+
+/**
+ * Fill Plot list
+ *
+ * @param ptr a pointer for current disk
+ */
+static void ebpf_fill_plot_disks(netdata_ebpf_disks_t *ptr)
+{
+ pthread_mutex_lock(&plot_mutex);
+ ebpf_publish_disk_t *w;
+ if (likely(plot_disks)) {
+ ebpf_publish_disk_t *move = plot_disks, *store = plot_disks;
+ while (move) {
+ if (move->plot == ptr) {
+ pthread_mutex_unlock(&plot_mutex);
+ return;
+ }
+
+ store = move;
+ move = move->next;
+ }
+
+ w = callocz(1, sizeof(ebpf_publish_disk_t));
+ w->plot = ptr;
+ store->next = w;
+ } else {
+ plot_disks = callocz(1, sizeof(ebpf_publish_disk_t));
+ plot_disks->plot = ptr;
+ }
+ pthread_mutex_unlock(&plot_mutex);
+
+ ptr->flags |= NETDATA_DISK_ADDED_TO_PLOT_LIST;
+}
+
+/**
+ * Read hard disk table
+ *
+ * @param table file descriptor for table
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_hard_disk_tables(int table)
+{
+ netdata_idx_t *values = disk_hash_values;
+ block_key_t key = {};
+ block_key_t next_key = {};
+
+ netdata_ebpf_disks_t *ret = NULL;
+
+ while (bpf_map_get_next_key(table, &key, &next_key) == 0) {
+ int test = bpf_map_lookup_elem(table, &key, values);
+ if (test < 0) {
+ key = next_key;
+ continue;
+ }
+
+ netdata_ebpf_disks_t find;
+ find.dev = key.dev;
+
+ if (likely(ret)) {
+ if (find.dev != ret->dev)
+ ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find);
+ } else
+ ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find);
+
+ // Disk was inserted after we parse /proc/partitions
+ if (!ret) {
+ if (read_local_disks()) {
+ key = next_key;
+ continue;
+ }
+
+ ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find);
+ if (!ret) {
+ // We should never reach this point, but we are adding it to keep a safe code
+ key = next_key;
+ continue;
+ }
+ }
+
+ uint64_t total = 0;
+ int i;
+ int end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs;
+ for (i = 0; i < end; i++) {
+ total += values[i];
+ }
+
+ ret->histogram.histogram[key.bin] = total;
+
+ if (!(ret->flags & NETDATA_DISK_ADDED_TO_PLOT_LIST))
+ ebpf_fill_plot_disks(ret);
+
+ key = next_key;
+ }
+}
+
+/**
+ * Disk read hash
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data on very busy socket.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_disk_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_disk_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ usec_t step = NETDATA_LATENCY_DISK_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_hard_disk_tables(disk_maps[NETDATA_DISK_READ].map_fd);
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Obsolete Hard Disk charts
+ *
+ * Make Hard disk charts and fill chart name
+ *
+ * @param w the structure with necessary information to create the chart
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_obsolete_hd_charts(netdata_ebpf_disks_t *w, int update_every)
+{
+ ebpf_write_chart_obsolete(w->histogram.name, w->family, w->histogram.title, EBPF_COMMON_DIMENSION_CALL,
+ w->family, NETDATA_EBPF_CHART_TYPE_STACKED, "disk.latency_io",
+ w->histogram.order, update_every);
+
+ w->flags = 0;
+}
+
+/**
+ * Create Hard Disk charts
+ *
+ * Make Hard disk charts and fill chart name
+ *
+ * @param w the structure with necessary information to create the chart
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_hd_charts(netdata_ebpf_disks_t *w, int update_every)
+{
+ int order = NETDATA_CHART_PRIO_DISK_LATENCY;
+ char *family = w->family;
+
+ w->histogram.name = strdupz("disk_latency_io");
+ w->histogram.title = NULL;
+ w->histogram.order = order;
+
+ ebpf_create_chart(w->histogram.name, family, "Disk latency", EBPF_COMMON_DIMENSION_CALL,
+ family, "disk.latency_io", NETDATA_EBPF_CHART_TYPE_STACKED, order,
+ ebpf_create_global_dimension, disk_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS,
+ update_every, NETDATA_EBPF_MODULE_NAME_DISK);
+ order++;
+
+ w->flags |= NETDATA_DISK_CHART_CREATED;
+}
+
+/**
+ * Remove pointer from plot
+ *
+ * Remove pointer from plot list when the disk is not present.
+ */
+static void ebpf_remove_pointer_from_plot_disk(ebpf_module_t *em)
+{
+ time_t current_time = now_realtime_sec();
+ time_t limit = 10 * em->update_every;
+ pthread_mutex_lock(&plot_mutex);
+ ebpf_publish_disk_t *move = plot_disks, *prev = plot_disks;
+ int update_every = em->update_every;
+ while (move) {
+ netdata_ebpf_disks_t *ned = move->plot;
+ uint32_t flags = ned->flags;
+
+ if (!(flags & NETDATA_DISK_IS_HERE) && ((current_time - ned->last_update) > limit)) {
+ ebpf_obsolete_hd_charts(ned, update_every);
+ avl_t *ret = (avl_t *)avl_remove_lock(&disk_tree, (avl_t *)ned);
+ UNUSED(ret);
+ if (move == plot_disks) {
+ freez(move);
+ plot_disks = NULL;
+ break;
+ } else {
+ prev->next = move->next;
+ ebpf_publish_disk_t *clean = move;
+ move = move->next;
+ freez(clean);
+ continue;
+ }
+ }
+
+ prev = move;
+ move = move->next;
+ }
+ pthread_mutex_unlock(&plot_mutex);
+}
+
+/**
+ * Send Hard disk data
+ *
+ * Send hard disk information to Netdata.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_latency_send_hd_data(int update_every)
+{
+ pthread_mutex_lock(&plot_mutex);
+ if (!plot_disks) {
+ pthread_mutex_unlock(&plot_mutex);
+ return;
+ }
+
+ ebpf_publish_disk_t *move = plot_disks;
+ while (move) {
+ netdata_ebpf_disks_t *ned = move->plot;
+ uint32_t flags = ned->flags;
+ if (!(flags & NETDATA_DISK_CHART_CREATED)) {
+ ebpf_create_hd_charts(ned, update_every);
+ }
+
+ if ((flags & NETDATA_DISK_CHART_CREATED)) {
+ write_histogram_chart(ned->histogram.name, ned->family,
+ ned->histogram.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+ }
+
+ ned->flags &= ~NETDATA_DISK_IS_HERE;
+
+ move = move->next;
+ }
+ pthread_mutex_unlock(&plot_mutex);
+}
+
+/**
+* Main loop for this collector.
+*/
+static void disk_collector(ebpf_module_t *em)
+{
+ disk_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t));
+ disk_threads.thread = mallocz(sizeof(netdata_thread_t));
+ disk_threads.start_routine = ebpf_disk_read_hash;
+
+ netdata_thread_create(disk_threads.thread, disk_threads.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_disk_read_hash, em);
+
+ int update_every = em->update_every;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ pthread_mutex_lock(&lock);
+ ebpf_remove_pointer_from_plot_disk(em);
+ ebpf_latency_send_hd_data(update_every);
+
+ pthread_mutex_unlock(&lock);
+
+ ebpf_update_disks(em);
+ }
+}
+
+/*****************************************************************
+ *
+ * EBPF DISK THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Enable tracepoints
+ *
+ * Enable necessary tracepoints for thread.
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+static int ebpf_disk_enable_tracepoints()
+{
+ int test = ebpf_is_tracepoint_enabled(tracepoint_block_type, tracepoint_block_issue);
+ if (test == -1)
+ return -1;
+ else if (!test) {
+ if (ebpf_enable_tracing_values(tracepoint_block_type, tracepoint_block_issue))
+ return -1;
+ }
+ was_block_issue_enabled = test;
+
+ test = ebpf_is_tracepoint_enabled(tracepoint_block_type, tracepoint_block_rq_complete);
+ if (test == -1)
+ return -1;
+ else if (!test) {
+ if (ebpf_enable_tracing_values(tracepoint_block_type, tracepoint_block_rq_complete))
+ return -1;
+ }
+ was_block_rq_complete_enabled = test;
+
+ return 0;
+}
+
+/**
+ * Disk thread
+ *
+ * Thread used to generate disk charts.
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always return NULL
+ */
+void *ebpf_disk_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_disk_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = disk_maps;
+
+ if (ebpf_disk_enable_tracepoints()) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto enddisk;
+ }
+
+ avl_init_lock(&disk_tree, ebpf_compare_disks);
+ if (read_local_disks()) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto enddisk;
+ }
+
+ if (pthread_mutex_init(&plot_mutex, NULL)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ error("Cannot initialize local mutex");
+ goto enddisk;
+ }
+
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto enddisk;
+ }
+
+ int algorithms[NETDATA_EBPF_HIST_MAX_BINS];
+ ebpf_fill_algorithms(algorithms, NETDATA_EBPF_HIST_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX);
+ dimensions = ebpf_fill_histogram_dimension(NETDATA_EBPF_HIST_MAX_BINS);
+
+ ebpf_global_labels(disk_aggregated_data, disk_publish_aggregated, dimensions, dimensions, algorithms,
+ NETDATA_EBPF_HIST_MAX_BINS);
+
+ pthread_mutex_lock(&lock);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ disk_collector(em);
+
+enddisk:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_disk.h b/collectors/ebpf.plugin/ebpf_disk.h
new file mode 100644
index 0000000..c14b887
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_disk.h
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_DISK_H
+#define NETDATA_EBPF_DISK_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_DISK "disk"
+
+#include "libnetdata/avl/avl.h"
+#include "libnetdata/ebpf/ebpf.h"
+
+#define NETDATA_EBPF_PROC_PARTITIONS "/proc/partitions"
+
+#define NETDATA_LATENCY_DISK_SLEEP_MS 650000ULL
+
+// Process configuration name
+#define NETDATA_DISK_CONFIG_FILE "disk.conf"
+
+// Decode function extracted from: https://elixir.bootlin.com/linux/v5.10.8/source/include/linux/kdev_t.h#L7
+#define MINORBITS 20
+#define MKDEV(ma,mi) (((ma) << MINORBITS) | (mi))
+
+enum netdata_latency_disks_flags {
+ NETDATA_DISK_ADDED_TO_PLOT_LIST = 1,
+ NETDATA_DISK_CHART_CREATED = 2,
+ NETDATA_DISK_IS_HERE = 4,
+ NETDATA_DISK_HAS_EFI = 8
+};
+
+/*
+ * The definition (DISK_NAME_LEN) has been a stable value since Kernel 3.0,
+ * I decided to bring it as internal definition, to avoid include linux/genhd.h.
+ */
+#define NETDATA_DISK_NAME_LEN 32
+typedef struct netdata_ebpf_disks {
+ // Search
+ avl_t avl;
+ uint32_t dev;
+ uint32_t major;
+ uint32_t minor;
+ uint32_t bootsector_key;
+ uint64_t start; // start sector
+ uint64_t end; // end sector
+
+ // Print information
+ char family[NETDATA_DISK_NAME_LEN + 1];
+ char *boot_chart;
+
+ netdata_ebpf_histogram_t histogram;
+
+ uint32_t flags;
+ time_t last_update;
+
+ struct netdata_ebpf_disks *main;
+ struct netdata_ebpf_disks *boot_partition;
+ struct netdata_ebpf_disks *next;
+} netdata_ebpf_disks_t;
+
+enum ebpf_disk_tables {
+ NETDATA_DISK_READ
+};
+
+typedef struct block_key {
+ uint32_t bin;
+ uint32_t dev;
+} block_key_t;
+
+typedef struct netdata_ebpf_publish_disk {
+ netdata_ebpf_disks_t *plot;
+ struct netdata_ebpf_publish_disk *next;
+} ebpf_publish_disk_t;
+
+extern struct config disk_config;
+
+void *ebpf_disk_thread(void *ptr);
+
+#endif /* NETDATA_EBPF_DISK_H */
+
diff --git a/collectors/ebpf.plugin/ebpf_fd.c b/collectors/ebpf.plugin/ebpf_fd.c
new file mode 100644
index 0000000..30b7f22
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_fd.c
@@ -0,0 +1,1183 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_fd.h"
+
+static char *fd_dimension_names[NETDATA_FD_SYSCALL_END] = { "open", "close" };
+static char *fd_id_names[NETDATA_FD_SYSCALL_END] = { "do_sys_open", "__close_fd" };
+
+static netdata_syscall_stat_t fd_aggregated_data[NETDATA_FD_SYSCALL_END];
+static netdata_publish_syscall_t fd_publish_aggregated[NETDATA_FD_SYSCALL_END];
+
+static ebpf_local_maps_t fd_maps[] = {{.name = "tbl_fd_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_fd_global", .internal_input = NETDATA_KEY_END_VECTOR,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "fd_ctrl", .internal_input = NETDATA_CONTROLLER_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
+
+
+struct config fd_config = { .first_section = NULL, .last_section = NULL, .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = {.avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+struct netdata_static_thread fd_thread = {"FD KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL};
+
+static netdata_idx_t fd_hash_values[NETDATA_FD_COUNTER];
+static netdata_idx_t *fd_values = NULL;
+
+netdata_fd_stat_t *fd_vector = NULL;
+netdata_fd_stat_t **fd_pid = NULL;
+
+netdata_ebpf_targets_t fd_targets[] = { {.name = "open", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "close", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/fd.skel.h" // BTF code
+
+static struct fd_bpf *bpf_obj = NULL;
+
+/**
+ * Disable probe
+ *
+ * Disable all probes to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects
+*/
+static inline void ebpf_fd_disable_probes(struct fd_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_sys_open_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_sys_open_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fd_kprobe, false);
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_11) {
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_kprobe, false);
+ } else {
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_kprobe, false);
+ }
+}
+
+/*
+ * Disable specific probe
+ *
+ * Disable probes according the kernel version
+ *
+ * @param obj is the main structure for bpf objects
+ */
+static inline void ebpf_disable_specific_probes(struct fd_bpf *obj)
+{
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_11) {
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_kprobe, false);
+ } else {
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_kprobe, false);
+ }
+}
+
+/*
+ * Disable trampoline
+ *
+ * Disable all trampoline to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void ebpf_disable_trampoline(struct fd_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_sys_open_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_sys_open_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fd_fentry, false);
+}
+
+/*
+ * Disable specific trampoline
+ *
+ * Disable trampoline according to kernel version.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void ebpf_disable_specific_trampoline(struct fd_bpf *obj)
+{
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_11) {
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata___close_fd_fexit, false);
+ } else {
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_close_fd_fexit, false);
+ }
+}
+
+/**
+ * Set trampoline target
+ *
+ * Set the targets we will monitor.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_set_trampoline_target(struct fd_bpf *obj)
+{
+ bpf_program__set_attach_target(obj->progs.netdata_sys_open_fentry, 0, fd_targets[NETDATA_FD_SYSCALL_OPEN].name);
+ bpf_program__set_attach_target(obj->progs.netdata_sys_open_fexit, 0, fd_targets[NETDATA_FD_SYSCALL_OPEN].name);
+ bpf_program__set_attach_target(obj->progs.netdata_release_task_fd_fentry, 0, EBPF_COMMON_FNCT_CLEAN_UP);
+
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_11) {
+ bpf_program__set_attach_target(
+ obj->progs.netdata_close_fd_fentry, 0, fd_targets[NETDATA_FD_SYSCALL_CLOSE].name);
+ bpf_program__set_attach_target(obj->progs.netdata_close_fd_fexit, 0, fd_targets[NETDATA_FD_SYSCALL_CLOSE].name);
+ } else {
+ bpf_program__set_attach_target(
+ obj->progs.netdata___close_fd_fentry, 0, fd_targets[NETDATA_FD_SYSCALL_CLOSE].name);
+ bpf_program__set_attach_target(
+ obj->progs.netdata___close_fd_fexit, 0, fd_targets[NETDATA_FD_SYSCALL_CLOSE].name);
+ }
+}
+
+/**
+ * Mount Attach Probe
+ *
+ * Attach probes to target
+ *
+ * @param obj is the main structure for bpf objects.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+static int ebpf_fd_attach_probe(struct fd_bpf *obj)
+{
+ obj->links.netdata_sys_open_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_sys_open_kprobe, false,
+ fd_targets[NETDATA_FD_SYSCALL_OPEN].name);
+ int ret = libbpf_get_error(obj->links.netdata_sys_open_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_sys_open_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_sys_open_kretprobe, true,
+ fd_targets[NETDATA_FD_SYSCALL_OPEN].name);
+ ret = libbpf_get_error(obj->links.netdata_sys_open_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_release_task_fd_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_release_task_fd_kprobe,
+ false,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+ ret = libbpf_get_error(obj->links.netdata_release_task_fd_kprobe);
+ if (ret)
+ return -1;
+
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_11) {
+ obj->links.netdata_close_fd_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_close_fd_kretprobe, true,
+ fd_targets[NETDATA_FD_SYSCALL_CLOSE].name);
+ ret = libbpf_get_error(obj->links.netdata_close_fd_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_close_fd_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_close_fd_kprobe, false,
+ fd_targets[NETDATA_FD_SYSCALL_CLOSE].name);
+ ret = libbpf_get_error(obj->links.netdata_close_fd_kprobe);
+ if (ret)
+ return -1;
+ } else {
+ obj->links.netdata___close_fd_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata___close_fd_kretprobe,
+ true,
+ fd_targets[NETDATA_FD_SYSCALL_CLOSE].name);
+ ret = libbpf_get_error(obj->links.netdata___close_fd_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata___close_fd_kprobe = bpf_program__attach_kprobe(obj->progs.netdata___close_fd_kprobe,
+ false,
+ fd_targets[NETDATA_FD_SYSCALL_CLOSE].name);
+ ret = libbpf_get_error(obj->links.netdata___close_fd_kprobe);
+ if (ret)
+ return -1;
+ }
+
+ return 0;
+}
+
+/**
+ * Set target values
+ *
+ * Set pointers used to laod data.
+ */
+static void ebpf_fd_set_target_values()
+{
+ static char *close_targets[] = {"close_fd", "__close_fd"};
+ static char *open_targets[] = {"do_sys_openat2", "do_sys_open"};
+ if (running_on_kernel >= NETDATA_EBPF_KERNEL_5_11) {
+ fd_targets[NETDATA_FD_SYSCALL_OPEN].name = open_targets[0];
+ fd_targets[NETDATA_FD_SYSCALL_CLOSE].name = close_targets[0];
+ } else {
+ fd_targets[NETDATA_FD_SYSCALL_OPEN].name = open_targets[1];
+ fd_targets[NETDATA_FD_SYSCALL_CLOSE].name = close_targets[1];
+ }
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_fd_set_hash_tables(struct fd_bpf *obj)
+{
+ fd_maps[NETDATA_FD_GLOBAL_STATS].map_fd = bpf_map__fd(obj->maps.tbl_fd_global);
+ fd_maps[NETDATA_FD_PID_STATS].map_fd = bpf_map__fd(obj->maps.tbl_fd_pid);
+ fd_maps[NETDATA_FD_CONTROLLER].map_fd = bpf_map__fd(obj->maps.fd_ctrl);
+}
+
+/**
+ * Adjust Map Size
+ *
+ * Resize maps according input from users.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ */
+static void ebpf_fd_adjust_map_size(struct fd_bpf *obj, ebpf_module_t *em)
+{
+ ebpf_update_map_size(obj->maps.tbl_fd_pid, &fd_maps[NETDATA_FD_PID_STATS],
+ em, bpf_map__name(obj->maps.tbl_fd_pid));
+}
+
+/**
+ * Disable Release Task
+ *
+ * Disable release task when apps is not enabled.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_fd_disable_release_task(struct fd_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fd_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fd_fentry, false);
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_fd_load_and_attach(struct fd_bpf *obj, ebpf_module_t *em)
+{
+ netdata_ebpf_targets_t *mt = em->targets;
+ netdata_ebpf_program_loaded_t test = mt[NETDATA_FD_SYSCALL_OPEN].mode;
+
+ ebpf_fd_set_target_values();
+ if (test == EBPF_LOAD_TRAMPOLINE) {
+ ebpf_fd_disable_probes(obj);
+ ebpf_disable_specific_trampoline(obj);
+
+ ebpf_set_trampoline_target(obj);
+ // TODO: Remove this in next PR, because this specific trampoline has an error.
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fd_fentry, false);
+ } else {
+ ebpf_disable_trampoline(obj);
+ ebpf_disable_specific_probes(obj);
+ }
+
+ ebpf_fd_adjust_map_size(obj, em);
+
+ if (!em->apps_charts && !em->cgroup_charts)
+ ebpf_fd_disable_release_task(obj);
+
+ int ret = fd_bpf__load(obj);
+ if (ret) {
+ return ret;
+ }
+
+ ret = (test == EBPF_LOAD_TRAMPOLINE) ? fd_bpf__attach(obj) : ebpf_fd_attach_probe(obj);
+ if (!ret) {
+ ebpf_fd_set_hash_tables(obj);
+
+ ebpf_update_controller(fd_maps[NETDATA_CACHESTAT_CTRL].map_fd, em);
+ }
+
+ return ret;
+}
+#endif
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * FD Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_fd_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ ebpf_cleanup_publish_syscall(fd_publish_aggregated);
+ freez(fd_thread.thread);
+ freez(fd_values);
+ freez(fd_vector);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ if (bpf_obj)
+ fd_bpf__destroy(bpf_obj);
+#endif
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * FD Exit
+ *
+ * Cancel child thread and exit.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_fd_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*fd_thread.thread);
+ ebpf_fd_free(em);
+}
+
+/**
+ * Clean up the main thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_fd_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_fd_free(em);
+}
+
+/*****************************************************************
+ *
+ * MAIN LOOP
+ *
+ *****************************************************************/
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the structure with thread information
+ */
+static void ebpf_fd_send_data(ebpf_module_t *em)
+{
+ fd_publish_aggregated[NETDATA_FD_SYSCALL_OPEN].ncall = fd_hash_values[NETDATA_KEY_CALLS_DO_SYS_OPEN];
+ fd_publish_aggregated[NETDATA_FD_SYSCALL_OPEN].nerr = fd_hash_values[NETDATA_KEY_ERROR_DO_SYS_OPEN];
+
+ fd_publish_aggregated[NETDATA_FD_SYSCALL_CLOSE].ncall = fd_hash_values[NETDATA_KEY_CALLS_CLOSE_FD];
+ fd_publish_aggregated[NETDATA_FD_SYSCALL_CLOSE].nerr = fd_hash_values[NETDATA_KEY_ERROR_CLOSE_FD];
+
+ write_count_chart(NETDATA_FILE_OPEN_CLOSE_COUNT, NETDATA_FILESYSTEM_FAMILY, fd_publish_aggregated,
+ NETDATA_FD_SYSCALL_END);
+
+ if (em->mode < MODE_ENTRY) {
+ write_err_chart(NETDATA_FILE_OPEN_ERR_COUNT, NETDATA_FILESYSTEM_FAMILY,
+ fd_publish_aggregated, NETDATA_FD_SYSCALL_END);
+ }
+}
+
+/**
+ * Read global counter
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_global_table()
+{
+ uint32_t idx;
+ netdata_idx_t *val = fd_hash_values;
+ netdata_idx_t *stored = fd_values;
+ int fd = fd_maps[NETDATA_FD_GLOBAL_STATS].map_fd;
+
+ for (idx = NETDATA_KEY_CALLS_DO_SYS_OPEN; idx < NETDATA_FD_COUNTER; idx++) {
+ if (!bpf_map_lookup_elem(fd, &idx, stored)) {
+ int i;
+ int end = ebpf_nprocs;
+ netdata_idx_t total = 0;
+ for (i = 0; i < end; i++)
+ total += stored[i];
+
+ val[idx] = total;
+ }
+ }
+}
+
+/**
+ * File descriptor read hash
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_fd_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_fd_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ usec_t step = NETDATA_FD_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_global_table();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Apps Accumulator
+ *
+ * Sum all values read from kernel and store in the first address.
+ *
+ * @param out the vector with read values.
+ */
+static void fd_apps_accumulator(netdata_fd_stat_t *out)
+{
+ int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1;
+ netdata_fd_stat_t *total = &out[0];
+ for (i = 1; i < end; i++) {
+ netdata_fd_stat_t *w = &out[i];
+ total->open_call += w->open_call;
+ total->close_call += w->close_call;
+ total->open_err += w->open_err;
+ total->close_err += w->close_err;
+ }
+}
+
+/**
+ * Fill PID
+ *
+ * Fill PID structures
+ *
+ * @param current_pid pid that we are collecting data
+ * @param out values read from hash tables;
+ */
+static void fd_fill_pid(uint32_t current_pid, netdata_fd_stat_t *publish)
+{
+ netdata_fd_stat_t *curr = fd_pid[current_pid];
+ if (!curr) {
+ curr = callocz(1, sizeof(netdata_fd_stat_t));
+ fd_pid[current_pid] = curr;
+ }
+
+ memcpy(curr, &publish[0], sizeof(netdata_fd_stat_t));
+}
+
+/**
+ * Read APPS table
+ *
+ * Read the apps table and store data inside the structure.
+ */
+static void read_apps_table()
+{
+ netdata_fd_stat_t *fv = fd_vector;
+ uint32_t key;
+ struct pid_stat *pids = root_of_pids;
+ int fd = fd_maps[NETDATA_FD_PID_STATS].map_fd;
+ size_t length = sizeof(netdata_fd_stat_t) * ebpf_nprocs;
+ while (pids) {
+ key = pids->pid;
+
+ if (bpf_map_lookup_elem(fd, &key, fv)) {
+ pids = pids->next;
+ continue;
+ }
+
+ fd_apps_accumulator(fv);
+
+ fd_fill_pid(key, fv);
+
+ // We are cleaning to avoid passing data read from one process to other.
+ memset(fv, 0, length);
+
+ pids = pids->next;
+ }
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ */
+static void ebpf_update_fd_cgroup()
+{
+ ebpf_cgroup_target_t *ect ;
+ netdata_fd_stat_t *fv = fd_vector;
+ int fd = fd_maps[NETDATA_FD_PID_STATS].map_fd;
+ size_t length = sizeof(netdata_fd_stat_t) * ebpf_nprocs;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ int pid = pids->pid;
+ netdata_fd_stat_t *out = &pids->fd;
+ if (likely(fd_pid) && fd_pid[pid]) {
+ netdata_fd_stat_t *in = fd_pid[pid];
+
+ memcpy(out, in, sizeof(netdata_fd_stat_t));
+ } else {
+ memset(fv, 0, length);
+ if (!bpf_map_lookup_elem(fd, &pid, fv)) {
+ fd_apps_accumulator(fv);
+
+ memcpy(out, fv, sizeof(netdata_fd_stat_t));
+ }
+ }
+ }
+ }
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Sum PIDs
+ *
+ * Sum values for all targets.
+ *
+ * @param fd the output
+ * @param root list of pids
+ */
+static void ebpf_fd_sum_pids(netdata_fd_stat_t *fd, struct pid_on_target *root)
+{
+ uint32_t open_call = 0;
+ uint32_t close_call = 0;
+ uint32_t open_err = 0;
+ uint32_t close_err = 0;
+
+ while (root) {
+ int32_t pid = root->pid;
+ netdata_fd_stat_t *w = fd_pid[pid];
+ if (w) {
+ open_call += w->open_call;
+ close_call += w->close_call;
+ open_err += w->open_err;
+ close_err += w->close_err;
+ }
+
+ root = root->next;
+ }
+
+ // These conditions were added, because we are using incremental algorithm
+ fd->open_call = (open_call >= fd->open_call) ? open_call : fd->open_call;
+ fd->close_call = (close_call >= fd->close_call) ? close_call : fd->close_call;
+ fd->open_err = (open_err >= fd->open_err) ? open_err : fd->open_err;
+ fd->close_err = (close_err >= fd->close_err) ? close_err : fd->close_err;
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the structure with thread information
+ * @param root the target list.
+*/
+void ebpf_fd_send_apps_data(ebpf_module_t *em, struct target *root)
+{
+ struct target *w;
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ ebpf_fd_sum_pids(&w->fd, w->root_pid);
+ }
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_FILE_OPEN);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->fd.open_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_FILE_OPEN_ERROR);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->fd.open_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_FILE_CLOSED);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->fd.close_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_FILE_CLOSE_ERROR);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->fd.close_err);
+ }
+ }
+ write_end_chart();
+ }
+}
+
+/**
+ * Sum PIDs
+ *
+ * Sum values for all targets.
+ *
+ * @param fd structure used to store data
+ * @param pids input data
+ */
+static void ebpf_fd_sum_cgroup_pids(netdata_fd_stat_t *fd, struct pid_on_target2 *pids)
+{
+ netdata_fd_stat_t accumulator;
+ memset(&accumulator, 0, sizeof(accumulator));
+
+ while (pids) {
+ netdata_fd_stat_t *w = &pids->fd;
+
+ accumulator.open_err += w->open_err;
+ accumulator.open_call += w->open_call;
+ accumulator.close_call += w->close_call;
+ accumulator.close_err += w->close_err;
+
+ pids = pids->next;
+ }
+
+ fd->open_call = (accumulator.open_call >= fd->open_call) ? accumulator.open_call : fd->open_call;
+ fd->open_err = (accumulator.open_err >= fd->open_err) ? accumulator.open_err : fd->open_err;
+ fd->close_call = (accumulator.close_call >= fd->close_call) ? accumulator.close_call : fd->close_call;
+ fd->close_err = (accumulator.close_err >= fd->close_err) ? accumulator.close_err : fd->close_err;
+}
+
+/**
+ * Create specific file descriptor charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param em the main thread structure.
+ */
+static void ebpf_create_specific_fd_charts(char *type, ebpf_module_t *em)
+{
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_FILE_OPEN, "Number of open files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_CGROUP_GROUP,
+ NETDATA_CGROUP_FD_OPEN_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5400,
+ ebpf_create_global_dimension,
+ &fd_publish_aggregated[NETDATA_FD_SYSCALL_OPEN],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_FILE_OPEN_ERROR, "Fails to open files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_CGROUP_GROUP,
+ NETDATA_CGROUP_FD_OPEN_ERR_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5401,
+ ebpf_create_global_dimension,
+ &fd_publish_aggregated[NETDATA_FD_SYSCALL_OPEN],
+ 1, em->update_every,
+ NETDATA_EBPF_MODULE_NAME_SWAP);
+ }
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_FILE_CLOSED, "Files closed",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_CGROUP_GROUP,
+ NETDATA_CGROUP_FD_CLOSE_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5402,
+ ebpf_create_global_dimension,
+ &fd_publish_aggregated[NETDATA_FD_SYSCALL_CLOSE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_FILE_CLOSE_ERROR, "Fails to close files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_CGROUP_GROUP,
+ NETDATA_CGROUP_FD_CLOSE_ERR_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5403,
+ ebpf_create_global_dimension,
+ &fd_publish_aggregated[NETDATA_FD_SYSCALL_CLOSE],
+ 1, em->update_every,
+ NETDATA_EBPF_MODULE_NAME_SWAP);
+ }
+}
+
+/**
+ * Obsolete specific file descriptor charts
+ *
+ * Obsolete charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param em the main thread structure.
+ */
+static void ebpf_obsolete_specific_fd_charts(char *type, ebpf_module_t *em)
+{
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_FILE_OPEN, "Number of open files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_FD_OPEN_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5400, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_FILE_OPEN_ERROR, "Fails to open files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_FD_OPEN_ERR_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5401, em->update_every);
+ }
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_FILE_CLOSED, "Files closed",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_FD_CLOSE_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5402, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_FILE_CLOSE_ERROR, "Fails to close files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_FD_CLOSE_ERR_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5403, em->update_every);
+ }
+}
+
+/*
+ * Send specific file descriptor data
+ *
+ * Send data for specific cgroup/apps.
+ *
+ * @param type chart type
+ * @param values structure with values that will be sent to netdata
+ */
+static void ebpf_send_specific_fd_data(char *type, netdata_fd_stat_t *values, ebpf_module_t *em)
+{
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_FILE_OPEN);
+ write_chart_dimension(fd_publish_aggregated[NETDATA_FD_SYSCALL_OPEN].name, (long long)values->open_call);
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_FILE_OPEN_ERROR);
+ write_chart_dimension(fd_publish_aggregated[NETDATA_FD_SYSCALL_OPEN].name, (long long)values->open_err);
+ write_end_chart();
+ }
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_FILE_CLOSED);
+ write_chart_dimension(fd_publish_aggregated[NETDATA_FD_SYSCALL_CLOSE].name, (long long)values->close_call);
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_FILE_CLOSE_ERROR);
+ write_chart_dimension(fd_publish_aggregated[NETDATA_FD_SYSCALL_CLOSE].name, (long long)values->close_err);
+ write_end_chart();
+ }
+}
+
+/**
+ * Create systemd file descriptor charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param em the main collector structure
+ **/
+static void ebpf_create_systemd_fd_charts(ebpf_module_t *em)
+{
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_FILE_OPEN, "Number of open files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20061,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_FD_OPEN_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_FILE_OPEN_ERROR, "Fails to open files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20062,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_FD_OPEN_ERR_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+ }
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_FILE_CLOSED, "Files closed",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20063,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_FD_CLOSE_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_FILE_CLOSE_ERROR, "Fails to close files",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20064,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_FD_CLOSE_ERR_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+ }
+}
+
+/**
+ * Send Systemd charts
+ *
+ * Send collected data to Netdata.
+ *
+ * @param em the main collector structure
+ */
+static void ebpf_send_systemd_fd_charts(ebpf_module_t *em)
+{
+ ebpf_cgroup_target_t *ect;
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_FILE_OPEN);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_fd.open_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_FILE_OPEN_ERROR);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_fd.open_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_FILE_CLOSED);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_fd.close_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_FILE_CLOSE_ERROR);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_fd.close_err);
+ }
+ }
+ write_end_chart();
+ }
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the main collector structure
+*/
+static void ebpf_fd_send_cgroup_data(ebpf_module_t *em)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ ebpf_fd_sum_cgroup_pids(&ect->publish_systemd_fd, ect->pids);
+ }
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_fd_charts(em);
+ }
+
+ ebpf_send_systemd_fd_charts(em);
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_FD_CHART) && ect->updated) {
+ ebpf_create_specific_fd_charts(ect->name, em);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_FD_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_FD_CHART ) {
+ if (ect->updated) {
+ ebpf_send_specific_fd_data(ect->name, &ect->publish_systemd_fd, em);
+ } else {
+ ebpf_obsolete_specific_fd_charts(ect->name, em);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_FD_CHART;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+* Main loop for this collector.
+*/
+static void fd_collector(ebpf_module_t *em)
+{
+ fd_thread.thread = mallocz(sizeof(netdata_thread_t));
+ fd_thread.start_routine = ebpf_fd_read_hash;
+
+ netdata_thread_create(fd_thread.thread, fd_thread.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_fd_read_hash, em);
+
+ int cgroups = em->cgroup_charts;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = em->update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ netdata_apps_integration_flags_t apps = em->apps_charts;
+ pthread_mutex_lock(&collect_data_mutex);
+ if (apps)
+ read_apps_table();
+
+ if (cgroups)
+ ebpf_update_fd_cgroup();
+
+ pthread_mutex_lock(&lock);
+
+ ebpf_fd_send_data(em);
+
+ if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED)
+ ebpf_fd_send_apps_data(em, apps_groups_root_target);
+
+ if (cgroups)
+ ebpf_fd_send_cgroup_data(em);
+
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+}
+
+/*****************************************************************
+ *
+ * CREATE CHARTS
+ *
+ *****************************************************************/
+
+/**
+ * Create apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+void ebpf_fd_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_FILE_OPEN,
+ "Number of open files",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_FILE_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20061,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_FILE_OPEN_ERROR,
+ "Fails to open files",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_FILE_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20062,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+ }
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_FILE_CLOSED,
+ "Files closed",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_FILE_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20063,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_FILE_CLOSE_ERROR,
+ "Fails to close files",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_FILE_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20064,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+ }
+
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/**
+ * Create global charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+static void ebpf_create_fd_global_charts(ebpf_module_t *em)
+{
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_FILE_OPEN_CLOSE_COUNT,
+ "Open and close calls",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_FILE_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_EBPF_FD_CHARTS,
+ ebpf_create_global_dimension,
+ fd_publish_aggregated,
+ NETDATA_FD_SYSCALL_END,
+ em->update_every, NETDATA_EBPF_MODULE_NAME_FD);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_FILE_OPEN_ERR_COUNT,
+ "Open fails",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_FILE_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_EBPF_FD_CHARTS + 1,
+ ebpf_create_global_dimension,
+ fd_publish_aggregated,
+ NETDATA_FD_SYSCALL_END,
+ em->update_every, NETDATA_EBPF_MODULE_NAME_FD);
+ }
+}
+
+/*****************************************************************
+ *
+ * MAIN THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Allocate vectors used with this thread.
+ *
+ * We are not testing the return, because callocz does this and shutdown the software
+ * case it was not possible to allocate.
+ *
+ * @param apps is apps enabled?
+ */
+static void ebpf_fd_allocate_global_vectors(int apps)
+{
+ if (apps)
+ fd_pid = callocz((size_t)pid_max, sizeof(netdata_fd_stat_t *));
+
+ fd_vector = callocz((size_t)ebpf_nprocs, sizeof(netdata_fd_stat_t));
+
+ fd_values = callocz((size_t)ebpf_nprocs, sizeof(netdata_idx_t));
+}
+
+/*
+ * Load BPF
+ *
+ * Load BPF files.
+ *
+ * @param em the structure with configuration
+ */
+static int ebpf_fd_load_bpf(ebpf_module_t *em)
+{
+ int ret = 0;
+ ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_FD_SYSCALL_OPEN].mode);
+ if (em->load & EBPF_LOAD_LEGACY) {
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->enabled = CONFIG_BOOLEAN_NO;
+ ret = -1;
+ }
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ bpf_obj = fd_bpf__open();
+ if (!bpf_obj)
+ ret = -1;
+ else
+ ret = ebpf_fd_load_and_attach(bpf_obj, em);
+ }
+#endif
+
+ if (ret)
+ error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->thread_name);
+
+ return ret;
+}
+
+/**
+ * Directory Cache thread
+ *
+ * Thread used to make dcstat thread
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always returns NULL
+ */
+void *ebpf_fd_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_fd_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = fd_maps;
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_fd_load_bpf(em)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endfd;
+ }
+
+ ebpf_fd_allocate_global_vectors(em->apps_charts);
+
+ int algorithms[NETDATA_FD_SYSCALL_END] = {
+ NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX
+ };
+
+ ebpf_global_labels(fd_aggregated_data, fd_publish_aggregated, fd_dimension_names, fd_id_names,
+ algorithms, NETDATA_FD_SYSCALL_END);
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_fd_global_charts(em);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ fd_collector(em);
+
+endfd:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_fd.h b/collectors/ebpf.plugin/ebpf_fd.h
new file mode 100644
index 0000000..914a34b
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_fd.h
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_FD_H
+#define NETDATA_EBPF_FD_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_FD "filedescriptor"
+
+#define NETDATA_FD_SLEEP_MS 850000ULL
+
+// Menu group
+#define NETDATA_FILE_GROUP "file_access"
+
+// Global chart name
+#define NETDATA_FILE_OPEN_CLOSE_COUNT "file_descriptor"
+#define NETDATA_FILE_OPEN_ERR_COUNT "file_error"
+
+// Charts created on Apps submenu
+#define NETDATA_SYSCALL_APPS_FILE_OPEN "file_open"
+#define NETDATA_SYSCALL_APPS_FILE_CLOSED "file_closed"
+#define NETDATA_SYSCALL_APPS_FILE_OPEN_ERROR "file_open_error"
+#define NETDATA_SYSCALL_APPS_FILE_CLOSE_ERROR "file_close_error"
+
+// Process configuration name
+#define NETDATA_FD_CONFIG_FILE "fd.conf"
+
+// Contexts
+#define NETDATA_CGROUP_FD_OPEN_CONTEXT "cgroup.fd_open"
+#define NETDATA_CGROUP_FD_OPEN_ERR_CONTEXT "cgroup.fd_open_error"
+#define NETDATA_CGROUP_FD_CLOSE_CONTEXT "cgroup.fd_close"
+#define NETDATA_CGROUP_FD_CLOSE_ERR_CONTEXT "cgroup.fd_close_error"
+
+#define NETDATA_SYSTEMD_FD_OPEN_CONTEXT "services.fd_open"
+#define NETDATA_SYSTEMD_FD_OPEN_ERR_CONTEXT "services.fd_open_error"
+#define NETDATA_SYSTEMD_FD_CLOSE_CONTEXT "services.fd_close"
+#define NETDATA_SYSTEMD_FD_CLOSE_ERR_CONTEXT "services.fd_close_error"
+
+typedef struct netdata_fd_stat {
+ uint64_t pid_tgid; // Unique identifier
+ uint32_t pid; // Process ID
+
+ uint32_t open_call; // Open syscalls (open and openat)
+ uint32_t close_call; // Close syscall (close)
+
+ // Errors
+ uint32_t open_err;
+ uint32_t close_err;
+} netdata_fd_stat_t;
+
+enum fd_tables {
+ NETDATA_FD_PID_STATS,
+ NETDATA_FD_GLOBAL_STATS,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_FD_CONTROLLER
+};
+
+enum fd_counters {
+ NETDATA_KEY_CALLS_DO_SYS_OPEN,
+ NETDATA_KEY_ERROR_DO_SYS_OPEN,
+
+ NETDATA_KEY_CALLS_CLOSE_FD,
+ NETDATA_KEY_ERROR_CLOSE_FD,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_FD_COUNTER
+};
+
+enum fd_syscalls {
+ NETDATA_FD_SYSCALL_OPEN,
+ NETDATA_FD_SYSCALL_CLOSE,
+
+ // Do not insert nothing after this value
+ NETDATA_FD_SYSCALL_END
+};
+
+
+void *ebpf_fd_thread(void *ptr);
+void ebpf_fd_create_apps_charts(struct ebpf_module *em, void *ptr);
+extern struct config fd_config;
+extern netdata_fd_stat_t **fd_pid;
+extern netdata_ebpf_targets_t fd_targets[];
+
+#endif /* NETDATA_EBPF_FD_H */
+
diff --git a/collectors/ebpf.plugin/ebpf_filesystem.c b/collectors/ebpf.plugin/ebpf_filesystem.c
new file mode 100644
index 0000000..7dbec74
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_filesystem.c
@@ -0,0 +1,638 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf_filesystem.h"
+
+struct config fs_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+static ebpf_local_maps_t fs_maps[] = {{.name = "tbl_ext4", .internal_input = NETDATA_KEY_CALLS_SYNC,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_xfs", .internal_input = NETDATA_KEY_CALLS_SYNC,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_nfs", .internal_input = NETDATA_KEY_CALLS_SYNC,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_zfs", .internal_input = NETDATA_KEY_CALLS_SYNC,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_btrfs", .internal_input = NETDATA_KEY_CALLS_SYNC,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_ext_addr", .internal_input = 1,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
+
+struct netdata_static_thread filesystem_threads = {
+ .name = "EBPF FS READ",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+static netdata_syscall_stat_t filesystem_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS];
+static netdata_publish_syscall_t filesystem_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS];
+
+char **dimensions = NULL;
+static netdata_idx_t *filesystem_hash_values = NULL;
+
+/*****************************************************************
+ *
+ * COMMON FUNCTIONS
+ *
+ *****************************************************************/
+
+/**
+ * Create Filesystem chart
+ *
+ * Create latency charts
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_obsolete_fs_charts(int update_every)
+{
+ int i;
+ uint32_t test = NETDATA_FILESYSTEM_FLAG_CHART_CREATED | NETDATA_FILESYSTEM_REMOVE_CHARTS;
+ for (i = 0; localfs[i].filesystem; i++) {
+ ebpf_filesystem_partitions_t *efp = &localfs[i];
+ uint32_t flags = efp->flags;
+ if ((flags & test) == test) {
+ flags &= ~NETDATA_FILESYSTEM_FLAG_CHART_CREATED;
+
+ ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY, efp->hread.name,
+ efp->hread.title,
+ EBPF_COMMON_DIMENSION_CALL, efp->family_name,
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, efp->hread.order, update_every);
+
+ ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY, efp->hwrite.name,
+ efp->hwrite.title,
+ EBPF_COMMON_DIMENSION_CALL, efp->family_name,
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, efp->hwrite.order, update_every);
+
+ ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY, efp->hopen.name, efp->hopen.title,
+ EBPF_COMMON_DIMENSION_CALL, efp->family_name,
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, efp->hopen.order, update_every);
+
+ ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY, efp->hadditional.name, efp->hadditional.title,
+ EBPF_COMMON_DIMENSION_CALL, efp->family_name,
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, efp->hadditional.order,
+ update_every);
+ }
+ efp->flags = flags;
+ }
+}
+
+/**
+ * Create Filesystem chart
+ *
+ * Create latency charts
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_fs_charts(int update_every)
+{
+ static int order = NETDATA_CHART_PRIO_EBPF_FILESYSTEM_CHARTS;
+ char chart_name[64], title[256], family[64];
+ int i;
+ uint32_t test = NETDATA_FILESYSTEM_FLAG_CHART_CREATED|NETDATA_FILESYSTEM_REMOVE_CHARTS;
+ for (i = 0; localfs[i].filesystem; i++) {
+ ebpf_filesystem_partitions_t *efp = &localfs[i];
+ uint32_t flags = efp->flags;
+ if (flags & NETDATA_FILESYSTEM_FLAG_HAS_PARTITION && !(flags & test)) {
+ snprintfz(title, 255, "%s latency for each read request.", efp->filesystem);
+ snprintfz(family, 63, "%s_latency", efp->family);
+ snprintfz(chart_name, 63, "%s_read_latency", efp->filesystem);
+ efp->hread.name = strdupz(chart_name);
+ efp->hread.title = strdupz(title);
+ efp->hread.order = order;
+ efp->family_name = strdupz(family);
+
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY, efp->hread.name,
+ title,
+ EBPF_COMMON_DIMENSION_CALL, family,
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
+ filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS,
+ update_every, NETDATA_EBPF_MODULE_NAME_FILESYSTEM);
+ order++;
+
+ snprintfz(title, 255, "%s latency for each write request.", efp->filesystem);
+ snprintfz(chart_name, 63, "%s_write_latency", efp->filesystem);
+ efp->hwrite.name = strdupz(chart_name);
+ efp->hwrite.title = strdupz(title);
+ efp->hwrite.order = order;
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY, efp->hwrite.name,
+ title,
+ EBPF_COMMON_DIMENSION_CALL, family,
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
+ filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS,
+ update_every, NETDATA_EBPF_MODULE_NAME_FILESYSTEM);
+ order++;
+
+ snprintfz(title, 255, "%s latency for each open request.", efp->filesystem);
+ snprintfz(chart_name, 63, "%s_open_latency", efp->filesystem);
+ efp->hopen.name = strdupz(chart_name);
+ efp->hopen.title = strdupz(title);
+ efp->hopen.order = order;
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY, efp->hopen.name,
+ title,
+ EBPF_COMMON_DIMENSION_CALL, family,
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
+ filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS,
+ update_every, NETDATA_EBPF_MODULE_NAME_FILESYSTEM);
+ order++;
+
+ char *type = (efp->flags & NETDATA_FILESYSTEM_ATTR_CHARTS) ? "attribute" : "sync";
+ snprintfz(title, 255, "%s latency for each %s request.", efp->filesystem, type);
+ snprintfz(chart_name, 63, "%s_%s_latency", efp->filesystem, type);
+ efp->hadditional.name = strdupz(chart_name);
+ efp->hadditional.title = strdupz(title);
+ efp->hadditional.order = order;
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY, efp->hadditional.name, title,
+ EBPF_COMMON_DIMENSION_CALL, family,
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
+ filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS,
+ update_every, NETDATA_EBPF_MODULE_NAME_FILESYSTEM);
+ order++;
+ efp->flags |= NETDATA_FILESYSTEM_FLAG_CHART_CREATED;
+ }
+ }
+}
+
+/**
+ * Initialize eBPF data
+ *
+ * @param em main thread structure.
+ *
+ * @return it returns 0 on success and -1 otherwise.
+ */
+int ebpf_filesystem_initialize_ebpf_data(ebpf_module_t *em)
+{
+ int i;
+ const char *saved_name = em->thread_name;
+ uint64_t kernels = em->kernels;
+ for (i = 0; localfs[i].filesystem; i++) {
+ ebpf_filesystem_partitions_t *efp = &localfs[i];
+ if (!efp->probe_links && efp->flags & NETDATA_FILESYSTEM_LOAD_EBPF_PROGRAM) {
+ em->thread_name = efp->filesystem;
+ em->kernels = efp->kernels;
+ efp->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &efp->objects);
+ if (!efp->probe_links) {
+ em->thread_name = saved_name;
+ em->kernels = kernels;
+ return -1;
+ }
+ efp->flags |= NETDATA_FILESYSTEM_FLAG_HAS_PARTITION;
+
+ // Nedeed for filesystems like btrfs
+ if ((efp->flags & NETDATA_FILESYSTEM_FILL_ADDRESS_TABLE) && (efp->addresses.function)) {
+ ebpf_load_addresses(&efp->addresses, fs_maps[i + 1].map_fd);
+ }
+ }
+ efp->flags &= ~NETDATA_FILESYSTEM_LOAD_EBPF_PROGRAM;
+ }
+ em->thread_name = saved_name;
+ em->kernels = kernels;
+
+ if (!dimensions) {
+ dimensions = ebpf_fill_histogram_dimension(NETDATA_EBPF_HIST_MAX_BINS);
+
+ memset(filesystem_aggregated_data, 0 , NETDATA_EBPF_HIST_MAX_BINS * sizeof(netdata_syscall_stat_t));
+ memset(filesystem_publish_aggregated, 0 , NETDATA_EBPF_HIST_MAX_BINS * sizeof(netdata_publish_syscall_t));
+
+ filesystem_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t));
+ }
+
+ return 0;
+}
+
+/**
+ * Read Local partitions
+ *
+ * @return the total of partitions that will be monitored
+ */
+static int ebpf_read_local_partitions()
+{
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/self/mountinfo", netdata_configured_host_prefix);
+ procfile *ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) {
+ snprintfz(filename, FILENAME_MAX, "%s/proc/1/mountinfo", netdata_configured_host_prefix);
+ ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 0;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0;
+
+ int count = 0;
+ unsigned long l, i, lines = procfile_lines(ff);
+ for (i = 0; localfs[i].filesystem; i++) {
+ localfs[i].flags |= NETDATA_FILESYSTEM_REMOVE_CHARTS;
+ }
+
+ for(l = 0; l < lines ; l++) {
+ // In "normal" situation the expected value is at column 7
+ // When `shared` options is added to mount information, the filesystem is at column 8
+ // Finally when we have systemd starting netdata, it will be at column 9
+ unsigned long index = procfile_linewords(ff, l) - 3;
+
+ char *fs = procfile_lineword(ff, l, index);
+
+ for (i = 0; localfs[i].filesystem; i++) {
+ ebpf_filesystem_partitions_t *w = &localfs[i];
+ if (w->enabled && (!strcmp(fs, w->filesystem) ||
+ (w->optional_filesystem && !strcmp(fs, w->optional_filesystem)))) {
+ localfs[i].flags |= NETDATA_FILESYSTEM_LOAD_EBPF_PROGRAM;
+ localfs[i].flags &= ~NETDATA_FILESYSTEM_REMOVE_CHARTS;
+ count++;
+ break;
+ }
+ }
+ }
+ procfile_close(ff);
+
+ return count;
+}
+
+/**
+ * Update partition
+ *
+ * Update the partition structures before to plot
+ *
+ * @param em main thread structure
+ *
+ * @return 0 on success and -1 otherwise.
+ */
+static int ebpf_update_partitions(ebpf_module_t *em)
+{
+ static time_t update_every = 0;
+ time_t curr = now_realtime_sec();
+ if (curr < update_every)
+ return 0;
+
+ update_every = curr + 5 * em->update_every;
+ if (!ebpf_read_local_partitions()) {
+ em->optional = -1;
+ return -1;
+ }
+
+ if (ebpf_filesystem_initialize_ebpf_data(em)) {
+ return -1;
+ }
+
+ return 0;
+}
+
+/*****************************************************************
+ *
+ * CLEANUP FUNCTIONS
+ *
+ *****************************************************************/
+
+/*
+ * Cleanup eBPF data
+ */
+void ebpf_filesystem_cleanup_ebpf_data()
+{
+ int i;
+ for (i = 0; localfs[i].filesystem; i++) {
+ ebpf_filesystem_partitions_t *efp = &localfs[i];
+ if (efp->probe_links) {
+ freez(efp->family_name);
+
+ freez(efp->hread.name);
+ freez(efp->hread.title);
+
+ freez(efp->hwrite.name);
+ freez(efp->hwrite.title);
+
+ freez(efp->hopen.name);
+ freez(efp->hopen.title);
+
+ freez(efp->hadditional.name);
+ freez(efp->hadditional.title);
+ }
+ }
+}
+
+/**
+ * Filesystem Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_filesystem_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ freez(filesystem_threads.thread);
+ ebpf_cleanup_publish_syscall(filesystem_publish_aggregated);
+
+ ebpf_filesystem_cleanup_ebpf_data();
+ if (dimensions)
+ ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+ freez(filesystem_hash_values);
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Filesystem exit
+ *
+ * Cancel child thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_filesystem_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*filesystem_threads.thread);
+ ebpf_filesystem_free(em);
+}
+
+/**
+ * File system cleanup
+ *
+ * Clean up allocated thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_filesystem_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_filesystem_free(em);
+}
+
+/*****************************************************************
+ *
+ * MAIN THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Select hist
+ *
+ * Select a histogram to store data.
+ *
+ * @param efp pointer for the structure with pointers.
+ * @param id histogram selector
+ *
+ * @return It returns a pointer for the histogram
+ */
+static inline netdata_ebpf_histogram_t *select_hist(ebpf_filesystem_partitions_t *efp, uint32_t *idx, uint32_t id)
+{
+ if (id < NETDATA_KEY_CALLS_READ) {
+ *idx = id;
+ return &efp->hread;
+ } else if (id < NETDATA_KEY_CALLS_WRITE) {
+ *idx = id - NETDATA_KEY_CALLS_READ;
+ return &efp->hwrite;
+ } else if (id < NETDATA_KEY_CALLS_OPEN) {
+ *idx = id - NETDATA_KEY_CALLS_WRITE;
+ return &efp->hopen;
+ } else if (id < NETDATA_KEY_CALLS_SYNC ){
+ *idx = id - NETDATA_KEY_CALLS_OPEN;
+ return &efp->hadditional;
+ }
+
+ return NULL;
+}
+
+/**
+ * Read hard disk table
+ *
+ * @param table index for the hash table
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_filesystem_table(ebpf_filesystem_partitions_t *efp, int fd)
+{
+ netdata_idx_t *values = filesystem_hash_values;
+ uint32_t key;
+ uint32_t idx;
+ for (key = 0; key < NETDATA_KEY_CALLS_SYNC; key++) {
+ netdata_ebpf_histogram_t *w = select_hist(efp, &idx, key);
+ if (!w) {
+ continue;
+ }
+
+ int test = bpf_map_lookup_elem(fd, &key, values);
+ if (test < 0) {
+ continue;
+ }
+
+ uint64_t total = 0;
+ int i;
+ int end = ebpf_nprocs;
+ for (i = 0; i < end; i++) {
+ total += values[i];
+ }
+
+ if (idx >= NETDATA_EBPF_HIST_MAX_BINS)
+ idx = NETDATA_EBPF_HIST_MAX_BINS - 1;
+ w->histogram[idx] = total;
+ }
+}
+
+/**
+ * Read hard disk table
+ *
+ * @param table index for the hash table
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_filesystem_tables()
+{
+ int i;
+ for (i = 0; localfs[i].filesystem; i++) {
+ ebpf_filesystem_partitions_t *efp = &localfs[i];
+ if (efp->flags & NETDATA_FILESYSTEM_FLAG_HAS_PARTITION) {
+ read_filesystem_table(efp, fs_maps[i].map_fd);
+ }
+ }
+}
+
+/**
+ * Socket read hash
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data on very busy socket.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_filesystem_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_filesystem_cleanup, ptr);
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = NETDATA_FILESYSTEM_READ_SLEEP_MS * em->update_every;
+ int update_every = em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ (void) ebpf_update_partitions(em);
+ ebpf_obsolete_fs_charts(update_every);
+
+ // No more partitions, it is not necessary to read tables
+ if (em->optional)
+ continue;
+
+ read_filesystem_tables();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Send Hard disk data
+ *
+ * Send hard disk information to Netdata.
+ */
+static void ebpf_histogram_send_data()
+{
+ uint32_t i;
+ uint32_t test = NETDATA_FILESYSTEM_FLAG_HAS_PARTITION | NETDATA_FILESYSTEM_REMOVE_CHARTS;
+ for (i = 0; localfs[i].filesystem; i++) {
+ ebpf_filesystem_partitions_t *efp = &localfs[i];
+ if ((efp->flags & test) == NETDATA_FILESYSTEM_FLAG_HAS_PARTITION) {
+ write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hread.name,
+ efp->hread.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+
+ write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hwrite.name,
+ efp->hwrite.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+
+ write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hopen.name,
+ efp->hopen.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+
+ write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hadditional.name,
+ efp->hadditional.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+ }
+ }
+}
+
+/**
+ * Main loop for this collector.
+ *
+ * @param em main structure for this thread
+ */
+static void filesystem_collector(ebpf_module_t *em)
+{
+ filesystem_threads.thread = mallocz(sizeof(netdata_thread_t));
+ filesystem_threads.start_routine = ebpf_filesystem_read_hash;
+
+ netdata_thread_create(filesystem_threads.thread, filesystem_threads.name,
+ NETDATA_THREAD_OPTION_DEFAULT, ebpf_filesystem_read_hash, em);
+
+ int update_every = em->update_every;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ pthread_mutex_lock(&lock);
+
+ ebpf_create_fs_charts(update_every);
+ ebpf_histogram_send_data();
+
+ pthread_mutex_unlock(&lock);
+ }
+}
+
+/*****************************************************************
+ *
+ * ENTRY THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Update Filesystem
+ *
+ * Update file system structure using values read from configuration file.
+ */
+static void ebpf_update_filesystem()
+{
+ char dist[NETDATA_FS_MAX_DIST_NAME + 1];
+ int i;
+ for (i = 0; localfs[i].filesystem; i++) {
+ snprintfz(dist, NETDATA_FS_MAX_DIST_NAME, "%sdist", localfs[i].filesystem);
+
+ localfs[i].enabled = appconfig_get_boolean(&fs_config, NETDATA_FILESYSTEM_CONFIG_NAME, dist,
+ CONFIG_BOOLEAN_YES);
+ }
+}
+
+/**
+ * Filesystem thread
+ *
+ * Thread used to generate socket charts.
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always return NULL
+ */
+void *ebpf_filesystem_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_filesystem_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = fs_maps;
+ ebpf_update_filesystem();
+
+ // Initialize optional as zero, to identify when there are not partitions to monitor
+ em->optional = 0;
+
+ if (ebpf_update_partitions(em)) {
+ if (em->optional)
+ info("Netdata cannot monitor the filesystems used on this host.");
+
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endfilesystem;
+ }
+
+ int algorithms[NETDATA_EBPF_HIST_MAX_BINS];
+ ebpf_fill_algorithms(algorithms, NETDATA_EBPF_HIST_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX);
+ ebpf_global_labels(filesystem_aggregated_data, filesystem_publish_aggregated, dimensions, dimensions,
+ algorithms, NETDATA_EBPF_HIST_MAX_BINS);
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_fs_charts(em->update_every);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ filesystem_collector(em);
+
+endfilesystem:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_filesystem.h b/collectors/ebpf.plugin/ebpf_filesystem.h
new file mode 100644
index 0000000..0d558df
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_filesystem.h
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_FILESYSTEM_H
+#define NETDATA_EBPF_FILESYSTEM_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_FILESYSTEM "filesystem"
+
+#include "ebpf.h"
+
+#define NETDATA_FS_MAX_DIST_NAME 64UL
+
+#define NETDATA_FILESYSTEM_CONFIG_NAME "filesystem"
+#define NETDATA_FILESYSTEM_READ_SLEEP_MS 600000ULL
+
+// Process configuration name
+#define NETDATA_FILESYSTEM_CONFIG_FILE "filesystem.conf"
+
+typedef struct netdata_fs_hist {
+ uint32_t hist_id;
+ uint32_t bin;
+} netdata_fs_hist_t;
+
+enum filesystem_limit {
+ NETDATA_KEY_CALLS_READ = 24,
+ NETDATA_KEY_CALLS_WRITE = 48,
+ NETDATA_KEY_CALLS_OPEN = 72,
+ NETDATA_KEY_CALLS_SYNC = 96
+};
+
+enum netdata_filesystem_flags {
+ NETDATA_FILESYSTEM_FLAG_NO_PARTITION = 0,
+ NETDATA_FILESYSTEM_LOAD_EBPF_PROGRAM = 1,
+ NETDATA_FILESYSTEM_FLAG_HAS_PARTITION = 2,
+ NETDATA_FILESYSTEM_FLAG_CHART_CREATED = 4,
+ NETDATA_FILESYSTEM_FILL_ADDRESS_TABLE = 8,
+ NETDATA_FILESYSTEM_REMOVE_CHARTS = 16,
+ NETDATA_FILESYSTEM_ATTR_CHARTS = 32
+};
+
+enum netdata_filesystem_table {
+ NETDATA_MAIN_FS_TABLE,
+ NETDATA_ADDR_FS_TABLE
+};
+
+void *ebpf_filesystem_thread(void *ptr);
+extern struct config fs_config;
+
+#endif /* NETDATA_EBPF_FILESYSTEM_H */
diff --git a/collectors/ebpf.plugin/ebpf_hardirq.c b/collectors/ebpf.plugin/ebpf_hardirq.c
new file mode 100644
index 0000000..b07dd24
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_hardirq.c
@@ -0,0 +1,507 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_hardirq.h"
+
+struct config hardirq_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+#define HARDIRQ_MAP_LATENCY 0
+#define HARDIRQ_MAP_LATENCY_STATIC 1
+static ebpf_local_maps_t hardirq_maps[] = {
+ {
+ .name = "tbl_hardirq",
+ .internal_input = NETDATA_HARDIRQ_MAX_IRQS,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ },
+ {
+ .name = "tbl_hardirq_static",
+ .internal_input = HARDIRQ_EBPF_STATIC_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ },
+ /* end */
+ {
+ .name = NULL,
+ .internal_input = 0,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ }
+};
+
+#define HARDIRQ_TP_CLASS_IRQ "irq"
+#define HARDIRQ_TP_CLASS_IRQ_VECTORS "irq_vectors"
+static ebpf_tracepoint_t hardirq_tracepoints[] = {
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ, .event = "irq_handler_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ, .event = "irq_handler_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "thermal_apic_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "thermal_apic_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "threshold_apic_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "threshold_apic_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "error_apic_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "error_apic_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "deferred_error_apic_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "deferred_error_apic_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "spurious_apic_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "spurious_apic_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "call_function_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "call_function_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "call_function_single_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "call_function_single_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "reschedule_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "reschedule_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "local_timer_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "local_timer_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "irq_work_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "irq_work_exit"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "x86_platform_ipi_entry"},
+ {.enabled = false, .class = HARDIRQ_TP_CLASS_IRQ_VECTORS, .event = "x86_platform_ipi_exit"},
+ /* end */
+ {.enabled = false, .class = NULL, .event = NULL}
+};
+
+static hardirq_static_val_t hardirq_static_vals[] = {
+ {
+ .idx = HARDIRQ_EBPF_STATIC_APIC_THERMAL,
+ .name = "apic_thermal",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_APIC_THRESHOLD,
+ .name = "apic_threshold",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_APIC_ERROR,
+ .name = "apic_error",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_APIC_DEFERRED_ERROR,
+ .name = "apic_deferred_error",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_APIC_SPURIOUS,
+ .name = "apic_spurious",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_FUNC_CALL,
+ .name = "func_call",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_FUNC_CALL_SINGLE,
+ .name = "func_call_single",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_RESCHEDULE,
+ .name = "reschedule",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_LOCAL_TIMER,
+ .name = "local_timer",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_IRQ_WORK,
+ .name = "irq_work",
+ .latency = 0
+ },
+ {
+ .idx = HARDIRQ_EBPF_STATIC_X86_PLATFORM_IPI,
+ .name = "x86_platform_ipi",
+ .latency = 0
+ },
+};
+
+// store for "published" data from the reader thread, which the collector
+// thread will write to netdata agent.
+static avl_tree_lock hardirq_pub;
+
+// tmp store for dynamic hard IRQ values we get from a per-CPU eBPF map.
+static hardirq_ebpf_val_t *hardirq_ebpf_vals = NULL;
+
+// tmp store for static hard IRQ values we get from a per-CPU eBPF map.
+static hardirq_ebpf_static_val_t *hardirq_ebpf_static_vals = NULL;
+
+static struct netdata_static_thread hardirq_threads = {
+ .name = "HARDIRQ KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+/**
+ * Hardirq Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_hardirq_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ freez(hardirq_threads.thread);
+ for (int i = 0; hardirq_tracepoints[i].class != NULL; i++) {
+ ebpf_disable_tracepoint(&hardirq_tracepoints[i]);
+ }
+ freez(hardirq_ebpf_vals);
+ freez(hardirq_ebpf_static_vals);
+
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+}
+
+/**
+ * Hardirq Exit
+ *
+ * Cancel child and exit.
+ *
+ * @param ptr thread data.
+ */
+static void hardirq_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*hardirq_threads.thread);
+ ebpf_hardirq_free(em);
+}
+
+/**
+ * Hardirq clean up
+ *
+ * Clean up allocated memory.
+ *
+ * @param ptr thread data.
+ */
+static void hardirq_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_hardirq_free(em);
+}
+
+/*****************************************************************
+ * MAIN LOOP
+ *****************************************************************/
+
+/**
+ * Compare hard IRQ values.
+ *
+ * @param a `hardirq_val_t *`.
+ * @param b `hardirq_val_t *`.
+ *
+ * @return 0 if a==b, 1 if a>b, -1 if a<b.
+*/
+static int hardirq_val_cmp(void *a, void *b)
+{
+ hardirq_val_t *ptr1 = a;
+ hardirq_val_t *ptr2 = b;
+
+ if (ptr1->irq > ptr2->irq) {
+ return 1;
+ }
+ else if (ptr1->irq < ptr2->irq) {
+ return -1;
+ }
+ else {
+ return 0;
+ }
+}
+
+static void hardirq_read_latency_map(int mapfd)
+{
+ hardirq_ebpf_key_t key = {};
+ hardirq_ebpf_key_t next_key = {};
+ hardirq_val_t search_v = {};
+ hardirq_val_t *v = NULL;
+
+ while (bpf_map_get_next_key(mapfd, &key, &next_key) == 0) {
+ // get val for this key.
+ int test = bpf_map_lookup_elem(mapfd, &key, hardirq_ebpf_vals);
+ if (unlikely(test < 0)) {
+ key = next_key;
+ continue;
+ }
+
+ // is this IRQ saved yet?
+ //
+ // if not, make a new one, mark it as unsaved for now, and continue; we
+ // will insert it at the end after all of its values are correctly set,
+ // so that we can safely publish it to the collector within a single,
+ // short locked operation.
+ //
+ // otherwise simply continue; we will only update the latency, which
+ // can be republished safely without a lock.
+ //
+ // NOTE: lock isn't strictly necessary for this initial search, as only
+ // this thread does writing, but the AVL is using a read-write lock so
+ // there is no congestion.
+ bool v_is_new = false;
+ search_v.irq = key.irq;
+ v = (hardirq_val_t *)avl_search_lock(&hardirq_pub, (avl_t *)&search_v);
+ if (unlikely(v == NULL)) {
+ // latency/name can only be added reliably at a later time.
+ // when they're added, only then will we AVL insert.
+ v = callocz(1, sizeof(hardirq_val_t));
+ v->irq = key.irq;
+ v->dim_exists = false;
+
+ v_is_new = true;
+ }
+
+ // note two things:
+ // 1. we must add up latency value for this IRQ across all CPUs.
+ // 2. the name is unfortunately *not* available on all CPU maps - only
+ // a single map contains the name, so we must find it. we only need
+ // to copy it though if the IRQ is new for us.
+ bool name_saved = false;
+ uint64_t total_latency = 0;
+ int i;
+ int end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs;
+ for (i = 0; i < end; i++) {
+ total_latency += hardirq_ebpf_vals[i].latency/1000;
+
+ // copy name for new IRQs.
+ if (v_is_new && !name_saved && hardirq_ebpf_vals[i].name[0] != '\0') {
+ strncpyz(
+ v->name,
+ hardirq_ebpf_vals[i].name,
+ NETDATA_HARDIRQ_NAME_LEN
+ );
+ name_saved = true;
+ }
+ }
+
+ // can now safely publish latency for existing IRQs.
+ v->latency = total_latency;
+
+ // can now safely publish new IRQ.
+ if (v_is_new) {
+ avl_t *check = avl_insert_lock(&hardirq_pub, (avl_t *)v);
+ if (check != (avl_t *)v) {
+ error("Internal error, cannot insert the AVL tree.");
+ }
+ }
+
+ key = next_key;
+ }
+}
+
+static void hardirq_read_latency_static_map(int mapfd)
+{
+ uint32_t i;
+ for (i = 0; i < HARDIRQ_EBPF_STATIC_END; i++) {
+ uint32_t map_i = hardirq_static_vals[i].idx;
+ int test = bpf_map_lookup_elem(mapfd, &map_i, hardirq_ebpf_static_vals);
+ if (unlikely(test < 0)) {
+ continue;
+ }
+
+ uint64_t total_latency = 0;
+ int cpu_i;
+ int end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs;
+ for (cpu_i = 0; cpu_i < end; cpu_i++) {
+ total_latency += hardirq_ebpf_static_vals[cpu_i].latency/1000;
+ }
+
+ hardirq_static_vals[i].latency = total_latency;
+ }
+}
+
+/**
+ * Read eBPF maps for hard IRQ.
+ */
+static void *hardirq_reader(void *ptr)
+{
+ netdata_thread_cleanup_push(hardirq_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ usec_t step = NETDATA_HARDIRQ_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ hardirq_read_latency_map(hardirq_maps[HARDIRQ_MAP_LATENCY].map_fd);
+ hardirq_read_latency_static_map(hardirq_maps[HARDIRQ_MAP_LATENCY_STATIC].map_fd);
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+static void hardirq_create_charts(int update_every)
+{
+ ebpf_create_chart(
+ NETDATA_EBPF_SYSTEM_GROUP,
+ "hardirq_latency",
+ "Hardware IRQ latency",
+ EBPF_COMMON_DIMENSION_MILLISECONDS,
+ "interrupts",
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ NETDATA_CHART_PRIO_HARDIRQ_LATENCY,
+ NULL, NULL, 0, update_every,
+ NETDATA_EBPF_MODULE_NAME_HARDIRQ
+ );
+
+ fflush(stdout);
+}
+
+static void hardirq_create_static_dims()
+{
+ uint32_t i;
+ for (i = 0; i < HARDIRQ_EBPF_STATIC_END; i++) {
+ ebpf_write_global_dimension(
+ hardirq_static_vals[i].name, hardirq_static_vals[i].name,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX]
+ );
+ }
+}
+
+// callback for avl tree traversal on `hardirq_pub`.
+static int hardirq_write_dims(void *entry, void *data)
+{
+ UNUSED(data);
+
+ hardirq_val_t *v = entry;
+
+ // IRQs get dynamically added in, so add the dimension if we haven't yet.
+ if (!v->dim_exists) {
+ ebpf_write_global_dimension(
+ v->name, v->name,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX]
+ );
+ v->dim_exists = true;
+ }
+
+ write_chart_dimension(v->name, v->latency);
+
+ return 1;
+}
+
+static inline void hardirq_write_static_dims()
+{
+ uint32_t i;
+ for (i = 0; i < HARDIRQ_EBPF_STATIC_END; i++) {
+ write_chart_dimension(
+ hardirq_static_vals[i].name,
+ hardirq_static_vals[i].latency
+ );
+ }
+}
+
+/**
+* Main loop for this collector.
+*/
+static void hardirq_collector(ebpf_module_t *em)
+{
+ hardirq_ebpf_vals = callocz(
+ (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs,
+ sizeof(hardirq_ebpf_val_t)
+ );
+ hardirq_ebpf_static_vals = callocz(
+ (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs,
+ sizeof(hardirq_ebpf_static_val_t)
+ );
+
+ avl_init_lock(&hardirq_pub, hardirq_val_cmp);
+
+ // create reader thread.
+ hardirq_threads.thread = mallocz(sizeof(netdata_thread_t));
+ hardirq_threads.start_routine = hardirq_reader;
+ netdata_thread_create(
+ hardirq_threads.thread,
+ hardirq_threads.name,
+ NETDATA_THREAD_OPTION_DEFAULT,
+ hardirq_reader,
+ em
+ );
+
+ // create chart and static dims.
+ pthread_mutex_lock(&lock);
+ hardirq_create_charts(em->update_every);
+ hardirq_create_static_dims();
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ // loop and read from published data until ebpf plugin is closed.
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = em->update_every * USEC_PER_SEC;
+ //This will be cancelled by its parent
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ pthread_mutex_lock(&lock);
+
+ // write dims now for all hitherto discovered IRQs.
+ write_begin_chart(NETDATA_EBPF_SYSTEM_GROUP, "hardirq_latency");
+ avl_traverse_lock(&hardirq_pub, hardirq_write_dims, NULL);
+ hardirq_write_static_dims();
+ write_end_chart();
+
+ pthread_mutex_unlock(&lock);
+ }
+}
+
+/*****************************************************************
+ * EBPF HARDIRQ THREAD
+ *****************************************************************/
+
+/**
+ * Hard IRQ latency thread.
+ *
+ * @param ptr a `ebpf_module_t *`.
+ * @return always NULL.
+ */
+void *ebpf_hardirq_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(hardirq_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = hardirq_maps;
+
+ if (ebpf_enable_tracepoints(hardirq_tracepoints) == 0) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endhardirq;
+ }
+
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endhardirq;
+ }
+
+ hardirq_collector(em);
+
+endhardirq:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_hardirq.h b/collectors/ebpf.plugin/ebpf_hardirq.h
new file mode 100644
index 0000000..381da57
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_hardirq.h
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_HARDIRQ_H
+#define NETDATA_EBPF_HARDIRQ_H 1
+
+/*****************************************************************
+ * copied from kernel-collectors repo, with modifications needed
+ * for inclusion here.
+ *****************************************************************/
+
+#define NETDATA_HARDIRQ_NAME_LEN 32
+#define NETDATA_HARDIRQ_MAX_IRQS 1024L
+
+typedef struct hardirq_ebpf_key {
+ int irq;
+} hardirq_ebpf_key_t;
+
+typedef struct hardirq_ebpf_val {
+ uint64_t latency;
+ uint64_t ts;
+ char name[NETDATA_HARDIRQ_NAME_LEN];
+} hardirq_ebpf_val_t;
+
+enum hardirq_ebpf_static {
+ HARDIRQ_EBPF_STATIC_APIC_THERMAL,
+ HARDIRQ_EBPF_STATIC_APIC_THRESHOLD,
+ HARDIRQ_EBPF_STATIC_APIC_ERROR,
+ HARDIRQ_EBPF_STATIC_APIC_DEFERRED_ERROR,
+ HARDIRQ_EBPF_STATIC_APIC_SPURIOUS,
+ HARDIRQ_EBPF_STATIC_FUNC_CALL,
+ HARDIRQ_EBPF_STATIC_FUNC_CALL_SINGLE,
+ HARDIRQ_EBPF_STATIC_RESCHEDULE,
+ HARDIRQ_EBPF_STATIC_LOCAL_TIMER,
+ HARDIRQ_EBPF_STATIC_IRQ_WORK,
+ HARDIRQ_EBPF_STATIC_X86_PLATFORM_IPI,
+
+ HARDIRQ_EBPF_STATIC_END
+};
+
+typedef struct hardirq_ebpf_static_val {
+ uint64_t latency;
+ uint64_t ts;
+} hardirq_ebpf_static_val_t;
+
+/*****************************************************************
+ * below this is eBPF plugin-specific code.
+ *****************************************************************/
+
+#define NETDATA_EBPF_MODULE_NAME_HARDIRQ "hardirq"
+#define NETDATA_HARDIRQ_SLEEP_MS 650000ULL
+#define NETDATA_HARDIRQ_CONFIG_FILE "hardirq.conf"
+
+typedef struct hardirq_val {
+ // must be at top for simplified AVL tree usage.
+ // if it's not at the top, we need to use `containerof` for almost all ops.
+ avl_t avl;
+
+ int irq;
+ bool dim_exists; // keep this after `int irq` for alignment byte savings.
+ uint64_t latency;
+ char name[NETDATA_HARDIRQ_NAME_LEN];
+} hardirq_val_t;
+
+typedef struct hardirq_static_val {
+ enum hardirq_ebpf_static idx;
+ char *name;
+ uint64_t latency;
+} hardirq_static_val_t;
+
+extern struct config hardirq_config;
+void *ebpf_hardirq_thread(void *ptr);
+
+#endif /* NETDATA_EBPF_HARDIRQ_H */
diff --git a/collectors/ebpf.plugin/ebpf_mdflush.c b/collectors/ebpf.plugin/ebpf_mdflush.c
new file mode 100644
index 0000000..dc805da
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_mdflush.c
@@ -0,0 +1,334 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_mdflush.h"
+
+struct config mdflush_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+#define MDFLUSH_MAP_COUNT 0
+static ebpf_local_maps_t mdflush_maps[] = {
+ {
+ .name = "tbl_mdflush",
+ .internal_input = 1024,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ },
+ /* end */
+ {
+ .name = NULL,
+ .internal_input = 0,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ }
+};
+
+// store for "published" data from the reader thread, which the collector
+// thread will write to netdata agent.
+static avl_tree_lock mdflush_pub;
+
+// tmp store for mdflush values we get from a per-CPU eBPF map.
+static mdflush_ebpf_val_t *mdflush_ebpf_vals = NULL;
+
+static struct netdata_static_thread mdflush_threads = {
+ .name = "MDFLUSH KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+/**
+ * MDflush Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_mdflush_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ freez(mdflush_ebpf_vals);
+ freez(mdflush_threads.thread);
+
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+}
+
+/**
+ * MDflush exit
+ *
+ * Cancel thread and exit.
+ *
+ * @param ptr thread data.
+ */
+static void mdflush_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_mdflush_free(em);
+}
+
+/**
+ * CLeanup
+ *
+ * Clean allocated memory.
+ *
+ * @param ptr thread data.
+ */
+static void mdflush_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*mdflush_threads.thread);
+ ebpf_mdflush_free(em);
+}
+
+/**
+ * Compare mdflush values.
+ *
+ * @param a `netdata_mdflush_t *`.
+ * @param b `netdata_mdflush_t *`.
+ *
+ * @return 0 if a==b, 1 if a>b, -1 if a<b.
+*/
+static int mdflush_val_cmp(void *a, void *b)
+{
+ netdata_mdflush_t *ptr1 = a;
+ netdata_mdflush_t *ptr2 = b;
+
+ if (ptr1->unit > ptr2->unit) {
+ return 1;
+ }
+ else if (ptr1->unit < ptr2->unit) {
+ return -1;
+ }
+ else {
+ return 0;
+ }
+}
+
+static void mdflush_read_count_map()
+{
+ int mapfd = mdflush_maps[MDFLUSH_MAP_COUNT].map_fd;
+ mdflush_ebpf_key_t curr_key = (uint32_t)-1;
+ mdflush_ebpf_key_t key = (uint32_t)-1;
+ netdata_mdflush_t search_v;
+ netdata_mdflush_t *v = NULL;
+
+ while (bpf_map_get_next_key(mapfd, &curr_key, &key) == 0) {
+ curr_key = key;
+
+ // get val for this key.
+ int test = bpf_map_lookup_elem(mapfd, &key, mdflush_ebpf_vals);
+ if (unlikely(test < 0)) {
+ continue;
+ }
+
+ // is this record saved yet?
+ //
+ // if not, make a new one, mark it as unsaved for now, and continue; we
+ // will insert it at the end after all of its values are correctly set,
+ // so that we can safely publish it to the collector within a single,
+ // short locked operation.
+ //
+ // otherwise simply continue; we will only update the flush count,
+ // which can be republished safely without a lock.
+ //
+ // NOTE: lock isn't strictly necessary for this initial search, as only
+ // this thread does writing, but the AVL is using a read-write lock so
+ // there is no congestion.
+ bool v_is_new = false;
+ search_v.unit = key;
+ v = (netdata_mdflush_t *)avl_search_lock(
+ &mdflush_pub,
+ (avl_t *)&search_v
+ );
+ if (unlikely(v == NULL)) {
+ // flush count can only be added reliably at a later time.
+ // when they're added, only then will we AVL insert.
+ v = callocz(1, sizeof(netdata_mdflush_t));
+ v->unit = key;
+ sprintf(v->disk_name, "md%u", key);
+ v->dim_exists = false;
+
+ v_is_new = true;
+ }
+
+ // we must add up count value for this record across all CPUs.
+ uint64_t total_cnt = 0;
+ int i;
+ int end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs;
+ for (i = 0; i < end; i++) {
+ total_cnt += mdflush_ebpf_vals[i];
+ }
+
+ // can now safely publish count for existing records.
+ v->cnt = total_cnt;
+
+ // can now safely publish new record.
+ if (v_is_new) {
+ avl_t *check = avl_insert_lock(&mdflush_pub, (avl_t *)v);
+ if (check != (avl_t *)v) {
+ error("Internal error, cannot insert the AVL tree.");
+ }
+ }
+ }
+}
+
+/**
+ * Read eBPF maps for mdflush.
+ */
+static void *mdflush_reader(void *ptr)
+{
+ netdata_thread_cleanup_push(mdflush_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ usec_t step = NETDATA_MDFLUSH_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ mdflush_read_count_map();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+static void mdflush_create_charts(int update_every)
+{
+ ebpf_create_chart(
+ "mdstat",
+ "mdstat_flush",
+ "MD flushes",
+ "flushes",
+ "flush (eBPF)",
+ "md.flush",
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ NETDATA_CHART_PRIO_MDSTAT_FLUSH,
+ NULL, NULL, 0, update_every,
+ NETDATA_EBPF_MODULE_NAME_MDFLUSH
+ );
+
+ fflush(stdout);
+}
+
+// callback for avl tree traversal on `mdflush_pub`.
+static int mdflush_write_dims(void *entry, void *data)
+{
+ UNUSED(data);
+
+ netdata_mdflush_t *v = entry;
+
+ // records get dynamically added in, so add the dim if we haven't yet.
+ if (!v->dim_exists) {
+ ebpf_write_global_dimension(
+ v->disk_name, v->disk_name,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX]
+ );
+ v->dim_exists = true;
+ }
+
+ write_chart_dimension(v->disk_name, v->cnt);
+
+ return 1;
+}
+
+/**
+* Main loop for this collector.
+*/
+static void mdflush_collector(ebpf_module_t *em)
+{
+ mdflush_ebpf_vals = callocz(ebpf_nprocs, sizeof(mdflush_ebpf_val_t));
+
+ avl_init_lock(&mdflush_pub, mdflush_val_cmp);
+
+ // create reader thread.
+ mdflush_threads.thread = mallocz(sizeof(netdata_thread_t));
+ mdflush_threads.start_routine = mdflush_reader;
+ netdata_thread_create(
+ mdflush_threads.thread,
+ mdflush_threads.name,
+ NETDATA_THREAD_OPTION_DEFAULT,
+ mdflush_reader,
+ em
+ );
+
+ // create chart and static dims.
+ pthread_mutex_lock(&lock);
+ mdflush_create_charts(em->update_every);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ // loop and read from published data until ebpf plugin is closed.
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = em->update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ // write dims now for all hitherto discovered devices.
+ write_begin_chart("mdstat", "mdstat_flush");
+ avl_traverse_lock(&mdflush_pub, mdflush_write_dims, NULL);
+ write_end_chart();
+
+ pthread_mutex_unlock(&lock);
+ }
+}
+
+/**
+ * mdflush thread.
+ *
+ * @param ptr a `ebpf_module_t *`.
+ * @return always NULL.
+ */
+void *ebpf_mdflush_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(mdflush_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = mdflush_maps;
+
+ char *md_flush_request = ebpf_find_symbol("md_flush_request");
+ if (!md_flush_request) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ error("Cannot monitor MD devices, because md is not loaded.");
+ }
+ freez(md_flush_request);
+
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_STOPPED) {
+ goto endmdflush;
+ }
+
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endmdflush;
+ }
+
+ mdflush_collector(em);
+
+endmdflush:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_mdflush.h b/collectors/ebpf.plugin/ebpf_mdflush.h
new file mode 100644
index 0000000..b04eefd
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_mdflush.h
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_MDFLUSH_H
+#define NETDATA_EBPF_MDFLUSH_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_MDFLUSH "mdflush"
+
+#define NETDATA_MDFLUSH_SLEEP_MS 850000ULL
+
+// charts
+#define NETDATA_MDFLUSH_GLOBAL_CHART "mdflush"
+
+// configuration file
+#define NETDATA_DIRECTORY_MDFLUSH_CONFIG_FILE "mdflush.conf"
+
+// copy of mdflush types from kernel-collectors repo.
+typedef uint32_t mdflush_ebpf_key_t;
+typedef uint64_t mdflush_ebpf_val_t;
+
+typedef struct netdata_mdflush {
+ // must be at top for simplified AVL tree usage.
+ // if it's not at the top, we need to use `containerof` for almost all ops.
+ avl_t avl;
+
+ // key & name of device.
+ // the name is generated by the key, usually as `md<unit>`.
+ uint32_t unit;
+ char disk_name[32];
+
+ // have we defined the dimension for this device yet?
+ bool dim_exists;
+
+ // incremental flush count value.
+ uint64_t cnt;
+} netdata_mdflush_t;
+
+void *ebpf_mdflush_thread(void *ptr);
+
+extern struct config mdflush_config;
+
+#endif
diff --git a/collectors/ebpf.plugin/ebpf_mount.c b/collectors/ebpf.plugin/ebpf_mount.c
new file mode 100644
index 0000000..ec1f07a
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_mount.c
@@ -0,0 +1,517 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_mount.h"
+
+static ebpf_local_maps_t mount_maps[] = {{.name = "tbl_mount", .internal_input = NETDATA_MOUNT_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
+
+static char *mount_dimension_name[NETDATA_EBPF_MOUNT_SYSCALL] = { "mount", "umount" };
+static netdata_syscall_stat_t mount_aggregated_data[NETDATA_EBPF_MOUNT_SYSCALL];
+static netdata_publish_syscall_t mount_publish_aggregated[NETDATA_EBPF_MOUNT_SYSCALL];
+
+struct config mount_config = { .first_section = NULL, .last_section = NULL, .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = {.avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+static netdata_idx_t *mount_values = NULL;
+
+static netdata_idx_t mount_hash_values[NETDATA_MOUNT_END];
+
+struct netdata_static_thread mount_thread = {
+ .name = "MOUNT KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+netdata_ebpf_targets_t mount_targets[] = { {.name = "mount", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "umount", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/mount.skel.h" // BTF code
+
+static struct mount_bpf *bpf_obj = NULL;
+
+/*****************************************************************
+ *
+ * BTF FUNCTIONS
+ *
+ *****************************************************************/
+
+/*
+ * Disable probe
+ *
+ * Disable all probes to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void ebpf_mount_disable_probe(struct mount_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_mount_probe, false);
+ bpf_program__set_autoload(obj->progs.netdata_umount_probe, false);
+
+ bpf_program__set_autoload(obj->progs.netdata_mount_retprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_umount_retprobe, false);
+}
+
+/*
+ * Disable tracepoint
+ *
+ * Disable all tracepoints to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void ebpf_mount_disable_tracepoint(struct mount_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_mount_exit, false);
+ bpf_program__set_autoload(obj->progs.netdata_umount_exit, false);
+}
+
+/*
+ * Disable trampoline
+ *
+ * Disable all trampoline to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void ebpf_mount_disable_trampoline(struct mount_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_mount_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_umount_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_mount_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_umount_fexit, false);
+}
+
+/**
+ * Set trampoline target
+ *
+ * Set the targets we will monitor.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void netdata_set_trampoline_target(struct mount_bpf *obj)
+{
+ char syscall[NETDATA_EBPF_MAX_SYSCALL_LENGTH + 1];
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ mount_targets[NETDATA_MOUNT_SYSCALL].name, running_on_kernel);
+
+ bpf_program__set_attach_target(obj->progs.netdata_mount_fentry, 0,
+ syscall);
+
+ bpf_program__set_attach_target(obj->progs.netdata_mount_fexit, 0,
+ syscall);
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ mount_targets[NETDATA_UMOUNT_SYSCALL].name, running_on_kernel);
+
+ bpf_program__set_attach_target(obj->progs.netdata_umount_fentry, 0,
+ syscall);
+
+ bpf_program__set_attach_target(obj->progs.netdata_umount_fexit, 0,
+ syscall);
+}
+
+/**
+ * Mount Attach Probe
+ *
+ * Attach probes to target
+ *
+ * @param obj is the main structure for bpf objects.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+static int ebpf_mount_attach_probe(struct mount_bpf *obj)
+{
+ char syscall[NETDATA_EBPF_MAX_SYSCALL_LENGTH + 1];
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ mount_targets[NETDATA_MOUNT_SYSCALL].name, running_on_kernel);
+
+ obj->links.netdata_mount_probe = bpf_program__attach_kprobe(obj->progs.netdata_mount_probe,
+ false, syscall);
+ int ret = (int)libbpf_get_error(obj->links.netdata_mount_probe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_mount_retprobe = bpf_program__attach_kprobe(obj->progs.netdata_mount_retprobe,
+ true, syscall);
+ ret = (int)libbpf_get_error(obj->links.netdata_mount_retprobe);
+ if (ret)
+ return -1;
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ mount_targets[NETDATA_UMOUNT_SYSCALL].name, running_on_kernel);
+
+ obj->links.netdata_umount_probe = bpf_program__attach_kprobe(obj->progs.netdata_umount_probe,
+ false, syscall);
+ ret = (int)libbpf_get_error(obj->links.netdata_umount_probe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_umount_retprobe = bpf_program__attach_kprobe(obj->progs.netdata_umount_retprobe,
+ true, syscall);
+ ret = (int)libbpf_get_error(obj->links.netdata_umount_retprobe);
+ if (ret)
+ return -1;
+
+ return 0;
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_mount_set_hash_tables(struct mount_bpf *obj)
+{
+ mount_maps[NETDATA_KEY_MOUNT_TABLE].map_fd = bpf_map__fd(obj->maps.tbl_mount);
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_mount_load_and_attach(struct mount_bpf *obj, ebpf_module_t *em)
+{
+ netdata_ebpf_targets_t *mt = em->targets;
+ netdata_ebpf_program_loaded_t test = mt[NETDATA_MOUNT_SYSCALL].mode;
+
+ // We are testing only one, because all will have the same behavior
+ if (test == EBPF_LOAD_TRAMPOLINE ) {
+ ebpf_mount_disable_probe(obj);
+ ebpf_mount_disable_tracepoint(obj);
+
+ netdata_set_trampoline_target(obj);
+ } else if (test == EBPF_LOAD_PROBE ||
+ test == EBPF_LOAD_RETPROBE ) {
+ ebpf_mount_disable_tracepoint(obj);
+ ebpf_mount_disable_trampoline(obj);
+ } else {
+ ebpf_mount_disable_probe(obj);
+ ebpf_mount_disable_trampoline(obj);
+ }
+
+ int ret = mount_bpf__load(obj);
+ if (!ret) {
+ if (test != EBPF_LOAD_PROBE && test != EBPF_LOAD_RETPROBE )
+ ret = mount_bpf__attach(obj);
+ else
+ ret = ebpf_mount_attach_probe(obj);
+
+ if (!ret)
+ ebpf_mount_set_hash_tables(obj);
+ }
+
+ return ret;
+}
+#endif
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Mount Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_mount_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ freez(mount_thread.thread);
+ freez(mount_values);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ if (bpf_obj)
+ mount_bpf__destroy(bpf_obj);
+#endif
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Mount Exit
+ *
+ * Cancel child thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_mount_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*mount_thread.thread);
+ ebpf_mount_free(em);
+}
+
+/**
+ * Mount cleanup
+ *
+ * Clean up allocated memory.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_mount_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_mount_free(em);
+}
+
+/*****************************************************************
+ *
+ * MAIN LOOP
+ *
+ *****************************************************************/
+
+/**
+ * Read global table
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_global_table()
+{
+ uint32_t idx;
+ netdata_idx_t *val = mount_hash_values;
+ netdata_idx_t *stored = mount_values;
+ int fd = mount_maps[NETDATA_KEY_MOUNT_TABLE].map_fd;
+
+ for (idx = NETDATA_KEY_MOUNT_CALL; idx < NETDATA_MOUNT_END; idx++) {
+ if (!bpf_map_lookup_elem(fd, &idx, stored)) {
+ int i;
+ int end = ebpf_nprocs;
+ netdata_idx_t total = 0;
+ for (i = 0; i < end; i++)
+ total += stored[i];
+
+ val[idx] = total;
+ }
+ }
+}
+
+/**
+ * Mount read hash
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_mount_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_mount_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ usec_t step = NETDATA_LATENCY_MOUNT_SLEEP_MS * em->update_every;
+ //This will be cancelled by its parent
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_global_table();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+*/
+static void ebpf_mount_send_data()
+{
+ int i, j;
+ int end = NETDATA_EBPF_MOUNT_SYSCALL;
+ for (i = NETDATA_KEY_MOUNT_CALL, j = NETDATA_KEY_MOUNT_ERROR; i < end; i++, j++) {
+ mount_publish_aggregated[i].ncall = mount_hash_values[i];
+ mount_publish_aggregated[i].nerr = mount_hash_values[j];
+ }
+
+ write_count_chart(NETDATA_EBPF_MOUNT_CALLS, NETDATA_EBPF_MOUNT_GLOBAL_FAMILY,
+ mount_publish_aggregated, NETDATA_EBPF_MOUNT_SYSCALL);
+
+ write_err_chart(NETDATA_EBPF_MOUNT_ERRORS, NETDATA_EBPF_MOUNT_GLOBAL_FAMILY,
+ mount_publish_aggregated, NETDATA_EBPF_MOUNT_SYSCALL);
+}
+
+/**
+* Main loop for this collector.
+*/
+static void mount_collector(ebpf_module_t *em)
+{
+ mount_thread.thread = mallocz(sizeof(netdata_thread_t));
+ mount_thread.start_routine = ebpf_mount_read_hash;
+ memset(mount_hash_values, 0, sizeof(mount_hash_values));
+
+ mount_values = callocz((size_t)ebpf_nprocs, sizeof(netdata_idx_t));
+
+ netdata_thread_create(mount_thread.thread, mount_thread.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_mount_read_hash, em);
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = em->update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ pthread_mutex_lock(&lock);
+
+ ebpf_mount_send_data();
+
+ pthread_mutex_unlock(&lock);
+ }
+}
+
+/*****************************************************************
+ *
+ * INITIALIZE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Create mount charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_mount_charts(int update_every)
+{
+ ebpf_create_chart(NETDATA_EBPF_MOUNT_GLOBAL_FAMILY, NETDATA_EBPF_MOUNT_CALLS,
+ "Calls to mount and umount syscalls",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_EBPF_MOUNT_FAMILY,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_EBPF_MOUNT_CHARTS,
+ ebpf_create_global_dimension,
+ mount_publish_aggregated, NETDATA_EBPF_MOUNT_SYSCALL,
+ update_every, NETDATA_EBPF_MODULE_NAME_MOUNT);
+
+ ebpf_create_chart(NETDATA_EBPF_MOUNT_GLOBAL_FAMILY, NETDATA_EBPF_MOUNT_ERRORS,
+ "Errors to mount and umount file systems",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_EBPF_MOUNT_FAMILY,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_EBPF_MOUNT_CHARTS + 1,
+ ebpf_create_global_dimension,
+ mount_publish_aggregated, NETDATA_EBPF_MOUNT_SYSCALL,
+ update_every, NETDATA_EBPF_MODULE_NAME_MOUNT);
+
+ fflush(stdout);
+}
+
+/*****************************************************************
+ *
+ * MAIN THREAD
+ *
+ *****************************************************************/
+
+/*
+ * Load BPF
+ *
+ * Load BPF files.
+ *
+ * @param em the structure with configuration
+ */
+static int ebpf_mount_load_bpf(ebpf_module_t *em)
+{
+ int ret = 0;
+ if (em->load & EBPF_LOAD_LEGACY) {
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->enabled = CONFIG_BOOLEAN_NO;
+ ret = -1;
+ }
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ bpf_obj = mount_bpf__open();
+ if (!bpf_obj)
+ ret = -1;
+ else
+ ret = ebpf_mount_load_and_attach(bpf_obj, em);
+ }
+#endif
+
+ if (ret)
+ error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->thread_name);
+
+ return ret;
+}
+
+/**
+ * Mount thread
+ *
+ * Thread used to make mount thread
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always returns NULL
+ */
+void *ebpf_mount_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_mount_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = mount_maps;
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_mount_load_bpf(em)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endmount;
+ }
+
+ int algorithms[NETDATA_EBPF_MOUNT_SYSCALL] = { NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX };
+
+ ebpf_global_labels(mount_aggregated_data, mount_publish_aggregated, mount_dimension_name, mount_dimension_name,
+ algorithms, NETDATA_EBPF_MOUNT_SYSCALL);
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_mount_charts(em->update_every);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ mount_collector(em);
+
+endmount:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_mount.h b/collectors/ebpf.plugin/ebpf_mount.h
new file mode 100644
index 0000000..5a8d11a
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_mount.h
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_MOUNT_H
+#define NETDATA_EBPF_MOUNT_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_MOUNT "mount"
+
+#define NETDATA_EBPF_MOUNT_SYSCALL 2
+
+#define NETDATA_LATENCY_MOUNT_SLEEP_MS 700000ULL
+
+#define NETDATA_EBPF_MOUNT_CALLS "call"
+#define NETDATA_EBPF_MOUNT_ERRORS "error"
+#define NETDATA_EBPF_MOUNT_FAMILY "mount (eBPF)"
+
+// Process configuration name
+#define NETDATA_MOUNT_CONFIG_FILE "mount.conf"
+
+enum mount_counters {
+ NETDATA_KEY_MOUNT_CALL,
+ NETDATA_KEY_UMOUNT_CALL,
+ NETDATA_KEY_MOUNT_ERROR,
+ NETDATA_KEY_UMOUNT_ERROR,
+
+ NETDATA_MOUNT_END
+};
+
+enum mount_tables {
+ NETDATA_KEY_MOUNT_TABLE
+};
+
+enum netdata_mount_syscalls {
+ NETDATA_MOUNT_SYSCALL,
+ NETDATA_UMOUNT_SYSCALL,
+
+ NETDATA_MOUNT_SYSCALLS_END
+};
+
+extern struct config mount_config;
+void *ebpf_mount_thread(void *ptr);
+extern netdata_ebpf_targets_t mount_targets[];
+
+#endif /* NETDATA_EBPF_MOUNT_H */
diff --git a/collectors/ebpf.plugin/ebpf_oomkill.c b/collectors/ebpf.plugin/ebpf_oomkill.c
new file mode 100644
index 0000000..d93e415
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_oomkill.c
@@ -0,0 +1,402 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_oomkill.h"
+
+struct config oomkill_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+#define OOMKILL_MAP_KILLCNT 0
+static ebpf_local_maps_t oomkill_maps[] = {
+ {
+ .name = "tbl_oomkill",
+ .internal_input = NETDATA_OOMKILL_MAX_ENTRIES,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ },
+ /* end */
+ {
+ .name = NULL,
+ .internal_input = 0,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ }
+};
+
+static ebpf_tracepoint_t oomkill_tracepoints[] = {
+ {.enabled = false, .class = "oom", .event = "mark_victim"},
+ /* end */
+ {.enabled = false, .class = NULL, .event = NULL}
+};
+
+static netdata_publish_syscall_t oomkill_publish_aggregated = {.name = "oomkill", .dimension = "oomkill",
+ .algorithm = "absolute",
+ .next = NULL};
+
+/**
+ * Clean up the main thread.
+ *
+ * @param ptr thread data.
+ */
+static void oomkill_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+}
+
+static void oomkill_write_data(int32_t *keys, uint32_t total)
+{
+ // for each app, see if it was OOM killed. record as 1 if so otherwise 0.
+ struct target *w;
+ for (w = apps_groups_root_target; w != NULL; w = w->next) {
+ if (likely(w->exposed && w->processes)) {
+ bool was_oomkilled = false;
+ struct pid_on_target *pids = w->root_pid;
+ while (pids) {
+ uint32_t j;
+ for (j = 0; j < total; j++) {
+ if (pids->pid == keys[j]) {
+ was_oomkilled = true;
+ // set to 0 so we consider it "done".
+ keys[j] = 0;
+ goto write_dim;
+ }
+ }
+ pids = pids->next;
+ }
+
+ write_dim:;
+ write_chart_dimension(w->name, was_oomkilled);
+ }
+ }
+
+ // for any remaining keys for which we couldn't find a group, this could be
+ // for various reasons, but the primary one is that the PID has not yet
+ // been picked up by the process thread when parsing the proc filesystem.
+ // since it's been OOM killed, it will never be parsed in the future, so
+ // we have no choice but to dump it into `other`.
+ uint32_t j;
+ uint32_t rem_count = 0;
+ for (j = 0; j < total; j++) {
+ int32_t key = keys[j];
+ if (key != 0) {
+ rem_count += 1;
+ }
+ }
+ if (rem_count > 0) {
+ write_chart_dimension("other", rem_count);
+ }
+}
+
+/**
+ * Create specific OOMkill charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_specific_oomkill_charts(char *type, int update_every)
+{
+ ebpf_create_chart(type, NETDATA_OOMKILL_CHART, "OOM kills. This chart is provided by eBPF plugin.",
+ EBPF_COMMON_DIMENSION_KILLS, NETDATA_EBPF_MEMORY_GROUP,
+ NETDATA_CGROUP_OOMKILLS_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5600,
+ ebpf_create_global_dimension,
+ &oomkill_publish_aggregated, 1, update_every, NETDATA_EBPF_MODULE_NAME_OOMKILL);
+}
+
+/**
+ * Create Systemd OOMkill Charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ **/
+static void ebpf_create_systemd_oomkill_charts(int update_every)
+{
+ ebpf_create_charts_on_systemd(NETDATA_OOMKILL_CHART, "OOM kills. This chart is provided by eBPF plugin.",
+ EBPF_COMMON_DIMENSION_KILLS, NETDATA_EBPF_MEMORY_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, 20191,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NULL,
+ NETDATA_EBPF_MODULE_NAME_OOMKILL, update_every);
+}
+
+/**
+ * Send Systemd charts
+ *
+ * Send collected data to Netdata.
+ */
+static void ebpf_send_systemd_oomkill_charts()
+{
+ ebpf_cgroup_target_t *ect;
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_OOMKILL_CHART);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long) ect->oomkill);
+ ect->oomkill = 0;
+ }
+ }
+ write_end_chart();
+}
+
+/*
+ * Send Specific OOMkill data
+ *
+ * Send data for specific cgroup/apps.
+ *
+ * @param type chart type
+ * @param value value for oomkill
+ */
+static void ebpf_send_specific_oomkill_data(char *type, int value)
+{
+ write_begin_chart(type, NETDATA_OOMKILL_CHART);
+ write_chart_dimension(oomkill_publish_aggregated.name, (long long)value);
+ write_end_chart();
+}
+
+/**
+ * Create specific OOMkill charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_obsolete_specific_oomkill_charts(char *type, int update_every)
+{
+ ebpf_write_chart_obsolete(type, NETDATA_OOMKILL_CHART, "OOM kills. This chart is provided by eBPF plugin.",
+ EBPF_COMMON_DIMENSION_KILLS, NETDATA_EBPF_MEMORY_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_OOMKILLS_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5600, update_every);
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+*/
+void ebpf_oomkill_send_cgroup_data(int update_every)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_oomkill_charts(update_every);
+ }
+ ebpf_send_systemd_oomkill_charts();
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART) && ect->updated) {
+ ebpf_create_specific_oomkill_charts(ect->name, update_every);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART && ect->updated) {
+ ebpf_send_specific_oomkill_data(ect->name, ect->oomkill);
+ } else {
+ ebpf_obsolete_specific_oomkill_charts(ect->name, update_every);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_OOMKILL_CHART;
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Read data
+ *
+ * Read OOMKILL events from table.
+ *
+ * @param keys vector where data will be stored
+ *
+ * @return It returns the number of read elements
+ */
+static uint32_t oomkill_read_data(int32_t *keys)
+{
+ // the first `i` entries of `keys` will contain the currently active PIDs
+ // in the eBPF map.
+ uint32_t i = 0;
+
+ uint32_t curr_key = 0;
+ uint32_t key = 0;
+ int mapfd = oomkill_maps[OOMKILL_MAP_KILLCNT].map_fd;
+ while (bpf_map_get_next_key(mapfd, &curr_key, &key) == 0) {
+ curr_key = key;
+
+ keys[i] = (int32_t)key;
+ i += 1;
+
+ // delete this key now that we've recorded its existence. there's no
+ // race here, as the same PID will only get OOM killed once.
+ int test = bpf_map_delete_elem(mapfd, &key);
+ if (unlikely(test < 0)) {
+ // since there's only 1 thread doing these deletions, it should be
+ // impossible to get this condition.
+ error("key unexpectedly not available for deletion.");
+ }
+ }
+
+ return i;
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ *
+ * @param keys vector with pids that had oomkill event
+ * @param total number of elements in keys vector.
+ */
+static void ebpf_update_oomkill_cgroup(int32_t *keys, uint32_t total)
+{
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ ect->oomkill = 0;
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ uint32_t j;
+ int32_t pid = pids->pid;
+ for (j = 0; j < total; j++) {
+ if (pid == keys[j]) {
+ ect->oomkill = 1;
+ break;
+ }
+ }
+ }
+ }
+}
+
+/**
+* Main loop for this collector.
+*/
+static void oomkill_collector(ebpf_module_t *em)
+{
+ int cgroups = em->cgroup_charts;
+ int update_every = em->update_every;
+ int32_t keys[NETDATA_OOMKILL_MAX_ENTRIES];
+ memset(keys, 0, sizeof(keys));
+
+ // loop and read until ebpf plugin is closed.
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ pthread_mutex_lock(&collect_data_mutex);
+ pthread_mutex_lock(&lock);
+
+ uint32_t count = oomkill_read_data(keys);
+ if (cgroups && count)
+ ebpf_update_oomkill_cgroup(keys, count);
+
+ // write everything from the ebpf map.
+ if (cgroups)
+ ebpf_oomkill_send_cgroup_data(update_every);
+
+ if (em->apps_charts & NETDATA_EBPF_APPS_FLAG_CHART_CREATED) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_OOMKILL_CHART);
+ oomkill_write_data(keys, count);
+ write_end_chart();
+ }
+
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+}
+
+/**
+ * Create apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+void ebpf_oomkill_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+ ebpf_create_charts_on_apps(NETDATA_OOMKILL_CHART,
+ "OOM kills",
+ EBPF_COMMON_DIMENSION_KILLS,
+ "mem",
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20020,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_OOMKILL);
+
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/**
+ * OOM kill tracking thread.
+ *
+ * @param ptr a `ebpf_module_t *`.
+ * @return always NULL.
+ */
+void *ebpf_oomkill_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(oomkill_cleanup, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = oomkill_maps;
+
+#define NETDATA_DEFAULT_OOM_DISABLED_MSG "Disabling OOMKILL thread, because"
+ if (unlikely(!all_pids || !em->apps_charts)) {
+ // When we are not running integration with apps, we won't fill necessary variables for this thread to run, so
+ // we need to disable it.
+ if (em->thread->enabled)
+ info("%s apps integration is completely disabled.", NETDATA_DEFAULT_OOM_DISABLED_MSG);
+
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ } else if (running_on_kernel < NETDATA_EBPF_KERNEL_4_14) {
+ if (em->thread->enabled)
+ info("%s kernel does not have necessary tracepoints.", NETDATA_DEFAULT_OOM_DISABLED_MSG);
+
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ }
+
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_STOPPED) {
+ goto endoomkill;
+ }
+
+ if (ebpf_enable_tracepoints(oomkill_tracepoints) == 0) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endoomkill;
+ }
+
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endoomkill;
+ }
+
+ pthread_mutex_lock(&lock);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ oomkill_collector(em);
+
+endoomkill:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_oomkill.h b/collectors/ebpf.plugin/ebpf_oomkill.h
new file mode 100644
index 0000000..7860863
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_oomkill.h
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_OOMKILL_H
+#define NETDATA_EBPF_OOMKILL_H 1
+
+/*****************************************************************
+ * copied from kernel-collectors repo, with modifications needed
+ * for inclusion here.
+ *****************************************************************/
+
+#define NETDATA_OOMKILL_MAX_ENTRIES 64
+
+typedef uint8_t oomkill_ebpf_val_t;
+
+/*****************************************************************
+ * below this is eBPF plugin-specific code.
+ *****************************************************************/
+
+#define NETDATA_EBPF_MODULE_NAME_OOMKILL "oomkill"
+#define NETDATA_OOMKILL_SLEEP_MS 650000ULL
+#define NETDATA_OOMKILL_CONFIG_FILE "oomkill.conf"
+
+#define NETDATA_OOMKILL_CHART "oomkills"
+
+// Contexts
+#define NETDATA_CGROUP_OOMKILLS_CONTEXT "cgroup.oomkills"
+
+extern struct config oomkill_config;
+void *ebpf_oomkill_thread(void *ptr);
+void ebpf_oomkill_create_apps_charts(struct ebpf_module *em, void *ptr);
+
+#endif /* NETDATA_EBPF_OOMKILL_H */
diff --git a/collectors/ebpf.plugin/ebpf_process.c b/collectors/ebpf.plugin/ebpf_process.c
new file mode 100644
index 0000000..682577d
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_process.c
@@ -0,0 +1,1327 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <sys/resource.h>
+
+#include "ebpf.h"
+#include "ebpf_process.h"
+
+/*****************************************************************
+ *
+ * GLOBAL VARIABLES
+ *
+ *****************************************************************/
+
+static char *process_dimension_names[NETDATA_KEY_PUBLISH_PROCESS_END] = { "process", "task", "process", "thread" };
+static char *process_id_names[NETDATA_KEY_PUBLISH_PROCESS_END] = { "do_exit", "release_task", "_do_fork", "sys_clone" };
+static char *status[] = { "process", "zombie" };
+
+static ebpf_local_maps_t process_maps[] = {{.name = "tbl_pid_stats", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_total_stats", .internal_input = NETDATA_KEY_END_VECTOR,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "process_ctrl", .internal_input = NETDATA_CONTROLLER_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
+
+char *tracepoint_sched_type = { "sched" } ;
+char *tracepoint_sched_process_exit = { "sched_process_exit" };
+char *tracepoint_sched_process_exec = { "sched_process_exec" };
+char *tracepoint_sched_process_fork = { "sched_process_fork" };
+static int was_sched_process_exit_enabled = 0;
+static int was_sched_process_exec_enabled = 0;
+static int was_sched_process_fork_enabled = 0;
+
+static netdata_idx_t *process_hash_values = NULL;
+static netdata_syscall_stat_t process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_END];
+static netdata_publish_syscall_t process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_END];
+
+ebpf_process_stat_t **global_process_stats = NULL;
+ebpf_process_publish_apps_t **current_apps_data = NULL;
+
+int process_enabled = 0;
+bool publish_internal_metrics = true;
+
+struct config process_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+static char *threads_stat[NETDATA_EBPF_THREAD_STAT_END] = {"total", "running"};
+static char *load_event_stat[NETDATA_EBPF_LOAD_STAT_END] = {"legacy", "co-re"};
+
+static struct netdata_static_thread cgroup_thread = {
+ .name = "EBPF CGROUP",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+/*****************************************************************
+ *
+ * PROCESS DATA AND SEND TO NETDATA
+ *
+ *****************************************************************/
+
+/**
+ * Update publish structure before to send data to Netdata.
+ *
+ * @param publish the first output structure with independent dimensions
+ * @param pvc the second output structure with correlated dimensions
+ * @param input the structure with the input data.
+ */
+static void ebpf_update_global_publish(netdata_publish_syscall_t *publish, netdata_publish_vfs_common_t *pvc,
+ netdata_syscall_stat_t *input)
+{
+ netdata_publish_syscall_t *move = publish;
+ int selector = NETDATA_KEY_PUBLISH_PROCESS_EXIT;
+ while (move) {
+ move->ncall = (input->call > move->pcall) ? input->call - move->pcall : move->pcall - input->call;
+ move->nbyte = (input->bytes > move->pbyte) ? input->bytes - move->pbyte : move->pbyte - input->bytes;
+ move->nerr = (input->ecall > move->nerr) ? input->ecall - move->perr : move->perr - input->ecall;
+
+ move->pcall = input->call;
+ move->pbyte = input->bytes;
+ move->perr = input->ecall;
+
+ input = input->next;
+ move = move->next;
+ selector++;
+ }
+
+ pvc->running = (long)publish[NETDATA_KEY_PUBLISH_PROCESS_FORK].ncall -
+ (long)publish[NETDATA_KEY_PUBLISH_PROCESS_CLONE].ncall;
+ publish[NETDATA_KEY_PUBLISH_PROCESS_RELEASE_TASK].ncall = -publish[NETDATA_KEY_PUBLISH_PROCESS_RELEASE_TASK].ncall;
+ pvc->zombie = (long)publish[NETDATA_KEY_PUBLISH_PROCESS_EXIT].ncall +
+ (long)publish[NETDATA_KEY_PUBLISH_PROCESS_RELEASE_TASK].ncall;
+}
+
+/**
+ * Call the necessary functions to create a chart.
+ *
+ * @param family the chart family
+ * @param move the pointer with the values that will be published
+ */
+static void write_status_chart(char *family, netdata_publish_vfs_common_t *pvc)
+{
+ write_begin_chart(family, NETDATA_PROCESS_STATUS_NAME);
+
+ write_chart_dimension(status[0], (long long)pvc->running);
+ write_chart_dimension(status[1], (long long)pvc->zombie);
+
+ write_end_chart();
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the structure with thread information
+ */
+static void ebpf_process_send_data(ebpf_module_t *em)
+{
+ netdata_publish_vfs_common_t pvc;
+ ebpf_update_global_publish(process_publish_aggregated, &pvc, process_aggregated_data);
+
+ write_count_chart(NETDATA_EXIT_SYSCALL, NETDATA_EBPF_SYSTEM_GROUP,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_EXIT], 2);
+ write_count_chart(NETDATA_PROCESS_SYSCALL, NETDATA_EBPF_SYSTEM_GROUP,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_FORK], 2);
+
+ write_status_chart(NETDATA_EBPF_SYSTEM_GROUP, &pvc);
+ if (em->mode < MODE_ENTRY) {
+ write_err_chart(NETDATA_PROCESS_ERROR_NAME, NETDATA_EBPF_SYSTEM_GROUP,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_FORK], 2);
+ }
+}
+
+/**
+ * Sum values for pid
+ *
+ * @param root the structure with all available PIDs
+ *
+ * @param offset the address that we are reading
+ *
+ * @return it returns the sum of all PIDs
+ */
+long long ebpf_process_sum_values_for_pids(struct pid_on_target *root, size_t offset)
+{
+ long long ret = 0;
+ while (root) {
+ int32_t pid = root->pid;
+ ebpf_process_publish_apps_t *w = current_apps_data[pid];
+ if (w) {
+ ret += get_value_from_structure((char *)w, offset);
+ }
+
+ root = root->next;
+ }
+
+ return ret;
+}
+
+/**
+ * Remove process pid
+ *
+ * Remove from PID task table when task_release was called.
+ */
+void ebpf_process_remove_pids()
+{
+ struct pid_stat *pids = root_of_pids;
+ int pid_fd = process_maps[NETDATA_PROCESS_PID_TABLE].map_fd;
+ while (pids) {
+ uint32_t pid = pids->pid;
+ ebpf_process_stat_t *w = global_process_stats[pid];
+ if (w) {
+ freez(w);
+ global_process_stats[pid] = NULL;
+ bpf_map_delete_elem(pid_fd, &pid);
+ }
+
+ pids = pids->next;
+ }
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param root the target list.
+ */
+void ebpf_process_send_apps_data(struct target *root, ebpf_module_t *em)
+{
+ struct target *w;
+ collected_number value;
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_TASK_PROCESS);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_process_sum_values_for_pids(w->root_pid, offsetof(ebpf_process_publish_apps_t, create_process));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_TASK_THREAD);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_process_sum_values_for_pids(w->root_pid, offsetof(ebpf_process_publish_apps_t, create_thread));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_TASK_EXIT);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_process_sum_values_for_pids(w->root_pid, offsetof(ebpf_process_publish_apps_t,
+ call_do_exit));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_TASK_CLOSE);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_process_sum_values_for_pids(w->root_pid, offsetof(ebpf_process_publish_apps_t,
+ call_release_task));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_TASK_ERROR);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_process_sum_values_for_pids(w->root_pid, offsetof(ebpf_process_publish_apps_t,
+ task_err));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+ }
+
+ ebpf_process_remove_pids();
+}
+
+/*****************************************************************
+ *
+ * READ INFORMATION FROM KERNEL RING
+ *
+ *****************************************************************/
+
+/**
+ * Read the hash table and store data to allocated vectors.
+ */
+static void read_hash_global_tables()
+{
+ uint64_t idx;
+ netdata_idx_t res[NETDATA_KEY_END_VECTOR];
+
+ netdata_idx_t *val = process_hash_values;
+ int fd = process_maps[NETDATA_PROCESS_GLOBAL_TABLE].map_fd;
+ for (idx = 0; idx < NETDATA_KEY_END_VECTOR; idx++) {
+ if (!bpf_map_lookup_elem(fd, &idx, val)) {
+ uint64_t total = 0;
+ int i;
+ int end = ebpf_nprocs;
+ for (i = 0; i < end; i++)
+ total += val[i];
+
+ res[idx] = total;
+ } else {
+ res[idx] = 0;
+ }
+ }
+
+ process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_EXIT].call = res[NETDATA_KEY_CALLS_DO_EXIT];
+ process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_RELEASE_TASK].call = res[NETDATA_KEY_CALLS_RELEASE_TASK];
+ process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_FORK].call = res[NETDATA_KEY_CALLS_DO_FORK];
+ process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_CLONE].call = res[NETDATA_KEY_CALLS_SYS_CLONE];
+
+ process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_FORK].ecall = res[NETDATA_KEY_ERROR_DO_FORK];
+ process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_CLONE].ecall = res[NETDATA_KEY_ERROR_SYS_CLONE];
+}
+
+/**
+ * Read the hash table and store data to allocated vectors.
+ */
+static void ebpf_process_update_apps_data()
+{
+ struct pid_stat *pids = root_of_pids;
+ while (pids) {
+ uint32_t current_pid = pids->pid;
+ ebpf_process_stat_t *ps = global_process_stats[current_pid];
+ if (!ps) {
+ pids = pids->next;
+ continue;
+ }
+
+ ebpf_process_publish_apps_t *cad = current_apps_data[current_pid];
+ if (!cad) {
+ cad = callocz(1, sizeof(ebpf_process_publish_apps_t));
+ current_apps_data[current_pid] = cad;
+ }
+
+ //Read data
+ cad->call_do_exit = ps->exit_call;
+ cad->call_release_task = ps->release_call;
+ cad->create_process = ps->create_process;
+ cad->create_thread = ps->create_thread;
+
+ cad->task_err = ps->task_err;
+
+ pids = pids->next;
+ }
+}
+
+/**
+ * Cgroup Exit
+ *
+ * Function used with netdata_thread_clean_push
+ *
+ * @param ptr unused argument
+ */
+static void ebpf_cgroup_exit(void *ptr)
+{
+ UNUSED(ptr);
+}
+
+/**
+ * Cgroup update shm
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data from shared memory.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_cgroup_update_shm(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_cgroup_exit, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ usec_t step = 3 * USEC_PER_SEC;
+ int counter = NETDATA_EBPF_CGROUP_UPDATE - 1;
+ //This will be cancelled by its parent
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ // We are using a small heartbeat time to wake up thread,
+ // but we should not update so frequently the shared memory data
+ if (++counter >= NETDATA_EBPF_CGROUP_UPDATE) {
+ counter = 0;
+ if (!shm_ebpf_cgroup.header)
+ ebpf_map_cgroup_shared_memory();
+
+ ebpf_parse_cgroup_shm_data();
+ }
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ */
+static void ebpf_update_process_cgroup()
+{
+ ebpf_cgroup_target_t *ect ;
+ int pid_fd = process_maps[NETDATA_PROCESS_PID_TABLE].map_fd;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ int pid = pids->pid;
+ ebpf_process_stat_t *out = &pids->ps;
+ if (global_process_stats[pid]) {
+ ebpf_process_stat_t *in = global_process_stats[pid];
+
+ memcpy(out, in, sizeof(ebpf_process_stat_t));
+ } else {
+ if (bpf_map_lookup_elem(pid_fd, &pid, out)) {
+ memset(out, 0, sizeof(ebpf_process_stat_t));
+ }
+ }
+ }
+ }
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CREATE CHARTS
+ *
+ *****************************************************************/
+
+/**
+ * Create process status chart
+ *
+ * @param family the chart family
+ * @param name the chart name
+ * @param axis the axis label
+ * @param web the group name used to attach the chart on dashboard
+ * @param order the order number of the specified chart
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_process_status_chart(char *family, char *name, char *axis,
+ char *web, char *algorithm, int order, int update_every)
+{
+ printf("CHART %s.%s '' 'Process not closed' '%s' '%s' '' line %d %d '' 'ebpf.plugin' 'process'\n",
+ family,
+ name,
+ axis,
+ web,
+ order,
+ update_every);
+
+ printf("DIMENSION %s '' %s 1 1\n", status[0], algorithm);
+ printf("DIMENSION %s '' %s 1 1\n", status[1], algorithm);
+}
+
+/**
+ * Create global charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+static void ebpf_create_global_charts(ebpf_module_t *em)
+{
+ ebpf_create_chart(NETDATA_EBPF_SYSTEM_GROUP,
+ NETDATA_PROCESS_SYSCALL,
+ "Start process",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_PROCESS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21002,
+ ebpf_create_global_dimension,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_FORK],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_create_chart(NETDATA_EBPF_SYSTEM_GROUP,
+ NETDATA_EXIT_SYSCALL,
+ "Exit process",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_PROCESS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21003,
+ ebpf_create_global_dimension,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_EXIT],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_process_status_chart(NETDATA_EBPF_SYSTEM_GROUP,
+ NETDATA_PROCESS_STATUS_NAME,
+ EBPF_COMMON_DIMENSION_DIFFERENCE,
+ NETDATA_PROCESS_GROUP,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ 21004, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(NETDATA_EBPF_SYSTEM_GROUP,
+ NETDATA_PROCESS_ERROR_NAME,
+ "Fails to create process",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_PROCESS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 21005,
+ ebpf_create_global_dimension,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_FORK],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+ }
+}
+
+/**
+ * Create chart for Statistic Thread
+ *
+ * Write to standard output current values for threads.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+static inline void ebpf_create_statistic_thread_chart(ebpf_module_t *em)
+{
+ ebpf_write_chart_cmd(NETDATA_MONITORING_FAMILY,
+ NETDATA_EBPF_THREADS,
+ "Threads info.",
+ "threads",
+ NETDATA_EBPF_FAMILY,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NULL,
+ 140000,
+ em->update_every,
+ NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_write_global_dimension(threads_stat[NETDATA_EBPF_THREAD_STAT_TOTAL],
+ threads_stat[NETDATA_EBPF_THREAD_STAT_TOTAL],
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX]);
+
+ ebpf_write_global_dimension(threads_stat[NETDATA_EBPF_THREAD_STAT_RUNNING],
+ threads_stat[NETDATA_EBPF_THREAD_STAT_RUNNING],
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX]);
+}
+
+/**
+ * Create chart for Load Thread
+ *
+ * Write to standard output current values for load mode.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+static inline void ebpf_create_statistic_load_chart(ebpf_module_t *em)
+{
+ ebpf_write_chart_cmd(NETDATA_MONITORING_FAMILY,
+ NETDATA_EBPF_LOAD_METHOD,
+ "Load info.",
+ "methods",
+ NETDATA_EBPF_FAMILY,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NULL,
+ 140001,
+ em->update_every,
+ NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_write_global_dimension(load_event_stat[NETDATA_EBPF_LOAD_STAT_LEGACY],
+ load_event_stat[NETDATA_EBPF_LOAD_STAT_LEGACY],
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX]);
+
+ ebpf_write_global_dimension(load_event_stat[NETDATA_EBPF_LOAD_STAT_CORE],
+ load_event_stat[NETDATA_EBPF_LOAD_STAT_CORE],
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX]);
+}
+
+/**
+ * Update Internal Metric variable
+ *
+ * By default eBPF.plugin sends internal metrics for netdata, but user can
+ * disable this.
+ *
+ * The function updates the variable used to send charts.
+ */
+static void update_internal_metric_variable()
+{
+ const char *s = getenv("NETDATA_INTERNALS_MONITORING");
+ if (s && *s && strcmp(s, "NO") == 0)
+ publish_internal_metrics = false;
+}
+
+/**
+ * Create Statistics Charts
+ *
+ * Create charts that will show statistics related to eBPF plugin.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+static void ebpf_create_statistic_charts(ebpf_module_t *em)
+{
+ update_internal_metric_variable();
+ if (!publish_internal_metrics)
+ return;
+
+ ebpf_create_statistic_thread_chart(em);
+
+ ebpf_create_statistic_load_chart(em);
+}
+
+/**
+ * Create process apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ * @param ptr a pointer for the targets.
+ */
+void ebpf_process_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_TASK_PROCESS,
+ "Process started",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20065,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_TASK_THREAD,
+ "Threads started",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20066,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_TASK_EXIT,
+ "Tasks starts exit process.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20067,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_TASK_CLOSE,
+ "Tasks closed",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20068,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_TASK_ERROR,
+ "Errors to create process or threads.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20069,
+ ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
+ root,
+ em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+ }
+
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/**
+ * Create apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param root a pointer for the targets.
+ */
+static void ebpf_create_apps_charts(struct target *root)
+{
+ if (unlikely(!all_pids))
+ return;
+
+ struct target *w;
+ int newly_added = 0;
+
+ for (w = root; w; w = w->next) {
+ if (w->target)
+ continue;
+
+ if (unlikely(w->processes && (debug_enabled || w->debug_enabled))) {
+ struct pid_on_target *pid_on_target;
+
+ fprintf(
+ stderr, "ebpf.plugin: target '%s' has aggregated %u process%s:", w->name, w->processes,
+ (w->processes == 1) ? "" : "es");
+
+ for (pid_on_target = w->root_pid; pid_on_target; pid_on_target = pid_on_target->next) {
+ fprintf(stderr, " %d", pid_on_target->pid);
+ }
+
+ fputc('\n', stderr);
+ }
+
+ if (!w->exposed && w->processes) {
+ newly_added++;
+ w->exposed = 1;
+ if (debug_enabled || w->debug_enabled)
+ debug_log_int("%s just added - regenerating charts.", w->name);
+ }
+ }
+
+ if (!newly_added)
+ return;
+
+ int counter;
+ for (counter = 0; ebpf_modules[counter].thread_name; counter++) {
+ ebpf_module_t *current = &ebpf_modules[counter];
+ if (current->enabled && current->apps_charts && current->apps_routine)
+ current->apps_routine(current, root);
+ }
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Process disable tracepoints
+ *
+ * Disable tracepoints when the plugin was responsible to enable it.
+ */
+static void ebpf_process_disable_tracepoints()
+{
+ char *default_message = { "Cannot disable the tracepoint" };
+ if (!was_sched_process_exit_enabled) {
+ if (ebpf_disable_tracing_values(tracepoint_sched_type, tracepoint_sched_process_exit))
+ error("%s %s/%s.", default_message, tracepoint_sched_type, tracepoint_sched_process_exit);
+ }
+
+ if (!was_sched_process_exec_enabled) {
+ if (ebpf_disable_tracing_values(tracepoint_sched_type, tracepoint_sched_process_exec))
+ error("%s %s/%s.", default_message, tracepoint_sched_type, tracepoint_sched_process_exec);
+ }
+
+ if (!was_sched_process_fork_enabled) {
+ if (ebpf_disable_tracing_values(tracepoint_sched_type, tracepoint_sched_process_fork))
+ error("%s %s/%s.", default_message, tracepoint_sched_type, tracepoint_sched_process_fork);
+ }
+}
+
+/**
+ * Process Exit
+ *
+ * Cancel child thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_process_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ ebpf_cleanup_publish_syscall(process_publish_aggregated);
+ freez(process_hash_values);
+
+ ebpf_process_disable_tracepoints();
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ pthread_cancel(*cgroup_thread.thread);
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS WITH THE MAIN LOOP
+ *
+ *****************************************************************/
+
+
+/**
+ * Sum PIDs
+ *
+ * Sum values for all targets.
+ *
+ * @param ps structure used to store data
+ * @param pids input data
+ */
+static void ebpf_process_sum_cgroup_pids(ebpf_process_stat_t *ps, struct pid_on_target2 *pids)
+{
+ ebpf_process_stat_t accumulator;
+ memset(&accumulator, 0, sizeof(accumulator));
+
+ while (pids) {
+ ebpf_process_stat_t *ps = &pids->ps;
+
+ accumulator.exit_call += ps->exit_call;
+ accumulator.release_call += ps->release_call;
+ accumulator.create_process += ps->create_process;
+ accumulator.create_thread += ps->create_thread;
+
+ accumulator.task_err += ps->task_err;
+
+ pids = pids->next;
+ }
+
+ ps->exit_call = (accumulator.exit_call >= ps->exit_call) ? accumulator.exit_call : ps->exit_call;
+ ps->release_call = (accumulator.release_call >= ps->release_call) ? accumulator.release_call : ps->release_call;
+ ps->create_process = (accumulator.create_process >= ps->create_process) ? accumulator.create_process : ps->create_process;
+ ps->create_thread = (accumulator.create_thread >= ps->create_thread) ? accumulator.create_thread : ps->create_thread;
+
+ ps->task_err = (accumulator.task_err >= ps->task_err) ? accumulator.task_err : ps->task_err;
+}
+
+/*
+ * Send Specific Process data
+ *
+ * Send data for specific cgroup/apps.
+ *
+ * @param type chart type
+ * @param values structure with values that will be sent to netdata
+ * @param em the structure with thread information
+ */
+static void ebpf_send_specific_process_data(char *type, ebpf_process_stat_t *values, ebpf_module_t *em)
+{
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_TASK_PROCESS);
+ write_chart_dimension(process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_FORK].name,
+ (long long) values->create_process);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_TASK_THREAD);
+ write_chart_dimension(process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_CLONE].name,
+ (long long) values->create_thread);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_TASK_EXIT);
+ write_chart_dimension(process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_EXIT].name,
+ (long long) values->release_call);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_TASK_CLOSE);
+ write_chart_dimension(process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_RELEASE_TASK].name,
+ (long long) values->release_call);
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_TASK_ERROR);
+ write_chart_dimension(process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_EXIT].name,
+ (long long) values->task_err);
+ write_end_chart();
+ }
+}
+
+/**
+ * Create specific process charts
+ *
+ * Create charts for cgroup/application
+ *
+ * @param type the chart type.
+ * @param em the structure with thread information
+ */
+static void ebpf_create_specific_process_charts(char *type, ebpf_module_t *em)
+{
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_TASK_PROCESS, "Process started",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_CGROUP_GROUP,
+ NETDATA_CGROUP_PROCESS_CREATE_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5000,
+ ebpf_create_global_dimension, &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_FORK],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_TASK_THREAD, "Threads started",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_CGROUP_GROUP,
+ NETDATA_CGROUP_THREAD_CREATE_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5001,
+ ebpf_create_global_dimension,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_CLONE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_TASK_EXIT, "Tasks starts exit process.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_CGROUP_GROUP,
+ NETDATA_CGROUP_PROCESS_EXIT_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5002,
+ ebpf_create_global_dimension,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_EXIT],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_TASK_CLOSE, "Tasks closed",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_CGROUP_GROUP,
+ NETDATA_CGROUP_PROCESS_CLOSE_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5003,
+ ebpf_create_global_dimension,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_RELEASE_TASK],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_TASK_ERROR, "Errors to create process or threads.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_CGROUP_GROUP,
+ NETDATA_CGROUP_PROCESS_ERROR_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5004,
+ ebpf_create_global_dimension,
+ &process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_EXIT],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_PROCESS);
+ }
+}
+
+/**
+ * Obsolete specific process charts
+ *
+ * Obsolete charts for cgroup/application
+ *
+ * @param type the chart type.
+ * @param em the structure with thread information
+ */
+static void ebpf_obsolete_specific_process_charts(char *type, ebpf_module_t *em)
+{
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_TASK_PROCESS, "Process started",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_GROUP, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CGROUP_PROCESS_CREATE_CONTEXT, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5000,
+ em->update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_TASK_THREAD, "Threads started",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_GROUP, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CGROUP_THREAD_CREATE_CONTEXT, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5001,
+ em->update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_TASK_EXIT,"Tasks starts exit process.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_GROUP, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CGROUP_PROCESS_EXIT_CONTEXT, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5002,
+ em->update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_TASK_CLOSE,"Tasks closed",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_GROUP, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CGROUP_PROCESS_CLOSE_CONTEXT, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5003,
+ em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_TASK_ERROR,"Errors to create process or threads.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_GROUP, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CGROUP_PROCESS_ERROR_CONTEXT, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5004,
+ em->update_every);
+ }
+}
+
+/**
+ * Create Systemd process Charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param em the structure with thread information
+ **/
+static void ebpf_create_systemd_process_charts(ebpf_module_t *em)
+{
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_TASK_PROCESS, "Process started",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20065,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_PROCESS_CREATE_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_TASK_THREAD, "Threads started",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20066,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_THREAD_CREATE_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_TASK_CLOSE, "Tasks starts exit process.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20067,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_PROCESS_EXIT_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_TASK_EXIT, "Tasks closed",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20068,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_PROCESS_CLOSE_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_TASK_ERROR, "Errors to create process or threads.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_PROCESS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20069,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_PROCESS_ERROR_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_PROCESS, em->update_every);
+ }
+}
+
+/**
+ * Send Systemd charts
+ *
+ * Send collected data to Netdata.
+ *
+ * @param em the structure with thread information
+ */
+static void ebpf_send_systemd_process_charts(ebpf_module_t *em)
+{
+ ebpf_cgroup_target_t *ect;
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_TASK_PROCESS);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_ps.create_process);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_TASK_THREAD);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_ps.create_thread);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_TASK_EXIT);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_ps.exit_call);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_TASK_CLOSE);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_ps.release_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_TASK_ERROR);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_ps.task_err);
+ }
+ }
+ write_end_chart();
+ }
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the structure with thread information
+*/
+static void ebpf_process_send_cgroup_data(ebpf_module_t *em)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ ebpf_process_sum_cgroup_pids(&ect->publish_systemd_ps, ect->pids);
+ }
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_process_charts(em);
+ }
+
+ ebpf_send_systemd_process_charts(em);
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_PROCESS_CHART) && ect->updated) {
+ ebpf_create_specific_process_charts(ect->name, em);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_PROCESS_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_PROCESS_CHART) {
+ if (ect->updated) {
+ ebpf_send_specific_process_data(ect->name, &ect->publish_systemd_ps, em);
+ } else {
+ ebpf_obsolete_specific_process_charts(ect->name, em);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_PROCESS_CHART;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Update Cgroup algorithm
+ *
+ * Change algorithm from absolute to incremental
+ */
+void ebpf_process_update_cgroup_algorithm()
+{
+ int i;
+ for (i = 0; i < NETDATA_KEY_PUBLISH_PROCESS_END; i++) {
+ netdata_publish_syscall_t *ptr = &process_publish_aggregated[i];
+ freez(ptr->algorithm);
+ ptr->algorithm = strdupz(ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX]);
+ }
+}
+
+/**
+ * Send Statistic Data
+ *
+ * Send statistic information to netdata.
+ */
+void ebpf_send_statistic_data()
+{
+ if (!publish_internal_metrics)
+ return;
+
+ write_begin_chart(NETDATA_MONITORING_FAMILY, NETDATA_EBPF_THREADS);
+ write_chart_dimension(threads_stat[NETDATA_EBPF_THREAD_STAT_TOTAL], (long long)plugin_statistics.threads);
+ write_chart_dimension(threads_stat[NETDATA_EBPF_THREAD_STAT_RUNNING], (long long)plugin_statistics.running);
+ write_end_chart();
+
+ write_begin_chart(NETDATA_MONITORING_FAMILY, NETDATA_EBPF_LOAD_METHOD);
+ write_chart_dimension(load_event_stat[NETDATA_EBPF_LOAD_STAT_LEGACY], (long long)plugin_statistics.legacy);
+ write_chart_dimension(load_event_stat[NETDATA_EBPF_LOAD_STAT_CORE], (long long)plugin_statistics.core);
+ write_end_chart();
+}
+
+/**
+ * Main loop for this collector.
+ *
+ * @param em the structure with thread information
+ */
+static void process_collector(ebpf_module_t *em)
+{
+ // Start cgroup integration before other threads
+ cgroup_thread.thread = mallocz(sizeof(netdata_thread_t));
+ cgroup_thread.start_routine = ebpf_cgroup_update_shm;
+
+ netdata_thread_create(cgroup_thread.thread, cgroup_thread.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_cgroup_update_shm, NULL);
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ int publish_global = em->global_charts;
+ int cgroups = em->cgroup_charts;
+ int thread_enabled = em->enabled;
+ if (cgroups)
+ ebpf_process_update_cgroup_algorithm();
+
+ int update_apps_every = (int) EBPF_CFG_UPDATE_APPS_EVERY_DEFAULT;
+ int pid_fd = process_maps[NETDATA_PROCESS_PID_TABLE].map_fd;
+ int update_every = em->update_every;
+ int counter = update_every - 1;
+ int update_apps_list = update_apps_every - 1;
+ while (!ebpf_exit_plugin) {
+ usec_t dt = heartbeat_next(&hb, USEC_PER_SEC);
+ (void)dt;
+ if (ebpf_exit_plugin)
+ break;
+
+ pthread_mutex_lock(&collect_data_mutex);
+ if (++update_apps_list == update_apps_every) {
+ update_apps_list = 0;
+ cleanup_exited_pids();
+ collect_data_for_all_processes(pid_fd);
+ }
+ pthread_mutex_unlock(&collect_data_mutex);
+
+ if (++counter == update_every) {
+ counter = 0;
+
+ read_hash_global_tables();
+
+ netdata_apps_integration_flags_t apps_enabled = em->apps_charts;
+ pthread_mutex_lock(&collect_data_mutex);
+
+ ebpf_create_apps_charts(apps_groups_root_target);
+ if (all_pids_count > 0) {
+ if (apps_enabled) {
+ ebpf_process_update_apps_data();
+ }
+
+ if (cgroups) {
+ ebpf_update_process_cgroup();
+ }
+ }
+
+ pthread_mutex_lock(&lock);
+ ebpf_send_statistic_data();
+
+ if (thread_enabled) {
+ if (publish_global) {
+ ebpf_process_send_data(em);
+ }
+
+ if (apps_enabled & NETDATA_EBPF_APPS_FLAG_CHART_CREATED) {
+ ebpf_process_send_apps_data(apps_groups_root_target, em);
+ }
+
+ if (cgroups) {
+ ebpf_process_send_cgroup_data(em);
+ }
+ }
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+
+ fflush(stdout);
+ }
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO START THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Allocate vectors used with this thread.
+ * We are not testing the return, because callocz does this and shutdown the software
+ * case it was not possible to allocate.
+ *
+ * @param length is the length for the vectors used inside the collector.
+ */
+static void ebpf_process_allocate_global_vectors(size_t length)
+{
+ memset(process_aggregated_data, 0, length * sizeof(netdata_syscall_stat_t));
+ memset(process_publish_aggregated, 0, length * sizeof(netdata_publish_syscall_t));
+ process_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t));
+
+ global_process_stats = callocz((size_t)pid_max, sizeof(ebpf_process_stat_t *));
+ current_apps_data = callocz((size_t)pid_max, sizeof(ebpf_process_publish_apps_t *));
+}
+
+static void change_syscalls()
+{
+ static char *lfork = { "do_fork" };
+ process_id_names[NETDATA_KEY_PUBLISH_PROCESS_FORK] = lfork;
+}
+
+/**
+ * Set local variables
+ *
+ */
+static void set_local_pointers()
+{
+ if (isrh >= NETDATA_MINIMUM_RH_VERSION && isrh < NETDATA_RH_8)
+ change_syscalls();
+}
+
+/*****************************************************************
+ *
+ * EBPF PROCESS THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Enable tracepoints
+ *
+ * Enable necessary tracepoints for thread.
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+static int ebpf_process_enable_tracepoints()
+{
+ int test = ebpf_is_tracepoint_enabled(tracepoint_sched_type, tracepoint_sched_process_exit);
+ if (test == -1)
+ return -1;
+ else if (!test) {
+ if (ebpf_enable_tracing_values(tracepoint_sched_type, tracepoint_sched_process_exit))
+ return -1;
+ }
+ was_sched_process_exit_enabled = test;
+
+ test = ebpf_is_tracepoint_enabled(tracepoint_sched_type, tracepoint_sched_process_exec);
+ if (test == -1)
+ return -1;
+ else if (!test) {
+ if (ebpf_enable_tracing_values(tracepoint_sched_type, tracepoint_sched_process_exec))
+ return -1;
+ }
+ was_sched_process_exec_enabled = test;
+
+ test = ebpf_is_tracepoint_enabled(tracepoint_sched_type, tracepoint_sched_process_fork);
+ if (test == -1)
+ return -1;
+ else if (!test) {
+ if (ebpf_enable_tracing_values(tracepoint_sched_type, tracepoint_sched_process_fork))
+ return -1;
+ }
+ was_sched_process_fork_enabled = test;
+
+ return 0;
+}
+
+/**
+ * Process thread
+ *
+ * Thread used to generate process charts.
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always return NULL
+ */
+void *ebpf_process_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_process_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = process_maps;
+
+ if (ebpf_process_enable_tracepoints()) {
+ em->enabled = em->global_charts = em->apps_charts = em->cgroup_charts = CONFIG_BOOLEAN_NO;
+ }
+ process_enabled = em->enabled;
+
+ pthread_mutex_lock(&lock);
+ ebpf_process_allocate_global_vectors(NETDATA_KEY_PUBLISH_PROCESS_END);
+
+ ebpf_update_pid_table(&process_maps[0], em);
+
+ set_local_pointers();
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->enabled = CONFIG_BOOLEAN_NO;
+ pthread_mutex_unlock(&lock);
+ goto endprocess;
+ }
+
+ int algorithms[NETDATA_KEY_PUBLISH_PROCESS_END] = {
+ NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX
+ };
+
+ ebpf_global_labels(
+ process_aggregated_data, process_publish_aggregated, process_dimension_names, process_id_names,
+ algorithms, NETDATA_KEY_PUBLISH_PROCESS_END);
+
+ if (process_enabled) {
+ ebpf_create_global_charts(em);
+ }
+
+ ebpf_update_stats(&plugin_statistics, em);
+ ebpf_create_statistic_charts(em);
+
+ pthread_mutex_unlock(&lock);
+
+ process_collector(em);
+
+endprocess:
+ if (!em->enabled)
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_process.h b/collectors/ebpf.plugin/ebpf_process.h
new file mode 100644
index 0000000..43df34d
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_process.h
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_PROCESS_H
+#define NETDATA_EBPF_PROCESS_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_PROCESS "process"
+
+// Groups used on Dashboard
+#define NETDATA_PROCESS_GROUP "processes"
+#define NETDATA_PROCESS_CGROUP_GROUP "processes (eBPF)"
+
+// Global chart name
+#define NETDATA_EXIT_SYSCALL "exit"
+#define NETDATA_PROCESS_SYSCALL "process_thread"
+#define NETDATA_PROCESS_ERROR_NAME "task_error"
+#define NETDATA_PROCESS_STATUS_NAME "process_status"
+
+// Charts created on Apps submenu
+#define NETDATA_SYSCALL_APPS_TASK_PROCESS "process_create"
+#define NETDATA_SYSCALL_APPS_TASK_THREAD "thread_create"
+#define NETDATA_SYSCALL_APPS_TASK_EXIT "task_exit"
+#define NETDATA_SYSCALL_APPS_TASK_CLOSE "task_close"
+#define NETDATA_SYSCALL_APPS_TASK_ERROR "task_error"
+
+// Process configuration name
+#define NETDATA_PROCESS_CONFIG_FILE "process.conf"
+
+// Contexts
+#define NETDATA_CGROUP_PROCESS_CREATE_CONTEXT "cgroup.process_create"
+#define NETDATA_CGROUP_THREAD_CREATE_CONTEXT "cgroup.thread_create"
+#define NETDATA_CGROUP_PROCESS_CLOSE_CONTEXT "cgroup.task_close"
+#define NETDATA_CGROUP_PROCESS_EXIT_CONTEXT "cgroup.task_exit"
+#define NETDATA_CGROUP_PROCESS_ERROR_CONTEXT "cgroup.task_error"
+
+#define NETDATA_SYSTEMD_PROCESS_CREATE_CONTEXT "services.process_create"
+#define NETDATA_SYSTEMD_THREAD_CREATE_CONTEXT "services.thread_create"
+#define NETDATA_SYSTEMD_PROCESS_CLOSE_CONTEXT "services.task_close"
+#define NETDATA_SYSTEMD_PROCESS_EXIT_CONTEXT "services.task_exit"
+#define NETDATA_SYSTEMD_PROCESS_ERROR_CONTEXT "services.task_error"
+
+#define NETDATA_EBPF_CGROUP_UPDATE 10
+
+// Statistical information
+enum netdata_ebpf_thread_stats{
+ NETDATA_EBPF_THREAD_STAT_TOTAL,
+ NETDATA_EBPF_THREAD_STAT_RUNNING,
+
+ NETDATA_EBPF_THREAD_STAT_END
+};
+
+enum netdata_ebpf_load_mode_stats{
+ NETDATA_EBPF_LOAD_STAT_LEGACY,
+ NETDATA_EBPF_LOAD_STAT_CORE,
+
+ NETDATA_EBPF_LOAD_STAT_END
+};
+
+// Index from kernel
+typedef enum ebpf_process_index {
+ NETDATA_KEY_CALLS_DO_EXIT,
+
+ NETDATA_KEY_CALLS_RELEASE_TASK,
+
+ NETDATA_KEY_CALLS_DO_FORK,
+ NETDATA_KEY_ERROR_DO_FORK,
+
+ NETDATA_KEY_CALLS_SYS_CLONE,
+ NETDATA_KEY_ERROR_SYS_CLONE,
+
+ NETDATA_KEY_END_VECTOR
+} ebpf_process_index_t;
+
+// This enum acts as an index for publish vector.
+// Do not change the enum order because we use
+// different algorithms to make charts with incremental
+// values (the three initial positions) and absolute values
+// (the remaining charts).
+typedef enum netdata_publish_process {
+ NETDATA_KEY_PUBLISH_PROCESS_EXIT,
+ NETDATA_KEY_PUBLISH_PROCESS_RELEASE_TASK,
+ NETDATA_KEY_PUBLISH_PROCESS_FORK,
+ NETDATA_KEY_PUBLISH_PROCESS_CLONE,
+
+ NETDATA_KEY_PUBLISH_PROCESS_END
+} netdata_publish_process_t;
+
+typedef struct ebpf_process_publish_apps {
+ // Number of calls during the last read
+ uint64_t call_do_exit;
+ uint64_t call_release_task;
+ uint64_t create_process;
+ uint64_t create_thread;
+
+ // Number of errors during the last read
+ uint64_t task_err;
+} ebpf_process_publish_apps_t;
+
+enum ebpf_process_tables {
+ NETDATA_PROCESS_PID_TABLE,
+ NETDATA_PROCESS_GLOBAL_TABLE,
+ NETDATA_PROCESS_CTRL_TABLE
+};
+
+extern struct config process_config;
+
+#endif /* NETDATA_EBPF_PROCESS_H */
diff --git a/collectors/ebpf.plugin/ebpf_shm.c b/collectors/ebpf.plugin/ebpf_shm.c
new file mode 100644
index 0000000..f81287d
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_shm.c
@@ -0,0 +1,1137 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_shm.h"
+
+static char *shm_dimension_name[NETDATA_SHM_END] = { "get", "at", "dt", "ctl" };
+static netdata_syscall_stat_t shm_aggregated_data[NETDATA_SHM_END];
+static netdata_publish_syscall_t shm_publish_aggregated[NETDATA_SHM_END];
+
+netdata_publish_shm_t *shm_vector = NULL;
+
+static netdata_idx_t shm_hash_values[NETDATA_SHM_END];
+static netdata_idx_t *shm_values = NULL;
+
+netdata_publish_shm_t **shm_pid = NULL;
+
+struct config shm_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+static ebpf_local_maps_t shm_maps[] = {{.name = "tbl_pid_shm", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "shm_ctrl", .internal_input = NETDATA_CONTROLLER_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_shm", .internal_input = NETDATA_SHM_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0}};
+
+struct netdata_static_thread shm_threads = {
+ .name = "SHM KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+netdata_ebpf_targets_t shm_targets[] = { {.name = "shmget", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "shmat", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "shmdt", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "shmctl", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/shm.skel.h"
+
+static struct shm_bpf *bpf_obj = NULL;
+
+/*****************************************************************
+ *
+ * BTF FUNCTIONS
+ *
+ *****************************************************************/
+
+/*
+ * Disable tracepoint
+ *
+ * Disable all tracepoints to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_shm_disable_tracepoint(struct shm_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_syscall_shmget, false);
+ bpf_program__set_autoload(obj->progs.netdata_syscall_shmat, false);
+ bpf_program__set_autoload(obj->progs.netdata_syscall_shmdt, false);
+ bpf_program__set_autoload(obj->progs.netdata_syscall_shmctl, false);
+}
+
+/*
+ * Disable probe
+ *
+ * Disable all probes to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_disable_probe(struct shm_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_shmget_probe, false);
+ bpf_program__set_autoload(obj->progs.netdata_shmat_probe, false);
+ bpf_program__set_autoload(obj->progs.netdata_shmdt_probe, false);
+ bpf_program__set_autoload(obj->progs.netdata_shmctl_probe, false);
+ bpf_program__set_autoload(obj->progs.netdata_shm_release_task_probe, false);
+}
+
+/*
+ * Disable trampoline
+ *
+ * Disable all trampoline to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_disable_trampoline(struct shm_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_shmget_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_shmat_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_shmdt_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_shmctl_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_shm_release_task_fentry, false);
+}
+
+/**
+ * Set trampoline target
+ *
+ * Set the targets we will monitor.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_set_trampoline_target(struct shm_bpf *obj)
+{
+ char syscall[NETDATA_EBPF_MAX_SYSCALL_LENGTH + 1];
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ shm_targets[NETDATA_KEY_SHMGET_CALL].name, running_on_kernel);
+
+ bpf_program__set_attach_target(obj->progs.netdata_shmget_fentry, 0,
+ syscall);
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ shm_targets[NETDATA_KEY_SHMAT_CALL].name, running_on_kernel);
+ bpf_program__set_attach_target(obj->progs.netdata_shmat_fentry, 0,
+ syscall);
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ shm_targets[NETDATA_KEY_SHMDT_CALL].name, running_on_kernel);
+ bpf_program__set_attach_target(obj->progs.netdata_shmdt_fentry, 0,
+ syscall);
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ shm_targets[NETDATA_KEY_SHMCTL_CALL].name, running_on_kernel);
+ bpf_program__set_attach_target(obj->progs.netdata_shmctl_fentry, 0,
+ syscall);
+
+ bpf_program__set_attach_target(obj->progs.netdata_shm_release_task_fentry, 0,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+}
+
+/**
+ * SHM Attach Probe
+ *
+ * Attach probes to target
+ *
+ * @param obj is the main structure for bpf objects.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+static int ebpf_shm_attach_probe(struct shm_bpf *obj)
+{
+ char syscall[NETDATA_EBPF_MAX_SYSCALL_LENGTH + 1];
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ shm_targets[NETDATA_KEY_SHMGET_CALL].name, running_on_kernel);
+
+ obj->links.netdata_shmget_probe = bpf_program__attach_kprobe(obj->progs.netdata_shmget_probe,
+ false, syscall);
+ int ret = (int)libbpf_get_error(obj->links.netdata_shmget_probe);
+ if (ret)
+ return -1;
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ shm_targets[NETDATA_KEY_SHMAT_CALL].name, running_on_kernel);
+ obj->links.netdata_shmat_probe = bpf_program__attach_kprobe(obj->progs.netdata_shmat_probe,
+ false, syscall);
+ ret = (int)libbpf_get_error(obj->links.netdata_shmat_probe);
+ if (ret)
+ return -1;
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ shm_targets[NETDATA_KEY_SHMDT_CALL].name, running_on_kernel);
+ obj->links.netdata_shmdt_probe = bpf_program__attach_kprobe(obj->progs.netdata_shmdt_probe,
+ false, syscall);
+ ret = (int)libbpf_get_error(obj->links.netdata_shmdt_probe);
+ if (ret)
+ return -1;
+
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH,
+ shm_targets[NETDATA_KEY_SHMCTL_CALL].name, running_on_kernel);
+ obj->links.netdata_shmctl_probe = bpf_program__attach_kprobe(obj->progs.netdata_shmctl_probe,
+ false, syscall);
+ ret = (int)libbpf_get_error(obj->links.netdata_shmctl_probe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_shm_release_task_probe = bpf_program__attach_kprobe(obj->progs.netdata_shm_release_task_probe,
+ false, EBPF_COMMON_FNCT_CLEAN_UP);
+ ret = (int)libbpf_get_error(obj->links.netdata_shm_release_task_probe);
+ if (ret)
+ return -1;
+
+
+ return 0;
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ */
+static void ebpf_shm_set_hash_tables(struct shm_bpf *obj)
+{
+ shm_maps[NETDATA_PID_SHM_TABLE].map_fd = bpf_map__fd(obj->maps.tbl_pid_shm);
+ shm_maps[NETDATA_SHM_CONTROLLER].map_fd = bpf_map__fd(obj->maps.shm_ctrl);
+ shm_maps[NETDATA_SHM_GLOBAL_TABLE].map_fd = bpf_map__fd(obj->maps.tbl_shm);
+}
+
+/**
+ * Disable Release Task
+ *
+ * Disable release task when apps is not enabled.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_shm_disable_release_task(struct shm_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_shm_release_task_probe, false);
+ bpf_program__set_autoload(obj->progs.netdata_shm_release_task_fentry, false);
+}
+
+/**
+ * Adjust Map Size
+ *
+ * Resize maps according input from users.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ */
+static void ebpf_shm_adjust_map_size(struct shm_bpf *obj, ebpf_module_t *em)
+{
+ ebpf_update_map_size(obj->maps.tbl_pid_shm, &shm_maps[NETDATA_PID_SHM_TABLE],
+ em, bpf_map__name(obj->maps.tbl_pid_shm));
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_shm_load_and_attach(struct shm_bpf *obj, ebpf_module_t *em)
+{
+ netdata_ebpf_targets_t *shmt = em->targets;
+ netdata_ebpf_program_loaded_t test = shmt[NETDATA_KEY_SHMGET_CALL].mode;
+
+ // We are testing only one, because all will have the same behavior
+ if (test == EBPF_LOAD_TRAMPOLINE ) {
+ ebpf_shm_disable_tracepoint(obj);
+ ebpf_disable_probe(obj);
+
+ ebpf_set_trampoline_target(obj);
+ } else if (test == EBPF_LOAD_PROBE || test == EBPF_LOAD_RETPROBE ) {
+ ebpf_shm_disable_tracepoint(obj);
+ ebpf_disable_trampoline(obj);
+ } else {
+ ebpf_disable_probe(obj);
+ ebpf_disable_trampoline(obj);
+ }
+
+ ebpf_shm_adjust_map_size(obj, em);
+ if (!em->apps_charts && !em->cgroup_charts)
+ ebpf_shm_disable_release_task(obj);
+
+ int ret = shm_bpf__load(obj);
+ if (!ret) {
+ if (test != EBPF_LOAD_PROBE && test != EBPF_LOAD_RETPROBE)
+ shm_bpf__attach(obj);
+ else
+ ret = ebpf_shm_attach_probe(obj);
+
+ if (!ret)
+ ebpf_shm_set_hash_tables(obj);
+ }
+
+ return ret;
+}
+#endif
+/*****************************************************************
+ * FUNCTIONS TO CLOSE THE THREAD
+ *****************************************************************/
+
+/**
+ * SHM Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_shm_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ ebpf_cleanup_publish_syscall(shm_publish_aggregated);
+
+ freez(shm_vector);
+ freez(shm_values);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ if (bpf_obj)
+ shm_bpf__destroy(bpf_obj);
+#endif
+
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+}
+
+/**
+ * SHM Exit
+ *
+ * Cancel child thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_shm_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*shm_threads.thread);
+ ebpf_shm_free(em);
+}
+
+/**
+ * SHM Cleanup
+ *
+ * Clean up allocated memory.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_shm_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_shm_free(em);
+}
+
+/*****************************************************************
+ * COLLECTOR THREAD
+ *****************************************************************/
+
+/**
+ * Apps Accumulator
+ *
+ * Sum all values read from kernel and store in the first address.
+ *
+ * @param out the vector with read values.
+ */
+static void shm_apps_accumulator(netdata_publish_shm_t *out)
+{
+ int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1;
+ netdata_publish_shm_t *total = &out[0];
+ for (i = 1; i < end; i++) {
+ netdata_publish_shm_t *w = &out[i];
+ total->get += w->get;
+ total->at += w->at;
+ total->dt += w->dt;
+ total->ctl += w->ctl;
+ }
+}
+
+/**
+ * Fill PID
+ *
+ * Fill PID structures
+ *
+ * @param current_pid pid that we are collecting data
+ * @param out values read from hash tables;
+ */
+static void shm_fill_pid(uint32_t current_pid, netdata_publish_shm_t *publish)
+{
+ netdata_publish_shm_t *curr = shm_pid[current_pid];
+ if (!curr) {
+ curr = callocz(1, sizeof(netdata_publish_shm_t));
+ shm_pid[current_pid] = curr;
+ }
+
+ memcpy(curr, publish, sizeof(netdata_publish_shm_t));
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ */
+static void ebpf_update_shm_cgroup()
+{
+ netdata_publish_shm_t *cv = shm_vector;
+ int fd = shm_maps[NETDATA_PID_SHM_TABLE].map_fd;
+ size_t length = sizeof(netdata_publish_shm_t) * ebpf_nprocs;
+ ebpf_cgroup_target_t *ect;
+
+ memset(cv, 0, length);
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ int pid = pids->pid;
+ netdata_publish_shm_t *out = &pids->shm;
+ if (likely(shm_pid) && shm_pid[pid]) {
+ netdata_publish_shm_t *in = shm_pid[pid];
+
+ memcpy(out, in, sizeof(netdata_publish_shm_t));
+ } else {
+ if (!bpf_map_lookup_elem(fd, &pid, cv)) {
+ shm_apps_accumulator(cv);
+
+ memcpy(out, cv, sizeof(netdata_publish_shm_t));
+
+ // now that we've consumed the value, zero it out in the map.
+ memset(cv, 0, length);
+ bpf_map_update_elem(fd, &pid, cv, BPF_EXIST);
+ }
+ }
+ }
+ }
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Read APPS table
+ *
+ * Read the apps table and store data inside the structure.
+ */
+static void read_apps_table()
+{
+ netdata_publish_shm_t *cv = shm_vector;
+ uint32_t key;
+ struct pid_stat *pids = root_of_pids;
+ int fd = shm_maps[NETDATA_PID_SHM_TABLE].map_fd;
+ size_t length = sizeof(netdata_publish_shm_t)*ebpf_nprocs;
+ while (pids) {
+ key = pids->pid;
+
+ if (bpf_map_lookup_elem(fd, &key, cv)) {
+ pids = pids->next;
+ continue;
+ }
+
+ shm_apps_accumulator(cv);
+
+ shm_fill_pid(key, cv);
+
+ // now that we've consumed the value, zero it out in the map.
+ memset(cv, 0, length);
+ bpf_map_update_elem(fd, &key, cv, BPF_EXIST);
+
+ pids = pids->next;
+ }
+}
+
+/**
+* Send global charts to netdata agent.
+*/
+static void shm_send_global()
+{
+ write_begin_chart(NETDATA_EBPF_SYSTEM_GROUP, NETDATA_SHM_GLOBAL_CHART);
+ write_chart_dimension(
+ shm_publish_aggregated[NETDATA_KEY_SHMGET_CALL].dimension,
+ (long long) shm_hash_values[NETDATA_KEY_SHMGET_CALL]
+ );
+ write_chart_dimension(
+ shm_publish_aggregated[NETDATA_KEY_SHMAT_CALL].dimension,
+ (long long) shm_hash_values[NETDATA_KEY_SHMAT_CALL]
+ );
+ write_chart_dimension(
+ shm_publish_aggregated[NETDATA_KEY_SHMDT_CALL].dimension,
+ (long long) shm_hash_values[NETDATA_KEY_SHMDT_CALL]
+ );
+ write_chart_dimension(
+ shm_publish_aggregated[NETDATA_KEY_SHMCTL_CALL].dimension,
+ (long long) shm_hash_values[NETDATA_KEY_SHMCTL_CALL]
+ );
+ write_end_chart();
+}
+
+/**
+ * Read global counter
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_global_table()
+{
+ netdata_idx_t *stored = shm_values;
+ netdata_idx_t *val = shm_hash_values;
+ int fd = shm_maps[NETDATA_SHM_GLOBAL_TABLE].map_fd;
+
+ uint32_t i, end = NETDATA_SHM_END;
+ for (i = NETDATA_KEY_SHMGET_CALL; i < end; i++) {
+ if (!bpf_map_lookup_elem(fd, &i, stored)) {
+ int j;
+ int last = ebpf_nprocs;
+ netdata_idx_t total = 0;
+ for (j = 0; j < last; j++)
+ total += stored[j];
+
+ val[i] = total;
+ }
+ }
+}
+
+/**
+ * Shared memory reader thread.
+ *
+ * @param ptr It is a NULL value for this thread.
+ * @return It always returns NULL.
+ */
+void *ebpf_shm_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_shm_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ usec_t step = NETDATA_SHM_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_global_table();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Sum values for all targets.
+ */
+static void ebpf_shm_sum_pids(netdata_publish_shm_t *shm, struct pid_on_target *root)
+{
+ while (root) {
+ int32_t pid = root->pid;
+ netdata_publish_shm_t *w = shm_pid[pid];
+ if (w) {
+ shm->get += w->get;
+ shm->at += w->at;
+ shm->dt += w->dt;
+ shm->ctl += w->ctl;
+
+ // reset for next collection.
+ w->get = 0;
+ w->at = 0;
+ w->dt = 0;
+ w->ctl = 0;
+ }
+ root = root->next;
+ }
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param root the target list.
+*/
+void ebpf_shm_send_apps_data(struct target *root)
+{
+ struct target *w;
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ ebpf_shm_sum_pids(&w->shm, w->root_pid);
+ }
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SHMGET_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, (long long) w->shm.get);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SHMAT_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, (long long) w->shm.at);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SHMDT_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, (long long) w->shm.dt);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SHMCTL_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, (long long) w->shm.ctl);
+ }
+ }
+ write_end_chart();
+}
+
+/**
+ * Sum values for all targets.
+ */
+static void ebpf_shm_sum_cgroup_pids(netdata_publish_shm_t *shm, struct pid_on_target2 *root)
+{
+ netdata_publish_shm_t shmv;
+ memset(&shmv, 0, sizeof(shmv));
+ while (root) {
+ netdata_publish_shm_t *w = &root->shm;
+ shmv.get += w->get;
+ shmv.at += w->at;
+ shmv.dt += w->dt;
+ shmv.ctl += w->ctl;
+
+ root = root->next;
+ }
+
+ memcpy(shm, &shmv, sizeof(shmv));
+}
+
+/**
+ * Create specific shared memory charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_specific_shm_charts(char *type, int update_every)
+{
+ ebpf_create_chart(type, NETDATA_SHMGET_CHART,
+ "Calls to syscall <code>shmget(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_CGROUP_SHM_GET_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5800,
+ ebpf_create_global_dimension,
+ &shm_publish_aggregated[NETDATA_KEY_SHMGET_CALL],
+ 1,
+ update_every,
+ NETDATA_EBPF_MODULE_NAME_SHM);
+
+ ebpf_create_chart(type, NETDATA_SHMAT_CHART,
+ "Calls to syscall <code>shmat(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_CGROUP_SHM_AT_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5801,
+ ebpf_create_global_dimension,
+ &shm_publish_aggregated[NETDATA_KEY_SHMAT_CALL],
+ 1,
+ update_every,
+ NETDATA_EBPF_MODULE_NAME_SHM);
+
+ ebpf_create_chart(type, NETDATA_SHMDT_CHART,
+ "Calls to syscall <code>shmdt(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_CGROUP_SHM_DT_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5802,
+ ebpf_create_global_dimension,
+ &shm_publish_aggregated[NETDATA_KEY_SHMDT_CALL],
+ 1,
+ update_every,
+ NETDATA_EBPF_MODULE_NAME_SHM);
+
+ ebpf_create_chart(type, NETDATA_SHMCTL_CHART,
+ "Calls to syscall <code>shmctl(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_CGROUP_SHM_CTL_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5803,
+ ebpf_create_global_dimension,
+ &shm_publish_aggregated[NETDATA_KEY_SHMCTL_CALL],
+ 1,
+ update_every,
+ NETDATA_EBPF_MODULE_NAME_SHM);
+}
+
+/**
+ * Obsolete specific shared memory charts
+ *
+ * Obsolete charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_obsolete_specific_shm_charts(char *type, int update_every)
+{
+ ebpf_write_chart_obsolete(type, NETDATA_SHMGET_CHART,
+ "Calls to syscall <code>shmget(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_SHM_GET_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5800, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SHMAT_CHART,
+ "Calls to syscall <code>shmat(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_SHM_AT_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5801, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SHMDT_CHART,
+ "Calls to syscall <code>shmdt(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_SHM_DT_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5802, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SHMCTL_CHART,
+ "Calls to syscall <code>shmctl(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_SHM_CTL_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5803, update_every);
+}
+
+/**
+ * Create Systemd Swap Charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ **/
+static void ebpf_create_systemd_shm_charts(int update_every)
+{
+ ebpf_create_charts_on_systemd(NETDATA_SHMGET_CHART,
+ "Calls to syscall <code>shmget(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20191,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SYSTEMD_SHM_GET_CONTEXT, NETDATA_EBPF_MODULE_NAME_SHM, update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SHMAT_CHART,
+ "Calls to syscall <code>shmat(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20192,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SYSTEMD_SHM_AT_CONTEXT, NETDATA_EBPF_MODULE_NAME_SHM, update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SHMDT_CHART,
+ "Calls to syscall <code>shmdt(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20193,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SYSTEMD_SHM_DT_CONTEXT, NETDATA_EBPF_MODULE_NAME_SHM, update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SHMCTL_CHART,
+ "Calls to syscall <code>shmctl(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20193,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SYSTEMD_SHM_CTL_CONTEXT, NETDATA_EBPF_MODULE_NAME_SHM, update_every);
+}
+
+/**
+ * Send Systemd charts
+ *
+ * Send collected data to Netdata.
+ */
+static void ebpf_send_systemd_shm_charts()
+{
+ ebpf_cgroup_target_t *ect;
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SHMGET_CHART);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_shm.get);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SHMAT_CHART);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_shm.at);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SHMDT_CHART);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_shm.dt);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SHMCTL_CHART);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_shm.ctl);
+ }
+ }
+ write_end_chart();
+}
+
+/*
+ * Send Specific Shared memory data
+ *
+ * Send data for specific cgroup/apps.
+ *
+ * @param type chart type
+ * @param values structure with values that will be sent to netdata
+ */
+static void ebpf_send_specific_shm_data(char *type, netdata_publish_shm_t *values)
+{
+ write_begin_chart(type, NETDATA_SHMGET_CHART);
+ write_chart_dimension(shm_publish_aggregated[NETDATA_KEY_SHMGET_CALL].name, (long long)values->get);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SHMAT_CHART);
+ write_chart_dimension(shm_publish_aggregated[NETDATA_KEY_SHMAT_CALL].name, (long long)values->at);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SHMDT_CHART);
+ write_chart_dimension(shm_publish_aggregated[NETDATA_KEY_SHMDT_CALL].name, (long long)values->dt);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SHMCTL_CHART);
+ write_chart_dimension(shm_publish_aggregated[NETDATA_KEY_SHMCTL_CALL].name, (long long)values->ctl);
+ write_end_chart();
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+*/
+void ebpf_shm_send_cgroup_data(int update_every)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ ebpf_shm_sum_cgroup_pids(&ect->publish_shm, ect->pids);
+ }
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_shm_charts(update_every);
+ }
+
+ ebpf_send_systemd_shm_charts();
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_SHM_CHART) && ect->updated) {
+ ebpf_create_specific_shm_charts(ect->name, update_every);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_SHM_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_SHM_CHART) {
+ if (ect->updated) {
+ ebpf_send_specific_shm_data(ect->name, &ect->publish_shm);
+ } else {
+ ebpf_obsolete_specific_shm_charts(ect->name, update_every);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_SWAP_CHART;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+* Main loop for this collector.
+*/
+static void shm_collector(ebpf_module_t *em)
+{
+ shm_threads.thread = mallocz(sizeof(netdata_thread_t));
+ shm_threads.start_routine = ebpf_shm_read_hash;
+
+ netdata_thread_create(
+ shm_threads.thread,
+ shm_threads.name,
+ NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_shm_read_hash,
+ em
+ );
+
+ int cgroups = em->cgroup_charts;
+ int update_every = em->update_every;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ netdata_apps_integration_flags_t apps = em->apps_charts;
+ pthread_mutex_lock(&collect_data_mutex);
+ if (apps) {
+ read_apps_table();
+ }
+
+ if (cgroups) {
+ ebpf_update_shm_cgroup();
+ }
+
+ pthread_mutex_lock(&lock);
+
+ shm_send_global();
+
+ if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED) {
+ ebpf_shm_send_apps_data(apps_groups_root_target);
+ }
+
+ if (cgroups) {
+ ebpf_shm_send_cgroup_data(update_every);
+ }
+
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+}
+
+/*****************************************************************
+ * INITIALIZE THREAD
+ *****************************************************************/
+
+/**
+ * Create apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+void ebpf_shm_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+ ebpf_create_charts_on_apps(NETDATA_SHMGET_CHART,
+ "Calls to syscall <code>shmget(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20191,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SHM);
+
+ ebpf_create_charts_on_apps(NETDATA_SHMAT_CHART,
+ "Calls to syscall <code>shmat(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20192,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SHM);
+
+ ebpf_create_charts_on_apps(NETDATA_SHMDT_CHART,
+ "Calls to syscall <code>shmdt(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20193,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SHM);
+
+ ebpf_create_charts_on_apps(NETDATA_SHMCTL_CHART,
+ "Calls to syscall <code>shmctl(2)</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_IPC_SHM_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20194,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SHM);
+
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/**
+ * Allocate vectors used with this thread.
+ *
+ * We are not testing the return, because callocz does this and shutdown the software
+ * case it was not possible to allocate.
+ *
+ * @param apps is apps enabled?
+ */
+static void ebpf_shm_allocate_global_vectors(int apps)
+{
+ if (apps)
+ shm_pid = callocz((size_t)pid_max, sizeof(netdata_publish_shm_t *));
+
+ shm_vector = callocz((size_t)ebpf_nprocs, sizeof(netdata_publish_shm_t));
+
+ shm_values = callocz((size_t)ebpf_nprocs, sizeof(netdata_idx_t));
+
+ memset(shm_hash_values, 0, sizeof(shm_hash_values));
+}
+
+/*****************************************************************
+ * MAIN THREAD
+ *****************************************************************/
+
+/**
+ * Create global charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_shm_charts(int update_every)
+{
+ ebpf_create_chart(
+ NETDATA_EBPF_SYSTEM_GROUP,
+ NETDATA_SHM_GLOBAL_CHART,
+ "Calls to shared memory system calls",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_SYSTEM_IPC_SHM_SUBMENU,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_SYSTEM_IPC_SHARED_MEM_CALLS,
+ ebpf_create_global_dimension,
+ shm_publish_aggregated,
+ NETDATA_SHM_END,
+ update_every, NETDATA_EBPF_MODULE_NAME_SHM
+ );
+
+ fflush(stdout);
+}
+
+/*
+ * Load BPF
+ *
+ * Load BPF files.
+ *
+ * @param em the structure with configuration
+ */
+static int ebpf_shm_load_bpf(ebpf_module_t *em)
+{
+ int ret = 0;
+
+ ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_KEY_SHMGET_CALL].mode);
+ if (em->load & EBPF_LOAD_LEGACY) {
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->enabled = CONFIG_BOOLEAN_NO;
+ ret = -1;
+ }
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ bpf_obj = shm_bpf__open();
+ if (!bpf_obj)
+ ret = -1;
+ else
+ ret = ebpf_shm_load_and_attach(bpf_obj, em);
+ }
+#endif
+
+
+ if (ret)
+ error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->thread_name);
+
+ return ret;
+}
+
+/**
+ * Shared memory thread.
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ * @return It always return NULL
+ */
+void *ebpf_shm_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_shm_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = shm_maps;
+
+ ebpf_update_pid_table(&shm_maps[NETDATA_PID_SHM_TABLE], em);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_shm_load_bpf(em)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endshm;
+ }
+
+ ebpf_shm_allocate_global_vectors(em->apps_charts);
+
+ int algorithms[NETDATA_SHM_END] = {
+ NETDATA_EBPF_INCREMENTAL_IDX,
+ NETDATA_EBPF_INCREMENTAL_IDX,
+ NETDATA_EBPF_INCREMENTAL_IDX,
+ NETDATA_EBPF_INCREMENTAL_IDX
+ };
+ ebpf_global_labels(
+ shm_aggregated_data,
+ shm_publish_aggregated,
+ shm_dimension_name,
+ shm_dimension_name,
+ algorithms,
+ NETDATA_SHM_END
+ );
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_shm_charts(em->update_every);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ shm_collector(em);
+
+endshm:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_shm.h b/collectors/ebpf.plugin/ebpf_shm.h
new file mode 100644
index 0000000..4e06881
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_shm.h
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_SHM_H
+#define NETDATA_EBPF_SHM_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_SHM "shm"
+
+#define NETDATA_SHM_SLEEP_MS 850000ULL
+
+// charts
+#define NETDATA_SHM_GLOBAL_CHART "shared_memory_calls"
+#define NETDATA_SHMGET_CHART "shmget_call"
+#define NETDATA_SHMAT_CHART "shmat_call"
+#define NETDATA_SHMDT_CHART "shmdt_call"
+#define NETDATA_SHMCTL_CHART "shmctl_call"
+
+// configuration file
+#define NETDATA_DIRECTORY_SHM_CONFIG_FILE "shm.conf"
+
+// Contexts
+#define NETDATA_CGROUP_SHM_GET_CONTEXT "cgroup.shmget"
+#define NETDATA_CGROUP_SHM_AT_CONTEXT "cgroup.shmat"
+#define NETDATA_CGROUP_SHM_DT_CONTEXT "cgroup.shmdt"
+#define NETDATA_CGROUP_SHM_CTL_CONTEXT "cgroup.shmctl"
+
+#define NETDATA_SYSTEMD_SHM_GET_CONTEXT "services.shmget"
+#define NETDATA_SYSTEMD_SHM_AT_CONTEXT "services.shmat"
+#define NETDATA_SYSTEMD_SHM_DT_CONTEXT "services.shmdt"
+#define NETDATA_SYSTEMD_SHM_CTL_CONTEXT "services.shmctl"
+
+typedef struct netdata_publish_shm {
+ uint64_t get;
+ uint64_t at;
+ uint64_t dt;
+ uint64_t ctl;
+} netdata_publish_shm_t;
+
+enum shm_tables {
+ NETDATA_PID_SHM_TABLE,
+ NETDATA_SHM_CONTROLLER,
+ NETDATA_SHM_GLOBAL_TABLE
+};
+
+enum shm_counters {
+ NETDATA_KEY_SHMGET_CALL,
+ NETDATA_KEY_SHMAT_CALL,
+ NETDATA_KEY_SHMDT_CALL,
+ NETDATA_KEY_SHMCTL_CALL,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_SHM_END
+};
+
+extern netdata_publish_shm_t **shm_pid;
+
+void *ebpf_shm_thread(void *ptr);
+void ebpf_shm_create_apps_charts(struct ebpf_module *em, void *ptr);
+extern netdata_ebpf_targets_t shm_targets[];
+
+extern struct config shm_config;
+
+#endif
diff --git a/collectors/ebpf.plugin/ebpf_socket.c b/collectors/ebpf.plugin/ebpf_socket.c
new file mode 100644
index 0000000..3a023e4
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_socket.c
@@ -0,0 +1,3976 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <sys/resource.h>
+
+#include "ebpf.h"
+#include "ebpf_socket.h"
+
+/*****************************************************************
+ *
+ * GLOBAL VARIABLES
+ *
+ *****************************************************************/
+
+static char *socket_dimension_names[NETDATA_MAX_SOCKET_VECTOR] = { "received", "sent", "close",
+ "received", "sent", "retransmitted",
+ "connected_V4", "connected_V6", "connected_tcp",
+ "connected_udp"};
+static char *socket_id_names[NETDATA_MAX_SOCKET_VECTOR] = { "tcp_cleanup_rbuf", "tcp_sendmsg", "tcp_close",
+ "udp_recvmsg", "udp_sendmsg", "tcp_retransmit_skb",
+ "tcp_connect_v4", "tcp_connect_v6", "inet_csk_accept_tcp",
+ "inet_csk_accept_udp" };
+
+static ebpf_local_maps_t socket_maps[] = {{.name = "tbl_bandwidth",
+ .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED,
+ .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED,
+ .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_global_sock",
+ .internal_input = NETDATA_SOCKET_COUNTER,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_lports",
+ .internal_input = NETDATA_SOCKET_COUNTER,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_conn_ipv4",
+ .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED,
+ .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_conn_ipv6",
+ .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED,
+ .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_nv_udp",
+ .internal_input = NETDATA_COMPILED_UDP_CONNECTIONS_ALLOWED,
+ .user_input = NETDATA_MAXIMUM_UDP_CONNECTIONS_ALLOWED,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "socket_ctrl", .internal_input = NETDATA_CONTROLLER_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0}};
+
+static netdata_idx_t *socket_hash_values = NULL;
+static netdata_syscall_stat_t socket_aggregated_data[NETDATA_MAX_SOCKET_VECTOR];
+static netdata_publish_syscall_t socket_publish_aggregated[NETDATA_MAX_SOCKET_VECTOR];
+
+ebpf_socket_publish_apps_t **socket_bandwidth_curr = NULL;
+static ebpf_bandwidth_t *bandwidth_vector = NULL;
+
+pthread_mutex_t nv_mutex;
+int wait_to_plot = 0;
+
+netdata_vector_plot_t inbound_vectors = { .plot = NULL, .next = 0, .last = 0 };
+netdata_vector_plot_t outbound_vectors = { .plot = NULL, .next = 0, .last = 0 };
+netdata_socket_t *socket_values;
+
+ebpf_network_viewer_port_list_t *listen_ports = NULL;
+
+struct config socket_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+netdata_ebpf_targets_t socket_targets[] = { {.name = "inet_csk_accept", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "tcp_retransmit_skb", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "tcp_cleanup_rbuf", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "tcp_close", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "udp_recvmsg", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "tcp_sendmsg", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "udp_sendmsg", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "tcp_v4_connect", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "tcp_v6_connect", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+struct netdata_static_thread socket_threads = {
+ .name = "EBPF SOCKET READ",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/socket.skel.h" // BTF code
+
+static struct socket_bpf *bpf_obj = NULL;
+
+/**
+ * Disable Probe
+ *
+ * Disable probes to use trampoline.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_socket_disable_probes(struct socket_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_inet_csk_accept_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_v4_connect_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_v6_connect_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_retransmit_skb_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_cleanup_rbuf_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_close_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_recvmsg_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_recvmsg_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_sendmsg_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_sendmsg_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_sendmsg_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_sendmsg_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_socket_release_task_kprobe, false);
+}
+
+/**
+ * Disable Trampoline
+ *
+ * Disable trampoline to use probes.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_socket_disable_trampoline(struct socket_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_inet_csk_accept_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_v4_connect_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_v6_connect_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_retransmit_skb_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_cleanup_rbuf_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_close_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_recvmsg_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_recvmsg_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_sendmsg_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_tcp_sendmsg_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_sendmsg_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_sendmsg_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_socket_release_task_fentry, false);
+}
+
+/**
+ * Set trampoline target.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_set_trampoline_target(struct socket_bpf *obj)
+{
+ bpf_program__set_attach_target(obj->progs.netdata_inet_csk_accept_fentry, 0,
+ socket_targets[NETDATA_FCNT_INET_CSK_ACCEPT].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_tcp_v4_connect_fexit, 0,
+ socket_targets[NETDATA_FCNT_TCP_V4_CONNECT].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_tcp_v6_connect_fexit, 0,
+ socket_targets[NETDATA_FCNT_TCP_V6_CONNECT].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_tcp_retransmit_skb_fentry, 0,
+ socket_targets[NETDATA_FCNT_TCP_RETRANSMIT].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_tcp_cleanup_rbuf_fentry, 0,
+ socket_targets[NETDATA_FCNT_CLEANUP_RBUF].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_tcp_close_fentry, 0, socket_targets[NETDATA_FCNT_TCP_CLOSE].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_udp_recvmsg_fentry, 0,
+ socket_targets[NETDATA_FCNT_UDP_RECEVMSG].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_udp_recvmsg_fexit, 0,
+ socket_targets[NETDATA_FCNT_UDP_RECEVMSG].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_tcp_sendmsg_fentry, 0,
+ socket_targets[NETDATA_FCNT_TCP_SENDMSG].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_tcp_sendmsg_fexit, 0,
+ socket_targets[NETDATA_FCNT_TCP_SENDMSG].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_udp_sendmsg_fentry, 0,
+ socket_targets[NETDATA_FCNT_UDP_SENDMSG].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_udp_sendmsg_fexit, 0,
+ socket_targets[NETDATA_FCNT_UDP_SENDMSG].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_socket_release_task_fentry, 0, EBPF_COMMON_FNCT_CLEAN_UP);
+}
+
+
+/**
+ * Disable specific trampoline
+ *
+ * Disable specific trampoline to match user selection.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param sel option selected by user.
+ */
+static inline void ebpf_socket_disable_specific_trampoline(struct socket_bpf *obj, netdata_run_mode_t sel)
+{
+ if (sel == MODE_RETURN) {
+ bpf_program__set_autoload(obj->progs.netdata_tcp_sendmsg_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_sendmsg_fentry, false);
+ } else {
+ bpf_program__set_autoload(obj->progs.netdata_tcp_sendmsg_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_sendmsg_fexit, false);
+ }
+}
+
+/**
+ * Disable specific probe
+ *
+ * Disable specific probe to match user selection.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param sel option selected by user.
+ */
+static inline void ebpf_socket_disable_specific_probe(struct socket_bpf *obj, netdata_run_mode_t sel)
+{
+ if (sel == MODE_RETURN) {
+ bpf_program__set_autoload(obj->progs.netdata_tcp_sendmsg_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_sendmsg_kprobe, false);
+ } else {
+ bpf_program__set_autoload(obj->progs.netdata_tcp_sendmsg_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_udp_sendmsg_kretprobe, false);
+ }
+}
+
+/**
+ * Attach probes
+ *
+ * Attach probes to targets.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param sel option selected by user.
+ */
+static int ebpf_socket_attach_probes(struct socket_bpf *obj, netdata_run_mode_t sel)
+{
+ obj->links.netdata_inet_csk_accept_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_inet_csk_accept_kretprobe,
+ true,
+ socket_targets[NETDATA_FCNT_INET_CSK_ACCEPT].name);
+ int ret = libbpf_get_error(obj->links.netdata_inet_csk_accept_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_tcp_v4_connect_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_tcp_v4_connect_kretprobe,
+ true,
+ socket_targets[NETDATA_FCNT_TCP_V4_CONNECT].name);
+ ret = libbpf_get_error(obj->links.netdata_tcp_v4_connect_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_tcp_v6_connect_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_tcp_v6_connect_kretprobe,
+ true,
+ socket_targets[NETDATA_FCNT_TCP_V6_CONNECT].name);
+ ret = libbpf_get_error(obj->links.netdata_tcp_v6_connect_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_tcp_retransmit_skb_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_tcp_retransmit_skb_kprobe,
+ false,
+ socket_targets[NETDATA_FCNT_TCP_RETRANSMIT].name);
+ ret = libbpf_get_error(obj->links.netdata_tcp_retransmit_skb_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_tcp_cleanup_rbuf_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_tcp_cleanup_rbuf_kprobe,
+ false,
+ socket_targets[NETDATA_FCNT_CLEANUP_RBUF].name);
+ ret = libbpf_get_error(obj->links.netdata_tcp_cleanup_rbuf_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_tcp_close_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_tcp_close_kprobe,
+ false,
+ socket_targets[NETDATA_FCNT_TCP_CLOSE].name);
+ ret = libbpf_get_error(obj->links.netdata_tcp_close_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_udp_recvmsg_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_udp_recvmsg_kprobe,
+ false,
+ socket_targets[NETDATA_FCNT_UDP_RECEVMSG].name);
+ ret = libbpf_get_error(obj->links.netdata_udp_recvmsg_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_udp_recvmsg_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_udp_recvmsg_kretprobe,
+ true,
+ socket_targets[NETDATA_FCNT_UDP_RECEVMSG].name);
+ ret = libbpf_get_error(obj->links.netdata_udp_recvmsg_kretprobe);
+ if (ret)
+ return -1;
+
+ if (sel == MODE_RETURN) {
+ obj->links.netdata_tcp_sendmsg_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_tcp_sendmsg_kretprobe,
+ true,
+ socket_targets[NETDATA_FCNT_TCP_SENDMSG].name);
+ ret = libbpf_get_error(obj->links.netdata_tcp_sendmsg_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_udp_sendmsg_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_udp_sendmsg_kretprobe,
+ true,
+ socket_targets[NETDATA_FCNT_UDP_SENDMSG].name);
+ ret = libbpf_get_error(obj->links.netdata_udp_sendmsg_kretprobe);
+ if (ret)
+ return -1;
+ } else {
+ obj->links.netdata_tcp_sendmsg_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_tcp_sendmsg_kprobe,
+ false,
+ socket_targets[NETDATA_FCNT_TCP_SENDMSG].name);
+ ret = libbpf_get_error(obj->links.netdata_tcp_sendmsg_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_udp_sendmsg_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_udp_sendmsg_kprobe,
+ false,
+ socket_targets[NETDATA_FCNT_UDP_SENDMSG].name);
+ ret = libbpf_get_error(obj->links.netdata_udp_sendmsg_kprobe);
+ if (ret)
+ return -1;
+ }
+
+ obj->links.netdata_socket_release_task_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_socket_release_task_kprobe,
+ false, EBPF_COMMON_FNCT_CLEAN_UP);
+ ret = libbpf_get_error(obj->links.netdata_socket_release_task_kprobe);
+ if (ret)
+ return -1;
+
+ return 0;
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_socket_set_hash_tables(struct socket_bpf *obj)
+{
+ socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH].map_fd = bpf_map__fd(obj->maps.tbl_bandwidth);
+ socket_maps[NETDATA_SOCKET_GLOBAL].map_fd = bpf_map__fd(obj->maps.tbl_global_sock);
+ socket_maps[NETDATA_SOCKET_LPORTS].map_fd = bpf_map__fd(obj->maps.tbl_lports);
+ socket_maps[NETDATA_SOCKET_TABLE_IPV4].map_fd = bpf_map__fd(obj->maps.tbl_conn_ipv4);
+ socket_maps[NETDATA_SOCKET_TABLE_IPV6].map_fd = bpf_map__fd(obj->maps.tbl_conn_ipv6);
+ socket_maps[NETDATA_SOCKET_TABLE_UDP].map_fd = bpf_map__fd(obj->maps.tbl_nv_udp);
+ socket_maps[NETDATA_SOCKET_TABLE_CTRL].map_fd = bpf_map__fd(obj->maps.socket_ctrl);
+}
+
+/**
+ * Adjust Map Size
+ *
+ * Resize maps according input from users.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ */
+static void ebpf_socket_adjust_map_size(struct socket_bpf *obj, ebpf_module_t *em)
+{
+ ebpf_update_map_size(obj->maps.tbl_bandwidth, &socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH],
+ em, bpf_map__name(obj->maps.tbl_bandwidth));
+
+ ebpf_update_map_size(obj->maps.tbl_conn_ipv4, &socket_maps[NETDATA_SOCKET_TABLE_IPV4],
+ em, bpf_map__name(obj->maps.tbl_conn_ipv4));
+
+ ebpf_update_map_size(obj->maps.tbl_conn_ipv6, &socket_maps[NETDATA_SOCKET_TABLE_IPV6],
+ em, bpf_map__name(obj->maps.tbl_conn_ipv6));
+
+ ebpf_update_map_size(obj->maps.tbl_nv_udp, &socket_maps[NETDATA_SOCKET_TABLE_UDP],
+ em, bpf_map__name(obj->maps.tbl_nv_udp));
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_socket_load_and_attach(struct socket_bpf *obj, ebpf_module_t *em)
+{
+ netdata_ebpf_targets_t *mt = em->targets;
+ netdata_ebpf_program_loaded_t test = mt[NETDATA_FCNT_INET_CSK_ACCEPT].mode;
+
+ if (test == EBPF_LOAD_TRAMPOLINE) {
+ ebpf_socket_disable_probes(obj);
+
+ ebpf_set_trampoline_target(obj);
+ ebpf_socket_disable_specific_trampoline(obj, em->mode);
+ } else { // We are not using tracepoints for this thread.
+ ebpf_socket_disable_trampoline(obj);
+
+ ebpf_socket_disable_specific_probe(obj, em->mode);
+ }
+
+ int ret = socket_bpf__load(obj);
+ if (ret) {
+ fprintf(stderr, "failed to load BPF object: %d\n", ret);
+ return ret;
+ }
+
+ ebpf_socket_adjust_map_size(obj, em);
+
+ if (test == EBPF_LOAD_TRAMPOLINE) {
+ ret = socket_bpf__attach(obj);
+ } else {
+ ret = ebpf_socket_attach_probes(obj, em->mode);
+ }
+
+ if (!ret) {
+ ebpf_socket_set_hash_tables(obj);
+
+ ebpf_update_controller(socket_maps[NETDATA_SOCKET_TABLE_CTRL].map_fd, em);
+ }
+
+ return ret;
+}
+#endif
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Clean internal socket plot
+ *
+ * Clean all structures allocated with strdupz.
+ *
+ * @param ptr the pointer with addresses to clean.
+ */
+static inline void clean_internal_socket_plot(netdata_socket_plot_t *ptr)
+{
+ freez(ptr->dimension_recv);
+ freez(ptr->dimension_sent);
+ freez(ptr->resolved_name);
+ freez(ptr->dimension_retransmit);
+}
+
+/**
+ * Clean socket plot
+ *
+ * Clean the allocated data for inbound and outbound vectors.
+ */
+static void clean_allocated_socket_plot()
+{
+ uint32_t i;
+ uint32_t end = inbound_vectors.last;
+ netdata_socket_plot_t *plot = inbound_vectors.plot;
+ for (i = 0; i < end; i++) {
+ clean_internal_socket_plot(&plot[i]);
+ }
+
+ clean_internal_socket_plot(&plot[inbound_vectors.last]);
+
+ end = outbound_vectors.last;
+ plot = outbound_vectors.plot;
+ for (i = 0; i < end; i++) {
+ clean_internal_socket_plot(&plot[i]);
+ }
+ clean_internal_socket_plot(&plot[outbound_vectors.last]);
+}
+
+/**
+ * Clean network ports allocated during initialization.
+ *
+ * @param ptr a pointer to the link list.
+ */
+static void clean_network_ports(ebpf_network_viewer_port_list_t *ptr)
+{
+ if (unlikely(!ptr))
+ return;
+
+ while (ptr) {
+ ebpf_network_viewer_port_list_t *next = ptr->next;
+ freez(ptr->value);
+ freez(ptr);
+ ptr = next;
+ }
+}
+
+/**
+ * Clean service names
+ *
+ * Clean the allocated link list that stores names.
+ *
+ * @param names the link list.
+ */
+static void clean_service_names(ebpf_network_viewer_dim_name_t *names)
+{
+ if (unlikely(!names))
+ return;
+
+ while (names) {
+ ebpf_network_viewer_dim_name_t *next = names->next;
+ freez(names->name);
+ freez(names);
+ names = next;
+ }
+}
+
+/**
+ * Clean hostnames
+ *
+ * @param hostnames the hostnames to clean
+ */
+static void clean_hostnames(ebpf_network_viewer_hostname_list_t *hostnames)
+{
+ if (unlikely(!hostnames))
+ return;
+
+ while (hostnames) {
+ ebpf_network_viewer_hostname_list_t *next = hostnames->next;
+ freez(hostnames->value);
+ simple_pattern_free(hostnames->value_pattern);
+ freez(hostnames);
+ hostnames = next;
+ }
+}
+
+/**
+ * Cleanup publish syscall
+ *
+ * @param nps list of structures to clean
+ */
+void ebpf_cleanup_publish_syscall(netdata_publish_syscall_t *nps)
+{
+ while (nps) {
+ freez(nps->algorithm);
+ nps = nps->next;
+ }
+}
+
+/**
+ * Clean port Structure
+ *
+ * Clean the allocated list.
+ *
+ * @param clean the list that will be cleaned
+ */
+void clean_port_structure(ebpf_network_viewer_port_list_t **clean)
+{
+ ebpf_network_viewer_port_list_t *move = *clean;
+ while (move) {
+ ebpf_network_viewer_port_list_t *next = move->next;
+ freez(move->value);
+ freez(move);
+
+ move = next;
+ }
+ *clean = NULL;
+}
+
+/**
+ * Clean IP structure
+ *
+ * Clean the allocated list.
+ *
+ * @param clean the list that will be cleaned
+ */
+static void clean_ip_structure(ebpf_network_viewer_ip_list_t **clean)
+{
+ ebpf_network_viewer_ip_list_t *move = *clean;
+ while (move) {
+ ebpf_network_viewer_ip_list_t *next = move->next;
+ freez(move->value);
+ freez(move);
+
+ move = next;
+ }
+ *clean = NULL;
+}
+
+/**
+ * Socket Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_socket_free(ebpf_module_t *em )
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ ebpf_cleanup_publish_syscall(socket_publish_aggregated);
+ freez(socket_hash_values);
+
+ freez(bandwidth_vector);
+
+ freez(socket_values);
+ clean_allocated_socket_plot();
+ freez(inbound_vectors.plot);
+ freez(outbound_vectors.plot);
+
+ clean_port_structure(&listen_ports);
+
+ ebpf_modules[EBPF_MODULE_SOCKET_IDX].enabled = 0;
+
+ clean_network_ports(network_viewer_opt.included_port);
+ clean_network_ports(network_viewer_opt.excluded_port);
+ clean_service_names(network_viewer_opt.names);
+ clean_hostnames(network_viewer_opt.included_hostnames);
+ clean_hostnames(network_viewer_opt.excluded_hostnames);
+
+ pthread_mutex_destroy(&nv_mutex);
+
+ freez(socket_threads.thread);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ if (bpf_obj)
+ socket_bpf__destroy(bpf_obj);
+#endif
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Socket exit
+ *
+ * Clean up the main thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_socket_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*socket_threads.thread);
+ ebpf_socket_free(em);
+}
+
+/**
+ * Socket cleanup
+ *
+ * Clean up allocated addresses.
+ *
+ * @param ptr thread data.
+ */
+void ebpf_socket_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_socket_free(em);
+}
+
+/*****************************************************************
+ *
+ * PROCESS DATA AND SEND TO NETDATA
+ *
+ *****************************************************************/
+
+/**
+ * Update publish structure before to send data to Netdata.
+ *
+ * @param publish the first output structure with independent dimensions
+ * @param tcp structure to store IO from tcp sockets
+ * @param udp structure to store IO from udp sockets
+ * @param input the structure with the input data.
+ */
+static void ebpf_update_global_publish(
+ netdata_publish_syscall_t *publish, netdata_publish_vfs_common_t *tcp, netdata_publish_vfs_common_t *udp,
+ netdata_syscall_stat_t *input)
+{
+ netdata_publish_syscall_t *move = publish;
+ while (move) {
+ if (input->call != move->pcall) {
+ // This condition happens to avoid initial values with dimensions higher than normal values.
+ if (move->pcall) {
+ move->ncall = (input->call > move->pcall) ? input->call - move->pcall : move->pcall - input->call;
+ move->nbyte = (input->bytes > move->pbyte) ? input->bytes - move->pbyte : move->pbyte - input->bytes;
+ move->nerr = (input->ecall > move->nerr) ? input->ecall - move->perr : move->perr - input->ecall;
+ } else {
+ move->ncall = 0;
+ move->nbyte = 0;
+ move->nerr = 0;
+ }
+
+ move->pcall = input->call;
+ move->pbyte = input->bytes;
+ move->perr = input->ecall;
+ } else {
+ move->ncall = 0;
+ move->nbyte = 0;
+ move->nerr = 0;
+ }
+
+ input = input->next;
+ move = move->next;
+ }
+
+ tcp->write = -(long)publish[0].nbyte;
+ tcp->read = (long)publish[1].nbyte;
+
+ udp->write = -(long)publish[3].nbyte;
+ udp->read = (long)publish[4].nbyte;
+}
+
+/**
+ * Update Network Viewer plot data
+ *
+ * @param plot the structure where the data will be stored
+ * @param sock the last update from the socket
+ */
+static inline void update_nv_plot_data(netdata_plot_values_t *plot, netdata_socket_t *sock)
+{
+ if (sock->ct > plot->last_time) {
+ plot->last_time = sock->ct;
+ plot->plot_recv_packets = sock->recv_packets;
+ plot->plot_sent_packets = sock->sent_packets;
+ plot->plot_recv_bytes = sock->recv_bytes;
+ plot->plot_sent_bytes = sock->sent_bytes;
+ plot->plot_retransmit = sock->retransmit;
+ }
+
+ sock->recv_packets = 0;
+ sock->sent_packets = 0;
+ sock->recv_bytes = 0;
+ sock->sent_bytes = 0;
+ sock->retransmit = 0;
+}
+
+/**
+ * Calculate Network Viewer Plot
+ *
+ * Do math with collected values before to plot data.
+ */
+static inline void calculate_nv_plot()
+{
+ uint32_t i;
+ uint32_t end = inbound_vectors.next;
+ for (i = 0; i < end; i++) {
+ update_nv_plot_data(&inbound_vectors.plot[i].plot, &inbound_vectors.plot[i].sock);
+ }
+ inbound_vectors.max_plot = end;
+
+ // The 'Other' dimension is always calculated for the chart to have at least one dimension
+ update_nv_plot_data(&inbound_vectors.plot[inbound_vectors.last].plot,
+ &inbound_vectors.plot[inbound_vectors.last].sock);
+
+ end = outbound_vectors.next;
+ for (i = 0; i < end; i++) {
+ update_nv_plot_data(&outbound_vectors.plot[i].plot, &outbound_vectors.plot[i].sock);
+ }
+ outbound_vectors.max_plot = end;
+
+ // The 'Other' dimension is always calculated for the chart to have at least one dimension
+ update_nv_plot_data(&outbound_vectors.plot[outbound_vectors.last].plot,
+ &outbound_vectors.plot[outbound_vectors.last].sock);
+}
+
+/**
+ * Network viewer send bytes
+ *
+ * @param ptr the structure with values to plot
+ * @param chart the chart name.
+ */
+static inline void ebpf_socket_nv_send_bytes(netdata_vector_plot_t *ptr, char *chart)
+{
+ uint32_t i;
+ uint32_t end = ptr->last_plot;
+ netdata_socket_plot_t *w = ptr->plot;
+ collected_number value;
+
+ write_begin_chart(NETDATA_EBPF_FAMILY, chart);
+ for (i = 0; i < end; i++) {
+ value = ((collected_number) w[i].plot.plot_sent_bytes);
+ write_chart_dimension(w[i].dimension_sent, value);
+ value = (collected_number) w[i].plot.plot_recv_bytes;
+ write_chart_dimension(w[i].dimension_recv, value);
+ }
+
+ i = ptr->last;
+ value = ((collected_number) w[i].plot.plot_sent_bytes);
+ write_chart_dimension(w[i].dimension_sent, value);
+ value = (collected_number) w[i].plot.plot_recv_bytes;
+ write_chart_dimension(w[i].dimension_recv, value);
+ write_end_chart();
+}
+
+/**
+ * Network Viewer Send packets
+ *
+ * @param ptr the structure with values to plot
+ * @param chart the chart name.
+ */
+static inline void ebpf_socket_nv_send_packets(netdata_vector_plot_t *ptr, char *chart)
+{
+ uint32_t i;
+ uint32_t end = ptr->last_plot;
+ netdata_socket_plot_t *w = ptr->plot;
+ collected_number value;
+
+ write_begin_chart(NETDATA_EBPF_FAMILY, chart);
+ for (i = 0; i < end; i++) {
+ value = ((collected_number)w[i].plot.plot_sent_packets);
+ write_chart_dimension(w[i].dimension_sent, value);
+ value = (collected_number) w[i].plot.plot_recv_packets;
+ write_chart_dimension(w[i].dimension_recv, value);
+ }
+
+ i = ptr->last;
+ value = ((collected_number)w[i].plot.plot_sent_packets);
+ write_chart_dimension(w[i].dimension_sent, value);
+ value = (collected_number)w[i].plot.plot_recv_packets;
+ write_chart_dimension(w[i].dimension_recv, value);
+ write_end_chart();
+}
+
+/**
+ * Network Viewer Send Retransmit
+ *
+ * @param ptr the structure with values to plot
+ * @param chart the chart name.
+ */
+static inline void ebpf_socket_nv_send_retransmit(netdata_vector_plot_t *ptr, char *chart)
+{
+ uint32_t i;
+ uint32_t end = ptr->last_plot;
+ netdata_socket_plot_t *w = ptr->plot;
+ collected_number value;
+
+ write_begin_chart(NETDATA_EBPF_FAMILY, chart);
+ for (i = 0; i < end; i++) {
+ value = (collected_number) w[i].plot.plot_retransmit;
+ write_chart_dimension(w[i].dimension_retransmit, value);
+ }
+
+ i = ptr->last;
+ value = (collected_number)w[i].plot.plot_retransmit;
+ write_chart_dimension(w[i].dimension_retransmit, value);
+ write_end_chart();
+}
+
+/**
+ * Send network viewer data
+ *
+ * @param ptr the pointer to plot data
+ */
+static void ebpf_socket_send_nv_data(netdata_vector_plot_t *ptr)
+{
+ if (!ptr->flags)
+ return;
+
+ if (ptr == (netdata_vector_plot_t *)&outbound_vectors) {
+ ebpf_socket_nv_send_bytes(ptr, NETDATA_NV_OUTBOUND_BYTES);
+ fflush(stdout);
+
+ ebpf_socket_nv_send_packets(ptr, NETDATA_NV_OUTBOUND_PACKETS);
+ fflush(stdout);
+
+ ebpf_socket_nv_send_retransmit(ptr, NETDATA_NV_OUTBOUND_RETRANSMIT);
+ fflush(stdout);
+ } else {
+ ebpf_socket_nv_send_bytes(ptr, NETDATA_NV_INBOUND_BYTES);
+ fflush(stdout);
+
+ ebpf_socket_nv_send_packets(ptr, NETDATA_NV_INBOUND_PACKETS);
+ fflush(stdout);
+ }
+}
+
+/**
+ * Send Global Inbound connection
+ *
+ * Send number of connections read per protocol.
+ */
+static void ebpf_socket_send_global_inbound_conn()
+{
+ uint64_t udp_conn = 0;
+ uint64_t tcp_conn = 0;
+ ebpf_network_viewer_port_list_t *move = listen_ports;
+ while (move) {
+ if (move->protocol == IPPROTO_TCP)
+ tcp_conn += move->connections;
+ else
+ udp_conn += move->connections;
+
+ move = move->next;
+ }
+
+ write_begin_chart(NETDATA_EBPF_IP_FAMILY, NETDATA_INBOUND_CONNECTIONS);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_INCOMING_CONNECTION_TCP].name, (long long) tcp_conn);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_INCOMING_CONNECTION_UDP].name, (long long) udp_conn);
+ write_end_chart();
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the structure with thread information
+ */
+static void ebpf_socket_send_data(ebpf_module_t *em)
+{
+ netdata_publish_vfs_common_t common_tcp;
+ netdata_publish_vfs_common_t common_udp;
+ ebpf_update_global_publish(socket_publish_aggregated, &common_tcp, &common_udp, socket_aggregated_data);
+
+ ebpf_socket_send_global_inbound_conn();
+ write_count_chart(NETDATA_TCP_OUTBOUND_CONNECTIONS, NETDATA_EBPF_IP_FAMILY,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_CONNECTION_V4], 2);
+
+ // We read bytes from function arguments, but bandwidth is given in bits,
+ // so we need to multiply by 8 to convert for the final value.
+ write_count_chart(NETDATA_TCP_FUNCTION_COUNT, NETDATA_EBPF_IP_FAMILY, socket_publish_aggregated, 3);
+ write_io_chart(NETDATA_TCP_FUNCTION_BITS, NETDATA_EBPF_IP_FAMILY, socket_id_names[0],
+ common_tcp.read * 8/BITS_IN_A_KILOBIT, socket_id_names[1],
+ common_tcp.write * 8/BITS_IN_A_KILOBIT);
+ if (em->mode < MODE_ENTRY) {
+ write_err_chart(NETDATA_TCP_FUNCTION_ERROR, NETDATA_EBPF_IP_FAMILY, socket_publish_aggregated, 2);
+ }
+ write_count_chart(NETDATA_TCP_RETRANSMIT, NETDATA_EBPF_IP_FAMILY,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_RETRANSMIT],1);
+
+ write_count_chart(NETDATA_UDP_FUNCTION_COUNT, NETDATA_EBPF_IP_FAMILY,
+ &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF],2);
+ write_io_chart(NETDATA_UDP_FUNCTION_BITS, NETDATA_EBPF_IP_FAMILY,
+ socket_id_names[3], (long long)common_udp.read * 8/BITS_IN_A_KILOBIT,
+ socket_id_names[4], (long long)common_udp.write * 8/BITS_IN_A_KILOBIT);
+ if (em->mode < MODE_ENTRY) {
+ write_err_chart(NETDATA_UDP_FUNCTION_ERROR, NETDATA_EBPF_IP_FAMILY,
+ &socket_publish_aggregated[NETDATA_UDP_START], 2);
+ }
+}
+
+/**
+ * Sum values for pid
+ *
+ * @param root the structure with all available PIDs
+ *
+ * @param offset the address that we are reading
+ *
+ * @return it returns the sum of all PIDs
+ */
+long long ebpf_socket_sum_values_for_pids(struct pid_on_target *root, size_t offset)
+{
+ long long ret = 0;
+ while (root) {
+ int32_t pid = root->pid;
+ ebpf_socket_publish_apps_t *w = socket_bandwidth_curr[pid];
+ if (w) {
+ ret += get_value_from_structure((char *)w, offset);
+ }
+
+ root = root->next;
+ }
+
+ return ret;
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the structure with thread information
+ * @param root the target list.
+ */
+void ebpf_socket_send_apps_data(ebpf_module_t *em, struct target *root)
+{
+ UNUSED(em);
+
+ struct target *w;
+ collected_number value;
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_CONNECTION_TCP_V4);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ call_tcp_v4_connection));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_CONNECTION_TCP_V6);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ call_tcp_v6_connection));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_BANDWIDTH_SENT);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ bytes_sent));
+ // We multiply by 0.008, because we read bytes, but we display bits
+ write_chart_dimension(w->name, ((value)*8)/1000);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_BANDWIDTH_RECV);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ bytes_received));
+ // We multiply by 0.008, because we read bytes, but we display bits
+ write_chart_dimension(w->name, ((value)*8)/1000);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_BANDWIDTH_TCP_SEND_CALLS);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ call_tcp_sent));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_BANDWIDTH_TCP_RECV_CALLS);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ call_tcp_received));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_BANDWIDTH_TCP_RETRANSMIT);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ retransmit));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_BANDWIDTH_UDP_SEND_CALLS);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ call_udp_sent));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_NET_APPS_BANDWIDTH_UDP_RECV_CALLS);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ value = ebpf_socket_sum_values_for_pids(w->root_pid, offsetof(ebpf_socket_publish_apps_t,
+ call_udp_received));
+ write_chart_dimension(w->name, value);
+ }
+ }
+ write_end_chart();
+
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CREATE CHARTS
+ *
+ *****************************************************************/
+
+/**
+ * Create global charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+static void ebpf_create_global_charts(ebpf_module_t *em)
+{
+ int order = 21070;
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY,
+ NETDATA_INBOUND_CONNECTIONS,
+ "Inbound connections.",
+ EBPF_COMMON_DIMENSION_CONNECTIONS,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_INCOMING_CONNECTION_TCP],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY,
+ NETDATA_TCP_OUTBOUND_CONNECTIONS,
+ "TCP outbound connections.",
+ EBPF_COMMON_DIMENSION_CONNECTIONS,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_CONNECTION_V4],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY,
+ NETDATA_TCP_FUNCTION_COUNT,
+ "Calls to internal functions",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ socket_publish_aggregated,
+ 3, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY, NETDATA_TCP_FUNCTION_BITS,
+ "TCP bandwidth", EBPF_COMMON_DIMENSION_BITS,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ socket_publish_aggregated,
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY,
+ NETDATA_TCP_FUNCTION_ERROR,
+ "TCP errors",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ socket_publish_aggregated,
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+ }
+
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY,
+ NETDATA_TCP_RETRANSMIT,
+ "Packages retransmitted",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_RETRANSMIT],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY,
+ NETDATA_UDP_FUNCTION_COUNT,
+ "UDP calls",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY, NETDATA_UDP_FUNCTION_BITS,
+ "UDP bandwidth", EBPF_COMMON_DIMENSION_BITS,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(NETDATA_EBPF_IP_FAMILY,
+ NETDATA_UDP_FUNCTION_ERROR,
+ "UDP errors",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_SOCKET_KERNEL_FUNCTIONS,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ order++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+ }
+}
+
+/**
+ * Create apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ * @param ptr a pointer for targets
+ */
+void ebpf_socket_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+ int order = 20080;
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_CONNECTION_TCP_V4,
+ "Calls to tcp_v4_connection", EBPF_COMMON_DIMENSION_CONNECTIONS,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_CONNECTION_TCP_V6,
+ "Calls to tcp_v6_connection", EBPF_COMMON_DIMENSION_CONNECTIONS,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_SENT,
+ "Bytes sent", EBPF_COMMON_DIMENSION_BITS,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_RECV,
+ "bytes received", EBPF_COMMON_DIMENSION_BITS,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_TCP_SEND_CALLS,
+ "Calls for tcp_sendmsg",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_TCP_RECV_CALLS,
+ "Calls for tcp_cleanup_rbuf",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_TCP_RETRANSMIT,
+ "Calls for tcp_retransmit",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_UDP_SEND_CALLS,
+ "Calls for udp_sendmsg",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_UDP_RECV_CALLS,
+ "Calls for udp_recvmsg",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/**
+ * Create network viewer chart
+ *
+ * Create common charts.
+ *
+ * @param id chart id
+ * @param title chart title
+ * @param units units label
+ * @param family group name used to attach the chart on dashboard
+ * @param order chart order
+ * @param update_every value to overwrite the update frequency set by the server.
+ * @param ptr plot structure with values.
+ */
+static void ebpf_socket_create_nv_chart(char *id, char *title, char *units,
+ char *family, int order, int update_every, netdata_vector_plot_t *ptr)
+{
+ ebpf_write_chart_cmd(NETDATA_EBPF_FAMILY,
+ id,
+ title,
+ units,
+ family,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ NULL,
+ order,
+ update_every,
+ NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ uint32_t i;
+ uint32_t end = ptr->last_plot;
+ netdata_socket_plot_t *w = ptr->plot;
+ for (i = 0; i < end; i++) {
+ fprintf(stdout, "DIMENSION %s '' incremental -1 1\n", w[i].dimension_sent);
+ fprintf(stdout, "DIMENSION %s '' incremental 1 1\n", w[i].dimension_recv);
+ }
+
+ end = ptr->last;
+ fprintf(stdout, "DIMENSION %s '' incremental -1 1\n", w[end].dimension_sent);
+ fprintf(stdout, "DIMENSION %s '' incremental 1 1\n", w[end].dimension_recv);
+}
+
+/**
+ * Create network viewer retransmit
+ *
+ * Create a specific chart.
+ *
+ * @param id the chart id
+ * @param title the chart title
+ * @param units the units label
+ * @param family the group name used to attach the chart on dashboard
+ * @param order the chart order
+ * @param update_every value to overwrite the update frequency set by the server.
+ * @param ptr the plot structure with values.
+ */
+static void ebpf_socket_create_nv_retransmit(char *id, char *title, char *units,
+ char *family, int order, int update_every, netdata_vector_plot_t *ptr)
+{
+ ebpf_write_chart_cmd(NETDATA_EBPF_FAMILY,
+ id,
+ title,
+ units,
+ family,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ NULL,
+ order,
+ update_every,
+ NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ uint32_t i;
+ uint32_t end = ptr->last_plot;
+ netdata_socket_plot_t *w = ptr->plot;
+ for (i = 0; i < end; i++) {
+ fprintf(stdout, "DIMENSION %s '' incremental 1 1\n", w[i].dimension_retransmit);
+ }
+
+ end = ptr->last;
+ fprintf(stdout, "DIMENSION %s '' incremental 1 1\n", w[end].dimension_retransmit);
+}
+
+/**
+ * Create Network Viewer charts
+ *
+ * Recreate the charts when new sockets are created.
+ *
+ * @param ptr a pointer for inbound or outbound vectors.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_socket_create_nv_charts(netdata_vector_plot_t *ptr, int update_every)
+{
+ // We do not have new sockets, so we do not need move forward
+ if (ptr->max_plot == ptr->last_plot)
+ return;
+
+ ptr->last_plot = ptr->max_plot;
+
+ if (ptr == (netdata_vector_plot_t *)&outbound_vectors) {
+ ebpf_socket_create_nv_chart(NETDATA_NV_OUTBOUND_BYTES,
+ "Outbound connections (bytes).", EBPF_COMMON_DIMENSION_BYTES,
+ NETDATA_NETWORK_CONNECTIONS_GROUP,
+ 21080,
+ update_every, ptr);
+
+ ebpf_socket_create_nv_chart(NETDATA_NV_OUTBOUND_PACKETS,
+ "Outbound connections (packets)",
+ EBPF_COMMON_DIMENSION_PACKETS,
+ NETDATA_NETWORK_CONNECTIONS_GROUP,
+ 21082,
+ update_every, ptr);
+
+ ebpf_socket_create_nv_retransmit(NETDATA_NV_OUTBOUND_RETRANSMIT,
+ "Retransmitted packets",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_NETWORK_CONNECTIONS_GROUP,
+ 21083,
+ update_every, ptr);
+ } else {
+ ebpf_socket_create_nv_chart(NETDATA_NV_INBOUND_BYTES,
+ "Inbound connections (bytes)", EBPF_COMMON_DIMENSION_BYTES,
+ NETDATA_NETWORK_CONNECTIONS_GROUP,
+ 21084,
+ update_every, ptr);
+
+ ebpf_socket_create_nv_chart(NETDATA_NV_INBOUND_PACKETS,
+ "Inbound connections (packets)",
+ EBPF_COMMON_DIMENSION_PACKETS,
+ NETDATA_NETWORK_CONNECTIONS_GROUP,
+ 21085,
+ update_every, ptr);
+ }
+
+ ptr->flags |= NETWORK_VIEWER_CHARTS_CREATED;
+}
+
+/*****************************************************************
+ *
+ * READ INFORMATION FROM KERNEL RING
+ *
+ *****************************************************************/
+
+/**
+ * Is specific ip inside the range
+ *
+ * Check if the ip is inside a IP range previously defined
+ *
+ * @param cmp the IP to compare
+ * @param family the IP family
+ *
+ * @return It returns 1 if the IP is inside the range and 0 otherwise
+ */
+static int is_specific_ip_inside_range(union netdata_ip_t *cmp, int family)
+{
+ if (!network_viewer_opt.excluded_ips && !network_viewer_opt.included_ips)
+ return 1;
+
+ uint32_t ipv4_test = ntohl(cmp->addr32[0]);
+ ebpf_network_viewer_ip_list_t *move = network_viewer_opt.excluded_ips;
+ while (move) {
+ if (family == AF_INET) {
+ if (ntohl(move->first.addr32[0]) <= ipv4_test &&
+ ipv4_test <= ntohl(move->last.addr32[0]) )
+ return 0;
+ } else {
+ if (memcmp(move->first.addr8, cmp->addr8, sizeof(union netdata_ip_t)) <= 0 &&
+ memcmp(move->last.addr8, cmp->addr8, sizeof(union netdata_ip_t)) >= 0) {
+ return 0;
+ }
+ }
+ move = move->next;
+ }
+
+ move = network_viewer_opt.included_ips;
+ while (move) {
+ if (family == AF_INET) {
+ if (ntohl(move->first.addr32[0]) <= ipv4_test &&
+ ntohl(move->last.addr32[0]) >= ipv4_test)
+ return 1;
+ } else {
+ if (memcmp(move->first.addr8, cmp->addr8, sizeof(union netdata_ip_t)) <= 0 &&
+ memcmp(move->last.addr8, cmp->addr8, sizeof(union netdata_ip_t)) >= 0) {
+ return 1;
+ }
+ }
+ move = move->next;
+ }
+
+ return 0;
+}
+
+/**
+ * Is port inside range
+ *
+ * Verify if the cmp port is inside the range [first, last].
+ * This function expects only the last parameter as big endian.
+ *
+ * @param cmp the value to compare
+ *
+ * @return It returns 1 when cmp is inside and 0 otherwise.
+ */
+static int is_port_inside_range(uint16_t cmp)
+{
+ // We do not have restrictions for ports.
+ if (!network_viewer_opt.excluded_port && !network_viewer_opt.included_port)
+ return 1;
+
+ // Test if port is excluded
+ ebpf_network_viewer_port_list_t *move = network_viewer_opt.excluded_port;
+ cmp = htons(cmp);
+ while (move) {
+ if (move->cmp_first <= cmp && cmp <= move->cmp_last)
+ return 0;
+
+ move = move->next;
+ }
+
+ // Test if the port is inside allowed range
+ move = network_viewer_opt.included_port;
+ while (move) {
+ if (move->cmp_first <= cmp && cmp <= move->cmp_last)
+ return 1;
+
+ move = move->next;
+ }
+
+ return 0;
+}
+
+/**
+ * Hostname matches pattern
+ *
+ * @param cmp the value to compare
+ *
+ * @return It returns 1 when the value matches and zero otherwise.
+ */
+int hostname_matches_pattern(char *cmp)
+{
+ if (!network_viewer_opt.included_hostnames && !network_viewer_opt.excluded_hostnames)
+ return 1;
+
+ ebpf_network_viewer_hostname_list_t *move = network_viewer_opt.excluded_hostnames;
+ while (move) {
+ if (simple_pattern_matches(move->value_pattern, cmp))
+ return 0;
+
+ move = move->next;
+ }
+
+ move = network_viewer_opt.included_hostnames;
+ while (move) {
+ if (simple_pattern_matches(move->value_pattern, cmp))
+ return 1;
+
+ move = move->next;
+ }
+
+
+ return 0;
+}
+
+/**
+ * Is socket allowed?
+ *
+ * Compare destination addresses and destination ports to define next steps
+ *
+ * @param key the socket read from kernel ring
+ * @param family the family used to compare IPs (AF_INET and AF_INET6)
+ *
+ * @return It returns 1 if this socket is inside the ranges and 0 otherwise.
+ */
+int is_socket_allowed(netdata_socket_idx_t *key, int family)
+{
+ if (!is_port_inside_range(key->dport))
+ return 0;
+
+ return is_specific_ip_inside_range(&key->daddr, family);
+}
+
+/**
+ * Compare sockets
+ *
+ * Compare destination address and destination port.
+ * We do not compare source port, because it is random.
+ * We also do not compare source address, because inbound and outbound connections are stored in separated AVL trees.
+ *
+ * @param a pointer to netdata_socket_plot
+ * @param b pointer to netdata_socket_plot
+ *
+ * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b.
+ */
+static int compare_sockets(void *a, void *b)
+{
+ struct netdata_socket_plot *val1 = a;
+ struct netdata_socket_plot *val2 = b;
+ int cmp;
+
+ // We do not need to compare val2 family, because data inside hash table is always from the same family
+ if (val1->family == AF_INET) { //IPV4
+ if (val1->flags & NETDATA_INBOUND_DIRECTION) {
+ if (val1->index.sport == val2->index.sport)
+ cmp = 0;
+ else {
+ cmp = (val1->index.sport > val2->index.sport)?1:-1;
+ }
+ } else {
+ cmp = memcmp(&val1->index.dport, &val2->index.dport, sizeof(uint16_t));
+ if (!cmp) {
+ cmp = memcmp(&val1->index.daddr.addr32[0], &val2->index.daddr.addr32[0], sizeof(uint32_t));
+ }
+ }
+ } else {
+ if (val1->flags & NETDATA_INBOUND_DIRECTION) {
+ if (val1->index.sport == val2->index.sport)
+ cmp = 0;
+ else {
+ cmp = (val1->index.sport > val2->index.sport)?1:-1;
+ }
+ } else {
+ cmp = memcmp(&val1->index.dport, &val2->index.dport, sizeof(uint16_t));
+ if (!cmp) {
+ cmp = memcmp(&val1->index.daddr.addr32, &val2->index.daddr.addr32, 4*sizeof(uint32_t));
+ }
+ }
+ }
+
+ return cmp;
+}
+
+/**
+ * Build dimension name
+ *
+ * Fill dimension name vector with values given
+ *
+ * @param dimname the output vector
+ * @param hostname the hostname for the socket.
+ * @param service_name the service used to connect.
+ * @param proto the protocol used in this connection
+ * @param family is this IPV4(AF_INET) or IPV6(AF_INET6)
+ *
+ * @return it returns the size of the data copied on success and -1 otherwise.
+ */
+static inline int build_outbound_dimension_name(char *dimname, char *hostname, char *service_name,
+ char *proto, int family)
+{
+ return snprintf(dimname, CONFIG_MAX_NAME - 7, (family == AF_INET)?"%s:%s:%s_":"%s:%s:[%s]_",
+ service_name, proto,
+ hostname);
+}
+
+/**
+ * Fill inbound dimension name
+ *
+ * Mount the dimension name with the input given
+ *
+ * @param dimname the output vector
+ * @param service_name the service used to connect.
+ * @param proto the protocol used in this connection
+ *
+ * @return it returns the size of the data copied on success and -1 otherwise.
+ */
+static inline int build_inbound_dimension_name(char *dimname, char *service_name, char *proto)
+{
+ return snprintf(dimname, CONFIG_MAX_NAME - 7, "%s:%s_", service_name,
+ proto);
+}
+
+/**
+ * Fill Resolved Name
+ *
+ * Fill the resolved name structure with the value given.
+ * The hostname is the largest value possible, if it is necessary to cut some value, it must be cut.
+ *
+ * @param ptr the output vector
+ * @param hostname the hostname resolved or IP.
+ * @param length the length for the hostname.
+ * @param service_name the service name associated to the connection
+ * @param is_outbound the is this an outbound connection
+ */
+static inline void fill_resolved_name(netdata_socket_plot_t *ptr, char *hostname, size_t length,
+ char *service_name, int is_outbound)
+{
+ if (length < NETDATA_MAX_NETWORK_COMBINED_LENGTH)
+ ptr->resolved_name = strdupz(hostname);
+ else {
+ length = NETDATA_MAX_NETWORK_COMBINED_LENGTH;
+ ptr->resolved_name = mallocz( NETDATA_MAX_NETWORK_COMBINED_LENGTH + 1);
+ memcpy(ptr->resolved_name, hostname, length);
+ ptr->resolved_name[length] = '\0';
+ }
+
+ char dimname[CONFIG_MAX_NAME];
+ int size;
+ char *protocol;
+ if (ptr->sock.protocol == IPPROTO_UDP) {
+ protocol = "UDP";
+ } else if (ptr->sock.protocol == IPPROTO_TCP) {
+ protocol = "TCP";
+ } else {
+ protocol = "ALL";
+ }
+
+ if (is_outbound)
+ size = build_outbound_dimension_name(dimname, hostname, service_name, protocol, ptr->family);
+ else
+ size = build_inbound_dimension_name(dimname,service_name, protocol);
+
+ if (size > 0) {
+ strcpy(&dimname[size], "sent");
+ dimname[size + 4] = '\0';
+ ptr->dimension_sent = strdupz(dimname);
+
+ strcpy(&dimname[size], "recv");
+ ptr->dimension_recv = strdupz(dimname);
+
+ dimname[size - 1] = '\0';
+ ptr->dimension_retransmit = strdupz(dimname);
+ }
+}
+
+/**
+ * Mount dimension names
+ *
+ * Fill the vector names after to resolve the addresses
+ *
+ * @param ptr a pointer to the structure where the values are stored.
+ * @param is_outbound is a outbound ptr value?
+ *
+ * @return It returns 1 if the name is valid and 0 otherwise.
+ */
+int fill_names(netdata_socket_plot_t *ptr, int is_outbound)
+{
+ char hostname[NI_MAXHOST], service_name[NI_MAXSERV];
+ if (ptr->resolved)
+ return 1;
+
+ int ret;
+ static int resolve_name = -1;
+ static int resolve_service = -1;
+ if (resolve_name == -1)
+ resolve_name = network_viewer_opt.hostname_resolution_enabled;
+
+ if (resolve_service == -1)
+ resolve_service = network_viewer_opt.service_resolution_enabled;
+
+ netdata_socket_idx_t *idx = &ptr->index;
+
+ char *errname = { "Not resolved" };
+ // Resolve Name
+ if (ptr->family == AF_INET) { //IPV4
+ struct sockaddr_in myaddr;
+ memset(&myaddr, 0 , sizeof(myaddr));
+
+ myaddr.sin_family = ptr->family;
+ if (is_outbound) {
+ myaddr.sin_port = idx->dport;
+ myaddr.sin_addr.s_addr = idx->daddr.addr32[0];
+ } else {
+ myaddr.sin_port = idx->sport;
+ myaddr.sin_addr.s_addr = idx->saddr.addr32[0];
+ }
+
+ ret = (!resolve_name)?-1:getnameinfo((struct sockaddr *)&myaddr, sizeof(myaddr), hostname,
+ sizeof(hostname), service_name, sizeof(service_name), NI_NAMEREQD);
+
+ if (!ret && !resolve_service) {
+ snprintf(service_name, sizeof(service_name), "%u", ntohs(myaddr.sin_port));
+ }
+
+ if (ret) {
+ // I cannot resolve the name, I will use the IP
+ if (!inet_ntop(AF_INET, &myaddr.sin_addr.s_addr, hostname, NI_MAXHOST)) {
+ strncpy(hostname, errname, 13);
+ }
+
+ snprintf(service_name, sizeof(service_name), "%u", ntohs(myaddr.sin_port));
+ ret = 1;
+ }
+ } else { // IPV6
+ struct sockaddr_in6 myaddr6;
+ memset(&myaddr6, 0 , sizeof(myaddr6));
+
+ myaddr6.sin6_family = AF_INET6;
+ if (is_outbound) {
+ myaddr6.sin6_port = idx->dport;
+ memcpy(myaddr6.sin6_addr.s6_addr, idx->daddr.addr8, sizeof(union netdata_ip_t));
+ } else {
+ myaddr6.sin6_port = idx->sport;
+ memcpy(myaddr6.sin6_addr.s6_addr, idx->saddr.addr8, sizeof(union netdata_ip_t));
+ }
+
+ ret = (!resolve_name)?-1:getnameinfo((struct sockaddr *)&myaddr6, sizeof(myaddr6), hostname,
+ sizeof(hostname), service_name, sizeof(service_name), NI_NAMEREQD);
+
+ if (!ret && !resolve_service) {
+ snprintf(service_name, sizeof(service_name), "%u", ntohs(myaddr6.sin6_port));
+ }
+
+ if (ret) {
+ // I cannot resolve the name, I will use the IP
+ if (!inet_ntop(AF_INET6, myaddr6.sin6_addr.s6_addr, hostname, NI_MAXHOST)) {
+ strncpy(hostname, errname, 13);
+ }
+
+ snprintf(service_name, sizeof(service_name), "%u", ntohs(myaddr6.sin6_port));
+
+ ret = 1;
+ }
+ }
+
+ fill_resolved_name(ptr, hostname,
+ strlen(hostname) + strlen(service_name)+ NETDATA_DOTS_PROTOCOL_COMBINED_LENGTH,
+ service_name, is_outbound);
+
+ if (resolve_name && !ret)
+ ret = hostname_matches_pattern(hostname);
+
+ ptr->resolved++;
+
+ return ret;
+}
+
+/**
+ * Fill last Network Viewer Dimension
+ *
+ * Fill the unique dimension that is always plotted.
+ *
+ * @param ptr the pointer for the last dimension
+ * @param is_outbound is this an inbound structure?
+ */
+static void fill_last_nv_dimension(netdata_socket_plot_t *ptr, int is_outbound)
+{
+ char hostname[NI_MAXHOST], service_name[NI_MAXSERV];
+ char *other = { "other" };
+ // We are also copying the NULL bytes to avoid warnings in new compilers
+ strncpy(hostname, other, 6);
+ strncpy(service_name, other, 6);
+
+ ptr->family = AF_INET;
+ ptr->sock.protocol = 255;
+ ptr->flags = (!is_outbound)?NETDATA_INBOUND_DIRECTION:NETDATA_OUTBOUND_DIRECTION;
+
+ fill_resolved_name(ptr, hostname, 10 + NETDATA_DOTS_PROTOCOL_COMBINED_LENGTH, service_name, is_outbound);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("Last %s dimension added: ID = %u, IP = OTHER, NAME = %s, DIM1 = %s, DIM2 = %s, DIM3 = %s",
+ (is_outbound)?"outbound":"inbound", network_viewer_opt.max_dim - 1, ptr->resolved_name,
+ ptr->dimension_recv, ptr->dimension_sent, ptr->dimension_retransmit);
+#endif
+}
+
+/**
+ * Update Socket Data
+ *
+ * Update the socket information with last collected data
+ *
+ * @param sock
+ * @param lvalues
+ */
+static inline void update_socket_data(netdata_socket_t *sock, netdata_socket_t *lvalues)
+{
+ sock->recv_packets += lvalues->recv_packets;
+ sock->sent_packets += lvalues->sent_packets;
+ sock->recv_bytes += lvalues->recv_bytes;
+ sock->sent_bytes += lvalues->sent_bytes;
+ sock->retransmit += lvalues->retransmit;
+
+ if (lvalues->ct > sock->ct)
+ sock->ct = lvalues->ct;
+}
+
+/**
+ * Store socket inside avl
+ *
+ * Store the socket values inside the avl tree.
+ *
+ * @param out the structure with information used to plot charts.
+ * @param lvalues Values read from socket ring.
+ * @param lindex the index information, the real socket.
+ * @param family the family associated to the socket
+ * @param flags the connection flags
+ */
+static void store_socket_inside_avl(netdata_vector_plot_t *out, netdata_socket_t *lvalues,
+ netdata_socket_idx_t *lindex, int family, uint32_t flags)
+{
+ netdata_socket_plot_t test, *ret ;
+
+ memcpy(&test.index, lindex, sizeof(netdata_socket_idx_t));
+ test.flags = flags;
+
+ ret = (netdata_socket_plot_t *) avl_search_lock(&out->tree, (avl_t *)&test);
+ if (ret) {
+ if (lvalues->ct > ret->plot.last_time) {
+ update_socket_data(&ret->sock, lvalues);
+ }
+ } else {
+ uint32_t curr = out->next;
+ uint32_t last = out->last;
+
+ netdata_socket_plot_t *w = &out->plot[curr];
+
+ int resolved;
+ if (curr == last) {
+ if (lvalues->ct > w->plot.last_time) {
+ update_socket_data(&w->sock, lvalues);
+ }
+ return;
+ } else {
+ memcpy(&w->sock, lvalues, sizeof(netdata_socket_t));
+ memcpy(&w->index, lindex, sizeof(netdata_socket_idx_t));
+ w->family = family;
+
+ resolved = fill_names(w, out != (netdata_vector_plot_t *)&inbound_vectors);
+ }
+
+ if (!resolved) {
+ freez(w->resolved_name);
+ freez(w->dimension_sent);
+ freez(w->dimension_recv);
+ freez(w->dimension_retransmit);
+
+ memset(w, 0, sizeof(netdata_socket_plot_t));
+
+ return;
+ }
+
+ w->flags = flags;
+ netdata_socket_plot_t *check ;
+ check = (netdata_socket_plot_t *) avl_insert_lock(&out->tree, (avl_t *)w);
+ if (check != w)
+ error("Internal error, cannot insert the AVL tree.");
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ char iptext[INET6_ADDRSTRLEN];
+ if (inet_ntop(family, &w->index.daddr.addr8, iptext, sizeof(iptext)))
+ info("New %s dimension added: ID = %u, IP = %s, NAME = %s, DIM1 = %s, DIM2 = %s, DIM3 = %s",
+ (out == &inbound_vectors)?"inbound":"outbound", curr, iptext, w->resolved_name,
+ w->dimension_recv, w->dimension_sent, w->dimension_retransmit);
+#endif
+ curr++;
+ if (curr > last)
+ curr = last;
+ out->next = curr;
+ }
+}
+
+/**
+ * Compare Vector to store
+ *
+ * Compare input values with local address to select table to store.
+ *
+ * @param direction store inbound and outbound direction.
+ * @param cmp index read from hash table.
+ * @param proto the protocol read.
+ *
+ * @return It returns the structure with address to compare.
+ */
+netdata_vector_plot_t * select_vector_to_store(uint32_t *direction, netdata_socket_idx_t *cmp, uint8_t proto)
+{
+ if (!listen_ports) {
+ *direction = NETDATA_OUTBOUND_DIRECTION;
+ return &outbound_vectors;
+ }
+
+ ebpf_network_viewer_port_list_t *move_ports = listen_ports;
+ while (move_ports) {
+ if (move_ports->protocol == proto && move_ports->first == cmp->sport) {
+ *direction = NETDATA_INBOUND_DIRECTION;
+ return &inbound_vectors;
+ }
+
+ move_ports = move_ports->next;
+ }
+
+ *direction = NETDATA_OUTBOUND_DIRECTION;
+ return &outbound_vectors;
+}
+
+/**
+ * Hash accumulator
+ *
+ * @param values the values used to calculate the data.
+ * @param key the key to store data.
+ * @param family the connection family
+ * @param end the values size.
+ */
+static void hash_accumulator(netdata_socket_t *values, netdata_socket_idx_t *key, int family, int end)
+{
+ uint64_t bsent = 0, brecv = 0, psent = 0, precv = 0;
+ uint16_t retransmit = 0;
+ int i;
+ uint8_t protocol = values[0].protocol;
+ uint64_t ct = values[0].ct;
+ for (i = 1; i < end; i++) {
+ netdata_socket_t *w = &values[i];
+
+ precv += w->recv_packets;
+ psent += w->sent_packets;
+ brecv += w->recv_bytes;
+ bsent += w->sent_bytes;
+ retransmit += w->retransmit;
+
+ if (!protocol)
+ protocol = w->protocol;
+
+ if (w->ct > ct)
+ ct = w->ct;
+ }
+
+ values[0].recv_packets += precv;
+ values[0].sent_packets += psent;
+ values[0].recv_bytes += brecv;
+ values[0].sent_bytes += bsent;
+ values[0].retransmit += retransmit;
+ values[0].protocol = (!protocol)?IPPROTO_TCP:protocol;
+ values[0].ct = ct;
+
+ if (is_socket_allowed(key, family)) {
+ uint32_t dir;
+ netdata_vector_plot_t *table = select_vector_to_store(&dir, key, protocol);
+ store_socket_inside_avl(table, &values[0], key, family, dir);
+ }
+}
+
+/**
+ * Read socket hash table
+ *
+ * Read data from hash tables created on kernel ring.
+ *
+ * @param fd the hash table with data.
+ * @param family the family associated to the hash table
+ *
+ * @return it returns 0 on success and -1 otherwise.
+ */
+static void read_socket_hash_table(int fd, int family, int network_connection)
+{
+ if (wait_to_plot)
+ return;
+
+ netdata_socket_idx_t key = {};
+ netdata_socket_idx_t next_key = {};
+
+ netdata_socket_t *values = socket_values;
+ size_t length = ebpf_nprocs*sizeof(netdata_socket_t);
+ int test, end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs;
+
+ while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
+ // We need to reset the values when we are working on kernel 4.15 or newer, because kernel does not create
+ // values for specific processor unless it is used to store data. As result of this behavior one the next socket
+ // can have values from the previous one.
+ memset(values, 0, length);
+ test = bpf_map_lookup_elem(fd, &key, values);
+ if (test < 0) {
+ key = next_key;
+ continue;
+ }
+
+ if (network_connection) {
+ hash_accumulator(values, &key, family, end);
+ }
+
+ key = next_key;
+ }
+}
+
+/**
+ * Fill Network Viewer Port list
+ *
+ * Fill the strcture with values read from /proc or hash table.
+ *
+ * @param out the structure where we will store data.
+ * @param value the ports we are listen to.
+ * @param proto the protocol used for this connection.
+ * @param in the strcuture with values read form different sources.
+ */
+static inline void fill_nv_port_list(ebpf_network_viewer_port_list_t *out, uint16_t value, uint16_t proto,
+ netdata_passive_connection_t *in)
+{
+ out->first = value;
+ out->protocol = proto;
+ out->pid = in->pid;
+ out->tgid = in->tgid;
+ out->connections = in->counter;
+}
+
+/**
+ * Update listen table
+ *
+ * Update link list when it is necessary.
+ *
+ * @param value the ports we are listen to.
+ * @param proto the protocol used with port connection.
+ * @param in the strcuture with values read form different sources.
+ */
+void update_listen_table(uint16_t value, uint16_t proto, netdata_passive_connection_t *in)
+{
+ ebpf_network_viewer_port_list_t *w;
+ if (likely(listen_ports)) {
+ ebpf_network_viewer_port_list_t *move = listen_ports, *store = listen_ports;
+ while (move) {
+ if (move->protocol == proto && move->first == value) {
+ move->pid = in->pid;
+ move->tgid = in->tgid;
+ move->connections = in->counter;
+ return;
+ }
+
+ store = move;
+ move = move->next;
+ }
+
+ w = callocz(1, sizeof(ebpf_network_viewer_port_list_t));
+ store->next = w;
+ } else {
+ w = callocz(1, sizeof(ebpf_network_viewer_port_list_t));
+
+ listen_ports = w;
+ }
+ fill_nv_port_list(w, value, proto, in);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("The network viewer is monitoring inbound connections for port %u", ntohs(value));
+#endif
+}
+
+/**
+ * Read listen table
+ *
+ * Read the table with all ports that we are listen on host.
+ */
+static void read_listen_table()
+{
+ netdata_passive_connection_idx_t key = {};
+ netdata_passive_connection_idx_t next_key = {};
+
+ int fd = socket_maps[NETDATA_SOCKET_LPORTS].map_fd;
+ netdata_passive_connection_t value = {};
+ while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
+ int test = bpf_map_lookup_elem(fd, &key, &value);
+ if (test < 0) {
+ key = next_key;
+ continue;
+ }
+
+ // The correct protocol must come from kernel
+ update_listen_table(key.port, key.protocol, &value);
+
+ key = next_key;
+ memset(&value, 0, sizeof(value));
+ }
+
+ if (next_key.port && value.pid) {
+ // The correct protocol must come from kernel
+ update_listen_table(next_key.port, next_key.protocol, &value);
+ }
+}
+
+/**
+ * Socket read hash
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data on very busy socket.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_socket_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_socket_cleanup, ptr);
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = NETDATA_SOCKET_READ_SLEEP_MS * em->update_every;
+ int fd_ipv4 = socket_maps[NETDATA_SOCKET_TABLE_IPV4].map_fd;
+ int fd_ipv6 = socket_maps[NETDATA_SOCKET_TABLE_IPV6].map_fd;
+ int network_connection = em->optional;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ pthread_mutex_lock(&nv_mutex);
+ read_listen_table();
+ read_socket_hash_table(fd_ipv4, AF_INET, network_connection);
+ read_socket_hash_table(fd_ipv6, AF_INET6, network_connection);
+ wait_to_plot = 1;
+ pthread_mutex_unlock(&nv_mutex);
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Read the hash table and store data to allocated vectors.
+ */
+static void read_hash_global_tables()
+{
+ uint64_t idx;
+ netdata_idx_t res[NETDATA_SOCKET_COUNTER];
+
+ netdata_idx_t *val = socket_hash_values;
+ int fd = socket_maps[NETDATA_SOCKET_GLOBAL].map_fd;
+ for (idx = 0; idx < NETDATA_SOCKET_COUNTER; idx++) {
+ if (!bpf_map_lookup_elem(fd, &idx, val)) {
+ uint64_t total = 0;
+ int i;
+ int end = ebpf_nprocs;
+ for (i = 0; i < end; i++)
+ total += val[i];
+
+ res[idx] = total;
+ } else {
+ res[idx] = 0;
+ }
+ }
+
+ socket_aggregated_data[NETDATA_IDX_TCP_SENDMSG].call = res[NETDATA_KEY_CALLS_TCP_SENDMSG];
+ socket_aggregated_data[NETDATA_IDX_TCP_CLEANUP_RBUF].call = res[NETDATA_KEY_CALLS_TCP_CLEANUP_RBUF];
+ socket_aggregated_data[NETDATA_IDX_TCP_CLOSE].call = res[NETDATA_KEY_CALLS_TCP_CLOSE];
+ socket_aggregated_data[NETDATA_IDX_UDP_RECVBUF].call = res[NETDATA_KEY_CALLS_UDP_RECVMSG];
+ socket_aggregated_data[NETDATA_IDX_UDP_SENDMSG].call = res[NETDATA_KEY_CALLS_UDP_SENDMSG];
+ socket_aggregated_data[NETDATA_IDX_TCP_RETRANSMIT].call = res[NETDATA_KEY_TCP_RETRANSMIT];
+ socket_aggregated_data[NETDATA_IDX_TCP_CONNECTION_V4].call = res[NETDATA_KEY_CALLS_TCP_CONNECT_IPV4];
+ socket_aggregated_data[NETDATA_IDX_TCP_CONNECTION_V6].call = res[NETDATA_KEY_CALLS_TCP_CONNECT_IPV6];
+
+ socket_aggregated_data[NETDATA_IDX_TCP_SENDMSG].ecall = res[NETDATA_KEY_ERROR_TCP_SENDMSG];
+ socket_aggregated_data[NETDATA_IDX_TCP_CLEANUP_RBUF].ecall = res[NETDATA_KEY_ERROR_TCP_CLEANUP_RBUF];
+ socket_aggregated_data[NETDATA_IDX_UDP_RECVBUF].ecall = res[NETDATA_KEY_ERROR_UDP_RECVMSG];
+ socket_aggregated_data[NETDATA_IDX_UDP_SENDMSG].ecall = res[NETDATA_KEY_ERROR_UDP_SENDMSG];
+ socket_aggregated_data[NETDATA_IDX_TCP_CONNECTION_V4].ecall = res[NETDATA_KEY_ERROR_TCP_CONNECT_IPV4];
+ socket_aggregated_data[NETDATA_IDX_TCP_CONNECTION_V6].ecall = res[NETDATA_KEY_ERROR_TCP_CONNECT_IPV6];
+
+ socket_aggregated_data[NETDATA_IDX_TCP_SENDMSG].bytes = res[NETDATA_KEY_BYTES_TCP_SENDMSG];
+ socket_aggregated_data[NETDATA_IDX_TCP_CLEANUP_RBUF].bytes = res[NETDATA_KEY_BYTES_TCP_CLEANUP_RBUF];
+ socket_aggregated_data[NETDATA_IDX_UDP_RECVBUF].bytes = res[NETDATA_KEY_BYTES_UDP_RECVMSG];
+ socket_aggregated_data[NETDATA_IDX_UDP_SENDMSG].bytes = res[NETDATA_KEY_BYTES_UDP_SENDMSG];
+}
+
+/**
+ * Fill publish apps when necessary.
+ *
+ * @param current_pid the PID that I am updating
+ * @param eb the structure with data read from memory.
+ */
+void ebpf_socket_fill_publish_apps(uint32_t current_pid, ebpf_bandwidth_t *eb)
+{
+ ebpf_socket_publish_apps_t *curr = socket_bandwidth_curr[current_pid];
+ if (!curr) {
+ curr = callocz(1, sizeof(ebpf_socket_publish_apps_t));
+ socket_bandwidth_curr[current_pid] = curr;
+ }
+
+ curr->bytes_sent = eb->bytes_sent;
+ curr->bytes_received = eb->bytes_received;
+ curr->call_tcp_sent = eb->call_tcp_sent;
+ curr->call_tcp_received = eb->call_tcp_received;
+ curr->retransmit = eb->retransmit;
+ curr->call_udp_sent = eb->call_udp_sent;
+ curr->call_udp_received = eb->call_udp_received;
+ curr->call_close = eb->close;
+ curr->call_tcp_v4_connection = eb->tcp_v4_connection;
+ curr->call_tcp_v6_connection = eb->tcp_v6_connection;
+}
+
+/**
+ * Bandwidth accumulator.
+ *
+ * @param out the vector with the values to sum
+ */
+void ebpf_socket_bandwidth_accumulator(ebpf_bandwidth_t *out)
+{
+ int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1;
+ ebpf_bandwidth_t *total = &out[0];
+ for (i = 1; i < end; i++) {
+ ebpf_bandwidth_t *move = &out[i];
+ total->bytes_sent += move->bytes_sent;
+ total->bytes_received += move->bytes_received;
+ total->call_tcp_sent += move->call_tcp_sent;
+ total->call_tcp_received += move->call_tcp_received;
+ total->retransmit += move->retransmit;
+ total->call_udp_sent += move->call_udp_sent;
+ total->call_udp_received += move->call_udp_received;
+ total->close += move->close;
+ total->tcp_v4_connection += move->tcp_v4_connection;
+ total->tcp_v6_connection += move->tcp_v6_connection;
+ }
+}
+
+/**
+ * Update the apps data reading information from the hash table
+ */
+static void ebpf_socket_update_apps_data()
+{
+ int fd = socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH].map_fd;
+ ebpf_bandwidth_t *eb = bandwidth_vector;
+ uint32_t key;
+ struct pid_stat *pids = root_of_pids;
+ while (pids) {
+ key = pids->pid;
+
+ if (bpf_map_lookup_elem(fd, &key, eb)) {
+ pids = pids->next;
+ continue;
+ }
+
+ ebpf_socket_bandwidth_accumulator(eb);
+
+ ebpf_socket_fill_publish_apps(key, eb);
+
+ pids = pids->next;
+ }
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ */
+static void ebpf_update_socket_cgroup()
+{
+ ebpf_cgroup_target_t *ect ;
+
+ ebpf_bandwidth_t *eb = bandwidth_vector;
+ int fd = socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH].map_fd;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ int pid = pids->pid;
+ ebpf_bandwidth_t *out = &pids->socket;
+ ebpf_socket_publish_apps_t *publish = &ect->publish_socket;
+ if (likely(socket_bandwidth_curr) && socket_bandwidth_curr[pid]) {
+ ebpf_socket_publish_apps_t *in = socket_bandwidth_curr[pid];
+
+ publish->bytes_sent = in->bytes_sent;
+ publish->bytes_received = in->bytes_received;
+ publish->call_tcp_sent = in->call_tcp_sent;
+ publish->call_tcp_received = in->call_tcp_received;
+ publish->retransmit = in->retransmit;
+ publish->call_udp_sent = in->call_udp_sent;
+ publish->call_udp_received = in->call_udp_received;
+ publish->call_close = in->call_close;
+ publish->call_tcp_v4_connection = in->call_tcp_v4_connection;
+ publish->call_tcp_v6_connection = in->call_tcp_v6_connection;
+ } else {
+ if (!bpf_map_lookup_elem(fd, &pid, eb)) {
+ ebpf_socket_bandwidth_accumulator(eb);
+
+ memcpy(out, eb, sizeof(ebpf_bandwidth_t));
+
+ publish->bytes_sent = out->bytes_sent;
+ publish->bytes_received = out->bytes_received;
+ publish->call_tcp_sent = out->call_tcp_sent;
+ publish->call_tcp_received = out->call_tcp_received;
+ publish->retransmit = out->retransmit;
+ publish->call_udp_sent = out->call_udp_sent;
+ publish->call_udp_received = out->call_udp_received;
+ publish->call_close = out->close;
+ publish->call_tcp_v4_connection = out->tcp_v4_connection;
+ publish->call_tcp_v6_connection = out->tcp_v6_connection;
+ }
+ }
+ }
+ }
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Sum PIDs
+ *
+ * Sum values for all targets.
+ *
+ * @param fd structure used to store data
+ * @param pids input data
+ */
+static void ebpf_socket_sum_cgroup_pids(ebpf_socket_publish_apps_t *socket, struct pid_on_target2 *pids)
+{
+ ebpf_socket_publish_apps_t accumulator;
+ memset(&accumulator, 0, sizeof(accumulator));
+
+ while (pids) {
+ ebpf_bandwidth_t *w = &pids->socket;
+
+ accumulator.bytes_received += w->bytes_received;
+ accumulator.bytes_sent += w->bytes_sent;
+ accumulator.call_tcp_received += w->call_tcp_received;
+ accumulator.call_tcp_sent += w->call_tcp_sent;
+ accumulator.retransmit += w->retransmit;
+ accumulator.call_udp_received += w->call_udp_received;
+ accumulator.call_udp_sent += w->call_udp_sent;
+ accumulator.call_close += w->close;
+ accumulator.call_tcp_v4_connection += w->tcp_v4_connection;
+ accumulator.call_tcp_v6_connection += w->tcp_v6_connection;
+
+ pids = pids->next;
+ }
+
+ socket->bytes_sent = (accumulator.bytes_sent >= socket->bytes_sent) ? accumulator.bytes_sent : socket->bytes_sent;
+ socket->bytes_received = (accumulator.bytes_received >= socket->bytes_received) ? accumulator.bytes_received : socket->bytes_received;
+ socket->call_tcp_sent = (accumulator.call_tcp_sent >= socket->call_tcp_sent) ? accumulator.call_tcp_sent : socket->call_tcp_sent;
+ socket->call_tcp_received = (accumulator.call_tcp_received >= socket->call_tcp_received) ? accumulator.call_tcp_received : socket->call_tcp_received;
+ socket->retransmit = (accumulator.retransmit >= socket->retransmit) ? accumulator.retransmit : socket->retransmit;
+ socket->call_udp_sent = (accumulator.call_udp_sent >= socket->call_udp_sent) ? accumulator.call_udp_sent : socket->call_udp_sent;
+ socket->call_udp_received = (accumulator.call_udp_received >= socket->call_udp_received) ? accumulator.call_udp_received : socket->call_udp_received;
+ socket->call_close = (accumulator.call_close >= socket->call_close) ? accumulator.call_close : socket->call_close;
+ socket->call_tcp_v4_connection = (accumulator.call_tcp_v4_connection >= socket->call_tcp_v4_connection) ?
+ accumulator.call_tcp_v4_connection : socket->call_tcp_v4_connection;
+ socket->call_tcp_v6_connection = (accumulator.call_tcp_v6_connection >= socket->call_tcp_v6_connection) ?
+ accumulator.call_tcp_v6_connection : socket->call_tcp_v6_connection;
+}
+
+/**
+ * Create specific socket charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_specific_socket_charts(char *type, int update_every)
+{
+ int order_basis = 5300;
+ ebpf_create_chart(type, NETDATA_NET_APPS_CONNECTION_TCP_V4,
+ "Calls to tcp_v4_connection",
+ EBPF_COMMON_DIMENSION_CONNECTIONS, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_TCP_V4_CONN_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_CONNECTION_V4], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(type, NETDATA_NET_APPS_CONNECTION_TCP_V6,
+ "Calls to tcp_v6_connection",
+ EBPF_COMMON_DIMENSION_CONNECTIONS, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_TCP_V6_CONN_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_CONNECTION_V6], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(type, NETDATA_NET_APPS_BANDWIDTH_RECV,
+ "Bytes received",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_SOCKET_BYTES_RECV_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_CLEANUP_RBUF], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(type, NETDATA_NET_APPS_BANDWIDTH_SENT,
+ "Bytes sent",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_SOCKET_BYTES_SEND_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ socket_publish_aggregated, 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(type, NETDATA_NET_APPS_BANDWIDTH_TCP_RECV_CALLS,
+ "Calls to tcp_cleanup_rbuf.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_SOCKET_TCP_RECV_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_CLEANUP_RBUF], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(type, NETDATA_NET_APPS_BANDWIDTH_TCP_SEND_CALLS,
+ "Calls to tcp_sendmsg.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_SOCKET_TCP_SEND_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ socket_publish_aggregated, 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(type, NETDATA_NET_APPS_BANDWIDTH_TCP_RETRANSMIT,
+ "Calls to tcp_retransmit.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_SOCKET_TCP_RETRANSMIT_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_TCP_RETRANSMIT], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(type, NETDATA_NET_APPS_BANDWIDTH_UDP_SEND_CALLS,
+ "Calls to udp_sendmsg",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_SOCKET_UDP_SEND_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_UDP_SENDMSG], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+
+ ebpf_create_chart(type, NETDATA_NET_APPS_BANDWIDTH_UDP_RECV_CALLS,
+ "Calls to udp_recvmsg",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_CGROUP_NET_GROUP,
+ NETDATA_CGROUP_SOCKET_UDP_RECV_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++,
+ ebpf_create_global_dimension,
+ &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SOCKET);
+}
+
+/**
+ * Obsolete specific socket charts
+ *
+ * Obsolete charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_obsolete_specific_socket_charts(char *type, int update_every)
+{
+ int order_basis = 5300;
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_CONNECTION_TCP_V4, "Calls to tcp_v4_connection",
+ EBPF_COMMON_DIMENSION_CONNECTIONS, NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_SERVICES_SOCKET_TCP_V4_CONN_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_CONNECTION_TCP_V6,"Calls to tcp_v6_connection",
+ EBPF_COMMON_DIMENSION_CONNECTIONS, NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_SERVICES_SOCKET_TCP_V6_CONN_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_BANDWIDTH_RECV, "Bytes received",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_SERVICES_SOCKET_BYTES_RECV_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_BANDWIDTH_SENT,"Bytes sent",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_SERVICES_SOCKET_BYTES_SEND_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_BANDWIDTH_TCP_RECV_CALLS, "Calls to tcp_cleanup_rbuf.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_SERVICES_SOCKET_TCP_RECV_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_BANDWIDTH_TCP_SEND_CALLS, "Calls to tcp_sendmsg.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_SERVICES_SOCKET_TCP_SEND_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_BANDWIDTH_TCP_RETRANSMIT, "Calls to tcp_retransmit.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_SERVICES_SOCKET_TCP_RETRANSMIT_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_BANDWIDTH_UDP_SEND_CALLS, "Calls to udp_sendmsg",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_SERVICES_SOCKET_UDP_SEND_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_NET_APPS_BANDWIDTH_UDP_RECV_CALLS, "Calls to udp_recvmsg",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_SERVICES_SOCKET_UDP_RECV_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + order_basis++, update_every);
+}
+
+/*
+ * Send Specific Swap data
+ *
+ * Send data for specific cgroup/apps.
+ *
+ * @param type chart type
+ * @param values structure with values that will be sent to netdata
+ */
+static void ebpf_send_specific_socket_data(char *type, ebpf_socket_publish_apps_t *values)
+{
+ write_begin_chart(type, NETDATA_NET_APPS_CONNECTION_TCP_V4);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_TCP_CONNECTION_V4].name,
+ (long long) values->call_tcp_v4_connection);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_NET_APPS_CONNECTION_TCP_V6);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_TCP_CONNECTION_V6].name,
+ (long long) values->call_tcp_v6_connection);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_NET_APPS_BANDWIDTH_SENT);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_TCP_SENDMSG].name,
+ (long long) values->bytes_sent);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_NET_APPS_BANDWIDTH_RECV);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_TCP_CLEANUP_RBUF].name,
+ (long long) values->bytes_received);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_NET_APPS_BANDWIDTH_TCP_SEND_CALLS);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_TCP_SENDMSG].name,
+ (long long) values->call_tcp_sent);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_NET_APPS_BANDWIDTH_TCP_RECV_CALLS);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_TCP_CLEANUP_RBUF].name,
+ (long long) values->call_tcp_received);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_NET_APPS_BANDWIDTH_TCP_RETRANSMIT);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_TCP_RETRANSMIT].name,
+ (long long) values->retransmit);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_NET_APPS_BANDWIDTH_UDP_SEND_CALLS);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_UDP_SENDMSG].name,
+ (long long) values->call_udp_sent);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_NET_APPS_BANDWIDTH_UDP_RECV_CALLS);
+ write_chart_dimension(socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF].name,
+ (long long) values->call_udp_received);
+ write_end_chart();
+}
+
+/**
+ * Create Systemd Socket Charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ **/
+static void ebpf_create_systemd_socket_charts(int update_every)
+{
+ int order = 20080;
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_CONNECTION_TCP_V4,
+ "Calls to tcp_v4_connection", EBPF_COMMON_DIMENSION_CONNECTIONS,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_TCP_V4_CONN_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_CONNECTION_TCP_V6,
+ "Calls to tcp_v6_connection", EBPF_COMMON_DIMENSION_CONNECTIONS,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_TCP_V6_CONN_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_BANDWIDTH_RECV,
+ "Bytes received", EBPF_COMMON_DIMENSION_BITS,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_BYTES_RECV_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_BANDWIDTH_SENT,
+ "Bytes sent", EBPF_COMMON_DIMENSION_BITS,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_BYTES_SEND_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_BANDWIDTH_TCP_RECV_CALLS,
+ "Calls to tcp_cleanup_rbuf.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_TCP_RECV_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_BANDWIDTH_TCP_SEND_CALLS,
+ "Calls to tcp_sendmsg.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_TCP_SEND_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_BANDWIDTH_TCP_RETRANSMIT,
+ "Calls to tcp_retransmit",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_TCP_RETRANSMIT_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_BANDWIDTH_UDP_SEND_CALLS,
+ "Calls to udp_sendmsg",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_UDP_SEND_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_NET_APPS_BANDWIDTH_UDP_RECV_CALLS,
+ "Calls to udp_recvmsg",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_APPS_NET_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ order++,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SERVICES_SOCKET_UDP_RECV_CONTEXT, NETDATA_EBPF_MODULE_NAME_SOCKET,
+ update_every);
+}
+
+/**
+ * Send Systemd charts
+ *
+ * Send collected data to Netdata.
+ */
+static void ebpf_send_systemd_socket_charts()
+{
+ ebpf_cgroup_target_t *ect;
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_CONNECTION_TCP_V4);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.call_tcp_v4_connection);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_CONNECTION_TCP_V6);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.call_tcp_v6_connection);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_BANDWIDTH_SENT);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.bytes_sent);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_BANDWIDTH_RECV);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.bytes_received);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_BANDWIDTH_TCP_SEND_CALLS);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.call_tcp_sent);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_BANDWIDTH_TCP_RECV_CALLS);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.call_tcp_received);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_BANDWIDTH_TCP_RETRANSMIT);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.retransmit);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_BANDWIDTH_UDP_SEND_CALLS);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.call_udp_sent);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_NET_APPS_BANDWIDTH_UDP_RECV_CALLS);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long)ect->publish_socket.call_udp_received);
+ }
+ }
+ write_end_chart();
+}
+
+/**
+ * Update Cgroup algorithm
+ *
+ * Change algorithm from absolute to incremental
+ */
+void ebpf_socket_update_cgroup_algorithm()
+{
+ int i;
+ for (i = 0; i < NETDATA_MAX_SOCKET_VECTOR; i++) {
+ netdata_publish_syscall_t *ptr = &socket_publish_aggregated[i];
+ freez(ptr->algorithm);
+ ptr->algorithm = strdupz(ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX]);
+ }
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+*/
+static void ebpf_socket_send_cgroup_data(int update_every)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ ebpf_socket_sum_cgroup_pids(&ect->publish_socket, ect->pids);
+ }
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_socket_charts(update_every);
+ }
+ ebpf_send_systemd_socket_charts();
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_SOCKET_CHART)) {
+ ebpf_create_specific_socket_charts(ect->name, update_every);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_SOCKET_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_SOCKET_CHART && ect->updated) {
+ ebpf_send_specific_socket_data(ect->name, &ect->publish_socket);
+ } else {
+ ebpf_obsolete_specific_socket_charts(ect->name, update_every);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_SOCKET_CHART;
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS WITH THE MAIN LOOP
+ *
+ *****************************************************************/
+
+/**
+ * Main loop for this collector.
+ *
+ * @param step the number of microseconds used with heart beat
+ * @param em the structure with thread information
+ */
+static void socket_collector(usec_t step, ebpf_module_t *em)
+{
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ socket_threads.thread = mallocz(sizeof(netdata_thread_t));
+ socket_threads.start_routine = ebpf_socket_read_hash;
+
+ netdata_thread_create(socket_threads.thread, socket_threads.name,
+ NETDATA_THREAD_OPTION_DEFAULT, ebpf_socket_read_hash, em);
+
+ int cgroups = em->cgroup_charts;
+ if (cgroups)
+ ebpf_socket_update_cgroup_algorithm();
+
+ int socket_global_enabled = em->global_charts;
+ int network_connection = em->optional;
+ int update_every = em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ netdata_apps_integration_flags_t socket_apps_enabled = em->apps_charts;
+ pthread_mutex_lock(&collect_data_mutex);
+ if (socket_global_enabled)
+ read_hash_global_tables();
+
+ if (socket_apps_enabled)
+ ebpf_socket_update_apps_data();
+
+ if (cgroups)
+ ebpf_update_socket_cgroup();
+
+ calculate_nv_plot();
+
+ pthread_mutex_lock(&lock);
+ if (socket_global_enabled)
+ ebpf_socket_send_data(em);
+
+ if (socket_apps_enabled & NETDATA_EBPF_APPS_FLAG_CHART_CREATED)
+ ebpf_socket_send_apps_data(em, apps_groups_root_target);
+
+ if (cgroups)
+ ebpf_socket_send_cgroup_data(update_every);
+
+ fflush(stdout);
+
+ if (network_connection) {
+ // We are calling fflush many times, because when we have a lot of dimensions
+ // we began to have not expected outputs and Netdata closed the plugin.
+ pthread_mutex_lock(&nv_mutex);
+ ebpf_socket_create_nv_charts(&inbound_vectors, update_every);
+ fflush(stdout);
+ ebpf_socket_send_nv_data(&inbound_vectors);
+
+ ebpf_socket_create_nv_charts(&outbound_vectors, update_every);
+ fflush(stdout);
+ ebpf_socket_send_nv_data(&outbound_vectors);
+ wait_to_plot = 0;
+ pthread_mutex_unlock(&nv_mutex);
+
+ }
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO START THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Allocate vectors used with this thread.
+ * We are not testing the return, because callocz does this and shutdown the software
+ * case it was not possible to allocate.
+ *
+ * @param apps is apps enabled?
+ */
+static void ebpf_socket_allocate_global_vectors(int apps)
+{
+ memset(socket_aggregated_data, 0 ,NETDATA_MAX_SOCKET_VECTOR * sizeof(netdata_syscall_stat_t));
+ memset(socket_publish_aggregated, 0 ,NETDATA_MAX_SOCKET_VECTOR * sizeof(netdata_publish_syscall_t));
+ socket_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t));
+
+ if (apps)
+ socket_bandwidth_curr = callocz((size_t)pid_max, sizeof(ebpf_socket_publish_apps_t *));
+
+ bandwidth_vector = callocz((size_t)ebpf_nprocs, sizeof(ebpf_bandwidth_t));
+
+ socket_values = callocz((size_t)ebpf_nprocs, sizeof(netdata_socket_t));
+ inbound_vectors.plot = callocz(network_viewer_opt.max_dim, sizeof(netdata_socket_plot_t));
+ outbound_vectors.plot = callocz(network_viewer_opt.max_dim, sizeof(netdata_socket_plot_t));
+}
+
+/**
+ * Initialize Inbound and Outbound
+ *
+ * Initialize the common outbound and inbound sockets.
+ */
+static void initialize_inbound_outbound()
+{
+ inbound_vectors.last = network_viewer_opt.max_dim - 1;
+ outbound_vectors.last = inbound_vectors.last;
+ fill_last_nv_dimension(&inbound_vectors.plot[inbound_vectors.last], 0);
+ fill_last_nv_dimension(&outbound_vectors.plot[outbound_vectors.last], 1);
+}
+
+/*****************************************************************
+ *
+ * EBPF SOCKET THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Fill Port list
+ *
+ * @param out a pointer to the link list.
+ * @param in the structure that will be linked.
+ */
+static inline void fill_port_list(ebpf_network_viewer_port_list_t **out, ebpf_network_viewer_port_list_t *in)
+{
+ if (likely(*out)) {
+ ebpf_network_viewer_port_list_t *move = *out, *store = *out;
+ uint16_t first = ntohs(in->first);
+ uint16_t last = ntohs(in->last);
+ while (move) {
+ uint16_t cmp_first = ntohs(move->first);
+ uint16_t cmp_last = ntohs(move->last);
+ if (cmp_first <= first && first <= cmp_last &&
+ cmp_first <= last && last <= cmp_last ) {
+ info("The range/value (%u, %u) is inside the range/value (%u, %u) already inserted, it will be ignored.",
+ first, last, cmp_first, cmp_last);
+ freez(in->value);
+ freez(in);
+ return;
+ } else if (first <= cmp_first && cmp_first <= last &&
+ first <= cmp_last && cmp_last <= last) {
+ info("The range (%u, %u) is bigger than previous range (%u, %u) already inserted, the previous will be ignored.",
+ first, last, cmp_first, cmp_last);
+ freez(move->value);
+ move->value = in->value;
+ move->first = in->first;
+ move->last = in->last;
+ freez(in);
+ return;
+ }
+
+ store = move;
+ move = move->next;
+ }
+
+ store->next = in;
+ } else {
+ *out = in;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("Adding values %s( %u, %u) to %s port list used on network viewer",
+ in->value, ntohs(in->first), ntohs(in->last),
+ (*out == network_viewer_opt.included_port)?"included":"excluded");
+#endif
+}
+
+/**
+ * Parse Service List
+ *
+ * @param out a pointer to store the link list
+ * @param service the service used to create the structure that will be linked.
+ */
+static void parse_service_list(void **out, char *service)
+{
+ ebpf_network_viewer_port_list_t **list = (ebpf_network_viewer_port_list_t **)out;
+ struct servent *serv = getservbyname((const char *)service, "tcp");
+ if (!serv)
+ serv = getservbyname((const char *)service, "udp");
+
+ if (!serv) {
+ info("Cannot resolv the service '%s' with protocols TCP and UDP, it will be ignored", service);
+ return;
+ }
+
+ ebpf_network_viewer_port_list_t *w = callocz(1, sizeof(ebpf_network_viewer_port_list_t));
+ w->value = strdupz(service);
+ w->hash = simple_hash(service);
+
+ w->first = w->last = (uint16_t)serv->s_port;
+
+ fill_port_list(list, w);
+}
+
+/**
+ * Netmask
+ *
+ * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h)
+ *
+ * @param prefix create the netmask based in the CIDR value.
+ *
+ * @return
+ */
+static inline in_addr_t netmask(int prefix) {
+
+ if (prefix == 0)
+ return (~((in_addr_t) - 1));
+ else
+ return (in_addr_t)(~((1 << (32 - prefix)) - 1));
+
+}
+
+/**
+ * Broadcast
+ *
+ * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h)
+ *
+ * @param addr is the ip address
+ * @param prefix is the CIDR value.
+ *
+ * @return It returns the last address of the range
+ */
+static inline in_addr_t broadcast(in_addr_t addr, int prefix)
+{
+ return (addr | ~netmask(prefix));
+}
+
+/**
+ * Network
+ *
+ * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h)
+ *
+ * @param addr is the ip address
+ * @param prefix is the CIDR value.
+ *
+ * @return It returns the first address of the range.
+ */
+static inline in_addr_t ipv4_network(in_addr_t addr, int prefix)
+{
+ return (addr & netmask(prefix));
+}
+
+/**
+ * IP to network long
+ *
+ * @param dst the vector to store the result
+ * @param ip the source ip given by our users.
+ * @param domain the ip domain (IPV4 or IPV6)
+ * @param source the original string
+ *
+ * @return it returns 0 on success and -1 otherwise.
+ */
+static inline int ip2nl(uint8_t *dst, char *ip, int domain, char *source)
+{
+ if (inet_pton(domain, ip, dst) <= 0) {
+ error("The address specified (%s) is invalid ", source);
+ return -1;
+ }
+
+ return 0;
+}
+
+/**
+ * Get IPV6 Last Address
+ *
+ * @param out the address to store the last address.
+ * @param in the address used to do the math.
+ * @param prefix number of bits used to calculate the address
+ */
+static void get_ipv6_last_addr(union netdata_ip_t *out, union netdata_ip_t *in, uint64_t prefix)
+{
+ uint64_t mask,tmp;
+ uint64_t ret[2];
+ memcpy(ret, in->addr32, sizeof(union netdata_ip_t));
+
+ if (prefix == 128) {
+ memcpy(out->addr32, in->addr32, sizeof(union netdata_ip_t));
+ return;
+ } else if (!prefix) {
+ ret[0] = ret[1] = 0xFFFFFFFFFFFFFFFF;
+ memcpy(out->addr32, ret, sizeof(union netdata_ip_t));
+ return;
+ } else if (prefix <= 64) {
+ ret[1] = 0xFFFFFFFFFFFFFFFFULL;
+
+ tmp = be64toh(ret[0]);
+ if (prefix > 0) {
+ mask = 0xFFFFFFFFFFFFFFFFULL << (64 - prefix);
+ tmp |= ~mask;
+ }
+ ret[0] = htobe64(tmp);
+ } else {
+ mask = 0xFFFFFFFFFFFFFFFFULL << (128 - prefix);
+ tmp = be64toh(ret[1]);
+ tmp |= ~mask;
+ ret[1] = htobe64(tmp);
+ }
+
+ memcpy(out->addr32, ret, sizeof(union netdata_ip_t));
+}
+
+/**
+ * Calculate ipv6 first address
+ *
+ * @param out the address to store the first address.
+ * @param in the address used to do the math.
+ * @param prefix number of bits used to calculate the address
+ */
+static void get_ipv6_first_addr(union netdata_ip_t *out, union netdata_ip_t *in, uint64_t prefix)
+{
+ uint64_t mask,tmp;
+ uint64_t ret[2];
+
+ memcpy(ret, in->addr32, sizeof(union netdata_ip_t));
+
+ if (prefix == 128) {
+ memcpy(out->addr32, in->addr32, sizeof(union netdata_ip_t));
+ return;
+ } else if (!prefix) {
+ ret[0] = ret[1] = 0;
+ memcpy(out->addr32, ret, sizeof(union netdata_ip_t));
+ return;
+ } else if (prefix <= 64) {
+ ret[1] = 0ULL;
+
+ tmp = be64toh(ret[0]);
+ if (prefix > 0) {
+ mask = 0xFFFFFFFFFFFFFFFFULL << (64 - prefix);
+ tmp &= mask;
+ }
+ ret[0] = htobe64(tmp);
+ } else {
+ mask = 0xFFFFFFFFFFFFFFFFULL << (128 - prefix);
+ tmp = be64toh(ret[1]);
+ tmp &= mask;
+ ret[1] = htobe64(tmp);
+ }
+
+ memcpy(out->addr32, ret, sizeof(union netdata_ip_t));
+}
+
+/**
+ * Is ip inside the range
+ *
+ * Check if the ip is inside a IP range
+ *
+ * @param rfirst the first ip address of the range
+ * @param rlast the last ip address of the range
+ * @param cmpfirst the first ip to compare
+ * @param cmplast the last ip to compare
+ * @param family the IP family
+ *
+ * @return It returns 1 if the IP is inside the range and 0 otherwise
+ */
+static int is_ip_inside_range(union netdata_ip_t *rfirst, union netdata_ip_t *rlast,
+ union netdata_ip_t *cmpfirst, union netdata_ip_t *cmplast, int family)
+{
+ if (family == AF_INET) {
+ if (ntohl(rfirst->addr32[0]) <= ntohl(cmpfirst->addr32[0]) &&
+ ntohl(rlast->addr32[0]) >= ntohl(cmplast->addr32[0]))
+ return 1;
+ } else {
+ if (memcmp(rfirst->addr8, cmpfirst->addr8, sizeof(union netdata_ip_t)) <= 0 &&
+ memcmp(rlast->addr8, cmplast->addr8, sizeof(union netdata_ip_t)) >= 0) {
+ return 1;
+ }
+
+ }
+ return 0;
+}
+
+/**
+ * Fill IP list
+ *
+ * @param out a pointer to the link list.
+ * @param in the structure that will be linked.
+ */
+void fill_ip_list(ebpf_network_viewer_ip_list_t **out, ebpf_network_viewer_ip_list_t *in, char *table)
+{
+#ifndef NETDATA_INTERNAL_CHECKS
+ UNUSED(table);
+#endif
+ if (likely(*out)) {
+ ebpf_network_viewer_ip_list_t *move = *out, *store = *out;
+ while (move) {
+ if (in->ver == move->ver && is_ip_inside_range(&move->first, &move->last, &in->first, &in->last, in->ver)) {
+ info("The range/value (%s) is inside the range/value (%s) already inserted, it will be ignored.",
+ in->value, move->value);
+ freez(in->value);
+ freez(in);
+ return;
+ }
+ store = move;
+ move = move->next;
+ }
+
+ store->next = in;
+ } else {
+ *out = in;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ char first[512], last[512];
+ if (in->ver == AF_INET) {
+ if (inet_ntop(AF_INET, in->first.addr8, first, INET_ADDRSTRLEN) &&
+ inet_ntop(AF_INET, in->last.addr8, last, INET_ADDRSTRLEN))
+ info("Adding values %s - %s to %s IP list \"%s\" used on network viewer",
+ first, last,
+ (*out == network_viewer_opt.included_ips)?"included":"excluded",
+ table);
+ } else {
+ if (inet_ntop(AF_INET6, in->first.addr8, first, INET6_ADDRSTRLEN) &&
+ inet_ntop(AF_INET6, in->last.addr8, last, INET6_ADDRSTRLEN))
+ info("Adding values %s - %s to %s IP list \"%s\" used on network viewer",
+ first, last,
+ (*out == network_viewer_opt.included_ips)?"included":"excluded",
+ table);
+ }
+#endif
+}
+
+/**
+ * Parse IP List
+ *
+ * Parse IP list and link it.
+ *
+ * @param out a pointer to store the link list
+ * @param ip the value given as parameter
+ */
+static void parse_ip_list(void **out, char *ip)
+{
+ ebpf_network_viewer_ip_list_t **list = (ebpf_network_viewer_ip_list_t **)out;
+
+ char *ipdup = strdupz(ip);
+ union netdata_ip_t first = { };
+ union netdata_ip_t last = { };
+ char *is_ipv6;
+ if (*ip == '*' && *(ip+1) == '\0') {
+ memset(first.addr8, 0, sizeof(first.addr8));
+ memset(last.addr8, 0xFF, sizeof(last.addr8));
+
+ is_ipv6 = ip;
+
+ clean_ip_structure(list);
+ goto storethisip;
+ }
+
+ char *end = ip;
+ // Move while I cannot find a separator
+ while (*end && *end != '/' && *end != '-') end++;
+
+ // We will use only the classic IPV6 for while, but we could consider the base 85 in a near future
+ // https://tools.ietf.org/html/rfc1924
+ is_ipv6 = strchr(ip, ':');
+
+ int select;
+ if (*end && !is_ipv6) { // IPV4 range
+ select = (*end == '/') ? 0 : 1;
+ *end++ = '\0';
+ if (*end == '!') {
+ info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup);
+ goto cleanipdup;
+ }
+
+ if (!select) { // CIDR
+ select = ip2nl(first.addr8, ip, AF_INET, ipdup);
+ if (select)
+ goto cleanipdup;
+
+ select = (int) str2i(end);
+ if (select < NETDATA_MINIMUM_IPV4_CIDR || select > NETDATA_MAXIMUM_IPV4_CIDR) {
+ info("The specified CIDR %s is not valid, the IP %s will be ignored.", end, ip);
+ goto cleanipdup;
+ }
+
+ last.addr32[0] = htonl(broadcast(ntohl(first.addr32[0]), select));
+ // This was added to remove
+ // https://app.codacy.com/manual/netdata/netdata/pullRequest?prid=5810941&bid=19021977
+ UNUSED(last.addr32[0]);
+
+ uint32_t ipv4_test = htonl(ipv4_network(ntohl(first.addr32[0]), select));
+ if (first.addr32[0] != ipv4_test) {
+ first.addr32[0] = ipv4_test;
+ struct in_addr ipv4_convert;
+ ipv4_convert.s_addr = ipv4_test;
+ char ipv4_msg[INET_ADDRSTRLEN];
+ if(inet_ntop(AF_INET, &ipv4_convert, ipv4_msg, INET_ADDRSTRLEN))
+ info("The network value of CIDR %s was updated for %s .", ipdup, ipv4_msg);
+ }
+ } else { // Range
+ select = ip2nl(first.addr8, ip, AF_INET, ipdup);
+ if (select)
+ goto cleanipdup;
+
+ select = ip2nl(last.addr8, end, AF_INET, ipdup);
+ if (select)
+ goto cleanipdup;
+ }
+
+ if (htonl(first.addr32[0]) > htonl(last.addr32[0])) {
+ info("The specified range %s is invalid, the second address is smallest than the first, it will be ignored.",
+ ipdup);
+ goto cleanipdup;
+ }
+ } else if (is_ipv6) { // IPV6
+ if (!*end) { // Unique
+ select = ip2nl(first.addr8, ip, AF_INET6, ipdup);
+ if (select)
+ goto cleanipdup;
+
+ memcpy(last.addr8, first.addr8, sizeof(first.addr8));
+ } else if (*end == '-') {
+ *end++ = 0x00;
+ if (*end == '!') {
+ info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup);
+ goto cleanipdup;
+ }
+
+ select = ip2nl(first.addr8, ip, AF_INET6, ipdup);
+ if (select)
+ goto cleanipdup;
+
+ select = ip2nl(last.addr8, end, AF_INET6, ipdup);
+ if (select)
+ goto cleanipdup;
+ } else { // CIDR
+ *end++ = 0x00;
+ if (*end == '!') {
+ info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup);
+ goto cleanipdup;
+ }
+
+ select = str2i(end);
+ if (select < 0 || select > 128) {
+ info("The CIDR %s is not valid, the address %s will be ignored.", end, ip);
+ goto cleanipdup;
+ }
+
+ uint64_t prefix = (uint64_t)select;
+ select = ip2nl(first.addr8, ip, AF_INET6, ipdup);
+ if (select)
+ goto cleanipdup;
+
+ get_ipv6_last_addr(&last, &first, prefix);
+
+ union netdata_ip_t ipv6_test;
+ get_ipv6_first_addr(&ipv6_test, &first, prefix);
+
+ if (memcmp(first.addr8, ipv6_test.addr8, sizeof(union netdata_ip_t)) != 0) {
+ memcpy(first.addr8, ipv6_test.addr8, sizeof(union netdata_ip_t));
+
+ struct in6_addr ipv6_convert;
+ memcpy(ipv6_convert.s6_addr, ipv6_test.addr8, sizeof(union netdata_ip_t));
+
+ char ipv6_msg[INET6_ADDRSTRLEN];
+ if(inet_ntop(AF_INET6, &ipv6_convert, ipv6_msg, INET6_ADDRSTRLEN))
+ info("The network value of CIDR %s was updated for %s .", ipdup, ipv6_msg);
+ }
+ }
+
+ if ((be64toh(*(uint64_t *)&first.addr32[2]) > be64toh(*(uint64_t *)&last.addr32[2]) &&
+ !memcmp(first.addr32, last.addr32, 2*sizeof(uint32_t))) ||
+ (be64toh(*(uint64_t *)&first.addr32) > be64toh(*(uint64_t *)&last.addr32)) ) {
+ info("The specified range %s is invalid, the second address is smallest than the first, it will be ignored.",
+ ipdup);
+ goto cleanipdup;
+ }
+ } else { // Unique ip
+ select = ip2nl(first.addr8, ip, AF_INET, ipdup);
+ if (select)
+ goto cleanipdup;
+
+ memcpy(last.addr8, first.addr8, sizeof(first.addr8));
+ }
+
+ ebpf_network_viewer_ip_list_t *store;
+
+ storethisip:
+ store = callocz(1, sizeof(ebpf_network_viewer_ip_list_t));
+ store->value = ipdup;
+ store->hash = simple_hash(ipdup);
+ store->ver = (uint8_t)(!is_ipv6)?AF_INET:AF_INET6;
+ memcpy(store->first.addr8, first.addr8, sizeof(first.addr8));
+ memcpy(store->last.addr8, last.addr8, sizeof(last.addr8));
+
+ fill_ip_list(list, store, "socket");
+ return;
+
+cleanipdup:
+ freez(ipdup);
+}
+
+/**
+ * Parse IP Range
+ *
+ * Parse the IP ranges given and create Network Viewer IP Structure
+ *
+ * @param ptr is a pointer with the text to parse.
+ */
+static void parse_ips(char *ptr)
+{
+ // No value
+ if (unlikely(!ptr))
+ return;
+
+ while (likely(ptr)) {
+ // Move forward until next valid character
+ while (isspace(*ptr)) ptr++;
+
+ // No valid value found
+ if (unlikely(!*ptr))
+ return;
+
+ // Find space that ends the list
+ char *end = strchr(ptr, ' ');
+ if (end) {
+ *end++ = '\0';
+ }
+
+ int neg = 0;
+ if (*ptr == '!') {
+ neg++;
+ ptr++;
+ }
+
+ if (isascii(*ptr)) { // Parse port
+ parse_ip_list((!neg)?(void **)&network_viewer_opt.included_ips:(void **)&network_viewer_opt.excluded_ips,
+ ptr);
+ }
+
+ ptr = end;
+ }
+}
+
+
+
+/**
+ * Parse port list
+ *
+ * Parse an allocated port list with the range given
+ *
+ * @param out a pointer to store the link list
+ * @param range the informed range for the user.
+ */
+static void parse_port_list(void **out, char *range)
+{
+ int first, last;
+ ebpf_network_viewer_port_list_t **list = (ebpf_network_viewer_port_list_t **)out;
+
+ char *copied = strdupz(range);
+ if (*range == '*' && *(range+1) == '\0') {
+ first = 1;
+ last = 65535;
+
+ clean_port_structure(list);
+ goto fillenvpl;
+ }
+
+ char *end = range;
+ //Move while I cannot find a separator
+ while (*end && *end != ':' && *end != '-') end++;
+
+ //It has a range
+ if (likely(*end)) {
+ *end++ = '\0';
+ if (*end == '!') {
+ info("The exclusion cannot be in the second part of the range, the range %s will be ignored.", copied);
+ freez(copied);
+ return;
+ }
+ last = str2i((const char *)end);
+ } else {
+ last = 0;
+ }
+
+ first = str2i((const char *)range);
+ if (first < NETDATA_MINIMUM_PORT_VALUE || first > NETDATA_MAXIMUM_PORT_VALUE) {
+ info("The first port %d of the range \"%s\" is invalid and it will be ignored!", first, copied);
+ freez(copied);
+ return;
+ }
+
+ if (!last)
+ last = first;
+
+ if (last < NETDATA_MINIMUM_PORT_VALUE || last > NETDATA_MAXIMUM_PORT_VALUE) {
+ info("The second port %d of the range \"%s\" is invalid and the whole range will be ignored!", last, copied);
+ freez(copied);
+ return;
+ }
+
+ if (first > last) {
+ info("The specified order %s is wrong, the smallest value is always the first, it will be ignored!", copied);
+ freez(copied);
+ return;
+ }
+
+ ebpf_network_viewer_port_list_t *w;
+fillenvpl:
+ w = callocz(1, sizeof(ebpf_network_viewer_port_list_t));
+ w->value = copied;
+ w->hash = simple_hash(copied);
+ w->first = (uint16_t)htons((uint16_t)first);
+ w->last = (uint16_t)htons((uint16_t)last);
+ w->cmp_first = (uint16_t)first;
+ w->cmp_last = (uint16_t)last;
+
+ fill_port_list(list, w);
+}
+
+/**
+ * Read max dimension.
+ *
+ * Netdata plot two dimensions per connection, so it is necessary to adjust the values.
+ *
+ * @param cfg the configuration structure
+ */
+static void read_max_dimension(struct config *cfg)
+{
+ int maxdim ;
+ maxdim = (int) appconfig_get_number(cfg,
+ EBPF_NETWORK_VIEWER_SECTION,
+ EBPF_MAXIMUM_DIMENSIONS,
+ NETDATA_NV_CAP_VALUE);
+ if (maxdim < 0) {
+ error("'maximum dimensions = %d' must be a positive number, Netdata will change for default value %ld.",
+ maxdim, NETDATA_NV_CAP_VALUE);
+ maxdim = NETDATA_NV_CAP_VALUE;
+ }
+
+ maxdim /= 2;
+ if (!maxdim) {
+ info("The number of dimensions is too small (%u), we are setting it to minimum 2", network_viewer_opt.max_dim);
+ network_viewer_opt.max_dim = 1;
+ return;
+ }
+
+ network_viewer_opt.max_dim = (uint32_t)maxdim;
+}
+
+/**
+ * Parse Port Range
+ *
+ * Parse the port ranges given and create Network Viewer Port Structure
+ *
+ * @param ptr is a pointer with the text to parse.
+ */
+static void parse_ports(char *ptr)
+{
+ // No value
+ if (unlikely(!ptr))
+ return;
+
+ while (likely(ptr)) {
+ // Move forward until next valid character
+ while (isspace(*ptr)) ptr++;
+
+ // No valid value found
+ if (unlikely(!*ptr))
+ return;
+
+ // Find space that ends the list
+ char *end = strchr(ptr, ' ');
+ if (end) {
+ *end++ = '\0';
+ }
+
+ int neg = 0;
+ if (*ptr == '!') {
+ neg++;
+ ptr++;
+ }
+
+ if (isdigit(*ptr)) { // Parse port
+ parse_port_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port,
+ ptr);
+ } else if (isalpha(*ptr)) { // Parse service
+ parse_service_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port,
+ ptr);
+ } else if (*ptr == '*') { // All
+ parse_port_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port,
+ ptr);
+ }
+
+ ptr = end;
+ }
+}
+
+/**
+ * Link hostname
+ *
+ * @param out is the output link list
+ * @param in the hostname to add to list.
+ */
+static void link_hostname(ebpf_network_viewer_hostname_list_t **out, ebpf_network_viewer_hostname_list_t *in)
+{
+ if (likely(*out)) {
+ ebpf_network_viewer_hostname_list_t *move = *out;
+ for (; move->next ; move = move->next ) {
+ if (move->hash == in->hash && !strcmp(move->value, in->value)) {
+ info("The hostname %s was already inserted, it will be ignored.", in->value);
+ freez(in->value);
+ simple_pattern_free(in->value_pattern);
+ freez(in);
+ return;
+ }
+ }
+
+ move->next = in;
+ } else {
+ *out = in;
+ }
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("Adding value %s to %s hostname list used on network viewer",
+ in->value,
+ (*out == network_viewer_opt.included_hostnames)?"included":"excluded");
+#endif
+}
+
+/**
+ * Link Hostnames
+ *
+ * Parse the list of hostnames to create the link list.
+ * This is not associated with the IP, because simple patterns like *example* cannot be resolved to IP.
+ *
+ * @param out is the output link list
+ * @param parse is a pointer with the text to parser.
+ */
+static void link_hostnames(char *parse)
+{
+ // No value
+ if (unlikely(!parse))
+ return;
+
+ while (likely(parse)) {
+ // Find the first valid value
+ while (isspace(*parse)) parse++;
+
+ // No valid value found
+ if (unlikely(!*parse))
+ return;
+
+ // Find space that ends the list
+ char *end = strchr(parse, ' ');
+ if (end) {
+ *end++ = '\0';
+ }
+
+ int neg = 0;
+ if (*parse == '!') {
+ neg++;
+ parse++;
+ }
+
+ ebpf_network_viewer_hostname_list_t *hostname = callocz(1 , sizeof(ebpf_network_viewer_hostname_list_t));
+ hostname->value = strdupz(parse);
+ hostname->hash = simple_hash(parse);
+ hostname->value_pattern = simple_pattern_create(parse, NULL, SIMPLE_PATTERN_EXACT);
+
+ link_hostname((!neg)?&network_viewer_opt.included_hostnames:&network_viewer_opt.excluded_hostnames,
+ hostname);
+
+ parse = end;
+ }
+}
+
+/**
+ * Parse network viewer section
+ *
+ * @param cfg the configuration structure
+ */
+void parse_network_viewer_section(struct config *cfg)
+{
+ read_max_dimension(cfg);
+
+ network_viewer_opt.hostname_resolution_enabled = appconfig_get_boolean(cfg,
+ EBPF_NETWORK_VIEWER_SECTION,
+ EBPF_CONFIG_RESOLVE_HOSTNAME,
+ CONFIG_BOOLEAN_NO);
+
+ network_viewer_opt.service_resolution_enabled = appconfig_get_boolean(cfg,
+ EBPF_NETWORK_VIEWER_SECTION,
+ EBPF_CONFIG_RESOLVE_SERVICE,
+ CONFIG_BOOLEAN_NO);
+
+ char *value = appconfig_get(cfg, EBPF_NETWORK_VIEWER_SECTION, EBPF_CONFIG_PORTS, NULL);
+ parse_ports(value);
+
+ if (network_viewer_opt.hostname_resolution_enabled) {
+ value = appconfig_get(cfg, EBPF_NETWORK_VIEWER_SECTION, EBPF_CONFIG_HOSTNAMES, NULL);
+ link_hostnames(value);
+ } else {
+ info("Name resolution is disabled, collector will not parser \"hostnames\" list.");
+ }
+
+ value = appconfig_get(cfg, EBPF_NETWORK_VIEWER_SECTION,
+ "ips", "!127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 !::1/128");
+ parse_ips(value);
+}
+
+/**
+ * Link dimension name
+ *
+ * Link user specified names inside a link list.
+ *
+ * @param port the port number associated to the dimension name.
+ * @param hash the calculated hash for the dimension name.
+ * @param name the dimension name.
+ */
+static void link_dimension_name(char *port, uint32_t hash, char *value)
+{
+ int test = str2i(port);
+ if (test < NETDATA_MINIMUM_PORT_VALUE || test > NETDATA_MAXIMUM_PORT_VALUE){
+ error("The dimension given (%s = %s) has an invalid value and it will be ignored.", port, value);
+ return;
+ }
+
+ ebpf_network_viewer_dim_name_t *w;
+ w = callocz(1, sizeof(ebpf_network_viewer_dim_name_t));
+
+ w->name = strdupz(value);
+ w->hash = hash;
+
+ w->port = (uint16_t) htons(test);
+
+ ebpf_network_viewer_dim_name_t *names = network_viewer_opt.names;
+ if (unlikely(!names)) {
+ network_viewer_opt.names = w;
+ } else {
+ for (; names->next; names = names->next) {
+ if (names->port == w->port) {
+ info("Duplicated definition for a service, the name %s will be ignored. ", names->name);
+ freez(names->name);
+ names->name = w->name;
+ names->hash = w->hash;
+ freez(w);
+ return;
+ }
+ }
+ names->next = w;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ info("Adding values %s( %u) to dimension name list used on network viewer", w->name, htons(w->port));
+#endif
+}
+
+/**
+ * Parse service Name section.
+ *
+ * This function gets the values that will be used to overwrite dimensions.
+ *
+ * @param cfg the configuration structure
+ */
+void parse_service_name_section(struct config *cfg)
+{
+ struct section *co = appconfig_get_section(cfg, EBPF_SERVICE_NAME_SECTION);
+ if (co) {
+ struct config_option *cv;
+ for (cv = co->values; cv ; cv = cv->next) {
+ link_dimension_name(cv->name, cv->hash, cv->value);
+ }
+ }
+
+ // Always associated the default port to Netdata
+ ebpf_network_viewer_dim_name_t *names = network_viewer_opt.names;
+ if (names) {
+ uint16_t default_port = htons(19999);
+ while (names) {
+ if (names->port == default_port)
+ return;
+
+ names = names->next;
+ }
+ }
+
+ char *port_string = getenv("NETDATA_LISTEN_PORT");
+ if (port_string) {
+ // if variable has an invalid value, we assume netdata is using 19999
+ int default_port = str2i(port_string);
+ if (default_port > 0 && default_port < 65536)
+ link_dimension_name(port_string, simple_hash(port_string), "Netdata");
+ }
+}
+
+void parse_table_size_options(struct config *cfg)
+{
+ socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH].user_input = (uint32_t) appconfig_get_number(cfg,
+ EBPF_GLOBAL_SECTION,
+ EBPF_CONFIG_BANDWIDTH_SIZE, NETDATA_MAXIMUM_CONNECTIONS_ALLOWED);
+
+ socket_maps[NETDATA_SOCKET_TABLE_IPV4].user_input = (uint32_t) appconfig_get_number(cfg,
+ EBPF_GLOBAL_SECTION,
+ EBPF_CONFIG_IPV4_SIZE, NETDATA_MAXIMUM_CONNECTIONS_ALLOWED);
+
+ socket_maps[NETDATA_SOCKET_TABLE_IPV6].user_input = (uint32_t) appconfig_get_number(cfg,
+ EBPF_GLOBAL_SECTION,
+ EBPF_CONFIG_IPV6_SIZE, NETDATA_MAXIMUM_CONNECTIONS_ALLOWED);
+
+ socket_maps[NETDATA_SOCKET_TABLE_UDP].user_input = (uint32_t) appconfig_get_number(cfg,
+ EBPF_GLOBAL_SECTION,
+ EBPF_CONFIG_UDP_SIZE, NETDATA_MAXIMUM_UDP_CONNECTIONS_ALLOWED);
+}
+
+/*
+ * Load BPF
+ *
+ * Load BPF files.
+ *
+ * @param em the structure with configuration
+ */
+static int ebpf_socket_load_bpf(ebpf_module_t *em)
+{
+ int ret = 0;
+
+ if (em->load & EBPF_LOAD_LEGACY) {
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ ret = -1;
+ }
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ bpf_obj = socket_bpf__open();
+ if (!bpf_obj)
+ ret = -1;
+ else
+ ret = ebpf_socket_load_and_attach(bpf_obj, em);
+ }
+#endif
+
+ if (ret) {
+ error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->thread_name);
+ }
+
+ return ret;
+}
+
+/**
+ * Socket thread
+ *
+ * Thread used to generate socket charts.
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always return NULL
+ */
+void *ebpf_socket_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_socket_exit, ptr);
+
+ memset(&inbound_vectors.tree, 0, sizeof(avl_tree_lock));
+ memset(&outbound_vectors.tree, 0, sizeof(avl_tree_lock));
+ avl_init_lock(&inbound_vectors.tree, compare_sockets);
+ avl_init_lock(&outbound_vectors.tree, compare_sockets);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = socket_maps;
+
+ parse_network_viewer_section(&socket_config);
+ parse_service_name_section(&socket_config);
+ parse_table_size_options(&socket_config);
+
+ if (pthread_mutex_init(&nv_mutex, NULL)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ error("Cannot initialize local mutex");
+ goto endsocket;
+ }
+
+ ebpf_socket_allocate_global_vectors(em->apps_charts);
+ initialize_inbound_outbound();
+
+ if (running_on_kernel < NETDATA_EBPF_KERNEL_5_0)
+ em->mode = MODE_ENTRY;
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_socket_load_bpf(em)) {
+ em->enabled = CONFIG_BOOLEAN_NO;
+ pthread_mutex_unlock(&lock);
+ goto endsocket;
+ }
+
+ int algorithms[NETDATA_MAX_SOCKET_VECTOR] = {
+ NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX,
+ NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX,
+ NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_INCREMENTAL_IDX,
+ NETDATA_EBPF_INCREMENTAL_IDX
+ };
+ ebpf_global_labels(
+ socket_aggregated_data, socket_publish_aggregated, socket_dimension_names, socket_id_names,
+ algorithms, NETDATA_MAX_SOCKET_VECTOR);
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_global_charts(em);
+
+ ebpf_update_stats(&plugin_statistics, em);
+
+ pthread_mutex_unlock(&lock);
+
+ socket_collector((usec_t)(em->update_every * USEC_PER_SEC), em);
+
+endsocket:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_socket.h b/collectors/ebpf.plugin/ebpf_socket.h
new file mode 100644
index 0000000..ca6b193
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_socket.h
@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#ifndef NETDATA_EBPF_SOCKET_H
+#define NETDATA_EBPF_SOCKET_H 1
+#include <stdint.h>
+#include "libnetdata/avl/avl.h"
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_SOCKET "socket"
+
+// Vector indexes
+#define NETDATA_UDP_START 3
+
+#define NETDATA_SOCKET_READ_SLEEP_MS 800000ULL
+
+// config file
+#define NETDATA_NETWORK_CONFIG_FILE "network.conf"
+#define EBPF_NETWORK_VIEWER_SECTION "network connections"
+#define EBPF_SERVICE_NAME_SECTION "service name"
+#define EBPF_CONFIG_RESOLVE_HOSTNAME "resolve hostnames"
+#define EBPF_CONFIG_RESOLVE_SERVICE "resolve service names"
+#define EBPF_CONFIG_PORTS "ports"
+#define EBPF_CONFIG_HOSTNAMES "hostnames"
+#define EBPF_CONFIG_BANDWIDTH_SIZE "bandwidth table size"
+#define EBPF_CONFIG_IPV4_SIZE "ipv4 connection table size"
+#define EBPF_CONFIG_IPV6_SIZE "ipv6 connection table size"
+#define EBPF_CONFIG_UDP_SIZE "udp connection table size"
+#define EBPF_MAXIMUM_DIMENSIONS "maximum dimensions"
+
+enum ebpf_socket_table_list {
+ NETDATA_SOCKET_TABLE_BANDWIDTH,
+ NETDATA_SOCKET_GLOBAL,
+ NETDATA_SOCKET_LPORTS,
+ NETDATA_SOCKET_TABLE_IPV4,
+ NETDATA_SOCKET_TABLE_IPV6,
+ NETDATA_SOCKET_TABLE_UDP,
+ NETDATA_SOCKET_TABLE_CTRL
+};
+
+enum ebpf_socket_publish_index {
+ NETDATA_IDX_TCP_SENDMSG,
+ NETDATA_IDX_TCP_CLEANUP_RBUF,
+ NETDATA_IDX_TCP_CLOSE,
+ NETDATA_IDX_UDP_RECVBUF,
+ NETDATA_IDX_UDP_SENDMSG,
+ NETDATA_IDX_TCP_RETRANSMIT,
+ NETDATA_IDX_TCP_CONNECTION_V4,
+ NETDATA_IDX_TCP_CONNECTION_V6,
+ NETDATA_IDX_INCOMING_CONNECTION_TCP,
+ NETDATA_IDX_INCOMING_CONNECTION_UDP,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_MAX_SOCKET_VECTOR
+};
+
+enum socket_functions {
+ NETDATA_FCNT_INET_CSK_ACCEPT,
+ NETDATA_FCNT_TCP_RETRANSMIT,
+ NETDATA_FCNT_CLEANUP_RBUF,
+ NETDATA_FCNT_TCP_CLOSE,
+ NETDATA_FCNT_UDP_RECEVMSG,
+ NETDATA_FCNT_TCP_SENDMSG,
+ NETDATA_FCNT_UDP_SENDMSG,
+ NETDATA_FCNT_TCP_V4_CONNECT,
+ NETDATA_FCNT_TCP_V6_CONNECT
+};
+
+typedef enum ebpf_socket_idx {
+ NETDATA_KEY_CALLS_TCP_SENDMSG,
+ NETDATA_KEY_ERROR_TCP_SENDMSG,
+ NETDATA_KEY_BYTES_TCP_SENDMSG,
+
+ NETDATA_KEY_CALLS_TCP_CLEANUP_RBUF,
+ NETDATA_KEY_ERROR_TCP_CLEANUP_RBUF,
+ NETDATA_KEY_BYTES_TCP_CLEANUP_RBUF,
+
+ NETDATA_KEY_CALLS_TCP_CLOSE,
+
+ NETDATA_KEY_CALLS_UDP_RECVMSG,
+ NETDATA_KEY_ERROR_UDP_RECVMSG,
+ NETDATA_KEY_BYTES_UDP_RECVMSG,
+
+ NETDATA_KEY_CALLS_UDP_SENDMSG,
+ NETDATA_KEY_ERROR_UDP_SENDMSG,
+ NETDATA_KEY_BYTES_UDP_SENDMSG,
+
+ NETDATA_KEY_TCP_RETRANSMIT,
+
+ NETDATA_KEY_CALLS_TCP_CONNECT_IPV4,
+ NETDATA_KEY_ERROR_TCP_CONNECT_IPV4,
+
+ NETDATA_KEY_CALLS_TCP_CONNECT_IPV6,
+ NETDATA_KEY_ERROR_TCP_CONNECT_IPV6,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_SOCKET_COUNTER
+} ebpf_socket_index_t;
+
+#define NETDATA_SOCKET_KERNEL_FUNCTIONS "kernel"
+#define NETDATA_NETWORK_CONNECTIONS_GROUP "network connections"
+#define NETDATA_CGROUP_NET_GROUP "network (eBPF)"
+
+// Global chart name
+#define NETDATA_TCP_OUTBOUND_CONNECTIONS "tcp_outbound_conn"
+#define NETDATA_INBOUND_CONNECTIONS "inbound_conn"
+#define NETDATA_TCP_FUNCTION_COUNT "tcp_functions"
+#define NETDATA_TCP_FUNCTION_BITS "total_tcp_bandwidth"
+#define NETDATA_TCP_FUNCTION_ERROR "tcp_error"
+#define NETDATA_TCP_RETRANSMIT "tcp_retransmit"
+#define NETDATA_UDP_FUNCTION_COUNT "udp_functions"
+#define NETDATA_UDP_FUNCTION_BITS "total_udp_bandwidth"
+#define NETDATA_UDP_FUNCTION_ERROR "udp_error"
+
+// Charts created on Apps submenu
+#define NETDATA_NET_APPS_CONNECTION_TCP_V4 "outbound_conn_v4"
+#define NETDATA_NET_APPS_CONNECTION_TCP_V6 "outbound_conn_v6"
+#define NETDATA_NET_APPS_BANDWIDTH_SENT "total_bandwidth_sent"
+#define NETDATA_NET_APPS_BANDWIDTH_RECV "total_bandwidth_recv"
+#define NETDATA_NET_APPS_BANDWIDTH_TCP_SEND_CALLS "bandwidth_tcp_send"
+#define NETDATA_NET_APPS_BANDWIDTH_TCP_RECV_CALLS "bandwidth_tcp_recv"
+#define NETDATA_NET_APPS_BANDWIDTH_TCP_RETRANSMIT "bandwidth_tcp_retransmit"
+#define NETDATA_NET_APPS_BANDWIDTH_UDP_SEND_CALLS "bandwidth_udp_send"
+#define NETDATA_NET_APPS_BANDWIDTH_UDP_RECV_CALLS "bandwidth_udp_recv"
+
+// Network viewer charts
+#define NETDATA_NV_OUTBOUND_BYTES "outbound_bytes"
+#define NETDATA_NV_OUTBOUND_PACKETS "outbound_packets"
+#define NETDATA_NV_OUTBOUND_RETRANSMIT "outbound_retransmit"
+#define NETDATA_NV_INBOUND_BYTES "inbound_bytes"
+#define NETDATA_NV_INBOUND_PACKETS "inbound_packets"
+
+// Port range
+#define NETDATA_MINIMUM_PORT_VALUE 1
+#define NETDATA_MAXIMUM_PORT_VALUE 65535
+#define NETDATA_COMPILED_CONNECTIONS_ALLOWED 65535U
+#define NETDATA_MAXIMUM_CONNECTIONS_ALLOWED 16384U
+#define NETDATA_COMPILED_UDP_CONNECTIONS_ALLOWED 8192U
+#define NETDATA_MAXIMUM_UDP_CONNECTIONS_ALLOWED 4096U
+
+#define NETDATA_MINIMUM_IPV4_CIDR 0
+#define NETDATA_MAXIMUM_IPV4_CIDR 32
+
+// Contexts
+#define NETDATA_CGROUP_TCP_V4_CONN_CONTEXT "cgroup.net_conn_ipv4"
+#define NETDATA_CGROUP_TCP_V6_CONN_CONTEXT "cgroup.net_conn_ipv6"
+#define NETDATA_CGROUP_SOCKET_BYTES_RECV_CONTEXT "cgroup.net_bytes_recv"
+#define NETDATA_CGROUP_SOCKET_BYTES_SEND_CONTEXT "cgroup.net_bytes_send"
+#define NETDATA_CGROUP_SOCKET_TCP_RECV_CONTEXT "cgroup.net_tcp_recv"
+#define NETDATA_CGROUP_SOCKET_TCP_SEND_CONTEXT "cgroup.net_tcp_send"
+#define NETDATA_CGROUP_SOCKET_TCP_RETRANSMIT_CONTEXT "cgroup.net_retransmit"
+#define NETDATA_CGROUP_SOCKET_UDP_RECV_CONTEXT "cgroup.net_udp_recv"
+#define NETDATA_CGROUP_SOCKET_UDP_SEND_CONTEXT "cgroup.net_udp_send"
+
+#define NETDATA_SERVICES_SOCKET_TCP_V4_CONN_CONTEXT "services.net_conn_ipv4"
+#define NETDATA_SERVICES_SOCKET_TCP_V6_CONN_CONTEXT "services.net_conn_ipv6"
+#define NETDATA_SERVICES_SOCKET_BYTES_RECV_CONTEXT "services.net_bytes_recv"
+#define NETDATA_SERVICES_SOCKET_BYTES_SEND_CONTEXT "services.net_bytes_send"
+#define NETDATA_SERVICES_SOCKET_TCP_RECV_CONTEXT "services.net_tcp_recv"
+#define NETDATA_SERVICES_SOCKET_TCP_SEND_CONTEXT "services.net_tcp_send"
+#define NETDATA_SERVICES_SOCKET_TCP_RETRANSMIT_CONTEXT "services.net_retransmit"
+#define NETDATA_SERVICES_SOCKET_UDP_RECV_CONTEXT "services.net_udp_recv"
+#define NETDATA_SERVICES_SOCKET_UDP_SEND_CONTEXT "services.net_udp_send"
+
+typedef struct ebpf_socket_publish_apps {
+ // Data read
+ uint64_t bytes_sent; // Bytes sent
+ uint64_t bytes_received; // Bytes received
+ uint64_t call_tcp_sent; // Number of times tcp_sendmsg was called
+ uint64_t call_tcp_received; // Number of times tcp_cleanup_rbuf was called
+ uint64_t retransmit; // Number of times tcp_retransmit was called
+ uint64_t call_udp_sent; // Number of times udp_sendmsg was called
+ uint64_t call_udp_received; // Number of times udp_recvmsg was called
+ uint64_t call_close; // Number of times tcp_close was called
+ uint64_t call_tcp_v4_connection;// Number of times tcp_v4_connect was called
+ uint64_t call_tcp_v6_connection;// Number of times tcp_v6_connect was called
+} ebpf_socket_publish_apps_t;
+
+typedef struct ebpf_network_viewer_dimension_names {
+ char *name;
+ uint32_t hash;
+
+ uint16_t port;
+
+ struct ebpf_network_viewer_dimension_names *next;
+} ebpf_network_viewer_dim_name_t ;
+
+typedef struct ebpf_network_viewer_port_list {
+ char *value;
+ uint32_t hash;
+
+ uint16_t first;
+ uint16_t last;
+
+ uint16_t cmp_first;
+ uint16_t cmp_last;
+
+ uint16_t protocol;
+ uint32_t pid;
+ uint32_t tgid;
+ uint64_t connections;
+ struct ebpf_network_viewer_port_list *next;
+} ebpf_network_viewer_port_list_t;
+
+typedef struct netdata_passive_connection {
+ uint32_t tgid;
+ uint32_t pid;
+ uint64_t counter;
+} netdata_passive_connection_t;
+
+typedef struct netdata_passive_connection_idx {
+ uint16_t protocol;
+ uint16_t port;
+} netdata_passive_connection_idx_t;
+
+/**
+ * Union used to store ip addresses
+ */
+union netdata_ip_t {
+ uint8_t addr8[16];
+ uint16_t addr16[8];
+ uint32_t addr32[4];
+ uint64_t addr64[2];
+};
+
+typedef struct ebpf_network_viewer_ip_list {
+ char *value; // IP value
+ uint32_t hash; // IP hash
+
+ uint8_t ver; // IP version
+
+ union netdata_ip_t first; // The IP address informed
+ union netdata_ip_t last; // The IP address informed
+
+ struct ebpf_network_viewer_ip_list *next;
+} ebpf_network_viewer_ip_list_t;
+
+typedef struct ebpf_network_viewer_hostname_list {
+ char *value; // IP value
+ uint32_t hash; // IP hash
+
+ SIMPLE_PATTERN *value_pattern;
+
+ struct ebpf_network_viewer_hostname_list *next;
+} ebpf_network_viewer_hostname_list_t;
+
+#define NETDATA_NV_CAP_VALUE 50L
+typedef struct ebpf_network_viewer_options {
+ uint32_t max_dim; // Store value read from 'maximum dimensions'
+
+ uint32_t hostname_resolution_enabled;
+ uint32_t service_resolution_enabled;
+
+ ebpf_network_viewer_port_list_t *excluded_port;
+ ebpf_network_viewer_port_list_t *included_port;
+
+ ebpf_network_viewer_dim_name_t *names;
+
+ ebpf_network_viewer_ip_list_t *excluded_ips;
+ ebpf_network_viewer_ip_list_t *included_ips;
+
+ ebpf_network_viewer_hostname_list_t *excluded_hostnames;
+ ebpf_network_viewer_hostname_list_t *included_hostnames;
+
+ ebpf_network_viewer_ip_list_t *ipv4_local_ip;
+ ebpf_network_viewer_ip_list_t *ipv6_local_ip;
+} ebpf_network_viewer_options_t;
+
+extern ebpf_network_viewer_options_t network_viewer_opt;
+
+/**
+ * Structure to store socket information
+ */
+typedef struct netdata_socket {
+ uint64_t recv_packets;
+ uint64_t sent_packets;
+ uint64_t recv_bytes;
+ uint64_t sent_bytes;
+ uint64_t first; // First timestamp
+ uint64_t ct; // Current timestamp
+ uint32_t retransmit; // It is never used with UDP
+ uint16_t protocol;
+ uint16_t reserved;
+} netdata_socket_t;
+
+typedef struct netdata_plot_values {
+ // Values used in the previous iteration
+ uint64_t recv_packets;
+ uint64_t sent_packets;
+ uint64_t recv_bytes;
+ uint64_t sent_bytes;
+ uint32_t retransmit;
+
+ uint64_t last_time;
+
+ // Values used to plot
+ uint64_t plot_recv_packets;
+ uint64_t plot_sent_packets;
+ uint64_t plot_recv_bytes;
+ uint64_t plot_sent_bytes;
+ uint16_t plot_retransmit;
+} netdata_plot_values_t;
+
+/**
+ * Index used together previous structure
+ */
+typedef struct netdata_socket_idx {
+ union netdata_ip_t saddr;
+ uint16_t sport;
+ union netdata_ip_t daddr;
+ uint16_t dport;
+} netdata_socket_idx_t;
+
+// Next values were defined according getnameinfo(3)
+#define NETDATA_MAX_NETWORK_COMBINED_LENGTH 1018
+#define NETDATA_DOTS_PROTOCOL_COMBINED_LENGTH 5 // :TCP:
+#define NETDATA_DIM_LENGTH_WITHOUT_SERVICE_PROTOCOL 979
+
+#define NETDATA_INBOUND_DIRECTION (uint32_t)1
+#define NETDATA_OUTBOUND_DIRECTION (uint32_t)2
+/**
+ * Allocate the maximum number of structures in the beginning, this can force the collector to use more memory
+ * in the long term, on the other had it is faster.
+ */
+typedef struct netdata_socket_plot {
+ // Search
+ avl_t avl;
+ netdata_socket_idx_t index;
+
+ // Current data
+ netdata_socket_t sock;
+
+ // Previous values and values used to write on chart.
+ netdata_plot_values_t plot;
+
+ int family; // AF_INET or AF_INET6
+ char *resolved_name; // Resolve only in the first call
+ unsigned char resolved;
+
+ char *dimension_sent;
+ char *dimension_recv;
+ char *dimension_retransmit;
+
+ uint32_t flags;
+} netdata_socket_plot_t;
+
+#define NETWORK_VIEWER_CHARTS_CREATED (uint32_t)1
+typedef struct netdata_vector_plot {
+ netdata_socket_plot_t *plot; // Vector used to plot charts
+
+ avl_tree_lock tree; // AVL tree to speed up search
+ uint32_t last; // The 'other' dimension, the last chart accepted.
+ uint32_t next; // The next position to store in the vector.
+ uint32_t max_plot; // Max number of elements to plot.
+ uint32_t last_plot; // Last element plot
+
+ uint32_t flags; // Flags
+
+} netdata_vector_plot_t;
+
+void clean_port_structure(ebpf_network_viewer_port_list_t **clean);
+extern ebpf_network_viewer_port_list_t *listen_ports;
+void update_listen_table(uint16_t value, uint16_t proto, netdata_passive_connection_t *values);
+void parse_network_viewer_section(struct config *cfg);
+void fill_ip_list(ebpf_network_viewer_ip_list_t **out, ebpf_network_viewer_ip_list_t *in, char *table);
+void parse_service_name_section(struct config *cfg);
+
+extern ebpf_socket_publish_apps_t **socket_bandwidth_curr;
+extern struct config socket_config;
+extern netdata_ebpf_targets_t socket_targets[];
+
+#endif
diff --git a/collectors/ebpf.plugin/ebpf_softirq.c b/collectors/ebpf.plugin/ebpf_softirq.c
new file mode 100644
index 0000000..3b5d159
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_softirq.c
@@ -0,0 +1,290 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_softirq.h"
+
+struct config softirq_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+#define SOFTIRQ_MAP_LATENCY 0
+static ebpf_local_maps_t softirq_maps[] = {
+ {
+ .name = "tbl_softirq",
+ .internal_input = NETDATA_SOFTIRQ_MAX_IRQS,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ },
+ /* end */
+ {
+ .name = NULL,
+ .internal_input = 0,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
+ }
+};
+
+#define SOFTIRQ_TP_CLASS_IRQ "irq"
+static ebpf_tracepoint_t softirq_tracepoints[] = {
+ {.enabled = false, .class = SOFTIRQ_TP_CLASS_IRQ, .event = "softirq_entry"},
+ {.enabled = false, .class = SOFTIRQ_TP_CLASS_IRQ, .event = "softirq_exit"},
+ /* end */
+ {.enabled = false, .class = NULL, .event = NULL}
+};
+
+// these must be in the order defined by the kernel:
+// https://elixir.bootlin.com/linux/v5.12.19/source/include/trace/events/irq.h#L13
+static softirq_val_t softirq_vals[] = {
+ {.name = "HI", .latency = 0},
+ {.name = "TIMER", .latency = 0},
+ {.name = "NET_TX", .latency = 0},
+ {.name = "NET_RX", .latency = 0},
+ {.name = "BLOCK", .latency = 0},
+ {.name = "IRQ_POLL", .latency = 0},
+ {.name = "TASKLET", .latency = 0},
+ {.name = "SCHED", .latency = 0},
+ {.name = "HRTIMER", .latency = 0},
+ {.name = "RCU", .latency = 0},
+};
+
+// tmp store for soft IRQ values we get from a per-CPU eBPF map.
+static softirq_ebpf_val_t *softirq_ebpf_vals = NULL;
+
+static struct netdata_static_thread softirq_threads = {
+ .name = "SOFTIRQ KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+/**
+ * Cachestat Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_softirq_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ freez(softirq_threads.thread);
+
+ for (int i = 0; softirq_tracepoints[i].class != NULL; i++) {
+ ebpf_disable_tracepoint(&softirq_tracepoints[i]);
+ }
+ freez(softirq_ebpf_vals);
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Exit
+ *
+ * Cancel thread.
+ *
+ * @param ptr thread data.
+ */
+static void softirq_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*softirq_threads.thread);
+ ebpf_softirq_free(em);
+}
+
+/**
+ * Cleanup
+ *
+ * Clean up allocated memory.
+ *
+ * @param ptr thread data.
+ */
+static void softirq_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_softirq_free(em);
+}
+
+/*****************************************************************
+ * MAIN LOOP
+ *****************************************************************/
+
+static void softirq_read_latency_map()
+{
+ int fd = softirq_maps[SOFTIRQ_MAP_LATENCY].map_fd;
+ int i;
+ for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) {
+ int test = bpf_map_lookup_elem(fd, &i, softirq_ebpf_vals);
+ if (unlikely(test < 0)) {
+ continue;
+ }
+
+ uint64_t total_latency = 0;
+ int cpu_i;
+ int end = ebpf_nprocs;
+ for (cpu_i = 0; cpu_i < end; cpu_i++) {
+ total_latency += softirq_ebpf_vals[cpu_i].latency/1000;
+ }
+
+ softirq_vals[i].latency = total_latency;
+ }
+}
+
+/**
+ * Read eBPF maps for soft IRQ.
+ */
+static void *softirq_reader(void *ptr)
+{
+ netdata_thread_cleanup_push(softirq_exit, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ usec_t step = NETDATA_SOFTIRQ_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ softirq_read_latency_map();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+static void softirq_create_charts(int update_every)
+{
+ ebpf_create_chart(
+ NETDATA_EBPF_SYSTEM_GROUP,
+ "softirq_latency",
+ "Software IRQ latency",
+ EBPF_COMMON_DIMENSION_MILLISECONDS,
+ "softirqs",
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ NETDATA_CHART_PRIO_SYSTEM_SOFTIRQS+1,
+ NULL, NULL, 0, update_every,
+ NETDATA_EBPF_MODULE_NAME_SOFTIRQ
+ );
+
+ fflush(stdout);
+}
+
+static void softirq_create_dims()
+{
+ uint32_t i;
+ for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) {
+ ebpf_write_global_dimension(
+ softirq_vals[i].name, softirq_vals[i].name,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX]
+ );
+ }
+}
+
+static inline void softirq_write_dims()
+{
+ uint32_t i;
+ for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) {
+ write_chart_dimension(softirq_vals[i].name, softirq_vals[i].latency);
+ }
+}
+
+/**
+* Main loop for this collector.
+*/
+static void softirq_collector(ebpf_module_t *em)
+{
+ softirq_ebpf_vals = callocz(ebpf_nprocs, sizeof(softirq_ebpf_val_t));
+
+ // create reader thread.
+ softirq_threads.thread = mallocz(sizeof(netdata_thread_t));
+ softirq_threads.start_routine = softirq_reader;
+ netdata_thread_create(
+ softirq_threads.thread,
+ softirq_threads.name,
+ NETDATA_THREAD_OPTION_DEFAULT,
+ softirq_reader,
+ em
+ );
+
+ // create chart and static dims.
+ pthread_mutex_lock(&lock);
+ softirq_create_charts(em->update_every);
+ softirq_create_dims();
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ // loop and read from published data until ebpf plugin is closed.
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = em->update_every * USEC_PER_SEC;
+ //This will be cancelled by its parent
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ pthread_mutex_lock(&lock);
+
+ // write dims now for all hitherto discovered IRQs.
+ write_begin_chart(NETDATA_EBPF_SYSTEM_GROUP, "softirq_latency");
+ softirq_write_dims();
+ write_end_chart();
+
+ pthread_mutex_unlock(&lock);
+ }
+}
+
+/*****************************************************************
+ * EBPF SOFTIRQ THREAD
+ *****************************************************************/
+
+/**
+ * Soft IRQ latency thread.
+ *
+ * @param ptr a `ebpf_module_t *`.
+ * @return always NULL.
+ */
+void *ebpf_softirq_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(softirq_cleanup, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = softirq_maps;
+
+ if (ebpf_enable_tracepoints(softirq_tracepoints) == 0) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endsoftirq;
+ }
+
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endsoftirq;
+ }
+
+ softirq_collector(em);
+
+endsoftirq:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_softirq.h b/collectors/ebpf.plugin/ebpf_softirq.h
new file mode 100644
index 0000000..7dcddbb
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_softirq.h
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_SOFTIRQ_H
+#define NETDATA_EBPF_SOFTIRQ_H 1
+
+/*****************************************************************
+ * copied from kernel-collectors repo, with modifications needed
+ * for inclusion here.
+ *****************************************************************/
+
+#define NETDATA_SOFTIRQ_MAX_IRQS 10
+
+typedef struct softirq_ebpf_val {
+ uint64_t latency;
+ uint64_t ts;
+} softirq_ebpf_val_t;
+
+/*****************************************************************
+ * below this is eBPF plugin-specific code.
+ *****************************************************************/
+
+#define NETDATA_EBPF_MODULE_NAME_SOFTIRQ "softirq"
+#define NETDATA_SOFTIRQ_SLEEP_MS 650000ULL
+#define NETDATA_SOFTIRQ_CONFIG_FILE "softirq.conf"
+
+typedef struct sofirq_val {
+ uint64_t latency;
+ char *name;
+} softirq_val_t;
+
+extern struct config softirq_config;
+void *ebpf_softirq_thread(void *ptr);
+
+#endif /* NETDATA_EBPF_SOFTIRQ_H */
diff --git a/collectors/ebpf.plugin/ebpf_swap.c b/collectors/ebpf.plugin/ebpf_swap.c
new file mode 100644
index 0000000..8199573
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_swap.c
@@ -0,0 +1,915 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_swap.h"
+
+static char *swap_dimension_name[NETDATA_SWAP_END] = { "read", "write" };
+static netdata_syscall_stat_t swap_aggregated_data[NETDATA_SWAP_END];
+static netdata_publish_syscall_t swap_publish_aggregated[NETDATA_SWAP_END];
+
+netdata_publish_swap_t *swap_vector = NULL;
+
+static netdata_idx_t swap_hash_values[NETDATA_SWAP_END];
+static netdata_idx_t *swap_values = NULL;
+
+netdata_publish_swap_t **swap_pid = NULL;
+
+struct config swap_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+static ebpf_local_maps_t swap_maps[] = {{.name = "tbl_pid_swap", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "swap_ctrl", .internal_input = NETDATA_CONTROLLER_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_swap", .internal_input = NETDATA_SWAP_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0}};
+
+struct netdata_static_thread swap_threads = {
+ .name = "SWAP KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+netdata_ebpf_targets_t swap_targets[] = { {.name = "swap_readpage", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "swap_writepage", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/swap.skel.h" // BTF code
+
+static struct swap_bpf *bpf_obj = NULL;
+
+/**
+ * Disable probe
+ *
+ * Disable all probes to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects
+ */
+static void ebpf_swap_disable_probe(struct swap_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_swap_readpage_probe, false);
+ bpf_program__set_autoload(obj->progs.netdata_swap_writepage_probe, false);
+}
+
+/*
+ * Disable trampoline
+ *
+ * Disable all trampoline to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_swap_disable_trampoline(struct swap_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_swap_readpage_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_swap_writepage_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fentry, false);
+}
+
+/**
+ * Set trampoline target
+ *
+ * Set the targets we will monitor.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_swap_set_trampoline_target(struct swap_bpf *obj)
+{
+ bpf_program__set_attach_target(obj->progs.netdata_swap_readpage_fentry, 0,
+ swap_targets[NETDATA_KEY_SWAP_READPAGE_CALL].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_swap_writepage_fentry, 0,
+ swap_targets[NETDATA_KEY_SWAP_WRITEPAGE_CALL].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_release_task_fentry, 0,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+}
+
+/**
+ * Mount Attach Probe
+ *
+ * Attach probes to target
+ *
+ * @param obj is the main structure for bpf objects.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+static int ebpf_swap_attach_kprobe(struct swap_bpf *obj)
+{
+ obj->links.netdata_swap_readpage_probe = bpf_program__attach_kprobe(obj->progs.netdata_swap_readpage_probe,
+ false,
+ swap_targets[NETDATA_KEY_SWAP_READPAGE_CALL].name);
+ int ret = libbpf_get_error(obj->links.netdata_swap_readpage_probe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_swap_writepage_probe = bpf_program__attach_kprobe(obj->progs.netdata_swap_writepage_probe,
+ false,
+ swap_targets[NETDATA_KEY_SWAP_WRITEPAGE_CALL].name);
+ ret = libbpf_get_error(obj->links.netdata_swap_writepage_probe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_release_task_probe = bpf_program__attach_kprobe(obj->progs.netdata_release_task_probe,
+ false,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+ ret = libbpf_get_error(obj->links.netdata_swap_writepage_probe);
+ if (ret)
+ return -1;
+
+ return 0;
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_swap_set_hash_tables(struct swap_bpf *obj)
+{
+ swap_maps[NETDATA_PID_SWAP_TABLE].map_fd = bpf_map__fd(obj->maps.tbl_pid_swap);
+ swap_maps[NETDATA_SWAP_CONTROLLER].map_fd = bpf_map__fd(obj->maps.swap_ctrl);
+ swap_maps[NETDATA_SWAP_GLOBAL_TABLE].map_fd = bpf_map__fd(obj->maps.tbl_swap);
+}
+
+/**
+ * Adjust Map Size
+ *
+ * Resize maps according input from users.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ */
+static void ebpf_swap_adjust_map_size(struct swap_bpf *obj, ebpf_module_t *em)
+{
+ ebpf_update_map_size(obj->maps.tbl_pid_swap, &swap_maps[NETDATA_PID_SWAP_TABLE],
+ em, bpf_map__name(obj->maps.tbl_pid_swap));
+}
+
+/**
+ * Disable Release Task
+ *
+ * Disable release task when apps is not enabled.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_swap_disable_release_task(struct swap_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_release_task_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_release_task_probe, false);
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_swap_load_and_attach(struct swap_bpf *obj, ebpf_module_t *em)
+{
+ netdata_ebpf_targets_t *mt = em->targets;
+ netdata_ebpf_program_loaded_t test = mt[NETDATA_KEY_SWAP_READPAGE_CALL].mode;
+
+ if (test == EBPF_LOAD_TRAMPOLINE) {
+ ebpf_swap_disable_probe(obj);
+
+ ebpf_swap_set_trampoline_target(obj);
+ } else {
+ ebpf_swap_disable_trampoline(obj);
+ }
+
+ ebpf_swap_adjust_map_size(obj, em);
+
+ if (!em->apps_charts && !em->cgroup_charts)
+ ebpf_swap_disable_release_task(obj);
+
+ int ret = swap_bpf__load(obj);
+ if (ret) {
+ return ret;
+ }
+
+ ret = (test == EBPF_LOAD_TRAMPOLINE) ? swap_bpf__attach(obj) : ebpf_swap_attach_kprobe(obj);
+ if (!ret) {
+ ebpf_swap_set_hash_tables(obj);
+
+ ebpf_update_controller(swap_maps[NETDATA_SWAP_CONTROLLER].map_fd, em);
+ }
+
+ return ret;
+}
+#endif
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Cachestat Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_swap_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ ebpf_cleanup_publish_syscall(swap_publish_aggregated);
+
+ freez(swap_vector);
+ freez(swap_values);
+ freez(swap_threads.thread);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ if (bpf_obj)
+ swap_bpf__destroy(bpf_obj);
+#endif
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Swap exit
+ *
+ * Cancel thread and exit.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_swap_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*swap_threads.thread);
+ ebpf_swap_free(em);
+}
+
+/**
+ * Swap cleanup
+ *
+ * Clean up allocated memory.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_swap_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_swap_free(em);
+}
+
+/*****************************************************************
+ *
+ * COLLECTOR THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Apps Accumulator
+ *
+ * Sum all values read from kernel and store in the first address.
+ *
+ * @param out the vector with read values.
+ */
+static void swap_apps_accumulator(netdata_publish_swap_t *out)
+{
+ int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1;
+ netdata_publish_swap_t *total = &out[0];
+ for (i = 1; i < end; i++) {
+ netdata_publish_swap_t *w = &out[i];
+ total->write += w->write;
+ total->read += w->read;
+ }
+}
+
+/**
+ * Fill PID
+ *
+ * Fill PID structures
+ *
+ * @param current_pid pid that we are collecting data
+ * @param out values read from hash tables;
+ */
+static void swap_fill_pid(uint32_t current_pid, netdata_publish_swap_t *publish)
+{
+ netdata_publish_swap_t *curr = swap_pid[current_pid];
+ if (!curr) {
+ curr = callocz(1, sizeof(netdata_publish_swap_t));
+ swap_pid[current_pid] = curr;
+ }
+
+ memcpy(curr, publish, sizeof(netdata_publish_swap_t));
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ */
+static void ebpf_update_swap_cgroup()
+{
+ ebpf_cgroup_target_t *ect ;
+ netdata_publish_swap_t *cv = swap_vector;
+ int fd = swap_maps[NETDATA_PID_SWAP_TABLE].map_fd;
+ size_t length = sizeof(netdata_publish_swap_t)*ebpf_nprocs;
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ int pid = pids->pid;
+ netdata_publish_swap_t *out = &pids->swap;
+ if (likely(swap_pid) && swap_pid[pid]) {
+ netdata_publish_swap_t *in = swap_pid[pid];
+
+ memcpy(out, in, sizeof(netdata_publish_swap_t));
+ } else {
+ memset(cv, 0, length);
+ if (!bpf_map_lookup_elem(fd, &pid, cv)) {
+ swap_apps_accumulator(cv);
+
+ memcpy(out, cv, sizeof(netdata_publish_swap_t));
+ }
+ }
+ }
+ }
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Read APPS table
+ *
+ * Read the apps table and store data inside the structure.
+ */
+static void read_apps_table()
+{
+ netdata_publish_swap_t *cv = swap_vector;
+ uint32_t key;
+ struct pid_stat *pids = root_of_pids;
+ int fd = swap_maps[NETDATA_PID_SWAP_TABLE].map_fd;
+ size_t length = sizeof(netdata_publish_swap_t)*ebpf_nprocs;
+ while (pids) {
+ key = pids->pid;
+
+ if (bpf_map_lookup_elem(fd, &key, cv)) {
+ pids = pids->next;
+ continue;
+ }
+
+ swap_apps_accumulator(cv);
+
+ swap_fill_pid(key, cv);
+
+ // We are cleaning to avoid passing data read from one process to other.
+ memset(cv, 0, length);
+
+ pids = pids->next;
+ }
+}
+
+/**
+* Send global
+*
+* Send global charts to Netdata
+*/
+static void swap_send_global()
+{
+ write_io_chart(NETDATA_MEM_SWAP_CHART, NETDATA_EBPF_SYSTEM_GROUP,
+ swap_publish_aggregated[NETDATA_KEY_SWAP_WRITEPAGE_CALL].dimension,
+ (long long) swap_hash_values[NETDATA_KEY_SWAP_WRITEPAGE_CALL],
+ swap_publish_aggregated[NETDATA_KEY_SWAP_READPAGE_CALL].dimension,
+ (long long) swap_hash_values[NETDATA_KEY_SWAP_READPAGE_CALL]);
+}
+
+/**
+ * Read global counter
+ *
+ * Read the table with number of calls to all functions
+ */
+static void read_global_table()
+{
+ netdata_idx_t *stored = swap_values;
+ netdata_idx_t *val = swap_hash_values;
+ int fd = swap_maps[NETDATA_SWAP_GLOBAL_TABLE].map_fd;
+
+ uint32_t i, end = NETDATA_SWAP_END;
+ for (i = NETDATA_KEY_SWAP_READPAGE_CALL; i < end; i++) {
+ if (!bpf_map_lookup_elem(fd, &i, stored)) {
+ int j;
+ int last = ebpf_nprocs;
+ netdata_idx_t total = 0;
+ for (j = 0; j < last; j++)
+ total += stored[j];
+
+ val[i] = total;
+ }
+ }
+}
+
+/**
+ * Swap read hash
+ *
+ * This is the thread callback.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_swap_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_swap_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ usec_t step = NETDATA_SWAP_SLEEP_MS * em->update_every;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_global_table();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Sum PIDs
+ *
+ * Sum values for all targets.
+ *
+ * @param swap
+ * @param root
+ */
+static void ebpf_swap_sum_pids(netdata_publish_swap_t *swap, struct pid_on_target *root)
+{
+ uint64_t local_read = 0;
+ uint64_t local_write = 0;
+
+ while (root) {
+ int32_t pid = root->pid;
+ netdata_publish_swap_t *w = swap_pid[pid];
+ if (w) {
+ local_write += w->write;
+ local_read += w->read;
+ }
+ root = root->next;
+ }
+
+ // These conditions were added, because we are using incremental algorithm
+ swap->write = (local_write >= swap->write) ? local_write : swap->write;
+ swap->read = (local_read >= swap->read) ? local_read : swap->read;
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param root the target list.
+*/
+void ebpf_swap_send_apps_data(struct target *root)
+{
+ struct target *w;
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ ebpf_swap_sum_pids(&w->swap, w->root_pid);
+ }
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_MEM_SWAP_READ_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, (long long) w->swap.read);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_MEM_SWAP_WRITE_CHART);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, (long long) w->swap.write);
+ }
+ }
+ write_end_chart();
+}
+
+/**
+ * Sum PIDs
+ *
+ * Sum values for all targets.
+ *
+ * @param swap
+ * @param root
+ */
+static void ebpf_swap_sum_cgroup_pids(netdata_publish_swap_t *swap, struct pid_on_target2 *pids)
+{
+ uint64_t local_read = 0;
+ uint64_t local_write = 0;
+
+ while (pids) {
+ netdata_publish_swap_t *w = &pids->swap;
+ local_write += w->write;
+ local_read += w->read;
+
+ pids = pids->next;
+ }
+
+ // These conditions were added, because we are using incremental algorithm
+ swap->write = (local_write >= swap->write) ? local_write : swap->write;
+ swap->read = (local_read >= swap->read) ? local_read : swap->read;
+}
+
+/**
+ * Send Systemd charts
+ *
+ * Send collected data to Netdata.
+ */
+static void ebpf_send_systemd_swap_charts()
+{
+ ebpf_cgroup_target_t *ect;
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_MEM_SWAP_READ_CHART);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long) ect->publish_systemd_swap.read);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_MEM_SWAP_WRITE_CHART);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, (long long) ect->publish_systemd_swap.write);
+ }
+ }
+ write_end_chart();
+}
+
+/**
+ * Create specific swap charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_specific_swap_charts(char *type, int update_every)
+{
+ ebpf_create_chart(type, NETDATA_MEM_SWAP_READ_CHART,
+ "Calls to function <code>swap_readpage</code>.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_SYSTEM_CGROUP_SWAP_SUBMENU,
+ NETDATA_CGROUP_SWAP_READ_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5100,
+ ebpf_create_global_dimension,
+ swap_publish_aggregated, 1, update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ ebpf_create_chart(type, NETDATA_MEM_SWAP_WRITE_CHART,
+ "Calls to function <code>swap_writepage</code>.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_SYSTEM_CGROUP_SWAP_SUBMENU,
+ NETDATA_CGROUP_SWAP_WRITE_CONTEXT, NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5101,
+ ebpf_create_global_dimension,
+ &swap_publish_aggregated[NETDATA_KEY_SWAP_WRITEPAGE_CALL], 1,
+ update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+}
+
+/**
+ * Create specific swap charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_obsolete_specific_swap_charts(char *type, int update_every)
+{
+ ebpf_write_chart_obsolete(type, NETDATA_MEM_SWAP_READ_CHART,"Calls to function <code>swap_readpage</code>.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_SYSTEM_CGROUP_SWAP_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_SWAP_READ_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5100, update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_MEM_SWAP_WRITE_CHART, "Calls to function <code>swap_writepage</code>.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_SYSTEM_CGROUP_SWAP_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_SWAP_WRITE_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5101, update_every);
+}
+
+/*
+ * Send Specific Swap data
+ *
+ * Send data for specific cgroup/apps.
+ *
+ * @param type chart type
+ * @param values structure with values that will be sent to netdata
+ */
+static void ebpf_send_specific_swap_data(char *type, netdata_publish_swap_t *values)
+{
+ write_begin_chart(type, NETDATA_MEM_SWAP_READ_CHART);
+ write_chart_dimension(swap_publish_aggregated[NETDATA_KEY_SWAP_READPAGE_CALL].name, (long long) values->read);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_MEM_SWAP_WRITE_CHART);
+ write_chart_dimension(swap_publish_aggregated[NETDATA_KEY_SWAP_WRITEPAGE_CALL].name, (long long) values->write);
+ write_end_chart();
+}
+
+/**
+ * Create Systemd Swap Charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ **/
+static void ebpf_create_systemd_swap_charts(int update_every)
+{
+ ebpf_create_charts_on_systemd(NETDATA_MEM_SWAP_READ_CHART,
+ "Calls to <code>swap_readpage</code>.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_SYSTEM_CGROUP_SWAP_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20191,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_SWAP_READ_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_SWAP, update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_MEM_SWAP_WRITE_CHART,
+ "Calls to function <code>swap_writepage</code>.",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_SYSTEM_CGROUP_SWAP_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20192,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_SWAP_WRITE_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_SWAP, update_every);
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+*/
+void ebpf_swap_send_cgroup_data(int update_every)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ ebpf_swap_sum_cgroup_pids(&ect->publish_systemd_swap, ect->pids);
+ }
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_swap_charts(update_every);
+ fflush(stdout);
+ }
+ ebpf_send_systemd_swap_charts();
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_SWAP_CHART) && ect->updated) {
+ ebpf_create_specific_swap_charts(ect->name, update_every);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_SWAP_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_SWAP_CHART) {
+ if (ect->updated) {
+ ebpf_send_specific_swap_data(ect->name, &ect->publish_systemd_swap);
+ } else {
+ ebpf_obsolete_specific_swap_charts(ect->name, update_every);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_SWAP_CHART;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+* Main loop for this collector.
+*/
+static void swap_collector(ebpf_module_t *em)
+{
+ swap_threads.thread = mallocz(sizeof(netdata_thread_t));
+ swap_threads.start_routine = ebpf_swap_read_hash;
+
+ netdata_thread_create(swap_threads.thread, swap_threads.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_swap_read_hash, em);
+
+ int cgroup = em->cgroup_charts;
+ int update_every = em->update_every;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ netdata_apps_integration_flags_t apps = em->apps_charts;
+ pthread_mutex_lock(&collect_data_mutex);
+ if (apps)
+ read_apps_table();
+
+ if (cgroup)
+ ebpf_update_swap_cgroup();
+
+ pthread_mutex_lock(&lock);
+
+ swap_send_global();
+
+ if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED)
+ ebpf_swap_send_apps_data(apps_groups_root_target);
+
+ if (cgroup)
+ ebpf_swap_send_cgroup_data(update_every);
+
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+}
+
+/*****************************************************************
+ *
+ * INITIALIZE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Create apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+void ebpf_swap_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+ ebpf_create_charts_on_apps(NETDATA_MEM_SWAP_READ_CHART,
+ "Calls to function <code>swap_readpage</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_SWAP_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20191,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ ebpf_create_charts_on_apps(NETDATA_MEM_SWAP_WRITE_CHART,
+ "Calls to function <code>swap_writepage</code>.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_SWAP_SUBMENU,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20192,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/**
+ * Allocate vectors used with this thread.
+ *
+ * We are not testing the return, because callocz does this and shutdown the software
+ * case it was not possible to allocate.
+ *
+ * @param apps is apps enabled?
+ */
+static void ebpf_swap_allocate_global_vectors(int apps)
+{
+ if (apps)
+ swap_pid = callocz((size_t)pid_max, sizeof(netdata_publish_swap_t *));
+
+ swap_vector = callocz((size_t)ebpf_nprocs, sizeof(netdata_publish_swap_t));
+
+ swap_values = callocz((size_t)ebpf_nprocs, sizeof(netdata_idx_t));
+
+ memset(swap_hash_values, 0, sizeof(swap_hash_values));
+}
+
+/*****************************************************************
+ *
+ * MAIN THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Create global charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_swap_charts(int update_every)
+{
+ ebpf_create_chart(NETDATA_EBPF_SYSTEM_GROUP, NETDATA_MEM_SWAP_CHART,
+ "Calls to access swap memory",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_SYSTEM_SWAP_SUBMENU,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ 202,
+ ebpf_create_global_dimension,
+ swap_publish_aggregated, NETDATA_SWAP_END,
+ update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+}
+
+/*
+ * Load BPF
+ *
+ * Load BPF files.
+ *
+ * @param em the structure with configuration
+ */
+static int ebpf_swap_load_bpf(ebpf_module_t *em)
+{
+ int ret = 0;
+ ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_KEY_SWAP_READPAGE_CALL].mode);
+ if (em->load & EBPF_LOAD_LEGACY) {
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ ret = -1;
+ }
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ bpf_obj = swap_bpf__open();
+ if (!bpf_obj)
+ ret = -1;
+ else
+ ret = ebpf_swap_load_and_attach(bpf_obj, em);
+ }
+#endif
+
+ if (ret)
+ error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->thread_name);
+
+ return ret;
+}
+
+/**
+ * SWAP thread
+ *
+ * Thread used to make swap thread
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always return NULL
+ */
+void *ebpf_swap_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_swap_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = swap_maps;
+
+ ebpf_update_pid_table(&swap_maps[NETDATA_PID_SWAP_TABLE], em);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_swap_load_bpf(em)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endswap;
+ }
+
+ ebpf_swap_allocate_global_vectors(em->apps_charts);
+
+ int algorithms[NETDATA_SWAP_END] = { NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX };
+ ebpf_global_labels(swap_aggregated_data, swap_publish_aggregated, swap_dimension_name, swap_dimension_name,
+ algorithms, NETDATA_SWAP_END);
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_swap_charts(em->update_every);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ swap_collector(em);
+
+endswap:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_swap.h b/collectors/ebpf.plugin/ebpf_swap.h
new file mode 100644
index 0000000..79182e5
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_swap.h
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_SWAP_H
+#define NETDATA_EBPF_SWAP_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_SWAP "swap"
+
+#define NETDATA_SWAP_SLEEP_MS 850000ULL
+
+// charts
+#define NETDATA_MEM_SWAP_CHART "swapcalls"
+#define NETDATA_MEM_SWAP_READ_CHART "swap_read_call"
+#define NETDATA_MEM_SWAP_WRITE_CHART "swap_write_call"
+#define NETDATA_SWAP_SUBMENU "swap"
+
+// configuration file
+#define NETDATA_DIRECTORY_SWAP_CONFIG_FILE "swap.conf"
+
+// Contexts
+#define NETDATA_CGROUP_SWAP_READ_CONTEXT "cgroup.swap_read"
+#define NETDATA_CGROUP_SWAP_WRITE_CONTEXT "cgroup.swap_write"
+#define NETDATA_SYSTEMD_SWAP_READ_CONTEXT "services.swap_read"
+#define NETDATA_SYSTEMD_SWAP_WRITE_CONTEXT "services.swap_write"
+
+typedef struct netdata_publish_swap {
+ uint64_t read;
+ uint64_t write;
+} netdata_publish_swap_t;
+
+enum swap_tables {
+ NETDATA_PID_SWAP_TABLE,
+ NETDATA_SWAP_CONTROLLER,
+ NETDATA_SWAP_GLOBAL_TABLE
+};
+
+enum swap_counters {
+ NETDATA_KEY_SWAP_READPAGE_CALL,
+ NETDATA_KEY_SWAP_WRITEPAGE_CALL,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_SWAP_END
+};
+
+extern netdata_publish_swap_t **swap_pid;
+
+void *ebpf_swap_thread(void *ptr);
+void ebpf_swap_create_apps_charts(struct ebpf_module *em, void *ptr);
+
+extern struct config swap_config;
+extern netdata_ebpf_targets_t swap_targets[];
+
+#endif
diff --git a/collectors/ebpf.plugin/ebpf_sync.c b/collectors/ebpf.plugin/ebpf_sync.c
new file mode 100644
index 0000000..8404975
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_sync.c
@@ -0,0 +1,608 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ebpf.h"
+#include "ebpf_sync.h"
+
+static char *sync_counter_dimension_name[NETDATA_SYNC_IDX_END] = { "sync", "syncfs", "msync", "fsync", "fdatasync",
+ "sync_file_range" };
+static netdata_syscall_stat_t sync_counter_aggregated_data[NETDATA_SYNC_IDX_END];
+static netdata_publish_syscall_t sync_counter_publish_aggregated[NETDATA_SYNC_IDX_END];
+
+static netdata_idx_t sync_hash_values[NETDATA_SYNC_IDX_END];
+
+struct netdata_static_thread sync_threads = {
+ .name = "SYNC KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+static ebpf_local_maps_t sync_maps[] = {{.name = "tbl_sync", .internal_input = NETDATA_SYNC_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_syncfs", .internal_input = NETDATA_SYNC_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_msync", .internal_input = NETDATA_SYNC_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_fsync", .internal_input = NETDATA_SYNC_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_fdatasync", .internal_input = NETDATA_SYNC_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_syncfr", .internal_input = NETDATA_SYNC_END,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
+
+struct config sync_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+netdata_ebpf_targets_t sync_targets[] = { {.name = NETDATA_SYSCALLS_SYNC, .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NETDATA_SYSCALLS_SYNCFS, .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NETDATA_SYSCALLS_MSYNC, .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NETDATA_SYSCALLS_FSYNC, .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NETDATA_SYSCALLS_FDATASYNC, .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NETDATA_SYSCALLS_SYNC_FILE_RANGE, .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+#ifdef LIBBPF_MAJOR_VERSION
+/*****************************************************************
+ *
+ * BTF FUNCTIONS
+ *
+ *****************************************************************/
+
+/**
+ * Disable probe
+ *
+ * Disable kprobe to use another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void ebpf_sync_disable_probe(struct sync_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_sync_kprobe, false);
+}
+
+/**
+ * Disable tramppoline
+ *
+ * Disable trampoline to use another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static inline void ebpf_sync_disable_trampoline(struct sync_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_sync_fentry, false);
+}
+
+/**
+ * Disable tracepoint
+ *
+ * Disable tracepoints according information given.
+ *
+ * @param obj object loaded
+ * @param idx Which syscall will not be disabled
+ */
+void ebpf_sync_disable_tracepoints(struct sync_bpf *obj, sync_syscalls_index_t idx)
+{
+ if (idx != NETDATA_SYNC_SYNC_IDX)
+ bpf_program__set_autoload(obj->progs.netdata_sync_entry, false);
+
+ if (idx != NETDATA_SYNC_SYNCFS_IDX)
+ bpf_program__set_autoload(obj->progs.netdata_syncfs_entry, false);
+
+ if (idx != NETDATA_SYNC_MSYNC_IDX)
+ bpf_program__set_autoload(obj->progs.netdata_msync_entry, false);
+
+ if (idx != NETDATA_SYNC_FSYNC_IDX)
+ bpf_program__set_autoload(obj->progs.netdata_fsync_entry, false);
+
+ if (idx != NETDATA_SYNC_FDATASYNC_IDX)
+ bpf_program__set_autoload(obj->progs.netdata_fdatasync_entry, false);
+
+ if (idx != NETDATA_SYNC_SYNC_FILE_RANGE_IDX)
+ bpf_program__set_autoload(obj->progs.netdata_sync_file_range_entry, false);
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param idx the index for the main structure
+ */
+static void ebpf_sync_set_hash_tables(struct sync_bpf *obj, sync_syscalls_index_t idx)
+{
+ sync_maps[idx].map_fd = bpf_map__fd(obj->maps.tbl_sync);
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em the structure with configuration
+ * @param target the syscall that we are attaching a tracer.
+ * @param idx the index for the main structure
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_sync_load_and_attach(struct sync_bpf *obj, ebpf_module_t *em, char *target,
+ sync_syscalls_index_t idx)
+{
+ netdata_ebpf_targets_t *synct = em->targets;
+ netdata_ebpf_program_loaded_t test = synct[NETDATA_SYNC_SYNC_IDX].mode;
+
+ if (test == EBPF_LOAD_TRAMPOLINE) {
+ ebpf_sync_disable_probe(obj);
+ ebpf_sync_disable_tracepoints(obj, NETDATA_SYNC_IDX_END);
+
+ bpf_program__set_attach_target(obj->progs.netdata_sync_fentry, 0,
+ target);
+ } else if (test == EBPF_LOAD_PROBE ||
+ test == EBPF_LOAD_RETPROBE) {
+ ebpf_sync_disable_tracepoints(obj, NETDATA_SYNC_IDX_END);
+ ebpf_sync_disable_trampoline(obj);
+ } else {
+ ebpf_sync_disable_probe(obj);
+ ebpf_sync_disable_trampoline(obj);
+
+ ebpf_sync_disable_tracepoints(obj, idx);
+ }
+
+ int ret = sync_bpf__load(obj);
+ if (!ret) {
+ if (test != EBPF_LOAD_PROBE && test != EBPF_LOAD_RETPROBE) {
+ ret = sync_bpf__attach(obj);
+ } else {
+ obj->links.netdata_sync_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_sync_kprobe,
+ false, target);
+ ret = (int)libbpf_get_error(obj->links.netdata_sync_kprobe);
+ }
+
+ if (!ret)
+ ebpf_sync_set_hash_tables(obj, idx);
+ }
+
+ return ret;
+}
+#endif
+
+/*****************************************************************
+ *
+ * CLEANUP THREAD
+ *
+ *****************************************************************/
+
+#ifdef LIBBPF_MAJOR_VERSION
+/**
+ * Cleanup Objects
+ *
+ * Cleanup loaded objects when thread was initialized.
+ */
+void ebpf_sync_cleanup_objects()
+{
+ int i;
+ for (i = 0; local_syscalls[i].syscall; i++) {
+ ebpf_sync_syscalls_t *w = &local_syscalls[i];
+ if (w->sync_obj)
+ sync_bpf__destroy(w->sync_obj);
+ }
+}
+#endif
+
+/**
+ * Sync Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_sync_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_sync_cleanup_objects();
+#endif
+ freez(sync_threads.thread);
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Exit
+ *
+ * Clean up the main thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_sync_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*sync_threads.thread);
+ ebpf_sync_free(em);
+}
+
+/**
+ * Clean up the main thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_sync_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_sync_free(em);
+}
+
+/*****************************************************************
+ *
+ * INITIALIZE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Load Legacy
+ *
+ * Load legacy code.
+ *
+ * @param w is the sync output structure with pointers to objects loaded.
+ * @param em is structure with configuration
+ *
+ * @return 0 on success and -1 otherwise.
+ */
+static int ebpf_sync_load_legacy(ebpf_sync_syscalls_t *w, ebpf_module_t *em)
+{
+ em->thread_name = w->syscall;
+ if (!w->probe_links) {
+ w->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &w->objects);
+ if (!w->probe_links) {
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Initialize Syscalls
+ *
+ * Load the eBPF programs to monitor syscalls
+ *
+ * @return 0 on success and -1 otherwise.
+ */
+static int ebpf_sync_initialize_syscall(ebpf_module_t *em)
+{
+ int i;
+ const char *saved_name = em->thread_name;
+ int errors = 0;
+ for (i = 0; local_syscalls[i].syscall; i++) {
+ ebpf_sync_syscalls_t *w = &local_syscalls[i];
+ if (w->enabled) {
+ if (em->load & EBPF_LOAD_LEGACY) {
+ if (ebpf_sync_load_legacy(w, em))
+ errors++;
+
+ em->thread_name = saved_name;
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ char syscall[NETDATA_EBPF_MAX_SYSCALL_LENGTH];
+ ebpf_select_host_prefix(syscall, NETDATA_EBPF_MAX_SYSCALL_LENGTH, w->syscall, running_on_kernel);
+ w->sync_obj = sync_bpf__open();
+ if (!w->sync_obj) {
+ errors++;
+ } else {
+ if (ebpf_is_function_inside_btf(default_btf, syscall)) {
+ if (ebpf_sync_load_and_attach(w->sync_obj, em, syscall, i)) {
+ errors++;
+ }
+ } else {
+ if (ebpf_sync_load_legacy(w, em))
+ errors++;
+ }
+ em->thread_name = saved_name;
+ }
+ }
+#endif
+ }
+ }
+ em->thread_name = saved_name;
+
+ memset(sync_counter_aggregated_data, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_syscall_stat_t));
+ memset(sync_counter_publish_aggregated, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_publish_syscall_t));
+ memset(sync_hash_values, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_idx_t));
+
+ return (errors) ? -1 : 0;
+}
+
+/*****************************************************************
+ *
+ * DATA THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Read global table
+ *
+ * Read the table with number of calls for all functions
+ */
+static void read_global_table()
+{
+ netdata_idx_t stored;
+ uint32_t idx = NETDATA_SYNC_CALL;
+ int i;
+ for (i = 0; local_syscalls[i].syscall; i++) {
+ if (local_syscalls[i].enabled) {
+ int fd = sync_maps[i].map_fd;
+ if (!bpf_map_lookup_elem(fd, &idx, &stored)) {
+ sync_hash_values[i] = stored;
+ }
+ }
+ }
+}
+
+/**
+ * Sync read hash
+ *
+ * This is the thread callback.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_sync_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_sync_cleanup, ptr);
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = NETDATA_EBPF_SYNC_SLEEP_MS * em->update_every;
+
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_global_table();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Create Sync charts
+ *
+ * Create charts and dimensions according user input.
+ *
+ * @param id chart id
+ * @param idx the first index with data.
+ * @param end the last index with data.
+ */
+static void ebpf_send_sync_chart(char *id,
+ int idx,
+ int end)
+{
+ write_begin_chart(NETDATA_EBPF_MEMORY_GROUP, id);
+
+ netdata_publish_syscall_t *move = &sync_counter_publish_aggregated[idx];
+
+ while (move && idx <= end) {
+ if (local_syscalls[idx].enabled)
+ write_chart_dimension(move->name, sync_hash_values[idx]);
+
+ move = move->next;
+ idx++;
+ }
+
+ write_end_chart();
+}
+
+/**
+ * Send data
+ *
+ * Send global charts to Netdata
+ */
+static void sync_send_data()
+{
+ if (local_syscalls[NETDATA_SYNC_FSYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_FDATASYNC_IDX].enabled) {
+ ebpf_send_sync_chart(NETDATA_EBPF_FILE_SYNC_CHART, NETDATA_SYNC_FSYNC_IDX, NETDATA_SYNC_FDATASYNC_IDX);
+ }
+
+ if (local_syscalls[NETDATA_SYNC_MSYNC_IDX].enabled)
+ ebpf_one_dimension_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_MSYNC_CHART,
+ sync_counter_publish_aggregated[NETDATA_SYNC_MSYNC_IDX].dimension,
+ sync_hash_values[NETDATA_SYNC_MSYNC_IDX]);
+
+ if (local_syscalls[NETDATA_SYNC_SYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_SYNCFS_IDX].enabled) {
+ ebpf_send_sync_chart(NETDATA_EBPF_SYNC_CHART, NETDATA_SYNC_SYNC_IDX, NETDATA_SYNC_SYNCFS_IDX);
+ }
+
+ if (local_syscalls[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].enabled)
+ ebpf_one_dimension_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_FILE_SEGMENT_CHART,
+ sync_counter_publish_aggregated[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].dimension,
+ sync_hash_values[NETDATA_SYNC_SYNC_FILE_RANGE_IDX]);
+}
+
+/**
+* Main loop for this collector.
+*/
+static void sync_collector(ebpf_module_t *em)
+{
+ sync_threads.thread = mallocz(sizeof(netdata_thread_t));
+ sync_threads.start_routine = ebpf_sync_read_hash;
+
+ netdata_thread_create(sync_threads.thread, sync_threads.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_sync_read_hash, em);
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = em->update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ pthread_mutex_lock(&lock);
+
+ sync_send_data();
+
+ pthread_mutex_unlock(&lock);
+ }
+}
+
+
+/*****************************************************************
+ *
+ * MAIN THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Create Sync charts
+ *
+ * Create charts and dimensions according user input.
+ *
+ * @param id chart id
+ * @param title chart title
+ * @param order order number of the specified chart
+ * @param idx the first index with data.
+ * @param end the last index with data.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_sync_chart(char *id,
+ char *title,
+ int order,
+ int idx,
+ int end,
+ int update_every)
+{
+ ebpf_write_chart_cmd(NETDATA_EBPF_MEMORY_GROUP, id, title, EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_EBPF_SYNC_SUBMENU, NETDATA_EBPF_CHART_TYPE_LINE, NULL, order,
+ update_every,
+ NETDATA_EBPF_MODULE_NAME_SYNC);
+
+ netdata_publish_syscall_t *move = &sync_counter_publish_aggregated[idx];
+
+ while (move && idx <= end) {
+ if (local_syscalls[idx].enabled)
+ ebpf_write_global_dimension(move->name, move->dimension, move->algorithm);
+
+ move = move->next;
+ idx++;
+ }
+}
+
+/**
+ * Create global charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_sync_charts(int update_every)
+{
+ if (local_syscalls[NETDATA_SYNC_FSYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_FDATASYNC_IDX].enabled)
+ ebpf_create_sync_chart(NETDATA_EBPF_FILE_SYNC_CHART,
+ "Monitor calls for <code>fsync(2)</code> and <code>fdatasync(2)</code>.", 21300,
+ NETDATA_SYNC_FSYNC_IDX, NETDATA_SYNC_FDATASYNC_IDX, update_every);
+
+ if (local_syscalls[NETDATA_SYNC_MSYNC_IDX].enabled)
+ ebpf_create_sync_chart(NETDATA_EBPF_MSYNC_CHART,
+ "Monitor calls for <code>msync(2)</code>.", 21301,
+ NETDATA_SYNC_MSYNC_IDX, NETDATA_SYNC_MSYNC_IDX, update_every);
+
+ if (local_syscalls[NETDATA_SYNC_SYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_SYNCFS_IDX].enabled)
+ ebpf_create_sync_chart(NETDATA_EBPF_SYNC_CHART,
+ "Monitor calls for <code>sync(2)</code> and <code>syncfs(2)</code>.", 21302,
+ NETDATA_SYNC_SYNC_IDX, NETDATA_SYNC_SYNCFS_IDX, update_every);
+
+ if (local_syscalls[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].enabled)
+ ebpf_create_sync_chart(NETDATA_EBPF_FILE_SEGMENT_CHART,
+ "Monitor calls for <code>sync_file_range(2)</code>.", 21303,
+ NETDATA_SYNC_SYNC_FILE_RANGE_IDX, NETDATA_SYNC_SYNC_FILE_RANGE_IDX, update_every);
+}
+
+/**
+ * Parse Syscalls
+ *
+ * Parse syscall options available inside ebpf.d/sync.conf
+ */
+static void ebpf_sync_parse_syscalls()
+{
+ int i;
+ for (i = 0; local_syscalls[i].syscall; i++) {
+ local_syscalls[i].enabled = appconfig_get_boolean(&sync_config, NETDATA_SYNC_CONFIG_NAME,
+ local_syscalls[i].syscall, CONFIG_BOOLEAN_YES);
+ }
+}
+
+/**
+ * Sync thread
+ *
+ * Thread used to make sync thread
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always return NULL
+ */
+void *ebpf_sync_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_sync_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = sync_maps;
+
+ ebpf_sync_parse_syscalls();
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_sync_initialize_syscall(em)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endsync;
+ }
+
+ int algorithms[NETDATA_SYNC_IDX_END] = { NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX,
+ NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX,
+ NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX };
+ ebpf_global_labels(sync_counter_aggregated_data, sync_counter_publish_aggregated,
+ sync_counter_dimension_name, sync_counter_dimension_name,
+ algorithms, NETDATA_SYNC_IDX_END);
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_sync_charts(em->update_every);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ sync_collector(em);
+
+endsync:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_sync.h b/collectors/ebpf.plugin/ebpf_sync.h
new file mode 100644
index 0000000..cace2a1
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_sync.h
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_SYNC_H
+#define NETDATA_EBPF_SYNC_H 1
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/sync.skel.h"
+#endif
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_SYNC "sync"
+
+// charts
+#define NETDATA_EBPF_SYNC_CHART "sync"
+#define NETDATA_EBPF_MSYNC_CHART "memory_map"
+#define NETDATA_EBPF_FILE_SYNC_CHART "file_sync"
+#define NETDATA_EBPF_FILE_SEGMENT_CHART "file_segment"
+#define NETDATA_EBPF_SYNC_SUBMENU "synchronization (eBPF)"
+
+#define NETDATA_SYSCALLS_SYNC "sync"
+#define NETDATA_SYSCALLS_SYNCFS "syncfs"
+#define NETDATA_SYSCALLS_MSYNC "msync"
+#define NETDATA_SYSCALLS_FSYNC "fsync"
+#define NETDATA_SYSCALLS_FDATASYNC "fdatasync"
+#define NETDATA_SYSCALLS_SYNC_FILE_RANGE "sync_file_range"
+
+#define NETDATA_EBPF_SYNC_SLEEP_MS 800000ULL
+
+// configuration file
+#define NETDATA_SYNC_CONFIG_FILE "sync.conf"
+#define NETDATA_SYNC_CONFIG_NAME "syscalls"
+
+typedef enum sync_syscalls_index {
+ NETDATA_SYNC_SYNC_IDX,
+ NETDATA_SYNC_SYNCFS_IDX,
+ NETDATA_SYNC_MSYNC_IDX,
+ NETDATA_SYNC_FSYNC_IDX,
+ NETDATA_SYNC_FDATASYNC_IDX,
+ NETDATA_SYNC_SYNC_FILE_RANGE_IDX,
+
+ NETDATA_SYNC_IDX_END
+} sync_syscalls_index_t;
+
+enum netdata_sync_charts {
+ NETDATA_SYNC_CALL,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_SYNC_END
+};
+
+enum netdata_sync_table {
+ NETDATA_SYNC_GLOBAL_TABLE
+};
+
+void *ebpf_sync_thread(void *ptr);
+extern struct config sync_config;
+extern netdata_ebpf_targets_t sync_targets[];
+
+#endif /* NETDATA_EBPF_SYNC_H */
diff --git a/collectors/ebpf.plugin/ebpf_vfs.c b/collectors/ebpf.plugin/ebpf_vfs.c
new file mode 100644
index 0000000..ad6de4a
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_vfs.c
@@ -0,0 +1,1983 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <sys/resource.h>
+
+#include "ebpf.h"
+#include "ebpf_vfs.h"
+
+static char *vfs_dimension_names[NETDATA_KEY_PUBLISH_VFS_END] = { "delete", "read", "write",
+ "fsync", "open", "create" };
+static char *vfs_id_names[NETDATA_KEY_PUBLISH_VFS_END] = { "vfs_unlink", "vfs_read", "vfs_write",
+ "vfs_fsync", "vfs_open", "vfs_create"};
+
+static netdata_idx_t *vfs_hash_values = NULL;
+static netdata_syscall_stat_t vfs_aggregated_data[NETDATA_KEY_PUBLISH_VFS_END];
+static netdata_publish_syscall_t vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_END];
+netdata_publish_vfs_t **vfs_pid = NULL;
+netdata_publish_vfs_t *vfs_vector = NULL;
+
+static ebpf_local_maps_t vfs_maps[] = {{.name = "tbl_vfs_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "tbl_vfs_stats", .internal_input = NETDATA_VFS_COUNTER,
+ .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = "vfs_ctrl", .internal_input = NETDATA_CONTROLLER_END,
+ .user_input = 0,
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
+ {.name = NULL, .internal_input = 0, .user_input = 0}};
+
+struct config vfs_config = { .first_section = NULL,
+ .last_section = NULL,
+ .mutex = NETDATA_MUTEX_INITIALIZER,
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+ .rwlock = AVL_LOCK_INITIALIZER } };
+
+struct netdata_static_thread vfs_threads = {
+ .name = "VFS KERNEL",
+ .config_section = NULL,
+ .config_name = NULL,
+ .env_name = NULL,
+ .enabled = 1,
+ .thread = NULL,
+ .init_routine = NULL,
+ .start_routine = NULL
+};
+
+netdata_ebpf_targets_t vfs_targets[] = { {.name = "vfs_write", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "vfs_writev", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "vfs_read", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "vfs_readv", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "vfs_unlink", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "vfs_fsync", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "vfs_open", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "vfs_create", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = "release_task", .mode = EBPF_LOAD_TRAMPOLINE},
+ {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
+
+#ifdef LIBBPF_MAJOR_VERSION
+#include "includes/vfs.skel.h" // BTF code
+
+static struct vfs_bpf *bpf_obj = NULL;
+
+/**
+ * Disable probe
+ *
+ * Disable all probes to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects
+ */
+static void ebpf_vfs_disable_probes(struct vfs_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_vfs_write_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_write_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_writev_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_writev_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_read_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_read_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_readv_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_readv_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_unlink_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_unlink_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_fsync_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_fsync_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_open_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_open_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_create_kprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_create_kretprobe, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_release_task_kprobe, false);
+}
+
+/*
+ * Disable trampoline
+ *
+ * Disable all trampoline to use exclusively another method.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_vfs_disable_trampoline(struct vfs_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_vfs_write_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_write_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_writev_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_writev_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_read_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_read_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_readv_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_readv_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_unlink_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_fsync_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_fsync_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_open_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_open_fexit, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_create_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_release_task_fentry, false);
+}
+
+/**
+ * Set trampoline target
+ *
+ * Set the targets we will monitor.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_vfs_set_trampoline_target(struct vfs_bpf *obj)
+{
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_write_fentry, 0, vfs_targets[NETDATA_EBPF_VFS_WRITE].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_write_fexit, 0, vfs_targets[NETDATA_EBPF_VFS_WRITE].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_writev_fentry, 0, vfs_targets[NETDATA_EBPF_VFS_WRITEV].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_writev_fexit, 0, vfs_targets[NETDATA_EBPF_VFS_WRITEV].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_read_fentry, 0, vfs_targets[NETDATA_EBPF_VFS_READ].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_read_fexit, 0, vfs_targets[NETDATA_EBPF_VFS_READ].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_readv_fentry, 0, vfs_targets[NETDATA_EBPF_VFS_READV].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_readv_fexit, 0, vfs_targets[NETDATA_EBPF_VFS_READV].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_unlink_fentry, 0, vfs_targets[NETDATA_EBPF_VFS_UNLINK].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_fsync_fentry, 0, vfs_targets[NETDATA_EBPF_VFS_FSYNC].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_fsync_fexit, 0, vfs_targets[NETDATA_EBPF_VFS_FSYNC].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_open_fentry, 0, vfs_targets[NETDATA_EBPF_VFS_OPEN].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_open_fexit, 0, vfs_targets[NETDATA_EBPF_VFS_OPEN].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_create_fentry, 0, vfs_targets[NETDATA_EBPF_VFS_CREATE].name);
+
+ bpf_program__set_attach_target(obj->progs.netdata_vfs_release_task_fentry, 0, EBPF_COMMON_FNCT_CLEAN_UP);
+}
+
+/**
+ * Attach Probe
+ *
+ * Attach probes to target
+ *
+ * @param obj is the main structure for bpf objects.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+static int ebpf_vfs_attach_probe(struct vfs_bpf *obj)
+{
+ obj->links.netdata_vfs_write_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_write_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_WRITE].name);
+ int ret = libbpf_get_error(obj->links.netdata_vfs_write_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_write_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_write_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_WRITE].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_write_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_writev_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_writev_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_WRITEV].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_writev_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_writev_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_writev_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_WRITEV].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_writev_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_read_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_read_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_READ].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_read_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_read_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_read_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_READ].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_read_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_readv_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_readv_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_READV].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_readv_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_readv_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_readv_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_READV].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_readv_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_unlink_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_unlink_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_UNLINK].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_unlink_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_unlink_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_unlink_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_UNLINK].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_unlink_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_fsync_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_fsync_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_FSYNC].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_fsync_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_fsync_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_fsync_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_FSYNC].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_fsync_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_open_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_open_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_OPEN].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_open_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_open_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_open_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_OPEN].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_open_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_create_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_create_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_CREATE].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_create_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_create_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_create_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_CREATE].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_create_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_fsync_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_fsync_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_FSYNC].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_fsync_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_fsync_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_fsync_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_FSYNC].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_fsync_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_open_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_open_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_OPEN].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_open_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_open_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_open_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_OPEN].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_open_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_create_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_create_kprobe, false,
+ vfs_targets[NETDATA_EBPF_VFS_CREATE].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_create_kprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_create_kretprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_create_kretprobe, true,
+ vfs_targets[NETDATA_EBPF_VFS_CREATE].name);
+ ret = libbpf_get_error(obj->links.netdata_vfs_create_kretprobe);
+ if (ret)
+ return -1;
+
+ obj->links.netdata_vfs_release_task_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_vfs_release_task_fentry,
+ true,
+ EBPF_COMMON_FNCT_CLEAN_UP);
+ ret = libbpf_get_error(obj->links.netdata_vfs_release_task_kprobe);
+ if (ret)
+ return -1;
+
+ return 0;
+}
+
+/**
+ * Adjust Map Size
+ *
+ * Resize maps according input from users.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ */
+static void ebpf_vfs_adjust_map_size(struct vfs_bpf *obj, ebpf_module_t *em)
+{
+ ebpf_update_map_size(obj->maps.tbl_vfs_pid, &vfs_maps[NETDATA_VFS_PID],
+ em, bpf_map__name(obj->maps.tbl_vfs_pid));
+}
+
+/**
+ * Set hash tables
+ *
+ * Set the values for maps according the value given by kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_vfs_set_hash_tables(struct vfs_bpf *obj)
+{
+ vfs_maps[NETDATA_VFS_ALL].map_fd = bpf_map__fd(obj->maps.tbl_vfs_stats);
+ vfs_maps[NETDATA_VFS_PID].map_fd = bpf_map__fd(obj->maps.tbl_vfs_pid);
+ vfs_maps[NETDATA_VFS_CTRL].map_fd = bpf_map__fd(obj->maps.vfs_ctrl);
+}
+
+/**
+ * Disable Release Task
+ *
+ * Disable release task when apps is not enabled.
+ *
+ * @param obj is the main structure for bpf objects.
+ */
+static void ebpf_vfs_disable_release_task(struct vfs_bpf *obj)
+{
+ bpf_program__set_autoload(obj->progs.netdata_vfs_release_task_fentry, false);
+ bpf_program__set_autoload(obj->progs.netdata_vfs_release_task_kprobe, false);
+}
+
+/**
+ * Load and attach
+ *
+ * Load and attach the eBPF code in kernel.
+ *
+ * @param obj is the main structure for bpf objects.
+ * @param em structure with configuration
+ *
+ * @return it returns 0 on succes and -1 otherwise
+ */
+static inline int ebpf_vfs_load_and_attach(struct vfs_bpf *obj, ebpf_module_t *em)
+{
+ netdata_ebpf_targets_t *mt = em->targets;
+ netdata_ebpf_program_loaded_t test = mt[NETDATA_EBPF_VFS_WRITE].mode;
+
+ if (test == EBPF_LOAD_TRAMPOLINE) {
+ ebpf_vfs_disable_probes(obj);
+
+ ebpf_vfs_set_trampoline_target(obj);
+ } else {
+ ebpf_vfs_disable_trampoline(obj);
+ }
+
+ ebpf_vfs_adjust_map_size(obj, em);
+
+ if (!em->apps_charts && !em->cgroup_charts)
+ ebpf_vfs_disable_release_task(obj);
+
+ int ret = vfs_bpf__load(obj);
+ if (ret) {
+ return ret;
+ }
+
+ ret = (test == EBPF_LOAD_TRAMPOLINE) ? vfs_bpf__attach(obj) : ebpf_vfs_attach_probe(obj);
+ if (!ret) {
+ ebpf_vfs_set_hash_tables(obj);
+
+ ebpf_update_controller(vfs_maps[NETDATA_VFS_CTRL].map_fd, em);
+ }
+
+ return ret;
+}
+#endif
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CLOSE THE THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Cachestat Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_vfs_free(ebpf_module_t *em)
+{
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+ return;
+ }
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+ freez(vfs_hash_values);
+ freez(vfs_vector);
+ freez(vfs_threads.thread);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ if (bpf_obj)
+ vfs_bpf__destroy(bpf_obj);
+#endif
+
+ pthread_mutex_lock(&ebpf_exit_cleanup);
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * Exit
+ *
+ * Cancel thread and exit.
+ *
+ * @param ptr thread data.
+**/
+static void ebpf_vfs_exit(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ netdata_thread_cancel(*vfs_threads.thread);
+ ebpf_vfs_free(em);
+}
+
+/**
+* Clean up the main thread.
+*
+* @param ptr thread data.
+**/
+static void ebpf_vfs_cleanup(void *ptr)
+{
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ ebpf_vfs_free(em);
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS WITH THE MAIN LOOP
+ *
+ *****************************************************************/
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the structure with thread information
+*/
+static void ebpf_vfs_send_data(ebpf_module_t *em)
+{
+ netdata_publish_vfs_common_t pvc;
+
+ pvc.write = (long)vfs_aggregated_data[NETDATA_KEY_PUBLISH_VFS_WRITE].bytes;
+ pvc.read = (long)vfs_aggregated_data[NETDATA_KEY_PUBLISH_VFS_READ].bytes;
+
+ write_count_chart(NETDATA_VFS_FILE_CLEAN_COUNT, NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_UNLINK], 1);
+
+ write_count_chart(NETDATA_VFS_FILE_IO_COUNT, NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ], 2);
+
+ if (em->mode < MODE_ENTRY) {
+ write_err_chart(NETDATA_VFS_FILE_ERR_COUNT, NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ], 2);
+ }
+
+ write_io_chart(NETDATA_VFS_IO_FILE_BYTES, NETDATA_FILESYSTEM_FAMILY, vfs_id_names[NETDATA_KEY_PUBLISH_VFS_WRITE],
+ (long long)pvc.write, vfs_id_names[NETDATA_KEY_PUBLISH_VFS_READ], (long long)pvc.read);
+
+ write_count_chart(NETDATA_VFS_FSYNC, NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC], 1);
+
+ if (em->mode < MODE_ENTRY) {
+ write_err_chart(NETDATA_VFS_FSYNC_ERR, NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC], 1);
+ }
+
+ write_count_chart(NETDATA_VFS_OPEN, NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN], 1);
+
+ if (em->mode < MODE_ENTRY) {
+ write_err_chart(NETDATA_VFS_OPEN_ERR, NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN], 1);
+ }
+
+ write_count_chart(NETDATA_VFS_CREATE, NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE], 1);
+
+ if (em->mode < MODE_ENTRY) {
+ write_err_chart(
+ NETDATA_VFS_CREATE_ERR,
+ NETDATA_FILESYSTEM_FAMILY,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE],
+ 1);
+ }
+}
+
+/**
+ * Read the hash table and store data to allocated vectors.
+ */
+static void read_global_table()
+{
+ uint64_t idx;
+ netdata_idx_t res[NETDATA_VFS_COUNTER];
+
+ netdata_idx_t *val = vfs_hash_values;
+ int fd = vfs_maps[NETDATA_VFS_ALL].map_fd;
+ for (idx = 0; idx < NETDATA_VFS_COUNTER; idx++) {
+ uint64_t total = 0;
+ if (!bpf_map_lookup_elem(fd, &idx, val)) {
+ int i;
+ int end = ebpf_nprocs;
+ for (i = 0; i < end; i++)
+ total += val[i];
+ }
+ res[idx] = total;
+ }
+
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_UNLINK].ncall = res[NETDATA_KEY_CALLS_VFS_UNLINK];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ].ncall = res[NETDATA_KEY_CALLS_VFS_READ] +
+ res[NETDATA_KEY_CALLS_VFS_READV];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_WRITE].ncall = res[NETDATA_KEY_CALLS_VFS_WRITE] +
+ res[NETDATA_KEY_CALLS_VFS_WRITEV];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC].ncall = res[NETDATA_KEY_CALLS_VFS_FSYNC];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN].ncall = res[NETDATA_KEY_CALLS_VFS_OPEN];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE].ncall = res[NETDATA_KEY_CALLS_VFS_CREATE];
+
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_UNLINK].nerr = res[NETDATA_KEY_ERROR_VFS_UNLINK];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ].nerr = res[NETDATA_KEY_ERROR_VFS_READ] +
+ res[NETDATA_KEY_ERROR_VFS_READV];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_WRITE].nerr = res[NETDATA_KEY_ERROR_VFS_WRITE] +
+ res[NETDATA_KEY_ERROR_VFS_WRITEV];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC].nerr = res[NETDATA_KEY_ERROR_VFS_FSYNC];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN].nerr = res[NETDATA_KEY_ERROR_VFS_OPEN];
+ vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE].nerr = res[NETDATA_KEY_ERROR_VFS_CREATE];
+
+ vfs_aggregated_data[NETDATA_KEY_PUBLISH_VFS_WRITE].bytes = (uint64_t)res[NETDATA_KEY_BYTES_VFS_WRITE] +
+ (uint64_t)res[NETDATA_KEY_BYTES_VFS_WRITEV];
+ vfs_aggregated_data[NETDATA_KEY_PUBLISH_VFS_READ].bytes = (uint64_t)res[NETDATA_KEY_BYTES_VFS_READ] +
+ (uint64_t)res[NETDATA_KEY_BYTES_VFS_READV];
+}
+
+/**
+ * Sum PIDs
+ *
+ * Sum values for all targets.
+ *
+ * @param swap output structure
+ * @param root link list with structure to be used
+ */
+static void ebpf_vfs_sum_pids(netdata_publish_vfs_t *vfs, struct pid_on_target *root)
+{
+ netdata_publish_vfs_t accumulator;
+ memset(&accumulator, 0, sizeof(accumulator));
+
+ while (root) {
+ int32_t pid = root->pid;
+ netdata_publish_vfs_t *w = vfs_pid[pid];
+ if (w) {
+ accumulator.write_call += w->write_call;
+ accumulator.writev_call += w->writev_call;
+ accumulator.read_call += w->read_call;
+ accumulator.readv_call += w->readv_call;
+ accumulator.unlink_call += w->unlink_call;
+ accumulator.fsync_call += w->fsync_call;
+ accumulator.open_call += w->open_call;
+ accumulator.create_call += w->create_call;
+
+ accumulator.write_bytes += w->write_bytes;
+ accumulator.writev_bytes += w->writev_bytes;
+ accumulator.read_bytes += w->read_bytes;
+ accumulator.readv_bytes += w->readv_bytes;
+
+ accumulator.write_err += w->write_err;
+ accumulator.writev_err += w->writev_err;
+ accumulator.read_err += w->read_err;
+ accumulator.readv_err += w->readv_err;
+ accumulator.unlink_err += w->unlink_err;
+ accumulator.fsync_err += w->fsync_err;
+ accumulator.open_err += w->open_err;
+ accumulator.create_err += w->create_err;
+ }
+ root = root->next;
+ }
+
+ // These conditions were added, because we are using incremental algorithm
+ vfs->write_call = (accumulator.write_call >= vfs->write_call) ? accumulator.write_call : vfs->write_call;
+ vfs->writev_call = (accumulator.writev_call >= vfs->writev_call) ? accumulator.writev_call : vfs->writev_call;
+ vfs->read_call = (accumulator.read_call >= vfs->read_call) ? accumulator.read_call : vfs->read_call;
+ vfs->readv_call = (accumulator.readv_call >= vfs->readv_call) ? accumulator.readv_call : vfs->readv_call;
+ vfs->unlink_call = (accumulator.unlink_call >= vfs->unlink_call) ? accumulator.unlink_call : vfs->unlink_call;
+ vfs->fsync_call = (accumulator.fsync_call >= vfs->fsync_call) ? accumulator.fsync_call : vfs->fsync_call;
+ vfs->open_call = (accumulator.open_call >= vfs->open_call) ? accumulator.open_call : vfs->open_call;
+ vfs->create_call = (accumulator.create_call >= vfs->create_call) ? accumulator.create_call : vfs->create_call;
+
+ vfs->write_bytes = (accumulator.write_bytes >= vfs->write_bytes) ? accumulator.write_bytes : vfs->write_bytes;
+ vfs->writev_bytes = (accumulator.writev_bytes >= vfs->writev_bytes) ? accumulator.writev_bytes : vfs->writev_bytes;
+ vfs->read_bytes = (accumulator.read_bytes >= vfs->read_bytes) ? accumulator.read_bytes : vfs->read_bytes;
+ vfs->readv_bytes = (accumulator.readv_bytes >= vfs->readv_bytes) ? accumulator.readv_bytes : vfs->readv_bytes;
+
+ vfs->write_err = (accumulator.write_err >= vfs->write_err) ? accumulator.write_err : vfs->write_err;
+ vfs->writev_err = (accumulator.writev_err >= vfs->writev_err) ? accumulator.writev_err : vfs->writev_err;
+ vfs->read_err = (accumulator.read_err >= vfs->read_err) ? accumulator.read_err : vfs->read_err;
+ vfs->readv_err = (accumulator.readv_err >= vfs->readv_err) ? accumulator.readv_err : vfs->readv_err;
+ vfs->unlink_err = (accumulator.unlink_err >= vfs->unlink_err) ? accumulator.unlink_err : vfs->unlink_err;
+ vfs->fsync_err = (accumulator.fsync_err >= vfs->fsync_err) ? accumulator.fsync_err : vfs->fsync_err;
+ vfs->open_err = (accumulator.open_err >= vfs->open_err) ? accumulator.open_err : vfs->open_err;
+ vfs->create_err = (accumulator.create_err >= vfs->create_err) ? accumulator.create_err : vfs->create_err;
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the structure with thread information
+ * @param root the target list.
+ */
+void ebpf_vfs_send_apps_data(ebpf_module_t *em, struct target *root)
+{
+ struct target *w;
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ ebpf_vfs_sum_pids(&w->vfs, w->root_pid);
+ }
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_FILE_DELETED);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.unlink_call);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.write_call + w->vfs.writev_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.write_err + w->vfs.writev_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_READ_CALLS);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.read_call + w->vfs.readv_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.read_err + w->vfs.readv_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.write_bytes + w->vfs.writev_bytes);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_READ_BYTES);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.read_bytes + w->vfs.readv_bytes);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_FSYNC);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.fsync_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.fsync_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_OPEN);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.open_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.open_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_CREATE);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.create_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR);
+ for (w = root; w; w = w->next) {
+ if (unlikely(w->exposed && w->processes)) {
+ write_chart_dimension(w->name, w->vfs.create_err);
+ }
+ }
+ write_end_chart();
+ }
+}
+
+/**
+ * Apps Accumulator
+ *
+ * Sum all values read from kernel and store in the first address.
+ *
+ * @param out the vector with read values.
+ */
+static void vfs_apps_accumulator(netdata_publish_vfs_t *out)
+{
+ int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1;
+ netdata_publish_vfs_t *total = &out[0];
+ for (i = 1; i < end; i++) {
+ netdata_publish_vfs_t *w = &out[i];
+
+ total->write_call += w->write_call;
+ total->writev_call += w->writev_call;
+ total->read_call += w->read_call;
+ total->readv_call += w->readv_call;
+ total->unlink_call += w->unlink_call;
+
+ total->write_bytes += w->write_bytes;
+ total->writev_bytes += w->writev_bytes;
+ total->read_bytes += w->read_bytes;
+ total->readv_bytes += w->readv_bytes;
+
+ total->write_err += w->write_err;
+ total->writev_err += w->writev_err;
+ total->read_err += w->read_err;
+ total->readv_err += w->readv_err;
+ total->unlink_err += w->unlink_err;
+ }
+}
+
+/**
+ * Fill PID
+ *
+ * Fill PID structures
+ *
+ * @param current_pid pid that we are collecting data
+ * @param out values read from hash tables;
+ */
+static void vfs_fill_pid(uint32_t current_pid, netdata_publish_vfs_t *publish)
+{
+ netdata_publish_vfs_t *curr = vfs_pid[current_pid];
+ if (!curr) {
+ curr = callocz(1, sizeof(netdata_publish_vfs_t));
+ vfs_pid[current_pid] = curr;
+ }
+
+ memcpy(curr, &publish[0], sizeof(netdata_publish_vfs_t));
+}
+
+/**
+ * Read the hash table and store data to allocated vectors.
+ */
+static void ebpf_vfs_read_apps()
+{
+ struct pid_stat *pids = root_of_pids;
+ netdata_publish_vfs_t *vv = vfs_vector;
+ int fd = vfs_maps[NETDATA_VFS_PID].map_fd;
+ size_t length = sizeof(netdata_publish_vfs_t) * ebpf_nprocs;
+ while (pids) {
+ uint32_t key = pids->pid;
+
+ if (bpf_map_lookup_elem(fd, &key, vv)) {
+ pids = pids->next;
+ continue;
+ }
+
+ vfs_apps_accumulator(vv);
+
+ vfs_fill_pid(key, vv);
+
+ // We are cleaning to avoid passing data read from one process to other.
+ memset(vv, 0, length);
+
+ pids = pids->next;
+ }
+}
+
+/**
+ * Update cgroup
+ *
+ * Update cgroup data based in
+ */
+static void read_update_vfs_cgroup()
+{
+ ebpf_cgroup_target_t *ect ;
+ netdata_publish_vfs_t *vv = vfs_vector;
+ int fd = vfs_maps[NETDATA_VFS_PID].map_fd;
+ size_t length = sizeof(netdata_publish_vfs_t) * ebpf_nprocs;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
+ struct pid_on_target2 *pids;
+ for (pids = ect->pids; pids; pids = pids->next) {
+ int pid = pids->pid;
+ netdata_publish_vfs_t *out = &pids->vfs;
+ if (likely(vfs_pid) && vfs_pid[pid]) {
+ netdata_publish_vfs_t *in = vfs_pid[pid];
+
+ memcpy(out, in, sizeof(netdata_publish_vfs_t));
+ } else {
+ memset(vv, 0, length);
+ if (!bpf_map_lookup_elem(fd, &pid, vv)) {
+ vfs_apps_accumulator(vv);
+
+ memcpy(out, vv, sizeof(netdata_publish_vfs_t));
+ }
+ }
+ }
+ }
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * VFS read hash
+ *
+ * This is the thread callback.
+ * This thread is necessary, because we cannot freeze the whole plugin to read the data.
+ *
+ * @param ptr It is a NULL value for this thread.
+ *
+ * @return It always returns NULL.
+ */
+void *ebpf_vfs_read_hash(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_vfs_cleanup, ptr);
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+
+ usec_t step = NETDATA_LATENCY_VFS_SLEEP_MS * em->update_every;
+ //This will be cancelled by its parent
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+
+ read_global_table();
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+/**
+ * Sum PIDs
+ *
+ * Sum values for all targets.
+ *
+ * @param vfs structure used to store data
+ * @param pids input data
+ */
+static void ebpf_vfs_sum_cgroup_pids(netdata_publish_vfs_t *vfs, struct pid_on_target2 *pids)
+ {
+ netdata_publish_vfs_t accumulator;
+ memset(&accumulator, 0, sizeof(accumulator));
+
+ while (pids) {
+ netdata_publish_vfs_t *w = &pids->vfs;
+
+ accumulator.write_call += w->write_call;
+ accumulator.writev_call += w->writev_call;
+ accumulator.read_call += w->read_call;
+ accumulator.readv_call += w->readv_call;
+ accumulator.unlink_call += w->unlink_call;
+ accumulator.fsync_call += w->fsync_call;
+ accumulator.open_call += w->open_call;
+ accumulator.create_call += w->create_call;
+
+ accumulator.write_bytes += w->write_bytes;
+ accumulator.writev_bytes += w->writev_bytes;
+ accumulator.read_bytes += w->read_bytes;
+ accumulator.readv_bytes += w->readv_bytes;
+
+ accumulator.write_err += w->write_err;
+ accumulator.writev_err += w->writev_err;
+ accumulator.read_err += w->read_err;
+ accumulator.readv_err += w->readv_err;
+ accumulator.unlink_err += w->unlink_err;
+ accumulator.fsync_err += w->fsync_err;
+ accumulator.open_err += w->open_err;
+ accumulator.create_err += w->create_err;
+
+ pids = pids->next;
+ }
+
+ // These conditions were added, because we are using incremental algorithm
+ vfs->write_call = (accumulator.write_call >= vfs->write_call) ? accumulator.write_call : vfs->write_call;
+ vfs->writev_call = (accumulator.writev_call >= vfs->writev_call) ? accumulator.writev_call : vfs->writev_call;
+ vfs->read_call = (accumulator.read_call >= vfs->read_call) ? accumulator.read_call : vfs->read_call;
+ vfs->readv_call = (accumulator.readv_call >= vfs->readv_call) ? accumulator.readv_call : vfs->readv_call;
+ vfs->unlink_call = (accumulator.unlink_call >= vfs->unlink_call) ? accumulator.unlink_call : vfs->unlink_call;
+ vfs->fsync_call = (accumulator.fsync_call >= vfs->fsync_call) ? accumulator.fsync_call : vfs->fsync_call;
+ vfs->open_call = (accumulator.open_call >= vfs->open_call) ? accumulator.open_call : vfs->open_call;
+ vfs->create_call = (accumulator.create_call >= vfs->create_call) ? accumulator.create_call : vfs->create_call;
+
+ vfs->write_bytes = (accumulator.write_bytes >= vfs->write_bytes) ? accumulator.write_bytes : vfs->write_bytes;
+ vfs->writev_bytes = (accumulator.writev_bytes >= vfs->writev_bytes) ? accumulator.writev_bytes : vfs->writev_bytes;
+ vfs->read_bytes = (accumulator.read_bytes >= vfs->read_bytes) ? accumulator.read_bytes : vfs->read_bytes;
+ vfs->readv_bytes = (accumulator.readv_bytes >= vfs->readv_bytes) ? accumulator.readv_bytes : vfs->readv_bytes;
+
+ vfs->write_err = (accumulator.write_err >= vfs->write_err) ? accumulator.write_err : vfs->write_err;
+ vfs->writev_err = (accumulator.writev_err >= vfs->writev_err) ? accumulator.writev_err : vfs->writev_err;
+ vfs->read_err = (accumulator.read_err >= vfs->read_err) ? accumulator.read_err : vfs->read_err;
+ vfs->readv_err = (accumulator.readv_err >= vfs->readv_err) ? accumulator.readv_err : vfs->readv_err;
+ vfs->unlink_err = (accumulator.unlink_err >= vfs->unlink_err) ? accumulator.unlink_err : vfs->unlink_err;
+ vfs->fsync_err = (accumulator.fsync_err >= vfs->fsync_err) ? accumulator.fsync_err : vfs->fsync_err;
+ vfs->open_err = (accumulator.open_err >= vfs->open_err) ? accumulator.open_err : vfs->open_err;
+ vfs->create_err = (accumulator.create_err >= vfs->create_err) ? accumulator.create_err : vfs->create_err;
+}
+
+/**
+ * Create specific VFS charts
+ *
+ * Create charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param em the main thread structure.
+ */
+static void ebpf_create_specific_vfs_charts(char *type, ebpf_module_t *em)
+{
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_FILE_DELETED,"Files deleted",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_UNLINK_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5500,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_UNLINK],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS, "Write to disk",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_WRITE_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5501,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_WRITE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR, "Fails to write",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_WRITE_ERROR_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5502,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_WRITE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+ }
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_READ_CALLS, "Read from disk",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_READ_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5503,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR, "Fails to read",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_READ_ERROR_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5504,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+ }
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES, "Bytes written on disk",
+ EBPF_COMMON_DIMENSION_BYTES, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_WRITE_BYTES_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5505,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_WRITE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_READ_BYTES, "Bytes read from disk",
+ EBPF_COMMON_DIMENSION_BYTES, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_READ_BYTES_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5506,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_FSYNC, "Calls for <code>vfs_fsync</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_FSYNC_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5507,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR, "Sync error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_FSYNC_ERROR_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5508,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+ }
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_OPEN, "Calls for <code>vfs_open</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_OPEN_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5509,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR, "Open error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_OPEN_ERROR_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5510,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+ }
+
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_CREATE, "Calls for <code>vfs_create</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_CREATE_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5511,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(type, NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR, "Create error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP, NETDATA_CGROUP_VFS_CREATE_ERROR_CONTEXT,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5512,
+ ebpf_create_global_dimension, &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_SWAP);
+ }
+}
+
+/**
+ * Obsolete specific VFS charts
+ *
+ * Obsolete charts for cgroup/application.
+ *
+ * @param type the chart type.
+ * @param em the main thread structure.
+ */
+static void ebpf_obsolete_specific_vfs_charts(char *type, ebpf_module_t *em)
+{
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_FILE_DELETED, "Files deleted",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_UNLINK_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5500, em->update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS, "Write to disk",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_WRITE_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5501, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR, "Fails to write",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_WRITE_ERROR_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5502, em->update_every);
+ }
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_READ_CALLS, "Read from disk",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_READ_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5503, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR, "Fails to read",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_READ_ERROR_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5504, em->update_every);
+ }
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES, "Bytes written on disk",
+ EBPF_COMMON_DIMENSION_BYTES, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_WRITE_BYTES_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5505, em->update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_READ_BYTES, "Bytes read from disk",
+ EBPF_COMMON_DIMENSION_BYTES, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_READ_BYTES_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5506, em->update_every);
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_FSYNC, "Calls for <code>vfs_fsync</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_FSYNC_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5507, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR, "Sync error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_FSYNC_ERROR_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5508, em->update_every);
+ }
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_OPEN, "Calls for <code>vfs_open</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_OPEN_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5509, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR, "Open error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_OPEN_ERROR_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5510, em->update_every);
+ }
+
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_CREATE, "Calls for <code>vfs_create</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_CREATE_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5511, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_write_chart_obsolete(type, NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR, "Create error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_VFS_CREATE_ERROR_CONTEXT,
+ NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5512, em->update_every);
+ }
+}
+
+/*
+ * Send specific VFS data
+ *
+ * Send data for specific cgroup/apps.
+ *
+ * @param type chart type
+ * @param values structure with values that will be sent to netdata
+ */
+static void ebpf_send_specific_vfs_data(char *type, netdata_publish_vfs_t *values, ebpf_module_t *em)
+{
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_FILE_DELETED);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_UNLINK].name, (long long)values->unlink_call);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_WRITE].name,
+ (long long)values->write_call + (long long)values->writev_call);
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_WRITE].name,
+ (long long)values->write_err + (long long)values->writev_err);
+ write_end_chart();
+ }
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_READ_CALLS);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ].name,
+ (long long)values->read_call + (long long)values->readv_call);
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ].name,
+ (long long)values->read_err + (long long)values->readv_err);
+ write_end_chart();
+ }
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_WRITE].name,
+ (long long)values->write_bytes + (long long)values->writev_bytes);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_READ_BYTES);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ].name,
+ (long long)values->read_bytes + (long long)values->readv_bytes);
+ write_end_chart();
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_FSYNC);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC].name,
+ (long long)values->fsync_call);
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC].name,
+ (long long)values->fsync_err);
+ write_end_chart();
+ }
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_OPEN);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN].name,
+ (long long)values->open_call);
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN].name,
+ (long long)values->open_err);
+ write_end_chart();
+ }
+
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_CREATE);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE].name,
+ (long long)values->create_call);
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(type, NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR);
+ write_chart_dimension(vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE].name,
+ (long long)values->create_err);
+ write_end_chart();
+ }
+}
+
+/**
+ * Create Systemd Socket Charts
+ *
+ * Create charts when systemd is enabled
+ *
+ * @param em the main collector structure
+ **/
+static void ebpf_create_systemd_vfs_charts(ebpf_module_t *em)
+{
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_FILE_DELETED, "Files deleted",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20065,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_UNLINK_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS, "Write to disk",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20066,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_WRITE_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR, "Fails to write",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20067,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SYSTEMD_VFS_WRITE_ERROR_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+ }
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_READ_CALLS, "Read from disk",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20068,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_READ_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR, "Fails to read",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20069,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ NETDATA_SYSTEMD_VFS_READ_ERROR_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+ }
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES, "Bytes written on disk",
+ EBPF_COMMON_DIMENSION_BYTES, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20070,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_WRITE_BYTES_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_READ_BYTES, "Bytes read from disk",
+ EBPF_COMMON_DIMENSION_BYTES, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20071,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_READ_BYTES_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_FSYNC, "Calls to <code>vfs_fsync</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20072,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_FSYNC_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR, "Sync error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20073,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_FSYNC_ERROR_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+ }
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_OPEN, "Calls to <code>vfs_open</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20074,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_OPEN_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR, "Open error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20075,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_OPEN_ERROR_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+ }
+
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_CREATE, "Calls to <code>vfs_create</code>",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20076,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_CREATE_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_systemd(NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR, "Create error",
+ EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_CGROUP_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED, 20077,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], NETDATA_SYSTEMD_VFS_CREATE_ERROR_CONTEXT,
+ NETDATA_EBPF_MODULE_NAME_VFS, em->update_every);
+ }
+}
+
+/**
+ * Send Systemd charts
+ *
+ * Send collected data to Netdata.
+ *
+ * @param em the main collector structure
+ */
+static void ebpf_send_systemd_vfs_charts(ebpf_module_t *em)
+{
+ ebpf_cgroup_target_t *ect;
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_FILE_DELETED);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.unlink_call);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.write_call +
+ ect->publish_systemd_vfs.writev_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.write_err +
+ ect->publish_systemd_vfs.writev_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_READ_CALLS);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.read_call +
+ ect->publish_systemd_vfs.readv_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.read_err +
+ ect->publish_systemd_vfs.readv_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.write_bytes +
+ ect->publish_systemd_vfs.writev_bytes);
+ }
+ }
+ write_end_chart();
+
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_READ_BYTES);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.read_bytes +
+ ect->publish_systemd_vfs.readv_bytes);
+ }
+ }
+ write_end_chart();
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_FSYNC);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.fsync_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.fsync_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_OPEN);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.open_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.open_err);
+ }
+ }
+ write_end_chart();
+ }
+
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_CREATE);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.create_call);
+ }
+ }
+ write_end_chart();
+
+ if (em->mode < MODE_ENTRY) {
+ write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR);
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (unlikely(ect->systemd) && unlikely(ect->updated)) {
+ write_chart_dimension(ect->name, ect->publish_systemd_vfs.create_err);
+ }
+ }
+ write_end_chart();
+ }
+}
+
+/**
+ * Send data to Netdata calling auxiliary functions.
+ *
+ * @param em the main collector structure
+*/
+static void ebpf_vfs_send_cgroup_data(ebpf_module_t *em)
+{
+ if (!ebpf_cgroup_pids)
+ return;
+
+ pthread_mutex_lock(&mutex_cgroup_shm);
+ ebpf_cgroup_target_t *ect;
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ ebpf_vfs_sum_cgroup_pids(&ect->publish_systemd_vfs, ect->pids);
+ }
+
+ int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
+ if (has_systemd) {
+ if (send_cgroup_chart) {
+ ebpf_create_systemd_vfs_charts(em);
+ }
+ ebpf_send_systemd_vfs_charts(em);
+ }
+
+ for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+ if (ect->systemd)
+ continue;
+
+ if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_VFS_CHART) && ect->updated) {
+ ebpf_create_specific_vfs_charts(ect->name, em);
+ ect->flags |= NETDATA_EBPF_CGROUP_HAS_VFS_CHART;
+ }
+
+ if (ect->flags & NETDATA_EBPF_CGROUP_HAS_VFS_CHART) {
+ if (ect->updated) {
+ ebpf_send_specific_vfs_data(ect->name, &ect->publish_systemd_vfs, em);
+ } else {
+ ebpf_obsolete_specific_vfs_charts(ect->name, em);
+ ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_VFS_CHART;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Main loop for this collector.
+ *
+ * @param step the number of microseconds used with heart beat
+ * @param em the structure with thread information
+ */
+static void vfs_collector(ebpf_module_t *em)
+{
+ vfs_threads.thread = mallocz(sizeof(netdata_thread_t));
+ vfs_threads.start_routine = ebpf_vfs_read_hash;
+
+ netdata_thread_create(vfs_threads.thread, vfs_threads.name, NETDATA_THREAD_OPTION_DEFAULT,
+ ebpf_vfs_read_hash, em);
+
+ int cgroups = em->cgroup_charts;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ usec_t step = em->update_every * USEC_PER_SEC;
+ while (!ebpf_exit_plugin) {
+ (void)heartbeat_next(&hb, step);
+ if (ebpf_exit_plugin)
+ break;
+
+ netdata_apps_integration_flags_t apps = em->apps_charts;
+ pthread_mutex_lock(&collect_data_mutex);
+ if (apps)
+ ebpf_vfs_read_apps();
+
+ if (cgroups)
+ read_update_vfs_cgroup();
+
+ pthread_mutex_lock(&lock);
+
+ ebpf_vfs_send_data(em);
+ fflush(stdout);
+
+ if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED)
+ ebpf_vfs_send_apps_data(em, apps_groups_root_target);
+
+ if (cgroups)
+ ebpf_vfs_send_cgroup_data(em);
+
+ pthread_mutex_unlock(&lock);
+ pthread_mutex_unlock(&collect_data_mutex);
+ }
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO CREATE CHARTS
+ *
+ *****************************************************************/
+
+/**
+ * Create IO chart
+ *
+ * @param family the chart family
+ * @param name the chart name
+ * @param axis the axis label
+ * @param web the group name used to attach the chart on dashboard
+ * @param order the order number of the specified chart
+ * @param algorithm the algorithm used to make the charts.
+ * @param update_every value to overwrite the update frequency set by the server.
+ */
+static void ebpf_create_io_chart(char *family, char *name, char *axis, char *web,
+ int order, int algorithm, int update_every)
+{
+ printf("CHART %s.%s '' 'Bytes written and read' '%s' '%s' '' line %d %d '' 'ebpf.plugin' 'filesystem'\n",
+ family,
+ name,
+ axis,
+ web,
+ order,
+ update_every);
+
+ printf("DIMENSION %s %s %s 1 1\n",
+ vfs_id_names[NETDATA_KEY_PUBLISH_VFS_READ],
+ vfs_dimension_names[NETDATA_KEY_PUBLISH_VFS_READ],
+ ebpf_algorithms[algorithm]);
+ printf("DIMENSION %s %s %s -1 1\n",
+ vfs_id_names[NETDATA_KEY_PUBLISH_VFS_WRITE],
+ vfs_dimension_names[NETDATA_KEY_PUBLISH_VFS_WRITE],
+ ebpf_algorithms[algorithm]);
+}
+
+/**
+ * Create global charts
+ *
+ * Call ebpf_create_chart to create the charts for the collector.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+static void ebpf_create_global_charts(ebpf_module_t *em)
+{
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_FILE_CLEAN_COUNT,
+ "Remove files",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_CLEAN,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_UNLINK],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_FILE_IO_COUNT,
+ "Calls to IO",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_COUNT,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ ebpf_create_io_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_IO_FILE_BYTES, EBPF_COMMON_DIMENSION_BYTES,
+ NETDATA_VFS_GROUP,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_BYTES,
+ NETDATA_EBPF_INCREMENTAL_IDX, em->update_every);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_FILE_ERR_COUNT,
+ "Fails to write or read",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EBYTES,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_READ],
+ 2, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_FSYNC,
+ "Calls for <code>vfs_fsync</code>",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_FSYNC,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_FSYNC_ERR,
+ "Fails to synchronize",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EFSYNC,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_FSYNC],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_OPEN,
+ "Calls for <code>vfs_open</code>",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_OPEN,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_OPEN_ERR,
+ "Fails to open a file",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EOPEN,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_OPEN],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_CREATE,
+ "Calls for <code>vfs_create</code>",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_CREATE,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY,
+ NETDATA_VFS_CREATE_ERR,
+ "Fails to create a file.",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NULL,
+ NETDATA_EBPF_CHART_TYPE_LINE,
+ NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_ECREATE,
+ ebpf_create_global_dimension,
+ &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE],
+ 1, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+}
+
+/**
+ * Create process apps charts
+ *
+ * Call ebpf_create_chart to create the charts on apps submenu.
+ *
+ * @param em a pointer to the structure with the default values.
+ * @param ptr a pointer for the targets.
+ **/
+void ebpf_vfs_create_apps_charts(struct ebpf_module *em, void *ptr)
+{
+ struct target *root = ptr;
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_FILE_DELETED,
+ "Files deleted",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20065,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS,
+ "Write to disk",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20066,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR,
+ "Fails to write",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20067,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_READ_CALLS,
+ "Read from disk",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20068,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR,
+ "Fails to read",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20069,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES,
+ "Bytes written on disk", EBPF_COMMON_DIMENSION_BYTES,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20070,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_READ_BYTES,
+ "Bytes read from disk", EBPF_COMMON_DIMENSION_BYTES,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20071,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_FSYNC,
+ "Calls for <code>vfs_fsync</code>", EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20072,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR,
+ "Sync error",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20073,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_OPEN,
+ "Calls for <code>vfs_open</code>", EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20074,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR,
+ "Open error",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20075,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_CREATE,
+ "Calls for <code>vfs_create</code>", EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20076,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+
+ if (em->mode < MODE_ENTRY) {
+ ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR,
+ "Create error",
+ EBPF_COMMON_DIMENSION_CALL,
+ NETDATA_VFS_GROUP,
+ NETDATA_EBPF_CHART_TYPE_STACKED,
+ 20077,
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
+ root, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
+ }
+
+ em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
+}
+
+/*****************************************************************
+ *
+ * FUNCTIONS TO START THREAD
+ *
+ *****************************************************************/
+
+/**
+ * Allocate vectors used with this thread.
+ * We are not testing the return, because callocz does this and shutdown the software
+ * case it was not possible to allocate.
+ *
+ * @param apps is apps enabled?
+ */
+static void ebpf_vfs_allocate_global_vectors(int apps)
+{
+ memset(vfs_aggregated_data, 0, sizeof(vfs_aggregated_data));
+ memset(vfs_publish_aggregated, 0, sizeof(vfs_publish_aggregated));
+
+ vfs_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t));
+ vfs_vector = callocz(ebpf_nprocs, sizeof(netdata_publish_vfs_t));
+
+ if (apps)
+ vfs_pid = callocz((size_t)pid_max, sizeof(netdata_publish_vfs_t *));
+}
+
+/*****************************************************************
+ *
+ * EBPF VFS THREAD
+ *
+ *****************************************************************/
+
+/*
+ * Load BPF
+ *
+ * Load BPF files.
+ *
+ * @param em the structure with configuration
+ */
+static int ebpf_vfs_load_bpf(ebpf_module_t *em)
+{
+ int ret = 0;
+ ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_EBPF_VFS_WRITE].mode);
+ if (em->load & EBPF_LOAD_LEGACY) {
+ em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
+ if (!em->probe_links) {
+ ret = -1;
+ }
+ }
+#ifdef LIBBPF_MAJOR_VERSION
+ else {
+ bpf_obj = vfs_bpf__open();
+ if (!bpf_obj)
+ ret = -1;
+ else
+ ret = ebpf_vfs_load_and_attach(bpf_obj, em);
+ }
+#endif
+
+ return ret;
+}
+
+/**
+ * Process thread
+ *
+ * Thread used to generate process charts.
+ *
+ * @param ptr a pointer to `struct ebpf_module`
+ *
+ * @return It always return NULL
+ */
+void *ebpf_vfs_thread(void *ptr)
+{
+ netdata_thread_cleanup_push(ebpf_vfs_exit, ptr);
+
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
+ em->maps = vfs_maps;
+
+ ebpf_update_pid_table(&vfs_maps[NETDATA_VFS_PID], em);
+
+ ebpf_vfs_allocate_global_vectors(em->apps_charts);
+
+#ifdef LIBBPF_MAJOR_VERSION
+ ebpf_adjust_thread_load(em, default_btf);
+#endif
+ if (ebpf_vfs_load_bpf(em)) {
+ em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+ goto endvfs;
+ }
+
+ int algorithms[NETDATA_KEY_PUBLISH_VFS_END] = {
+ NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX,NETDATA_EBPF_INCREMENTAL_IDX,
+ NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX,NETDATA_EBPF_INCREMENTAL_IDX
+ };
+
+ ebpf_global_labels(vfs_aggregated_data, vfs_publish_aggregated, vfs_dimension_names,
+ vfs_id_names, algorithms, NETDATA_KEY_PUBLISH_VFS_END);
+
+ pthread_mutex_lock(&lock);
+ ebpf_create_global_charts(em);
+ ebpf_update_stats(&plugin_statistics, em);
+ pthread_mutex_unlock(&lock);
+
+ vfs_collector(em);
+
+endvfs:
+ ebpf_update_disabled_plugin_stats(em);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/ebpf.plugin/ebpf_vfs.h b/collectors/ebpf.plugin/ebpf_vfs.h
new file mode 100644
index 0000000..2e3c7cc
--- /dev/null
+++ b/collectors/ebpf.plugin/ebpf_vfs.h
@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_VFS_H
+#define NETDATA_EBPF_VFS_H 1
+
+// Module name
+#define NETDATA_EBPF_MODULE_NAME_VFS "vfs"
+
+#define NETDATA_DIRECTORY_VFS_CONFIG_FILE "vfs.conf"
+
+#define NETDATA_LATENCY_VFS_SLEEP_MS 750000ULL
+
+// Global chart name
+#define NETDATA_VFS_FILE_CLEAN_COUNT "vfs_deleted_objects"
+#define NETDATA_VFS_FILE_IO_COUNT "vfs_io"
+#define NETDATA_VFS_FILE_ERR_COUNT "vfs_io_error"
+#define NETDATA_VFS_IO_FILE_BYTES "vfs_io_bytes"
+#define NETDATA_VFS_FSYNC "vfs_fsync"
+#define NETDATA_VFS_FSYNC_ERR "vfs_fsync_error"
+#define NETDATA_VFS_OPEN "vfs_open"
+#define NETDATA_VFS_OPEN_ERR "vfs_open_error"
+#define NETDATA_VFS_CREATE "vfs_create"
+#define NETDATA_VFS_CREATE_ERR "vfs_create_error"
+
+// Charts created on Apps submenu
+#define NETDATA_SYSCALL_APPS_FILE_DELETED "file_deleted"
+#define NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS "vfs_write_call"
+#define NETDATA_SYSCALL_APPS_VFS_READ_CALLS "vfs_read_call"
+#define NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES "vfs_write_bytes"
+#define NETDATA_SYSCALL_APPS_VFS_READ_BYTES "vfs_read_bytes"
+#define NETDATA_SYSCALL_APPS_VFS_FSYNC "vfs_fsync"
+#define NETDATA_SYSCALL_APPS_VFS_OPEN "vfs_open"
+#define NETDATA_SYSCALL_APPS_VFS_CREATE "vfs_create"
+
+#define NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR "vfs_write_error"
+#define NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR "vfs_read_error"
+#define NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR "vfs_fsync_error"
+#define NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR "vfs_open_error"
+#define NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR "vfs_create_error"
+
+// Group used on Dashboard
+#define NETDATA_VFS_GROUP "vfs"
+#define NETDATA_VFS_CGROUP_GROUP "vfs (eBPF)"
+
+// Contexts
+#define NETDATA_CGROUP_VFS_UNLINK_CONTEXT "cgroup.vfs_unlink"
+#define NETDATA_CGROUP_VFS_WRITE_CONTEXT "cgroup.vfs_write"
+#define NETDATA_CGROUP_VFS_WRITE_ERROR_CONTEXT "cgroup.vfs_write_error"
+#define NETDATA_CGROUP_VFS_READ_CONTEXT "cgroup.vfs_read"
+#define NETDATA_CGROUP_VFS_READ_ERROR_CONTEXT "cgroup.vfs_read_error"
+#define NETDATA_CGROUP_VFS_WRITE_BYTES_CONTEXT "cgroup.vfs_write_bytes"
+#define NETDATA_CGROUP_VFS_READ_BYTES_CONTEXT "cgroup.vfs_read_bytes"
+#define NETDATA_CGROUP_VFS_CREATE_CONTEXT "cgroup.vfs_create"
+#define NETDATA_CGROUP_VFS_CREATE_ERROR_CONTEXT "cgroup.vfs_create_error"
+#define NETDATA_CGROUP_VFS_OPEN_CONTEXT "cgroup.vfs_open"
+#define NETDATA_CGROUP_VFS_OPEN_ERROR_CONTEXT "cgroup.vfs_open_error"
+#define NETDATA_CGROUP_VFS_FSYNC_CONTEXT "cgroup.vfs_fsync"
+#define NETDATA_CGROUP_VFS_FSYNC_ERROR_CONTEXT "cgroup.vfs_fsync_error"
+
+#define NETDATA_SYSTEMD_VFS_UNLINK_CONTEXT "services.vfs_unlink"
+#define NETDATA_SYSTEMD_VFS_WRITE_CONTEXT "services.vfs_write"
+#define NETDATA_SYSTEMD_VFS_WRITE_ERROR_CONTEXT "services.vfs_write_error"
+#define NETDATA_SYSTEMD_VFS_READ_CONTEXT "services.vfs_read"
+#define NETDATA_SYSTEMD_VFS_READ_ERROR_CONTEXT "services.vfs_read_error"
+#define NETDATA_SYSTEMD_VFS_WRITE_BYTES_CONTEXT "services.vfs_write_bytes"
+#define NETDATA_SYSTEMD_VFS_READ_BYTES_CONTEXT "services.vfs_read_bytes"
+#define NETDATA_SYSTEMD_VFS_CREATE_CONTEXT "services.vfs_create"
+#define NETDATA_SYSTEMD_VFS_CREATE_ERROR_CONTEXT "services.vfs_create_error"
+#define NETDATA_SYSTEMD_VFS_OPEN_CONTEXT "services.vfs_open"
+#define NETDATA_SYSTEMD_VFS_OPEN_ERROR_CONTEXT "services.vfs_open_error"
+#define NETDATA_SYSTEMD_VFS_FSYNC_CONTEXT "services.vfs_fsync"
+#define NETDATA_SYSTEMD_VFS_FSYNC_ERROR_CONTEXT "services.vfs_fsync_error"
+
+typedef struct netdata_publish_vfs {
+ uint64_t pid_tgid;
+ uint32_t pid;
+ uint32_t pad;
+
+ //Counter
+ uint32_t write_call;
+ uint32_t writev_call;
+ uint32_t read_call;
+ uint32_t readv_call;
+ uint32_t unlink_call;
+ uint32_t fsync_call;
+ uint32_t open_call;
+ uint32_t create_call;
+
+ //Accumulator
+ uint64_t write_bytes;
+ uint64_t writev_bytes;
+ uint64_t readv_bytes;
+ uint64_t read_bytes;
+
+ //Counter
+ uint32_t write_err;
+ uint32_t writev_err;
+ uint32_t read_err;
+ uint32_t readv_err;
+ uint32_t unlink_err;
+ uint32_t fsync_err;
+ uint32_t open_err;
+ uint32_t create_err;
+} netdata_publish_vfs_t;
+
+enum netdata_publish_vfs_list {
+ NETDATA_KEY_PUBLISH_VFS_UNLINK,
+ NETDATA_KEY_PUBLISH_VFS_READ,
+ NETDATA_KEY_PUBLISH_VFS_WRITE,
+ NETDATA_KEY_PUBLISH_VFS_FSYNC,
+ NETDATA_KEY_PUBLISH_VFS_OPEN,
+ NETDATA_KEY_PUBLISH_VFS_CREATE,
+
+ NETDATA_KEY_PUBLISH_VFS_END
+};
+
+enum vfs_counters {
+ NETDATA_KEY_CALLS_VFS_WRITE,
+ NETDATA_KEY_ERROR_VFS_WRITE,
+ NETDATA_KEY_BYTES_VFS_WRITE,
+
+ NETDATA_KEY_CALLS_VFS_WRITEV,
+ NETDATA_KEY_ERROR_VFS_WRITEV,
+ NETDATA_KEY_BYTES_VFS_WRITEV,
+
+ NETDATA_KEY_CALLS_VFS_READ,
+ NETDATA_KEY_ERROR_VFS_READ,
+ NETDATA_KEY_BYTES_VFS_READ,
+
+ NETDATA_KEY_CALLS_VFS_READV,
+ NETDATA_KEY_ERROR_VFS_READV,
+ NETDATA_KEY_BYTES_VFS_READV,
+
+ NETDATA_KEY_CALLS_VFS_UNLINK,
+ NETDATA_KEY_ERROR_VFS_UNLINK,
+
+ NETDATA_KEY_CALLS_VFS_FSYNC,
+ NETDATA_KEY_ERROR_VFS_FSYNC,
+
+ NETDATA_KEY_CALLS_VFS_OPEN,
+ NETDATA_KEY_ERROR_VFS_OPEN,
+
+ NETDATA_KEY_CALLS_VFS_CREATE,
+ NETDATA_KEY_ERROR_VFS_CREATE,
+
+ // Keep this as last and don't skip numbers as it is used as element counter
+ NETDATA_VFS_COUNTER
+};
+
+enum netdata_vfs_tables {
+ NETDATA_VFS_PID,
+ NETDATA_VFS_ALL,
+ NETDATA_VFS_CTRL
+};
+
+enum netdata_vfs_calls_name {
+ NETDATA_EBPF_VFS_WRITE,
+ NETDATA_EBPF_VFS_WRITEV,
+ NETDATA_EBPF_VFS_READ,
+ NETDATA_EBPF_VFS_READV,
+ NETDATA_EBPF_VFS_UNLINK,
+ NETDATA_EBPF_VFS_FSYNC,
+ NETDATA_EBPF_VFS_OPEN,
+ NETDATA_EBPF_VFS_CREATE,
+
+ NETDATA_VFS_END_LIST
+};
+
+extern netdata_publish_vfs_t **vfs_pid;
+
+void *ebpf_vfs_thread(void *ptr);
+void ebpf_vfs_create_apps_charts(struct ebpf_module *em, void *ptr);
+extern netdata_ebpf_targets_t vfs_targets[];
+
+extern struct config vfs_config;
+
+#endif /* NETDATA_EBPF_VFS_H */
diff --git a/collectors/fping.plugin/Makefile.am b/collectors/fping.plugin/Makefile.am
new file mode 100644
index 0000000..9065483
--- /dev/null
+++ b/collectors/fping.plugin/Makefile.am
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+CLEANFILES = \
+ fping.plugin \
+ $(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+dist_plugins_SCRIPTS = \
+ fping.plugin \
+ $(NULL)
+
+dist_noinst_DATA = \
+ fping.plugin.in \
+ README.md \
+ $(NULL)
+
+dist_libconfig_DATA = \
+ fping.conf \
+ $(NULL)
diff --git a/collectors/fping.plugin/README.md b/collectors/fping.plugin/README.md
new file mode 100644
index 0000000..e32d391
--- /dev/null
+++ b/collectors/fping.plugin/README.md
@@ -0,0 +1,110 @@
+<!--
+title: "fping.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/fping.plugin/README.md
+-->
+
+# fping.plugin
+
+The fping plugin supports monitoring latency, packet loss and uptime of any number of network end points,
+by pinging them with `fping`.
+
+This plugin requires version 5.1 or newer of `fping` (earlier versions may or may not work). Our static builds and
+Docker images come bundled with a known working version of `fping`. Native packages and local builds will need to
+have a working version installed before the plugin is usable.
+
+## Installing fping locally
+
+If your distribution’s repositories do not include a working version of `fping`, the supplied plugin can install
+it, by running:
+
+```sh
+/usr/libexec/netdata/plugins.d/fping.plugin install
+```
+
+The above will download, build and install the right version as `/usr/local/bin/fping`. This requires a working C
+compiler, GNU autotools (at least autoconf and automake), and GNU make. On Debian or Ubuntu, you can pull in most
+of the required tools by installing the `build-essential` package (this should include everything except automake
+and autoconf).
+
+## Configuration
+
+Then you need to edit `/etc/netdata/fping.conf` (to edit it on your system run
+`/etc/netdata/edit-config fping.conf`) like this:
+
+```sh
+# set here all the hosts you need to ping
+# I suggest to use hostnames and put their IPs in /etc/hosts
+hosts="host1 host2 host3"
+
+# override the chart update frequency - the default is inherited from Netdata
+update_every=1
+
+# time in milliseconds (1 sec = 1000 ms) to ping the hosts
+# 200 = 5 pings per second
+ping_every=200
+
+# other fping options - these are the defaults
+fping_opts="-R -b 56 -i 1 -r 0 -t 5000"
+```
+
+## alarms
+
+Netdata will automatically attach a few alarms for each host.
+Check the [latest versions of the fping alarms](https://raw.githubusercontent.com/netdata/netdata/master/health/health.d/fping.conf)
+
+## Additional Tips
+
+### Customizing Amount of Pings Per Second
+
+For example, to update the chart every 10 seconds and use 2 pings every 10 seconds, use this:
+
+```sh
+# Chart Update Frequency (Time in Seconds)
+update_every=10
+
+# Time in Milliseconds (1 sec = 1000 ms) to Ping the Hosts
+# The Following Example Sends 1 Ping Every 5000 ms
+# Calculation Formula: ping_every = (update_every * 1000 ) / 2
+ping_every=5000
+```
+
+### Multiple fping Plugins With Different Settings
+
+You may need to run multiple fping plugins with different settings for different end points.
+For example, you may need to ping a few hosts 10 times per second, and others once per second.
+
+Netdata allows you to add as many `fping` plugins as you like.
+
+Follow this procedure:
+
+**1. Create New fping Configuration File**
+
+```sh
+# Step Into Configuration Directory
+cd /etc/netdata
+
+# Copy Original fping Configuration File To New Configuration File
+cp fping.conf fping2.conf
+```
+
+Edit `fping2.conf` and set the settings and the hosts you need for the seconds instance.
+
+**2. Soft Link Original fping Plugin to New Plugin File**
+
+```sh
+# Become root (If The Step Step Is Performed As Non-Root User)
+sudo su
+
+# Step Into The Plugins Directory
+cd /usr/libexec/netdata/plugins.d
+
+# Link fping.plugin to fping2.plugin
+ln -s fping.plugin fping2.plugin
+```
+
+That's it. Netdata will detect the new plugin and start it.
+
+You can name the new plugin any name you like.
+Just make sure the plugin and the configuration file have the same name.
+
+
diff --git a/collectors/fping.plugin/fping.conf b/collectors/fping.plugin/fping.conf
new file mode 100644
index 0000000..63a7f7a
--- /dev/null
+++ b/collectors/fping.plugin/fping.conf
@@ -0,0 +1,44 @@
+# no need for shebang - this file is sourced from fping.plugin
+
+# fping.plugin requires a recent version of fping.
+#
+# You can get it on your system, by running:
+#
+# /usr/libexec/netdata/plugins.d/fping.plugin install
+
+# -----------------------------------------------------------------------------
+# configuration options
+
+# The fping binary to use. We need one that can output netdata friendly info
+# (supporting: -N). If you have multiple versions, put here the full filename
+# of the right one
+
+#fping="/usr/local/bin/fping"
+
+
+# a space separated list of hosts to fping
+# we suggest to put names here and the IPs of these names in /etc/hosts
+
+hosts=""
+
+
+# The update frequency of the chart - the default is inherited from netdata
+
+#update_every=2
+
+
+# The time in milliseconds (1 sec = 1000 ms) to ping the hosts
+# by default 5 pings per host per iteration
+# fping will not allow this to be below 20ms
+
+#ping_every="200"
+
+
+# other fping options - defaults:
+# -R = send packets with random data
+# -b 56 = the number of bytes per packet
+# -i 1 = 1 ms when sending packets to others hosts (switching hosts)
+# -r 0 = never retry packets
+# -t 5000 = per packet timeout at 5000 ms
+
+#fping_opts="-R -b 56 -i 1 -r 0 -t 5000"
diff --git a/collectors/fping.plugin/fping.plugin.in b/collectors/fping.plugin/fping.plugin.in
new file mode 100755
index 0000000..4b3d1d1
--- /dev/null
+++ b/collectors/fping.plugin/fping.plugin.in
@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+#
+# This plugin requires a latest version of fping.
+# You can compile it from source, by running me with option: install
+
+export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
+export LC_ALL=C
+
+if [ "${1}" = "install" ]
+ then
+ [ "${UID}" != 0 ] && echo >&2 "Please run me as root. This will install a single binary file: /usr/local/bin/fping." && exit 1
+
+ [ -z "${2}" ] && fping_version="5.1" || fping_version="${2}"
+
+ run() {
+ printf >&2 " > "
+ printf >&2 "%q " "${@}"
+ printf >&2 "\n"
+ "${@}" || exit 1
+ }
+
+ download() {
+ local curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
+ [ ! -z "${curl}" ] && run curl -s -L "${1}" && return 0
+
+ local wget="$(which wget 2>/dev/null || command -v wget 2>/dev/null)"
+ [ ! -z "${wget}" ] && run wget -q -O - "${1}" && return 0
+
+ echo >&2 "Cannot find 'curl' or 'wget' in this system." && exit 1
+ }
+
+ [ ! -d /usr/src ] && run mkdir -p /usr/src
+ [ ! -d /usr/local/bin ] && run mkdir -p /usr/local/bin
+
+ run cd /usr/src
+
+ if [ -d "fping-${fping_version}" ]
+ then
+ run rm -rf "fping-${fping_version}" || exit 1
+ fi
+
+ download "https://github.com/schweikert/fping/releases/download/v${fping_version}/fping-${fping_version}.tar.gz" | run tar -zxvpf -
+ [ $? -ne 0 ] && exit 1
+ run cd "fping-${fping_version}" || exit 1
+
+ run ./configure --prefix=/usr/local
+ run make clean
+ run make
+ if [ -f /usr/local/bin/fping ]
+ then
+ run mv -f /usr/local/bin/fping /usr/local/bin/fping.old
+ fi
+ run mv src/fping /usr/local/bin/fping
+ run chown root:root /usr/local/bin/fping
+ run chmod 4755 /usr/local/bin/fping
+ echo >&2
+ echo >&2 "All done, you have a compatible fping now at /usr/local/bin/fping."
+ echo >&2
+
+ fping="$(which fping 2>/dev/null || command -v fping 2>/dev/null)"
+ if [ "${fping}" != "/usr/local/bin/fping" ]
+ then
+ echo >&2 "You have another fping installed at: ${fping}."
+ echo >&2 "Please set:"
+ echo >&2
+ echo >&2 " fping=\"/usr/local/bin/fping\""
+ echo >&2
+ echo >&2 "at /etc/netdata/fping.conf"
+ echo >&2
+ fi
+ exit 0
+fi
+
+# -----------------------------------------------------------------------------
+
+PROGRAM_NAME="$(basename "${0}")"
+
+logdate() {
+ date "+%Y-%m-%d %H:%M:%S"
+}
+
+log() {
+ local status="${1}"
+ shift
+
+ echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
+
+}
+
+warning() {
+ log WARNING "${@}"
+}
+
+error() {
+ log ERROR "${@}"
+}
+
+info() {
+ log INFO "${@}"
+}
+
+fatal() {
+ log FATAL "${@}"
+ echo "DISABLE"
+ exit 1
+}
+
+debug=0
+debug() {
+ [ $debug -eq 1 ] && log DEBUG "${@}"
+}
+
+# -----------------------------------------------------------------------------
+
+# store in ${plugin} the name we run under
+# this allows us to copy/link fping.plugin under a different name
+# to have multiple fping plugins running with different settings
+plugin="${PROGRAM_NAME/.plugin/}"
+
+
+# -----------------------------------------------------------------------------
+
+# the frequency to send info to netdata
+# passed by netdata as the first parameter
+update_every="${1-1}"
+
+# the netdata configuration directory
+# passed by netdata as an environment variable
+[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
+[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
+
+# -----------------------------------------------------------------------------
+# configuration options
+# can be overwritten at /etc/netdata/fping.conf
+
+# the fping binary to use
+# we need one that can output netdata friendly info (supporting: -N)
+# if you have multiple versions, put here the full filename of the right one
+fping="$( which fping 2>/dev/null || command -v fping 2>/dev/null )"
+
+# a space separated list of hosts to fping
+# we suggest to put names here and the IPs of these names in /etc/hosts
+hosts=""
+
+# the time in milliseconds (1 sec = 1000 ms)
+# to ping the hosts - by default 5 pings per host per iteration
+ping_every="$((update_every * 1000 / 5))"
+
+# fping options
+fping_opts="-R -b 56 -i 1 -r 0 -t 5000"
+
+# -----------------------------------------------------------------------------
+# load the configuration files
+
+for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/${plugin}.conf" "${NETDATA_USER_CONFIG_DIR}/${plugin}.conf"
+do
+ if [ -f "${CONFIG}" ]
+ then
+ info "Loading config file '${CONFIG}'..."
+ source "${CONFIG}"
+ [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'."
+ else
+ warning "Cannot find file '${CONFIG}'."
+ fi
+done
+
+if [ -z "${hosts}" ]
+then
+ fatal "no hosts configured - nothing to do."
+fi
+
+if [ -z "${fping}" ]
+then
+ fatal "fping command is not found. Please set its full path in '${NETDATA_USER_CONFIG_DIR}/${plugin}.conf'"
+fi
+
+if [ ! -x "${fping}" ]
+then
+ fatal "fping command '${fping}' is not executable - cannot proceed."
+fi
+
+if [ ${ping_every} -lt 20 ]
+ then
+ warning "ping every was set to ${ping_every} but 20 is the minimum for non-root users. Setting it to 20 ms."
+ ping_every=20
+fi
+
+# the fping options we will use
+options=( -N -l -Q ${update_every} -p ${ping_every} ${fping_opts} ${hosts} )
+
+# execute fping
+info "starting fping: ${fping} ${options[*]}"
+exec "${fping}" "${options[@]}"
+
+# if we cannot execute fping, stop
+fatal "command '${fping} ${options[*]}' failed to be executed (returned code $?)."
diff --git a/collectors/freebsd.plugin/Makefile.am b/collectors/freebsd.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/freebsd.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/freebsd.plugin/README.md b/collectors/freebsd.plugin/README.md
new file mode 100644
index 0000000..9a97a7e
--- /dev/null
+++ b/collectors/freebsd.plugin/README.md
@@ -0,0 +1,12 @@
+<!--
+title: "freebsd.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/freebsd.plugin/README.md
+-->
+
+# freebsd.plugin
+
+Collects resource usage and performance data on FreeBSD systems
+
+By default, Netdata will enable monitoring metrics for disks, memory, and network only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). Use `yes` instead of `auto` in plugin configuration sections to enable these charts permanently. You can also set the `enable zero metrics` option to `yes` in the `[global]` section which enables charts with zero metrics for all internal Netdata plugins.
+
+
diff --git a/collectors/freebsd.plugin/freebsd_devstat.c b/collectors/freebsd.plugin/freebsd_devstat.c
new file mode 100644
index 0000000..0f03774
--- /dev/null
+++ b/collectors/freebsd.plugin/freebsd_devstat.c
@@ -0,0 +1,759 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_freebsd.h"
+
+#include <sys/devicestat.h>
+
+struct disk {
+ char *name;
+ uint32_t hash;
+ size_t len;
+
+ // flags
+ int configured;
+ int enabled;
+ int updated;
+
+ int do_io;
+ int do_ops;
+ int do_qops;
+ int do_util;
+ int do_iotime;
+ int do_await;
+ int do_avagsz;
+ int do_svctm;
+
+
+ // data for differential charts
+
+ struct prev_dstat {
+ collected_number bytes_read;
+ collected_number bytes_write;
+ collected_number bytes_free;
+ collected_number operations_read;
+ collected_number operations_write;
+ collected_number operations_other;
+ collected_number operations_free;
+ collected_number duration_read_ms;
+ collected_number duration_write_ms;
+ collected_number duration_other_ms;
+ collected_number duration_free_ms;
+ collected_number busy_time_ms;
+ } prev_dstat;
+
+ // charts and dimensions
+
+ RRDSET *st_io;
+ RRDDIM *rd_io_in;
+ RRDDIM *rd_io_out;
+ RRDDIM *rd_io_free;
+
+ RRDSET *st_ops;
+ RRDDIM *rd_ops_in;
+ RRDDIM *rd_ops_out;
+ RRDDIM *rd_ops_other;
+ RRDDIM *rd_ops_free;
+
+ RRDSET *st_qops;
+ RRDDIM *rd_qops;
+
+ RRDSET *st_util;
+ RRDDIM *rd_util;
+
+ RRDSET *st_iotime;
+ RRDDIM *rd_iotime_in;
+ RRDDIM *rd_iotime_out;
+ RRDDIM *rd_iotime_other;
+ RRDDIM *rd_iotime_free;
+
+ RRDSET *st_await;
+ RRDDIM *rd_await_in;
+ RRDDIM *rd_await_out;
+ RRDDIM *rd_await_other;
+ RRDDIM *rd_await_free;
+
+ RRDSET *st_avagsz;
+ RRDDIM *rd_avagsz_in;
+ RRDDIM *rd_avagsz_out;
+ RRDDIM *rd_avagsz_free;
+
+ RRDSET *st_svctm;
+ RRDDIM *rd_svctm;
+
+ struct disk *next;
+};
+
+static struct disk *disks_root = NULL, *disks_last_used = NULL;
+
+static size_t disks_added = 0, disks_found = 0;
+
+static void disk_free(struct disk *dm) {
+ if (likely(dm->st_io))
+ rrdset_is_obsolete(dm->st_io);
+ if (likely(dm->st_ops))
+ rrdset_is_obsolete(dm->st_ops);
+ if (likely(dm->st_qops))
+ rrdset_is_obsolete(dm->st_qops);
+ if (likely(dm->st_util))
+ rrdset_is_obsolete(dm->st_util);
+ if (likely(dm->st_iotime))
+ rrdset_is_obsolete(dm->st_iotime);
+ if (likely(dm->st_await))
+ rrdset_is_obsolete(dm->st_await);
+ if (likely(dm->st_avagsz))
+ rrdset_is_obsolete(dm->st_avagsz);
+ if (likely(dm->st_svctm))
+ rrdset_is_obsolete(dm->st_svctm);
+
+ disks_added--;
+ freez(dm->name);
+ freez(dm);
+}
+
+static void disks_cleanup() {
+ if (likely(disks_found == disks_added)) return;
+
+ struct disk *dm = disks_root, *last = NULL;
+ while(dm) {
+ if (unlikely(!dm->updated)) {
+ // info("Removing disk '%s', linked after '%s'", dm->name, last?last->name:"ROOT");
+
+ if (disks_last_used == dm)
+ disks_last_used = last;
+
+ struct disk *t = dm;
+
+ if (dm == disks_root || !last)
+ disks_root = dm = dm->next;
+
+ else
+ last->next = dm = dm->next;
+
+ t->next = NULL;
+ disk_free(t);
+ }
+ else {
+ last = dm;
+ dm->updated = 0;
+ dm = dm->next;
+ }
+ }
+}
+
+static struct disk *get_disk(const char *name) {
+ struct disk *dm;
+
+ uint32_t hash = simple_hash(name);
+
+ // search it, from the last position to the end
+ for(dm = disks_last_used ; dm ; dm = dm->next) {
+ if (unlikely(hash == dm->hash && !strcmp(name, dm->name))) {
+ disks_last_used = dm->next;
+ return dm;
+ }
+ }
+
+ // search it from the beginning to the last position we used
+ for(dm = disks_root ; dm != disks_last_used ; dm = dm->next) {
+ if (unlikely(hash == dm->hash && !strcmp(name, dm->name))) {
+ disks_last_used = dm->next;
+ return dm;
+ }
+ }
+
+ // create a new one
+ dm = callocz(1, sizeof(struct disk));
+ dm->name = strdupz(name);
+ dm->hash = simple_hash(dm->name);
+ dm->len = strlen(dm->name);
+ disks_added++;
+
+ // link it to the end
+ if (disks_root) {
+ struct disk *e;
+ for(e = disks_root; e->next ; e = e->next) ;
+ e->next = dm;
+ }
+ else
+ disks_root = dm;
+
+ return dm;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// kern.devstat
+
+int do_kern_devstat(int update_every, usec_t dt) {
+
+#define DEFAULT_EXCLUDED_DISKS ""
+#define CONFIG_SECTION_KERN_DEVSTAT "plugin:freebsd:kern.devstat"
+#define BINTIME_SCALE 5.42101086242752217003726400434970855712890625e-17 // this is 1000/2^64
+
+ static int enable_new_disks = -1;
+ static int enable_pass_devices = -1, do_system_io = -1, do_io = -1, do_ops = -1, do_qops = -1, do_util = -1,
+ do_iotime = -1, do_await = -1, do_avagsz = -1, do_svctm = -1;
+ static SIMPLE_PATTERN *excluded_disks = NULL;
+
+ if (unlikely(enable_new_disks == -1)) {
+ enable_new_disks = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT,
+ "enable new disks detected at runtime", CONFIG_BOOLEAN_AUTO);
+
+ enable_pass_devices = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT,
+ "performance metrics for pass devices", CONFIG_BOOLEAN_AUTO);
+
+ do_system_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "total bandwidth for all disks",
+ CONFIG_BOOLEAN_YES);
+
+ do_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "bandwidth for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_ops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "operations for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_qops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "queued operations for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_util = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "utilization percentage for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_iotime = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "i/o time for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_await = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o time for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_avagsz = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o bandwidth for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_svctm = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average service time for all disks",
+ CONFIG_BOOLEAN_AUTO);
+
+ excluded_disks = simple_pattern_create(
+ config_get(CONFIG_SECTION_KERN_DEVSTAT, "disable by default disks matching", DEFAULT_EXCLUDED_DISKS)
+ , NULL
+ , SIMPLE_PATTERN_EXACT
+ );
+ }
+
+ if (likely(do_system_io || do_io || do_ops || do_qops || do_util || do_iotime || do_await || do_avagsz || do_svctm)) {
+ static int mib_numdevs[3] = {0, 0, 0};
+ int numdevs;
+ int common_error = 0;
+
+ if (unlikely(GETSYSCTL_SIMPLE("kern.devstat.numdevs", mib_numdevs, numdevs))) {
+ common_error = 1;
+ } else {
+ static int mib_devstat[3] = {0, 0, 0};
+ static void *devstat_data = NULL;
+ static int old_numdevs = 0;
+
+ if (unlikely(numdevs != old_numdevs)) {
+ devstat_data = reallocz(devstat_data, sizeof(long) + sizeof(struct devstat) *
+ numdevs); // there is generation number before devstat structures
+ old_numdevs = numdevs;
+ }
+ if (unlikely(GETSYSCTL_WSIZE("kern.devstat.all", mib_devstat, devstat_data,
+ sizeof(long) + sizeof(struct devstat) * numdevs))) {
+ common_error = 1;
+ } else {
+ struct devstat *dstat;
+ int i;
+ collected_number total_disk_kbytes_read = 0;
+ collected_number total_disk_kbytes_write = 0;
+
+ disks_found = 0;
+
+ dstat = (struct devstat*)((char*)devstat_data + sizeof(long)); // skip generation number
+
+ for (i = 0; i < numdevs; i++) {
+ if (likely(do_system_io)) {
+ if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) ||
+ ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) {
+ total_disk_kbytes_read += dstat[i].bytes[DEVSTAT_READ] / KILO_FACTOR;
+ total_disk_kbytes_write += dstat[i].bytes[DEVSTAT_WRITE] / KILO_FACTOR;
+ }
+ }
+
+ if (unlikely(!enable_pass_devices))
+ if ((dstat[i].device_type & DEVSTAT_TYPE_PASS) == DEVSTAT_TYPE_PASS)
+ continue;
+
+ if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) ||
+ ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) {
+ char disk[DEVSTAT_NAME_LEN + MAX_INT_DIGITS + 1];
+ struct cur_dstat {
+ collected_number duration_read_ms;
+ collected_number duration_write_ms;
+ collected_number duration_other_ms;
+ collected_number duration_free_ms;
+ collected_number busy_time_ms;
+ } cur_dstat;
+
+ sprintf(disk, "%s%d", dstat[i].device_name, dstat[i].unit_number);
+
+ struct disk *dm = get_disk(disk);
+ dm->updated = 1;
+ disks_found++;
+
+ if(unlikely(!dm->configured)) {
+ char var_name[4096 + 1];
+
+ // this is the first time we see this disk
+
+ // remember we configured it
+ dm->configured = 1;
+
+ dm->enabled = enable_new_disks;
+
+ if (likely(dm->enabled))
+ dm->enabled = !simple_pattern_matches(excluded_disks, disk);
+
+ snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_KERN_DEVSTAT, disk);
+ dm->enabled = config_get_boolean_ondemand(var_name, "enabled", dm->enabled);
+
+ dm->do_io = config_get_boolean_ondemand(var_name, "bandwidth", do_io);
+ dm->do_ops = config_get_boolean_ondemand(var_name, "operations", do_ops);
+ dm->do_qops = config_get_boolean_ondemand(var_name, "queued operations", do_qops);
+ dm->do_util = config_get_boolean_ondemand(var_name, "utilization percentage", do_util);
+ dm->do_iotime = config_get_boolean_ondemand(var_name, "i/o time", do_iotime);
+ dm->do_await = config_get_boolean_ondemand(var_name, "average completed i/o time",
+ do_await);
+ dm->do_avagsz = config_get_boolean_ondemand(var_name, "average completed i/o bandwidth",
+ do_avagsz);
+ dm->do_svctm = config_get_boolean_ondemand(var_name, "average service time", do_svctm);
+
+ // initialise data for differential charts
+
+ dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ];
+ dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE];
+ dm->prev_dstat.bytes_free = dstat[i].bytes[DEVSTAT_FREE];
+ dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ];
+ dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE];
+ dm->prev_dstat.operations_other = dstat[i].operations[DEVSTAT_NO_DATA];
+ dm->prev_dstat.operations_free = dstat[i].operations[DEVSTAT_FREE];
+ dm->prev_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000
+ + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
+ dm->prev_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000
+ + dstat[i].duration[DEVSTAT_WRITE].frac * BINTIME_SCALE;
+ dm->prev_dstat.duration_other_ms = dstat[i].duration[DEVSTAT_NO_DATA].sec * 1000
+ + dstat[i].duration[DEVSTAT_NO_DATA].frac * BINTIME_SCALE;
+ dm->prev_dstat.duration_free_ms = dstat[i].duration[DEVSTAT_FREE].sec * 1000
+ + dstat[i].duration[DEVSTAT_FREE].frac * BINTIME_SCALE;
+ dm->prev_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000
+ + dstat[i].busy_time.frac * BINTIME_SCALE;
+ }
+
+ cur_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000
+ + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
+ cur_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000
+ + dstat[i].duration[DEVSTAT_WRITE].frac * BINTIME_SCALE;
+ cur_dstat.duration_other_ms = dstat[i].duration[DEVSTAT_NO_DATA].sec * 1000
+ + dstat[i].duration[DEVSTAT_NO_DATA].frac * BINTIME_SCALE;
+ cur_dstat.duration_free_ms = dstat[i].duration[DEVSTAT_FREE].sec * 1000
+ + dstat[i].duration[DEVSTAT_FREE].frac * BINTIME_SCALE;
+
+ cur_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + dstat[i].busy_time.frac * BINTIME_SCALE;
+
+ if(dm->do_io == CONFIG_BOOLEAN_YES || (dm->do_io == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].bytes[DEVSTAT_READ] ||
+ dstat[i].bytes[DEVSTAT_WRITE] ||
+ dstat[i].bytes[DEVSTAT_FREE] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!dm->st_io)) {
+ dm->st_io = rrdset_create_localhost("disk",
+ disk,
+ NULL,
+ disk,
+ "disk.io",
+ "Disk I/O Bandwidth",
+ "KiB/s",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_DISK_IO,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ dm->rd_io_in = rrddim_add(dm->st_io, "reads", NULL, 1, KILO_FACTOR,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_io_out = rrddim_add(dm->st_io, "writes", NULL, -1, KILO_FACTOR,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_io_free = rrddim_add(dm->st_io, "frees", NULL, -1, KILO_FACTOR,
+ RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(dm->st_io, dm->rd_io_in, dstat[i].bytes[DEVSTAT_READ]);
+ rrddim_set_by_pointer(dm->st_io, dm->rd_io_out, dstat[i].bytes[DEVSTAT_WRITE]);
+ rrddim_set_by_pointer(dm->st_io, dm->rd_io_free, dstat[i].bytes[DEVSTAT_FREE]);
+ rrdset_done(dm->st_io);
+ }
+
+ if(dm->do_ops == CONFIG_BOOLEAN_YES || (dm->do_ops == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].operations[DEVSTAT_READ] ||
+ dstat[i].operations[DEVSTAT_WRITE] ||
+ dstat[i].operations[DEVSTAT_NO_DATA] ||
+ dstat[i].operations[DEVSTAT_FREE] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!dm->st_ops)) {
+ dm->st_ops = rrdset_create_localhost("disk_ops",
+ disk,
+ NULL,
+ disk,
+ "disk.ops",
+ "Disk Completed I/O Operations",
+ "operations/s",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_DISK_OPS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_ops, RRDSET_FLAG_DETAIL);
+
+ dm->rd_ops_in = rrddim_add(dm->st_ops, "reads", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_ops_out = rrddim_add(dm->st_ops, "writes", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_ops_other = rrddim_add(dm->st_ops, "other", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_ops_free = rrddim_add(dm->st_ops, "frees", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_in, dstat[i].operations[DEVSTAT_READ]);
+ rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_out, dstat[i].operations[DEVSTAT_WRITE]);
+ rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_other, dstat[i].operations[DEVSTAT_NO_DATA]);
+ rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_free, dstat[i].operations[DEVSTAT_FREE]);
+ rrdset_done(dm->st_ops);
+ }
+
+ if(dm->do_qops == CONFIG_BOOLEAN_YES || (dm->do_qops == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].start_count ||
+ dstat[i].end_count ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!dm->st_qops)) {
+ dm->st_qops = rrdset_create_localhost("disk_qops",
+ disk,
+ NULL,
+ disk,
+ "disk.qops",
+ "Disk Current I/O Operations",
+ "operations",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_DISK_QOPS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_qops, RRDSET_FLAG_DETAIL);
+
+ dm->rd_qops = rrddim_add(dm->st_qops, "operations", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(dm->st_qops, dm->rd_qops, dstat[i].start_count - dstat[i].end_count);
+ rrdset_done(dm->st_qops);
+ }
+
+ if(dm->do_util == CONFIG_BOOLEAN_YES || (dm->do_util == CONFIG_BOOLEAN_AUTO &&
+ (cur_dstat.busy_time_ms ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!dm->st_util)) {
+ dm->st_util = rrdset_create_localhost("disk_util",
+ disk,
+ NULL,
+ disk,
+ "disk.util",
+ "Disk Utilization Time",
+ "% of time working",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_DISK_UTIL,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(dm->st_util, RRDSET_FLAG_DETAIL);
+
+ dm->rd_util = rrddim_add(dm->st_util, "utilization", NULL, 1, 10,
+ RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(dm->st_util, dm->rd_util, cur_dstat.busy_time_ms);
+ rrdset_done(dm->st_util);
+ }
+
+ if(dm->do_iotime == CONFIG_BOOLEAN_YES || (dm->do_iotime == CONFIG_BOOLEAN_AUTO &&
+ (cur_dstat.duration_read_ms ||
+ cur_dstat.duration_write_ms ||
+ cur_dstat.duration_other_ms ||
+ cur_dstat.duration_free_ms ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!dm->st_iotime)) {
+ dm->st_iotime = rrdset_create_localhost("disk_iotime",
+ disk,
+ NULL,
+ disk,
+ "disk.iotime",
+ "Disk Total I/O Time",
+ "milliseconds/s",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_DISK_IOTIME,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_iotime, RRDSET_FLAG_DETAIL);
+
+ dm->rd_iotime_in = rrddim_add(dm->st_iotime, "reads", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_iotime_out = rrddim_add(dm->st_iotime, "writes", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_iotime_other = rrddim_add(dm->st_iotime, "other", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_iotime_free = rrddim_add(dm->st_iotime, "frees", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_in, cur_dstat.duration_read_ms);
+ rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_out, cur_dstat.duration_write_ms);
+ rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_other, cur_dstat.duration_other_ms);
+ rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_free, cur_dstat.duration_free_ms);
+ rrdset_done(dm->st_iotime);
+ }
+
+ // calculate differential charts
+ // only if this is not the first time we run
+
+ if (likely(dt)) {
+ if(dm->do_await == CONFIG_BOOLEAN_YES || (dm->do_await == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].operations[DEVSTAT_READ] ||
+ dstat[i].operations[DEVSTAT_WRITE] ||
+ dstat[i].operations[DEVSTAT_NO_DATA] ||
+ dstat[i].operations[DEVSTAT_FREE] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!dm->st_await)) {
+ dm->st_await = rrdset_create_localhost("disk_await",
+ disk,
+ NULL,
+ disk,
+ "disk.await",
+ "Average Completed I/O Operation Time",
+ "milliseconds/operation",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_DISK_AWAIT,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_await, RRDSET_FLAG_DETAIL);
+
+ dm->rd_await_in = rrddim_add(dm->st_await, "reads", NULL, 1, 1,
+ RRD_ALGORITHM_ABSOLUTE);
+ dm->rd_await_out = rrddim_add(dm->st_await, "writes", NULL, -1, 1,
+ RRD_ALGORITHM_ABSOLUTE);
+ dm->rd_await_other = rrddim_add(dm->st_await, "other", NULL, 1, 1,
+ RRD_ALGORITHM_ABSOLUTE);
+ dm->rd_await_free = rrddim_add(dm->st_await, "frees", NULL, -1, 1,
+ RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(dm->st_await, dm->rd_await_in,
+ (dstat[i].operations[DEVSTAT_READ] -
+ dm->prev_dstat.operations_read) ?
+ (cur_dstat.duration_read_ms - dm->prev_dstat.duration_read_ms) /
+ (dstat[i].operations[DEVSTAT_READ] -
+ dm->prev_dstat.operations_read) :
+ 0);
+ rrddim_set_by_pointer(dm->st_await, dm->rd_await_out,
+ (dstat[i].operations[DEVSTAT_WRITE] -
+ dm->prev_dstat.operations_write) ?
+ (cur_dstat.duration_write_ms - dm->prev_dstat.duration_write_ms) /
+ (dstat[i].operations[DEVSTAT_WRITE] -
+ dm->prev_dstat.operations_write) :
+ 0);
+ rrddim_set_by_pointer(dm->st_await, dm->rd_await_other,
+ (dstat[i].operations[DEVSTAT_NO_DATA] -
+ dm->prev_dstat.operations_other) ?
+ (cur_dstat.duration_other_ms - dm->prev_dstat.duration_other_ms) /
+ (dstat[i].operations[DEVSTAT_NO_DATA] -
+ dm->prev_dstat.operations_other) :
+ 0);
+ rrddim_set_by_pointer(dm->st_await, dm->rd_await_free,
+ (dstat[i].operations[DEVSTAT_FREE] -
+ dm->prev_dstat.operations_free) ?
+ (cur_dstat.duration_free_ms - dm->prev_dstat.duration_free_ms) /
+ (dstat[i].operations[DEVSTAT_FREE] -
+ dm->prev_dstat.operations_free) :
+ 0);
+ rrdset_done(dm->st_await);
+ }
+
+ if(dm->do_avagsz == CONFIG_BOOLEAN_YES || (dm->do_avagsz == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].operations[DEVSTAT_READ] ||
+ dstat[i].operations[DEVSTAT_WRITE] ||
+ dstat[i].operations[DEVSTAT_FREE] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!dm->st_avagsz)) {
+ dm->st_avagsz = rrdset_create_localhost("disk_avgsz",
+ disk,
+ NULL,
+ disk,
+ "disk.avgsz",
+ "Average Completed I/O Operation Bandwidth",
+ "KiB/operation",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_DISK_AVGSZ,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(dm->st_avagsz, RRDSET_FLAG_DETAIL);
+
+ dm->rd_avagsz_in = rrddim_add(dm->st_avagsz, "reads", NULL, 1, KILO_FACTOR,
+ RRD_ALGORITHM_ABSOLUTE);
+ dm->rd_avagsz_out = rrddim_add(dm->st_avagsz, "writes", NULL, -1, KILO_FACTOR,
+ RRD_ALGORITHM_ABSOLUTE);
+ dm->rd_avagsz_free = rrddim_add(dm->st_avagsz, "frees", NULL, -1, KILO_FACTOR,
+ RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_in,
+ (dstat[i].operations[DEVSTAT_READ] -
+ dm->prev_dstat.operations_read) ?
+ (dstat[i].bytes[DEVSTAT_READ] - dm->prev_dstat.bytes_read) /
+ (dstat[i].operations[DEVSTAT_READ] -
+ dm->prev_dstat.operations_read) :
+ 0);
+ rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_out,
+ (dstat[i].operations[DEVSTAT_WRITE] -
+ dm->prev_dstat.operations_write) ?
+ (dstat[i].bytes[DEVSTAT_WRITE] - dm->prev_dstat.bytes_write) /
+ (dstat[i].operations[DEVSTAT_WRITE] -
+ dm->prev_dstat.operations_write) :
+ 0);
+ rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_free,
+ (dstat[i].operations[DEVSTAT_FREE] -
+ dm->prev_dstat.operations_free) ?
+ (dstat[i].bytes[DEVSTAT_FREE] - dm->prev_dstat.bytes_free) /
+ (dstat[i].operations[DEVSTAT_FREE] -
+ dm->prev_dstat.operations_free) :
+ 0);
+ rrdset_done(dm->st_avagsz);
+ }
+
+ if(dm->do_svctm == CONFIG_BOOLEAN_YES || (dm->do_svctm == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].operations[DEVSTAT_READ] ||
+ dstat[i].operations[DEVSTAT_WRITE] ||
+ dstat[i].operations[DEVSTAT_NO_DATA] ||
+ dstat[i].operations[DEVSTAT_FREE] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!dm->st_svctm)) {
+ dm->st_svctm = rrdset_create_localhost("disk_svctm",
+ disk,
+ NULL,
+ disk,
+ "disk.svctm",
+ "Average Service Time",
+ "milliseconds/operation",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_DISK_SVCTM,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_svctm, RRDSET_FLAG_DETAIL);
+
+ dm->rd_svctm = rrddim_add(dm->st_svctm, "svctm", NULL, 1, 1,
+ RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(dm->st_svctm, dm->rd_svctm,
+ ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) +
+ (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write) +
+ (dstat[i].operations[DEVSTAT_NO_DATA] - dm->prev_dstat.operations_other) +
+ (dstat[i].operations[DEVSTAT_FREE] - dm->prev_dstat.operations_free)) ?
+ (cur_dstat.busy_time_ms - dm->prev_dstat.busy_time_ms) /
+ ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) +
+ (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write) +
+ (dstat[i].operations[DEVSTAT_NO_DATA] - dm->prev_dstat.operations_other) +
+ (dstat[i].operations[DEVSTAT_FREE] - dm->prev_dstat.operations_free)) :
+ 0);
+ rrdset_done(dm->st_svctm);
+ }
+
+ dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ];
+ dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE];
+ dm->prev_dstat.bytes_free = dstat[i].bytes[DEVSTAT_FREE];
+ dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ];
+ dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE];
+ dm->prev_dstat.operations_other = dstat[i].operations[DEVSTAT_NO_DATA];
+ dm->prev_dstat.operations_free = dstat[i].operations[DEVSTAT_FREE];
+ dm->prev_dstat.duration_read_ms = cur_dstat.duration_read_ms;
+ dm->prev_dstat.duration_write_ms = cur_dstat.duration_write_ms;
+ dm->prev_dstat.duration_other_ms = cur_dstat.duration_other_ms;
+ dm->prev_dstat.duration_free_ms = cur_dstat.duration_free_ms;
+ dm->prev_dstat.busy_time_ms = cur_dstat.busy_time_ms;
+ }
+ }
+ }
+
+ if (likely(do_system_io)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost("system",
+ "io",
+ NULL,
+ "disk",
+ NULL,
+ "Disk I/O",
+ "KiB/s",
+ "freebsd.plugin",
+ "devstat",
+ NETDATA_CHART_PRIO_SYSTEM_IO,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, total_disk_kbytes_read);
+ rrddim_set_by_pointer(st, rd_out, total_disk_kbytes_write);
+ rrdset_done(st);
+ }
+ }
+ }
+
+ if (unlikely(common_error)) {
+ do_system_io = 0;
+ error("DISABLED: system.io chart");
+ do_io = 0;
+ error("DISABLED: disk.* charts");
+ do_ops = 0;
+ error("DISABLED: disk_ops.* charts");
+ do_qops = 0;
+ error("DISABLED: disk_qops.* charts");
+ do_util = 0;
+ error("DISABLED: disk_util.* charts");
+ do_iotime = 0;
+ error("DISABLED: disk_iotime.* charts");
+ do_await = 0;
+ error("DISABLED: disk_await.* charts");
+ do_avagsz = 0;
+ error("DISABLED: disk_avgsz.* charts");
+ do_svctm = 0;
+ error("DISABLED: disk_svctm.* charts");
+ error("DISABLED: kern.devstat module");
+ return 1;
+ }
+ } else {
+ error("DISABLED: kern.devstat module");
+ return 1;
+ }
+
+ disks_cleanup();
+
+ return 0;
+}
diff --git a/collectors/freebsd.plugin/freebsd_getifaddrs.c b/collectors/freebsd.plugin/freebsd_getifaddrs.c
new file mode 100644
index 0000000..1e870c0
--- /dev/null
+++ b/collectors/freebsd.plugin/freebsd_getifaddrs.c
@@ -0,0 +1,599 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_freebsd.h"
+
+#include <ifaddrs.h>
+
+struct cgroup_network_interface {
+ char *name;
+ uint32_t hash;
+ size_t len;
+
+ // flags
+ int configured;
+ int enabled;
+ int updated;
+
+ int do_bandwidth;
+ int do_packets;
+ int do_errors;
+ int do_drops;
+ int do_events;
+
+ // charts and dimensions
+
+ RRDSET *st_bandwidth;
+ RRDDIM *rd_bandwidth_in;
+ RRDDIM *rd_bandwidth_out;
+
+ RRDSET *st_packets;
+ RRDDIM *rd_packets_in;
+ RRDDIM *rd_packets_out;
+ RRDDIM *rd_packets_m_in;
+ RRDDIM *rd_packets_m_out;
+
+ RRDSET *st_errors;
+ RRDDIM *rd_errors_in;
+ RRDDIM *rd_errors_out;
+
+ RRDSET *st_drops;
+ RRDDIM *rd_drops_in;
+ RRDDIM *rd_drops_out;
+
+ RRDSET *st_events;
+ RRDDIM *rd_events_coll;
+
+ struct cgroup_network_interface *next;
+};
+
+static struct cgroup_network_interface *network_interfaces_root = NULL, *network_interfaces_last_used = NULL;
+
+static size_t network_interfaces_added = 0, network_interfaces_found = 0;
+
+static void network_interface_free(struct cgroup_network_interface *ifm) {
+ if (likely(ifm->st_bandwidth))
+ rrdset_is_obsolete(ifm->st_bandwidth);
+ if (likely(ifm->st_packets))
+ rrdset_is_obsolete(ifm->st_packets);
+ if (likely(ifm->st_errors))
+ rrdset_is_obsolete(ifm->st_errors);
+ if (likely(ifm->st_drops))
+ rrdset_is_obsolete(ifm->st_drops);
+ if (likely(ifm->st_events))
+ rrdset_is_obsolete(ifm->st_events);
+
+ network_interfaces_added--;
+ freez(ifm->name);
+ freez(ifm);
+}
+
+static void network_interfaces_cleanup() {
+ if (likely(network_interfaces_found == network_interfaces_added)) return;
+
+ struct cgroup_network_interface *ifm = network_interfaces_root, *last = NULL;
+ while(ifm) {
+ if (unlikely(!ifm->updated)) {
+ // info("Removing network interface '%s', linked after '%s'", ifm->name, last?last->name:"ROOT");
+
+ if (network_interfaces_last_used == ifm)
+ network_interfaces_last_used = last;
+
+ struct cgroup_network_interface *t = ifm;
+
+ if (ifm == network_interfaces_root || !last)
+ network_interfaces_root = ifm = ifm->next;
+
+ else
+ last->next = ifm = ifm->next;
+
+ t->next = NULL;
+ network_interface_free(t);
+ }
+ else {
+ last = ifm;
+ ifm->updated = 0;
+ ifm = ifm->next;
+ }
+ }
+}
+
+static struct cgroup_network_interface *get_network_interface(const char *name) {
+ struct cgroup_network_interface *ifm;
+
+ uint32_t hash = simple_hash(name);
+
+ // search it, from the last position to the end
+ for(ifm = network_interfaces_last_used ; ifm ; ifm = ifm->next) {
+ if (unlikely(hash == ifm->hash && !strcmp(name, ifm->name))) {
+ network_interfaces_last_used = ifm->next;
+ return ifm;
+ }
+ }
+
+ // search it from the beginning to the last position we used
+ for(ifm = network_interfaces_root ; ifm != network_interfaces_last_used ; ifm = ifm->next) {
+ if (unlikely(hash == ifm->hash && !strcmp(name, ifm->name))) {
+ network_interfaces_last_used = ifm->next;
+ return ifm;
+ }
+ }
+
+ // create a new one
+ ifm = callocz(1, sizeof(struct cgroup_network_interface));
+ ifm->name = strdupz(name);
+ ifm->hash = simple_hash(ifm->name);
+ ifm->len = strlen(ifm->name);
+ network_interfaces_added++;
+
+ // link it to the end
+ if (network_interfaces_root) {
+ struct cgroup_network_interface *e;
+ for(e = network_interfaces_root; e->next ; e = e->next) ;
+ e->next = ifm;
+ }
+ else
+ network_interfaces_root = ifm;
+
+ return ifm;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// getifaddrs
+
+int do_getifaddrs(int update_every, usec_t dt) {
+ (void)dt;
+
+#define DEFAULT_EXCLUDED_INTERFACES "lo*"
+#define DEFAULT_PHYSICAL_INTERFACES "igb* ix* cxl* em* ixl* ixlv* bge* ixgbe* vtnet* vmx* re* igc* dwc*"
+#define CONFIG_SECTION_GETIFADDRS "plugin:freebsd:getifaddrs"
+
+ static int enable_new_interfaces = -1;
+ static int do_bandwidth_ipv4 = -1, do_bandwidth_ipv6 = -1, do_bandwidth = -1, do_packets = -1, do_bandwidth_net = -1, do_packets_net = -1,
+ do_errors = -1, do_drops = -1, do_events = -1;
+ static SIMPLE_PATTERN *excluded_interfaces = NULL, *physical_interfaces = NULL;
+
+ if (unlikely(enable_new_interfaces == -1)) {
+ enable_new_interfaces = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS,
+ "enable new interfaces detected at runtime",
+ CONFIG_BOOLEAN_AUTO);
+
+ do_bandwidth_net = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for physical interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_packets_net = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total packets for physical interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_bandwidth_ipv4 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv4 interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_bandwidth_ipv6 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv6 interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_bandwidth = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "bandwidth for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_packets = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "packets for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_errors = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "errors for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_drops = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "drops for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_events = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "collisions for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+
+ excluded_interfaces = simple_pattern_create(
+ config_get(CONFIG_SECTION_GETIFADDRS, "disable by default interfaces matching", DEFAULT_EXCLUDED_INTERFACES)
+ , NULL
+ , SIMPLE_PATTERN_EXACT
+ );
+ physical_interfaces = simple_pattern_create(
+ config_get(CONFIG_SECTION_GETIFADDRS, "set physical interfaces for system.net", DEFAULT_PHYSICAL_INTERFACES)
+ , NULL
+ , SIMPLE_PATTERN_EXACT
+ );
+ }
+
+ if (likely(do_bandwidth_ipv4 || do_bandwidth_ipv6 || do_bandwidth || do_packets || do_errors || do_bandwidth_net || do_packets_net ||
+ do_drops || do_events)) {
+ struct ifaddrs *ifap;
+
+ if (unlikely(getifaddrs(&ifap))) {
+ error("FREEBSD: getifaddrs() failed");
+ do_bandwidth_net = 0;
+ error("DISABLED: system.net chart");
+ do_packets_net = 0;
+ error("DISABLED: system.packets chart");
+ do_bandwidth_ipv4 = 0;
+ error("DISABLED: system.ipv4 chart");
+ do_bandwidth_ipv6 = 0;
+ error("DISABLED: system.ipv6 chart");
+ do_bandwidth = 0;
+ error("DISABLED: net.* charts");
+ do_packets = 0;
+ error("DISABLED: net_packets.* charts");
+ do_errors = 0;
+ error("DISABLED: net_errors.* charts");
+ do_drops = 0;
+ error("DISABLED: net_drops.* charts");
+ do_events = 0;
+ error("DISABLED: net_events.* charts");
+ error("DISABLED: getifaddrs module");
+ return 1;
+ } else {
+#define IFA_DATA(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s)
+ struct ifaddrs *ifa;
+ struct iftot {
+ u_long ift_ibytes;
+ u_long ift_obytes;
+ u_long ift_ipackets;
+ u_long ift_opackets;
+ u_long ift_imcasts;
+ u_long ift_omcasts;
+ } iftot = {0, 0, 0, 0, 0, 0};
+
+ if (likely(do_bandwidth_net)) {
+
+ iftot.ift_ibytes = iftot.ift_obytes = 0;
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_LINK)
+ continue;
+ if (!simple_pattern_matches(physical_interfaces, ifa->ifa_name))
+ continue;
+ iftot.ift_ibytes += IFA_DATA(ibytes);
+ iftot.ift_obytes += IFA_DATA(obytes);
+ }
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost("system",
+ "net",
+ NULL,
+ "network",
+ NULL,
+ "Network Traffic",
+ "kilobits/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_SYSTEM_NET,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes);
+ rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes);
+ rrdset_done(st);
+ }
+
+ if (likely(do_packets_net)) {
+ iftot.ift_ipackets = iftot.ift_opackets = iftot.ift_imcasts = iftot.ift_omcasts = 0;
+
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_LINK)
+ continue;
+ if (!simple_pattern_matches(physical_interfaces, ifa->ifa_name))
+ continue;
+ iftot.ift_ipackets += IFA_DATA(ipackets);
+ iftot.ift_opackets += IFA_DATA(opackets);
+ iftot.ift_imcasts += IFA_DATA(imcasts);
+ iftot.ift_omcasts += IFA_DATA(omcasts);
+ }
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_packets_in = NULL, *rd_packets_out = NULL, *rd_packets_m_in = NULL, *rd_packets_m_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost("system",
+ "packets",
+ NULL,
+ "network",
+ NULL,
+ "Network Packets",
+ "packets/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_SYSTEM_PACKETS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_packets_in = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_packets_out = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_packets_m_in = rrddim_add(st, "multicast_received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_packets_m_out = rrddim_add(st, "multicast_sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_packets_in, iftot.ift_ipackets);
+ rrddim_set_by_pointer(st, rd_packets_out, iftot.ift_opackets);
+ rrddim_set_by_pointer(st, rd_packets_m_in, iftot.ift_imcasts);
+ rrddim_set_by_pointer(st, rd_packets_m_out, iftot.ift_omcasts);
+ rrdset_done(st);
+ }
+
+ if (likely(do_bandwidth_ipv4)) {
+ iftot.ift_ibytes = iftot.ift_obytes = 0;
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_INET)
+ continue;
+ iftot.ift_ibytes += IFA_DATA(ibytes);
+ iftot.ift_obytes += IFA_DATA(obytes);
+ }
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost("system",
+ "ipv4",
+ NULL,
+ "network",
+ NULL,
+ "IPv4 Bandwidth",
+ "kilobits/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_SYSTEM_IPV4,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes);
+ rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes);
+ rrdset_done(st);
+ }
+
+ if (likely(do_bandwidth_ipv6)) {
+ iftot.ift_ibytes = iftot.ift_obytes = 0;
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_INET6)
+ continue;
+ iftot.ift_ibytes += IFA_DATA(ibytes);
+ iftot.ift_obytes += IFA_DATA(obytes);
+ }
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost("system",
+ "ipv6",
+ NULL,
+ "network",
+ NULL,
+ "IPv6 Bandwidth",
+ "kilobits/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_SYSTEM_IPV6,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes);
+ rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes);
+ rrdset_done(st);
+ }
+
+ network_interfaces_found = 0;
+
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_LINK)
+ continue;
+
+ struct cgroup_network_interface *ifm = get_network_interface(ifa->ifa_name);
+ ifm->updated = 1;
+ network_interfaces_found++;
+
+ if (unlikely(!ifm->configured)) {
+ char var_name[4096 + 1];
+
+ // this is the first time we see this network interface
+
+ // remember we configured it
+ ifm->configured = 1;
+
+ ifm->enabled = enable_new_interfaces;
+
+ if (likely(ifm->enabled))
+ ifm->enabled = !simple_pattern_matches(excluded_interfaces, ifa->ifa_name);
+
+ snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETIFADDRS, ifa->ifa_name);
+ ifm->enabled = config_get_boolean_ondemand(var_name, "enabled", ifm->enabled);
+
+ if (unlikely(ifm->enabled == CONFIG_BOOLEAN_NO))
+ continue;
+
+ ifm->do_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", do_bandwidth);
+ ifm->do_packets = config_get_boolean_ondemand(var_name, "packets", do_packets);
+ ifm->do_errors = config_get_boolean_ondemand(var_name, "errors", do_errors);
+ ifm->do_drops = config_get_boolean_ondemand(var_name, "drops", do_drops);
+ ifm->do_events = config_get_boolean_ondemand(var_name, "events", do_events);
+ }
+
+ if (unlikely(!ifm->enabled))
+ continue;
+
+ if (ifm->do_bandwidth == CONFIG_BOOLEAN_YES || (ifm->do_bandwidth == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(ibytes) ||
+ IFA_DATA(obytes) ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!ifm->st_bandwidth)) {
+ ifm->st_bandwidth = rrdset_create_localhost("net",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.net",
+ "Bandwidth",
+ "kilobits/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_FIRST_NET_IFACE,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ ifm->rd_bandwidth_in = rrddim_add(ifm->st_bandwidth, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_bandwidth_out = rrddim_add(ifm->st_bandwidth, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_in, IFA_DATA(ibytes));
+ rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_out, IFA_DATA(obytes));
+ rrdset_done(ifm->st_bandwidth);
+ }
+
+ if (ifm->do_packets == CONFIG_BOOLEAN_YES || (ifm->do_packets == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(ipackets) ||
+ IFA_DATA(opackets) ||
+ IFA_DATA(imcasts) ||
+ IFA_DATA(omcasts) ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!ifm->st_packets)) {
+ ifm->st_packets = rrdset_create_localhost("net_packets",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.packets",
+ "Packets",
+ "packets/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_FIRST_NET_PACKETS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(ifm->st_packets, RRDSET_FLAG_DETAIL);
+
+ ifm->rd_packets_in = rrddim_add(ifm->st_packets, "received", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_packets_out = rrddim_add(ifm->st_packets, "sent", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_packets_m_in = rrddim_add(ifm->st_packets, "multicast_received", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_packets_m_out = rrddim_add(ifm->st_packets, "multicast_sent", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_in, IFA_DATA(ipackets));
+ rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_out, IFA_DATA(opackets));
+ rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_in, IFA_DATA(imcasts));
+ rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_out, IFA_DATA(omcasts));
+ rrdset_done(ifm->st_packets);
+ }
+
+ if (ifm->do_errors == CONFIG_BOOLEAN_YES || (ifm->do_errors == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(ierrors) ||
+ IFA_DATA(oerrors) ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!ifm->st_errors)) {
+ ifm->st_errors = rrdset_create_localhost("net_errors",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.errors",
+ "Interface Errors",
+ "errors/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_FIRST_NET_ERRORS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(ifm->st_errors, RRDSET_FLAG_DETAIL);
+
+ ifm->rd_errors_in = rrddim_add(ifm->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_errors_out = rrddim_add(ifm->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_in, IFA_DATA(ierrors));
+ rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_out, IFA_DATA(oerrors));
+ rrdset_done(ifm->st_errors);
+ }
+
+ if (ifm->do_drops == CONFIG_BOOLEAN_YES || (ifm->do_drops == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(iqdrops) ||
+ #if __FreeBSD__ >= 11
+ IFA_DATA(oqdrops) ||
+ #endif
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!ifm->st_drops)) {
+ ifm->st_drops = rrdset_create_localhost("net_drops",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.drops",
+ "Interface Drops",
+ "drops/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_FIRST_NET_DROPS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(ifm->st_drops, RRDSET_FLAG_DETAIL);
+
+ ifm->rd_drops_in = rrddim_add(ifm->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#if __FreeBSD__ >= 11
+ ifm->rd_drops_out = rrddim_add(ifm->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+#endif
+ }
+
+ rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_in, IFA_DATA(iqdrops));
+#if __FreeBSD__ >= 11
+ rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_out, IFA_DATA(oqdrops));
+#endif
+ rrdset_done(ifm->st_drops);
+ }
+
+ if (ifm->do_events == CONFIG_BOOLEAN_YES || (ifm->do_events == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(collisions) ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!ifm->st_events)) {
+ ifm->st_events = rrdset_create_localhost("net_events",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.events",
+ "Network Interface Events",
+ "events/s",
+ "freebsd.plugin",
+ "getifaddrs",
+ NETDATA_CHART_PRIO_FIRST_NET_EVENTS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(ifm->st_events, RRDSET_FLAG_DETAIL);
+
+ ifm->rd_events_coll = rrddim_add(ifm->st_events, "collisions", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(ifm->st_events, ifm->rd_events_coll, IFA_DATA(collisions));
+ rrdset_done(ifm->st_events);
+ }
+ }
+
+ freeifaddrs(ifap);
+ }
+ } else {
+ error("DISABLED: getifaddrs module");
+ return 1;
+ }
+
+ network_interfaces_cleanup();
+
+ return 0;
+}
diff --git a/collectors/freebsd.plugin/freebsd_getmntinfo.c b/collectors/freebsd.plugin/freebsd_getmntinfo.c
new file mode 100644
index 0000000..e8feefc
--- /dev/null
+++ b/collectors/freebsd.plugin/freebsd_getmntinfo.c
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_freebsd.h"
+
+#include <sys/mount.h>
+
+struct mount_point {
+ char *name;
+ uint32_t hash;
+ size_t len;
+
+ // flags
+ int configured;
+ int enabled;
+ int updated;
+
+ int do_space;
+ int do_inodes;
+
+ size_t collected; // the number of times this has been collected
+
+ // charts and dimensions
+
+ RRDSET *st_space;
+ RRDDIM *rd_space_used;
+ RRDDIM *rd_space_avail;
+ RRDDIM *rd_space_reserved;
+
+ RRDSET *st_inodes;
+ RRDDIM *rd_inodes_used;
+ RRDDIM *rd_inodes_avail;
+
+ struct mount_point *next;
+};
+
+static struct mount_point *mount_points_root = NULL, *mount_points_last_used = NULL;
+
+static size_t mount_points_added = 0, mount_points_found = 0;
+
+static void mount_point_free(struct mount_point *m) {
+ if (likely(m->st_space))
+ rrdset_is_obsolete(m->st_space);
+ if (likely(m->st_inodes))
+ rrdset_is_obsolete(m->st_inodes);
+
+ mount_points_added--;
+ freez(m->name);
+ freez(m);
+}
+
+static void mount_points_cleanup() {
+ if (likely(mount_points_found == mount_points_added)) return;
+
+ struct mount_point *m = mount_points_root, *last = NULL;
+ while(m) {
+ if (unlikely(!m->updated)) {
+ // info("Removing mount point '%s', linked after '%s'", m->name, last?last->name:"ROOT");
+
+ if (mount_points_last_used == m)
+ mount_points_last_used = last;
+
+ struct mount_point *t = m;
+
+ if (m == mount_points_root || !last)
+ mount_points_root = m = m->next;
+
+ else
+ last->next = m = m->next;
+
+ t->next = NULL;
+ mount_point_free(t);
+ }
+ else {
+ last = m;
+ m->updated = 0;
+ m = m->next;
+ }
+ }
+}
+
+static struct mount_point *get_mount_point(const char *name) {
+ struct mount_point *m;
+
+ uint32_t hash = simple_hash(name);
+
+ // search it, from the last position to the end
+ for(m = mount_points_last_used ; m ; m = m->next) {
+ if (unlikely(hash == m->hash && !strcmp(name, m->name))) {
+ mount_points_last_used = m->next;
+ return m;
+ }
+ }
+
+ // search it from the beginning to the last position we used
+ for(m = mount_points_root ; m != mount_points_last_used ; m = m->next) {
+ if (unlikely(hash == m->hash && !strcmp(name, m->name))) {
+ mount_points_last_used = m->next;
+ return m;
+ }
+ }
+
+ // create a new one
+ m = callocz(1, sizeof(struct mount_point));
+ m->name = strdupz(name);
+ m->hash = simple_hash(m->name);
+ m->len = strlen(m->name);
+ mount_points_added++;
+
+ // link it to the end
+ if (mount_points_root) {
+ struct mount_point *e;
+ for(e = mount_points_root; e->next ; e = e->next) ;
+ e->next = m;
+ }
+ else
+ mount_points_root = m;
+
+ return m;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// getmntinfo
+
+int do_getmntinfo(int update_every, usec_t dt) {
+ (void)dt;
+
+#define DEFAULT_EXCLUDED_PATHS "/proc/*"
+// taken from gnulib/mountlist.c and shortened to FreeBSD related fstypes
+#define DEFAULT_EXCLUDED_FILESYSTEMS "autofs procfs subfs devfs none"
+#define CONFIG_SECTION_GETMNTINFO "plugin:freebsd:getmntinfo"
+
+ static int enable_new_mount_points = -1;
+ static int do_space = -1, do_inodes = -1;
+ static SIMPLE_PATTERN *excluded_mountpoints = NULL;
+ static SIMPLE_PATTERN *excluded_filesystems = NULL;
+
+ if (unlikely(enable_new_mount_points == -1)) {
+ enable_new_mount_points = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO,
+ "enable new mount points detected at runtime",
+ CONFIG_BOOLEAN_AUTO);
+
+ do_space = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "space usage for all disks", CONFIG_BOOLEAN_AUTO);
+ do_inodes = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "inodes usage for all disks", CONFIG_BOOLEAN_AUTO);
+
+ excluded_mountpoints = simple_pattern_create(
+ config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on paths",
+ DEFAULT_EXCLUDED_PATHS)
+ , NULL
+ , SIMPLE_PATTERN_EXACT
+ );
+
+ excluded_filesystems = simple_pattern_create(
+ config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on filesystems",
+ DEFAULT_EXCLUDED_FILESYSTEMS)
+ , NULL
+ , SIMPLE_PATTERN_EXACT
+ );
+ }
+
+ if (likely(do_space || do_inodes)) {
+ struct statfs *mntbuf;
+ int mntsize;
+
+ // there is no mount info in sysctl MIBs
+ if (unlikely(!(mntsize = getmntinfo(&mntbuf, MNT_NOWAIT)))) {
+ error("FREEBSD: getmntinfo() failed");
+ do_space = 0;
+ error("DISABLED: disk_space.* charts");
+ do_inodes = 0;
+ error("DISABLED: disk_inodes.* charts");
+ error("DISABLED: getmntinfo module");
+ return 1;
+ } else {
+ int i;
+
+ mount_points_found = 0;
+
+ for (i = 0; i < mntsize; i++) {
+ char title[4096 + 1];
+
+ struct mount_point *m = get_mount_point(mntbuf[i].f_mntonname);
+ m->updated = 1;
+ mount_points_found++;
+
+ if (unlikely(!m->configured)) {
+ char var_name[4096 + 1];
+
+ // this is the first time we see this filesystem
+
+ // remember we configured it
+ m->configured = 1;
+
+ m->enabled = enable_new_mount_points;
+
+ if (likely(m->enabled))
+ m->enabled = !(simple_pattern_matches(excluded_mountpoints, mntbuf[i].f_mntonname)
+ || simple_pattern_matches(excluded_filesystems, mntbuf[i].f_fstypename));
+
+ snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETMNTINFO, mntbuf[i].f_mntonname);
+ m->enabled = config_get_boolean_ondemand(var_name, "enabled", m->enabled);
+
+ if (unlikely(m->enabled == CONFIG_BOOLEAN_NO))
+ continue;
+
+ m->do_space = config_get_boolean_ondemand(var_name, "space usage", do_space);
+ m->do_inodes = config_get_boolean_ondemand(var_name, "inodes usage", do_inodes);
+ }
+
+ if (unlikely(!m->enabled))
+ continue;
+
+ if (unlikely(mntbuf[i].f_flags & MNT_RDONLY && !m->collected))
+ continue;
+
+ int rendered = 0;
+
+ if (m->do_space == CONFIG_BOOLEAN_YES || (m->do_space == CONFIG_BOOLEAN_AUTO &&
+ (mntbuf[i].f_blocks > 2 ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!m->st_space)) {
+ snprintfz(title, 4096, "Disk Space Usage for %s [%s]",
+ mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
+ m->st_space = rrdset_create_localhost("disk_space",
+ mntbuf[i].f_mntonname,
+ NULL,
+ mntbuf[i].f_mntonname,
+ "disk.space",
+ title,
+ "GiB",
+ "freebsd.plugin",
+ "getmntinfo",
+ NETDATA_CHART_PRIO_DISKSPACE_SPACE,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL,
+ mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_space_used = rrddim_add(m->st_space, "used", NULL,
+ mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root",
+ mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number) mntbuf[i].f_bavail);
+ rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number) (mntbuf[i].f_blocks -
+ mntbuf[i].f_bfree));
+ rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number) (mntbuf[i].f_bfree -
+ mntbuf[i].f_bavail));
+ rrdset_done(m->st_space);
+
+ rendered++;
+ }
+
+ if (m->do_inodes == CONFIG_BOOLEAN_YES || (m->do_inodes == CONFIG_BOOLEAN_AUTO &&
+ (mntbuf[i].f_files > 1 ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ if (unlikely(!m->st_inodes)) {
+ snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]",
+ mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
+ m->st_inodes = rrdset_create_localhost("disk_inodes",
+ mntbuf[i].f_mntonname,
+ NULL,
+ mntbuf[i].f_mntonname,
+ "disk.inodes",
+ title,
+ "inodes",
+ "freebsd.plugin",
+ "getmntinfo",
+ NETDATA_CHART_PRIO_DISKSPACE_INODES,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number) mntbuf[i].f_ffree);
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number) (mntbuf[i].f_files -
+ mntbuf[i].f_ffree));
+ rrdset_done(m->st_inodes);
+
+ rendered++;
+ }
+
+ if (likely(rendered))
+ m->collected++;
+ }
+ }
+ } else {
+ error("DISABLED: getmntinfo module");
+ return 1;
+ }
+
+ mount_points_cleanup();
+
+ return 0;
+}
diff --git a/collectors/freebsd.plugin/freebsd_ipfw.c b/collectors/freebsd.plugin/freebsd_ipfw.c
new file mode 100644
index 0000000..178eaa3
--- /dev/null
+++ b/collectors/freebsd.plugin/freebsd_ipfw.c
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_freebsd.h"
+
+#include <netinet/ip_fw.h>
+
+#define FREE_MEM_THRESHOLD 10000 // number of unused chunks that trigger memory freeing
+
+#define COMMON_IPFW_ERROR() error("DISABLED: ipfw.packets chart"); \
+ error("DISABLED: ipfw.bytes chart"); \
+ error("DISABLED: ipfw.dyn_active chart"); \
+ error("DISABLED: ipfw.dyn_expired chart"); \
+ error("DISABLED: ipfw.mem chart");
+
+// --------------------------------------------------------------------------------------------------------------------
+// ipfw
+
+int do_ipfw(int update_every, usec_t dt) {
+ (void)dt;
+#if __FreeBSD__ >= 11
+
+ static int do_static = -1, do_dynamic = -1, do_mem = -1;
+
+ if (unlikely(do_static == -1)) {
+ do_static = config_get_boolean("plugin:freebsd:ipfw", "counters for static rules", 1);
+ do_dynamic = config_get_boolean("plugin:freebsd:ipfw", "number of dynamic rules", 1);
+ do_mem = config_get_boolean("plugin:freebsd:ipfw", "allocated memory", 1);
+ }
+
+ // variables for getting ipfw configuration
+
+ int error;
+ static int ipfw_socket = -1;
+ static ipfw_cfg_lheader *cfg = NULL;
+ ip_fw3_opheader *op3 = NULL;
+ static socklen_t *optlen = NULL, cfg_size = 0;
+
+ // variables for static rules handling
+
+ ipfw_obj_ctlv *ctlv = NULL;
+ ipfw_obj_tlv *rbase = NULL;
+ int rcnt = 0;
+
+ int n, seen;
+ struct ip_fw_rule *rule;
+ struct ip_fw_bcounter *cntr;
+ int c = 0;
+
+ char rule_num_str[12];
+
+ // variables for dynamic rules handling
+
+ caddr_t dynbase = NULL;
+ size_t dynsz = 0;
+ size_t readsz = sizeof(*cfg);;
+ int ttype = 0;
+ ipfw_obj_tlv *tlv;
+ ipfw_dyn_rule *dyn_rule;
+ uint16_t rulenum, prev_rulenum = IPFW_DEFAULT_RULE;
+ unsigned srn, static_rules_num = 0;
+ static size_t dyn_rules_num_size = 0;
+
+ static struct dyn_rule_num {
+ uint16_t rule_num;
+ uint32_t active_rules;
+ uint32_t expired_rules;
+ } *dyn_rules_num = NULL;
+
+ uint32_t *dyn_rules_counter;
+
+ if (likely(do_static | do_dynamic | do_mem)) {
+
+ // initialize the smallest ipfw_cfg_lheader possible
+
+ if (unlikely((optlen == NULL) || (cfg == NULL))) {
+ optlen = reallocz(optlen, sizeof(socklen_t));
+ *optlen = cfg_size = 32;
+ cfg = reallocz(cfg, *optlen);
+ }
+
+ // get socket descriptor and initialize ipfw_cfg_lheader structure
+
+ if (unlikely(ipfw_socket == -1))
+ ipfw_socket = socket(AF_INET, SOCK_RAW, IPPROTO_RAW);
+ if (unlikely(ipfw_socket == -1)) {
+ error("FREEBSD: can't get socket for ipfw configuration");
+ error("FREEBSD: run netdata as root to get access to ipfw data");
+ COMMON_IPFW_ERROR();
+ return 1;
+ }
+
+ bzero(cfg, 32);
+ cfg->flags = IPFW_CFG_GET_STATIC | IPFW_CFG_GET_COUNTERS | IPFW_CFG_GET_STATES;
+ op3 = &cfg->opheader;
+ op3->opcode = IP_FW_XGET;
+
+ // get ifpw configuration size than get configuration
+
+ *optlen = cfg_size;
+ error = getsockopt(ipfw_socket, IPPROTO_IP, IP_FW3, op3, optlen);
+ if (error)
+ if (errno != ENOMEM) {
+ error("FREEBSD: ipfw socket reading error");
+ COMMON_IPFW_ERROR();
+ return 1;
+ }
+ if ((cfg->size > cfg_size) || ((cfg_size - cfg->size) > sizeof(struct dyn_rule_num) * FREE_MEM_THRESHOLD)) {
+ *optlen = cfg_size = cfg->size;
+ cfg = reallocz(cfg, *optlen);
+ bzero(cfg, 32);
+ cfg->flags = IPFW_CFG_GET_STATIC | IPFW_CFG_GET_COUNTERS | IPFW_CFG_GET_STATES;
+ op3 = &cfg->opheader;
+ op3->opcode = IP_FW_XGET;
+ error = getsockopt(ipfw_socket, IPPROTO_IP, IP_FW3, op3, optlen);
+ if (error) {
+ error("FREEBSD: ipfw socket reading error");
+ COMMON_IPFW_ERROR();
+ return 1;
+ }
+ }
+
+ // go through static rules configuration structures
+
+ ctlv = (ipfw_obj_ctlv *) (cfg + 1);
+
+ if (cfg->flags & IPFW_CFG_GET_STATIC) {
+ /* We've requested static rules */
+ if (ctlv->head.type == IPFW_TLV_TBLNAME_LIST) {
+ readsz += ctlv->head.length;
+ ctlv = (ipfw_obj_ctlv *) ((caddr_t) ctlv +
+ ctlv->head.length);
+ }
+
+ if (ctlv->head.type == IPFW_TLV_RULE_LIST) {
+ rbase = (ipfw_obj_tlv *) (ctlv + 1);
+ rcnt = ctlv->count;
+ readsz += ctlv->head.length;
+ ctlv = (ipfw_obj_ctlv *) ((caddr_t) ctlv + ctlv->head.length);
+ }
+ }
+
+ if ((cfg->flags & IPFW_CFG_GET_STATES) && (readsz != *optlen)) {
+ /* We may have some dynamic states */
+ dynsz = *optlen - readsz;
+ /* Skip empty header */
+ if (dynsz != sizeof(ipfw_obj_ctlv))
+ dynbase = (caddr_t) ctlv;
+ else
+ dynsz = 0;
+ }
+
+ if (likely(do_mem)) {
+ static RRDSET *st_mem = NULL;
+ static RRDDIM *rd_dyn_mem = NULL;
+ static RRDDIM *rd_stat_mem = NULL;
+
+ if (unlikely(!st_mem)) {
+ st_mem = rrdset_create_localhost("ipfw",
+ "mem",
+ NULL,
+ "memory allocated",
+ NULL,
+ "Memory allocated by rules",
+ "bytes",
+ "freebsd.plugin",
+ "ipfw",
+ NETDATA_CHART_PRIO_IPFW_MEM,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ rrdset_flag_set(st_mem, RRDSET_FLAG_DETAIL);
+
+ rd_dyn_mem = rrddim_add(st_mem, "dynamic", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_stat_mem = rrddim_add(st_mem, "static", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem, rd_dyn_mem, dynsz);
+ rrddim_set_by_pointer(st_mem, rd_stat_mem, *optlen - dynsz);
+ rrdset_done(st_mem);
+ }
+
+ static RRDSET *st_packets = NULL, *st_bytes = NULL;
+ RRDDIM *rd_packets = NULL, *rd_bytes = NULL;
+
+ if (likely(do_static || do_dynamic)) {
+ if (likely(do_static)) {
+ if (unlikely(!st_packets)) {
+ st_packets = rrdset_create_localhost("ipfw",
+ "packets",
+ NULL,
+ "static rules",
+ NULL,
+ "Packets",
+ "packets/s",
+ "freebsd.plugin",
+ "ipfw",
+ NETDATA_CHART_PRIO_IPFW_PACKETS,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ }
+
+ if (unlikely(!st_bytes)) {
+ st_bytes = rrdset_create_localhost("ipfw",
+ "bytes",
+ NULL,
+ "static rules",
+ NULL,
+ "Bytes",
+ "bytes/s",
+ "freebsd.plugin",
+ "ipfw",
+ NETDATA_CHART_PRIO_IPFW_BYTES,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ }
+ }
+
+ for (n = seen = 0; n < rcnt; n++, rbase = (ipfw_obj_tlv *) ((caddr_t) rbase + rbase->length)) {
+ cntr = (struct ip_fw_bcounter *) (rbase + 1);
+ rule = (struct ip_fw_rule *) ((caddr_t) cntr + cntr->size);
+ if (rule->rulenum != prev_rulenum)
+ static_rules_num++;
+ if (rule->rulenum > IPFW_DEFAULT_RULE)
+ break;
+
+ if (likely(do_static)) {
+ sprintf(rule_num_str, "%"PRIu32"_%"PRIu32"", (uint32_t)rule->rulenum, (uint32_t)rule->id);
+
+ rd_packets = rrddim_find_active(st_packets, rule_num_str);
+ if (unlikely(!rd_packets))
+ rd_packets = rrddim_add(st_packets, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_set_by_pointer(st_packets, rd_packets, cntr->pcnt);
+
+ rd_bytes = rrddim_find_active(st_bytes, rule_num_str);
+ if (unlikely(!rd_bytes))
+ rd_bytes = rrddim_add(st_bytes, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_set_by_pointer(st_bytes, rd_bytes, cntr->bcnt);
+ }
+
+ c += rbase->length;
+ seen++;
+ }
+
+ if (likely(do_static)) {
+ rrdset_done(st_packets);
+ rrdset_done(st_bytes);
+ }
+ }
+
+ // go through dynamic rules configuration structures
+
+ if (likely(do_dynamic && (dynsz > 0))) {
+ if ((dyn_rules_num_size < sizeof(struct dyn_rule_num) * static_rules_num) ||
+ ((dyn_rules_num_size - sizeof(struct dyn_rule_num) * static_rules_num) >
+ sizeof(struct dyn_rule_num) * FREE_MEM_THRESHOLD)) {
+ dyn_rules_num_size = sizeof(struct dyn_rule_num) * static_rules_num;
+ dyn_rules_num = reallocz(dyn_rules_num, dyn_rules_num_size);
+ }
+ bzero(dyn_rules_num, sizeof(struct dyn_rule_num) * static_rules_num);
+ dyn_rules_num->rule_num = IPFW_DEFAULT_RULE;
+
+ if (dynsz > 0 && ctlv->head.type == IPFW_TLV_DYNSTATE_LIST) {
+ dynbase += sizeof(*ctlv);
+ dynsz -= sizeof(*ctlv);
+ ttype = IPFW_TLV_DYN_ENT;
+ }
+
+ while (dynsz > 0) {
+ tlv = (ipfw_obj_tlv *) dynbase;
+ if (tlv->type != ttype)
+ break;
+
+ dyn_rule = (ipfw_dyn_rule *) (tlv + 1);
+ bcopy(&dyn_rule->rule, &rulenum, sizeof(rulenum));
+
+ for (srn = 0; srn < (static_rules_num - 1); srn++) {
+ if (dyn_rule->expire > 0)
+ dyn_rules_counter = &dyn_rules_num[srn].active_rules;
+ else
+ dyn_rules_counter = &dyn_rules_num[srn].expired_rules;
+ if (dyn_rules_num[srn].rule_num == rulenum) {
+ (*dyn_rules_counter)++;
+ break;
+ }
+ if (dyn_rules_num[srn].rule_num == IPFW_DEFAULT_RULE) {
+ dyn_rules_num[srn].rule_num = rulenum;
+ dyn_rules_num[srn + 1].rule_num = IPFW_DEFAULT_RULE;
+ (*dyn_rules_counter)++;
+ break;
+ }
+ }
+
+ dynsz -= tlv->length;
+ dynbase += tlv->length;
+ }
+
+ static RRDSET *st_active = NULL, *st_expired = NULL;
+ RRDDIM *rd_active = NULL, *rd_expired = NULL;
+
+ if (unlikely(!st_active)) {
+ st_active = rrdset_create_localhost("ipfw",
+ "active",
+ NULL,
+ "dynamic_rules",
+ NULL,
+ "Active rules",
+ "rules",
+ "freebsd.plugin",
+ "ipfw",
+ NETDATA_CHART_PRIO_IPFW_ACTIVE,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ }
+
+ if (unlikely(!st_expired)) {
+ st_expired = rrdset_create_localhost("ipfw",
+ "expired",
+ NULL,
+ "dynamic_rules",
+ NULL,
+ "Expired rules",
+ "rules",
+ "freebsd.plugin",
+ "ipfw",
+ NETDATA_CHART_PRIO_IPFW_EXPIRED,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ }
+
+ for (srn = 0; (srn < (static_rules_num - 1)) && (dyn_rules_num[srn].rule_num != IPFW_DEFAULT_RULE); srn++) {
+ sprintf(rule_num_str, "%d", dyn_rules_num[srn].rule_num);
+
+ rd_active = rrddim_find_active(st_active, rule_num_str);
+ if (unlikely(!rd_active))
+ rd_active = rrddim_add(st_active, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_set_by_pointer(st_active, rd_active, dyn_rules_num[srn].active_rules);
+
+ rd_expired = rrddim_find_active(st_expired, rule_num_str);
+ if (unlikely(!rd_expired))
+ rd_expired = rrddim_add(st_expired, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_set_by_pointer(st_expired, rd_expired, dyn_rules_num[srn].expired_rules);
+ }
+
+ rrdset_done(st_active);
+ rrdset_done(st_expired);
+ }
+ }
+
+ return 0;
+#else
+ error("FREEBSD: ipfw charts supported for FreeBSD 11.0 and newer releases only");
+ COMMON_IPFW_ERROR();
+ return 1;
+#endif
+}
diff --git a/collectors/freebsd.plugin/freebsd_kstat_zfs.c b/collectors/freebsd.plugin/freebsd_kstat_zfs.c
new file mode 100644
index 0000000..046a1e6
--- /dev/null
+++ b/collectors/freebsd.plugin/freebsd_kstat_zfs.c
@@ -0,0 +1,304 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_freebsd.h"
+#include "collectors/proc.plugin/zfs_common.h"
+
+extern struct arcstats arcstats;
+
+unsigned long long zfs_arcstats_shrinkable_cache_size_bytes = 0;
+
+// kstat.zfs.misc.arcstats
+
+int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt) {
+ (void)dt;
+
+ static int show_zero_charts = -1;
+ if(unlikely(show_zero_charts == -1))
+ show_zero_charts = config_get_boolean_ondemand("plugin:freebsd:zfs_arcstats", "show zero charts", CONFIG_BOOLEAN_NO);
+
+ unsigned long long l2_size;
+ size_t uint64_t_size = sizeof(uint64_t);
+ static struct mibs {
+ int hits[5];
+ int misses[5];
+ int demand_data_hits[5];
+ int demand_data_misses[5];
+ int demand_metadata_hits[5];
+ int demand_metadata_misses[5];
+ int prefetch_data_hits[5];
+ int prefetch_data_misses[5];
+ int prefetch_metadata_hits[5];
+ int prefetch_metadata_misses[5];
+ int mru_hits[5];
+ int mru_ghost_hits[5];
+ int mfu_hits[5];
+ int mfu_ghost_hits[5];
+ int deleted[5];
+ int mutex_miss[5];
+ int evict_skip[5];
+ // int evict_not_enough[5];
+ // int evict_l2_cached[5];
+ // int evict_l2_eligible[5];
+ // int evict_l2_ineligible[5];
+ // int evict_l2_skip[5];
+ int hash_elements[5];
+ int hash_elements_max[5];
+ int hash_collisions[5];
+ int hash_chains[5];
+ int hash_chain_max[5];
+ int p[5];
+ int c[5];
+ int c_min[5];
+ int c_max[5];
+ int size[5];
+ // int hdr_size[5];
+ // int data_size[5];
+ // int metadata_size[5];
+ // int other_size[5];
+ // int anon_size[5];
+ // int anon_evictable_data[5];
+ // int anon_evictable_metadata[5];
+ int mru_size[5];
+ // int mru_evictable_data[5];
+ // int mru_evictable_metadata[5];
+ // int mru_ghost_size[5];
+ // int mru_ghost_evictable_data[5];
+ // int mru_ghost_evictable_metadata[5];
+ int mfu_size[5];
+ // int mfu_evictable_data[5];
+ // int mfu_evictable_metadata[5];
+ // int mfu_ghost_size[5];
+ // int mfu_ghost_evictable_data[5];
+ // int mfu_ghost_evictable_metadata[5];
+ int l2_hits[5];
+ int l2_misses[5];
+ // int l2_feeds[5];
+ // int l2_rw_clash[5];
+ int l2_read_bytes[5];
+ int l2_write_bytes[5];
+ // int l2_writes_sent[5];
+ // int l2_writes_done[5];
+ // int l2_writes_error[5];
+ // int l2_writes_lock_retry[5];
+ // int l2_evict_lock_retry[5];
+ // int l2_evict_reading[5];
+ // int l2_evict_l1cached[5];
+ // int l2_free_on_write[5];
+ // int l2_cdata_free_on_write[5];
+ // int l2_abort_lowmem[5];
+ // int l2_cksum_bad[5];
+ // int l2_io_error[5];
+ int l2_size[5];
+ int l2_asize[5];
+ // int l2_hdr_size[5];
+ // int l2_compress_successes[5];
+ // int l2_compress_zeros[5];
+ // int l2_compress_failures[5];
+ int memory_throttle_count[5];
+ // int duplicate_buffers[5];
+ // int duplicate_buffers_size[5];
+ // int duplicate_reads[5];
+ // int memory_direct_count[5];
+ // int memory_indirect_count[5];
+ // int arc_no_grow[5];
+ // int arc_tempreserve[5];
+ // int arc_loaned_bytes[5];
+ // int arc_prune[5];
+ // int arc_meta_used[5];
+ // int arc_meta_limit[5];
+ // int arc_meta_max[5];
+ // int arc_meta_min[5];
+ // int arc_need_free[5];
+ // int arc_sys_free[5];
+ } mibs;
+
+ arcstats.l2exist = -1;
+
+ if(unlikely(sysctlbyname("kstat.zfs.misc.arcstats.l2_size", &l2_size, &uint64_t_size, NULL, 0)))
+ return 0;
+
+ if(likely(l2_size))
+ arcstats.l2exist = 1;
+ else
+ arcstats.l2exist = 0;
+
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hits", mibs.hits, arcstats.hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.misses", mibs.misses, arcstats.misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_data_hits", mibs.demand_data_hits, arcstats.demand_data_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_data_misses", mibs.demand_data_misses, arcstats.demand_data_misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_metadata_hits", mibs.demand_metadata_hits, arcstats.demand_metadata_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_metadata_misses", mibs.demand_metadata_misses, arcstats.demand_metadata_misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_data_hits", mibs.prefetch_data_hits, arcstats.prefetch_data_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_data_misses", mibs.prefetch_data_misses, arcstats.prefetch_data_misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_metadata_hits", mibs.prefetch_metadata_hits, arcstats.prefetch_metadata_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_metadata_misses", mibs.prefetch_metadata_misses, arcstats.prefetch_metadata_misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_hits", mibs.mru_hits, arcstats.mru_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_hits", mibs.mru_ghost_hits, arcstats.mru_ghost_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_hits", mibs.mfu_hits, arcstats.mfu_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_hits", mibs.mfu_ghost_hits, arcstats.mfu_ghost_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.deleted", mibs.deleted, arcstats.deleted);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mutex_miss", mibs.mutex_miss, arcstats.mutex_miss);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_skip", mibs.evict_skip, arcstats.evict_skip);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_not_enough", mibs.evict_not_enough, arcstats.evict_not_enough);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_cached", mibs.evict_l2_cached, arcstats.evict_l2_cached);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_eligible", mibs.evict_l2_eligible, arcstats.evict_l2_eligible);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_ineligible", mibs.evict_l2_ineligible, arcstats.evict_l2_ineligible);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_skip", mibs.evict_l2_skip, arcstats.evict_l2_skip);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_elements", mibs.hash_elements, arcstats.hash_elements);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_elements_max", mibs.hash_elements_max, arcstats.hash_elements_max);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_collisions", mibs.hash_collisions, arcstats.hash_collisions);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_chains", mibs.hash_chains, arcstats.hash_chains);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_chain_max", mibs.hash_chain_max, arcstats.hash_chain_max);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.p", mibs.p, arcstats.p);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c", mibs.c, arcstats.c);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c_min", mibs.c_min, arcstats.c_min);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c_max", mibs.c_max, arcstats.c_max);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.size", mibs.size, arcstats.size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hdr_size", mibs.hdr_size, arcstats.hdr_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.data_size", mibs.data_size, arcstats.data_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.metadata_size", mibs.metadata_size, arcstats.metadata_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.other_size", mibs.other_size, arcstats.other_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_size", mibs.anon_size, arcstats.anon_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_evictable_data", mibs.anon_evictable_data, arcstats.anon_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_evictable_metadata", mibs.anon_evictable_metadata, arcstats.anon_evictable_metadata);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_size", mibs.mru_size, arcstats.mru_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_evictable_data", mibs.mru_evictable_data, arcstats.mru_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_evictable_metadata", mibs.mru_evictable_metadata, arcstats.mru_evictable_metadata);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_size", mibs.mru_ghost_size, arcstats.mru_ghost_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_evictable_data", mibs.mru_ghost_evictable_data, arcstats.mru_ghost_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_evictable_metadata", mibs.mru_ghost_evictable_metadata, arcstats.mru_ghost_evictable_metadata);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_size", mibs.mfu_size, arcstats.mfu_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_evictable_data", mibs.mfu_evictable_data, arcstats.mfu_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_evictable_metadata", mibs.mfu_evictable_metadata, arcstats.mfu_evictable_metadata);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_size", mibs.mfu_ghost_size, arcstats.mfu_ghost_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_evictable_data", mibs.mfu_ghost_evictable_data, arcstats.mfu_ghost_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_evictable_metadata", mibs.mfu_ghost_evictable_metadata, arcstats.mfu_ghost_evictable_metadata);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_hits", mibs.l2_hits, arcstats.l2_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_misses", mibs.l2_misses, arcstats.l2_misses);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_feeds", mibs.l2_feeds, arcstats.l2_feeds);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_rw_clash", mibs.l2_rw_clash, arcstats.l2_rw_clash);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_read_bytes", mibs.l2_read_bytes, arcstats.l2_read_bytes);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_write_bytes", mibs.l2_write_bytes, arcstats.l2_write_bytes);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_sent", mibs.l2_writes_sent, arcstats.l2_writes_sent);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_done", mibs.l2_writes_done, arcstats.l2_writes_done);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_error", mibs.l2_writes_error, arcstats.l2_writes_error);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_lock_retry", mibs.l2_writes_lock_retry, arcstats.l2_writes_lock_retry);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_lock_retry", mibs.l2_evict_lock_retry, arcstats.l2_evict_lock_retry);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_reading", mibs.l2_evict_reading, arcstats.l2_evict_reading);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_l1cached", mibs.l2_evict_l1cached, arcstats.l2_evict_l1cached);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_free_on_write", mibs.l2_free_on_write, arcstats.l2_free_on_write);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_cdata_free_on_write", mibs.l2_cdata_free_on_write, arcstats.l2_cdata_free_on_write);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_abort_lowmem", mibs.l2_abort_lowmem, arcstats.l2_abort_lowmem);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_cksum_bad", mibs.l2_cksum_bad, arcstats.l2_cksum_bad);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_io_error", mibs.l2_io_error, arcstats.l2_io_error);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_size", mibs.l2_size, arcstats.l2_size);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_asize", mibs.l2_asize, arcstats.l2_asize);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_hdr_size", mibs.l2_hdr_size, arcstats.l2_hdr_size);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_successes", mibs.l2_compress_successes, arcstats.l2_compress_successes);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_zeros", mibs.l2_compress_zeros, arcstats.l2_compress_zeros);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_failures", mibs.l2_compress_failures, arcstats.l2_compress_failures);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_throttle_count", mibs.memory_throttle_count, arcstats.memory_throttle_count);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_buffers", mibs.duplicate_buffers, arcstats.duplicate_buffers);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_buffers_size", mibs.duplicate_buffers_size, arcstats.duplicate_buffers_size);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_reads", mibs.duplicate_reads, arcstats.duplicate_reads);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_direct_count", mibs.memory_direct_count, arcstats.memory_direct_count);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_indirect_count", mibs.memory_indirect_count, arcstats.memory_indirect_count);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_no_grow", mibs.arc_no_grow, arcstats.arc_no_grow);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_tempreserve", mibs.arc_tempreserve, arcstats.arc_tempreserve);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_loaned_bytes", mibs.arc_loaned_bytes, arcstats.arc_loaned_bytes);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_prune", mibs.arc_prune, arcstats.arc_prune);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_used", mibs.arc_meta_used, arcstats.arc_meta_used);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_limit", mibs.arc_meta_limit, arcstats.arc_meta_limit);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_max", mibs.arc_meta_max, arcstats.arc_meta_max);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_min", mibs.arc_meta_min, arcstats.arc_meta_min);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_need_free", mibs.arc_need_free, arcstats.arc_need_free);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_sys_free", mibs.arc_sys_free, arcstats.arc_sys_free);
+
+ if (arcstats.size > arcstats.c_min) {
+ zfs_arcstats_shrinkable_cache_size_bytes = arcstats.size - arcstats.c_min;
+ } else {
+ zfs_arcstats_shrinkable_cache_size_bytes = 0;
+ }
+
+ generate_charts_arcstats("freebsd.plugin", "zfs", show_zero_charts, update_every);
+ generate_charts_arc_summary("freebsd.plugin", "zfs", show_zero_charts, update_every);
+
+ return 0;
+}
+
+// kstat.zfs.misc.zio_trim
+
+int do_kstat_zfs_misc_zio_trim(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib_bytes[5] = {0, 0, 0, 0, 0}, mib_success[5] = {0, 0, 0, 0, 0},
+ mib_failed[5] = {0, 0, 0, 0, 0}, mib_unsupported[5] = {0, 0, 0, 0, 0};
+ uint64_t bytes, success, failed, unsupported;
+
+ if (unlikely(GETSYSCTL_SIMPLE("kstat.zfs.misc.zio_trim.bytes", mib_bytes, bytes) ||
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.zio_trim.success", mib_success, success) ||
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.zio_trim.failed", mib_failed, failed) ||
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.zio_trim.unsupported", mib_unsupported, unsupported))) {
+ error("DISABLED: zfs.trim_bytes chart");
+ error("DISABLED: zfs.trim_success chart");
+ error("DISABLED: kstat.zfs.misc.zio_trim module");
+ return 1;
+ } else {
+
+ static RRDSET *st_bytes = NULL;
+ static RRDDIM *rd_bytes = NULL;
+
+ if (unlikely(!st_bytes)) {
+ st_bytes = rrdset_create_localhost(
+ "zfs",
+ "trim_bytes",
+ NULL,
+ "trim",
+ NULL,
+ "Successfully TRIMmed bytes",
+ "bytes",
+ "freebsd.plugin",
+ "zfs",
+ 2320,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_bytes = rrddim_add(st_bytes, "TRIMmed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_bytes, rd_bytes, bytes);
+ rrdset_done(st_bytes);
+
+ static RRDSET *st_requests = NULL;
+ static RRDDIM *rd_successful = NULL, *rd_failed = NULL, *rd_unsupported = NULL;
+
+ if (unlikely(!st_requests)) {
+ st_requests = rrdset_create_localhost(
+ "zfs",
+ "trim_requests",
+ NULL,
+ "trim",
+ NULL,
+ "TRIM requests",
+ "requests",
+ "freebsd.plugin",
+ "zfs",
+ 2321,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ rd_successful = rrddim_add(st_requests, "successful", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st_requests, "failed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_unsupported = rrddim_add(st_requests, "unsupported", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_requests, rd_successful, success);
+ rrddim_set_by_pointer(st_requests, rd_failed, failed);
+ rrddim_set_by_pointer(st_requests, rd_unsupported, unsupported);
+ rrdset_done(st_requests);
+
+ }
+
+ return 0;
+}
diff --git a/collectors/freebsd.plugin/freebsd_sysctl.c b/collectors/freebsd.plugin/freebsd_sysctl.c
new file mode 100644
index 0000000..dd94a16
--- /dev/null
+++ b/collectors/freebsd.plugin/freebsd_sysctl.c
@@ -0,0 +1,3062 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_freebsd.h"
+
+#include <sys/vmmeter.h>
+#include <vm/vm_param.h>
+
+#define _KERNEL
+#include <sys/sem.h>
+#include <sys/shm.h>
+#include <sys/msg.h>
+#undef _KERNEL
+
+#include <net/netisr.h>
+
+#include <netinet/ip.h>
+#include <netinet/ip_var.h>
+#include <netinet/ip_icmp.h>
+#include <netinet/icmp_var.h>
+#include <netinet6/ip6_var.h>
+#include <netinet/icmp6.h>
+#include <netinet/tcp_var.h>
+#include <netinet/tcp_fsm.h>
+#include <netinet/udp.h>
+#include <netinet/udp_var.h>
+
+// --------------------------------------------------------------------------------------------------------------------
+// common definitions and variables
+
+int system_pagesize = PAGE_SIZE;
+int number_of_cpus = 1;
+#if __FreeBSD_version >= 1200029
+struct __vmmeter {
+ uint64_t v_swtch;
+ uint64_t v_trap;
+ uint64_t v_syscall;
+ uint64_t v_intr;
+ uint64_t v_soft;
+ uint64_t v_vm_faults;
+ uint64_t v_io_faults;
+ uint64_t v_cow_faults;
+ uint64_t v_cow_optim;
+ uint64_t v_zfod;
+ uint64_t v_ozfod;
+ uint64_t v_swapin;
+ uint64_t v_swapout;
+ uint64_t v_swappgsin;
+ uint64_t v_swappgsout;
+ uint64_t v_vnodein;
+ uint64_t v_vnodeout;
+ uint64_t v_vnodepgsin;
+ uint64_t v_vnodepgsout;
+ uint64_t v_intrans;
+ uint64_t v_reactivated;
+ uint64_t v_pdwakeups;
+ uint64_t v_pdpages;
+ uint64_t v_pdshortfalls;
+ uint64_t v_dfree;
+ uint64_t v_pfree;
+ uint64_t v_tfree;
+ uint64_t v_forks;
+ uint64_t v_vforks;
+ uint64_t v_rforks;
+ uint64_t v_kthreads;
+ uint64_t v_forkpages;
+ uint64_t v_vforkpages;
+ uint64_t v_rforkpages;
+ uint64_t v_kthreadpages;
+ u_int v_page_size;
+ u_int v_page_count;
+ u_int v_free_reserved;
+ u_int v_free_target;
+ u_int v_free_min;
+ u_int v_free_count;
+ u_int v_wire_count;
+ u_int v_active_count;
+ u_int v_inactive_target;
+ u_int v_inactive_count;
+ u_int v_laundry_count;
+ u_int v_pageout_free_min;
+ u_int v_interrupt_free_min;
+ u_int v_free_severe;
+};
+typedef struct __vmmeter vmmeter_t;
+#else
+typedef struct vmmeter vmmeter_t;
+#endif
+
+#if (__FreeBSD_version >= 1101516 && __FreeBSD_version < 1200000) || __FreeBSD_version >= 1200015
+#define NETDATA_COLLECT_LAUNDRY 1
+#endif
+
+// FreeBSD plugin initialization
+
+int freebsd_plugin_init()
+{
+ system_pagesize = getpagesize();
+ if (system_pagesize <= 0) {
+ error("FREEBSD: can't get system page size");
+ return 1;
+ }
+
+ if (unlikely(GETSYSCTL_BY_NAME("kern.smp.cpus", number_of_cpus))) {
+ error("FREEBSD: can't get number of cpus");
+ return 1;
+ }
+
+ if (unlikely(!number_of_cpus)) {
+ error("FREEBSD: wrong number of cpus");
+ return 1;
+ }
+
+ return 0;
+}
+
+// vm.loadavg
+
+// FreeBSD calculates load averages once every 5 seconds
+#define MIN_LOADAVG_UPDATE_EVERY 5
+
+int do_vm_loadavg(int update_every, usec_t dt){
+ static usec_t next_loadavg_dt = 0;
+
+ if (next_loadavg_dt <= dt) {
+ static int mib[2] = {0, 0};
+ struct loadavg sysload;
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.loadavg", mib, sysload))) {
+ error("DISABLED: system.load chart");
+ error("DISABLED: vm.loadavg module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_load1 = NULL, *rd_load2 = NULL, *rd_load3 = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "load",
+ NULL,
+ "load",
+ NULL,
+ "System Load Average",
+ "load",
+ "freebsd.plugin",
+ "vm.loadavg",
+ NETDATA_CHART_PRIO_SYSTEM_LOAD,
+ (update_every < MIN_LOADAVG_UPDATE_EVERY) ?
+ MIN_LOADAVG_UPDATE_EVERY : update_every, RRDSET_TYPE_LINE
+ );
+ rd_load1 = rrddim_add(st, "load1", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ rd_load2 = rrddim_add(st, "load5", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ rd_load3 = rrddim_add(st, "load15", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_load1, (collected_number) ((double) sysload.ldavg[0] / sysload.fscale * 1000));
+ rrddim_set_by_pointer(st, rd_load2, (collected_number) ((double) sysload.ldavg[1] / sysload.fscale * 1000));
+ rrddim_set_by_pointer(st, rd_load3, (collected_number) ((double) sysload.ldavg[2] / sysload.fscale * 1000));
+ rrdset_done(st);
+
+ next_loadavg_dt = st->update_every * USEC_PER_SEC;
+ }
+ }
+ else
+ next_loadavg_dt -= dt;
+
+ return 0;
+}
+
+// vm.vmtotal
+
+int do_vm_vmtotal(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_all_processes = -1, do_processes = -1, do_mem_real = -1;
+
+ if (unlikely(do_all_processes == -1)) {
+ do_all_processes = config_get_boolean("plugin:freebsd:vm.vmtotal", "enable total processes", 1);
+ do_processes = config_get_boolean("plugin:freebsd:vm.vmtotal", "processes running", 1);
+ do_mem_real = config_get_boolean("plugin:freebsd:vm.vmtotal", "real memory", 1);
+ }
+
+ if (likely(do_all_processes | do_processes | do_mem_real)) {
+ static int mib[2] = {0, 0};
+ struct vmtotal vmtotal_data;
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.vmtotal", mib, vmtotal_data))) {
+ do_all_processes = 0;
+ error("DISABLED: system.active_processes chart");
+ do_processes = 0;
+ error("DISABLED: system.processes chart");
+ do_mem_real = 0;
+ error("DISABLED: mem.real chart");
+ error("DISABLED: vm.vmtotal module");
+ return 1;
+ } else {
+ if (likely(do_all_processes)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "active_processes",
+ NULL,
+ "processes",
+ NULL,
+ "System Active Processes",
+ "processes",
+ "freebsd.plugin",
+ "vm.vmtotal",
+ NETDATA_CHART_PRIO_SYSTEM_ACTIVE_PROCESSES,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+ rd = rrddim_add(st, "active", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd, (vmtotal_data.t_rq + vmtotal_data.t_dw + vmtotal_data.t_pw + vmtotal_data.t_sl + vmtotal_data.t_sw));
+ rrdset_done(st);
+ }
+
+ if (likely(do_processes)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_running = NULL, *rd_blocked = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "processes",
+ NULL,
+ "processes",
+ NULL,
+ "System Processes",
+ "processes",
+ "freebsd.plugin",
+ "vm.vmtotal",
+ NETDATA_CHART_PRIO_SYSTEM_PROCESSES,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_running = rrddim_add(st, "running", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_blocked = rrddim_add(st, "blocked", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_running, vmtotal_data.t_rq);
+ rrddim_set_by_pointer(st, rd_blocked, (vmtotal_data.t_dw + vmtotal_data.t_pw));
+ rrdset_done(st);
+ }
+
+ if (likely(do_mem_real)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "mem",
+ "real",
+ NULL,
+ "system",
+ NULL,
+ "Total Real Memory In Use",
+ "MiB",
+ "freebsd.plugin",
+ "vm.vmtotal",
+ NETDATA_CHART_PRIO_MEM_SYSTEM_COMMITTED,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd = rrddim_add(st, "used", NULL, system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd, vmtotal_data.t_rm);
+ rrdset_done(st);
+ }
+ }
+ } else {
+ error("DISABLED: vm.vmtotal module");
+ return 1;
+ }
+
+ return 0;
+}
+
+// kern.cp_time
+
+int do_kern_cp_time(int update_every, usec_t dt) {
+ (void)dt;
+
+ if (unlikely(CPUSTATES != 5)) {
+ error("FREEBSD: There are %d CPU states (5 was expected)", CPUSTATES);
+ error("DISABLED: system.cpu chart");
+ error("DISABLED: kern.cp_time module");
+ return 1;
+ } else {
+ static int mib[2] = {0, 0};
+ long cp_time[CPUSTATES];
+
+ if (unlikely(GETSYSCTL_SIMPLE("kern.cp_time", mib, cp_time))) {
+ error("DISABLED: system.cpu chart");
+ error("DISABLED: kern.cp_time module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_nice = NULL, *rd_system = NULL, *rd_user = NULL, *rd_interrupt = NULL, *rd_idle = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "cpu",
+ NULL,
+ "cpu",
+ "system.cpu",
+ "Total CPU utilization",
+ "percentage",
+ "freebsd.plugin",
+ "kern.cp_time",
+ NETDATA_CHART_PRIO_SYSTEM_CPU,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ rd_nice = rrddim_add(st, "nice", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_system = rrddim_add(st, "system", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_user = rrddim_add(st, "user", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_interrupt = rrddim_add(st, "interrupt", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_idle = rrddim_add(st, "idle", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rrddim_hide(st, "idle");
+ }
+
+ rrddim_set_by_pointer(st, rd_nice, cp_time[1]);
+ rrddim_set_by_pointer(st, rd_system, cp_time[2]);
+ rrddim_set_by_pointer(st, rd_user, cp_time[0]);
+ rrddim_set_by_pointer(st, rd_interrupt, cp_time[3]);
+ rrddim_set_by_pointer(st, rd_idle, cp_time[4]);
+ rrdset_done(st);
+ }
+ }
+
+ return 0;
+}
+
+// kern.cp_times
+
+int do_kern_cp_times(int update_every, usec_t dt) {
+ (void)dt;
+
+ if (unlikely(CPUSTATES != 5)) {
+ error("FREEBSD: There are %d CPU states (5 was expected)", CPUSTATES);
+ error("DISABLED: cpu.cpuXX charts");
+ error("DISABLED: kern.cp_times module");
+ return 1;
+ } else {
+ static int mib[2] = {0, 0};
+ long cp_time[CPUSTATES];
+ static long *pcpu_cp_time = NULL;
+ static int old_number_of_cpus = 0;
+
+ if(unlikely(number_of_cpus != old_number_of_cpus))
+ pcpu_cp_time = reallocz(pcpu_cp_time, sizeof(cp_time) * number_of_cpus);
+ if (unlikely(GETSYSCTL_WSIZE("kern.cp_times", mib, pcpu_cp_time, sizeof(cp_time) * number_of_cpus))) {
+ error("DISABLED: cpu.cpuXX charts");
+ error("DISABLED: kern.cp_times module");
+ return 1;
+ } else {
+ int i;
+ static struct cpu_chart {
+ char cpuid[MAX_INT_DIGITS + 4];
+ RRDSET *st;
+ RRDDIM *rd_user;
+ RRDDIM *rd_nice;
+ RRDDIM *rd_system;
+ RRDDIM *rd_interrupt;
+ RRDDIM *rd_idle;
+ } *all_cpu_charts = NULL;
+
+ if(unlikely(number_of_cpus > old_number_of_cpus)) {
+ all_cpu_charts = reallocz(all_cpu_charts, sizeof(struct cpu_chart) * number_of_cpus);
+ memset(&all_cpu_charts[old_number_of_cpus], 0, sizeof(struct cpu_chart) * (number_of_cpus - old_number_of_cpus));
+ }
+
+ for (i = 0; i < number_of_cpus; i++) {
+ if (unlikely(!all_cpu_charts[i].st)) {
+ snprintfz(all_cpu_charts[i].cpuid, MAX_INT_DIGITS, "cpu%d", i);
+ all_cpu_charts[i].st = rrdset_create_localhost(
+ "cpu",
+ all_cpu_charts[i].cpuid,
+ NULL,
+ "utilization",
+ "cpu.cpu",
+ "Core utilization",
+ "percentage",
+ "freebsd.plugin",
+ "kern.cp_times",
+ NETDATA_CHART_PRIO_CPU_PER_CORE,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ all_cpu_charts[i].rd_nice = rrddim_add(all_cpu_charts[i].st, "nice", NULL, 1, 1,
+ RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ all_cpu_charts[i].rd_system = rrddim_add(all_cpu_charts[i].st, "system", NULL, 1, 1,
+ RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ all_cpu_charts[i].rd_user = rrddim_add(all_cpu_charts[i].st, "user", NULL, 1, 1,
+ RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ all_cpu_charts[i].rd_interrupt = rrddim_add(all_cpu_charts[i].st, "interrupt", NULL, 1, 1,
+ RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ all_cpu_charts[i].rd_idle = rrddim_add(all_cpu_charts[i].st, "idle", NULL, 1, 1,
+ RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rrddim_hide(all_cpu_charts[i].st, "idle");
+ }
+
+ rrddim_set_by_pointer(all_cpu_charts[i].st, all_cpu_charts[i].rd_nice, pcpu_cp_time[i * 5 + 1]);
+ rrddim_set_by_pointer(all_cpu_charts[i].st, all_cpu_charts[i].rd_system, pcpu_cp_time[i * 5 + 2]);
+ rrddim_set_by_pointer(all_cpu_charts[i].st, all_cpu_charts[i].rd_user, pcpu_cp_time[i * 5 + 0]);
+ rrddim_set_by_pointer(all_cpu_charts[i].st, all_cpu_charts[i].rd_interrupt, pcpu_cp_time[i * 5 + 3]);
+ rrddim_set_by_pointer(all_cpu_charts[i].st, all_cpu_charts[i].rd_idle, pcpu_cp_time[i * 5 + 4]);
+ rrdset_done(all_cpu_charts[i].st);
+ }
+ }
+
+ old_number_of_cpus = number_of_cpus;
+ }
+
+ return 0;
+}
+
+// dev.cpu.temperature
+
+int do_dev_cpu_temperature(int update_every, usec_t dt) {
+ (void)dt;
+
+ int i;
+ static int *mib = NULL;
+ static int *pcpu_temperature = NULL;
+ static int old_number_of_cpus = 0;
+ char char_mib[MAX_INT_DIGITS + 21];
+ char char_rd[MAX_INT_DIGITS + 9];
+
+ if (unlikely(number_of_cpus != old_number_of_cpus)) {
+ pcpu_temperature = reallocz(pcpu_temperature, sizeof(int) * number_of_cpus);
+ mib = reallocz(mib, sizeof(int) * number_of_cpus * 4);
+ if (unlikely(number_of_cpus > old_number_of_cpus))
+ memset(&mib[old_number_of_cpus * 4], 0, sizeof(int) * (number_of_cpus - old_number_of_cpus) * 4);
+ }
+ for (i = 0; i < number_of_cpus; i++) {
+ if (unlikely(!(mib[i * 4])))
+ sprintf(char_mib, "dev.cpu.%d.temperature", i);
+ if (unlikely(getsysctl_simple(char_mib, &mib[i * 4], 4, &pcpu_temperature[i], sizeof(int)))) {
+ error("DISABLED: cpu.temperature chart");
+ error("DISABLED: dev.cpu.temperature module");
+ return 1;
+ }
+ }
+
+ static RRDSET *st;
+ static RRDDIM **rd_pcpu_temperature;
+
+ if (unlikely(number_of_cpus != old_number_of_cpus)) {
+ rd_pcpu_temperature = reallocz(rd_pcpu_temperature, sizeof(RRDDIM) * number_of_cpus);
+ if (unlikely(number_of_cpus > old_number_of_cpus))
+ memset(&rd_pcpu_temperature[old_number_of_cpus], 0, sizeof(RRDDIM) * (number_of_cpus - old_number_of_cpus));
+ }
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "cpu",
+ "temperature",
+ NULL,
+ "temperature",
+ "cpu.temperature",
+ "Core temperature",
+ "Celsius",
+ "freebsd.plugin",
+ "dev.cpu.temperature",
+ NETDATA_CHART_PRIO_CPU_TEMPERATURE,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+ }
+
+ for (i = 0; i < number_of_cpus; i++) {
+ if (unlikely(!rd_pcpu_temperature[i])) {
+ sprintf(char_rd, "cpu%d.temp", i);
+ rd_pcpu_temperature[i] = rrddim_add(st, char_rd, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_pcpu_temperature[i], (collected_number) ((double)pcpu_temperature[i] / 10 - 273.15));
+ }
+
+ rrdset_done(st);
+
+ old_number_of_cpus = number_of_cpus;
+
+ return 0;
+}
+
+// dev.cpu.0.freq
+
+int do_dev_cpu_0_freq(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib[4] = {0, 0, 0, 0};
+ int cpufreq;
+
+ if (unlikely(GETSYSCTL_SIMPLE("dev.cpu.0.freq", mib, cpufreq))) {
+ error("DISABLED: cpu.scaling_cur_freq chart");
+ error("DISABLED: dev.cpu.0.freq module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "cpu",
+ "scaling_cur_freq",
+ NULL,
+ "cpufreq",
+ NULL,
+ "Current CPU Scaling Frequency",
+ "MHz",
+ "freebsd.plugin",
+ "dev.cpu.0.freq",
+ NETDATA_CHART_PRIO_CPUFREQ_SCALING_CUR_FREQ,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd = rrddim_add(st, "frequency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd, cpufreq);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// hw.intrcnt
+
+int do_hw_intcnt(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib_hw_intrcnt[2] = {0, 0};
+ size_t intrcnt_size = 0;
+
+ if (unlikely(GETSYSCTL_SIZE("hw.intrcnt", mib_hw_intrcnt, intrcnt_size))) {
+ error("DISABLED: system.intr chart");
+ error("DISABLED: system.interrupts chart");
+ error("DISABLED: hw.intrcnt module");
+ return 1;
+ } else {
+ unsigned long nintr = 0;
+ static unsigned long old_nintr = 0;
+ static unsigned long *intrcnt = NULL;
+
+ nintr = intrcnt_size / sizeof(u_long);
+ if (unlikely(nintr != old_nintr))
+ intrcnt = reallocz(intrcnt, nintr * sizeof(u_long));
+ if (unlikely(GETSYSCTL_WSIZE("hw.intrcnt", mib_hw_intrcnt, intrcnt, nintr * sizeof(u_long)))) {
+ error("DISABLED: system.intr chart");
+ error("DISABLED: system.interrupts chart");
+ error("DISABLED: hw.intrcnt module");
+ return 1;
+ } else {
+ unsigned long long totalintr = 0;
+ unsigned long i;
+
+ for (i = 0; i < nintr; i++)
+ totalintr += intrcnt[i];
+
+ static RRDSET *st_intr = NULL;
+ static RRDDIM *rd_intr = NULL;
+
+ if (unlikely(!st_intr)) {
+ st_intr = rrdset_create_localhost(
+ "system",
+ "intr",
+ NULL,
+ "interrupts",
+ NULL,
+ "Total Hardware Interrupts",
+ "interrupts/s",
+ "freebsd.plugin",
+ "hw.intrcnt",
+ NETDATA_CHART_PRIO_SYSTEM_INTR,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st_intr, RRDSET_FLAG_DETAIL);
+
+ rd_intr = rrddim_add(st_intr, "interrupts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_intr, rd_intr, totalintr);
+ rrdset_done(st_intr);
+
+ size_t size;
+ static int mib_hw_intrnames[2] = {0, 0};
+ static char *intrnames = NULL;
+
+ if (unlikely(GETSYSCTL_SIZE("hw.intrnames", mib_hw_intrnames, size))) {
+ error("DISABLED: system.intr chart");
+ error("DISABLED: system.interrupts chart");
+ error("DISABLED: hw.intrcnt module");
+ return 1;
+ } else {
+ if (unlikely(nintr != old_nintr))
+ intrnames = reallocz(intrnames, size);
+ if (unlikely(GETSYSCTL_WSIZE("hw.intrnames", mib_hw_intrnames, intrnames, size))) {
+ error("DISABLED: system.intr chart");
+ error("DISABLED: system.interrupts chart");
+ error("DISABLED: hw.intrcnt module");
+ return 1;
+ } else {
+ static RRDSET *st_interrupts = NULL;
+
+ if (unlikely(!st_interrupts)) {
+ st_interrupts = rrdset_create_localhost(
+ "system",
+ "interrupts",
+ NULL,
+ "interrupts",
+ NULL,
+ "System interrupts",
+ "interrupts/s",
+ "freebsd.plugin",
+ "hw.intrcnt",
+ NETDATA_CHART_PRIO_SYSTEM_INTERRUPTS,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ }
+
+ for (i = 0; i < nintr; i++) {
+ void *p;
+
+ p = intrnames + i * (strlen(intrnames) + 1);
+ if (unlikely((intrcnt[i] != 0) && (*(char *) p != 0))) {
+ RRDDIM *rd_interrupts = rrddim_find_active(st_interrupts, p);
+
+ if (unlikely(!rd_interrupts))
+ rd_interrupts = rrddim_add(st_interrupts, p, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st_interrupts, rd_interrupts, intrcnt[i]);
+ }
+ }
+ rrdset_done(st_interrupts);
+ }
+ }
+ }
+
+ old_nintr = nintr;
+ }
+
+ return 0;
+}
+
+// vm.stats.sys.v_intr
+
+int do_vm_stats_sys_v_intr(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib[4] = {0, 0, 0, 0};
+ u_int int_number;
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.stats.sys.v_intr", mib, int_number))) {
+ error("DISABLED: system.dev_intr chart");
+ error("DISABLED: vm.stats.sys.v_intr module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "dev_intr",
+ NULL,
+ "interrupts",
+ NULL,
+ "Device Interrupts",
+ "interrupts/s",
+ "freebsd.plugin",
+ "vm.stats.sys.v_intr",
+ NETDATA_CHART_PRIO_SYSTEM_DEV_INTR,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd = rrddim_add(st, "interrupts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd, int_number);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// vm.stats.sys.v_soft
+
+int do_vm_stats_sys_v_soft(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib[4] = {0, 0, 0, 0};
+ u_int soft_intr_number;
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.stats.sys.v_soft", mib, soft_intr_number))) {
+ error("DISABLED: system.dev_intr chart");
+ error("DISABLED: vm.stats.sys.v_soft module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "soft_intr",
+ NULL,
+ "interrupts",
+ NULL,
+ "Software Interrupts",
+ "interrupts/s",
+ "freebsd.plugin",
+ "vm.stats.sys.v_soft",
+ NETDATA_CHART_PRIO_SYSTEM_SOFT_INTR,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd = rrddim_add(st, "interrupts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd, soft_intr_number);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// vm.stats.sys.v_swtch
+
+int do_vm_stats_sys_v_swtch(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib[4] = {0, 0, 0, 0};
+ u_int ctxt_number;
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.stats.sys.v_swtch", mib, ctxt_number))) {
+ error("DISABLED: system.ctxt chart");
+ error("DISABLED: vm.stats.sys.v_swtch module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "ctxt",
+ NULL,
+ "processes",
+ NULL,
+ "CPU Context Switches",
+ "context switches/s",
+ "freebsd.plugin",
+ "vm.stats.sys.v_swtch",
+ NETDATA_CHART_PRIO_SYSTEM_CTXT,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd = rrddim_add(st, "switches", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd, ctxt_number);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// vm.stats.vm.v_forks
+
+int do_vm_stats_sys_v_forks(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib[4] = {0, 0, 0, 0};
+ u_int forks_number;
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.stats.vm.v_forks", mib, forks_number))) {
+ error("DISABLED: system.forks chart");
+ error("DISABLED: vm.stats.sys.v_swtch module");
+ return 1;
+ } else {
+
+ // --------------------------------------------------------------------
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "forks",
+ NULL,
+ "processes",
+ NULL,
+ "Started Processes",
+ "processes/s",
+ "freebsd.plugin",
+ "vm.stats.sys.v_swtch",
+ NETDATA_CHART_PRIO_SYSTEM_FORKS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd = rrddim_add(st, "started", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd, forks_number);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// vm.swap_info
+
+int do_vm_swap_info(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib[3] = {0, 0, 0};
+
+ if (unlikely(getsysctl_mib("vm.swap_info", mib, 2))) {
+ error("DISABLED: system.swap chart");
+ error("DISABLED: vm.swap_info module");
+ return 1;
+ } else {
+ int i;
+ struct xswdev xsw;
+ struct total_xsw {
+ collected_number bytes_used;
+ collected_number bytes_total;
+ } total_xsw = {0, 0};
+
+ for (i = 0; ; i++) {
+ size_t size;
+
+ mib[2] = i;
+ size = sizeof(xsw);
+ if (unlikely(sysctl(mib, 3, &xsw, &size, NULL, 0) == -1 )) {
+ if (unlikely(errno != ENOENT)) {
+ error("FREEBSD: sysctl(%s...) failed: %s", "vm.swap_info", strerror(errno));
+ error("DISABLED: system.swap chart");
+ error("DISABLED: vm.swap_info module");
+ return 1;
+ } else {
+ if (unlikely(size != sizeof(xsw))) {
+ error("FREEBSD: sysctl(%s...) expected %lu, got %lu", "vm.swap_info", (unsigned long)sizeof(xsw), (unsigned long)size);
+ error("DISABLED: system.swap chart");
+ error("DISABLED: vm.swap_info module");
+ return 1;
+ } else break;
+ }
+ }
+ total_xsw.bytes_used += xsw.xsw_used;
+ total_xsw.bytes_total += xsw.xsw_nblks;
+ }
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_free = NULL, *rd_used = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "swap",
+ NULL,
+ "swap",
+ NULL,
+ "System Swap",
+ "MiB",
+ "freebsd.plugin",
+ "vm.swap_info",
+ NETDATA_CHART_PRIO_SYSTEM_SWAP,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_free = rrddim_add(st, "free", NULL, system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ rd_used = rrddim_add(st, "used", NULL, system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_free, total_xsw.bytes_total - total_xsw.bytes_used);
+ rrddim_set_by_pointer(st, rd_used, total_xsw.bytes_used);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// system.ram
+
+int do_system_ram(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib_active_count[4] = {0, 0, 0, 0},
+ mib_inactive_count[4] = {0, 0, 0, 0},
+ mib_wire_count[4] = {0, 0, 0, 0},
+#if __FreeBSD_version < 1200016
+ mib_cache_count[4] = {0, 0, 0, 0},
+#endif
+ mib_vfs_bufspace[2] = {0, 0},
+ mib_free_count[4] = {0, 0, 0, 0};
+ vmmeter_t vmmeter_data;
+ size_t vfs_bufspace_count;
+
+#if defined(NETDATA_COLLECT_LAUNDRY)
+ static int mib_laundry_count[4] = {0, 0, 0, 0};
+#endif
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.stats.vm.v_active_count", mib_active_count, vmmeter_data.v_active_count) ||
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_inactive_count", mib_inactive_count, vmmeter_data.v_inactive_count) ||
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_wire_count", mib_wire_count, vmmeter_data.v_wire_count) ||
+#if __FreeBSD_version < 1200016
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_cache_count", mib_cache_count, vmmeter_data.v_cache_count) ||
+#endif
+#if defined(NETDATA_COLLECT_LAUNDRY)
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_laundry_count", mib_laundry_count, vmmeter_data.v_laundry_count) ||
+#endif
+ GETSYSCTL_SIMPLE("vfs.bufspace", mib_vfs_bufspace, vfs_bufspace_count) ||
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_free_count", mib_free_count, vmmeter_data.v_free_count))) {
+ error("DISABLED: system.ram chart");
+ error("DISABLED: system.ram module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL, *st_mem_available = NULL;
+ static RRDDIM *rd_free = NULL, *rd_active = NULL, *rd_inactive = NULL, *rd_wired = NULL,
+ *rd_cache = NULL, *rd_buffers = NULL, *rd_avail = NULL;
+
+#if defined(NETDATA_COLLECT_LAUNDRY)
+ static RRDDIM *rd_laundry = NULL;
+#endif
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "ram",
+ NULL,
+ "ram",
+ NULL,
+ "System RAM",
+ "MiB",
+ "freebsd.plugin",
+ "system.ram",
+ NETDATA_CHART_PRIO_SYSTEM_RAM,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ rd_free = rrddim_add(st, "free", NULL, system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ rd_active = rrddim_add(st, "active", NULL, system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ rd_inactive = rrddim_add(st, "inactive", NULL, system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ rd_wired = rrddim_add(st, "wired", NULL, 1, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ rd_cache = rrddim_add(st, "cache", NULL, 1, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+#if defined(NETDATA_COLLECT_LAUNDRY)
+ rd_laundry = rrddim_add(st, "laundry", NULL, system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+#endif
+ rd_buffers = rrddim_add(st, "buffers", NULL, 1, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_free, vmmeter_data.v_free_count);
+ rrddim_set_by_pointer(st, rd_active, vmmeter_data.v_active_count);
+ rrddim_set_by_pointer(st, rd_inactive, vmmeter_data.v_inactive_count);
+ rrddim_set_by_pointer(st, rd_wired, (unsigned long long)vmmeter_data.v_wire_count * (unsigned long long)system_pagesize - zfs_arcstats_shrinkable_cache_size_bytes);
+#if __FreeBSD_version < 1200016
+ rrddim_set_by_pointer(st, rd_cache, (unsigned long long)vmmeter_data.v_cache_count * (unsigned long long)system_pagesize + zfs_arcstats_shrinkable_cache_size_bytes);
+#else
+ rrddim_set_by_pointer(st, rd_cache, zfs_arcstats_shrinkable_cache_size_bytes);
+#endif
+#if defined(NETDATA_COLLECT_LAUNDRY)
+ rrddim_set_by_pointer(st, rd_laundry, vmmeter_data.v_laundry_count);
+#endif
+ rrddim_set_by_pointer(st, rd_buffers, vfs_bufspace_count);
+ rrdset_done(st);
+
+ if (unlikely(!st_mem_available)) {
+ st_mem_available = rrdset_create_localhost(
+ "mem",
+ "available",
+ NULL,
+ "system",
+ NULL,
+ "Available RAM for applications",
+ "MiB",
+ "freebsd.plugin",
+ "system.ram",
+ NETDATA_CHART_PRIO_MEM_SYSTEM_AVAILABLE,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_avail = rrddim_add(st_mem_available, "MemAvailable", "avail", system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+#if __FreeBSD_version < 1200016
+ rrddim_set_by_pointer(st_mem_available, rd_avail, vmmeter_data.v_inactive_count + vmmeter_data.v_free_count + vmmeter_data.v_cache_count + zfs_arcstats_shrinkable_cache_size_bytes / system_pagesize);
+#else
+ rrddim_set_by_pointer(st_mem_available, rd_avail, vmmeter_data.v_inactive_count + vmmeter_data.v_free_count + zfs_arcstats_shrinkable_cache_size_bytes / system_pagesize);
+#endif
+
+ rrdset_done(st_mem_available);
+ }
+
+ return 0;
+}
+
+// vm.stats.vm.v_swappgs
+
+int do_vm_stats_sys_v_swappgs(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib_swappgsin[4] = {0, 0, 0, 0}, mib_swappgsout[4] = {0, 0, 0, 0};
+ vmmeter_t vmmeter_data;
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.stats.vm.v_swappgsin", mib_swappgsin, vmmeter_data.v_swappgsin) ||
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_swappgsout", mib_swappgsout, vmmeter_data.v_swappgsout))) {
+ error("DISABLED: system.swapio chart");
+ error("DISABLED: vm.stats.vm.v_swappgs module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "swapio",
+ NULL,
+ "swap",
+ NULL,
+ "Swap I/O",
+ "KiB/s",
+ "freebsd.plugin",
+ "vm.stats.vm.v_swappgs",
+ NETDATA_CHART_PRIO_SYSTEM_SWAPIO,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st, "in", NULL, system_pagesize, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "out", NULL, -system_pagesize, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, vmmeter_data.v_swappgsin);
+ rrddim_set_by_pointer(st, rd_out, vmmeter_data.v_swappgsout);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// vm.stats.vm.v_pgfaults
+
+int do_vm_stats_sys_v_pgfaults(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib_vm_faults[4] = {0, 0, 0, 0}, mib_io_faults[4] = {0, 0, 0, 0}, mib_cow_faults[4] = {0, 0, 0, 0},
+ mib_cow_optim[4] = {0, 0, 0, 0}, mib_intrans[4] = {0, 0, 0, 0};
+ vmmeter_t vmmeter_data;
+
+ if (unlikely(GETSYSCTL_SIMPLE("vm.stats.vm.v_vm_faults", mib_vm_faults, vmmeter_data.v_vm_faults) ||
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_io_faults", mib_io_faults, vmmeter_data.v_io_faults) ||
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_cow_faults", mib_cow_faults, vmmeter_data.v_cow_faults) ||
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_cow_optim", mib_cow_optim, vmmeter_data.v_cow_optim) ||
+ GETSYSCTL_SIMPLE("vm.stats.vm.v_intrans", mib_intrans, vmmeter_data.v_intrans))) {
+ error("DISABLED: mem.pgfaults chart");
+ error("DISABLED: vm.stats.vm.v_pgfaults module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_memory = NULL, *rd_io_requiring = NULL, *rd_cow = NULL,
+ *rd_cow_optimized = NULL, *rd_in_transit = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "mem",
+ "pgfaults",
+ NULL,
+ "system",
+ NULL,
+ "Memory Page Faults",
+ "page faults/s",
+ "freebsd.plugin",
+ "vm.stats.vm.v_pgfaults",
+ NETDATA_CHART_PRIO_MEM_SYSTEM_PGFAULTS,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_memory = rrddim_add(st, "memory", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_io_requiring = rrddim_add(st, "io_requiring", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_cow = rrddim_add(st, "cow", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_cow_optimized = rrddim_add(st, "cow_optimized", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_transit = rrddim_add(st, "in_transit", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_memory, vmmeter_data.v_vm_faults);
+ rrddim_set_by_pointer(st, rd_io_requiring, vmmeter_data.v_io_faults);
+ rrddim_set_by_pointer(st, rd_cow, vmmeter_data.v_cow_faults);
+ rrddim_set_by_pointer(st, rd_cow_optimized, vmmeter_data.v_cow_optim);
+ rrddim_set_by_pointer(st, rd_in_transit, vmmeter_data.v_intrans);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// kern.ipc.sem
+
+int do_kern_ipc_sem(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib_semmni[3] = {0, 0, 0};
+ struct ipc_sem {
+ int semmni;
+ collected_number sets;
+ collected_number semaphores;
+ } ipc_sem = {0, 0, 0};
+
+ if (unlikely(GETSYSCTL_SIMPLE("kern.ipc.semmni", mib_semmni, ipc_sem.semmni))) {
+ error("DISABLED: system.ipc_semaphores chart");
+ error("DISABLED: system.ipc_semaphore_arrays chart");
+ error("DISABLED: kern.ipc.sem module");
+ return 1;
+ } else {
+ static struct semid_kernel *ipc_sem_data = NULL;
+ static int old_semmni = 0;
+ static int mib_sema[3] = {0, 0, 0};
+
+ if (unlikely(ipc_sem.semmni != old_semmni)) {
+ ipc_sem_data = reallocz(ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni);
+ old_semmni = ipc_sem.semmni;
+ }
+ if (unlikely(GETSYSCTL_WSIZE("kern.ipc.sema", mib_sema, ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni))) {
+ error("DISABLED: system.ipc_semaphores chart");
+ error("DISABLED: system.ipc_semaphore_arrays chart");
+ error("DISABLED: kern.ipc.sem module");
+ return 1;
+ } else {
+ int i;
+
+ for (i = 0; i < ipc_sem.semmni; i++) {
+ if (unlikely(ipc_sem_data[i].u.sem_perm.mode & SEM_ALLOC)) {
+ ipc_sem.sets += 1;
+ ipc_sem.semaphores += ipc_sem_data[i].u.sem_nsems;
+ }
+ }
+
+ static RRDSET *st_semaphores = NULL, *st_semaphore_arrays = NULL;
+ static RRDDIM *rd_semaphores = NULL, *rd_semaphore_arrays = NULL;
+
+ if (unlikely(!st_semaphores)) {
+ st_semaphores = rrdset_create_localhost(
+ "system",
+ "ipc_semaphores",
+ NULL,
+ "ipc semaphores",
+ NULL,
+ "IPC Semaphores",
+ "semaphores",
+ "freebsd.plugin",
+ "kern.ipc.sem",
+ NETDATA_CHART_PRIO_SYSTEM_IPC_SEMAPHORES,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_semaphores = rrddim_add(st_semaphores, "semaphores", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_semaphores, rd_semaphores, ipc_sem.semaphores);
+ rrdset_done(st_semaphores);
+
+ if (unlikely(!st_semaphore_arrays)) {
+ st_semaphore_arrays = rrdset_create_localhost(
+ "system",
+ "ipc_semaphore_arrays",
+ NULL,
+ "ipc semaphores",
+ NULL,
+ "IPC Semaphore Arrays",
+ "arrays",
+ "freebsd.plugin",
+ "kern.ipc.sem",
+ NETDATA_CHART_PRIO_SYSTEM_IPC_SEM_ARRAYS,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_semaphore_arrays = rrddim_add(st_semaphore_arrays, "arrays", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_semaphore_arrays, rd_semaphore_arrays, ipc_sem.sets);
+ rrdset_done(st_semaphore_arrays);
+ }
+ }
+
+ return 0;
+}
+
+// kern.ipc.shm
+
+int do_kern_ipc_shm(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib_shmmni[3] = {0, 0, 0};
+ struct ipc_shm {
+ u_long shmmni;
+ collected_number segs;
+ collected_number segsize;
+ } ipc_shm = {0, 0, 0};
+
+ if (unlikely(GETSYSCTL_SIMPLE("kern.ipc.shmmni", mib_shmmni, ipc_shm.shmmni))) {
+ error("DISABLED: system.ipc_shared_mem_segs chart");
+ error("DISABLED: system.ipc_shared_mem_size chart");
+ error("DISABLED: kern.ipc.shmmodule");
+ return 1;
+ } else {
+ static struct shmid_kernel *ipc_shm_data = NULL;
+ static u_long old_shmmni = 0;
+ static int mib_shmsegs[3] = {0, 0, 0};
+
+ if (unlikely(ipc_shm.shmmni != old_shmmni)) {
+ ipc_shm_data = reallocz(ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni);
+ old_shmmni = ipc_shm.shmmni;
+ }
+ if (unlikely(
+ GETSYSCTL_WSIZE("kern.ipc.shmsegs", mib_shmsegs, ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni))) {
+ error("DISABLED: system.ipc_shared_mem_segs chart");
+ error("DISABLED: system.ipc_shared_mem_size chart");
+ error("DISABLED: kern.ipc.shmmodule");
+ return 1;
+ } else {
+ unsigned long i;
+
+ for (i = 0; i < ipc_shm.shmmni; i++) {
+ if (unlikely(ipc_shm_data[i].u.shm_perm.mode & 0x0800)) {
+ ipc_shm.segs += 1;
+ ipc_shm.segsize += ipc_shm_data[i].u.shm_segsz;
+ }
+ }
+
+ static RRDSET *st_segs = NULL, *st_size = NULL;
+ static RRDDIM *rd_segments = NULL, *rd_allocated = NULL;
+
+ if (unlikely(!st_segs)) {
+ st_segs = rrdset_create_localhost(
+ "system",
+ "ipc_shared_mem_segs",
+ NULL,
+ "ipc shared memory",
+ NULL,
+ "IPC Shared Memory Segments",
+ "segments",
+ "freebsd.plugin",
+ "kern.ipc.shm",
+ NETDATA_CHART_PRIO_SYSTEM_IPC_SHARED_MEM_SEGS,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_segments = rrddim_add(st_segs, "segments", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_segs, rd_segments, ipc_shm.segs);
+ rrdset_done(st_segs);
+
+ if (unlikely(!st_size)) {
+ st_size = rrdset_create_localhost(
+ "system",
+ "ipc_shared_mem_size",
+ NULL,
+ "ipc shared memory",
+ NULL,
+ "IPC Shared Memory Segments Size",
+ "KiB",
+ "freebsd.plugin",
+ "kern.ipc.shm",
+ NETDATA_CHART_PRIO_SYSTEM_IPC_SHARED_MEM_SIZE,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_allocated = rrddim_add(st_size, "allocated", NULL, 1, KILO_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_size, rd_allocated, ipc_shm.segsize);
+ rrdset_done(st_size);
+ }
+ }
+
+ return 0;
+}
+
+// kern.ipc.msq
+
+int do_kern_ipc_msq(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib_msgmni[3] = {0, 0, 0};
+ struct ipc_msq {
+ int msgmni;
+ collected_number queues;
+ collected_number messages;
+ collected_number usedsize;
+ collected_number allocsize;
+ } ipc_msq = {0, 0, 0, 0, 0};
+
+ if (unlikely(GETSYSCTL_SIMPLE("kern.ipc.msgmni", mib_msgmni, ipc_msq.msgmni))) {
+ error("DISABLED: system.ipc_msq_queues chart");
+ error("DISABLED: system.ipc_msq_messages chart");
+ error("DISABLED: system.ipc_msq_size chart");
+ error("DISABLED: kern.ipc.msg module");
+ return 1;
+ } else {
+ static struct msqid_kernel *ipc_msq_data = NULL;
+ static int old_msgmni = 0;
+ static int mib_msqids[3] = {0, 0, 0};
+
+ if (unlikely(ipc_msq.msgmni != old_msgmni)) {
+ ipc_msq_data = reallocz(ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni);
+ old_msgmni = ipc_msq.msgmni;
+ }
+ if (unlikely(
+ GETSYSCTL_WSIZE("kern.ipc.msqids", mib_msqids, ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni))) {
+ error("DISABLED: system.ipc_msq_queues chart");
+ error("DISABLED: system.ipc_msq_messages chart");
+ error("DISABLED: system.ipc_msq_size chart");
+ error("DISABLED: kern.ipc.msg module");
+ return 1;
+ } else {
+ int i;
+
+ for (i = 0; i < ipc_msq.msgmni; i++) {
+ if (unlikely(ipc_msq_data[i].u.msg_qbytes != 0)) {
+ ipc_msq.queues += 1;
+ ipc_msq.messages += ipc_msq_data[i].u.msg_qnum;
+ ipc_msq.usedsize += ipc_msq_data[i].u.msg_cbytes;
+ ipc_msq.allocsize += ipc_msq_data[i].u.msg_qbytes;
+ }
+ }
+
+ static RRDSET *st_queues = NULL, *st_messages = NULL, *st_size = NULL;
+ static RRDDIM *rd_queues = NULL, *rd_messages = NULL, *rd_allocated = NULL, *rd_used = NULL;
+
+ if (unlikely(!st_queues)) {
+ st_queues = rrdset_create_localhost(
+ "system",
+ "ipc_msq_queues",
+ NULL,
+ "ipc message queues",
+ NULL,
+ "Number of IPC Message Queues",
+ "queues",
+ "freebsd.plugin",
+ "kern.ipc.msq",
+ NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_QUEUES,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_queues = rrddim_add(st_queues, "queues", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_queues, rd_queues, ipc_msq.queues);
+ rrdset_done(st_queues);
+
+ if (unlikely(!st_messages)) {
+ st_messages = rrdset_create_localhost(
+ "system",
+ "ipc_msq_messages",
+ NULL,
+ "ipc message queues",
+ NULL,
+ "Number of Messages in IPC Message Queues",
+ "messages",
+ "freebsd.plugin",
+ "kern.ipc.msq",
+ NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_MESSAGES,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_messages = rrddim_add(st_messages, "messages", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_messages, rd_messages, ipc_msq.messages);
+ rrdset_done(st_messages);
+
+ if (unlikely(!st_size)) {
+ st_size = rrdset_create_localhost(
+ "system",
+ "ipc_msq_size",
+ NULL,
+ "ipc message queues",
+ NULL,
+ "Size of IPC Message Queues",
+ "bytes",
+ "freebsd.plugin",
+ "kern.ipc.msq",
+ NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_SIZE,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_allocated = rrddim_add(st_size, "allocated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_used = rrddim_add(st_size, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_size, rd_allocated, ipc_msq.allocsize);
+ rrddim_set_by_pointer(st_size, rd_used, ipc_msq.usedsize);
+ rrdset_done(st_size);
+ }
+ }
+
+ return 0;
+}
+
+// uptime
+
+int do_uptime(int update_every, usec_t dt) {
+ (void)dt;
+ struct timespec up_time;
+
+ clock_gettime(CLOCK_UPTIME, &up_time);
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "uptime",
+ NULL,
+ "uptime",
+ NULL,
+ "System Uptime",
+ "seconds",
+ "freebsd.plugin",
+ "uptime",
+ NETDATA_CHART_PRIO_SYSTEM_UPTIME,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd = rrddim_add(st, "uptime", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd, up_time.tv_sec);
+ rrdset_done(st);
+ return 0;
+}
+
+// net.isr
+
+int do_net_isr(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_netisr = -1, do_netisr_per_core = -1;
+
+ if (unlikely(do_netisr == -1)) {
+ do_netisr = config_get_boolean("plugin:freebsd:net.isr", "netisr", 1);
+ do_netisr_per_core = config_get_boolean("plugin:freebsd:net.isr", "netisr per core", 1);
+ }
+
+ static struct netisr_stats {
+ collected_number dispatched;
+ collected_number hybrid_dispatched;
+ collected_number qdrops;
+ collected_number queued;
+ } *netisr_stats = NULL;
+
+ if (likely(do_netisr || do_netisr_per_core)) {
+ static int mib_workstream[3] = {0, 0, 0}, mib_work[3] = {0, 0, 0};
+ size_t netisr_workstream_size = 0, netisr_work_size = 0;
+ static struct sysctl_netisr_workstream *netisr_workstream = NULL;
+ static struct sysctl_netisr_work *netisr_work = NULL;
+ unsigned long num_netisr_workstreams = 0, num_netisr_works = 0;
+ int common_error = 0;
+
+ if (unlikely(GETSYSCTL_SIZE("net.isr.workstream", mib_workstream, netisr_workstream_size))) {
+ common_error = 1;
+ } else if (unlikely(GETSYSCTL_SIZE("net.isr.work", mib_work, netisr_work_size))) {
+ common_error = 1;
+ } else {
+ static size_t old_netisr_workstream_size = 0;
+
+ num_netisr_workstreams = netisr_workstream_size / sizeof(struct sysctl_netisr_workstream);
+ if (unlikely(netisr_workstream_size != old_netisr_workstream_size)) {
+ netisr_workstream = reallocz(netisr_workstream,
+ num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream));
+ old_netisr_workstream_size = netisr_workstream_size;
+ }
+ if (unlikely(GETSYSCTL_WSIZE("net.isr.workstream", mib_workstream, netisr_workstream,
+ num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream)))){
+ common_error = 1;
+ } else {
+ static size_t old_netisr_work_size = 0;
+
+ num_netisr_works = netisr_work_size / sizeof(struct sysctl_netisr_work);
+ if (unlikely(netisr_work_size != old_netisr_work_size)) {
+ netisr_work = reallocz(netisr_work, num_netisr_works * sizeof(struct sysctl_netisr_work));
+ old_netisr_work_size = netisr_work_size;
+ }
+ if (unlikely(GETSYSCTL_WSIZE("net.isr.work", mib_work, netisr_work,
+ num_netisr_works * sizeof(struct sysctl_netisr_work)))){
+ common_error = 1;
+ }
+ }
+ }
+ if (unlikely(common_error)) {
+ do_netisr = 0;
+ error("DISABLED: system.softnet_stat chart");
+ do_netisr_per_core = 0;
+ error("DISABLED: system.cpuX_softnet_stat chart");
+ common_error = 0;
+ error("DISABLED: net.isr module");
+ return 1;
+ } else {
+ unsigned long i, n;
+ int j;
+ static int old_number_of_cpus = 0;
+
+ if (unlikely(number_of_cpus != old_number_of_cpus)) {
+ netisr_stats = reallocz(netisr_stats, (number_of_cpus + 1) * sizeof(struct netisr_stats));
+ old_number_of_cpus = number_of_cpus;
+ }
+ memset(netisr_stats, 0, (number_of_cpus + 1) * sizeof(struct netisr_stats));
+ for (i = 0; i < num_netisr_workstreams; i++) {
+ for (n = 0; n < num_netisr_works; n++) {
+ if (netisr_workstream[i].snws_wsid == netisr_work[n].snw_wsid) {
+ netisr_stats[netisr_workstream[i].snws_cpu].dispatched += netisr_work[n].snw_dispatched;
+ netisr_stats[netisr_workstream[i].snws_cpu].hybrid_dispatched += netisr_work[n].snw_hybrid_dispatched;
+ netisr_stats[netisr_workstream[i].snws_cpu].qdrops += netisr_work[n].snw_qdrops;
+ netisr_stats[netisr_workstream[i].snws_cpu].queued += netisr_work[n].snw_queued;
+ }
+ }
+ }
+ for (j = 0; j < number_of_cpus; j++) {
+ netisr_stats[number_of_cpus].dispatched += netisr_stats[j].dispatched;
+ netisr_stats[number_of_cpus].hybrid_dispatched += netisr_stats[j].hybrid_dispatched;
+ netisr_stats[number_of_cpus].qdrops += netisr_stats[j].qdrops;
+ netisr_stats[number_of_cpus].queued += netisr_stats[j].queued;
+ }
+ }
+ } else {
+ error("DISABLED: net.isr module");
+ return 1;
+ }
+
+ if (likely(do_netisr)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_dispatched = NULL, *rd_hybrid_dispatched = NULL, *rd_qdrops = NULL, *rd_queued = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system",
+ "softnet_stat",
+ NULL,
+ "softnet_stat",
+ NULL,
+ "System softnet_stat",
+ "events/s",
+ "freebsd.plugin",
+ "net.isr",
+ NETDATA_CHART_PRIO_SYSTEM_SOFTNET_STAT,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_dispatched = rrddim_add(st, "dispatched", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_hybrid_dispatched = rrddim_add(st, "hybrid_dispatched", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_qdrops = rrddim_add(st, "qdrops", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_queued = rrddim_add(st, "queued", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_dispatched, netisr_stats[number_of_cpus].dispatched);
+ rrddim_set_by_pointer(st, rd_hybrid_dispatched, netisr_stats[number_of_cpus].hybrid_dispatched);
+ rrddim_set_by_pointer(st, rd_qdrops, netisr_stats[number_of_cpus].qdrops);
+ rrddim_set_by_pointer(st, rd_queued, netisr_stats[number_of_cpus].queued);
+ rrdset_done(st);
+ }
+
+ if (likely(do_netisr_per_core)) {
+ static struct softnet_chart {
+ char netisr_cpuid[MAX_INT_DIGITS + 17];
+ RRDSET *st;
+ RRDDIM *rd_dispatched;
+ RRDDIM *rd_hybrid_dispatched;
+ RRDDIM *rd_qdrops;
+ RRDDIM *rd_queued;
+ } *all_softnet_charts = NULL;
+ static int old_number_of_cpus = 0;
+ int i;
+
+ if(unlikely(number_of_cpus > old_number_of_cpus)) {
+ all_softnet_charts = reallocz(all_softnet_charts, sizeof(struct softnet_chart) * number_of_cpus);
+ memset(&all_softnet_charts[old_number_of_cpus], 0, sizeof(struct softnet_chart) * (number_of_cpus - old_number_of_cpus));
+ old_number_of_cpus = number_of_cpus;
+ }
+
+ for (i = 0; i < number_of_cpus ;i++) {
+ snprintfz(all_softnet_charts[i].netisr_cpuid, MAX_INT_DIGITS + 17, "cpu%d_softnet_stat", i);
+
+ if (unlikely(!all_softnet_charts[i].st)) {
+ all_softnet_charts[i].st = rrdset_create_localhost(
+ "cpu",
+ all_softnet_charts[i].netisr_cpuid,
+ NULL,
+ "softnet_stat",
+ NULL,
+ "Per CPU netisr statistics",
+ "events/s",
+ "freebsd.plugin",
+ "net.isr",
+ NETDATA_CHART_PRIO_SOFTNET_PER_CORE + i,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ all_softnet_charts[i].rd_dispatched = rrddim_add(all_softnet_charts[i].st, "dispatched",
+ NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ all_softnet_charts[i].rd_hybrid_dispatched = rrddim_add(all_softnet_charts[i].st, "hybrid_dispatched",
+ NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ all_softnet_charts[i].rd_qdrops = rrddim_add(all_softnet_charts[i].st, "qdrops",
+ NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ all_softnet_charts[i].rd_queued = rrddim_add(all_softnet_charts[i].st, "queued",
+ NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(all_softnet_charts[i].st, all_softnet_charts[i].rd_dispatched,
+ netisr_stats[i].dispatched);
+ rrddim_set_by_pointer(all_softnet_charts[i].st, all_softnet_charts[i].rd_hybrid_dispatched,
+ netisr_stats[i].hybrid_dispatched);
+ rrddim_set_by_pointer(all_softnet_charts[i].st, all_softnet_charts[i].rd_qdrops,
+ netisr_stats[i].qdrops);
+ rrddim_set_by_pointer(all_softnet_charts[i].st, all_softnet_charts[i].rd_queued,
+ netisr_stats[i].queued);
+ rrdset_done(all_softnet_charts[i].st);
+ }
+ }
+
+ return 0;
+}
+
+// net.inet.tcp.states
+
+int do_net_inet_tcp_states(int update_every, usec_t dt) {
+ (void)dt;
+ static int mib[4] = {0, 0, 0, 0};
+ uint64_t tcps_states[TCP_NSTATES];
+
+ // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html
+ if (unlikely(GETSYSCTL_SIMPLE("net.inet.tcp.states", mib, tcps_states))) {
+ error("DISABLED: ipv4.tcpsock chart");
+ error("DISABLED: net.inet.tcp.states module");
+ return 1;
+ } else {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "tcpsock",
+ NULL,
+ "tcp",
+ NULL,
+ "IPv4 TCP Connections",
+ "active connections",
+ "freebsd.plugin",
+ "net.inet.tcp.states",
+ 2500,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd = rrddim_add(st, "CurrEstab", "connections", 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd, tcps_states[TCPS_ESTABLISHED]);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
+// net.inet.tcp.stats
+
+int do_net_inet_tcp_stats(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1, do_tcpext_connaborts = -1, do_tcpext_ofo = -1,
+ do_tcpext_syncookies = -1, do_tcpext_listen = -1, do_ecn = -1;
+
+ if (unlikely(do_tcp_packets == -1)) {
+ do_tcp_packets = config_get_boolean("plugin:freebsd:net.inet.tcp.stats", "ipv4 TCP packets", 1);
+ do_tcp_errors = config_get_boolean("plugin:freebsd:net.inet.tcp.stats", "ipv4 TCP errors", 1);
+ do_tcp_handshake = config_get_boolean("plugin:freebsd:net.inet.tcp.stats", "ipv4 TCP handshake issues", 1);
+ do_tcpext_connaborts = config_get_boolean_ondemand("plugin:freebsd:net.inet.tcp.stats", "TCP connection aborts",
+ CONFIG_BOOLEAN_AUTO);
+ do_tcpext_ofo = config_get_boolean_ondemand("plugin:freebsd:net.inet.tcp.stats", "TCP out-of-order queue",
+ CONFIG_BOOLEAN_AUTO);
+ do_tcpext_syncookies = config_get_boolean_ondemand("plugin:freebsd:net.inet.tcp.stats", "TCP SYN cookies",
+ CONFIG_BOOLEAN_AUTO);
+ do_tcpext_listen = config_get_boolean_ondemand("plugin:freebsd:net.inet.tcp.stats", "TCP listen issues",
+ CONFIG_BOOLEAN_AUTO);
+ do_ecn = config_get_boolean_ondemand("plugin:freebsd:net.inet.tcp.stats", "ECN packets",
+ CONFIG_BOOLEAN_AUTO);
+ }
+
+ // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html
+ if (likely(do_tcp_packets || do_tcp_errors || do_tcp_handshake || do_tcpext_connaborts || do_tcpext_ofo ||
+ do_tcpext_syncookies || do_tcpext_listen || do_ecn)) {
+ static int mib[4] = {0, 0, 0, 0};
+ struct tcpstat tcpstat;
+
+ if (unlikely(GETSYSCTL_SIMPLE("net.inet.tcp.stats", mib, tcpstat))) {
+ do_tcp_packets = 0;
+ error("DISABLED: ipv4.tcppackets chart");
+ do_tcp_errors = 0;
+ error("DISABLED: ipv4.tcperrors chart");
+ do_tcp_handshake = 0;
+ error("DISABLED: ipv4.tcphandshake chart");
+ do_tcpext_connaborts = 0;
+ error("DISABLED: ipv4.tcpconnaborts chart");
+ do_tcpext_ofo = 0;
+ error("DISABLED: ipv4.tcpofo chart");
+ do_tcpext_syncookies = 0;
+ error("DISABLED: ipv4.tcpsyncookies chart");
+ do_tcpext_listen = 0;
+ error("DISABLED: ipv4.tcplistenissues chart");
+ do_ecn = 0;
+ error("DISABLED: ipv4.ecnpkts chart");
+ error("DISABLED: net.inet.tcp.stats module");
+ return 1;
+ } else {
+ if (likely(do_tcp_packets)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_segs = NULL, *rd_out_segs = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "tcppackets",
+ NULL,
+ "tcp",
+ NULL,
+ "IPv4 TCP Packets",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.tcp.stats",
+ 2600,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_in_segs = rrddim_add(st, "InSegs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_segs = rrddim_add(st, "OutSegs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_segs, tcpstat.tcps_rcvtotal);
+ rrddim_set_by_pointer(st, rd_out_segs, tcpstat.tcps_sndtotal);
+ rrdset_done(st);
+ }
+
+ if (likely(do_tcp_errors)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_errs = NULL, *rd_in_csum_errs = NULL, *rd_retrans_segs = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "tcperrors",
+ NULL,
+ "tcp",
+ NULL,
+ "IPv4 TCP Errors",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.tcp.stats",
+ 2700,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_in_errs = rrddim_add(st, "InErrs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_csum_errs = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_retrans_segs = rrddim_add(st, "RetransSegs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+#if __FreeBSD__ >= 11
+ rrddim_set_by_pointer(st, rd_in_errs, tcpstat.tcps_rcvbadoff + tcpstat.tcps_rcvreassfull +
+ tcpstat.tcps_rcvshort);
+#else
+ rrddim_set_by_pointer(st, rd_in_errs, tcpstat.tcps_rcvbadoff + tcpstat.tcps_rcvshort);
+#endif
+ rrddim_set_by_pointer(st, rd_in_csum_errs, tcpstat.tcps_rcvbadsum);
+ rrddim_set_by_pointer(st, rd_retrans_segs, tcpstat.tcps_sndrexmitpack);
+ rrdset_done(st);
+ }
+
+ if (likely(do_tcp_handshake)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_estab_resets = NULL, *rd_active_opens = NULL, *rd_passive_opens = NULL,
+ *rd_attempt_fails = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "tcphandshake",
+ NULL,
+ "tcp",
+ NULL,
+ "IPv4 TCP Handshake Issues",
+ "events/s",
+ "freebsd.plugin",
+ "net.inet.tcp.stats",
+ 2900,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_estab_resets = rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_active_opens = rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_passive_opens = rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_attempt_fails = rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_estab_resets, tcpstat.tcps_drops);
+ rrddim_set_by_pointer(st, rd_active_opens, tcpstat.tcps_connattempt);
+ rrddim_set_by_pointer(st, rd_passive_opens, tcpstat.tcps_accepts);
+ rrddim_set_by_pointer(st, rd_attempt_fails, tcpstat.tcps_conndrops);
+ rrdset_done(st);
+ }
+
+ if (do_tcpext_connaborts == CONFIG_BOOLEAN_YES || (do_tcpext_connaborts == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_rcvpackafterwin ||
+ tcpstat.tcps_rcvafterclose ||
+ tcpstat.tcps_rcvmemdrop ||
+ tcpstat.tcps_persistdrop ||
+ tcpstat.tcps_finwait2_drops ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_connaborts = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_on_data = NULL, *rd_on_close = NULL, *rd_on_memory = NULL,
+ *rd_on_timeout = NULL, *rd_on_linger = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "tcpconnaborts",
+ NULL,
+ "tcp",
+ NULL,
+ "TCP Connection Aborts",
+ "connections/s",
+ "freebsd.plugin",
+ "net.inet.tcp.stats",
+ 3010,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_on_data = rrddim_add(st, "TCPAbortOnData", "baddata", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_on_close = rrddim_add(st, "TCPAbortOnClose", "userclosed", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_on_memory = rrddim_add(st, "TCPAbortOnMemory", "nomemory", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_on_timeout = rrddim_add(st, "TCPAbortOnTimeout", "timeout", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_on_linger = rrddim_add(st, "TCPAbortOnLinger", "linger", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_on_data, tcpstat.tcps_rcvpackafterwin);
+ rrddim_set_by_pointer(st, rd_on_close, tcpstat.tcps_rcvafterclose);
+ rrddim_set_by_pointer(st, rd_on_memory, tcpstat.tcps_rcvmemdrop);
+ rrddim_set_by_pointer(st, rd_on_timeout, tcpstat.tcps_persistdrop);
+ rrddim_set_by_pointer(st, rd_on_linger, tcpstat.tcps_finwait2_drops);
+ rrdset_done(st);
+ }
+
+ if (do_tcpext_ofo == CONFIG_BOOLEAN_YES || (do_tcpext_ofo == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_rcvoopack ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_ofo = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ofo_queue = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "tcpofo",
+ NULL,
+ "tcp",
+ NULL,
+ "TCP Out-Of-Order Queue",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.tcp.stats",
+ 3050,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_ofo_queue = rrddim_add(st, "TCPOFOQueue", "inqueue", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ofo_queue, tcpstat.tcps_rcvoopack);
+ rrdset_done(st);
+ }
+
+ if (do_tcpext_syncookies == CONFIG_BOOLEAN_YES || (do_tcpext_syncookies == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_sc_sendcookie ||
+ tcpstat.tcps_sc_recvcookie ||
+ tcpstat.tcps_sc_zonefail ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_syncookies = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_recv = NULL, *rd_send = NULL, *rd_failed = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "tcpsyncookies",
+ NULL,
+ "tcp",
+ NULL,
+ "TCP SYN Cookies",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.tcp.stats",
+ 3100,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_recv = rrddim_add(st, "SyncookiesRecv", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_send = rrddim_add(st, "SyncookiesSent", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st, "SyncookiesFailed", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_recv, tcpstat.tcps_sc_recvcookie);
+ rrddim_set_by_pointer(st, rd_send, tcpstat.tcps_sc_sendcookie);
+ rrddim_set_by_pointer(st, rd_failed, tcpstat.tcps_sc_zonefail);
+ rrdset_done(st);
+ }
+
+ if(do_tcpext_listen == CONFIG_BOOLEAN_YES || (do_tcpext_listen == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_listendrop ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_listen = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_listen = NULL;
+ static RRDDIM *rd_overflows = NULL;
+
+ if(unlikely(!st_listen)) {
+
+ st_listen = rrdset_create_localhost(
+ "ipv4",
+ "tcplistenissues",
+ NULL,
+ "tcp",
+ NULL,
+ "TCP Listen Socket Issues",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.tcp.stats",
+ 3015,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_overflows = rrddim_add(st_listen, "ListenOverflows", "overflows", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_listen, rd_overflows, tcpstat.tcps_listendrop);
+ rrdset_done(st_listen);
+ }
+
+ if (do_ecn == CONFIG_BOOLEAN_YES || (do_ecn == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_ecn_ce ||
+ tcpstat.tcps_ecn_ect0 ||
+ tcpstat.tcps_ecn_ect1 ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ecn = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ce = NULL, *rd_no_ect = NULL, *rd_ect0 = NULL, *rd_ect1 = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "ecnpkts",
+ NULL,
+ "ecn",
+ NULL,
+ "IPv4 ECN Statistics",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.tcp.stats",
+ 8700,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ce = rrddim_add(st, "InCEPkts", "CEP", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_no_ect = rrddim_add(st, "InNoECTPkts", "NoECTP", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ect0 = rrddim_add(st, "InECT0Pkts", "ECTP0", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ect1 = rrddim_add(st, "InECT1Pkts", "ECTP1", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ce, tcpstat.tcps_ecn_ce);
+ rrddim_set_by_pointer(st, rd_no_ect, tcpstat.tcps_ecn_ce - (tcpstat.tcps_ecn_ect0 +
+ tcpstat.tcps_ecn_ect1));
+ rrddim_set_by_pointer(st, rd_ect0, tcpstat.tcps_ecn_ect0);
+ rrddim_set_by_pointer(st, rd_ect1, tcpstat.tcps_ecn_ect1);
+ rrdset_done(st);
+ }
+
+ }
+ } else {
+ error("DISABLED: net.inet.tcp.stats module");
+ return 1;
+ }
+
+ return 0;
+}
+
+// net.inet.udp.stats
+
+int do_net_inet_udp_stats(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_udp_packets = -1, do_udp_errors = -1;
+
+ if (unlikely(do_udp_packets == -1)) {
+ do_udp_packets = config_get_boolean("plugin:freebsd:net.inet.udp.stats", "ipv4 UDP packets", 1);
+ do_udp_errors = config_get_boolean("plugin:freebsd:net.inet.udp.stats", "ipv4 UDP errors", 1);
+ }
+
+ // see http://net-snmp.sourceforge.net/docs/mibs/udp.html
+ if (likely(do_udp_packets || do_udp_errors)) {
+ static int mib[4] = {0, 0, 0, 0};
+ struct udpstat udpstat;
+
+ if (unlikely(GETSYSCTL_SIMPLE("net.inet.udp.stats", mib, udpstat))) {
+ do_udp_packets = 0;
+ error("DISABLED: ipv4.udppackets chart");
+ do_udp_errors = 0;
+ error("DISABLED: ipv4.udperrors chart");
+ error("DISABLED: net.inet.udp.stats module");
+ return 1;
+ } else {
+ if (likely(do_udp_packets)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "udppackets",
+ NULL,
+ "udp",
+ NULL,
+ "IPv4 UDP Packets",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.udp.stats",
+ 2601,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_in = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, udpstat.udps_ipackets);
+ rrddim_set_by_pointer(st, rd_out, udpstat.udps_opackets);
+ rrdset_done(st);
+ }
+
+ if (likely(do_udp_errors)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_errors = NULL, *rd_no_ports = NULL, *rd_recv_buf_errors = NULL,
+ *rd_in_csum_errors = NULL, *rd_ignored_multi = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "udperrors",
+ NULL,
+ "udp",
+ NULL,
+ "IPv4 UDP Errors",
+ "events/s",
+ "freebsd.plugin",
+ "net.inet.udp.stats",
+ 2701,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_in_errors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_no_ports = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_recv_buf_errors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_csum_errors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ignored_multi = rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_errors, udpstat.udps_hdrops + udpstat.udps_badlen);
+ rrddim_set_by_pointer(st, rd_no_ports, udpstat.udps_noport);
+ rrddim_set_by_pointer(st, rd_recv_buf_errors, udpstat.udps_fullsock);
+ rrddim_set_by_pointer(st, rd_in_csum_errors, udpstat.udps_badsum + udpstat.udps_nosum);
+ rrddim_set_by_pointer(st, rd_ignored_multi, udpstat.udps_filtermcast);
+ rrdset_done(st);
+ }
+ }
+ } else {
+ error("DISABLED: net.inet.udp.stats module");
+ return 1;
+ }
+
+ return 0;
+}
+
+// net.inet.icmp.stats
+
+int do_net_inet_icmp_stats(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_icmp_packets = -1, do_icmp_errors = -1, do_icmpmsg = -1;
+
+ if (unlikely(do_icmp_packets == -1)) {
+ do_icmp_packets = config_get_boolean("plugin:freebsd:net.inet.icmp.stats", "ipv4 ICMP packets", 1);
+ do_icmp_errors = config_get_boolean("plugin:freebsd:net.inet.icmp.stats", "ipv4 ICMP errors", 1);
+ do_icmpmsg = config_get_boolean("plugin:freebsd:net.inet.icmp.stats", "ipv4 ICMP messages", 1);
+ }
+
+ if (likely(do_icmp_packets || do_icmp_errors || do_icmpmsg)) {
+ static int mib[4] = {0, 0, 0, 0};
+ struct icmpstat icmpstat;
+ struct icmp_total {
+ u_long msgs_in;
+ u_long msgs_out;
+ } icmp_total = {0, 0};
+
+ if (unlikely(GETSYSCTL_SIMPLE("net.inet.icmp.stats", mib, icmpstat))) {
+ do_icmp_packets = 0;
+ error("DISABLED: ipv4.icmp chart");
+ do_icmp_errors = 0;
+ error("DISABLED: ipv4.icmp_errors chart");
+ do_icmpmsg = 0;
+ error("DISABLED: ipv4.icmpmsg chart");
+ error("DISABLED: net.inet.icmp.stats module");
+ return 1;
+ } else {
+ int i;
+
+ for (i = 0; i <= ICMP_MAXTYPE; i++) {
+ icmp_total.msgs_in += icmpstat.icps_inhist[i];
+ icmp_total.msgs_out += icmpstat.icps_outhist[i];
+ }
+ icmp_total.msgs_in += icmpstat.icps_badcode + icmpstat.icps_badlen + icmpstat.icps_checksum + icmpstat.icps_tooshort;
+
+ if (likely(do_icmp_packets)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "icmp"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Packets"
+ , "packets/s"
+ , "freebsd.plugin"
+ , "net.inet.icmp.stats"
+ , 2602
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_in = rrddim_add(st, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, icmp_total.msgs_in);
+ rrddim_set_by_pointer(st, rd_out, icmp_total.msgs_out);
+ rrdset_done(st);
+ }
+
+ if (likely(do_icmp_errors)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL, *rd_in_csum = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "icmp_errors"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Errors"
+ , "packets/s"
+ , "freebsd.plugin"
+ , "net.inet.icmp.stats"
+ , 2603
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_in = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_csum = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, icmpstat.icps_badcode + icmpstat.icps_badlen +
+ icmpstat.icps_checksum + icmpstat.icps_tooshort);
+ rrddim_set_by_pointer(st, rd_out, icmpstat.icps_error);
+ rrddim_set_by_pointer(st, rd_in_csum, icmpstat.icps_checksum);
+
+ rrdset_done(st);
+ }
+
+ if (likely(do_icmpmsg)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_reps = NULL, *rd_out_reps = NULL, *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "icmpmsg"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Messages"
+ , "packets/s"
+ , "freebsd.plugin"
+ , "net.inet.icmp.stats"
+ , 2604
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_in_reps = rrddim_add(st, "InEchoReps", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_reps = rrddim_add(st, "OutEchoReps", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in = rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "OutEchos", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_reps, icmpstat.icps_inhist[ICMP_ECHOREPLY]);
+ rrddim_set_by_pointer(st, rd_out_reps, icmpstat.icps_outhist[ICMP_ECHOREPLY]);
+ rrddim_set_by_pointer(st, rd_in, icmpstat.icps_inhist[ICMP_ECHO]);
+ rrddim_set_by_pointer(st, rd_out, icmpstat.icps_outhist[ICMP_ECHO]);
+ rrdset_done(st);
+ }
+ }
+ } else {
+ error("DISABLED: net.inet.icmp.stats module");
+ return 1;
+ }
+
+ return 0;
+}
+
+// net.inet.ip.stats
+
+int do_net_inet_ip_stats(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_ip_packets = -1, do_ip_fragsout = -1, do_ip_fragsin = -1, do_ip_errors = -1;
+
+ if (unlikely(do_ip_packets == -1)) {
+ do_ip_packets = config_get_boolean("plugin:freebsd:net.inet.ip.stats", "ipv4 packets", 1);
+ do_ip_fragsout = config_get_boolean("plugin:freebsd:net.inet.ip.stats", "ipv4 fragments sent", 1);
+ do_ip_fragsin = config_get_boolean("plugin:freebsd:net.inet.ip.stats", "ipv4 fragments assembly", 1);
+ do_ip_errors = config_get_boolean("plugin:freebsd:net.inet.ip.stats", "ipv4 errors", 1);
+ }
+
+ // see also http://net-snmp.sourceforge.net/docs/mibs/ip.html
+ if (likely(do_ip_packets || do_ip_fragsout || do_ip_fragsin || do_ip_errors)) {
+ static int mib[4] = {0, 0, 0, 0};
+ struct ipstat ipstat;
+
+ if (unlikely(GETSYSCTL_SIMPLE("net.inet.ip.stats", mib, ipstat))) {
+ do_ip_packets = 0;
+ error("DISABLED: ipv4.packets chart");
+ do_ip_fragsout = 0;
+ error("DISABLED: ipv4.fragsout chart");
+ do_ip_fragsin = 0;
+ error("DISABLED: ipv4.fragsin chart");
+ do_ip_errors = 0;
+ error("DISABLED: ipv4.errors chart");
+ error("DISABLED: net.inet.ip.stats module");
+ return 1;
+ } else {
+ if (likely(do_ip_packets)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_receives = NULL, *rd_out_requests = NULL, *rd_forward_datagrams = NULL,
+ *rd_in_delivers = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "packets",
+ NULL,
+ "packets",
+ NULL,
+ "IPv4 Packets",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.ip.stats",
+ 3000,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_in_receives = rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_requests = rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_forward_datagrams = rrddim_add(st, "ForwDatagrams", "forwarded", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_delivers = rrddim_add(st, "InDelivers", "delivered", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_receives, ipstat.ips_total);
+ rrddim_set_by_pointer(st, rd_out_requests, ipstat.ips_localout);
+ rrddim_set_by_pointer(st, rd_forward_datagrams, ipstat.ips_forward);
+ rrddim_set_by_pointer(st, rd_in_delivers, ipstat.ips_delivered);
+ rrdset_done(st);
+ }
+
+ if (likely(do_ip_fragsout)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ok = NULL, *rd_fails = NULL, *rd_created = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "fragsout",
+ NULL,
+ "fragments",
+ NULL,
+ "IPv4 Fragments Sent",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.ip.stats",
+ 3010,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ok = rrddim_add(st, "FragOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_fails = rrddim_add(st, "FragFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_created = rrddim_add(st, "FragCreates", "created", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ok, ipstat.ips_fragmented);
+ rrddim_set_by_pointer(st, rd_fails, ipstat.ips_cantfrag);
+ rrddim_set_by_pointer(st, rd_created, ipstat.ips_ofragments);
+ rrdset_done(st);
+ }
+
+ if (likely(do_ip_fragsin)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ok = NULL, *rd_failed = NULL, *rd_all = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "fragsin",
+ NULL,
+ "fragments",
+ NULL,
+ "IPv4 Fragments Reassembly",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.ip.stats",
+ 3011,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ok = rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st, "ReasmFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_all = rrddim_add(st, "ReasmReqds", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ok, ipstat.ips_fragments);
+ rrddim_set_by_pointer(st, rd_failed, ipstat.ips_fragdropped);
+ rrddim_set_by_pointer(st, rd_all, ipstat.ips_reassembled);
+ rrdset_done(st);
+ }
+
+ if (likely(do_ip_errors)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_discards = NULL, *rd_out_discards = NULL,
+ *rd_in_hdr_errors = NULL, *rd_out_no_routes = NULL,
+ *rd_in_addr_errors = NULL, *rd_in_unknown_protos = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4",
+ "errors",
+ NULL,
+ "errors",
+ NULL,
+ "IPv4 Errors",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet.ip.stats",
+ 3002,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_in_discards = rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_discards = rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_hdr_errors = rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_no_routes = rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_addr_errors = rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_unknown_protos = rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_discards, ipstat.ips_badsum + ipstat.ips_tooshort +
+ ipstat.ips_toosmall + ipstat.ips_toolong);
+ rrddim_set_by_pointer(st, rd_out_discards, ipstat.ips_odropped);
+ rrddim_set_by_pointer(st, rd_in_hdr_errors, ipstat.ips_badhlen + ipstat.ips_badlen +
+ ipstat.ips_badoptions + ipstat.ips_badvers);
+ rrddim_set_by_pointer(st, rd_out_no_routes, ipstat.ips_noroute);
+ rrddim_set_by_pointer(st, rd_in_addr_errors, ipstat.ips_badaddr);
+ rrddim_set_by_pointer(st, rd_in_unknown_protos, ipstat.ips_noproto);
+ rrdset_done(st);
+ }
+ }
+ } else {
+ error("DISABLED: net.inet.ip.stats module");
+ return 1;
+ }
+
+ return 0;
+}
+
+// net.inet6.ip6.stats
+
+int do_net_inet6_ip6_stats(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_ip6_packets = -1, do_ip6_fragsout = -1, do_ip6_fragsin = -1, do_ip6_errors = -1;
+
+ if (unlikely(do_ip6_packets == -1)) {
+ do_ip6_packets = config_get_boolean_ondemand("plugin:freebsd:net.inet6.ip6.stats", "ipv6 packets",
+ CONFIG_BOOLEAN_AUTO);
+ do_ip6_fragsout = config_get_boolean_ondemand("plugin:freebsd:net.inet6.ip6.stats", "ipv6 fragments sent",
+ CONFIG_BOOLEAN_AUTO);
+ do_ip6_fragsin = config_get_boolean_ondemand("plugin:freebsd:net.inet6.ip6.stats", "ipv6 fragments assembly",
+ CONFIG_BOOLEAN_AUTO);
+ do_ip6_errors = config_get_boolean_ondemand("plugin:freebsd:net.inet6.ip6.stats", "ipv6 errors",
+ CONFIG_BOOLEAN_AUTO);
+ }
+
+ if (likely(do_ip6_packets || do_ip6_fragsout || do_ip6_fragsin || do_ip6_errors)) {
+ static int mib[4] = {0, 0, 0, 0};
+ struct ip6stat ip6stat;
+
+ if (unlikely(GETSYSCTL_SIMPLE("net.inet6.ip6.stats", mib, ip6stat))) {
+ do_ip6_packets = 0;
+ error("DISABLED: ipv6.packets chart");
+ do_ip6_fragsout = 0;
+ error("DISABLED: ipv6.fragsout chart");
+ do_ip6_fragsin = 0;
+ error("DISABLED: ipv6.fragsin chart");
+ do_ip6_errors = 0;
+ error("DISABLED: ipv6.errors chart");
+ error("DISABLED: net.inet6.ip6.stats module");
+ return 1;
+ } else {
+ if (do_ip6_packets == CONFIG_BOOLEAN_YES || (do_ip6_packets == CONFIG_BOOLEAN_AUTO &&
+ (ip6stat.ip6s_localout ||
+ ip6stat.ip6s_total ||
+ ip6stat.ip6s_forward ||
+ ip6stat.ip6s_delivered ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_packets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_received = NULL, *rd_sent = NULL, *rd_forwarded = NULL, *rd_delivers = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "packets",
+ NULL,
+ "packets",
+ NULL,
+ "IPv6 Packets",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet6.ip6.stats",
+ 3000,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_forwarded = rrddim_add(st, "forwarded", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_delivers = rrddim_add(st, "delivers", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_sent, ip6stat.ip6s_localout);
+ rrddim_set_by_pointer(st, rd_received, ip6stat.ip6s_total);
+ rrddim_set_by_pointer(st, rd_forwarded, ip6stat.ip6s_forward);
+ rrddim_set_by_pointer(st, rd_delivers, ip6stat.ip6s_delivered);
+ rrdset_done(st);
+ }
+
+ if (do_ip6_fragsout == CONFIG_BOOLEAN_YES || (do_ip6_fragsout == CONFIG_BOOLEAN_AUTO &&
+ (ip6stat.ip6s_fragmented ||
+ ip6stat.ip6s_cantfrag ||
+ ip6stat.ip6s_ofragments ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_fragsout = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ok = NULL, *rd_failed = NULL, *rd_all = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "fragsout",
+ NULL,
+ "fragments",
+ NULL,
+ "IPv6 Fragments Sent",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet6.ip6.stats",
+ 3010,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ok = rrddim_add(st, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st, "failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_all = rrddim_add(st, "all", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ok, ip6stat.ip6s_fragmented);
+ rrddim_set_by_pointer(st, rd_failed, ip6stat.ip6s_cantfrag);
+ rrddim_set_by_pointer(st, rd_all, ip6stat.ip6s_ofragments);
+ rrdset_done(st);
+ }
+
+ if (do_ip6_fragsin == CONFIG_BOOLEAN_YES || (do_ip6_fragsin == CONFIG_BOOLEAN_AUTO &&
+ (ip6stat.ip6s_reassembled ||
+ ip6stat.ip6s_fragdropped ||
+ ip6stat.ip6s_fragtimeout ||
+ ip6stat.ip6s_fragments ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_fragsin = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ok = NULL, *rd_failed = NULL, *rd_timeout = NULL, *rd_all = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "fragsin",
+ NULL,
+ "fragments",
+ NULL,
+ "IPv6 Fragments Reassembly",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet6.ip6.stats",
+ 3011,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ok = rrddim_add(st, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st, "failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_timeout = rrddim_add(st, "timeout", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_all = rrddim_add(st, "all", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ok, ip6stat.ip6s_reassembled);
+ rrddim_set_by_pointer(st, rd_failed, ip6stat.ip6s_fragdropped);
+ rrddim_set_by_pointer(st, rd_timeout, ip6stat.ip6s_fragtimeout);
+ rrddim_set_by_pointer(st, rd_all, ip6stat.ip6s_fragments);
+ rrdset_done(st);
+ }
+
+ if (do_ip6_errors == CONFIG_BOOLEAN_YES || (do_ip6_errors == CONFIG_BOOLEAN_AUTO &&
+ (ip6stat.ip6s_toosmall ||
+ ip6stat.ip6s_odropped ||
+ ip6stat.ip6s_badoptions ||
+ ip6stat.ip6s_badvers ||
+ ip6stat.ip6s_exthdrtoolong ||
+ ip6stat.ip6s_sources_none ||
+ ip6stat.ip6s_tooshort ||
+ ip6stat.ip6s_cantforward ||
+ ip6stat.ip6s_noroute ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_errors = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_discards = NULL, *rd_out_discards = NULL,
+ *rd_in_hdr_errors = NULL, *rd_in_addr_errors = NULL, *rd_in_truncated_pkts = NULL,
+ *rd_in_no_routes = NULL, *rd_out_no_routes = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "errors",
+ NULL,
+ "errors",
+ NULL,
+ "IPv6 Errors",
+ "packets/s",
+ "freebsd.plugin",
+ "net.inet6.ip6.stats",
+ 3002,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_in_discards = rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_discards = rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_hdr_errors = rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_addr_errors = rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_truncated_pkts = rrddim_add(st, "InTruncatedPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_no_routes = rrddim_add(st, "InNoRoutes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_no_routes = rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_discards, ip6stat.ip6s_toosmall);
+ rrddim_set_by_pointer(st, rd_out_discards, ip6stat.ip6s_odropped);
+ rrddim_set_by_pointer(st, rd_in_hdr_errors, ip6stat.ip6s_badoptions + ip6stat.ip6s_badvers +
+ ip6stat.ip6s_exthdrtoolong);
+ rrddim_set_by_pointer(st, rd_in_addr_errors, ip6stat.ip6s_sources_none);
+ rrddim_set_by_pointer(st, rd_in_truncated_pkts, ip6stat.ip6s_tooshort);
+ rrddim_set_by_pointer(st, rd_in_no_routes, ip6stat.ip6s_cantforward);
+ rrddim_set_by_pointer(st, rd_out_no_routes, ip6stat.ip6s_noroute);
+ rrdset_done(st);
+ }
+ }
+ } else {
+ error("DISABLED: net.inet6.ip6.stats module");
+ return 1;
+ }
+
+ return 0;
+}
+
+// net.inet6.icmp6.stats
+
+int do_net_inet6_icmp6_stats(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_icmp6 = -1, do_icmp6_redir = -1, do_icmp6_errors = -1, do_icmp6_echos = -1, do_icmp6_router = -1,
+ do_icmp6_neighbor = -1, do_icmp6_types = -1;
+
+ if (unlikely(do_icmp6 == -1)) {
+ do_icmp6 = config_get_boolean_ondemand("plugin:freebsd:net.inet6.icmp6.stats", "icmp",
+ CONFIG_BOOLEAN_AUTO);
+ do_icmp6_redir = config_get_boolean_ondemand("plugin:freebsd:net.inet6.icmp6.stats", "icmp redirects",
+ CONFIG_BOOLEAN_AUTO);
+ do_icmp6_errors = config_get_boolean_ondemand("plugin:freebsd:net.inet6.icmp6.stats", "icmp errors",
+ CONFIG_BOOLEAN_AUTO);
+ do_icmp6_echos = config_get_boolean_ondemand("plugin:freebsd:net.inet6.icmp6.stats", "icmp echos",
+ CONFIG_BOOLEAN_AUTO);
+ do_icmp6_router = config_get_boolean_ondemand("plugin:freebsd:net.inet6.icmp6.stats", "icmp router",
+ CONFIG_BOOLEAN_AUTO);
+ do_icmp6_neighbor = config_get_boolean_ondemand("plugin:freebsd:net.inet6.icmp6.stats", "icmp neighbor",
+ CONFIG_BOOLEAN_AUTO);
+ do_icmp6_types = config_get_boolean_ondemand("plugin:freebsd:net.inet6.icmp6.stats", "icmp types",
+ CONFIG_BOOLEAN_AUTO);
+ }
+
+ if (likely(do_icmp6 || do_icmp6_redir || do_icmp6_errors || do_icmp6_echos || do_icmp6_router || do_icmp6_neighbor || do_icmp6_types)) {
+ static int mib[4] = {0, 0, 0, 0};
+ struct icmp6stat icmp6stat;
+
+ if (unlikely(GETSYSCTL_SIMPLE("net.inet6.icmp6.stats", mib, icmp6stat))) {
+ do_icmp6 = 0;
+ error("DISABLED: ipv6.icmp chart");
+ do_icmp6_redir = 0;
+ error("DISABLED: ipv6.icmpredir chart");
+ do_icmp6_errors = 0;
+ error("DISABLED: ipv6.icmperrors chart");
+ do_icmp6_echos = 0;
+ error("DISABLED: ipv6.icmpechos chart");
+ do_icmp6_router = 0;
+ error("DISABLED: ipv6.icmprouter chart");
+ do_icmp6_neighbor = 0;
+ error("DISABLED: ipv6.icmpneighbor chart");
+ do_icmp6_types = 0;
+ error("DISABLED: ipv6.icmptypes chart");
+ error("DISABLED: net.inet6.icmp6.stats module");
+ return 1;
+ } else {
+ int i;
+ struct icmp6_total {
+ u_long msgs_in;
+ u_long msgs_out;
+ } icmp6_total = {0, 0};
+
+ for (i = 0; i <= ICMP6_MAXTYPE; i++) {
+ icmp6_total.msgs_in += icmp6stat.icp6s_inhist[i];
+ icmp6_total.msgs_out += icmp6stat.icp6s_outhist[i];
+ }
+ icmp6_total.msgs_in += icmp6stat.icp6s_badcode + icmp6stat.icp6s_badlen + icmp6stat.icp6s_checksum + icmp6stat.icp6s_tooshort;
+
+ // --------------------------------------------------------------------
+
+ if (do_icmp6 == CONFIG_BOOLEAN_YES || (do_icmp6 == CONFIG_BOOLEAN_AUTO &&
+ (icmp6_total.msgs_in ||
+ icmp6_total.msgs_out ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6 = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_received = NULL, *rd_sent = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "icmp",
+ NULL,
+ "icmp",
+ NULL,
+ "IPv6 ICMP Messages",
+ "messages/s",
+ "freebsd.plugin",
+ "net.inet6.icmp6.stats",
+ 10000,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_received, icmp6_total.msgs_out);
+ rrddim_set_by_pointer(st, rd_sent, icmp6_total.msgs_in);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_redir == CONFIG_BOOLEAN_YES || (do_icmp6_redir == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[ND_REDIRECT] ||
+ icmp6stat.icp6s_outhist[ND_REDIRECT] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_redir = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_received = NULL, *rd_sent = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "icmpredir",
+ NULL,
+ "icmp",
+ NULL,
+ "IPv6 ICMP Redirects",
+ "redirects/s",
+ "freebsd.plugin",
+ "net.inet6.icmp6.stats",
+ 10050,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_received, icmp6stat.icp6s_outhist[ND_REDIRECT]);
+ rrddim_set_by_pointer(st, rd_sent, icmp6stat.icp6s_inhist[ND_REDIRECT]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_errors == CONFIG_BOOLEAN_YES || (do_icmp6_errors == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_badcode ||
+ icmp6stat.icp6s_badlen ||
+ icmp6stat.icp6s_checksum ||
+ icmp6stat.icp6s_tooshort ||
+ icmp6stat.icp6s_error ||
+ icmp6stat.icp6s_inhist[ICMP6_DST_UNREACH] ||
+ icmp6stat.icp6s_inhist[ICMP6_TIME_EXCEEDED] ||
+ icmp6stat.icp6s_inhist[ICMP6_PARAM_PROB] ||
+ icmp6stat.icp6s_outhist[ICMP6_DST_UNREACH] ||
+ icmp6stat.icp6s_outhist[ICMP6_TIME_EXCEEDED] ||
+ icmp6stat.icp6s_outhist[ICMP6_PARAM_PROB] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_errors = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_errors = NULL, *rd_out_errors = NULL, *rd_in_csum_errors = NULL,
+ *rd_in_dest_unreachs = NULL, *rd_in_pkt_too_bigs = NULL, *rd_in_time_excds = NULL,
+ *rd_in_parm_problems = NULL, *rd_out_dest_unreachs = NULL, *rd_out_time_excds = NULL,
+ *rd_out_parm_problems = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "icmperrors",
+ NULL, "icmp",
+ NULL,
+ "IPv6 ICMP Errors",
+ "errors/s",
+ "freebsd.plugin",
+ "net.inet6.icmp6.stats",
+ 10100,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_in_errors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_errors = rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_csum_errors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_dest_unreachs = rrddim_add(st, "InDestUnreachs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_pkt_too_bigs = rrddim_add(st, "InPktTooBigs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_time_excds = rrddim_add(st, "InTimeExcds", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_parm_problems = rrddim_add(st, "InParmProblems", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_dest_unreachs = rrddim_add(st, "OutDestUnreachs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_time_excds = rrddim_add(st, "OutTimeExcds", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_parm_problems = rrddim_add(st, "OutParmProblems", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_errors, icmp6stat.icp6s_badcode + icmp6stat.icp6s_badlen +
+ icmp6stat.icp6s_checksum + icmp6stat.icp6s_tooshort);
+ rrddim_set_by_pointer(st, rd_out_errors, icmp6stat.icp6s_error);
+ rrddim_set_by_pointer(st, rd_in_csum_errors, icmp6stat.icp6s_checksum);
+ rrddim_set_by_pointer(st, rd_in_dest_unreachs, icmp6stat.icp6s_inhist[ICMP6_DST_UNREACH]);
+ rrddim_set_by_pointer(st, rd_in_pkt_too_bigs, icmp6stat.icp6s_badlen);
+ rrddim_set_by_pointer(st, rd_in_time_excds, icmp6stat.icp6s_inhist[ICMP6_TIME_EXCEEDED]);
+ rrddim_set_by_pointer(st, rd_in_parm_problems, icmp6stat.icp6s_inhist[ICMP6_PARAM_PROB]);
+ rrddim_set_by_pointer(st, rd_out_dest_unreachs, icmp6stat.icp6s_outhist[ICMP6_DST_UNREACH]);
+ rrddim_set_by_pointer(st, rd_out_time_excds, icmp6stat.icp6s_outhist[ICMP6_TIME_EXCEEDED]);
+ rrddim_set_by_pointer(st, rd_out_parm_problems, icmp6stat.icp6s_outhist[ICMP6_PARAM_PROB]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_echos == CONFIG_BOOLEAN_YES || (do_icmp6_echos == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[ICMP6_ECHO_REQUEST] ||
+ icmp6stat.icp6s_outhist[ICMP6_ECHO_REQUEST] ||
+ icmp6stat.icp6s_inhist[ICMP6_ECHO_REPLY] ||
+ icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_echos = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL, *rd_in_replies = NULL, *rd_out_replies = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "icmpechos",
+ NULL,
+ "icmp",
+ NULL,
+ "IPv6 ICMP Echo",
+ "messages/s",
+ "freebsd.plugin",
+ "net.inet6.icmp6.stats",
+ 10200,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_in = rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "OutEchos", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_replies = rrddim_add(st, "InEchoReplies", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_replies = rrddim_add(st, "OutEchoReplies", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in, icmp6stat.icp6s_inhist[ICMP6_ECHO_REQUEST]);
+ rrddim_set_by_pointer(st, rd_out, icmp6stat.icp6s_outhist[ICMP6_ECHO_REQUEST]);
+ rrddim_set_by_pointer(st, rd_in_replies, icmp6stat.icp6s_inhist[ICMP6_ECHO_REPLY]);
+ rrddim_set_by_pointer(st, rd_out_replies, icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_router == CONFIG_BOOLEAN_YES || (do_icmp6_router == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[ND_ROUTER_SOLICIT] ||
+ icmp6stat.icp6s_outhist[ND_ROUTER_SOLICIT] ||
+ icmp6stat.icp6s_inhist[ND_ROUTER_ADVERT] ||
+ icmp6stat.icp6s_outhist[ND_ROUTER_ADVERT] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_router = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_solicits = NULL, *rd_out_solicits = NULL,
+ *rd_in_advertisements = NULL, *rd_out_advertisements = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "icmprouter",
+ NULL,
+ "icmp",
+ NULL,
+ "IPv6 Router Messages",
+ "messages/s",
+ "freebsd.plugin",
+ "net.inet6.icmp6.stats",
+ 10400,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_in_solicits = rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_solicits = rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_advertisements = rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_advertisements = rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_solicits, icmp6stat.icp6s_inhist[ND_ROUTER_SOLICIT]);
+ rrddim_set_by_pointer(st, rd_out_solicits, icmp6stat.icp6s_outhist[ND_ROUTER_SOLICIT]);
+ rrddim_set_by_pointer(st, rd_in_advertisements, icmp6stat.icp6s_inhist[ND_ROUTER_ADVERT]);
+ rrddim_set_by_pointer(st, rd_out_advertisements, icmp6stat.icp6s_outhist[ND_ROUTER_ADVERT]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_neighbor == CONFIG_BOOLEAN_YES || (do_icmp6_neighbor == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[ND_NEIGHBOR_SOLICIT] ||
+ icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT] ||
+ icmp6stat.icp6s_inhist[ND_NEIGHBOR_ADVERT] ||
+ icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_neighbor = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_solicits = NULL, *rd_out_solicits = NULL,
+ *rd_in_advertisements = NULL, *rd_out_advertisements = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "icmpneighbor",
+ NULL,
+ "icmp",
+ NULL,
+ "IPv6 Neighbor Messages",
+ "messages/s",
+ "freebsd.plugin",
+ "net.inet6.icmp6.stats",
+ 10500,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_in_solicits = rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_solicits = rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_advertisements = rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_advertisements = rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_solicits, icmp6stat.icp6s_inhist[ND_NEIGHBOR_SOLICIT]);
+ rrddim_set_by_pointer(st, rd_out_solicits, icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT]);
+ rrddim_set_by_pointer(st, rd_in_advertisements, icmp6stat.icp6s_inhist[ND_NEIGHBOR_ADVERT]);
+ rrddim_set_by_pointer(st, rd_out_advertisements, icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_types == CONFIG_BOOLEAN_YES || (do_icmp6_types == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[1] ||
+ icmp6stat.icp6s_inhist[128] ||
+ icmp6stat.icp6s_inhist[129] ||
+ icmp6stat.icp6s_inhist[136] ||
+ icmp6stat.icp6s_outhist[1] ||
+ icmp6stat.icp6s_outhist[128] ||
+ icmp6stat.icp6s_outhist[129] ||
+ icmp6stat.icp6s_outhist[133] ||
+ icmp6stat.icp6s_outhist[135] ||
+ icmp6stat.icp6s_outhist[136] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_types = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in_1 = NULL, *rd_in_128 = NULL, *rd_in_129 = NULL, *rd_in_136 = NULL,
+ *rd_out_1 = NULL, *rd_out_128 = NULL, *rd_out_129 = NULL, *rd_out_133 = NULL,
+ *rd_out_135 = NULL, *rd_out_143 = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6",
+ "icmptypes",
+ NULL,
+ "icmp",
+ NULL,
+ "IPv6 ICMP Types",
+ "messages/s",
+ "freebsd.plugin",
+ "net.inet6.icmp6.stats",
+ 10700,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rd_in_1 = rrddim_add(st, "InType1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_128 = rrddim_add(st, "InType128", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_129 = rrddim_add(st, "InType129", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_in_136 = rrddim_add(st, "InType136", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_1 = rrddim_add(st, "OutType1", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_128 = rrddim_add(st, "OutType128", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_129 = rrddim_add(st, "OutType129", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_133 = rrddim_add(st, "OutType133", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_135 = rrddim_add(st, "OutType135", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out_143 = rrddim_add(st, "OutType143", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_in_1, icmp6stat.icp6s_inhist[1]);
+ rrddim_set_by_pointer(st, rd_in_128, icmp6stat.icp6s_inhist[128]);
+ rrddim_set_by_pointer(st, rd_in_129, icmp6stat.icp6s_inhist[129]);
+ rrddim_set_by_pointer(st, rd_in_136, icmp6stat.icp6s_inhist[136]);
+ rrddim_set_by_pointer(st, rd_out_1, icmp6stat.icp6s_outhist[1]);
+ rrddim_set_by_pointer(st, rd_out_128, icmp6stat.icp6s_outhist[128]);
+ rrddim_set_by_pointer(st, rd_out_129, icmp6stat.icp6s_outhist[129]);
+ rrddim_set_by_pointer(st, rd_out_133, icmp6stat.icp6s_outhist[133]);
+ rrddim_set_by_pointer(st, rd_out_135, icmp6stat.icp6s_outhist[135]);
+ rrddim_set_by_pointer(st, rd_out_143, icmp6stat.icp6s_outhist[143]);
+ rrdset_done(st);
+ }
+ }
+ } else {
+ error("DISABLED: net.inet6.icmp6.stats module");
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c
new file mode 100644
index 0000000..a52ece3
--- /dev/null
+++ b/collectors/freebsd.plugin/plugin_freebsd.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_freebsd.h"
+
+static struct freebsd_module {
+ const char *name;
+ const char *dim;
+
+ int enabled;
+
+ int (*func)(int update_every, usec_t dt);
+
+ RRDDIM *rd;
+
+} freebsd_modules[] = {
+
+ // system metrics
+ {.name = "kern.cp_time", .dim = "cp_time", .enabled = 1, .func = do_kern_cp_time},
+ {.name = "vm.loadavg", .dim = "loadavg", .enabled = 1, .func = do_vm_loadavg},
+ {.name = "system.ram", .dim = "system_ram", .enabled = 1, .func = do_system_ram},
+ {.name = "vm.swap_info", .dim = "swap", .enabled = 1, .func = do_vm_swap_info},
+ {.name = "vm.stats.vm.v_swappgs", .dim = "swap_io", .enabled = 1, .func = do_vm_stats_sys_v_swappgs},
+ {.name = "vm.vmtotal", .dim = "vmtotal", .enabled = 1, .func = do_vm_vmtotal},
+ {.name = "vm.stats.vm.v_forks", .dim = "forks", .enabled = 1, .func = do_vm_stats_sys_v_forks},
+ {.name = "vm.stats.sys.v_swtch", .dim = "context_swtch", .enabled = 1, .func = do_vm_stats_sys_v_swtch},
+ {.name = "hw.intrcnt", .dim = "hw_intr", .enabled = 1, .func = do_hw_intcnt},
+ {.name = "vm.stats.sys.v_intr", .dim = "dev_intr", .enabled = 1, .func = do_vm_stats_sys_v_intr},
+ {.name = "vm.stats.sys.v_soft", .dim = "soft_intr", .enabled = 1, .func = do_vm_stats_sys_v_soft},
+ {.name = "net.isr", .dim = "net_isr", .enabled = 1, .func = do_net_isr},
+ {.name = "kern.ipc.sem", .dim = "semaphores", .enabled = 1, .func = do_kern_ipc_sem},
+ {.name = "kern.ipc.shm", .dim = "shared_memory", .enabled = 1, .func = do_kern_ipc_shm},
+ {.name = "kern.ipc.msq", .dim = "message_queues", .enabled = 1, .func = do_kern_ipc_msq},
+ {.name = "uptime", .dim = "uptime", .enabled = 1, .func = do_uptime},
+
+ // memory metrics
+ {.name = "vm.stats.vm.v_pgfaults", .dim = "pgfaults", .enabled = 1, .func = do_vm_stats_sys_v_pgfaults},
+
+ // CPU metrics
+ {.name = "kern.cp_times", .dim = "cp_times", .enabled = 1, .func = do_kern_cp_times},
+ {.name = "dev.cpu.temperature", .dim = "cpu_temperature", .enabled = 1, .func = do_dev_cpu_temperature},
+ {.name = "dev.cpu.0.freq", .dim = "cpu_frequency", .enabled = 1, .func = do_dev_cpu_0_freq},
+
+ // disk metrics
+ {.name = "kern.devstat", .dim = "kern_devstat", .enabled = 1, .func = do_kern_devstat},
+ {.name = "getmntinfo", .dim = "getmntinfo", .enabled = 1, .func = do_getmntinfo},
+
+ // network metrics
+ {.name = "net.inet.tcp.states", .dim = "tcp_states", .enabled = 1, .func = do_net_inet_tcp_states},
+ {.name = "net.inet.tcp.stats", .dim = "tcp_stats", .enabled = 1, .func = do_net_inet_tcp_stats},
+ {.name = "net.inet.udp.stats", .dim = "udp_stats", .enabled = 1, .func = do_net_inet_udp_stats},
+ {.name = "net.inet.icmp.stats", .dim = "icmp_stats", .enabled = 1, .func = do_net_inet_icmp_stats},
+ {.name = "net.inet.ip.stats", .dim = "ip_stats", .enabled = 1, .func = do_net_inet_ip_stats},
+ {.name = "net.inet6.ip6.stats", .dim = "ip6_stats", .enabled = 1, .func = do_net_inet6_ip6_stats},
+ {.name = "net.inet6.icmp6.stats", .dim = "icmp6_stats", .enabled = 1, .func = do_net_inet6_icmp6_stats},
+
+ // network interfaces metrics
+ {.name = "getifaddrs", .dim = "getifaddrs", .enabled = 1, .func = do_getifaddrs},
+
+ // ZFS metrics
+ {.name = "kstat.zfs.misc.arcstats", .dim = "arcstats", .enabled = 1, .func = do_kstat_zfs_misc_arcstats},
+ {.name = "kstat.zfs.misc.zio_trim", .dim = "trim", .enabled = 1, .func = do_kstat_zfs_misc_zio_trim},
+
+ // ipfw metrics
+ {.name = "ipfw", .dim = "ipfw", .enabled = 1, .func = do_ipfw},
+
+ // the terminator of this array
+ {.name = NULL, .dim = NULL, .enabled = 0, .func = NULL}
+};
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 33
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 33
+#endif
+
+static void freebsd_main_cleanup(void *ptr)
+{
+ worker_unregister();
+
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ info("cleaning up...");
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+void *freebsd_main(void *ptr)
+{
+ worker_register("FREEBSD");
+
+ netdata_thread_cleanup_push(freebsd_main_cleanup, ptr);
+
+ // initialize FreeBSD plugin
+ if (freebsd_plugin_init())
+ netdata_cleanup_and_exit(1);
+
+ // check the enabled status for each module
+ int i;
+ for (i = 0; freebsd_modules[i].name; i++) {
+ struct freebsd_module *pm = &freebsd_modules[i];
+
+ pm->enabled = config_get_boolean("plugin:freebsd", pm->name, pm->enabled);
+ pm->rd = NULL;
+
+ worker_register_job_name(i, freebsd_modules[i].dim);
+ }
+
+ usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ while (!netdata_exit) {
+ worker_is_idle();
+
+ usec_t hb_dt = heartbeat_next(&hb, step);
+
+ if (unlikely(netdata_exit))
+ break;
+
+ for (i = 0; freebsd_modules[i].name; i++) {
+ struct freebsd_module *pm = &freebsd_modules[i];
+ if (unlikely(!pm->enabled))
+ continue;
+
+ debug(D_PROCNETDEV_LOOP, "FREEBSD calling %s.", pm->name);
+
+ worker_is_busy(i);
+ pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
+
+ if (unlikely(netdata_exit))
+ break;
+ }
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/freebsd.plugin/plugin_freebsd.h b/collectors/freebsd.plugin/plugin_freebsd.h
new file mode 100644
index 0000000..af7d082
--- /dev/null
+++ b/collectors/freebsd.plugin/plugin_freebsd.h
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PLUGIN_FREEBSD_H
+#define NETDATA_PLUGIN_FREEBSD_H 1
+
+#include "daemon/common.h"
+
+#include <sys/sysctl.h>
+
+#define KILO_FACTOR 1024
+#define MEGA_FACTOR 1048576 // 1024 * 1024
+#define GIGA_FACTOR 1073741824 // 1024 * 1024 * 1024
+
+#define MAX_INT_DIGITS 10 // maximum number of digits for int
+
+int freebsd_plugin_init();
+
+int do_vm_loadavg(int update_every, usec_t dt);
+int do_vm_vmtotal(int update_every, usec_t dt);
+int do_kern_cp_time(int update_every, usec_t dt);
+int do_kern_cp_times(int update_every, usec_t dt);
+int do_dev_cpu_temperature(int update_every, usec_t dt);
+int do_dev_cpu_0_freq(int update_every, usec_t dt);
+int do_hw_intcnt(int update_every, usec_t dt);
+int do_vm_stats_sys_v_intr(int update_every, usec_t dt);
+int do_vm_stats_sys_v_soft(int update_every, usec_t dt);
+int do_vm_stats_sys_v_swtch(int update_every, usec_t dt);
+int do_vm_stats_sys_v_forks(int update_every, usec_t dt);
+int do_vm_swap_info(int update_every, usec_t dt);
+int do_system_ram(int update_every, usec_t dt);
+int do_vm_stats_sys_v_swappgs(int update_every, usec_t dt);
+int do_vm_stats_sys_v_pgfaults(int update_every, usec_t dt);
+int do_kern_ipc_sem(int update_every, usec_t dt);
+int do_kern_ipc_shm(int update_every, usec_t dt);
+int do_kern_ipc_msq(int update_every, usec_t dt);
+int do_uptime(int update_every, usec_t dt);
+int do_net_isr(int update_every, usec_t dt);
+int do_net_inet_tcp_states(int update_every, usec_t dt);
+int do_net_inet_tcp_stats(int update_every, usec_t dt);
+int do_net_inet_udp_stats(int update_every, usec_t dt);
+int do_net_inet_icmp_stats(int update_every, usec_t dt);
+int do_net_inet_ip_stats(int update_every, usec_t dt);
+int do_net_inet6_ip6_stats(int update_every, usec_t dt);
+int do_net_inet6_icmp6_stats(int update_every, usec_t dt);
+int do_getifaddrs(int update_every, usec_t dt);
+int do_getmntinfo(int update_every, usec_t dt);
+int do_kern_devstat(int update_every, usec_t dt);
+int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt);
+int do_kstat_zfs_misc_zio_trim(int update_every, usec_t dt);
+int do_ipfw(int update_every, usec_t dt);
+
+// metrics that need to be shared among data collectors
+extern unsigned long long zfs_arcstats_shrinkable_cache_size_bytes;
+
+#endif /* NETDATA_PLUGIN_FREEBSD_H */
diff --git a/collectors/freeipmi.plugin/Makefile.am b/collectors/freeipmi.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/freeipmi.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/freeipmi.plugin/README.md b/collectors/freeipmi.plugin/README.md
new file mode 100644
index 0000000..ff13717
--- /dev/null
+++ b/collectors/freeipmi.plugin/README.md
@@ -0,0 +1,202 @@
+<!--
+title: "freeipmi.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/freeipmi.plugin/README.md
+-->
+
+# freeipmi.plugin
+
+Netdata has a [freeipmi](https://www.gnu.org/software/freeipmi/) plugin.
+
+> FreeIPMI provides in-band and out-of-band IPMI software based on the IPMI v1.5/2.0 specification. The IPMI specification defines a set of interfaces for platform management and is implemented by a number vendors for system management. The features of IPMI that most users will be interested in are sensor monitoring, system event monitoring, power control, and serial-over-LAN (SOL).
+
+## Installing the FreeIPMI plugin
+
+When using our official DEB/RPM packages, the FreeIPMI plugin is included in a separate package named
+`netdata-plugin-freeipmi` which needs to be manually installed using your system package manager. It is not
+installed automatically due to the large number of dependencies it requires.
+
+When using a static build of Netdata, the FreeIPMI plugin will be included and installed automatically, though
+you will still need to have FreeIPMI installed on your system to be able to use the plugin.
+
+When using a local build of Netdata, you need to ensure that the FreeIPMI development packages (typically called `libipmimonitoring-dev`, `libipmimonitoring-devel`, or `freeipmi-devel`) are installed when building Netdata.
+
+### Special Considerations
+
+Accessing IPMI requires root access, so the FreeIPMI plugin is automatically installed setuid root.
+
+FreeIPMI does not work correctly on IBM POWER systems, thus Netdata’s FreeIPMI plugin is not usable on such systems.
+
+If you have not previously used IPMI on your system, you will probably need to run the `ipmimonitoring` command as root to initiailze IPMI settings so that the Netdata plugin works correctly. It should return information about available seensors on the system.
+
+In some distributions `libipmimonitoring.pc` is located in a non-standard directory, which
+can cause building the plugin to fail when building Netdata from source. In that case you
+should find the file and link it to the standard pkg-config directory. Usually, running `sudo ln -s
+/usr/lib/$(uname -m)-linux-gnu/pkgconfig/libipmimonitoring.pc/libipmimonitoring.pc /usr/lib/pkgconfig/libipmimonitoring.pc`
+resolves this issue.
+
+## Netdata use
+
+The plugin creates (up to) 8 charts, based on the information collected from IPMI:
+
+1. number of sensors by state
+2. number of events in SEL
+3. Temperatures CELSIUS
+4. Temperatures FAHRENHEIT
+5. Voltages
+6. Currents
+7. Power
+8. Fans
+
+It also adds 2 alarms:
+
+1. Sensors in non-nominal state (i.e. warning and critical)
+2. SEL is non empty
+
+![image](https://cloud.githubusercontent.com/assets/2662304/23674138/88926a20-037d-11e7-89c0-20e74ee10cd1.png)
+
+The plugin does a speed test when it starts, to find out the duration needed by the IPMI processor to respond. Depending on the speed of your IPMI processor, charts may need several seconds to show up on the dashboard.
+
+## `freeipmi.plugin` configuration
+
+The plugin supports a few options. To see them, run:
+
+```text
+# /usr/libexec/netdata/plugins.d/freeipmi.plugin -h
+
+ netdata freeipmi.plugin 1.8.0-546-g72ce5d6b_rolling
+ Copyright (C) 2016-2017 Costa Tsaousis <costa@tsaousis.gr>
+ Released under GNU General Public License v3 or later.
+ All rights reserved.
+
+ This program is a data collector plugin for netdata.
+
+ Available command line options:
+
+ SECONDS data collection frequency
+ minimum: 5
+
+ debug enable verbose output
+ default: disabled
+
+ sel
+ no-sel enable/disable SEL collection
+ default: enabled
+
+ hostname HOST
+ username USER
+ password PASS connect to remote IPMI host
+ default: local IPMI processor
+
+ noauthcodecheck don't check the authentication codes returned
+
+ driver-type IPMIDRIVER
+ Specify the driver type to use instead of doing an auto selection.
+ The currently available outofband drivers are LAN and LAN_2_0,
+ which perform IPMI 1.5 and IPMI 2.0 respectively.
+ The currently available inband drivers are KCS, SSIF, OPENIPMI and SUNBMC.
+
+ sdr-cache-dir PATH directory for SDR cache files
+ default: /tmp
+
+ sensor-config-file FILE filename to read sensor configuration
+ default: system default
+
+ ignore N1,N2,N3,... sensor IDs to ignore
+ default: none
+
+ -v
+ -V
+ version print version and exit
+
+ Linux kernel module for IPMI is CPU hungry.
+ On Linux run this to lower kipmiN CPU utilization:
+ # echo 10 > /sys/module/ipmi_si/parameters/kipmid_max_busy_us
+
+ or create: /etc/modprobe.d/ipmi.conf with these contents:
+ options ipmi_si kipmid_max_busy_us=10
+
+ For more information:
+ https://github.com/netdata/netdata/tree/master/collectors/freeipmi.plugin
+```
+
+You can set these options in `/etc/netdata/netdata.conf` at this section:
+
+```
+[plugin:freeipmi]
+ update every = 5
+ command options =
+```
+
+Append to `command options =` the settings you need. The minimum `update every` is 5 (enforced internally by the plugin). IPMI is slow and CPU hungry. So, once every 5 seconds is pretty acceptable.
+
+## Ignoring specific sensors
+
+Specific sensor IDs can be excluded from freeipmi tools by editing `/etc/freeipmi/freeipmi.conf` and setting the IDs to be ignored at `ipmi-sensors-exclude-record-ids`. **However this file is not used by `libipmimonitoring`** (the library used by Netdata's `freeipmi.plugin`).
+
+So, `freeipmi.plugin` supports the option `ignore` that accepts a comma separated list of sensor IDs to ignore. To configure it, edit `/etc/netdata/netdata.conf` and set:
+
+```
+[plugin:freeipmi]
+ command options = ignore 1,2,3,4,...
+```
+
+To find the IDs to ignore, run the command `ipmimonitoring`. The first column is the wanted ID:
+
+```
+ID | Name | Type | State | Reading | Units | Event
+1 | Ambient Temp | Temperature | Nominal | 26.00 | C | 'OK'
+2 | Altitude | Other Units Based Sensor | Nominal | 480.00 | ft | 'OK'
+3 | Avg Power | Current | Nominal | 100.00 | W | 'OK'
+4 | Planar 3.3V | Voltage | Nominal | 3.29 | V | 'OK'
+5 | Planar 5V | Voltage | Nominal | 4.90 | V | 'OK'
+6 | Planar 12V | Voltage | Nominal | 11.99 | V | 'OK'
+7 | Planar VBAT | Voltage | Nominal | 2.95 | V | 'OK'
+8 | Fan 1A Tach | Fan | Nominal | 3132.00 | RPM | 'OK'
+9 | Fan 1B Tach | Fan | Nominal | 2150.00 | RPM | 'OK'
+10 | Fan 2A Tach | Fan | Nominal | 2494.00 | RPM | 'OK'
+11 | Fan 2B Tach | Fan | Nominal | 1825.00 | RPM | 'OK'
+12 | Fan 3A Tach | Fan | Nominal | 3538.00 | RPM | 'OK'
+13 | Fan 3B Tach | Fan | Nominal | 2625.00 | RPM | 'OK'
+14 | Fan 1 | Entity Presence | Nominal | N/A | N/A | 'Entity Present'
+15 | Fan 2 | Entity Presence | Nominal | N/A | N/A | 'Entity Present'
+...
+```
+
+## Debugging
+
+You can run the plugin by hand:
+
+```sh
+# become user netdata
+sudo su -s /bin/sh netdata
+
+# run the plugin in debug mode
+/usr/libexec/netdata/plugins.d/freeipmi.plugin 5 debug
+```
+
+You will get verbose output on what the plugin does.
+
+## kipmi0 CPU usage
+
+There have been reports that kipmi is showing increased CPU when the IPMI is queried. To lower the CPU consumption of
+the system you can issue this command:
+
+```sh
+echo 10 > /sys/module/ipmi_si/parameters/kipmid_max_busy_us
+```
+
+You can also permanently set the above setting by creating the file `/etc/modprobe.d/ipmi.conf` with this content:
+
+```sh
+# prevent kipmi from consuming 100% CPU
+options ipmi_si kipmid_max_busy_us=10
+```
+
+This instructs the kernel IPMI module to pause for a tick between checking IPMI. Querying IPMI will be a lot slower now (e.g. several seconds for IPMI to respond), but `kipmi` will not use any noticeable CPU. You can also use a higher number (this is the number of microseconds to poll IPMI for a response, before waiting for a tick).
+
+If you need to disable IPMI for Netdata, edit `/etc/netdata/netdata.conf` and set:
+
+```
+[plugins]
+ freeipmi = no
+```
diff --git a/collectors/freeipmi.plugin/freeipmi_plugin.c b/collectors/freeipmi.plugin/freeipmi_plugin.c
new file mode 100644
index 0000000..351b6e3
--- /dev/null
+++ b/collectors/freeipmi.plugin/freeipmi_plugin.c
@@ -0,0 +1,1857 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+/*
+ * netdata freeipmi.plugin
+ * Copyright (C) 2017 Costa Tsaousis
+ * GPL v3+
+ *
+ * Based on:
+ * ipmimonitoring-sensors.c,v 1.51 2016/11/02 23:46:24 chu11 Exp
+ * ipmimonitoring-sel.c,v 1.51 2016/11/02 23:46:24 chu11 Exp
+ *
+ * Copyright (C) 2007-2015 Lawrence Livermore National Security, LLC.
+ * Copyright (C) 2006-2007 The Regents of the University of California.
+ * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+ * Written by Albert Chu <chu11@llnl.gov>
+ * UCRL-CODE-222073
+ */
+
+#include "libnetdata/libnetdata.h"
+#include "libnetdata/required_dummies.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/time.h>
+
+#define IPMI_PARSE_DEVICE_LAN_STR "lan"
+#define IPMI_PARSE_DEVICE_LAN_2_0_STR "lan_2_0"
+#define IPMI_PARSE_DEVICE_LAN_2_0_STR2 "lan20"
+#define IPMI_PARSE_DEVICE_LAN_2_0_STR3 "lan_20"
+#define IPMI_PARSE_DEVICE_LAN_2_0_STR4 "lan2_0"
+#define IPMI_PARSE_DEVICE_LAN_2_0_STR5 "lanplus"
+#define IPMI_PARSE_DEVICE_KCS_STR "kcs"
+#define IPMI_PARSE_DEVICE_SSIF_STR "ssif"
+#define IPMI_PARSE_DEVICE_OPENIPMI_STR "openipmi"
+#define IPMI_PARSE_DEVICE_OPENIPMI_STR2 "open"
+#define IPMI_PARSE_DEVICE_SUNBMC_STR "sunbmc"
+#define IPMI_PARSE_DEVICE_SUNBMC_STR2 "bmc"
+#define IPMI_PARSE_DEVICE_INTELDCMI_STR "inteldcmi"
+
+#include <ipmi_monitoring.h>
+#include <ipmi_monitoring_bitmasks.h>
+
+/* Communication Configuration - Initialize accordingly */
+
+/* Hostname, NULL for In-band communication, non-null for a hostname */
+char *hostname = NULL;
+
+/* In-band Communication Configuration */
+int driver_type = -1; // IPMI_MONITORING_DRIVER_TYPE_KCS; /* or -1 for default */
+int disable_auto_probe = 0; /* probe for in-band device */
+unsigned int driver_address = 0; /* not used if probing */
+unsigned int register_spacing = 0; /* not used if probing */
+char *driver_device = NULL; /* not used if probing */
+
+/* Out-of-band Communication Configuration */
+int protocol_version = -1; //IPMI_MONITORING_PROTOCOL_VERSION_1_5; /* or -1 for default */
+char *username = "foousername";
+char *password = "foopassword";
+unsigned char *ipmi_k_g = NULL;
+unsigned int ipmi_k_g_len = 0;
+int privilege_level = -1; // IPMI_MONITORING_PRIVILEGE_LEVEL_USER; /* or -1 for default */
+int authentication_type = -1; // IPMI_MONITORING_AUTHENTICATION_TYPE_MD5; /* or -1 for default */
+int cipher_suite_id = 0; /* or -1 for default */
+int session_timeout = 0; /* 0 for default */
+int retransmission_timeout = 0; /* 0 for default */
+
+/* Workarounds - specify workaround flags if necessary */
+unsigned int workaround_flags = 0;
+
+/* Initialize w/ record id numbers to only monitor specific record ids */
+unsigned int record_ids[] = {0};
+unsigned int record_ids_length = 0;
+
+/* Initialize w/ sensor types to only monitor specific sensor types
+ * see ipmi_monitoring.h sensor types list.
+ */
+unsigned int sensor_types[] = {0};
+unsigned int sensor_types_length = 0;
+
+/* Set to an appropriate alternate if desired */
+char *sdr_cache_directory = "/tmp";
+char *sensor_config_file = NULL;
+
+/* Set to 1 or 0 to enable these sensor reading flags
+ * - See ipmi_monitoring.h for descriptions of these flags.
+ */
+int reread_sdr_cache = 0;
+int ignore_non_interpretable_sensors = 0;
+int bridge_sensors = 0;
+int interpret_oem_data = 0;
+int shared_sensors = 0;
+int discrete_reading = 1;
+int ignore_scanning_disabled = 0;
+int assume_bmc_owner = 0;
+int entity_sensor_names = 0;
+
+/* Initialization flags
+ *
+ * Most commonly bitwise OR IPMI_MONITORING_FLAGS_DEBUG and/or
+ * IPMI_MONITORING_FLAGS_DEBUG_IPMI_PACKETS for extra debugging
+ * information.
+ */
+unsigned int ipmimonitoring_init_flags = 0;
+
+int errnum;
+
+// ----------------------------------------------------------------------------
+// SEL only variables
+
+/* Initialize w/ date range to only monitoring specific date range */
+char *date_begin = NULL; /* use MM/DD/YYYY format */
+char *date_end = NULL; /* use MM/DD/YYYY format */
+
+int assume_system_event_record = 0;
+
+char *sel_config_file = NULL;
+
+
+// ----------------------------------------------------------------------------
+// functions common to sensors and SEL
+
+static void
+_init_ipmi_config (struct ipmi_monitoring_ipmi_config *ipmi_config)
+{
+ fatal_assert(ipmi_config);
+
+ ipmi_config->driver_type = driver_type;
+ ipmi_config->disable_auto_probe = disable_auto_probe;
+ ipmi_config->driver_address = driver_address;
+ ipmi_config->register_spacing = register_spacing;
+ ipmi_config->driver_device = driver_device;
+
+ ipmi_config->protocol_version = protocol_version;
+ ipmi_config->username = username;
+ ipmi_config->password = password;
+ ipmi_config->k_g = ipmi_k_g;
+ ipmi_config->k_g_len = ipmi_k_g_len;
+ ipmi_config->privilege_level = privilege_level;
+ ipmi_config->authentication_type = authentication_type;
+ ipmi_config->cipher_suite_id = cipher_suite_id;
+ ipmi_config->session_timeout_len = session_timeout;
+ ipmi_config->retransmission_timeout_len = retransmission_timeout;
+
+ ipmi_config->workaround_flags = workaround_flags;
+}
+
+#ifdef NETDATA_COMMENTED
+static const char *
+_get_sensor_type_string (int sensor_type)
+{
+ switch (sensor_type)
+ {
+ case IPMI_MONITORING_SENSOR_TYPE_RESERVED:
+ return ("Reserved");
+ case IPMI_MONITORING_SENSOR_TYPE_TEMPERATURE:
+ return ("Temperature");
+ case IPMI_MONITORING_SENSOR_TYPE_VOLTAGE:
+ return ("Voltage");
+ case IPMI_MONITORING_SENSOR_TYPE_CURRENT:
+ return ("Current");
+ case IPMI_MONITORING_SENSOR_TYPE_FAN:
+ return ("Fan");
+ case IPMI_MONITORING_SENSOR_TYPE_PHYSICAL_SECURITY:
+ return ("Physical Security");
+ case IPMI_MONITORING_SENSOR_TYPE_PLATFORM_SECURITY_VIOLATION_ATTEMPT:
+ return ("Platform Security Violation Attempt");
+ case IPMI_MONITORING_SENSOR_TYPE_PROCESSOR:
+ return ("Processor");
+ case IPMI_MONITORING_SENSOR_TYPE_POWER_SUPPLY:
+ return ("Power Supply");
+ case IPMI_MONITORING_SENSOR_TYPE_POWER_UNIT:
+ return ("Power Unit");
+ case IPMI_MONITORING_SENSOR_TYPE_COOLING_DEVICE:
+ return ("Cooling Device");
+ case IPMI_MONITORING_SENSOR_TYPE_OTHER_UNITS_BASED_SENSOR:
+ return ("Other Units Based Sensor");
+ case IPMI_MONITORING_SENSOR_TYPE_MEMORY:
+ return ("Memory");
+ case IPMI_MONITORING_SENSOR_TYPE_DRIVE_SLOT:
+ return ("Drive Slot");
+ case IPMI_MONITORING_SENSOR_TYPE_POST_MEMORY_RESIZE:
+ return ("POST Memory Resize");
+ case IPMI_MONITORING_SENSOR_TYPE_SYSTEM_FIRMWARE_PROGRESS:
+ return ("System Firmware Progress");
+ case IPMI_MONITORING_SENSOR_TYPE_EVENT_LOGGING_DISABLED:
+ return ("Event Logging Disabled");
+ case IPMI_MONITORING_SENSOR_TYPE_WATCHDOG1:
+ return ("Watchdog 1");
+ case IPMI_MONITORING_SENSOR_TYPE_SYSTEM_EVENT:
+ return ("System Event");
+ case IPMI_MONITORING_SENSOR_TYPE_CRITICAL_INTERRUPT:
+ return ("Critical Interrupt");
+ case IPMI_MONITORING_SENSOR_TYPE_BUTTON_SWITCH:
+ return ("Button/Switch");
+ case IPMI_MONITORING_SENSOR_TYPE_MODULE_BOARD:
+ return ("Module/Board");
+ case IPMI_MONITORING_SENSOR_TYPE_MICROCONTROLLER_COPROCESSOR:
+ return ("Microcontroller/Coprocessor");
+ case IPMI_MONITORING_SENSOR_TYPE_ADD_IN_CARD:
+ return ("Add In Card");
+ case IPMI_MONITORING_SENSOR_TYPE_CHASSIS:
+ return ("Chassis");
+ case IPMI_MONITORING_SENSOR_TYPE_CHIP_SET:
+ return ("Chip Set");
+ case IPMI_MONITORING_SENSOR_TYPE_OTHER_FRU:
+ return ("Other Fru");
+ case IPMI_MONITORING_SENSOR_TYPE_CABLE_INTERCONNECT:
+ return ("Cable/Interconnect");
+ case IPMI_MONITORING_SENSOR_TYPE_TERMINATOR:
+ return ("Terminator");
+ case IPMI_MONITORING_SENSOR_TYPE_SYSTEM_BOOT_INITIATED:
+ return ("System Boot Initiated");
+ case IPMI_MONITORING_SENSOR_TYPE_BOOT_ERROR:
+ return ("Boot Error");
+ case IPMI_MONITORING_SENSOR_TYPE_OS_BOOT:
+ return ("OS Boot");
+ case IPMI_MONITORING_SENSOR_TYPE_OS_CRITICAL_STOP:
+ return ("OS Critical Stop");
+ case IPMI_MONITORING_SENSOR_TYPE_SLOT_CONNECTOR:
+ return ("Slot/Connector");
+ case IPMI_MONITORING_SENSOR_TYPE_SYSTEM_ACPI_POWER_STATE:
+ return ("System ACPI Power State");
+ case IPMI_MONITORING_SENSOR_TYPE_WATCHDOG2:
+ return ("Watchdog 2");
+ case IPMI_MONITORING_SENSOR_TYPE_PLATFORM_ALERT:
+ return ("Platform Alert");
+ case IPMI_MONITORING_SENSOR_TYPE_ENTITY_PRESENCE:
+ return ("Entity Presence");
+ case IPMI_MONITORING_SENSOR_TYPE_MONITOR_ASIC_IC:
+ return ("Monitor ASIC/IC");
+ case IPMI_MONITORING_SENSOR_TYPE_LAN:
+ return ("LAN");
+ case IPMI_MONITORING_SENSOR_TYPE_MANAGEMENT_SUBSYSTEM_HEALTH:
+ return ("Management Subsystem Health");
+ case IPMI_MONITORING_SENSOR_TYPE_BATTERY:
+ return ("Battery");
+ case IPMI_MONITORING_SENSOR_TYPE_SESSION_AUDIT:
+ return ("Session Audit");
+ case IPMI_MONITORING_SENSOR_TYPE_VERSION_CHANGE:
+ return ("Version Change");
+ case IPMI_MONITORING_SENSOR_TYPE_FRU_STATE:
+ return ("FRU State");
+ }
+
+ return ("Unrecognized");
+}
+#endif // NETDATA_COMMENTED
+
+
+// ----------------------------------------------------------------------------
+// BEGIN NETDATA CODE
+
+static int debug = 0;
+
+static int netdata_update_every = 5; // this is the minimum update frequency
+static int netdata_priority = 90000;
+static int netdata_do_sel = 1;
+
+static size_t netdata_sensors_updated = 0;
+static size_t netdata_sensors_collected = 0;
+static size_t netdata_sel_events = 0;
+static size_t netdata_sensors_states_nominal = 0;
+static size_t netdata_sensors_states_warning = 0;
+static size_t netdata_sensors_states_critical = 0;
+
+struct sensor {
+ int record_id;
+ int sensor_number;
+ int sensor_type;
+ int sensor_state;
+ int sensor_units;
+ char *sensor_name;
+
+ int sensor_reading_type;
+ union {
+ uint8_t bool_value;
+ uint32_t uint32_value;
+ double double_value;
+ } sensor_reading;
+
+ int sent;
+ int ignore;
+ int exposed;
+ int updated;
+ struct sensor *next;
+} *sensors_root = NULL;
+
+static void netdata_mark_as_not_updated() {
+ struct sensor *sn;
+ for(sn = sensors_root; sn ;sn = sn->next)
+ sn->updated = sn->sent = 0;
+
+ netdata_sensors_updated = 0;
+ netdata_sensors_collected = 0;
+ netdata_sel_events = 0;
+
+ netdata_sensors_states_nominal = 0;
+ netdata_sensors_states_warning = 0;
+ netdata_sensors_states_critical = 0;
+}
+
+static void send_chart_to_netdata_for_units(int units) {
+ struct sensor *sn, *sn_stored;
+ int dupfound, multiplier;
+
+ switch(units) {
+ case IPMI_MONITORING_SENSOR_UNITS_CELSIUS:
+ printf("CHART ipmi.temperatures_c '' 'System Celsius Temperatures read by IPMI' 'Celsius' 'temperatures' 'ipmi.temperatures_c' 'line' %d %d\n"
+ , netdata_priority + 10
+ , netdata_update_every
+ );
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_FAHRENHEIT:
+ printf("CHART ipmi.temperatures_f '' 'System Fahrenheit Temperatures read by IPMI' 'Fahrenheit' 'temperatures' 'ipmi.temperatures_f' 'line' %d %d\n"
+ , netdata_priority + 11
+ , netdata_update_every
+ );
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_VOLTS:
+ printf("CHART ipmi.volts '' 'System Voltages read by IPMI' 'Volts' 'voltages' 'ipmi.voltages' 'line' %d %d\n"
+ , netdata_priority + 12
+ , netdata_update_every
+ );
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_AMPS:
+ printf("CHART ipmi.amps '' 'System Current read by IPMI' 'Amps' 'current' 'ipmi.amps' 'line' %d %d\n"
+ , netdata_priority + 13
+ , netdata_update_every
+ );
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_RPM:
+ printf("CHART ipmi.rpm '' 'System Fans read by IPMI' 'RPM' 'fans' 'ipmi.rpm' 'line' %d %d\n"
+ , netdata_priority + 14
+ , netdata_update_every
+ );
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_WATTS:
+ printf("CHART ipmi.watts '' 'System Power read by IPMI' 'Watts' 'power' 'ipmi.watts' 'line' %d %d\n"
+ , netdata_priority + 5
+ , netdata_update_every
+ );
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_PERCENT:
+ printf("CHART ipmi.percent '' 'System Metrics read by IPMI' '%%' 'other' 'ipmi.percent' 'line' %d %d\n"
+ , netdata_priority + 15
+ , netdata_update_every
+ );
+ break;
+
+ default:
+ for(sn = sensors_root; sn; sn = sn->next)
+ if(sn->sensor_units == units)
+ sn->ignore = 1;
+ return;
+ }
+
+ for(sn = sensors_root; sn; sn = sn->next) {
+ dupfound = 0;
+ if(sn->sensor_units == units && sn->updated && !sn->ignore) {
+ sn->exposed = 1;
+ multiplier = 1;
+
+ switch(sn->sensor_reading_type) {
+ case IPMI_MONITORING_SENSOR_READING_TYPE_DOUBLE:
+ multiplier = 1000;
+ // fallthrough
+ case IPMI_MONITORING_SENSOR_READING_TYPE_UNSIGNED_INTEGER8_BOOL:
+ case IPMI_MONITORING_SENSOR_READING_TYPE_UNSIGNED_INTEGER32:
+ for (sn_stored = sensors_root; sn_stored; sn_stored = sn_stored->next) {
+ if (sn_stored == sn) continue;
+ // If the name is a duplicate, append the sensor number
+ if ( !strcmp(sn_stored->sensor_name, sn->sensor_name) ) {
+ dupfound = 1;
+ printf("DIMENSION i%d_n%d_r%d '%s i%d' absolute 1 %d\n"
+ , sn->sensor_number
+ , sn->record_id
+ , sn->sensor_reading_type
+ , sn->sensor_name
+ , sn->sensor_number
+ , multiplier
+ );
+ break;
+ }
+ }
+ // No duplicate name was found, display it just with Name
+ if (!dupfound) {
+ // display without ID
+ printf("DIMENSION i%d_n%d_r%d '%s' absolute 1 %d\n"
+ , sn->sensor_number
+ , sn->record_id
+ , sn->sensor_reading_type
+ , sn->sensor_name
+ , multiplier
+ );
+ }
+ break;
+
+ default:
+ sn->ignore = 1;
+ break;
+ }
+ }
+ }
+}
+
+static void send_metrics_to_netdata_for_units(int units) {
+ struct sensor *sn;
+
+ switch(units) {
+ case IPMI_MONITORING_SENSOR_UNITS_CELSIUS:
+ printf("BEGIN ipmi.temperatures_c\n");
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_FAHRENHEIT:
+ printf("BEGIN ipmi.temperatures_f\n");
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_VOLTS:
+ printf("BEGIN ipmi.volts\n");
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_AMPS:
+ printf("BEGIN ipmi.amps\n");
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_RPM:
+ printf("BEGIN ipmi.rpm\n");
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_WATTS:
+ printf("BEGIN ipmi.watts\n");
+ break;
+
+ case IPMI_MONITORING_SENSOR_UNITS_PERCENT:
+ printf("BEGIN ipmi.percent\n");
+ break;
+
+ default:
+ for(sn = sensors_root; sn; sn = sn->next)
+ if(sn->sensor_units == units)
+ sn->ignore = 1;
+ return;
+ }
+
+ for(sn = sensors_root; sn; sn = sn->next) {
+ if(sn->sensor_units == units && sn->updated && !sn->sent && !sn->ignore) {
+ netdata_sensors_updated++;
+
+ sn->sent = 1;
+
+ switch(sn->sensor_reading_type) {
+ case IPMI_MONITORING_SENSOR_READING_TYPE_UNSIGNED_INTEGER8_BOOL:
+ printf("SET i%d_n%d_r%d = %u\n"
+ , sn->sensor_number
+ , sn->record_id
+ , sn->sensor_reading_type
+ , sn->sensor_reading.bool_value
+ );
+ break;
+
+ case IPMI_MONITORING_SENSOR_READING_TYPE_UNSIGNED_INTEGER32:
+ printf("SET i%d_n%d_r%d = %u\n"
+ , sn->sensor_number
+ , sn->record_id
+ , sn->sensor_reading_type
+ , sn->sensor_reading.uint32_value
+ );
+ break;
+
+ case IPMI_MONITORING_SENSOR_READING_TYPE_DOUBLE:
+ printf("SET i%d_n%d_r%d = %lld\n"
+ , sn->sensor_number
+ , sn->record_id
+ , sn->sensor_reading_type
+ , (long long int)(sn->sensor_reading.double_value * 1000)
+ );
+ break;
+
+ default:
+ sn->ignore = 1;
+ break;
+ }
+ }
+ }
+
+ printf("END\n");
+}
+
+static void send_metrics_to_netdata() {
+ static int sel_chart_generated = 0, sensors_states_chart_generated = 0;
+ struct sensor *sn;
+
+ if(netdata_do_sel && !sel_chart_generated) {
+ sel_chart_generated = 1;
+ printf("CHART ipmi.events '' 'IPMI Events' 'events' 'events' ipmi.sel area %d %d\n"
+ , netdata_priority + 2
+ , netdata_update_every
+ );
+ printf("DIMENSION events '' absolute 1 1\n");
+ }
+
+ if(!sensors_states_chart_generated) {
+ sensors_states_chart_generated = 1;
+ printf("CHART ipmi.sensors_states '' 'IPMI Sensors State' 'sensors' 'states' ipmi.sensors_states line %d %d\n"
+ , netdata_priority + 1
+ , netdata_update_every
+ );
+ printf("DIMENSION nominal '' absolute 1 1\n");
+ printf("DIMENSION critical '' absolute 1 1\n");
+ printf("DIMENSION warning '' absolute 1 1\n");
+ }
+
+ // generate the CHART/DIMENSION lines, if we have to
+ for(sn = sensors_root; sn; sn = sn->next)
+ if(sn->updated && !sn->exposed && !sn->ignore)
+ send_chart_to_netdata_for_units(sn->sensor_units);
+
+ if(netdata_do_sel) {
+ printf(
+ "BEGIN ipmi.events\n"
+ "SET events = %zu\n"
+ "END\n"
+ , netdata_sel_events
+ );
+ }
+
+ printf(
+ "BEGIN ipmi.sensors_states\n"
+ "SET nominal = %zu\n"
+ "SET warning = %zu\n"
+ "SET critical = %zu\n"
+ "END\n"
+ , netdata_sensors_states_nominal
+ , netdata_sensors_states_warning
+ , netdata_sensors_states_critical
+ );
+
+ // send metrics to netdata
+ for(sn = sensors_root; sn; sn = sn->next)
+ if(sn->updated && sn->exposed && !sn->sent && !sn->ignore)
+ send_metrics_to_netdata_for_units(sn->sensor_units);
+
+}
+
+static int *excluded_record_ids = NULL;
+size_t excluded_record_ids_length = 0;
+
+static void excluded_record_ids_parse(const char *s) {
+ if(!s) return;
+
+ while(*s) {
+ while(*s && !isdigit(*s)) s++;
+
+ if(isdigit(*s)) {
+ char *e;
+ unsigned long n = strtoul(s, &e, 10);
+ s = e;
+
+ if(n != 0) {
+ excluded_record_ids = realloc(excluded_record_ids, (excluded_record_ids_length + 1) * sizeof(int));
+ if(!excluded_record_ids) {
+ fprintf(stderr, "freeipmi.plugin: failed to allocate memory. Exiting.");
+ exit(1);
+ }
+ excluded_record_ids[excluded_record_ids_length++] = (int)n;
+ }
+ }
+ }
+
+ if(debug) {
+ fprintf(stderr, "freeipmi.plugin: excluded record ids:");
+ size_t i;
+ for(i = 0; i < excluded_record_ids_length; i++) {
+ fprintf(stderr, " %d", excluded_record_ids[i]);
+ }
+ fprintf(stderr, "\n");
+ }
+}
+
+static int *excluded_status_record_ids = NULL;
+size_t excluded_status_record_ids_length = 0;
+
+static void excluded_status_record_ids_parse(const char *s) {
+ if(!s) return;
+
+ while(*s) {
+ while(*s && !isdigit(*s)) s++;
+
+ if(isdigit(*s)) {
+ char *e;
+ unsigned long n = strtoul(s, &e, 10);
+ s = e;
+
+ if(n != 0) {
+ excluded_status_record_ids = realloc(excluded_status_record_ids, (excluded_status_record_ids_length + 1) * sizeof(int));
+ if(!excluded_status_record_ids) {
+ fprintf(stderr, "freeipmi.plugin: failed to allocate memory. Exiting.");
+ exit(1);
+ }
+ excluded_status_record_ids[excluded_status_record_ids_length++] = (int)n;
+ }
+ }
+ }
+
+ if(debug) {
+ fprintf(stderr, "freeipmi.plugin: excluded status record ids:");
+ size_t i;
+ for(i = 0; i < excluded_status_record_ids_length; i++) {
+ fprintf(stderr, " %d", excluded_status_record_ids[i]);
+ }
+ fprintf(stderr, "\n");
+ }
+}
+
+
+static int excluded_record_ids_check(int record_id) {
+ size_t i;
+
+ for(i = 0; i < excluded_record_ids_length; i++) {
+ if(excluded_record_ids[i] == record_id)
+ return 1;
+ }
+
+ return 0;
+}
+
+static int excluded_status_record_ids_check(int record_id) {
+ size_t i;
+
+ for(i = 0; i < excluded_status_record_ids_length; i++) {
+ if(excluded_status_record_ids[i] == record_id)
+ return 1;
+ }
+
+ return 0;
+}
+
+static void netdata_get_sensor(
+ int record_id
+ , int sensor_number
+ , int sensor_type
+ , int sensor_state
+ , int sensor_units
+ , int sensor_reading_type
+ , char *sensor_name
+ , void *sensor_reading
+) {
+ // find the sensor record
+ struct sensor *sn;
+ for(sn = sensors_root; sn ;sn = sn->next)
+ if( sn->record_id == record_id &&
+ sn->sensor_number == sensor_number &&
+ sn->sensor_reading_type == sensor_reading_type &&
+ sn->sensor_units == sensor_units &&
+ !strcmp(sn->sensor_name, sensor_name)
+ )
+ break;
+
+ if(!sn) {
+ // not found, create it
+ // check if it is excluded
+ if(excluded_record_ids_check(record_id)) {
+ if(debug) fprintf(stderr, "Sensor '%s' is excluded by excluded_record_ids_check()\n", sensor_name);
+ return;
+ }
+
+ if(debug) fprintf(stderr, "Allocating new sensor data record for sensor '%s', id %d, number %d, type %d, state %d, units %d, reading_type %d\n", sensor_name, record_id, sensor_number, sensor_type, sensor_state, sensor_units, sensor_reading_type);
+
+ sn = calloc(1, sizeof(struct sensor));
+ if(!sn) {
+ fatal("cannot allocate %zu bytes of memory.", sizeof(struct sensor));
+ }
+
+ sn->record_id = record_id;
+ sn->sensor_number = sensor_number;
+ sn->sensor_type = sensor_type;
+ sn->sensor_state = sensor_state;
+ sn->sensor_units = sensor_units;
+ sn->sensor_reading_type = sensor_reading_type;
+ sn->sensor_name = strdup(sensor_name);
+ if(!sn->sensor_name) {
+ fatal("cannot allocate %zu bytes of memory.", strlen(sensor_name));
+ }
+
+ sn->next = sensors_root;
+ sensors_root = sn;
+ }
+ else {
+ if(debug) fprintf(stderr, "Reusing sensor record for sensor '%s', id %d, number %d, type %d, state %d, units %d, reading_type %d\n", sensor_name, record_id, sensor_number, sensor_type, sensor_state, sensor_units, sensor_reading_type);
+ }
+
+ switch(sensor_reading_type) {
+ case IPMI_MONITORING_SENSOR_READING_TYPE_UNSIGNED_INTEGER8_BOOL:
+ sn->sensor_reading.bool_value = *((uint8_t *)sensor_reading);
+ sn->updated = 1;
+ netdata_sensors_collected++;
+ break;
+
+ case IPMI_MONITORING_SENSOR_READING_TYPE_UNSIGNED_INTEGER32:
+ sn->sensor_reading.uint32_value = *((uint32_t *)sensor_reading);
+ sn->updated = 1;
+ netdata_sensors_collected++;
+ break;
+
+ case IPMI_MONITORING_SENSOR_READING_TYPE_DOUBLE:
+ sn->sensor_reading.double_value = *((double *)sensor_reading);
+ sn->updated = 1;
+ netdata_sensors_collected++;
+ break;
+
+ default:
+ if(debug) fprintf(stderr, "Unknown reading type - Ignoring sensor record for sensor '%s', id %d, number %d, type %d, state %d, units %d, reading_type %d\n", sensor_name, record_id, sensor_number, sensor_type, sensor_state, sensor_units, sensor_reading_type);
+ sn->ignore = 1;
+ break;
+ }
+
+ // check if it is excluded
+ if(excluded_status_record_ids_check(record_id)) {
+ if(debug) fprintf(stderr, "Sensor '%s' is excluded for status check, by excluded_status_record_ids_check()\n", sensor_name);
+ return;
+ }
+
+ switch(sensor_state) {
+ case IPMI_MONITORING_STATE_NOMINAL:
+ netdata_sensors_states_nominal++;
+ break;
+
+ case IPMI_MONITORING_STATE_WARNING:
+ netdata_sensors_states_warning++;
+ break;
+
+ case IPMI_MONITORING_STATE_CRITICAL:
+ netdata_sensors_states_critical++;
+ break;
+
+ default:
+ break;
+ }
+}
+
+static void netdata_get_sel(
+ int record_id
+ , int record_type_class
+ , int sel_state
+) {
+ (void)record_id;
+ (void)record_type_class;
+ (void)sel_state;
+
+ netdata_sel_events++;
+}
+
+
+// END NETDATA CODE
+// ----------------------------------------------------------------------------
+
+
+static int
+_ipmimonitoring_sensors (struct ipmi_monitoring_ipmi_config *ipmi_config)
+{
+ ipmi_monitoring_ctx_t ctx = NULL;
+ unsigned int sensor_reading_flags = 0;
+ int i;
+ int sensor_count;
+ int rv = -1;
+
+ if (!(ctx = ipmi_monitoring_ctx_create ())) {
+ error("ipmi_monitoring_ctx_create()");
+ goto cleanup;
+ }
+
+ if (sdr_cache_directory)
+ {
+ if (ipmi_monitoring_ctx_sdr_cache_directory (ctx,
+ sdr_cache_directory) < 0)
+ {
+ error("ipmi_monitoring_ctx_sdr_cache_directory(): %s\n",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+
+ /* Must call otherwise only default interpretations ever used */
+ if (sensor_config_file)
+ {
+ if (ipmi_monitoring_ctx_sensor_config_file (ctx,
+ sensor_config_file) < 0)
+ {
+ error( "ipmi_monitoring_ctx_sensor_config_file(): %s\n",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+ else
+ {
+ if (ipmi_monitoring_ctx_sensor_config_file (ctx, NULL) < 0)
+ {
+ error( "ipmi_monitoring_ctx_sensor_config_file(): %s\n",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+
+ if (reread_sdr_cache)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_REREAD_SDR_CACHE;
+
+ if (ignore_non_interpretable_sensors)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_IGNORE_NON_INTERPRETABLE_SENSORS;
+
+ if (bridge_sensors)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_BRIDGE_SENSORS;
+
+ if (interpret_oem_data)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_INTERPRET_OEM_DATA;
+
+ if (shared_sensors)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_SHARED_SENSORS;
+
+ if (discrete_reading)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_DISCRETE_READING;
+
+ if (ignore_scanning_disabled)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_IGNORE_SCANNING_DISABLED;
+
+ if (assume_bmc_owner)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_ASSUME_BMC_OWNER;
+
+#ifdef IPMI_MONITORING_SENSOR_READING_FLAGS_ENTITY_SENSOR_NAMES
+ if (entity_sensor_names)
+ sensor_reading_flags |= IPMI_MONITORING_SENSOR_READING_FLAGS_ENTITY_SENSOR_NAMES;
+#endif // IPMI_MONITORING_SENSOR_READING_FLAGS_ENTITY_SENSOR_NAMES
+
+ if (!record_ids_length && !sensor_types_length)
+ {
+ if ((sensor_count = ipmi_monitoring_sensor_readings_by_record_id (ctx,
+ hostname,
+ ipmi_config,
+ sensor_reading_flags,
+ NULL,
+ 0,
+ NULL,
+ NULL)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_readings_by_record_id(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+ else if (record_ids_length)
+ {
+ if ((sensor_count = ipmi_monitoring_sensor_readings_by_record_id (ctx,
+ hostname,
+ ipmi_config,
+ sensor_reading_flags,
+ record_ids,
+ record_ids_length,
+ NULL,
+ NULL)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_readings_by_record_id(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+ else
+ {
+ if ((sensor_count = ipmi_monitoring_sensor_readings_by_sensor_type (ctx,
+ hostname,
+ ipmi_config,
+ sensor_reading_flags,
+ sensor_types,
+ sensor_types_length,
+ NULL,
+ NULL)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_readings_by_sensor_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+
+#ifdef NETDATA_COMMENTED
+ printf ("%s, %s, %s, %s, %s, %s, %s, %s, %s, %s\n",
+ "Record ID",
+ "Sensor Name",
+ "Sensor Number",
+ "Sensor Type",
+ "Sensor State",
+ "Sensor Reading",
+ "Sensor Units",
+ "Sensor Event/Reading Type Code",
+ "Sensor Event Bitmask",
+ "Sensor Event String");
+#endif // NETDATA_COMMENTED
+
+ for (i = 0; i < sensor_count; i++, ipmi_monitoring_sensor_iterator_next (ctx))
+ {
+ int record_id, sensor_number, sensor_type, sensor_state, sensor_units,
+ sensor_reading_type;
+
+#ifdef NETDATA_COMMENTED
+ int sensor_bitmask_type, sensor_bitmask, event_reading_type_code;
+ char **sensor_bitmask_strings = NULL;
+ const char *sensor_type_str;
+ const char *sensor_state_str;
+#endif // NETDATA_COMMENTED
+
+ char *sensor_name = NULL;
+ void *sensor_reading;
+
+ if ((record_id = ipmi_monitoring_sensor_read_record_id (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_read_record_id(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((sensor_number = ipmi_monitoring_sensor_read_sensor_number (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_read_sensor_number(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((sensor_type = ipmi_monitoring_sensor_read_sensor_type (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_read_sensor_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if (!(sensor_name = ipmi_monitoring_sensor_read_sensor_name (ctx)))
+ {
+ error( "ipmi_monitoring_sensor_read_sensor_name(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((sensor_state = ipmi_monitoring_sensor_read_sensor_state (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_read_sensor_state(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((sensor_units = ipmi_monitoring_sensor_read_sensor_units (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_read_sensor_units(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+#ifdef NETDATA_COMMENTED
+ if ((sensor_bitmask_type = ipmi_monitoring_sensor_read_sensor_bitmask_type (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_read_sensor_bitmask_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ if ((sensor_bitmask = ipmi_monitoring_sensor_read_sensor_bitmask (ctx)) < 0)
+ {
+ error(
+ "ipmi_monitoring_sensor_read_sensor_bitmask(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ /* it's ok for this to be NULL, i.e. sensor_bitmask ==
+ * IPMI_MONITORING_SENSOR_BITMASK_TYPE_UNKNOWN
+ */
+ sensor_bitmask_strings = ipmi_monitoring_sensor_read_sensor_bitmask_strings (ctx);
+
+
+
+#endif // NETDATA_COMMENTED
+
+ if ((sensor_reading_type = ipmi_monitoring_sensor_read_sensor_reading_type (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_read_sensor_reading_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ sensor_reading = ipmi_monitoring_sensor_read_sensor_reading (ctx);
+
+#ifdef NETDATA_COMMENTED
+ if ((event_reading_type_code = ipmi_monitoring_sensor_read_event_reading_type_code (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sensor_read_event_reading_type_code(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+#endif // NETDATA_COMMENTED
+
+ netdata_get_sensor(
+ record_id
+ , sensor_number
+ , sensor_type
+ , sensor_state
+ , sensor_units
+ , sensor_reading_type
+ , sensor_name
+ , sensor_reading
+ );
+
+#ifdef NETDATA_COMMENTED
+ if (!strlen (sensor_name))
+ sensor_name = "N/A";
+
+ sensor_type_str = _get_sensor_type_string (sensor_type);
+
+ printf ("%d, %s, %d, %s",
+ record_id,
+ sensor_name,
+ sensor_number,
+ sensor_type_str);
+
+ if (sensor_state == IPMI_MONITORING_STATE_NOMINAL)
+ sensor_state_str = "Nominal";
+ else if (sensor_state == IPMI_MONITORING_STATE_WARNING)
+ sensor_state_str = "Warning";
+ else if (sensor_state == IPMI_MONITORING_STATE_CRITICAL)
+ sensor_state_str = "Critical";
+ else
+ sensor_state_str = "N/A";
+
+ printf (", %s", sensor_state_str);
+
+ if (sensor_reading)
+ {
+ const char *sensor_units_str;
+
+ if (sensor_reading_type == IPMI_MONITORING_SENSOR_READING_TYPE_UNSIGNED_INTEGER8_BOOL)
+ printf (", %s",
+ (*((uint8_t *)sensor_reading) ? "true" : "false"));
+ else if (sensor_reading_type == IPMI_MONITORING_SENSOR_READING_TYPE_UNSIGNED_INTEGER32)
+ printf (", %u",
+ *((uint32_t *)sensor_reading));
+ else if (sensor_reading_type == IPMI_MONITORING_SENSOR_READING_TYPE_DOUBLE)
+ printf (", %.2f",
+ *((double *)sensor_reading));
+ else
+ printf (", N/A");
+
+ if (sensor_units == IPMI_MONITORING_SENSOR_UNITS_CELSIUS)
+ sensor_units_str = "C";
+ else if (sensor_units == IPMI_MONITORING_SENSOR_UNITS_FAHRENHEIT)
+ sensor_units_str = "F";
+ else if (sensor_units == IPMI_MONITORING_SENSOR_UNITS_VOLTS)
+ sensor_units_str = "V";
+ else if (sensor_units == IPMI_MONITORING_SENSOR_UNITS_AMPS)
+ sensor_units_str = "A";
+ else if (sensor_units == IPMI_MONITORING_SENSOR_UNITS_RPM)
+ sensor_units_str = "RPM";
+ else if (sensor_units == IPMI_MONITORING_SENSOR_UNITS_WATTS)
+ sensor_units_str = "W";
+ else if (sensor_units == IPMI_MONITORING_SENSOR_UNITS_PERCENT)
+ sensor_units_str = "%";
+ else
+ sensor_units_str = "N/A";
+
+ printf (", %s", sensor_units_str);
+ }
+ else
+ printf (", N/A, N/A");
+
+ printf (", %Xh", event_reading_type_code);
+
+ /* It is possible you may want to monitor specific event
+ * conditions that may occur. If that is the case, you may want
+ * to check out what specific bitmask type and bitmask events
+ * occurred. See ipmi_monitoring_bitmasks.h for a list of
+ * bitmasks and types.
+ */
+
+ if (sensor_bitmask_type != IPMI_MONITORING_SENSOR_BITMASK_TYPE_UNKNOWN)
+ printf (", %Xh", sensor_bitmask);
+ else
+ printf (", N/A");
+
+ if (sensor_bitmask_type != IPMI_MONITORING_SENSOR_BITMASK_TYPE_UNKNOWN
+ && sensor_bitmask_strings)
+ {
+ unsigned int i = 0;
+
+ printf (",");
+
+ while (sensor_bitmask_strings[i])
+ {
+ printf (" ");
+
+ printf ("'%s'",
+ sensor_bitmask_strings[i]);
+
+ i++;
+ }
+ }
+ else
+ printf (", N/A");
+
+ printf ("\n");
+#endif // NETDATA_COMMENTED
+ }
+
+ rv = 0;
+ cleanup:
+ if (ctx)
+ ipmi_monitoring_ctx_destroy (ctx);
+ return (rv);
+}
+
+
+static int
+_ipmimonitoring_sel (struct ipmi_monitoring_ipmi_config *ipmi_config)
+{
+ ipmi_monitoring_ctx_t ctx = NULL;
+ unsigned int sel_flags = 0;
+ int i;
+ int sel_count;
+ int rv = -1;
+
+ if (!(ctx = ipmi_monitoring_ctx_create ()))
+ {
+ error("ipmi_monitoring_ctx_create()");
+ goto cleanup;
+ }
+
+ if (sdr_cache_directory)
+ {
+ if (ipmi_monitoring_ctx_sdr_cache_directory (ctx,
+ sdr_cache_directory) < 0)
+ {
+ error( "ipmi_monitoring_ctx_sdr_cache_directory(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+
+ /* Must call otherwise only default interpretations ever used */
+ if (sel_config_file)
+ {
+ if (ipmi_monitoring_ctx_sel_config_file (ctx,
+ sel_config_file) < 0)
+ {
+ error( "ipmi_monitoring_ctx_sel_config_file(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+ else
+ {
+ if (ipmi_monitoring_ctx_sel_config_file (ctx, NULL) < 0)
+ {
+ error( "ipmi_monitoring_ctx_sel_config_file(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+
+ if (reread_sdr_cache)
+ sel_flags |= IPMI_MONITORING_SEL_FLAGS_REREAD_SDR_CACHE;
+
+ if (interpret_oem_data)
+ sel_flags |= IPMI_MONITORING_SEL_FLAGS_INTERPRET_OEM_DATA;
+
+ if (assume_system_event_record)
+ sel_flags |= IPMI_MONITORING_SEL_FLAGS_ASSUME_SYSTEM_EVENT_RECORD;
+
+#ifdef IPMI_MONITORING_SEL_FLAGS_ENTITY_SENSOR_NAMES
+ if (entity_sensor_names)
+ sel_flags |= IPMI_MONITORING_SEL_FLAGS_ENTITY_SENSOR_NAMES;
+#endif // IPMI_MONITORING_SEL_FLAGS_ENTITY_SENSOR_NAMES
+
+ if (record_ids_length)
+ {
+ if ((sel_count = ipmi_monitoring_sel_by_record_id (ctx,
+ hostname,
+ ipmi_config,
+ sel_flags,
+ record_ids,
+ record_ids_length,
+ NULL,
+ NULL)) < 0)
+ {
+ error( "ipmi_monitoring_sel_by_record_id(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+ else if (sensor_types_length)
+ {
+ if ((sel_count = ipmi_monitoring_sel_by_sensor_type (ctx,
+ hostname,
+ ipmi_config,
+ sel_flags,
+ sensor_types,
+ sensor_types_length,
+ NULL,
+ NULL)) < 0)
+ {
+ error( "ipmi_monitoring_sel_by_sensor_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+ else if (date_begin
+ || date_end)
+ {
+ if ((sel_count = ipmi_monitoring_sel_by_date_range (ctx,
+ hostname,
+ ipmi_config,
+ sel_flags,
+ date_begin,
+ date_end,
+ NULL,
+ NULL)) < 0)
+ {
+ error( "ipmi_monitoring_sel_by_sensor_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+ else
+ {
+ if ((sel_count = ipmi_monitoring_sel_by_record_id (ctx,
+ hostname,
+ ipmi_config,
+ sel_flags,
+ NULL,
+ 0,
+ NULL,
+ NULL)) < 0)
+ {
+ error( "ipmi_monitoring_sel_by_record_id(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+ }
+
+#ifdef NETDATA_COMMENTED
+ printf ("%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s\n",
+ "Record ID",
+ "Record Type",
+ "SEL State",
+ "Timestamp",
+ "Sensor Name",
+ "Sensor Type",
+ "Event Direction",
+ "Event Type Code",
+ "Event Data",
+ "Event Offset",
+ "Event Offset String");
+#endif // NETDATA_COMMENTED
+
+ for (i = 0; i < sel_count; i++, ipmi_monitoring_sel_iterator_next (ctx))
+ {
+ int record_id, record_type, sel_state, record_type_class;
+#ifdef NETDATA_COMMENTED
+ int sensor_type, sensor_number, event_direction,
+ event_offset_type, event_offset, event_type_code, manufacturer_id;
+ unsigned int timestamp, event_data1, event_data2, event_data3;
+ char *event_offset_string = NULL;
+ const char *sensor_type_str;
+ const char *event_direction_str;
+ const char *sel_state_str;
+ char *sensor_name = NULL;
+ unsigned char oem_data[64];
+ int oem_data_len;
+ unsigned int j;
+#endif // NETDATA_COMMENTED
+
+ if ((record_id = ipmi_monitoring_sel_read_record_id (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_record_id(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((record_type = ipmi_monitoring_sel_read_record_type (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_record_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((record_type_class = ipmi_monitoring_sel_read_record_type_class (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_record_type_class(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((sel_state = ipmi_monitoring_sel_read_sel_state (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_sel_state(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ netdata_get_sel(
+ record_id
+ , record_type_class
+ , sel_state
+ );
+
+#ifdef NETDATA_COMMENTED
+ if (sel_state == IPMI_MONITORING_STATE_NOMINAL)
+ sel_state_str = "Nominal";
+ else if (sel_state == IPMI_MONITORING_STATE_WARNING)
+ sel_state_str = "Warning";
+ else if (sel_state == IPMI_MONITORING_STATE_CRITICAL)
+ sel_state_str = "Critical";
+ else
+ sel_state_str = "N/A";
+
+ printf ("%d, %d, %s",
+ record_id,
+ record_type,
+ sel_state_str);
+
+ if (record_type_class == IPMI_MONITORING_SEL_RECORD_TYPE_CLASS_SYSTEM_EVENT_RECORD
+ || record_type_class == IPMI_MONITORING_SEL_RECORD_TYPE_CLASS_TIMESTAMPED_OEM_RECORD)
+ {
+
+ if (ipmi_monitoring_sel_read_timestamp (ctx, &timestamp) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_timestamp(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ /* XXX: This should be converted to a nice date output using
+ * your favorite timestamp -> string conversion functions.
+ */
+ printf (", %u", timestamp);
+ }
+ else
+ printf (", N/A");
+
+ if (record_type_class == IPMI_MONITORING_SEL_RECORD_TYPE_CLASS_SYSTEM_EVENT_RECORD)
+ {
+ /* If you are integrating ipmimonitoring SEL into a monitoring application,
+ * you may wish to count the number of times a specific error occurred
+ * and report that to the monitoring application.
+ *
+ * In this particular case, you'll probably want to check out
+ * what sensor type each SEL event is reporting, the
+ * event offset type, and the specific event offset that occurred.
+ *
+ * See ipmi_monitoring_offsets.h for a list of event offsets
+ * and types.
+ */
+
+ if (!(sensor_name = ipmi_monitoring_sel_read_sensor_name (ctx)))
+ {
+ error( "ipmi_monitoring_sel_read_sensor_name(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((sensor_type = ipmi_monitoring_sel_read_sensor_type (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_sensor_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((sensor_number = ipmi_monitoring_sel_read_sensor_number (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_sensor_number(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((event_direction = ipmi_monitoring_sel_read_event_direction (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_event_direction(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((event_type_code = ipmi_monitoring_sel_read_event_type_code (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_event_type_code(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if (ipmi_monitoring_sel_read_event_data (ctx,
+ &event_data1,
+ &event_data2,
+ &event_data3) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_event_data(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((event_offset_type = ipmi_monitoring_sel_read_event_offset_type (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_event_offset_type(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if ((event_offset = ipmi_monitoring_sel_read_event_offset (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_event_offset(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if (!(event_offset_string = ipmi_monitoring_sel_read_event_offset_string (ctx)))
+ {
+ error( "ipmi_monitoring_sel_read_event_offset_string(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ if (!strlen (sensor_name))
+ sensor_name = "N/A";
+
+ sensor_type_str = _get_sensor_type_string (sensor_type);
+
+ if (event_direction == IPMI_MONITORING_SEL_EVENT_DIRECTION_ASSERTION)
+ event_direction_str = "Assertion";
+ else
+ event_direction_str = "Deassertion";
+
+ printf (", %s, %s, %d, %s, %Xh, %Xh-%Xh-%Xh",
+ sensor_name,
+ sensor_type_str,
+ sensor_number,
+ event_direction_str,
+ event_type_code,
+ event_data1,
+ event_data2,
+ event_data3);
+
+ if (event_offset_type != IPMI_MONITORING_EVENT_OFFSET_TYPE_UNKNOWN)
+ printf (", %Xh", event_offset);
+ else
+ printf (", N/A");
+
+ if (event_offset_type != IPMI_MONITORING_EVENT_OFFSET_TYPE_UNKNOWN)
+ printf (", %s", event_offset_string);
+ else
+ printf (", N/A");
+ }
+ else if (record_type_class == IPMI_MONITORING_SEL_RECORD_TYPE_CLASS_TIMESTAMPED_OEM_RECORD
+ || record_type_class == IPMI_MONITORING_SEL_RECORD_TYPE_CLASS_NON_TIMESTAMPED_OEM_RECORD)
+ {
+ if (record_type_class == IPMI_MONITORING_SEL_RECORD_TYPE_CLASS_TIMESTAMPED_OEM_RECORD)
+ {
+ if ((manufacturer_id = ipmi_monitoring_sel_read_manufacturer_id (ctx)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_manufacturer_id(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ printf (", Manufacturer ID = %Xh", manufacturer_id);
+ }
+
+ if ((oem_data_len = ipmi_monitoring_sel_read_oem_data (ctx, oem_data, 1024)) < 0)
+ {
+ error( "ipmi_monitoring_sel_read_oem_data(): %s",
+ ipmi_monitoring_ctx_errormsg (ctx));
+ goto cleanup;
+ }
+
+ printf (", OEM Data = ");
+
+ for (j = 0; j < oem_data_len; j++)
+ printf ("%02Xh ", oem_data[j]);
+ }
+ else
+ printf (", N/A, N/A, N/A, N/A, N/A, N/A, N/A");
+
+ printf ("\n");
+#endif // NETDATA_COMMENTED
+ }
+
+ rv = 0;
+ cleanup:
+ if (ctx)
+ ipmi_monitoring_ctx_destroy (ctx);
+ return (rv);
+}
+
+// ----------------------------------------------------------------------------
+// MAIN PROGRAM FOR NETDATA PLUGIN
+
+int ipmi_collect_data(struct ipmi_monitoring_ipmi_config *ipmi_config) {
+ errno = 0;
+
+ if (_ipmimonitoring_sensors(ipmi_config) < 0) return -1;
+
+ if(netdata_do_sel) {
+ if(_ipmimonitoring_sel(ipmi_config) < 0) return -2;
+ }
+
+ return 0;
+}
+
+int ipmi_detect_speed_secs(struct ipmi_monitoring_ipmi_config *ipmi_config) {
+ int i, checks = 10;
+ unsigned long long total = 0;
+
+ for(i = 0 ; i < checks ; i++) {
+ if(debug) fprintf(stderr, "freeipmi.plugin: checking data collection speed iteration %d of %d\n", i+1, checks);
+
+ // measure the time a data collection needs
+ unsigned long long start = now_realtime_usec();
+ if(ipmi_collect_data(ipmi_config) < 0)
+ fatal("freeipmi.plugin: data collection failed.");
+
+ unsigned long long end = now_realtime_usec();
+
+ if(debug) fprintf(stderr, "freeipmi.plugin: data collection speed was %llu usec\n", end - start);
+
+ // add it to our total
+ total += end - start;
+
+ // wait the same time
+ // to avoid flooding the IPMI processor with requests
+ sleep_usec(end - start);
+ }
+
+ // so, we assume it needed 2x the time
+ // we find the average in microseconds
+ // and we round-up to the closest second
+
+ return (int)(( total * 2 / checks / 1000000 ) + 1);
+}
+
+int parse_inband_driver_type (const char *str)
+{
+ fatal_assert(str);
+
+ if (strcasecmp (str, IPMI_PARSE_DEVICE_KCS_STR) == 0)
+ return (IPMI_MONITORING_DRIVER_TYPE_KCS);
+ else if (strcasecmp (str, IPMI_PARSE_DEVICE_SSIF_STR) == 0)
+ return (IPMI_MONITORING_DRIVER_TYPE_SSIF);
+ /* support "open" for those that might be used to
+ * ipmitool.
+ */
+ else if (strcasecmp (str, IPMI_PARSE_DEVICE_OPENIPMI_STR) == 0
+ || strcasecmp (str, IPMI_PARSE_DEVICE_OPENIPMI_STR2) == 0)
+ return (IPMI_MONITORING_DRIVER_TYPE_OPENIPMI);
+ /* support "bmc" for those that might be used to
+ * ipmitool.
+ */
+ else if (strcasecmp (str, IPMI_PARSE_DEVICE_SUNBMC_STR) == 0
+ || strcasecmp (str, IPMI_PARSE_DEVICE_SUNBMC_STR2) == 0)
+ return (IPMI_MONITORING_DRIVER_TYPE_SUNBMC);
+
+ return (-1);
+}
+
+int parse_outofband_driver_type (const char *str)
+{
+ fatal_assert(str);
+
+ if (strcasecmp (str, IPMI_PARSE_DEVICE_LAN_STR) == 0)
+ return (IPMI_MONITORING_PROTOCOL_VERSION_1_5);
+ /* support "lanplus" for those that might be used to ipmitool.
+ * support typo variants to ease.
+ */
+ else if (strcasecmp (str, IPMI_PARSE_DEVICE_LAN_2_0_STR) == 0
+ || strcasecmp (str, IPMI_PARSE_DEVICE_LAN_2_0_STR2) == 0
+ || strcasecmp (str, IPMI_PARSE_DEVICE_LAN_2_0_STR3) == 0
+ || strcasecmp (str, IPMI_PARSE_DEVICE_LAN_2_0_STR4) == 0
+ || strcasecmp (str, IPMI_PARSE_DEVICE_LAN_2_0_STR5) == 0)
+ return (IPMI_MONITORING_PROTOCOL_VERSION_2_0);
+
+ return (-1);
+}
+
+int host_is_local(const char *host)
+{
+ if (host && (!strcmp(host, "localhost") || !strcmp(host, "127.0.0.1") || !strcmp(host, "::1")))
+ return (1);
+
+ return (0);
+}
+
+int main (int argc, char **argv) {
+ clocks_init();
+
+ // ------------------------------------------------------------------------
+ // initialization of netdata plugin
+
+ program_name = "freeipmi.plugin";
+
+ // disable syslog
+ error_log_syslog = 0;
+
+ // set errors flood protection to 100 logs per hour
+ error_log_errors_per_period = 100;
+ error_log_throttle_period = 3600;
+
+
+ // ------------------------------------------------------------------------
+ // parse command line parameters
+
+ int i, freq = 0;
+ for(i = 1; i < argc ; i++) {
+ if(isdigit(*argv[i]) && !freq) {
+ int n = str2i(argv[i]);
+ if(n > 0 && n < 86400) {
+ freq = n;
+ continue;
+ }
+ }
+ else if(strcmp("version", argv[i]) == 0 || strcmp("-version", argv[i]) == 0 || strcmp("--version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) {
+ printf("freeipmi.plugin %s\n", VERSION);
+ exit(0);
+ }
+ else if(strcmp("debug", argv[i]) == 0) {
+ debug = 1;
+ continue;
+ }
+ else if(strcmp("sel", argv[i]) == 0) {
+ netdata_do_sel = 1;
+ continue;
+ }
+ else if(strcmp("no-sel", argv[i]) == 0) {
+ netdata_do_sel = 0;
+ continue;
+ }
+ else if(strcmp("-h", argv[i]) == 0 || strcmp("--help", argv[i]) == 0) {
+ fprintf(stderr,
+ "\n"
+ " netdata freeipmi.plugin %s\n"
+ " Copyright (C) 2016-2017 Costa Tsaousis <costa@tsaousis.gr>\n"
+ " Released under GNU General Public License v3 or later.\n"
+ " All rights reserved.\n"
+ "\n"
+ " This program is a data collector plugin for netdata.\n"
+ "\n"
+ " Available command line options:\n"
+ "\n"
+ " SECONDS data collection frequency\n"
+ " minimum: %d\n"
+ "\n"
+ " debug enable verbose output\n"
+ " default: disabled\n"
+ "\n"
+ " sel\n"
+ " no-sel enable/disable SEL collection\n"
+ " default: %s\n"
+ "\n"
+ " hostname HOST\n"
+ " username USER\n"
+ " password PASS connect to remote IPMI host\n"
+ " default: local IPMI processor\n"
+ "\n"
+ " noauthcodecheck don't check the authentication codes returned\n"
+ "\n"
+ " driver-type IPMIDRIVER\n"
+ " Specify the driver type to use instead of doing an auto selection. \n"
+ " The currently available outofband drivers are LAN and LAN_2_0,\n"
+ " which perform IPMI 1.5 and IPMI 2.0 respectively. \n"
+ " The currently available inband drivers are KCS, SSIF, OPENIPMI and SUNBMC.\n"
+ "\n"
+ " sdr-cache-dir PATH directory for SDR cache files\n"
+ " default: %s\n"
+ "\n"
+ " sensor-config-file FILE filename to read sensor configuration\n"
+ " default: %s\n"
+ "\n"
+ " ignore N1,N2,N3,... sensor IDs to ignore\n"
+ " default: none\n"
+ "\n"
+ " ignore-status N1,N2,N3,... sensor IDs to ignore status (nominal/warning/critical)\n"
+ " default: none\n"
+ "\n"
+ " -v\n"
+ " -V\n"
+ " version print version and exit\n"
+ "\n"
+ " Linux kernel module for IPMI is CPU hungry.\n"
+ " On Linux run this to lower kipmiN CPU utilization:\n"
+ " # echo 10 > /sys/module/ipmi_si/parameters/kipmid_max_busy_us\n"
+ "\n"
+ " or create: /etc/modprobe.d/ipmi.conf with these contents:\n"
+ " options ipmi_si kipmid_max_busy_us=10\n"
+ "\n"
+ " For more information:\n"
+ " https://github.com/netdata/netdata/tree/master/collectors/freeipmi.plugin\n"
+ "\n"
+ , VERSION
+ , netdata_update_every
+ , netdata_do_sel?"enabled":"disabled"
+ , sdr_cache_directory?sdr_cache_directory:"system default"
+ , sensor_config_file?sensor_config_file:"system default"
+ );
+ exit(1);
+ }
+ else if(i < argc && strcmp("hostname", argv[i]) == 0) {
+ hostname = strdupz(argv[++i]);
+ char *s = argv[i];
+ // mask it be hidden from the process tree
+ while(*s) *s++ = 'x';
+ if(debug) fprintf(stderr, "freeipmi.plugin: hostname set to '%s'\n", hostname);
+ continue;
+ }
+ else if(i < argc && strcmp("username", argv[i]) == 0) {
+ username = strdupz(argv[++i]);
+ char *s = argv[i];
+ // mask it be hidden from the process tree
+ while(*s) *s++ = 'x';
+ if(debug) fprintf(stderr, "freeipmi.plugin: username set to '%s'\n", username);
+ continue;
+ }
+ else if(i < argc && strcmp("password", argv[i]) == 0) {
+ password = strdupz(argv[++i]);
+ char *s = argv[i];
+ // mask it be hidden from the process tree
+ while(*s) *s++ = 'x';
+ if(debug) fprintf(stderr, "freeipmi.plugin: password set to '%s'\n", password);
+ continue;
+ }
+ else if(strcmp("driver-type", argv[i]) == 0) {
+ if (hostname) {
+ protocol_version=parse_outofband_driver_type(argv[++i]);
+ if(debug) fprintf(stderr, "freeipmi.plugin: outband protocol version set to '%d'\n", protocol_version);
+ }
+ else {
+ driver_type=parse_inband_driver_type(argv[++i]);
+ if(debug) fprintf(stderr, "freeipmi.plugin: inband driver type set to '%d'\n", driver_type);
+ }
+ continue;
+ } else if (i < argc && strcmp("noauthcodecheck", argv[i]) == 0) {
+ if (!hostname || host_is_local(hostname)) {
+ if (debug)
+ fprintf(
+ stderr,
+ "freeipmi.plugin: noauthcodecheck workaround flag is ignored for inband configuration\n");
+ } else if (protocol_version < 0 || protocol_version == IPMI_MONITORING_PROTOCOL_VERSION_1_5) {
+ workaround_flags |= IPMI_MONITORING_WORKAROUND_FLAGS_PROTOCOL_VERSION_1_5_NO_AUTH_CODE_CHECK;
+ if (debug)
+ fprintf(stderr, "freeipmi.plugin: noauthcodecheck workaround flag enabled\n");
+ } else {
+ if (debug)
+ fprintf(
+ stderr,
+ "freeipmi.plugin: noauthcodecheck workaround flag is ignored for protocol version 2.0\n");
+ }
+ continue;
+ }
+ else if(i < argc && strcmp("sdr-cache-dir", argv[i]) == 0) {
+ sdr_cache_directory = argv[++i];
+ if(debug) fprintf(stderr, "freeipmi.plugin: SDR cache directory set to '%s'\n", sdr_cache_directory);
+ continue;
+ }
+ else if(i < argc && strcmp("sensor-config-file", argv[i]) == 0) {
+ sensor_config_file = argv[++i];
+ if(debug) fprintf(stderr, "freeipmi.plugin: sensor config file set to '%s'\n", sensor_config_file);
+ continue;
+ }
+ else if(i < argc && strcmp("ignore", argv[i]) == 0) {
+ excluded_record_ids_parse(argv[++i]);
+ continue;
+ }
+ else if(i < argc && strcmp("ignore-status", argv[i]) == 0) {
+ excluded_status_record_ids_parse(argv[++i]);
+ continue;
+ }
+
+ error("freeipmi.plugin: ignoring parameter '%s'", argv[i]);
+ }
+
+ errno = 0;
+
+ if(freq >= netdata_update_every)
+ netdata_update_every = freq;
+
+ else if(freq)
+ error("update frequency %d seconds is too small for IPMI. Using %d.", freq, netdata_update_every);
+
+
+ // ------------------------------------------------------------------------
+ // initialize IPMI
+
+ struct ipmi_monitoring_ipmi_config ipmi_config;
+
+ if(debug) fprintf(stderr, "freeipmi.plugin: calling _init_ipmi_config()\n");
+
+ _init_ipmi_config(&ipmi_config);
+
+ if(debug) {
+ fprintf(stderr, "freeipmi.plugin: calling ipmi_monitoring_init()\n");
+ ipmimonitoring_init_flags|=IPMI_MONITORING_FLAGS_DEBUG|IPMI_MONITORING_FLAGS_DEBUG_IPMI_PACKETS;
+ }
+
+ if(ipmi_monitoring_init(ipmimonitoring_init_flags, &errnum) < 0)
+ fatal("ipmi_monitoring_init: %s", ipmi_monitoring_ctx_strerror(errnum));
+
+ if(debug) fprintf(stderr, "freeipmi.plugin: detecting IPMI minimum update frequency...\n");
+ freq = ipmi_detect_speed_secs(&ipmi_config);
+ if(debug) fprintf(stderr, "freeipmi.plugin: IPMI minimum update frequency was calculated to %d seconds.\n", freq);
+
+ if(freq > netdata_update_every) {
+ info("enforcing minimum data collection frequency, calculated to %d seconds.", freq);
+ netdata_update_every = freq;
+ }
+
+
+ // ------------------------------------------------------------------------
+ // the main loop
+
+ if(debug) fprintf(stderr, "freeipmi.plugin: starting data collection\n");
+
+ time_t started_t = now_monotonic_sec();
+
+ size_t iteration = 0;
+ usec_t step = netdata_update_every * USEC_PER_SEC;
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ for(iteration = 0; 1 ; iteration++) {
+ usec_t dt = heartbeat_next(&hb, step);
+
+ if(debug && iteration)
+ fprintf(stderr, "freeipmi.plugin: iteration %zu, dt %llu usec, sensors collected %zu, sensors sent to netdata %zu \n"
+ , iteration
+ , dt
+ , netdata_sensors_collected
+ , netdata_sensors_updated
+ );
+
+ netdata_mark_as_not_updated();
+
+ if(debug) fprintf(stderr, "freeipmi.plugin: calling ipmi_collect_data()\n");
+ if(ipmi_collect_data(&ipmi_config) < 0)
+ fatal("data collection failed.");
+
+ if(debug) fprintf(stderr, "freeipmi.plugin: calling send_metrics_to_netdata()\n");
+ send_metrics_to_netdata();
+ fflush(stdout);
+
+ // restart check (14400 seconds)
+ if(now_monotonic_sec() - started_t > 14400) exit(0);
+ }
+}
diff --git a/collectors/idlejitter.plugin/Makefile.am b/collectors/idlejitter.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/idlejitter.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/idlejitter.plugin/README.md b/collectors/idlejitter.plugin/README.md
new file mode 100644
index 0000000..5a92d53
--- /dev/null
+++ b/collectors/idlejitter.plugin/README.md
@@ -0,0 +1,32 @@
+<!--
+title: "idlejitter.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/idlejitter.plugin/README.md
+-->
+
+# idlejitter.plugin
+
+Idle jitter is a measure of delays in timing for user processes caused by scheduling limitations.
+
+## How Netdata measures idle jitter
+
+A thread is spawned that requests to sleep for 20000 microseconds (20ms).
+When the system wakes it up, it measures how many microseconds have passed.
+The difference between the requested and the actual duration of the sleep, is the idle jitter.
+This is done at most 50 times per second, to ensure we have a good average.
+
+This number is useful:
+
+- In multimedia-streaming environments such as VoIP gateways, where the CPU jitter can affect the quality of the service.
+- On time servers and other systems that require very precise timing, where CPU jitter can actively interfere with timing precision.
+- On gaming systems, where CPU jitter can cause frame drops and stuttering.
+- In cloud infrastructure that can pause the VM or container for a small duration to perform operations at the host.
+
+## Charts
+
+idlejitter.plugin generates the idlejitter chart which measures CPU idle jitter in milliseconds lost per second.
+
+## Configuration
+
+This chart is available without any configuration.
+
+
diff --git a/collectors/idlejitter.plugin/plugin_idlejitter.c b/collectors/idlejitter.plugin/plugin_idlejitter.c
new file mode 100644
index 0000000..b6339cc
--- /dev/null
+++ b/collectors/idlejitter.plugin/plugin_idlejitter.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "daemon/common.h"
+
+#define CPU_IDLEJITTER_SLEEP_TIME_MS 20
+
+static void cpuidlejitter_main_cleanup(void *ptr) {
+ worker_unregister();
+
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ info("cleaning up...");
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+void *cpuidlejitter_main(void *ptr) {
+ worker_register("IDLEJITTER");
+ worker_register_job_name(0, "measurements");
+
+ netdata_thread_cleanup_push(cpuidlejitter_main_cleanup, ptr);
+
+ usec_t sleep_ut = config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS) * USEC_PER_MS;
+ if(sleep_ut <= 0) {
+ config_set_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS);
+ sleep_ut = CPU_IDLEJITTER_SLEEP_TIME_MS * USEC_PER_MS;
+ }
+
+ RRDSET *st = rrdset_create_localhost(
+ "system"
+ , "idlejitter"
+ , NULL
+ , "idlejitter"
+ , NULL
+ , "CPU Idle Jitter"
+ , "microseconds lost/s"
+ , "idlejitter.plugin"
+ , NULL
+ , NETDATA_CHART_PRIO_SYSTEM_IDLEJITTER
+ , localhost->rrd_update_every
+ , RRDSET_TYPE_AREA
+ );
+ RRDDIM *rd_min = rrddim_add(st, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_avg = rrddim_add(st, "average", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ usec_t update_every_ut = localhost->rrd_update_every * USEC_PER_SEC;
+ struct timeval before, after;
+
+ while (!netdata_exit) {
+ int iterations = 0;
+ usec_t error_total = 0,
+ error_min = 0,
+ error_max = 0,
+ elapsed = 0;
+
+ while (elapsed < update_every_ut) {
+ now_monotonic_high_precision_timeval(&before);
+ worker_is_idle();
+ sleep_usec(sleep_ut);
+ worker_is_busy(0);
+ now_monotonic_high_precision_timeval(&after);
+
+ usec_t dt = dt_usec(&after, &before);
+ elapsed += dt;
+
+ usec_t error = dt - sleep_ut;
+ error_total += error;
+
+ if(unlikely(!iterations))
+ error_min = error;
+ else if(error < error_min)
+ error_min = error;
+
+ if(error > error_max)
+ error_max = error;
+
+ iterations++;
+ }
+
+ if(iterations) {
+ rrddim_set_by_pointer(st, rd_min, error_min);
+ rrddim_set_by_pointer(st, rd_max, error_max);
+ rrddim_set_by_pointer(st, rd_avg, error_total / iterations);
+ rrdset_done(st);
+ }
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
diff --git a/collectors/ioping.plugin/Makefile.am b/collectors/ioping.plugin/Makefile.am
new file mode 100644
index 0000000..a9cd7c4
--- /dev/null
+++ b/collectors/ioping.plugin/Makefile.am
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+CLEANFILES = \
+ ioping.plugin \
+ $(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+dist_plugins_SCRIPTS = \
+ ioping.plugin \
+ $(NULL)
+
+dist_noinst_DATA = \
+ ioping.plugin.in \
+ README.md \
+ $(NULL)
+
+dist_libconfig_DATA = \
+ ioping.conf \
+ $(NULL)
diff --git a/collectors/ioping.plugin/README.md b/collectors/ioping.plugin/README.md
new file mode 100644
index 0000000..c4c3695
--- /dev/null
+++ b/collectors/ioping.plugin/README.md
@@ -0,0 +1,86 @@
+<!--
+title: "ioping.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/ioping.plugin/README.md
+-->
+
+# ioping.plugin
+
+The ioping plugin supports monitoring latency for any number of directories/files/devices,
+by pinging them with `ioping`.
+
+A recent version of `ioping` is required (one that supports option `-N`).
+The supplied plugin can install it, by running:
+
+```sh
+/usr/libexec/netdata/plugins.d/ioping.plugin install
+```
+
+The `-e` option can be supplied to indicate where the Netdata environment file is installed. The default path is `/etc/netdata/.environment`.
+
+The above will download, build and install the right version as `/usr/libexec/netdata/plugins.d/ioping`.
+
+Then you need to edit `/etc/netdata/ioping.conf` (to edit it on your system run
+`/etc/netdata/edit-config ioping.conf`) like this:
+
+```sh
+# uncomment the following line - it should already be there
+ioping="/usr/libexec/netdata/plugins.d/ioping"
+
+# set here the directory/file/device, you need to ping
+destination="destination"
+
+# override the chart update frequency - the default is inherited from Netdata
+update_every="1s"
+
+# the request size in bytes to ping the destination
+request_size="4k"
+
+# other iping options - these are the defaults
+ioping_opts="-T 1000000 -R"
+```
+
+## alarms
+
+Netdata will automatically attach a few alarms for each host.
+Check the [latest versions of the ioping alarms](https://raw.githubusercontent.com/netdata/netdata/master/health/health.d/ioping.conf)
+
+## Multiple ioping Plugins With Different Settings
+
+You may need to run multiple ioping plugins with different settings or different end points.
+For example, you may need to ping one destination once per 10 seconds, and another once per second.
+
+Netdata allows you to add as many `ioping` plugins as you like.
+
+Follow this procedure:
+
+**1. Create New ioping Configuration File**
+
+```sh
+# Step Into Configuration Directory
+cd /etc/netdata
+
+# Copy Original ioping Configuration File To New Configuration File
+cp ioping.conf ioping2.conf
+```
+
+Edit `ioping2.conf` and set the settings and the destination you need for the seconds instance.
+
+**2. Soft Link Original ioping Plugin to New Plugin File**
+
+```sh
+# Become root (If The Step Step Is Performed As Non-Root User)
+sudo su
+
+# Step Into The Plugins Directory
+cd /usr/libexec/netdata/plugins.d
+
+# Link ioping.plugin to ioping2.plugin
+ln -s ioping.plugin ioping2.plugin
+```
+
+That's it. Netdata will detect the new plugin and start it.
+
+You can name the new plugin any name you like.
+Just make sure the plugin and the configuration file have the same name.
+
+
diff --git a/collectors/ioping.plugin/ioping.conf b/collectors/ioping.plugin/ioping.conf
new file mode 100644
index 0000000..86f0de7
--- /dev/null
+++ b/collectors/ioping.plugin/ioping.conf
@@ -0,0 +1,40 @@
+# no need for shebang - this file is sourced from ioping.plugin
+
+# ioping.plugin requires a recent version of ioping.
+#
+# You can get it on your system, by running:
+#
+# /usr/libexec/netdata/plugins.d/ioping.plugin install
+
+# -----------------------------------------------------------------------------
+# configuration options
+
+# The ioping binary to use. We need one that can output netdata friendly info
+# (supporting: -N). If you have multiple versions, put here the full filename
+# of the right one
+
+#ioping="/usr/libexec/netdata/plugins.d/ioping"
+
+
+# The directory/file/device to ioping
+
+destination=""
+
+
+# The update frequency of the chart in seconds (symbolic modifiers are supported)
+# the default is inherited from netdata
+
+#update_every="1s"
+
+
+# The request size in bytes to ioping the destination (symbolic modifiers are supported)
+# by default 4k chunks are used
+
+#request_size="4k"
+
+
+# Other ioping options
+# the defaults:
+# -T 1000000 = maximum valid request time (us)
+
+#ioping_opts="-T 1000000"
diff --git a/collectors/ioping.plugin/ioping.plugin.in b/collectors/ioping.plugin/ioping.plugin.in
new file mode 100755
index 0000000..1d79eb7
--- /dev/null
+++ b/collectors/ioping.plugin/ioping.plugin.in
@@ -0,0 +1,210 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# GPL v3+
+#
+# This plugin requires a latest version of ioping.
+# You can compile it from source, by running me with option: install
+
+export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
+export LC_ALL=C
+
+usage="$(basename "$0") [install] [-h] [-e]
+
+where:
+ install install ioping binary
+ -e, --env path to environment file (defaults to '/etc/netdata/.environment'
+ -h show this help text"
+
+INSTALL=0
+ENVIRONMENT_FILE="/etc/netdata/.environment"
+
+while :; do
+ case "$1" in
+ -h | --help)
+ echo "$usage" >&2
+ exit 1
+ ;;
+ install)
+ INSTALL=1
+ shift
+ ;;
+ -e | --env)
+ ENVIRONMENT_FILE="$2"
+ shift 2
+ ;;
+ -*)
+ echo "$usage" >&2
+ exit 1
+ ;;
+ *) break ;;
+ esac
+done
+
+if [ "$INSTALL" == "1" ]
+ then
+ [ "${UID}" != 0 ] && echo >&2 "Please run me as root. This will install a single binary file: /usr/libexec/netdata/plugins.d/ioping." && exit 1
+
+ source "${ENVIRONMENT_FILE}" || exit 1
+
+ run() {
+ printf >&2 " > "
+ printf >&2 "%q " "${@}"
+ printf >&2 "\n"
+ "${@}" || exit 1
+ }
+
+ download() {
+ local git="$(which git 2>/dev/null || command -v git 2>/dev/null)"
+ [ ! -z "${git}" ] && run git clone "${1}" "${2}" && return 0
+
+ echo >&2 "Cannot find 'git' in this system." && exit 1
+ }
+
+ tmp=$(mktemp -d /tmp/netdata-ioping-XXXXXX)
+ [ ! -d "${NETDATA_PREFIX}/usr/libexec/netdata" ] && run mkdir -p "${NETDATA_PREFIX}/usr/libexec/netdata"
+
+ run cd "${tmp}"
+
+ if [ -d ioping-netdata ]
+ then
+ run rm -rf ioping-netdata || exit 1
+ fi
+
+ download 'https://github.com/netdata/ioping.git' 'ioping-netdata'
+ [ $? -ne 0 ] && exit 1
+ run cd ioping-netdata || exit 1
+
+ INSTALL_PATH="${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/ioping"
+
+ run make clean
+ run make
+ run mv ioping "${INSTALL_PATH}"
+ run chown root:"${NETDATA_GROUP}" "${INSTALL_PATH}"
+ run chmod 4750 "${INSTALL_PATH}"
+ echo >&2
+ echo >&2 "All done, you have a compatible ioping now at ${INSTALL_PATH}."
+ echo >&2
+
+ exit 0
+fi
+
+# -----------------------------------------------------------------------------
+
+PROGRAM_NAME="$(basename "${0}")"
+
+logdate() {
+ date "+%Y-%m-%d %H:%M:%S"
+}
+
+log() {
+ local status="${1}"
+ shift
+
+ echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
+
+}
+
+warning() {
+ log WARNING "${@}"
+}
+
+error() {
+ log ERROR "${@}"
+}
+
+info() {
+ log INFO "${@}"
+}
+
+fatal() {
+ log FATAL "${@}"
+ echo "DISABLE"
+ exit 1
+}
+
+debug=0
+debug() {
+ [ $debug -eq 1 ] && log DEBUG "${@}"
+}
+
+# -----------------------------------------------------------------------------
+
+# store in ${plugin} the name we run under
+# this allows us to copy/link ioping.plugin under a different name
+# to have multiple ioping plugins running with different settings
+plugin="${PROGRAM_NAME/.plugin/}"
+
+
+# -----------------------------------------------------------------------------
+
+# the frequency to send info to netdata
+# passed by netdata as the first parameter
+update_every="${1-1}"
+
+# the netdata configuration directory
+# passed by netdata as an environment variable
+[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
+[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
+
+# the netdata directory for internal binaries
+[ -z "${NETDATA_PLUGINS_DIR}" ] && NETDATA_PLUGINS_DIR="@pluginsdir_POST@"
+
+# -----------------------------------------------------------------------------
+# configuration options
+# can be overwritten at /etc/netdata/ioping.conf
+
+# the ioping binary to use
+# we need one that can output netdata friendly info (supporting: -N)
+# if you have multiple versions, put here the full filename of the right one
+ioping="${NETDATA_PLUGINS_DIR}/ioping"
+
+# the destination to ioping
+destination=""
+
+# the request size in bytes to ping the disk
+request_size="4k"
+
+# ioping options
+ioping_opts="-T 1000000"
+
+# -----------------------------------------------------------------------------
+# load the configuration files
+
+for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/${plugin}.conf" "${NETDATA_USER_CONFIG_DIR}/${plugin}.conf"; do
+ if [ -f "${CONFIG}" ]; then
+ info "Loading config file '${CONFIG}'..."
+ source "${CONFIG}"
+ [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'."
+ elif [[ $CONFIG =~ ^$NETDATA_USER_CONFIG_DIR ]]; then
+ warning "Cannot find file '${CONFIG}'."
+ fi
+done
+
+if [ -z "${destination}" ]
+then
+ fatal "destination is not configured - nothing to do."
+fi
+
+if [ ! -f "${ioping}" ]
+then
+ fatal "ioping command is not found. Please set its full path in '${NETDATA_USER_CONFIG_DIR}/${plugin}.conf'"
+fi
+
+if [ ! -x "${ioping}" ]
+then
+ fatal "ioping command '${ioping}' is not executable - cannot proceed."
+fi
+
+# the ioping options we will use
+options=( -N -i ${update_every} -s ${request_size} ${ioping_opts} ${destination} )
+
+# execute ioping
+info "starting ioping: ${ioping} ${options[*]}"
+exec "${ioping}" "${options[@]}"
+
+# if we cannot execute ioping, stop
+fatal "command '${ioping} ${options[*]}' failed to be executed (returned code $?)."
diff --git a/collectors/macos.plugin/Makefile.am b/collectors/macos.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/macos.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/macos.plugin/README.md b/collectors/macos.plugin/README.md
new file mode 100644
index 0000000..92bbf1e
--- /dev/null
+++ b/collectors/macos.plugin/README.md
@@ -0,0 +1,12 @@
+<!--
+title: "macos.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/macos.plugin/README.md
+-->
+
+# macos.plugin
+
+Collects resource usage and performance data on macOS systems
+
+By default, Netdata will enable monitoring metrics for disks, memory, and network only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). Use `yes` instead of `auto` in plugin configuration sections to enable these charts permanently. You can also set the `enable zero metrics` option to `yes` in the `[global]` section which enables charts with zero metrics for all internal Netdata plugins.
+
+
diff --git a/collectors/macos.plugin/macos_fw.c b/collectors/macos.plugin/macos_fw.c
new file mode 100644
index 0000000..07f7d77
--- /dev/null
+++ b/collectors/macos.plugin/macos_fw.c
@@ -0,0 +1,648 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_macos.h"
+
+#include <CoreFoundation/CoreFoundation.h>
+#include <IOKit/IOKitLib.h>
+#include <IOKit/storage/IOBlockStorageDriver.h>
+#include <IOKit/IOBSD.h>
+// NEEDED BY do_space, do_inodes
+#include <sys/mount.h>
+// NEEDED BY: struct ifaddrs, getifaddrs()
+#include <net/if.h>
+#include <ifaddrs.h>
+
+// NEEDED BY: do_bandwidth
+#define IFA_DATA(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s)
+
+#define MAXDRIVENAME 31
+
+#define KILO_FACTOR 1024
+#define MEGA_FACTOR 1048576 // 1024 * 1024
+#define GIGA_FACTOR 1073741824 // 1024 * 1024 * 1024
+
+int do_macos_iokit(int update_every, usec_t dt) {
+ (void)dt;
+
+ static int do_io = -1, do_space = -1, do_inodes = -1, do_bandwidth = -1;
+
+ if (unlikely(do_io == -1)) {
+ do_io = config_get_boolean("plugin:macos:iokit", "disk i/o", 1);
+ do_space = config_get_boolean("plugin:macos:sysctl", "space usage for all disks", 1);
+ do_inodes = config_get_boolean("plugin:macos:sysctl", "inodes usage for all disks", 1);
+ do_bandwidth = config_get_boolean("plugin:macos:sysctl", "bandwidth", 1);
+ }
+
+ RRDSET *st;
+
+ mach_port_t main_port;
+ io_registry_entry_t drive, drive_media;
+ io_iterator_t drive_list;
+ CFDictionaryRef properties, statistics;
+ CFStringRef name;
+ CFNumberRef number;
+ kern_return_t status;
+ collected_number total_disk_reads = 0;
+ collected_number total_disk_writes = 0;
+ struct diskstat {
+ char name[MAXDRIVENAME];
+ collected_number bytes_read;
+ collected_number bytes_write;
+ collected_number reads;
+ collected_number writes;
+ collected_number time_read;
+ collected_number time_write;
+ collected_number latency_read;
+ collected_number latency_write;
+ } diskstat;
+ struct cur_diskstat {
+ collected_number duration_read_ns;
+ collected_number duration_write_ns;
+ collected_number busy_time_ns;
+ } cur_diskstat;
+ struct prev_diskstat {
+ collected_number bytes_read;
+ collected_number bytes_write;
+ collected_number operations_read;
+ collected_number operations_write;
+ collected_number duration_read_ns;
+ collected_number duration_write_ns;
+ collected_number busy_time_ns;
+ } prev_diskstat;
+
+ // NEEDED BY: do_space, do_inodes
+ struct statfs *mntbuf;
+ int mntsize, i;
+ char title[4096 + 1];
+
+ // NEEDED BY: do_bandwidth
+ struct ifaddrs *ifa, *ifap;
+
+#if !defined(MAC_OS_VERSION_12_0) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_VERSION_12_0)
+#define IOMainPort IOMasterPort
+#endif
+
+ /* Get ports and services for drive statistics. */
+ if (unlikely(IOMainPort(bootstrap_port, &main_port))) {
+ error("MACOS: IOMasterPort() failed");
+ do_io = 0;
+ error("DISABLED: system.io");
+ /* Get the list of all drive objects. */
+ } else if (unlikely(IOServiceGetMatchingServices(main_port, IOServiceMatching("IOBlockStorageDriver"), &drive_list))) {
+ error("MACOS: IOServiceGetMatchingServices() failed");
+ do_io = 0;
+ error("DISABLED: system.io");
+ } else {
+ while ((drive = IOIteratorNext(drive_list)) != 0) {
+ properties = 0;
+ statistics = 0;
+ number = 0;
+ bzero(&diskstat, sizeof(diskstat));
+
+ /* Get drive media object. */
+ status = IORegistryEntryGetChildEntry(drive, kIOServicePlane, &drive_media);
+ if (unlikely(status != KERN_SUCCESS)) {
+ IOObjectRelease(drive);
+ continue;
+ }
+
+ /* Get drive media properties. */
+ if (likely(!IORegistryEntryCreateCFProperties(drive_media, (CFMutableDictionaryRef *)&properties, kCFAllocatorDefault, 0))) {
+ /* Get disk name. */
+ if (likely(name = (CFStringRef)CFDictionaryGetValue(properties, CFSTR(kIOBSDNameKey)))) {
+ CFStringGetCString(name, diskstat.name, MAXDRIVENAME, kCFStringEncodingUTF8);
+ }
+ }
+
+ /* Release. */
+ CFRelease(properties);
+ IOObjectRelease(drive_media);
+
+ if(unlikely(!*diskstat.name)) {
+ IOObjectRelease(drive);
+ continue;
+ }
+
+ /* Obtain the properties for this drive object. */
+ if (unlikely(IORegistryEntryCreateCFProperties(drive, (CFMutableDictionaryRef *)&properties, kCFAllocatorDefault, 0))) {
+ IOObjectRelease(drive);
+ error("MACOS: IORegistryEntryCreateCFProperties() failed");
+ do_io = 0;
+ error("DISABLED: system.io");
+ break;
+ } else if (likely(properties)) {
+ /* Obtain the statistics from the drive properties. */
+ if (likely(statistics = (CFDictionaryRef)CFDictionaryGetValue(properties, CFSTR(kIOBlockStorageDriverStatisticsKey)))) {
+
+ // --------------------------------------------------------------------
+
+ /* Get bytes read. */
+ if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsBytesReadKey)))) {
+ CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.bytes_read);
+ total_disk_reads += diskstat.bytes_read;
+ }
+
+ /* Get bytes written. */
+ if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsBytesWrittenKey)))) {
+ CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.bytes_write);
+ total_disk_writes += diskstat.bytes_write;
+ }
+
+ st = rrdset_find_active_bytype_localhost("disk", diskstat.name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "disk"
+ , diskstat.name
+ , NULL
+ , diskstat.name
+ , "disk.io"
+ , "Disk I/O Bandwidth"
+ , "KiB/s"
+ , "macos.plugin"
+ , "iokit"
+ , 2000
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrddim_add(st, "reads", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "writes", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ prev_diskstat.bytes_read = rrddim_set(st, "reads", diskstat.bytes_read);
+ prev_diskstat.bytes_write = rrddim_set(st, "writes", diskstat.bytes_write);
+ rrdset_done(st);
+
+ /* Get number of reads. */
+ if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsReadsKey)))) {
+ CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.reads);
+ }
+
+ /* Get number of writes. */
+ if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsWritesKey)))) {
+ CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes);
+ }
+
+ st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "disk_ops"
+ , diskstat.name
+ , NULL
+ , diskstat.name
+ , "disk.ops"
+ , "Disk Completed I/O Operations"
+ , "operations/s"
+ , "macos.plugin"
+ , "iokit"
+ , 2001
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ prev_diskstat.operations_read = rrddim_set(st, "reads", diskstat.reads);
+ prev_diskstat.operations_write = rrddim_set(st, "writes", diskstat.writes);
+ rrdset_done(st);
+
+ /* Get reads time. */
+ if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsTotalReadTimeKey)))) {
+ CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_read);
+ }
+
+ /* Get writes time. */
+ if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsTotalWriteTimeKey)))) {
+ CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write);
+ }
+
+ st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "disk_util"
+ , diskstat.name
+ , NULL
+ , diskstat.name
+ , "disk.util"
+ , "Disk Utilization Time"
+ , "% of time working"
+ , "macos.plugin"
+ , "iokit"
+ , 2004
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "utilization", NULL, 1, 10000000, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ cur_diskstat.busy_time_ns = (diskstat.time_read + diskstat.time_write);
+ prev_diskstat.busy_time_ns = rrddim_set(st, "utilization", cur_diskstat.busy_time_ns);
+ rrdset_done(st);
+
+ /* Get reads latency. */
+ if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsLatentReadTimeKey)))) {
+ CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_read);
+ }
+
+ /* Get writes latency. */
+ if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsLatentWriteTimeKey)))) {
+ CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write);
+ }
+
+ st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "disk_iotime"
+ , diskstat.name
+ , NULL
+ , diskstat.name
+ , "disk.iotime"
+ , "Disk Total I/O Time"
+ , "milliseconds/s"
+ , "macos.plugin"
+ , "iokit"
+ , 2022
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "reads", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "writes", NULL, -1, 1000000, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ cur_diskstat.duration_read_ns = diskstat.time_read + diskstat.latency_read;
+ cur_diskstat.duration_write_ns = diskstat.time_write + diskstat.latency_write;
+ prev_diskstat.duration_read_ns = rrddim_set(st, "reads", cur_diskstat.duration_read_ns);
+ prev_diskstat.duration_write_ns = rrddim_set(st, "writes", cur_diskstat.duration_write_ns);
+ rrdset_done(st);
+
+ // calculate differential charts
+ // only if this is not the first time we run
+
+ if (likely(dt)) {
+ st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "disk_await"
+ , diskstat.name
+ , NULL
+ , diskstat.name
+ , "disk.await"
+ , "Average Completed I/O Operation Time"
+ , "milliseconds/operation"
+ , "macos.plugin"
+ , "iokit"
+ , 2005
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "reads", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "writes", NULL, -1, 1000000, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "reads", (diskstat.reads - prev_diskstat.operations_read) ?
+ (cur_diskstat.duration_read_ns - prev_diskstat.duration_read_ns) / (diskstat.reads - prev_diskstat.operations_read) : 0);
+ rrddim_set(st, "writes", (diskstat.writes - prev_diskstat.operations_write) ?
+ (cur_diskstat.duration_write_ns - prev_diskstat.duration_write_ns) / (diskstat.writes - prev_diskstat.operations_write) : 0);
+ rrdset_done(st);
+
+ st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "disk_avgsz"
+ , diskstat.name
+ , NULL
+ , diskstat.name
+ , "disk.avgsz"
+ , "Average Completed I/O Operation Bandwidth"
+ , "KiB/operation"
+ , "macos.plugin"
+ , "iokit"
+ , 2006
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "reads", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "writes", NULL, -1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "reads", (diskstat.reads - prev_diskstat.operations_read) ?
+ (diskstat.bytes_read - prev_diskstat.bytes_read) / (diskstat.reads - prev_diskstat.operations_read) : 0);
+ rrddim_set(st, "writes", (diskstat.writes - prev_diskstat.operations_write) ?
+ (diskstat.bytes_write - prev_diskstat.bytes_write) / (diskstat.writes - prev_diskstat.operations_write) : 0);
+ rrdset_done(st);
+
+ st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "disk_svctm"
+ , diskstat.name
+ , NULL
+ , diskstat.name
+ , "disk.svctm"
+ , "Average Service Time"
+ , "milliseconds/operation"
+ , "macos.plugin"
+ , "iokit"
+ , 2007
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "svctm", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "svctm", ((diskstat.reads - prev_diskstat.operations_read) + (diskstat.writes - prev_diskstat.operations_write)) ?
+ (cur_diskstat.busy_time_ns - prev_diskstat.busy_time_ns) / ((diskstat.reads - prev_diskstat.operations_read) + (diskstat.writes - prev_diskstat.operations_write)) : 0);
+ rrdset_done(st);
+ }
+ }
+
+ /* Release. */
+ CFRelease(properties);
+ }
+
+ /* Release. */
+ IOObjectRelease(drive);
+ }
+ IOIteratorReset(drive_list);
+
+ /* Release. */
+ IOObjectRelease(drive_list);
+ }
+
+ if (likely(do_io)) {
+ st = rrdset_find_active_bytype_localhost("system", "io");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "io"
+ , NULL
+ , "disk"
+ , NULL
+ , "Disk I/O"
+ , "KiB/s"
+ , "macos.plugin"
+ , "iokit"
+ , 150
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+ rrddim_add(st, "in", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "out", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "in", total_disk_reads);
+ rrddim_set(st, "out", total_disk_writes);
+ rrdset_done(st);
+ }
+
+ // Can be merged with FreeBSD plugin
+
+ if (likely(do_space || do_inodes)) {
+ // there is no mount info in sysctl MIBs
+ if (unlikely(!(mntsize = getmntinfo(&mntbuf, MNT_NOWAIT)))) {
+ error("MACOS: getmntinfo() failed");
+ do_space = 0;
+ error("DISABLED: disk_space.X");
+ do_inodes = 0;
+ error("DISABLED: disk_inodes.X");
+ } else {
+ for (i = 0; i < mntsize; i++) {
+ if (mntbuf[i].f_flags == MNT_RDONLY ||
+ mntbuf[i].f_blocks == 0 ||
+ // taken from gnulib/mountlist.c and shortened to FreeBSD related fstypes
+ strcmp(mntbuf[i].f_fstypename, "autofs") == 0 ||
+ strcmp(mntbuf[i].f_fstypename, "procfs") == 0 ||
+ strcmp(mntbuf[i].f_fstypename, "subfs") == 0 ||
+ strcmp(mntbuf[i].f_fstypename, "devfs") == 0 ||
+ strcmp(mntbuf[i].f_fstypename, "none") == 0)
+ continue;
+
+ // --------------------------------------------------------------------------
+
+ if (likely(do_space)) {
+ st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
+ if (unlikely(!st)) {
+ snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
+ st = rrdset_create_localhost(
+ "disk_space"
+ , mntbuf[i].f_mntonname
+ , NULL
+ , mntbuf[i].f_mntonname
+ , "disk.space"
+ , title
+ , "GiB"
+ , "macos.plugin"
+ , "iokit"
+ , 2023
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrddim_add(st, "avail", NULL, mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "used", NULL, mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "reserved_for_root", "reserved for root", mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "avail", (collected_number) mntbuf[i].f_bavail);
+ rrddim_set(st, "used", (collected_number) (mntbuf[i].f_blocks - mntbuf[i].f_bfree));
+ rrddim_set(st, "reserved_for_root", (collected_number) (mntbuf[i].f_bfree - mntbuf[i].f_bavail));
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------------
+
+ if (likely(do_inodes)) {
+ st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
+ if (unlikely(!st)) {
+ snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
+ st = rrdset_create_localhost(
+ "disk_inodes"
+ , mntbuf[i].f_mntonname
+ , NULL
+ , mntbuf[i].f_mntonname
+ , "disk.inodes"
+ , title
+ , "inodes"
+ , "macos.plugin"
+ , "iokit"
+ , 2024
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrddim_add(st, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "reserved_for_root", "reserved for root", 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "avail", (collected_number) mntbuf[i].f_ffree);
+ rrddim_set(st, "used", (collected_number) (mntbuf[i].f_files - mntbuf[i].f_ffree));
+ rrdset_done(st);
+ }
+ }
+ }
+ }
+
+ // Can be merged with FreeBSD plugin
+
+ if (likely(do_bandwidth)) {
+ if (unlikely(getifaddrs(&ifap))) {
+ error("MACOS: getifaddrs()");
+ do_bandwidth = 0;
+ error("DISABLED: system.ipv4");
+ } else {
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_LINK)
+ continue;
+
+ st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "net"
+ , ifa->ifa_name
+ , NULL
+ , ifa->ifa_name
+ , "net.net"
+ , "Bandwidth"
+ , "kilobits/s"
+ , "macos.plugin"
+ , "iokit"
+ , 7000
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "received", IFA_DATA(ibytes));
+ rrddim_set(st, "sent", IFA_DATA(obytes));
+ rrdset_done(st);
+
+ st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "net_packets"
+ , ifa->ifa_name
+ , NULL
+ , ifa->ifa_name
+ , "net.packets"
+ , "Packets"
+ , "packets/s"
+ , "macos.plugin"
+ , "iokit"
+ , 7001
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "multicast_received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "multicast_sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "received", IFA_DATA(ipackets));
+ rrddim_set(st, "sent", IFA_DATA(opackets));
+ rrddim_set(st, "multicast_received", IFA_DATA(imcasts));
+ rrddim_set(st, "multicast_sent", IFA_DATA(omcasts));
+ rrdset_done(st);
+
+ st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "net_errors"
+ , ifa->ifa_name
+ , NULL
+ , ifa->ifa_name
+ , "net.errors"
+ , "Interface Errors"
+ , "errors/s"
+ , "macos.plugin"
+ , "iokit"
+ , 7002
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "inbound", IFA_DATA(ierrors));
+ rrddim_set(st, "outbound", IFA_DATA(oerrors));
+ rrdset_done(st);
+
+ st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "net_drops"
+ , ifa->ifa_name
+ , NULL
+ , ifa->ifa_name
+ , "net.drops"
+ , "Interface Drops"
+ , "drops/s"
+ , "macos.plugin"
+ , "iokit"
+ , 7003
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "inbound", IFA_DATA(iqdrops));
+ rrdset_done(st);
+
+ st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name);
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "net_events"
+ , ifa->ifa_name
+ , NULL
+ , ifa->ifa_name
+ , "net.events"
+ , "Network Interface Events"
+ , "events/s"
+ , "macos.plugin"
+ , "iokit"
+ , 7006
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "frames", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "collisions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "carrier", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "collisions", IFA_DATA(collisions));
+ rrdset_done(st);
+ }
+
+ freeifaddrs(ifap);
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/macos.plugin/macos_mach_smi.c b/collectors/macos.plugin/macos_mach_smi.c
new file mode 100644
index 0000000..53b2607
--- /dev/null
+++ b/collectors/macos.plugin/macos_mach_smi.c
@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_macos.h"
+
+#include <mach/mach.h>
+
+int do_macos_mach_smi(int update_every, usec_t dt) {
+ (void)dt;
+
+ static int do_cpu = -1, do_ram = - 1, do_swapio = -1, do_pgfaults = -1;
+
+ if (unlikely(do_cpu == -1)) {
+ do_cpu = config_get_boolean("plugin:macos:mach_smi", "cpu utilization", 1);
+ do_ram = config_get_boolean("plugin:macos:mach_smi", "system ram", 1);
+ do_swapio = config_get_boolean("plugin:macos:mach_smi", "swap i/o", 1);
+ do_pgfaults = config_get_boolean("plugin:macos:mach_smi", "memory page faults", 1);
+ }
+
+ RRDSET *st;
+
+ kern_return_t kr;
+ mach_msg_type_number_t count;
+ host_t host;
+ vm_size_t system_pagesize;
+
+
+ // NEEDED BY: do_cpu
+ natural_t cp_time[CPU_STATE_MAX];
+
+ // NEEDED BY: do_ram, do_swapio, do_pgfaults
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060)
+ vm_statistics64_data_t vm_statistics;
+#else
+ vm_statistics_data_t vm_statistics;
+#endif
+
+ host = mach_host_self();
+ kr = host_page_size(host, &system_pagesize);
+ if (unlikely(kr != KERN_SUCCESS))
+ return -1;
+
+ if (likely(do_cpu)) {
+ if (unlikely(HOST_CPU_LOAD_INFO_COUNT != 4)) {
+ error("MACOS: There are %d CPU states (4 was expected)", HOST_CPU_LOAD_INFO_COUNT);
+ do_cpu = 0;
+ error("DISABLED: system.cpu");
+ } else {
+ count = HOST_CPU_LOAD_INFO_COUNT;
+ kr = host_statistics(host, HOST_CPU_LOAD_INFO, (host_info_t)cp_time, &count);
+ if (unlikely(kr != KERN_SUCCESS)) {
+ error("MACOS: host_statistics() failed: %s", mach_error_string(kr));
+ do_cpu = 0;
+ error("DISABLED: system.cpu");
+ } else {
+
+ st = rrdset_find_active_bytype_localhost("system", "cpu");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "cpu"
+ , NULL
+ , "cpu"
+ , "system.cpu"
+ , "Total CPU utilization"
+ , "percentage"
+ , "macos.plugin"
+ , "mach_smi"
+ , 100
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrddim_add(st, "user", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rrddim_add(st, "nice", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rrddim_add(st, "system", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rrddim_add(st, "idle", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rrddim_hide(st, "idle");
+ }
+
+ rrddim_set(st, "user", cp_time[CPU_STATE_USER]);
+ rrddim_set(st, "nice", cp_time[CPU_STATE_NICE]);
+ rrddim_set(st, "system", cp_time[CPU_STATE_SYSTEM]);
+ rrddim_set(st, "idle", cp_time[CPU_STATE_IDLE]);
+ rrdset_done(st);
+ }
+ }
+ }
+
+ if (likely(do_ram || do_swapio || do_pgfaults)) {
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060)
+ count = sizeof(vm_statistics64_data_t);
+ kr = host_statistics64(host, HOST_VM_INFO64, (host_info64_t)&vm_statistics, &count);
+#else
+ count = sizeof(vm_statistics_data_t);
+ kr = host_statistics(host, HOST_VM_INFO, (host_info_t)&vm_statistics, &count);
+#endif
+ if (unlikely(kr != KERN_SUCCESS)) {
+ error("MACOS: host_statistics64() failed: %s", mach_error_string(kr));
+ do_ram = 0;
+ error("DISABLED: system.ram");
+ do_swapio = 0;
+ error("DISABLED: system.swapio");
+ do_pgfaults = 0;
+ error("DISABLED: mem.pgfaults");
+ } else {
+ if (likely(do_ram)) {
+ st = rrdset_find_active_localhost("system.ram");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "ram"
+ , NULL
+ , "ram"
+ , NULL
+ , "System RAM"
+ , "MiB"
+ , "macos.plugin"
+ , "mach_smi"
+ , 200
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrddim_add(st, "active", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "wired", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE);
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090)
+ rrddim_add(st, "throttled", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "compressor", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE);
+#endif
+ rrddim_add(st, "inactive", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "purgeable", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "speculative", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "free", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "active", vm_statistics.active_count);
+ rrddim_set(st, "wired", vm_statistics.wire_count);
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090)
+ rrddim_set(st, "throttled", vm_statistics.throttled_count);
+ rrddim_set(st, "compressor", vm_statistics.compressor_page_count);
+#endif
+ rrddim_set(st, "inactive", vm_statistics.inactive_count);
+ rrddim_set(st, "purgeable", vm_statistics.purgeable_count);
+ rrddim_set(st, "speculative", vm_statistics.speculative_count);
+ rrddim_set(st, "free", (vm_statistics.free_count - vm_statistics.speculative_count));
+ rrdset_done(st);
+ }
+
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090)
+ if (likely(do_swapio)) {
+ st = rrdset_find_active_localhost("system.swapio");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "swapio"
+ , NULL
+ , "swap"
+ , NULL
+ , "Swap I/O"
+ , "KiB/s"
+ , "macos.plugin"
+ , "mach_smi"
+ , 250
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrddim_add(st, "in", NULL, system_pagesize, 1024, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "out", NULL, -system_pagesize, 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "in", vm_statistics.swapins);
+ rrddim_set(st, "out", vm_statistics.swapouts);
+ rrdset_done(st);
+ }
+#endif
+
+ if (likely(do_pgfaults)) {
+ st = rrdset_find_active_localhost("mem.pgfaults");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "mem"
+ , "pgfaults"
+ , NULL
+ , "system"
+ , NULL
+ , "Memory Page Faults"
+ , "faults/s"
+ , "macos.plugin"
+ , "mach_smi"
+ , NETDATA_CHART_PRIO_MEM_SYSTEM_PGFAULTS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "memory", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "cow", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "pagein", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "pageout", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090)
+ rrddim_add(st, "compress", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "decompress", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#endif
+ rrddim_add(st, "zero_fill", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "reactivate", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "purge", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "memory", vm_statistics.faults);
+ rrddim_set(st, "cow", vm_statistics.cow_faults);
+ rrddim_set(st, "pagein", vm_statistics.pageins);
+ rrddim_set(st, "pageout", vm_statistics.pageouts);
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090)
+ rrddim_set(st, "compress", vm_statistics.compressions);
+ rrddim_set(st, "decompress", vm_statistics.decompressions);
+#endif
+ rrddim_set(st, "zero_fill", vm_statistics.zero_fill_count);
+ rrddim_set(st, "reactivate", vm_statistics.reactivations);
+ rrddim_set(st, "purge", vm_statistics.purges);
+ rrdset_done(st);
+ }
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/macos.plugin/macos_sysctl.c b/collectors/macos.plugin/macos_sysctl.c
new file mode 100644
index 0000000..1f04f6e
--- /dev/null
+++ b/collectors/macos.plugin/macos_sysctl.c
@@ -0,0 +1,1424 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_macos.h"
+
+#include <Availability.h>
+// NEEDED BY: do_bandwidth
+#include <net/route.h>
+// NEEDED BY do_tcp...
+#include <sys/socketvar.h>
+#include <netinet/tcp_var.h>
+#include <netinet/tcp_fsm.h>
+// NEEDED BY do_udp..., do_ip...
+#include <netinet/ip_var.h>
+// NEEDED BY do_udp...
+#include <netinet/udp.h>
+#include <netinet/udp_var.h>
+// NEEDED BY do_icmp...
+#include <netinet/ip.h>
+#include <netinet/ip_icmp.h>
+#include <netinet/icmp_var.h>
+// NEEDED BY do_icmp6...
+#include <netinet/icmp6.h>
+// NEEDED BY do_uptime
+#include <time.h>
+
+// MacOS calculates load averages once every 5 seconds
+#define MIN_LOADAVG_UPDATE_EVERY 5
+
+int do_macos_sysctl(int update_every, usec_t dt) {
+ static int do_loadavg = -1, do_swap = -1, do_bandwidth = -1,
+ do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1, do_ecn = -1,
+ do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1,
+ do_udp_packets = -1, do_udp_errors = -1, do_icmp_packets = -1, do_icmpmsg = -1,
+ do_ip_packets = -1, do_ip_fragsout = -1, do_ip_fragsin = -1, do_ip_errors = -1,
+ do_ip6_packets = -1, do_ip6_fragsout = -1, do_ip6_fragsin = -1, do_ip6_errors = -1,
+ do_icmp6 = -1, do_icmp6_redir = -1, do_icmp6_errors = -1, do_icmp6_echos = -1,
+ do_icmp6_router = -1, do_icmp6_neighbor = -1, do_icmp6_types = -1, do_uptime = -1;
+
+
+ if (unlikely(do_loadavg == -1)) {
+ do_loadavg = config_get_boolean("plugin:macos:sysctl", "enable load average", 1);
+ do_swap = config_get_boolean("plugin:macos:sysctl", "system swap", 1);
+ do_bandwidth = config_get_boolean("plugin:macos:sysctl", "bandwidth", 1);
+ do_tcp_packets = config_get_boolean("plugin:macos:sysctl", "ipv4 TCP packets", 1);
+ do_tcp_errors = config_get_boolean("plugin:macos:sysctl", "ipv4 TCP errors", 1);
+ do_tcp_handshake = config_get_boolean("plugin:macos:sysctl", "ipv4 TCP handshake issues", 1);
+ do_ecn = config_get_boolean_ondemand("plugin:macos:sysctl", "ECN packets", CONFIG_BOOLEAN_AUTO);
+ do_tcpext_syscookies = config_get_boolean_ondemand("plugin:macos:sysctl", "TCP SYN cookies", CONFIG_BOOLEAN_AUTO);
+ do_tcpext_ofo = config_get_boolean_ondemand("plugin:macos:sysctl", "TCP out-of-order queue", CONFIG_BOOLEAN_AUTO);
+ do_tcpext_connaborts = config_get_boolean_ondemand("plugin:macos:sysctl", "TCP connection aborts", CONFIG_BOOLEAN_AUTO);
+ do_udp_packets = config_get_boolean("plugin:macos:sysctl", "ipv4 UDP packets", 1);
+ do_udp_errors = config_get_boolean("plugin:macos:sysctl", "ipv4 UDP errors", 1);
+ do_icmp_packets = config_get_boolean("plugin:macos:sysctl", "ipv4 ICMP packets", 1);
+ do_icmpmsg = config_get_boolean("plugin:macos:sysctl", "ipv4 ICMP messages", 1);
+ do_ip_packets = config_get_boolean("plugin:macos:sysctl", "ipv4 packets", 1);
+ do_ip_fragsout = config_get_boolean("plugin:macos:sysctl", "ipv4 fragments sent", 1);
+ do_ip_fragsin = config_get_boolean("plugin:macos:sysctl", "ipv4 fragments assembly", 1);
+ do_ip_errors = config_get_boolean("plugin:macos:sysctl", "ipv4 errors", 1);
+ do_ip6_packets = config_get_boolean_ondemand("plugin:macos:sysctl", "ipv6 packets", CONFIG_BOOLEAN_AUTO);
+ do_ip6_fragsout = config_get_boolean_ondemand("plugin:macos:sysctl", "ipv6 fragments sent", CONFIG_BOOLEAN_AUTO);
+ do_ip6_fragsin = config_get_boolean_ondemand("plugin:macos:sysctl", "ipv6 fragments assembly", CONFIG_BOOLEAN_AUTO);
+ do_ip6_errors = config_get_boolean_ondemand("plugin:macos:sysctl", "ipv6 errors", CONFIG_BOOLEAN_AUTO);
+ do_icmp6 = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp", CONFIG_BOOLEAN_AUTO);
+ do_icmp6_redir = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp redirects", CONFIG_BOOLEAN_AUTO);
+ do_icmp6_errors = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp errors", CONFIG_BOOLEAN_AUTO);
+ do_icmp6_echos = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp echos", CONFIG_BOOLEAN_AUTO);
+ do_icmp6_router = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp router", CONFIG_BOOLEAN_AUTO);
+ do_icmp6_neighbor = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp neighbor", CONFIG_BOOLEAN_AUTO);
+ do_icmp6_types = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp types", CONFIG_BOOLEAN_AUTO);
+ do_uptime = config_get_boolean("plugin:macos:sysctl", "system uptime", 1);
+ }
+
+ RRDSET *st = NULL;
+
+ int i;
+ size_t size;
+
+ // NEEDED BY: do_loadavg
+ static usec_t next_loadavg_dt = 0;
+ struct loadavg sysload;
+
+ // NEEDED BY: do_swap
+ struct xsw_usage swap_usage;
+
+ // NEEDED BY: do_bandwidth
+ int mib[6];
+ static char *ifstatdata = NULL;
+ char *lim, *next;
+ struct if_msghdr *ifm;
+ struct iftot {
+ u_long ift_ibytes;
+ u_long ift_obytes;
+ } iftot = {0, 0};
+
+ // NEEDED BY: do_tcp...
+ struct tcpstat tcpstat;
+
+ // NEEDED BY: do_udp...
+ struct udpstat udpstat;
+
+ // NEEDED BY: do_icmp...
+ struct icmpstat icmpstat;
+ struct icmp_total {
+ u_long msgs_in;
+ u_long msgs_out;
+ } icmp_total = {0, 0};
+
+ // NEEDED BY: do_ip...
+ struct ipstat ipstat;
+
+ // NEEDED BY: do_ip6...
+ /*
+ * Dirty workaround for /usr/include/netinet6/ip6_var.h absence.
+ * Struct ip6stat was copied from bsd/netinet6/ip6_var.h from xnu sources.
+ * Do the same for previously missing scope6_var.h on OS X < 10.11.
+ */
+#define IP6S_SRCRULE_COUNT 16
+
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED < 101100)
+#ifndef _NETINET6_SCOPE6_VAR_H_
+#define _NETINET6_SCOPE6_VAR_H_
+#include <sys/appleapiopts.h>
+
+#define SCOPE6_ID_MAX 16
+#endif
+#else
+#include <netinet6/scope6_var.h>
+#endif
+
+ struct ip6stat {
+ u_quad_t ip6s_total; /* total packets received */
+ u_quad_t ip6s_tooshort; /* packet too short */
+ u_quad_t ip6s_toosmall; /* not enough data */
+ u_quad_t ip6s_fragments; /* fragments received */
+ u_quad_t ip6s_fragdropped; /* frags dropped(dups, out of space) */
+ u_quad_t ip6s_fragtimeout; /* fragments timed out */
+ u_quad_t ip6s_fragoverflow; /* fragments that exceeded limit */
+ u_quad_t ip6s_forward; /* packets forwarded */
+ u_quad_t ip6s_cantforward; /* packets rcvd for unreachable dest */
+ u_quad_t ip6s_redirectsent; /* packets forwarded on same net */
+ u_quad_t ip6s_delivered; /* datagrams delivered to upper level */
+ u_quad_t ip6s_localout; /* total ip packets generated here */
+ u_quad_t ip6s_odropped; /* lost packets due to nobufs, etc. */
+ u_quad_t ip6s_reassembled; /* total packets reassembled ok */
+ u_quad_t ip6s_atmfrag_rcvd; /* atomic fragments received */
+ u_quad_t ip6s_fragmented; /* datagrams successfully fragmented */
+ u_quad_t ip6s_ofragments; /* output fragments created */
+ u_quad_t ip6s_cantfrag; /* don't fragment flag was set, etc. */
+ u_quad_t ip6s_badoptions; /* error in option processing */
+ u_quad_t ip6s_noroute; /* packets discarded due to no route */
+ u_quad_t ip6s_badvers; /* ip6 version != 6 */
+ u_quad_t ip6s_rawout; /* total raw ip packets generated */
+ u_quad_t ip6s_badscope; /* scope error */
+ u_quad_t ip6s_notmember; /* don't join this multicast group */
+ u_quad_t ip6s_nxthist[256]; /* next header history */
+ u_quad_t ip6s_m1; /* one mbuf */
+ u_quad_t ip6s_m2m[32]; /* two or more mbuf */
+ u_quad_t ip6s_mext1; /* one ext mbuf */
+ u_quad_t ip6s_mext2m; /* two or more ext mbuf */
+ u_quad_t ip6s_exthdrtoolong; /* ext hdr are not continuous */
+ u_quad_t ip6s_nogif; /* no match gif found */
+ u_quad_t ip6s_toomanyhdr; /* discarded due to too many headers */
+
+ /*
+ * statistics for improvement of the source address selection
+ * algorithm:
+ */
+ /* number of times that address selection fails */
+ u_quad_t ip6s_sources_none;
+ /* number of times that an address on the outgoing I/F is chosen */
+ u_quad_t ip6s_sources_sameif[SCOPE6_ID_MAX];
+ /* number of times that an address on a non-outgoing I/F is chosen */
+ u_quad_t ip6s_sources_otherif[SCOPE6_ID_MAX];
+ /*
+ * number of times that an address that has the same scope
+ * from the destination is chosen.
+ */
+ u_quad_t ip6s_sources_samescope[SCOPE6_ID_MAX];
+ /*
+ * number of times that an address that has a different scope
+ * from the destination is chosen.
+ */
+ u_quad_t ip6s_sources_otherscope[SCOPE6_ID_MAX];
+ /* number of times that a deprecated address is chosen */
+ u_quad_t ip6s_sources_deprecated[SCOPE6_ID_MAX];
+
+ u_quad_t ip6s_forward_cachehit;
+ u_quad_t ip6s_forward_cachemiss;
+
+ /* number of times that each rule of source selection is applied. */
+ u_quad_t ip6s_sources_rule[IP6S_SRCRULE_COUNT];
+
+ /* number of times we ignored address on expensive secondary interfaces */
+ u_quad_t ip6s_sources_skip_expensive_secondary_if;
+
+ /* pkt dropped, no mbufs for control data */
+ u_quad_t ip6s_pktdropcntrl;
+
+ /* total packets trimmed/adjusted */
+ u_quad_t ip6s_adj;
+ /* hwcksum info discarded during adjustment */
+ u_quad_t ip6s_adj_hwcsum_clr;
+
+ /* duplicate address detection collisions */
+ u_quad_t ip6s_dad_collide;
+
+ /* DAD NS looped back */
+ u_quad_t ip6s_dad_loopcount;
+ } ip6stat;
+
+ // NEEDED BY: do_icmp6...
+ struct icmp6stat icmp6stat;
+ struct icmp6_total {
+ u_long msgs_in;
+ u_long msgs_out;
+ } icmp6_total = {0, 0};
+
+ // NEEDED BY: do_uptime
+ struct timespec boot_time, cur_time;
+
+ if (next_loadavg_dt <= dt) {
+ if (likely(do_loadavg)) {
+ if (unlikely(GETSYSCTL_BY_NAME("vm.loadavg", sysload))) {
+ do_loadavg = 0;
+ error("DISABLED: system.load");
+ } else {
+
+ st = rrdset_find_active_bytype_localhost("system", "load");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "load"
+ , NULL
+ , "load"
+ , NULL
+ , "System Load Average"
+ , "load"
+ , "macos.plugin"
+ , "sysctl"
+ , 100
+ , (update_every < MIN_LOADAVG_UPDATE_EVERY) ? MIN_LOADAVG_UPDATE_EVERY : update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrddim_add(st, "load1", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "load5", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "load15", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "load1", (collected_number) ((double)sysload.ldavg[0] / sysload.fscale * 1000));
+ rrddim_set(st, "load5", (collected_number) ((double)sysload.ldavg[1] / sysload.fscale * 1000));
+ rrddim_set(st, "load15", (collected_number) ((double)sysload.ldavg[2] / sysload.fscale * 1000));
+ rrdset_done(st);
+ }
+ }
+
+ next_loadavg_dt = st->update_every * USEC_PER_SEC;
+ }
+ else next_loadavg_dt -= dt;
+
+ if (likely(do_swap)) {
+ if (unlikely(GETSYSCTL_BY_NAME("vm.swapusage", swap_usage))) {
+ do_swap = 0;
+ error("DISABLED: system.swap");
+ } else {
+ st = rrdset_find_active_localhost("system.swap");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "swap"
+ , NULL
+ , "swap"
+ , NULL
+ , "System Swap"
+ , "MiB"
+ , "macos.plugin"
+ , "sysctl"
+ , 201
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "free", NULL, 1, 1048576, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_add(st, "used", NULL, 1, 1048576, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "free", swap_usage.xsu_avail);
+ rrddim_set(st, "used", swap_usage.xsu_used);
+ rrdset_done(st);
+ }
+ }
+
+ if (likely(do_bandwidth)) {
+ mib[0] = CTL_NET;
+ mib[1] = PF_ROUTE;
+ mib[2] = 0;
+ mib[3] = AF_INET;
+ mib[4] = NET_RT_IFLIST2;
+ mib[5] = 0;
+ if (unlikely(sysctl(mib, 6, NULL, &size, NULL, 0))) {
+ error("MACOS: sysctl(%s...) failed: %s", "net interfaces", strerror(errno));
+ do_bandwidth = 0;
+ error("DISABLED: system.ipv4");
+ } else {
+ ifstatdata = reallocz(ifstatdata, size);
+ if (unlikely(sysctl(mib, 6, ifstatdata, &size, NULL, 0) < 0)) {
+ error("MACOS: sysctl(%s...) failed: %s", "net interfaces", strerror(errno));
+ do_bandwidth = 0;
+ error("DISABLED: system.ipv4");
+ } else {
+ lim = ifstatdata + size;
+ iftot.ift_ibytes = iftot.ift_obytes = 0;
+ for (next = ifstatdata; next < lim; ) {
+ ifm = (struct if_msghdr *)next;
+ next += ifm->ifm_msglen;
+
+ if (ifm->ifm_type == RTM_IFINFO2) {
+ struct if_msghdr2 *if2m = (struct if_msghdr2 *)ifm;
+
+ iftot.ift_ibytes += if2m->ifm_data.ifi_ibytes;
+ iftot.ift_obytes += if2m->ifm_data.ifi_obytes;
+ }
+ }
+ st = rrdset_find_active_localhost("system.ipv4");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "ipv4"
+ , NULL
+ , "network"
+ , NULL
+ , "IPv4 Bandwidth"
+ , "kilobits/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 500
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrddim_add(st, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InOctets", iftot.ift_ibytes);
+ rrddim_set(st, "OutOctets", iftot.ift_obytes);
+ rrdset_done(st);
+ }
+ }
+ }
+
+ // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html
+ if (likely(do_tcp_packets || do_tcp_errors || do_tcp_handshake || do_tcpext_connaborts || do_tcpext_ofo || do_tcpext_syscookies || do_ecn)) {
+ if (unlikely(GETSYSCTL_BY_NAME("net.inet.tcp.stats", tcpstat))){
+ do_tcp_packets = 0;
+ error("DISABLED: ipv4.tcppackets");
+ do_tcp_errors = 0;
+ error("DISABLED: ipv4.tcperrors");
+ do_tcp_handshake = 0;
+ error("DISABLED: ipv4.tcphandshake");
+ do_tcpext_connaborts = 0;
+ error("DISABLED: ipv4.tcpconnaborts");
+ do_tcpext_ofo = 0;
+ error("DISABLED: ipv4.tcpofo");
+ do_tcpext_syscookies = 0;
+ error("DISABLED: ipv4.tcpsyncookies");
+ do_ecn = 0;
+ error("DISABLED: ipv4.ecnpkts");
+ } else {
+ if (likely(do_tcp_packets)) {
+ st = rrdset_find_active_localhost("ipv4.tcppackets");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "tcppackets"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Packets"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 2600
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InSegs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutSegs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InSegs", tcpstat.tcps_rcvtotal);
+ rrddim_set(st, "OutSegs", tcpstat.tcps_sndtotal);
+ rrdset_done(st);
+ }
+
+ if (likely(do_tcp_errors)) {
+ st = rrdset_find_active_localhost("ipv4.tcperrors");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "tcperrors"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Errors"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 2700
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "InErrs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "RetransSegs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InErrs", tcpstat.tcps_rcvbadoff + tcpstat.tcps_rcvshort);
+ rrddim_set(st, "InCsumErrors", tcpstat.tcps_rcvbadsum);
+ rrddim_set(st, "RetransSegs", tcpstat.tcps_sndrexmitpack);
+ rrdset_done(st);
+ }
+
+ if (likely(do_tcp_handshake)) {
+ st = rrdset_find_active_localhost("ipv4.tcphandshake");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "tcphandshake"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Handshake Issues"
+ , "events/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 2900
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "EstabResets", tcpstat.tcps_drops);
+ rrddim_set(st, "ActiveOpens", tcpstat.tcps_connattempt);
+ rrddim_set(st, "PassiveOpens", tcpstat.tcps_accepts);
+ rrddim_set(st, "AttemptFails", tcpstat.tcps_conndrops);
+ rrdset_done(st);
+ }
+
+ if (do_tcpext_connaborts == CONFIG_BOOLEAN_YES || (do_tcpext_connaborts == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_rcvpackafterwin ||
+ tcpstat.tcps_rcvafterclose ||
+ tcpstat.tcps_rcvmemdrop ||
+ tcpstat.tcps_persistdrop ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_connaborts = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv4.tcpconnaborts");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "tcpconnaborts"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP Connection Aborts"
+ , "connections/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3010
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "TCPAbortOnData", "baddata", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "TCPAbortOnClose", "userclosed", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "TCPAbortOnMemory", "nomemory", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "TCPAbortOnTimeout", "timeout", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "TCPAbortOnData", tcpstat.tcps_rcvpackafterwin);
+ rrddim_set(st, "TCPAbortOnClose", tcpstat.tcps_rcvafterclose);
+ rrddim_set(st, "TCPAbortOnMemory", tcpstat.tcps_rcvmemdrop);
+ rrddim_set(st, "TCPAbortOnTimeout", tcpstat.tcps_persistdrop);
+ rrdset_done(st);
+ }
+
+ if (do_tcpext_ofo == CONFIG_BOOLEAN_YES || (do_tcpext_ofo == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_rcvoopack ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_ofo = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv4.tcpofo");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "tcpofo"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP Out-Of-Order Queue"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3050
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "TCPOFOQueue", "inqueue", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "TCPOFOQueue", tcpstat.tcps_rcvoopack);
+ rrdset_done(st);
+ }
+
+ if (do_tcpext_syscookies == CONFIG_BOOLEAN_YES || (do_tcpext_syscookies == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_sc_sendcookie ||
+ tcpstat.tcps_sc_recvcookie ||
+ tcpstat.tcps_sc_zonefail ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_syscookies = CONFIG_BOOLEAN_YES;
+
+ st = rrdset_find_active_localhost("ipv4.tcpsyncookies");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "tcpsyncookies"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP SYN Cookies"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3100
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "SyncookiesRecv", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "SyncookiesSent", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "SyncookiesFailed", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "SyncookiesRecv", tcpstat.tcps_sc_recvcookie);
+ rrddim_set(st, "SyncookiesSent", tcpstat.tcps_sc_sendcookie);
+ rrddim_set(st, "SyncookiesFailed", tcpstat.tcps_sc_zonefail);
+ rrdset_done(st);
+ }
+
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101100)
+ if (do_ecn == CONFIG_BOOLEAN_YES || (do_ecn == CONFIG_BOOLEAN_AUTO &&
+ (tcpstat.tcps_ecn_recv_ce ||
+ tcpstat.tcps_ecn_not_supported ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ecn = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv4.ecnpkts");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "ecnpkts"
+ , NULL
+ , "ecn"
+ , NULL
+ , "IPv4 ECN Statistics"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 8700
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "InCEPkts", "CEP", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InNoECTPkts", "NoECTP", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InCEPkts", tcpstat.tcps_ecn_recv_ce);
+ rrddim_set(st, "InNoECTPkts", tcpstat.tcps_ecn_not_supported);
+ rrdset_done(st);
+ }
+#endif
+
+ }
+ }
+
+ // see http://net-snmp.sourceforge.net/docs/mibs/udp.html
+ if (likely(do_udp_packets || do_udp_errors)) {
+ if (unlikely(GETSYSCTL_BY_NAME("net.inet.udp.stats", udpstat))) {
+ do_udp_packets = 0;
+ error("DISABLED: ipv4.udppackets");
+ do_udp_errors = 0;
+ error("DISABLED: ipv4.udperrors");
+ } else {
+ if (likely(do_udp_packets)) {
+ st = rrdset_find_active_localhost("ipv4.udppackets");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "udppackets"
+ , NULL
+ , "udp"
+ , NULL
+ , "IPv4 UDP Packets"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 2601
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InDatagrams", udpstat.udps_ipackets);
+ rrddim_set(st, "OutDatagrams", udpstat.udps_opackets);
+ rrdset_done(st);
+ }
+
+ if (likely(do_udp_errors)) {
+ st = rrdset_find_active_localhost("ipv4.udperrors");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "udperrors"
+ , NULL
+ , "udp"
+ , NULL
+ , "IPv4 UDP Errors"
+ , "events/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 2701
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090)
+ rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#endif
+ }
+
+ rrddim_set(st, "InErrors", udpstat.udps_hdrops + udpstat.udps_badlen);
+ rrddim_set(st, "NoPorts", udpstat.udps_noport);
+ rrddim_set(st, "RcvbufErrors", udpstat.udps_fullsock);
+#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090)
+ rrddim_set(st, "InCsumErrors", udpstat.udps_badsum + udpstat.udps_nosum);
+ rrddim_set(st, "IgnoredMulti", udpstat.udps_filtermcast);
+#else
+ rrddim_set(st, "InCsumErrors", udpstat.udps_badsum);
+#endif
+ rrdset_done(st);
+ }
+ }
+ }
+
+ if (likely(do_icmp_packets || do_icmpmsg)) {
+ if (unlikely(GETSYSCTL_BY_NAME("net.inet.icmp.stats", icmpstat))) {
+ do_icmp_packets = 0;
+ error("DISABLED: ipv4.icmp");
+ error("DISABLED: ipv4.icmp_errors");
+ do_icmpmsg = 0;
+ error("DISABLED: ipv4.icmpmsg");
+ } else {
+ for (i = 0; i <= ICMP_MAXTYPE; i++) {
+ icmp_total.msgs_in += icmpstat.icps_inhist[i];
+ icmp_total.msgs_out += icmpstat.icps_outhist[i];
+ }
+ icmp_total.msgs_in += icmpstat.icps_badcode + icmpstat.icps_badlen + icmpstat.icps_checksum + icmpstat.icps_tooshort;
+
+ // --------------------------------------------------------------------
+
+ if (likely(do_icmp_packets)) {
+ st = rrdset_find_active_localhost("ipv4.icmp");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "icmp"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Packets"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 2602
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InMsgs", icmp_total.msgs_in);
+ rrddim_set(st, "OutMsgs", icmp_total.msgs_out);
+ rrdset_done(st);
+
+ st = rrdset_find_active_localhost("ipv4.icmp_errors");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "icmp_errors"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Errors"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 2603
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InErrors", icmpstat.icps_badcode + icmpstat.icps_badlen + icmpstat.icps_checksum + icmpstat.icps_tooshort);
+ rrddim_set(st, "OutErrors", icmpstat.icps_error);
+ rrddim_set(st, "InCsumErrors", icmpstat.icps_checksum);
+ rrdset_done(st);
+ }
+
+ if (likely(do_icmpmsg)) {
+ st = rrdset_find_active_localhost("ipv4.icmpmsg");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "icmpmsg"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Messages"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 2604
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InEchoReps", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutEchoReps", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutEchos", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InEchoReps", icmpstat.icps_inhist[ICMP_ECHOREPLY]);
+ rrddim_set(st, "OutEchoReps", icmpstat.icps_outhist[ICMP_ECHOREPLY]);
+ rrddim_set(st, "InEchos", icmpstat.icps_inhist[ICMP_ECHO]);
+ rrddim_set(st, "OutEchos", icmpstat.icps_outhist[ICMP_ECHO]);
+ rrdset_done(st);
+ }
+ }
+ }
+
+ // see also http://net-snmp.sourceforge.net/docs/mibs/ip.html
+ if (likely(do_ip_packets || do_ip_fragsout || do_ip_fragsin || do_ip_errors)) {
+ if (unlikely(GETSYSCTL_BY_NAME("net.inet.ip.stats", ipstat))) {
+ do_ip_packets = 0;
+ error("DISABLED: ipv4.packets");
+ do_ip_fragsout = 0;
+ error("DISABLED: ipv4.fragsout");
+ do_ip_fragsin = 0;
+ error("DISABLED: ipv4.fragsin");
+ do_ip_errors = 0;
+ error("DISABLED: ipv4.errors");
+ } else {
+ if (likely(do_ip_packets)) {
+ st = rrdset_find_active_localhost("ipv4.packets");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "packets"
+ , NULL
+ , "packets"
+ , NULL
+ , "IPv4 Packets"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3000
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "ForwDatagrams", "forwarded", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InDelivers", "delivered", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "OutRequests", ipstat.ips_localout);
+ rrddim_set(st, "InReceives", ipstat.ips_total);
+ rrddim_set(st, "ForwDatagrams", ipstat.ips_forward);
+ rrddim_set(st, "InDelivers", ipstat.ips_delivered);
+ rrdset_done(st);
+ }
+
+ if (likely(do_ip_fragsout)) {
+ st = rrdset_find_active_localhost("ipv4.fragsout");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "fragsout"
+ , NULL
+ , "fragments"
+ , NULL
+ , "IPv4 Fragments Sent"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3010
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "FragOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "FragFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "FragCreates", "created", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "FragOKs", ipstat.ips_fragmented);
+ rrddim_set(st, "FragFails", ipstat.ips_cantfrag);
+ rrddim_set(st, "FragCreates", ipstat.ips_ofragments);
+ rrdset_done(st);
+ }
+
+ if (likely(do_ip_fragsin)) {
+ st = rrdset_find_active_localhost("ipv4.fragsin");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "fragsin"
+ , NULL
+ , "fragments"
+ , NULL
+ , "IPv4 Fragments Reassembly"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3011
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "ReasmFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "ReasmReqds", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "ReasmOKs", ipstat.ips_fragments);
+ rrddim_set(st, "ReasmFails", ipstat.ips_fragdropped);
+ rrddim_set(st, "ReasmReqds", ipstat.ips_reassembled);
+ rrdset_done(st);
+ }
+
+ if (likely(do_ip_errors)) {
+ st = rrdset_find_active_localhost("ipv4.errors");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "errors"
+ , NULL
+ , "errors"
+ , NULL
+ , "IPv4 Errors"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3002
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InDiscards", ipstat.ips_badsum + ipstat.ips_tooshort + ipstat.ips_toosmall + ipstat.ips_toolong);
+ rrddim_set(st, "OutDiscards", ipstat.ips_odropped);
+ rrddim_set(st, "InHdrErrors", ipstat.ips_badhlen + ipstat.ips_badlen + ipstat.ips_badoptions + ipstat.ips_badvers);
+ rrddim_set(st, "InAddrErrors", ipstat.ips_badaddr);
+ rrddim_set(st, "InUnknownProtos", ipstat.ips_noproto);
+ rrddim_set(st, "OutNoRoutes", ipstat.ips_noroute);
+ rrdset_done(st);
+ }
+ }
+ }
+
+ if (likely(do_ip6_packets || do_ip6_fragsout || do_ip6_fragsin || do_ip6_errors)) {
+ if (unlikely(GETSYSCTL_BY_NAME("net.inet6.ip6.stats", ip6stat))) {
+ do_ip6_packets = 0;
+ error("DISABLED: ipv6.packets");
+ do_ip6_fragsout = 0;
+ error("DISABLED: ipv6.fragsout");
+ do_ip6_fragsin = 0;
+ error("DISABLED: ipv6.fragsin");
+ do_ip6_errors = 0;
+ error("DISABLED: ipv6.errors");
+ } else {
+ if (do_ip6_packets == CONFIG_BOOLEAN_YES || (do_ip6_packets == CONFIG_BOOLEAN_AUTO &&
+ (ip6stat.ip6s_localout ||
+ ip6stat.ip6s_total ||
+ ip6stat.ip6s_forward ||
+ ip6stat.ip6s_delivered ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_packets = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.packets");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "packets"
+ , NULL
+ , "packets"
+ , NULL
+ , "IPv6 Packets"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3000
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "forwarded", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "delivers", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "sent", ip6stat.ip6s_localout);
+ rrddim_set(st, "received", ip6stat.ip6s_total);
+ rrddim_set(st, "forwarded", ip6stat.ip6s_forward);
+ rrddim_set(st, "delivers", ip6stat.ip6s_delivered);
+ rrdset_done(st);
+ }
+
+ if (do_ip6_fragsout == CONFIG_BOOLEAN_YES || (do_ip6_fragsout == CONFIG_BOOLEAN_AUTO &&
+ (ip6stat.ip6s_fragmented ||
+ ip6stat.ip6s_cantfrag ||
+ ip6stat.ip6s_ofragments ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_fragsout = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.fragsout");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "fragsout"
+ , NULL
+ , "fragments"
+ , NULL
+ , "IPv6 Fragments Sent"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3010
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "all", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "ok", ip6stat.ip6s_fragmented);
+ rrddim_set(st, "failed", ip6stat.ip6s_cantfrag);
+ rrddim_set(st, "all", ip6stat.ip6s_ofragments);
+ rrdset_done(st);
+ }
+
+ if (do_ip6_fragsin == CONFIG_BOOLEAN_YES || (do_ip6_fragsin == CONFIG_BOOLEAN_AUTO &&
+ (ip6stat.ip6s_reassembled ||
+ ip6stat.ip6s_fragdropped ||
+ ip6stat.ip6s_fragtimeout ||
+ ip6stat.ip6s_fragments ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_fragsin = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.fragsin");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "fragsin"
+ , NULL
+ , "fragments"
+ , NULL
+ , "IPv6 Fragments Reassembly"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3011
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "timeout", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "all", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "ok", ip6stat.ip6s_reassembled);
+ rrddim_set(st, "failed", ip6stat.ip6s_fragdropped);
+ rrddim_set(st, "timeout", ip6stat.ip6s_fragtimeout);
+ rrddim_set(st, "all", ip6stat.ip6s_fragments);
+ rrdset_done(st);
+ }
+
+ if (do_ip6_errors == CONFIG_BOOLEAN_YES || (do_ip6_errors == CONFIG_BOOLEAN_AUTO &&
+ (ip6stat.ip6s_toosmall ||
+ ip6stat.ip6s_odropped ||
+ ip6stat.ip6s_badoptions ||
+ ip6stat.ip6s_badvers ||
+ ip6stat.ip6s_exthdrtoolong ||
+ ip6stat.ip6s_sources_none ||
+ ip6stat.ip6s_tooshort ||
+ ip6stat.ip6s_cantforward ||
+ ip6stat.ip6s_noroute ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_errors = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.errors");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "errors"
+ , NULL
+ , "errors"
+ , NULL
+ , "IPv6 Errors"
+ , "packets/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 3002
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InTruncatedPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InNoRoutes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InDiscards", ip6stat.ip6s_toosmall);
+ rrddim_set(st, "OutDiscards", ip6stat.ip6s_odropped);
+
+ rrddim_set(st, "InHdrErrors",
+ ip6stat.ip6s_badoptions + ip6stat.ip6s_badvers + ip6stat.ip6s_exthdrtoolong);
+ rrddim_set(st, "InAddrErrors", ip6stat.ip6s_sources_none);
+ rrddim_set(st, "InTruncatedPkts", ip6stat.ip6s_tooshort);
+ rrddim_set(st, "InNoRoutes", ip6stat.ip6s_cantforward);
+
+ rrddim_set(st, "OutNoRoutes", ip6stat.ip6s_noroute);
+ rrdset_done(st);
+ }
+ }
+ }
+
+ if (likely(do_icmp6 || do_icmp6_redir || do_icmp6_errors || do_icmp6_echos || do_icmp6_router || do_icmp6_neighbor || do_icmp6_types)) {
+ if (unlikely(GETSYSCTL_BY_NAME("net.inet6.icmp6.stats", icmp6stat))) {
+ do_icmp6 = 0;
+ error("DISABLED: ipv6.icmp");
+ } else {
+ for (i = 0; i <= ICMP6_MAXTYPE; i++) {
+ icmp6_total.msgs_in += icmp6stat.icp6s_inhist[i];
+ icmp6_total.msgs_out += icmp6stat.icp6s_outhist[i];
+ }
+ icmp6_total.msgs_in += icmp6stat.icp6s_badcode + icmp6stat.icp6s_badlen + icmp6stat.icp6s_checksum + icmp6stat.icp6s_tooshort;
+ if (do_icmp6 == CONFIG_BOOLEAN_YES || (do_icmp6 == CONFIG_BOOLEAN_AUTO &&
+ (icmp6_total.msgs_in ||
+ icmp6_total.msgs_out ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6 = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.icmp");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "icmp"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv6 ICMP Messages"
+ , "messages/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 10000
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "sent", icmp6_total.msgs_in);
+ rrddim_set(st, "received", icmp6_total.msgs_out);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_redir == CONFIG_BOOLEAN_YES || (do_icmp6_redir == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[ND_REDIRECT] ||
+ icmp6stat.icp6s_outhist[ND_REDIRECT] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_redir = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.icmpredir");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "icmpredir"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv6 ICMP Redirects"
+ , "redirects/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 10050
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "sent", icmp6stat.icp6s_inhist[ND_REDIRECT]);
+ rrddim_set(st, "received", icmp6stat.icp6s_outhist[ND_REDIRECT]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_errors == CONFIG_BOOLEAN_YES || (do_icmp6_errors == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_badcode ||
+ icmp6stat.icp6s_badlen ||
+ icmp6stat.icp6s_checksum ||
+ icmp6stat.icp6s_tooshort ||
+ icmp6stat.icp6s_error ||
+ icmp6stat.icp6s_inhist[ICMP6_DST_UNREACH] ||
+ icmp6stat.icp6s_inhist[ICMP6_TIME_EXCEEDED] ||
+ icmp6stat.icp6s_inhist[ICMP6_PARAM_PROB] ||
+ icmp6stat.icp6s_outhist[ICMP6_DST_UNREACH] ||
+ icmp6stat.icp6s_outhist[ICMP6_TIME_EXCEEDED] ||
+ icmp6stat.icp6s_outhist[ICMP6_PARAM_PROB] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_errors = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.icmperrors");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "icmperrors"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv6 ICMP Errors"
+ , "errors/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 10100
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InDestUnreachs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InPktTooBigs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InTimeExcds", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InParmProblems", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutDestUnreachs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutTimeExcds", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutParmProblems", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InErrors", icmp6stat.icp6s_badcode + icmp6stat.icp6s_badlen + icmp6stat.icp6s_checksum + icmp6stat.icp6s_tooshort);
+ rrddim_set(st, "OutErrors", icmp6stat.icp6s_error);
+ rrddim_set(st, "InCsumErrors", icmp6stat.icp6s_checksum);
+ rrddim_set(st, "InDestUnreachs", icmp6stat.icp6s_inhist[ICMP6_DST_UNREACH]);
+ rrddim_set(st, "InPktTooBigs", icmp6stat.icp6s_badlen);
+ rrddim_set(st, "InTimeExcds", icmp6stat.icp6s_inhist[ICMP6_TIME_EXCEEDED]);
+ rrddim_set(st, "InParmProblems", icmp6stat.icp6s_inhist[ICMP6_PARAM_PROB]);
+ rrddim_set(st, "OutDestUnreachs", icmp6stat.icp6s_outhist[ICMP6_DST_UNREACH]);
+ rrddim_set(st, "OutTimeExcds", icmp6stat.icp6s_outhist[ICMP6_TIME_EXCEEDED]);
+ rrddim_set(st, "OutParmProblems", icmp6stat.icp6s_outhist[ICMP6_PARAM_PROB]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_echos == CONFIG_BOOLEAN_YES || (do_icmp6_echos == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[ICMP6_ECHO_REQUEST] ||
+ icmp6stat.icp6s_outhist[ICMP6_ECHO_REQUEST] ||
+ icmp6stat.icp6s_inhist[ICMP6_ECHO_REPLY] ||
+ icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_echos = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.icmpechos");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "icmpechos"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv6 ICMP Echo"
+ , "messages/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 10200
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutEchos", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InEchoReplies", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutEchoReplies", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InEchos", icmp6stat.icp6s_inhist[ICMP6_ECHO_REQUEST]);
+ rrddim_set(st, "OutEchos", icmp6stat.icp6s_outhist[ICMP6_ECHO_REQUEST]);
+ rrddim_set(st, "InEchoReplies", icmp6stat.icp6s_inhist[ICMP6_ECHO_REPLY]);
+ rrddim_set(st, "OutEchoReplies", icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_router == CONFIG_BOOLEAN_YES || (do_icmp6_router == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[ND_ROUTER_SOLICIT] ||
+ icmp6stat.icp6s_outhist[ND_ROUTER_SOLICIT] ||
+ icmp6stat.icp6s_inhist[ND_ROUTER_ADVERT] ||
+ icmp6stat.icp6s_outhist[ND_ROUTER_ADVERT] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_router = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.icmprouter");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "icmprouter"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv6 Router Messages"
+ , "messages/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 10400
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InSolicits", icmp6stat.icp6s_inhist[ND_ROUTER_SOLICIT]);
+ rrddim_set(st, "OutSolicits", icmp6stat.icp6s_outhist[ND_ROUTER_SOLICIT]);
+ rrddim_set(st, "InAdvertisements", icmp6stat.icp6s_inhist[ND_ROUTER_ADVERT]);
+ rrddim_set(st, "OutAdvertisements", icmp6stat.icp6s_outhist[ND_ROUTER_ADVERT]);
+ rrdset_done(st);
+ }
+
+ if (do_icmp6_neighbor == CONFIG_BOOLEAN_YES || (do_icmp6_neighbor == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[ND_NEIGHBOR_SOLICIT] ||
+ icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT] ||
+ icmp6stat.icp6s_inhist[ND_NEIGHBOR_ADVERT] ||
+ icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_neighbor = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.icmpneighbor");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "icmpneighbor"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv6 Neighbor Messages"
+ , "messages/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 10500
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InSolicits", icmp6stat.icp6s_inhist[ND_NEIGHBOR_SOLICIT]);
+ rrddim_set(st, "OutSolicits", icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT]);
+ rrddim_set(st, "InAdvertisements", icmp6stat.icp6s_inhist[ND_NEIGHBOR_ADVERT]);
+ rrddim_set(st, "OutAdvertisements", icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT]);
+ }
+
+ if (do_icmp6_types == CONFIG_BOOLEAN_YES || (do_icmp6_types == CONFIG_BOOLEAN_AUTO &&
+ (icmp6stat.icp6s_inhist[1] ||
+ icmp6stat.icp6s_inhist[128] ||
+ icmp6stat.icp6s_inhist[129] ||
+ icmp6stat.icp6s_inhist[136] ||
+ icmp6stat.icp6s_outhist[1] ||
+ icmp6stat.icp6s_outhist[128] ||
+ icmp6stat.icp6s_outhist[129] ||
+ icmp6stat.icp6s_outhist[133] ||
+ icmp6stat.icp6s_outhist[135] ||
+ icmp6stat.icp6s_outhist[136] ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp6_types = CONFIG_BOOLEAN_YES;
+ st = rrdset_find_active_localhost("ipv6.icmptypes");
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "icmptypes"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv6 ICMP Types"
+ , "messages/s"
+ , "macos.plugin"
+ , "sysctl"
+ , 10700
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "InType1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InType128", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InType129", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "InType136", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutType1", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutType128", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutType129", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutType133", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutType135", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutType143", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "InType1", icmp6stat.icp6s_inhist[1]);
+ rrddim_set(st, "InType128", icmp6stat.icp6s_inhist[128]);
+ rrddim_set(st, "InType129", icmp6stat.icp6s_inhist[129]);
+ rrddim_set(st, "InType136", icmp6stat.icp6s_inhist[136]);
+ rrddim_set(st, "OutType1", icmp6stat.icp6s_outhist[1]);
+ rrddim_set(st, "OutType128", icmp6stat.icp6s_outhist[128]);
+ rrddim_set(st, "OutType129", icmp6stat.icp6s_outhist[129]);
+ rrddim_set(st, "OutType133", icmp6stat.icp6s_outhist[133]);
+ rrddim_set(st, "OutType135", icmp6stat.icp6s_outhist[135]);
+ rrddim_set(st, "OutType143", icmp6stat.icp6s_outhist[143]);
+ rrdset_done(st);
+ }
+ }
+ }
+
+ if (likely(do_uptime)) {
+ if (unlikely(GETSYSCTL_BY_NAME("kern.boottime", boot_time))) {
+ do_uptime = 0;
+ error("DISABLED: system.uptime");
+ } else {
+ clock_gettime(CLOCK_REALTIME, &cur_time);
+ st = rrdset_find_active_localhost("system.uptime");
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "uptime"
+ , NULL
+ , "uptime"
+ , NULL
+ , "System Uptime"
+ , "seconds"
+ , "macos.plugin"
+ , "sysctl"
+ , 1000
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrddim_add(st, "uptime", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set(st, "uptime", cur_time.tv_sec - boot_time.tv_sec);
+ rrdset_done(st);
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/macos.plugin/plugin_macos.c b/collectors/macos.plugin/plugin_macos.c
new file mode 100644
index 0000000..10472bd
--- /dev/null
+++ b/collectors/macos.plugin/plugin_macos.c
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_macos.h"
+
+static struct macos_module {
+ const char *name;
+ const char *dim;
+
+ int enabled;
+
+ int (*func)(int update_every, usec_t dt);
+
+ RRDDIM *rd;
+
+} macos_modules[] = {
+ {.name = "sysctl", .dim = "sysctl", .enabled = 1, .func = do_macos_sysctl},
+ {.name = "mach system management interface", .dim = "mach_smi", .enabled = 1, .func = do_macos_mach_smi},
+ {.name = "iokit", .dim = "iokit", .enabled = 1, .func = do_macos_iokit},
+
+ // the terminator of this array
+ {.name = NULL, .dim = NULL, .enabled = 0, .func = NULL}
+};
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
+#endif
+
+static void macos_main_cleanup(void *ptr)
+{
+ worker_unregister();
+
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ info("cleaning up...");
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+void *macos_main(void *ptr)
+{
+ worker_register("MACOS");
+
+ netdata_thread_cleanup_push(macos_main_cleanup, ptr);
+
+ // check the enabled status for each module
+ for (int i = 0; macos_modules[i].name; i++) {
+ struct macos_module *pm = &macos_modules[i];
+
+ pm->enabled = config_get_boolean("plugin:macos", pm->name, pm->enabled);
+ pm->rd = NULL;
+
+ worker_register_job_name(i, macos_modules[i].dim);
+ }
+
+ usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ while (!netdata_exit) {
+ worker_is_idle();
+ usec_t hb_dt = heartbeat_next(&hb, step);
+
+ for (int i = 0; macos_modules[i].name; i++) {
+ struct macos_module *pm = &macos_modules[i];
+ if (unlikely(!pm->enabled))
+ continue;
+
+ debug(D_PROCNETDEV_LOOP, "macos calling %s.", pm->name);
+
+ worker_is_busy(i);
+ pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
+
+ if (unlikely(netdata_exit))
+ break;
+ }
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/macos.plugin/plugin_macos.h b/collectors/macos.plugin/plugin_macos.h
new file mode 100644
index 0000000..2c673a2
--- /dev/null
+++ b/collectors/macos.plugin/plugin_macos.h
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PLUGIN_MACOS_H
+#define NETDATA_PLUGIN_MACOS_H 1
+
+#include "daemon/common.h"
+
+int do_macos_sysctl(int update_every, usec_t dt);
+int do_macos_mach_smi(int update_every, usec_t dt);
+int do_macos_iokit(int update_every, usec_t dt);
+
+#endif /* NETDATA_PLUGIN_MACOS_H */
diff --git a/collectors/nfacct.plugin/Makefile.am b/collectors/nfacct.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/nfacct.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/nfacct.plugin/README.md b/collectors/nfacct.plugin/README.md
new file mode 100644
index 0000000..bacc8b7
--- /dev/null
+++ b/collectors/nfacct.plugin/README.md
@@ -0,0 +1,54 @@
+<!--
+title: "nfacct.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/nfacct.plugin/README.md
+-->
+
+# nfacct.plugin
+
+`nfacct.plugin` collects Netfilter statistics.
+
+## Prerequisites
+
+1. install `libmnl-dev` and `libnetfilter-acct-dev` using the package manager of your system.
+
+2. re-install Netdata from source. The installer will detect that the required libraries are now available and will also build `netdata.plugin`.
+
+Keep in mind that NFACCT requires root access, so the plugin is setuid to root.
+
+## Charts
+
+The plugin provides Netfilter connection tracker statistics and nfacct packet and bandwidth accounting:
+
+Connection tracker:
+
+1. Connections.
+2. Changes.
+3. Expectations.
+4. Errors.
+5. Searches.
+
+Netfilter accounting:
+
+1. Packets.
+2. Bandwidth.
+
+## Configuration
+
+If you need to disable NFACCT for Netdata, edit /etc/netdata/netdata.conf and set:
+
+```
+[plugins]
+ nfacct = no
+```
+
+## Debugging
+
+You can run the plugin by hand:
+
+```
+sudo /usr/libexec/netdata/plugins.d/nfacct.plugin 1 debug
+```
+
+You will get verbose output on what the plugin does.
+
+
diff --git a/collectors/nfacct.plugin/plugin_nfacct.c b/collectors/nfacct.plugin/plugin_nfacct.c
new file mode 100644
index 0000000..eeadb3c
--- /dev/null
+++ b/collectors/nfacct.plugin/plugin_nfacct.c
@@ -0,0 +1,886 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata/libnetdata.h"
+#include "libnetdata/required_dummies.h"
+
+#include <linux/netfilter/nfnetlink_conntrack.h>
+#include <libmnl/libmnl.h>
+#include <libnetfilter_acct/libnetfilter_acct.h>
+
+#define PLUGIN_NFACCT_NAME "nfacct.plugin"
+
+#define NETDATA_CHART_PRIO_NETFILTER_NEW 8701
+#define NETDATA_CHART_PRIO_NETFILTER_CHANGES 8702
+#define NETDATA_CHART_PRIO_NETFILTER_EXPECT 8703
+#define NETDATA_CHART_PRIO_NETFILTER_ERRORS 8705
+#define NETDATA_CHART_PRIO_NETFILTER_SEARCH 8710
+
+#define NETDATA_CHART_PRIO_NETFILTER_PACKETS 8906
+#define NETDATA_CHART_PRIO_NETFILTER_BYTES 8907
+
+static inline size_t mnl_buffer_size() {
+ long s = MNL_SOCKET_BUFFER_SIZE;
+ if(s <= 0) return 8192;
+ return (size_t)s;
+}
+
+// variables
+static int debug = 0;
+static int netdata_update_every = 1;
+
+#define RRD_TYPE_NET_STAT_NETFILTER "netfilter"
+#define RRD_TYPE_NET_STAT_CONNTRACK "netlink"
+
+static struct {
+ int update_every;
+ char *buf;
+ size_t buf_size;
+ struct mnl_socket *mnl;
+ struct nlmsghdr *nlh;
+ struct nfgenmsg *nfh;
+ unsigned int seq;
+ uint32_t portid;
+
+ struct nlattr *tb[CTA_STATS_MAX+1];
+ const char *attr2name[CTA_STATS_MAX+1];
+ kernel_uint_t metrics[CTA_STATS_MAX+1];
+
+ struct nlattr *tb_exp[CTA_STATS_EXP_MAX+1];
+ const char *attr2name_exp[CTA_STATS_EXP_MAX+1];
+ kernel_uint_t metrics_exp[CTA_STATS_EXP_MAX+1];
+} nfstat_root = {
+ .update_every = 1,
+ .buf = NULL,
+ .buf_size = 0,
+ .mnl = NULL,
+ .nlh = NULL,
+ .nfh = NULL,
+ .seq = 0,
+ .portid = 0,
+ .tb = {},
+ .attr2name = {
+ [CTA_STATS_SEARCHED] = "searched",
+ [CTA_STATS_FOUND] = "found",
+ [CTA_STATS_NEW] = "new",
+ [CTA_STATS_INVALID] = "invalid",
+ [CTA_STATS_IGNORE] = "ignore",
+ [CTA_STATS_DELETE] = "delete",
+ [CTA_STATS_DELETE_LIST] = "delete_list",
+ [CTA_STATS_INSERT] = "insert",
+ [CTA_STATS_INSERT_FAILED] = "insert_failed",
+ [CTA_STATS_DROP] = "drop",
+ [CTA_STATS_EARLY_DROP] = "early_drop",
+ [CTA_STATS_ERROR] = "icmp_error",
+ [CTA_STATS_SEARCH_RESTART] = "search_restart",
+ },
+ .metrics = {},
+ .tb_exp = {},
+ .attr2name_exp = {
+ [CTA_STATS_EXP_NEW] = "new",
+ [CTA_STATS_EXP_CREATE] = "created",
+ [CTA_STATS_EXP_DELETE] = "deleted",
+ },
+ .metrics_exp = {}
+};
+
+
+static int nfstat_init(int update_every) {
+ nfstat_root.update_every = update_every;
+
+ nfstat_root.buf_size = mnl_buffer_size();
+ nfstat_root.buf = mallocz(nfstat_root.buf_size);
+
+ nfstat_root.mnl = mnl_socket_open(NETLINK_NETFILTER);
+ if(!nfstat_root.mnl) {
+ error("NFSTAT: mnl_socket_open() failed");
+ return 1;
+ }
+
+ nfstat_root.seq = (unsigned int)now_realtime_sec() - 1;
+
+ if(mnl_socket_bind(nfstat_root.mnl, 0, MNL_SOCKET_AUTOPID) < 0) {
+ error("NFSTAT: mnl_socket_bind() failed");
+ return 1;
+ }
+ nfstat_root.portid = mnl_socket_get_portid(nfstat_root.mnl);
+
+ return 0;
+}
+
+static struct nlmsghdr * nfct_mnl_nlmsghdr_put(char *buf, uint16_t subsys, uint16_t type, uint8_t family, uint32_t seq) {
+ struct nlmsghdr *nlh;
+ struct nfgenmsg *nfh;
+
+ nlh = mnl_nlmsg_put_header(buf);
+ nlh->nlmsg_type = (subsys << 8) | type;
+ nlh->nlmsg_flags = NLM_F_REQUEST|NLM_F_DUMP;
+ nlh->nlmsg_seq = seq;
+
+ nfh = mnl_nlmsg_put_extra_header(nlh, sizeof(struct nfgenmsg));
+ nfh->nfgen_family = family;
+ nfh->version = NFNETLINK_V0;
+ nfh->res_id = 0;
+
+ return nlh;
+}
+
+static int nfct_stats_attr_cb(const struct nlattr *attr, void *data) {
+ const struct nlattr **tb = data;
+ int type = mnl_attr_get_type(attr);
+
+ if (mnl_attr_type_valid(attr, CTA_STATS_MAX) < 0)
+ return MNL_CB_OK;
+
+ if (mnl_attr_validate(attr, MNL_TYPE_U32) < 0) {
+ error("NFSTAT: mnl_attr_validate() failed");
+ return MNL_CB_ERROR;
+ }
+
+ tb[type] = attr;
+ return MNL_CB_OK;
+}
+
+static int nfstat_callback(const struct nlmsghdr *nlh, void *data) {
+ (void)data;
+
+ struct nfgenmsg *nfg = mnl_nlmsg_get_payload(nlh);
+
+ mnl_attr_parse(nlh, sizeof(*nfg), nfct_stats_attr_cb, nfstat_root.tb);
+
+ // printf("cpu=%-4u\t", ntohs(nfg->res_id));
+
+ int i;
+ // add the metrics of this CPU into the metrics
+ for (i = 0; i < CTA_STATS_MAX+1; i++) {
+ if (nfstat_root.tb[i]) {
+ // printf("%s=%u ", nfstat_root.attr2name[i], ntohl(mnl_attr_get_u32(nfstat_root.tb[i])));
+ nfstat_root.metrics[i] += ntohl(mnl_attr_get_u32(nfstat_root.tb[i]));
+ }
+ }
+ // printf("\n");
+
+ return MNL_CB_OK;
+}
+
+static int nfstat_collect_conntrack() {
+ // zero all metrics - we will sum the metrics of all CPUs later
+ int i;
+ for (i = 0; i < CTA_STATS_MAX+1; i++)
+ nfstat_root.metrics[i] = 0;
+
+ // prepare the request
+ nfstat_root.nlh = nfct_mnl_nlmsghdr_put(nfstat_root.buf, NFNL_SUBSYS_CTNETLINK, IPCTNL_MSG_CT_GET_STATS_CPU, AF_UNSPEC, nfstat_root.seq);
+
+ // send the request
+ if(mnl_socket_sendto(nfstat_root.mnl, nfstat_root.nlh, nfstat_root.nlh->nlmsg_len) < 0) {
+ error("NFSTAT: mnl_socket_sendto() failed");
+ return 1;
+ }
+
+ // get the reply
+ ssize_t ret;
+ while ((ret = mnl_socket_recvfrom(nfstat_root.mnl, nfstat_root.buf, nfstat_root.buf_size)) > 0) {
+ if(mnl_cb_run(
+ nfstat_root.buf
+ , (size_t)ret
+ , nfstat_root.nlh->nlmsg_seq
+ , nfstat_root.portid
+ , nfstat_callback
+ , NULL
+ ) <= MNL_CB_STOP)
+ break;
+ }
+
+ // verify we run without issues
+ if (ret == -1) {
+ error("NFSTAT: error communicating with kernel. This plugin can only work when netdata runs as root.");
+ return 1;
+ }
+
+ return 0;
+}
+
+static int nfexp_stats_attr_cb(const struct nlattr *attr, void *data)
+{
+ const struct nlattr **tb = data;
+ int type = mnl_attr_get_type(attr);
+
+ if (mnl_attr_type_valid(attr, CTA_STATS_EXP_MAX) < 0)
+ return MNL_CB_OK;
+
+ if (mnl_attr_validate(attr, MNL_TYPE_U32) < 0) {
+ error("NFSTAT EXP: mnl_attr_validate() failed");
+ return MNL_CB_ERROR;
+ }
+
+ tb[type] = attr;
+ return MNL_CB_OK;
+}
+
+static int nfstat_callback_exp(const struct nlmsghdr *nlh, void *data) {
+ (void)data;
+
+ struct nfgenmsg *nfg = mnl_nlmsg_get_payload(nlh);
+
+ mnl_attr_parse(nlh, sizeof(*nfg), nfexp_stats_attr_cb, nfstat_root.tb_exp);
+
+ int i;
+ for (i = 0; i < CTA_STATS_EXP_MAX+1; i++) {
+ if (nfstat_root.tb_exp[i]) {
+ nfstat_root.metrics_exp[i] += ntohl(mnl_attr_get_u32(nfstat_root.tb_exp[i]));
+ }
+ }
+
+ return MNL_CB_OK;
+}
+
+static int nfstat_collect_conntrack_expectations() {
+ // zero all metrics - we will sum the metrics of all CPUs later
+ int i;
+ for (i = 0; i < CTA_STATS_EXP_MAX+1; i++)
+ nfstat_root.metrics_exp[i] = 0;
+
+ // prepare the request
+ nfstat_root.nlh = nfct_mnl_nlmsghdr_put(nfstat_root.buf, NFNL_SUBSYS_CTNETLINK_EXP, IPCTNL_MSG_EXP_GET_STATS_CPU, AF_UNSPEC, nfstat_root.seq);
+
+ // send the request
+ if(mnl_socket_sendto(nfstat_root.mnl, nfstat_root.nlh, nfstat_root.nlh->nlmsg_len) < 0) {
+ error("NFSTAT: mnl_socket_sendto() failed");
+ return 1;
+ }
+
+ // get the reply
+ ssize_t ret;
+ while ((ret = mnl_socket_recvfrom(nfstat_root.mnl, nfstat_root.buf, nfstat_root.buf_size)) > 0) {
+ if(mnl_cb_run(
+ nfstat_root.buf
+ , (size_t)ret
+ , nfstat_root.nlh->nlmsg_seq
+ , nfstat_root.portid
+ , nfstat_callback_exp
+ , NULL
+ ) <= MNL_CB_STOP)
+ break;
+ }
+
+ // verify we run without issues
+ if (ret == -1) {
+ error("NFSTAT: error communicating with kernel. This plugin can only work when netdata runs as root.");
+ return 1;
+ }
+
+ return 0;
+}
+
+static int nfstat_collect() {
+ nfstat_root.seq++;
+
+ if(nfstat_collect_conntrack())
+ return 1;
+
+ if(nfstat_collect_conntrack_expectations())
+ return 1;
+
+ return 0;
+}
+
+static void nfstat_send_metrics() {
+ static int new_chart_generated = 0, changes_chart_generated = 0, search_chart_generated = 0, errors_chart_generated = 0, expect_chart_generated = 0;
+
+ if(!new_chart_generated) {
+ new_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Connection Tracker New Connections' 'connections/s' %s '' line %d %d %s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_new"
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NETDATA_CHART_PRIO_NETFILTER_NEW
+ , nfstat_root.update_every
+ , PLUGIN_NFACCT_NAME
+ );
+ printf("DIMENSION %s '' incremental 1 1\n", nfstat_root.attr2name[CTA_STATS_NEW]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_IGNORE]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_INVALID]);
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_new"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_NEW]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_NEW]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_IGNORE]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_IGNORE]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_INVALID]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_INVALID]
+ );
+ printf("END\n");
+
+ // ----------------------------------------------------------------
+
+ if(!changes_chart_generated) {
+ changes_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Connection Tracker Changes' 'changes/s' %s '' line %d %d detail %s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_changes"
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NETDATA_CHART_PRIO_NETFILTER_CHANGES
+ , nfstat_root.update_every
+ , PLUGIN_NFACCT_NAME
+ );
+ printf("DIMENSION %s '' incremental 1 1\n", nfstat_root.attr2name[CTA_STATS_INSERT]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_DELETE]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_DELETE_LIST]);
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_changes"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_INSERT]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_INSERT]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_DELETE]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_DELETE]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_DELETE_LIST]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_DELETE_LIST]
+ );
+ printf("END\n");
+
+ // ----------------------------------------------------------------
+
+ if(!search_chart_generated) {
+ search_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Connection Tracker Searches' 'searches/s' %s '' line %d %d detail %s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_search"
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NETDATA_CHART_PRIO_NETFILTER_SEARCH
+ , nfstat_root.update_every
+ , PLUGIN_NFACCT_NAME
+ );
+ printf("DIMENSION %s '' incremental 1 1\n", nfstat_root.attr2name[CTA_STATS_SEARCHED]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_SEARCH_RESTART]);
+ printf("DIMENSION %s '' incremental 1 1\n", nfstat_root.attr2name[CTA_STATS_FOUND]);
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_search"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_SEARCHED]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_SEARCHED]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_SEARCH_RESTART]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_SEARCH_RESTART]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_FOUND]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_FOUND]
+ );
+ printf("END\n");
+
+ // ----------------------------------------------------------------
+
+ if(!errors_chart_generated) {
+ errors_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Connection Tracker Errors' 'events/s' %s '' line %d %d detail %s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_errors"
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NETDATA_CHART_PRIO_NETFILTER_ERRORS
+ , nfstat_root.update_every
+ , PLUGIN_NFACCT_NAME
+ );
+ printf("DIMENSION %s '' incremental 1 1\n", nfstat_root.attr2name[CTA_STATS_ERROR]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_INSERT_FAILED]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_DROP]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_EARLY_DROP]);
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_errors"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_ERROR]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_ERROR]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_INSERT_FAILED]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_INSERT_FAILED]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_DROP]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_DROP]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_EARLY_DROP]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_EARLY_DROP]
+ );
+ printf("END\n");
+
+ // ----------------------------------------------------------------
+
+ if(!expect_chart_generated) {
+ expect_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Connection Tracker Expectations' 'expectations/s' %s '' line %d %d detail %s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_expect"
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NETDATA_CHART_PRIO_NETFILTER_EXPECT
+ , nfstat_root.update_every
+ , PLUGIN_NFACCT_NAME
+ );
+ printf("DIMENSION %s '' incremental 1 1\n", nfstat_root.attr2name[CTA_STATS_EXP_CREATE]);
+ printf("DIMENSION %s '' incremental -1 1\n", nfstat_root.attr2name[CTA_STATS_EXP_DELETE]);
+ printf("DIMENSION %s '' incremental 1 1\n", nfstat_root.attr2name[CTA_STATS_EXP_NEW]);
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_expect"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_EXP_CREATE]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_EXP_CREATE]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_EXP_DELETE]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_EXP_DELETE]
+ );
+ printf(
+ "SET %s = %lld\n"
+ , nfstat_root.attr2name[CTA_STATS_EXP_NEW]
+ , (collected_number) nfstat_root.metrics[CTA_STATS_EXP_NEW]
+ );
+ printf("END\n");
+}
+
+
+struct nfacct_data {
+ char *name;
+ uint32_t hash;
+
+ uint64_t pkts;
+ uint64_t bytes;
+
+ int packets_dimension_added;
+ int bytes_dimension_added;
+
+ int updated;
+
+ struct nfacct_data *next;
+};
+
+static struct {
+ int update_every;
+ char *buf;
+ size_t buf_size;
+ struct mnl_socket *mnl;
+ struct nlmsghdr *nlh;
+ unsigned int seq;
+ uint32_t portid;
+ struct nfacct *nfacct_buffer;
+ struct nfacct_data *nfacct_metrics;
+} nfacct_root = {
+ .update_every = 1,
+ .buf = NULL,
+ .buf_size = 0,
+ .mnl = NULL,
+ .nlh = NULL,
+ .seq = 0,
+ .portid = 0,
+ .nfacct_buffer = NULL,
+ .nfacct_metrics = NULL
+};
+
+static inline struct nfacct_data *nfacct_data_get(const char *name, uint32_t hash) {
+ struct nfacct_data *d = NULL, *last = NULL;
+ for(d = nfacct_root.nfacct_metrics; d ; last = d, d = d->next) {
+ if(unlikely(d->hash == hash && !strcmp(d->name, name)))
+ return d;
+ }
+
+ d = callocz(1, sizeof(struct nfacct_data));
+ d->name = strdupz(name);
+ d->hash = hash;
+
+ if(!last) {
+ d->next = nfacct_root.nfacct_metrics;
+ nfacct_root.nfacct_metrics = d;
+ }
+ else {
+ d->next = last->next;
+ last->next = d;
+ }
+
+ return d;
+}
+
+static int nfacct_init(int update_every) {
+ nfacct_root.update_every = update_every;
+
+ nfacct_root.buf_size = mnl_buffer_size();
+ nfacct_root.buf = mallocz(nfacct_root.buf_size);
+
+ nfacct_root.nfacct_buffer = nfacct_alloc();
+ if(!nfacct_root.nfacct_buffer) {
+ error("nfacct.plugin: nfacct_alloc() failed.");
+ return 0;
+ }
+
+ nfacct_root.seq = (unsigned int)now_realtime_sec() - 1;
+
+ nfacct_root.mnl = mnl_socket_open(NETLINK_NETFILTER);
+ if(!nfacct_root.mnl) {
+ error("nfacct.plugin: mnl_socket_open() failed");
+ return 1;
+ }
+
+ if(mnl_socket_bind(nfacct_root.mnl, 0, MNL_SOCKET_AUTOPID) < 0) {
+ error("nfacct.plugin: mnl_socket_bind() failed");
+ return 1;
+ }
+ nfacct_root.portid = mnl_socket_get_portid(nfacct_root.mnl);
+
+ return 0;
+}
+
+static int nfacct_callback(const struct nlmsghdr *nlh, void *data) {
+ (void)data;
+
+ if(nfacct_nlmsg_parse_payload(nlh, nfacct_root.nfacct_buffer) < 0) {
+ error("NFACCT: nfacct_nlmsg_parse_payload() failed.");
+ return MNL_CB_OK;
+ }
+
+ const char *name = nfacct_attr_get_str(nfacct_root.nfacct_buffer, NFACCT_ATTR_NAME);
+ uint32_t hash = simple_hash(name);
+
+ struct nfacct_data *d = nfacct_data_get(name, hash);
+
+ d->pkts = nfacct_attr_get_u64(nfacct_root.nfacct_buffer, NFACCT_ATTR_PKTS);
+ d->bytes = nfacct_attr_get_u64(nfacct_root.nfacct_buffer, NFACCT_ATTR_BYTES);
+ d->updated = 1;
+
+ return MNL_CB_OK;
+}
+
+static int nfacct_collect() {
+ // mark all old metrics as not-updated
+ struct nfacct_data *d;
+ for(d = nfacct_root.nfacct_metrics; d ; d = d->next)
+ d->updated = 0;
+
+ // prepare the request
+ nfacct_root.seq++;
+ nfacct_root.nlh = nfacct_nlmsg_build_hdr(nfacct_root.buf, NFNL_MSG_ACCT_GET, NLM_F_DUMP, (uint32_t)nfacct_root.seq);
+ if(!nfacct_root.nlh) {
+ error("NFACCT: nfacct_nlmsg_build_hdr() failed");
+ return 1;
+ }
+
+ // send the request
+ if(mnl_socket_sendto(nfacct_root.mnl, nfacct_root.nlh, nfacct_root.nlh->nlmsg_len) < 0) {
+ error("NFACCT: mnl_socket_sendto() failed");
+ return 1;
+ }
+
+ // get the reply
+ ssize_t ret;
+ while((ret = mnl_socket_recvfrom(nfacct_root.mnl, nfacct_root.buf, nfacct_root.buf_size)) > 0) {
+ if(mnl_cb_run(
+ nfacct_root.buf
+ , (size_t)ret
+ , nfacct_root.seq
+ , nfacct_root.portid
+ , nfacct_callback
+ , NULL
+ ) <= 0)
+ break;
+ }
+
+ // verify we run without issues
+ if (ret == -1) {
+ error("NFACCT: error communicating with kernel. This plugin can only work when netdata runs as root.");
+ return 1;
+ }
+
+ return 0;
+}
+
+static void nfacct_send_metrics() {
+ static int bytes_chart_generated = 0, packets_chart_generated = 0;
+
+ if(!nfacct_root.nfacct_metrics) return;
+ struct nfacct_data *d;
+
+ if(!packets_chart_generated) {
+ packets_chart_generated = 1;
+ printf("CHART netfilter.nfacct_packets '' 'Netfilter Accounting Packets' 'packets/s' 'nfacct' '' stacked %d %d %s\n"
+ , NETDATA_CHART_PRIO_NETFILTER_PACKETS
+ , nfacct_root.update_every
+ , PLUGIN_NFACCT_NAME
+ );
+ }
+
+ for(d = nfacct_root.nfacct_metrics; d ; d = d->next) {
+ if(likely(d->updated)) {
+ if(unlikely(!d->packets_dimension_added)) {
+ d->packets_dimension_added = 1;
+ printf(
+ "CHART netfilter.nfacct_packets '' 'Netfilter Accounting Packets' 'packets/s' 'nfacct' '' stacked %d %d %s\n",
+ NETDATA_CHART_PRIO_NETFILTER_PACKETS,
+ nfacct_root.update_every,
+ PLUGIN_NFACCT_NAME);
+ printf("DIMENSION %s '' incremental 1 %d\n", d->name, nfacct_root.update_every);
+ }
+ }
+ }
+ printf("BEGIN netfilter.nfacct_packets\n");
+ for(d = nfacct_root.nfacct_metrics; d ; d = d->next) {
+ if(likely(d->updated)) {
+ printf("SET %s = %lld\n"
+ , d->name
+ , (collected_number)d->pkts
+ );
+ }
+ }
+ printf("END\n");
+
+ // ----------------------------------------------------------------
+
+ if(!bytes_chart_generated) {
+ bytes_chart_generated = 1;
+ printf("CHART netfilter.nfacct_bytes '' 'Netfilter Accounting Bandwidth' 'kilobytes/s' 'nfacct' '' stacked %d %d %s\n"
+ , NETDATA_CHART_PRIO_NETFILTER_BYTES
+ , nfacct_root.update_every
+ , PLUGIN_NFACCT_NAME
+ );
+ }
+
+ for(d = nfacct_root.nfacct_metrics; d ; d = d->next) {
+ if(likely(d->updated)) {
+ if(unlikely(!d->bytes_dimension_added)) {
+ d->bytes_dimension_added = 1;
+ printf(
+ "CHART netfilter.nfacct_bytes '' 'Netfilter Accounting Bandwidth' 'kilobytes/s' 'nfacct' '' stacked %d %d %s\n",
+ NETDATA_CHART_PRIO_NETFILTER_BYTES,
+ nfacct_root.update_every,
+ PLUGIN_NFACCT_NAME);
+ printf("DIMENSION %s '' incremental 1 %d\n", d->name, 1000 * nfacct_root.update_every);
+ }
+ }
+ }
+ printf("BEGIN netfilter.nfacct_bytes\n");
+ for(d = nfacct_root.nfacct_metrics; d ; d = d->next) {
+ if(likely(d->updated)) {
+ printf("SET %s = %lld\n"
+ , d->name
+ , (collected_number)d->bytes
+ );
+ }
+ }
+ printf("END\n");
+}
+
+static void nfacct_signal_handler(int signo)
+{
+ exit((signo == SIGPIPE)?1:0);
+}
+
+// When Netdata crashes this plugin was becoming zombie,
+// this function was added to remove it when sigpipe and other signals are received.
+void nfacct_signals()
+{
+ int signals[] = { SIGPIPE, SIGINT, SIGTERM, 0};
+ int i;
+ struct sigaction sa;
+ sa.sa_flags = 0;
+ sa.sa_handler = nfacct_signal_handler;
+
+ // ignore all signals while we run in a signal handler
+ sigfillset(&sa.sa_mask);
+
+ for (i = 0; signals[i]; i++) {
+ if(sigaction(signals[i], &sa, NULL) == -1)
+ error("Cannot add the handler to signal %d", signals[i]);
+ }
+}
+
+int main(int argc, char **argv) {
+ clocks_init();
+
+ // ------------------------------------------------------------------------
+ // initialization of netdata plugin
+
+ program_name = "nfacct.plugin";
+
+ // disable syslog
+ error_log_syslog = 0;
+
+ // set errors flood protection to 100 logs per hour
+ error_log_errors_per_period = 100;
+ error_log_throttle_period = 3600;
+
+ // ------------------------------------------------------------------------
+ // parse command line parameters
+
+ int i, freq = 0;
+ for(i = 1; i < argc ; i++) {
+ if(isdigit(*argv[i]) && !freq) {
+ int n = str2i(argv[i]);
+ if(n > 0 && n < 86400) {
+ freq = n;
+ continue;
+ }
+ }
+ else if(strcmp("version", argv[i]) == 0 || strcmp("-version", argv[i]) == 0 || strcmp("--version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) {
+ printf("nfacct.plugin %s\n", VERSION);
+ exit(0);
+ }
+ else if(strcmp("debug", argv[i]) == 0) {
+ debug = 1;
+ continue;
+ }
+ else if(strcmp("-h", argv[i]) == 0 || strcmp("--help", argv[i]) == 0) {
+ fprintf(stderr,
+ "\n"
+ " netdata nfacct.plugin %s\n"
+ " Copyright (C) 2015-2017 Costa Tsaousis <costa@tsaousis.gr>\n"
+ " Released under GNU General Public License v3 or later.\n"
+ " All rights reserved.\n"
+ "\n"
+ " This program is a data collector plugin for netdata.\n"
+ "\n"
+ " Available command line options:\n"
+ "\n"
+ " COLLECTION_FREQUENCY data collection frequency in seconds\n"
+ " minimum: %d\n"
+ "\n"
+ " debug enable verbose output\n"
+ " default: disabled\n"
+ "\n"
+ " -v\n"
+ " -V\n"
+ " --version print version and exit\n"
+ "\n"
+ " -h\n"
+ " --help print this message and exit\n"
+ "\n"
+ " For more information:\n"
+ " https://github.com/netdata/netdata/tree/master/collectors/nfacct.plugin\n"
+ "\n"
+ , VERSION
+ , netdata_update_every
+ );
+ exit(1);
+ }
+
+ error("nfacct.plugin: ignoring parameter '%s'", argv[i]);
+ }
+
+ nfacct_signals();
+
+ errno = 0;
+
+ if(freq >= netdata_update_every)
+ netdata_update_every = freq;
+ else if(freq)
+ error("update frequency %d seconds is too small for NFACCT. Using %d.", freq, netdata_update_every);
+
+ if (debug)
+ fprintf(stderr, "nfacct.plugin: calling nfacct_init()\n");
+ int nfacct = !nfacct_init(netdata_update_every);
+
+ if (debug)
+ fprintf(stderr, "nfacct.plugin: calling nfstat_init()\n");
+ int nfstat = !nfstat_init(netdata_update_every);
+
+ // ------------------------------------------------------------------------
+ // the main loop
+
+ if(debug) fprintf(stderr, "nfacct.plugin: starting data collection\n");
+
+ time_t started_t = now_monotonic_sec();
+
+ size_t iteration;
+ usec_t step = netdata_update_every * USEC_PER_SEC;
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ for(iteration = 0; 1; iteration++) {
+ usec_t dt = heartbeat_next(&hb, step);
+
+ if(unlikely(netdata_exit)) break;
+
+ if(debug && iteration)
+ fprintf(stderr, "nfacct.plugin: iteration %zu, dt %llu usec\n"
+ , iteration
+ , dt
+ );
+
+ if(likely(nfacct)) {
+ if(debug) fprintf(stderr, "nfacct.plugin: calling nfacct_collect()\n");
+ nfacct = !nfacct_collect();
+
+ if(likely(nfacct)) {
+ if(debug) fprintf(stderr, "nfacct.plugin: calling nfacct_send_metrics()\n");
+ nfacct_send_metrics();
+ }
+ }
+
+ if(likely(nfstat)) {
+ if(debug) fprintf(stderr, "nfacct.plugin: calling nfstat_collect()\n");
+ nfstat = !nfstat_collect();
+
+ if(likely(nfstat)) {
+ if(debug) fprintf(stderr, "nfacct.plugin: calling nfstat_send_metrics()\n");
+ nfstat_send_metrics();
+ }
+ }
+
+ fflush(stdout);
+
+ // restart check (14400 seconds)
+ if(now_monotonic_sec() - started_t > 14400) break;
+ }
+
+ info("NFACCT process exiting");
+}
diff --git a/collectors/perf.plugin/Makefile.am b/collectors/perf.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/perf.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/perf.plugin/README.md b/collectors/perf.plugin/README.md
new file mode 100644
index 0000000..a7a87ac
--- /dev/null
+++ b/collectors/perf.plugin/README.md
@@ -0,0 +1,80 @@
+<!--
+title: "perf.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/perf.plugin/README.md
+-->
+
+# perf.plugin
+
+`perf.plugin` collects system-wide CPU performance statistics from Performance Monitoring Units (PMU) using
+the `perf_event_open()` system call.
+
+## Important Notes
+
+Accessing hardware PMUs requires root permissions, so the plugin is setuid to root.
+
+Keep in mind that the number of PMUs in a system is usually quite limited and every hardware monitoring
+event for every CPU core needs a separate file descriptor to be opened.
+
+## Charts
+
+The plugin provides statistics for general hardware and software performance monitoring events:
+
+Hardware events:
+
+1. CPU cycles
+2. Instructions
+3. Branch instructions
+4. Cache operations
+5. BUS cycles
+6. Stalled frontend and backend cycles
+
+Software events:
+
+1. CPU migrations
+2. Alignment faults
+3. Emulation faults
+
+Hardware cache events:
+
+1. L1D cache operations
+2. L1D prefetch cache operations
+3. L1I cache operations
+4. LL cache operations
+5. DTLB cache operations
+6. ITLB cache operations
+7. PBU cache operations
+
+## Configuration
+
+The plugin is disabled by default because the number of PMUs is usually quite limited and it is not desired to
+allow Netdata to struggle silently for PMUs, interfering with other performance monitoring software. If you need to
+enable the perf plugin, edit /etc/netdata/netdata.conf and set:
+
+```raw
+[plugins]
+ perf = yes
+```
+
+```raw
+[plugin:perf]
+ update every = 1
+ command options = all
+```
+
+You can use the `command options` parameter to pick what data should be collected and which charts should be
+displayed. If `all` is used, all general performance monitoring counters are probed and corresponding charts
+are enabled for the available counters. You can also define a particular set of enabled charts using the
+following keywords: `cycles`, `instructions`, `branch`, `cache`, `bus`, `stalled`, `migrations`, `alignment`,
+`emulation`, `L1D`, `L1D-prefetch`, `L1I`, `LL`, `DTLB`, `ITLB`, `PBU`.
+
+## Debugging
+
+You can run the plugin by hand:
+
+```raw
+sudo /usr/libexec/netdata/plugins.d/perf.plugin 1 all debug
+```
+
+You will get verbose output on what the plugin does.
+
+
diff --git a/collectors/perf.plugin/perf_plugin.c b/collectors/perf.plugin/perf_plugin.c
new file mode 100644
index 0000000..b2f7d2e
--- /dev/null
+++ b/collectors/perf.plugin/perf_plugin.c
@@ -0,0 +1,1353 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata/libnetdata.h"
+#include "libnetdata/required_dummies.h"
+
+#include <linux/perf_event.h>
+
+#define PLUGIN_PERF_NAME "perf.plugin"
+
+// Hardware counters
+#define NETDATA_CHART_PRIO_PERF_CPU_CYCLES 8800
+#define NETDATA_CHART_PRIO_PERF_INSTRUCTIONS 8801
+#define NETDATA_CHART_PRIO_PERF_IPC 8802
+#define NETDATA_CHART_PRIO_PERF_BRANCH_INSTRUCTIONS 8803
+#define NETDATA_CHART_PRIO_PERF_CACHE 8804
+#define NETDATA_CHART_PRIO_PERF_BUS_CYCLES 8805
+#define NETDATA_CHART_PRIO_PERF_FRONT_BACK_CYCLES 8806
+
+// Software counters
+#define NETDATA_CHART_PRIO_PERF_MIGRATIONS 8810
+#define NETDATA_CHART_PRIO_PERF_ALIGNMENT 8811
+#define NETDATA_CHART_PRIO_PERF_EMULATION 8812
+
+// Hardware cache counters
+#define NETDATA_CHART_PRIO_PERF_L1D 8820
+#define NETDATA_CHART_PRIO_PERF_L1D_PREFETCH 8821
+#define NETDATA_CHART_PRIO_PERF_L1I 8822
+#define NETDATA_CHART_PRIO_PERF_LL 8823
+#define NETDATA_CHART_PRIO_PERF_DTLB 8824
+#define NETDATA_CHART_PRIO_PERF_ITLB 8825
+#define NETDATA_CHART_PRIO_PERF_PBU 8826
+
+#define RRD_TYPE_PERF "perf"
+#define RRD_FAMILY_HW "hardware"
+#define RRD_FAMILY_SW "software"
+#define RRD_FAMILY_CACHE "cache"
+
+#define NO_FD -1
+#define ALL_PIDS -1
+#define RUNNING_THRESHOLD 100
+
+static int debug = 0;
+
+static int update_every = 1;
+static int freq = 0;
+
+typedef enum perf_event_id {
+ // Hardware counters
+ EV_ID_CPU_CYCLES,
+ EV_ID_INSTRUCTIONS,
+ EV_ID_CACHE_REFERENCES,
+ EV_ID_CACHE_MISSES,
+ EV_ID_BRANCH_INSTRUCTIONS,
+ EV_ID_BRANCH_MISSES,
+ EV_ID_BUS_CYCLES,
+ EV_ID_STALLED_CYCLES_FRONTEND,
+ EV_ID_STALLED_CYCLES_BACKEND,
+ EV_ID_REF_CPU_CYCLES,
+
+ // Software counters
+ // EV_ID_CPU_CLOCK,
+ // EV_ID_TASK_CLOCK,
+ // EV_ID_PAGE_FAULTS,
+ // EV_ID_CONTEXT_SWITCHES,
+ EV_ID_CPU_MIGRATIONS,
+ // EV_ID_PAGE_FAULTS_MIN,
+ // EV_ID_PAGE_FAULTS_MAJ,
+ EV_ID_ALIGNMENT_FAULTS,
+ EV_ID_EMULATION_FAULTS,
+
+ // Hardware cache counters
+ EV_ID_L1D_READ_ACCESS,
+ EV_ID_L1D_READ_MISS,
+ EV_ID_L1D_WRITE_ACCESS,
+ EV_ID_L1D_WRITE_MISS,
+ EV_ID_L1D_PREFETCH_ACCESS,
+
+ EV_ID_L1I_READ_ACCESS,
+ EV_ID_L1I_READ_MISS,
+
+ EV_ID_LL_READ_ACCESS,
+ EV_ID_LL_READ_MISS,
+ EV_ID_LL_WRITE_ACCESS,
+ EV_ID_LL_WRITE_MISS,
+
+ EV_ID_DTLB_READ_ACCESS,
+ EV_ID_DTLB_READ_MISS,
+ EV_ID_DTLB_WRITE_ACCESS,
+ EV_ID_DTLB_WRITE_MISS,
+
+ EV_ID_ITLB_READ_ACCESS,
+ EV_ID_ITLB_READ_MISS,
+
+ EV_ID_PBU_READ_ACCESS,
+
+ EV_ID_END
+} perf_event_id_t;
+
+enum perf_event_group {
+ EV_GROUP_CYCLES,
+ EV_GROUP_INSTRUCTIONS_AND_CACHE,
+ EV_GROUP_SOFTWARE,
+ EV_GROUP_CACHE_L1D,
+ EV_GROUP_CACHE_L1I_LL_DTLB,
+ EV_GROUP_CACHE_ITLB_BPU,
+
+ EV_GROUP_NUM
+};
+
+static int number_of_cpus;
+
+static int *group_leader_fds[EV_GROUP_NUM];
+
+static struct perf_event {
+ perf_event_id_t id;
+
+ int type;
+ int config;
+
+ int **group_leader_fd;
+ int *fd;
+
+ int disabled;
+ int updated;
+
+ uint64_t value;
+
+ uint64_t *prev_value;
+ uint64_t *prev_time_enabled;
+ uint64_t *prev_time_running;
+} perf_events[] = {
+ // Hardware counters
+ {EV_ID_CPU_CYCLES, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES, &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_INSTRUCTIONS, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_CACHE_REFERENCES, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_CACHE_MISSES, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_BRANCH_INSTRUCTIONS, PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_INSTRUCTIONS, &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_BRANCH_MISSES, PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_MISSES, &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_BUS_CYCLES, PERF_TYPE_HARDWARE, PERF_COUNT_HW_BUS_CYCLES, &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_STALLED_CYCLES_FRONTEND, PERF_TYPE_HARDWARE, PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_STALLED_CYCLES_BACKEND, PERF_TYPE_HARDWARE, PERF_COUNT_HW_STALLED_CYCLES_BACKEND, &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_REF_CPU_CYCLES, PERF_TYPE_HARDWARE, PERF_COUNT_HW_REF_CPU_CYCLES, &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+
+ // Software counters
+ // {EV_ID_CPU_CLOCK, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CPU_CLOCK, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ // {EV_ID_TASK_CLOCK, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_TASK_CLOCK, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ // {EV_ID_PAGE_FAULTS, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ // {EV_ID_CONTEXT_SWITCHES, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CONTEXT_SWITCHES, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_CPU_MIGRATIONS, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CPU_MIGRATIONS, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ // {EV_ID_PAGE_FAULTS_MIN, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MIN, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ // {EV_ID_PAGE_FAULTS_MAJ, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MAJ, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_ALIGNMENT_FAULTS, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_ALIGNMENT_FAULTS, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+ {EV_ID_EMULATION_FAULTS, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_EMULATION_FAULTS, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+
+ // Hardware cache counters
+ {
+ EV_ID_L1D_READ_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_L1D_READ_MISS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_L1D_WRITE_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_L1D_WRITE_MISS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_L1D_PREFETCH_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+ },
+
+ {
+ EV_ID_L1I_READ_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_L1I) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_L1I_READ_MISS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_L1I) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ },
+
+ {
+ EV_ID_LL_READ_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_LL) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_LL_READ_MISS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_LL) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_LL_WRITE_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_LL) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_LL_WRITE_MISS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_LL) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ },
+
+ {
+ EV_ID_DTLB_READ_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_DTLB) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_DTLB_READ_MISS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_DTLB) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_DTLB_WRITE_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_DTLB) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_DTLB_WRITE_MISS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_DTLB) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_ITLB_BPU], NULL, 1, 0, 0, NULL, NULL, NULL
+ },
+
+ {
+ EV_ID_ITLB_READ_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_ITLB) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_ITLB_BPU], NULL, 1, 0, 0, NULL, NULL, NULL
+ }, {
+ EV_ID_ITLB_READ_MISS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_ITLB) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_ITLB_BPU], NULL, 1, 0, 0, NULL, NULL, NULL
+ },
+
+ {
+ EV_ID_PBU_READ_ACCESS, PERF_TYPE_HW_CACHE,
+ (PERF_COUNT_HW_CACHE_BPU) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+ &group_leader_fds[EV_GROUP_CACHE_ITLB_BPU], NULL, 1, 0, 0, NULL, NULL, NULL
+ },
+
+ {EV_ID_END, 0, 0, NULL, NULL, 0, 0, 0, NULL, NULL, NULL}
+};
+
+static int perf_init() {
+ int cpu, group;
+ struct perf_event_attr perf_event_attr;
+ struct perf_event *current_event = NULL;
+ unsigned long flags = 0;
+
+ number_of_cpus = (int)get_system_cpus();
+
+ // initialize all perf event file descriptors
+ for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) {
+ current_event->fd = mallocz(number_of_cpus * sizeof(int));
+ memset(current_event->fd, NO_FD, number_of_cpus * sizeof(int));
+
+ current_event->prev_value = mallocz(number_of_cpus * sizeof(uint64_t));
+ memset(current_event->prev_value, 0, number_of_cpus * sizeof(uint64_t));
+
+ current_event->prev_time_enabled = mallocz(number_of_cpus * sizeof(uint64_t));
+ memset(current_event->prev_time_enabled, 0, number_of_cpus * sizeof(uint64_t));
+
+ current_event->prev_time_running = mallocz(number_of_cpus * sizeof(uint64_t));
+ memset(current_event->prev_time_running, 0, number_of_cpus * sizeof(uint64_t));
+ }
+
+ for(group = 0; group < EV_GROUP_NUM; group++) {
+ group_leader_fds[group] = mallocz(number_of_cpus * sizeof(int));
+ memset(group_leader_fds[group], NO_FD, number_of_cpus * sizeof(int));
+ }
+
+ memset(&perf_event_attr, 0, sizeof(perf_event_attr));
+
+ for(cpu = 0; cpu < number_of_cpus; cpu++) {
+ for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) {
+ if(unlikely(current_event->disabled)) continue;
+
+ perf_event_attr.type = current_event->type;
+ perf_event_attr.config = current_event->config;
+ perf_event_attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING;
+
+ int fd, group_leader_fd = *(*current_event->group_leader_fd + cpu);
+
+ fd = syscall(
+ __NR_perf_event_open,
+ &perf_event_attr,
+ ALL_PIDS,
+ cpu,
+ group_leader_fd,
+ flags
+ );
+
+ if(unlikely(group_leader_fd == NO_FD)) group_leader_fd = fd;
+
+ if(unlikely(fd < 0)) {
+ switch errno {
+ case EACCES:
+ error("Cannot access to the PMU: Permission denied");
+ break;
+ case EBUSY:
+ error("Another event already has exclusive access to the PMU");
+ break;
+ default:
+ error("Cannot open perf event");
+ }
+ error("Disabling event %u", current_event->id);
+ current_event->disabled = 1;
+ }
+
+ *(current_event->fd + cpu) = fd;
+ *(*current_event->group_leader_fd + cpu) = group_leader_fd;
+
+ if(unlikely(debug)) fprintf(stderr, "perf.plugin: event id = %u, cpu = %d, fd = %d, leader_fd = %d\n", current_event->id, cpu, fd, group_leader_fd);
+ }
+ }
+
+ return 0;
+}
+
+static void perf_free(void) {
+ int cpu, group;
+ struct perf_event *current_event = NULL;
+
+ for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) {
+ for(cpu = 0; cpu < number_of_cpus; cpu++)
+ if(*(current_event->fd + cpu) != NO_FD) close(*(current_event->fd + cpu));
+
+ free(current_event->fd);
+ free(current_event->prev_value);
+ free(current_event->prev_time_enabled);
+ free(current_event->prev_time_running);
+ }
+
+ for(group = 0; group < EV_GROUP_NUM; group++)
+ free(group_leader_fds[group]);
+}
+
+static void reenable_events() {
+ int group, cpu;
+
+ for(group = 0; group < EV_GROUP_NUM; group++) {
+ for(cpu = 0; cpu < number_of_cpus; cpu++) {
+ int current_fd = *(group_leader_fds[group] + cpu);
+
+ if(unlikely(current_fd == NO_FD)) continue;
+
+ if(ioctl(current_fd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP) == -1
+ || ioctl(current_fd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP) == -1)
+ {
+ error("Cannot reenable event group");
+ }
+ }
+ }
+}
+
+static int perf_collect() {
+ int cpu;
+ struct perf_event *current_event = NULL;
+ static uint64_t prev_cpu_cycles_value = 0;
+ struct {
+ uint64_t value;
+ uint64_t time_enabled;
+ uint64_t time_running;
+ } read_result;
+
+ for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) {
+ current_event->updated = 0;
+ current_event->value = 0;
+
+ if(unlikely(current_event->disabled)) continue;
+
+ for(cpu = 0; cpu < number_of_cpus; cpu++) {
+
+ ssize_t read_size = read(current_event->fd[cpu], &read_result, sizeof(read_result));
+
+ if(likely(read_size == sizeof(read_result))) {
+ if (likely(read_result.time_running
+ && read_result.time_running != *(current_event->prev_time_running + cpu)
+ && (read_result.time_enabled / read_result.time_running < RUNNING_THRESHOLD))) {
+ current_event->value += (read_result.value - *(current_event->prev_value + cpu)) \
+ * (read_result.time_enabled - *(current_event->prev_time_enabled + cpu)) \
+ / (read_result.time_running - *(current_event->prev_time_running + cpu));
+ }
+
+ *(current_event->prev_value + cpu) = read_result.value;
+ *(current_event->prev_time_enabled + cpu) = read_result.time_enabled;
+ *(current_event->prev_time_running + cpu) = read_result.time_running;
+
+ current_event->updated = 1;
+ }
+ else {
+ error("Cannot update value for event %u", current_event->id);
+ return 1;
+ }
+ }
+
+ if(unlikely(debug)) fprintf(stderr, "perf.plugin: successfully read event id = %u, value = %"PRIu64"\n", current_event->id, current_event->value);
+ }
+
+ if(unlikely(perf_events[EV_ID_CPU_CYCLES].value == prev_cpu_cycles_value))
+ reenable_events();
+ prev_cpu_cycles_value = perf_events[EV_ID_CPU_CYCLES].value;
+
+ return 0;
+}
+
+static void perf_send_metrics() {
+ static int // Hardware counters
+ cpu_cycles_chart_generated = 0,
+ instructions_chart_generated = 0,
+ ipc_chart_generated = 0,
+ branch_chart_generated = 0,
+ cache_chart_generated = 0,
+ bus_cycles_chart_generated = 0,
+ stalled_cycles_chart_generated = 0,
+
+ // Software counters
+ migrations_chart_generated = 0,
+ alignment_chart_generated = 0,
+ emulation_chart_generated = 0,
+
+ // Hardware cache counters
+ L1D_chart_generated = 0,
+ L1D_prefetch_chart_generated = 0,
+ L1I_chart_generated = 0,
+ LL_chart_generated = 0,
+ DTLB_chart_generated = 0,
+ ITLB_chart_generated = 0,
+ PBU_chart_generated = 0;
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_CPU_CYCLES].updated || perf_events[EV_ID_REF_CPU_CYCLES].updated)) {
+ if(unlikely(!cpu_cycles_chart_generated)) {
+ cpu_cycles_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'CPU cycles' 'cycles/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "cpu_cycles"
+ , RRD_FAMILY_HW
+ , NETDATA_CHART_PRIO_PERF_CPU_CYCLES
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "cpu");
+ printf("DIMENSION %s '' absolute 1 1\n", "ref_cpu");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "cpu_cycles"
+ );
+ if(likely(perf_events[EV_ID_CPU_CYCLES].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "cpu"
+ , (collected_number) perf_events[EV_ID_CPU_CYCLES].value
+ );
+ }
+ if(likely(perf_events[EV_ID_REF_CPU_CYCLES].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "ref_cpu"
+ , (collected_number) perf_events[EV_ID_REF_CPU_CYCLES].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_INSTRUCTIONS].updated)) {
+ if(unlikely(!instructions_chart_generated)) {
+ instructions_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Instructions' 'instructions/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "instructions"
+ , RRD_FAMILY_HW
+ , NETDATA_CHART_PRIO_PERF_INSTRUCTIONS
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "instructions");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "instructions"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , "instructions"
+ , (collected_number) perf_events[EV_ID_INSTRUCTIONS].value
+ );
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_INSTRUCTIONS].updated) && likely(perf_events[EV_ID_CPU_CYCLES].updated)) {
+ if(unlikely(!ipc_chart_generated)) {
+ ipc_chart_generated = 1;
+
+ printf("CHART %s.%s '' '%s' 'instructions/cycle' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "instructions_per_cycle"
+ , "Instructions per Cycle(IPC)"
+ , RRD_FAMILY_HW
+ , NETDATA_CHART_PRIO_PERF_IPC
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 100\n", "ipc");
+ }
+
+ printf("BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "instructions_per_cycle"
+ );
+
+ NETDATA_DOUBLE result = ((NETDATA_DOUBLE)perf_events[EV_ID_INSTRUCTIONS].value /
+ (NETDATA_DOUBLE)perf_events[EV_ID_CPU_CYCLES].value) * 100.0;
+ printf("SET %s = %lld\n"
+ , "ipc"
+ , (collected_number) result
+ );
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_BRANCH_INSTRUCTIONS].updated || perf_events[EV_ID_BRANCH_MISSES].updated)) {
+ if(unlikely(!branch_chart_generated)) {
+ branch_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Branch instructions' 'instructions/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "branch_instructions"
+ , RRD_FAMILY_HW
+ , NETDATA_CHART_PRIO_PERF_BRANCH_INSTRUCTIONS
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "instructions");
+ printf("DIMENSION %s '' absolute 1 1\n", "misses");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "branch_instructions"
+ );
+ if(likely(perf_events[EV_ID_BRANCH_INSTRUCTIONS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "instructions"
+ , (collected_number) perf_events[EV_ID_BRANCH_INSTRUCTIONS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_BRANCH_MISSES].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "misses"
+ , (collected_number) perf_events[EV_ID_BRANCH_MISSES].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_CACHE_REFERENCES].updated || perf_events[EV_ID_CACHE_MISSES].updated)) {
+ if(unlikely(!cache_chart_generated)) {
+ cache_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Cache operations' 'operations/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "cache"
+ , RRD_FAMILY_HW
+ , NETDATA_CHART_PRIO_PERF_CACHE
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "references");
+ printf("DIMENSION %s '' absolute 1 1\n", "misses");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "cache"
+ );
+ if(likely(perf_events[EV_ID_CACHE_REFERENCES].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "references"
+ , (collected_number) perf_events[EV_ID_CACHE_REFERENCES].value
+ );
+ }
+ if(likely(perf_events[EV_ID_CACHE_MISSES].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "misses"
+ , (collected_number) perf_events[EV_ID_CACHE_MISSES].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_BUS_CYCLES].updated)) {
+ if(unlikely(!bus_cycles_chart_generated)) {
+ bus_cycles_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Bus cycles' 'cycles/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "bus_cycles"
+ , RRD_FAMILY_HW
+ , NETDATA_CHART_PRIO_PERF_BUS_CYCLES
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "bus");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "bus_cycles"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , "bus"
+ , (collected_number) perf_events[EV_ID_BUS_CYCLES].value
+ );
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_STALLED_CYCLES_FRONTEND].updated || perf_events[EV_ID_STALLED_CYCLES_BACKEND].updated)) {
+ if(unlikely(!stalled_cycles_chart_generated)) {
+ stalled_cycles_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Stalled frontend and backend cycles' 'cycles/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "stalled_cycles"
+ , RRD_FAMILY_HW
+ , NETDATA_CHART_PRIO_PERF_FRONT_BACK_CYCLES
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "frontend");
+ printf("DIMENSION %s '' absolute 1 1\n", "backend");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "stalled_cycles"
+ );
+ if(likely(perf_events[EV_ID_STALLED_CYCLES_FRONTEND].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "frontend"
+ , (collected_number) perf_events[EV_ID_STALLED_CYCLES_FRONTEND].value
+ );
+ }
+ if(likely(perf_events[EV_ID_STALLED_CYCLES_BACKEND].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "backend"
+ , (collected_number) perf_events[EV_ID_STALLED_CYCLES_BACKEND].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_CPU_MIGRATIONS].updated)) {
+ if(unlikely(!migrations_chart_generated)) {
+ migrations_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'CPU migrations' 'migrations' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "migrations"
+ , RRD_FAMILY_SW
+ , NETDATA_CHART_PRIO_PERF_MIGRATIONS
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "migrations");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "migrations"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , "migrations"
+ , (collected_number) perf_events[EV_ID_CPU_MIGRATIONS].value
+ );
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_ALIGNMENT_FAULTS].updated)) {
+ if(unlikely(!alignment_chart_generated)) {
+ alignment_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Alignment faults' 'faults' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "alignment_faults"
+ , RRD_FAMILY_SW
+ , NETDATA_CHART_PRIO_PERF_ALIGNMENT
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "faults");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "alignment_faults"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , "faults"
+ , (collected_number) perf_events[EV_ID_ALIGNMENT_FAULTS].value
+ );
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_EMULATION_FAULTS].updated)) {
+ if(unlikely(!emulation_chart_generated)) {
+ emulation_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'Emulation faults' 'faults' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "emulation_faults"
+ , RRD_FAMILY_SW
+ , NETDATA_CHART_PRIO_PERF_EMULATION
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "faults");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "emulation_faults"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , "faults"
+ , (collected_number) perf_events[EV_ID_EMULATION_FAULTS].value
+ );
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_L1D_READ_ACCESS].updated || perf_events[EV_ID_L1D_READ_MISS].updated
+ || perf_events[EV_ID_L1D_WRITE_ACCESS].updated || perf_events[EV_ID_L1D_WRITE_MISS].updated)) {
+ if(unlikely(!L1D_chart_generated)) {
+ L1D_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'L1D cache operations' 'events/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "l1d_cache"
+ , RRD_FAMILY_CACHE
+ , NETDATA_CHART_PRIO_PERF_L1D
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "read_access");
+ printf("DIMENSION %s '' absolute 1 1\n", "read_misses");
+ printf("DIMENSION %s '' absolute -1 1\n", "write_access");
+ printf("DIMENSION %s '' absolute -1 1\n", "write_misses");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "l1d_cache"
+ );
+ if(likely(perf_events[EV_ID_L1D_READ_ACCESS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_access"
+ , (collected_number) perf_events[EV_ID_L1D_READ_ACCESS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_L1D_READ_MISS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_misses"
+ , (collected_number) perf_events[EV_ID_L1D_READ_MISS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_L1D_WRITE_ACCESS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "write_access"
+ , (collected_number) perf_events[EV_ID_L1D_WRITE_ACCESS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_L1D_WRITE_MISS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "write_misses"
+ , (collected_number) perf_events[EV_ID_L1D_WRITE_MISS].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_L1D_PREFETCH_ACCESS].updated)) {
+ if(unlikely(!L1D_prefetch_chart_generated)) {
+ L1D_prefetch_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'L1D prefetch cache operations' 'prefetches/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "l1d_cache_prefetch"
+ , RRD_FAMILY_CACHE
+ , NETDATA_CHART_PRIO_PERF_L1D_PREFETCH
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "prefetches");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "l1d_cache_prefetch"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , "prefetches"
+ , (collected_number) perf_events[EV_ID_L1D_PREFETCH_ACCESS].value
+ );
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_L1I_READ_ACCESS].updated || perf_events[EV_ID_L1I_READ_MISS].updated)) {
+ if(unlikely(!L1I_chart_generated)) {
+ L1I_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'L1I cache operations' 'events/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "l1i_cache"
+ , RRD_FAMILY_CACHE
+ , NETDATA_CHART_PRIO_PERF_L1I
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "read_access");
+ printf("DIMENSION %s '' absolute 1 1\n", "read_misses");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "l1i_cache"
+ );
+ if(likely(perf_events[EV_ID_L1I_READ_ACCESS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_access"
+ , (collected_number) perf_events[EV_ID_L1I_READ_ACCESS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_L1I_READ_MISS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_misses"
+ , (collected_number) perf_events[EV_ID_L1I_READ_MISS].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_LL_READ_ACCESS].updated || perf_events[EV_ID_LL_READ_MISS].updated
+ || perf_events[EV_ID_LL_WRITE_ACCESS].updated || perf_events[EV_ID_LL_WRITE_MISS].updated)) {
+ if(unlikely(!LL_chart_generated)) {
+ LL_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'LL cache operations' 'events/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "ll_cache"
+ , RRD_FAMILY_CACHE
+ , NETDATA_CHART_PRIO_PERF_LL
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "read_access");
+ printf("DIMENSION %s '' absolute 1 1\n", "read_misses");
+ printf("DIMENSION %s '' absolute -1 1\n", "write_access");
+ printf("DIMENSION %s '' absolute -1 1\n", "write_misses");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "ll_cache"
+ );
+ if(likely(perf_events[EV_ID_LL_READ_ACCESS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_access"
+ , (collected_number) perf_events[EV_ID_LL_READ_ACCESS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_LL_READ_MISS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_misses"
+ , (collected_number) perf_events[EV_ID_LL_READ_MISS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_LL_WRITE_ACCESS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "write_access"
+ , (collected_number) perf_events[EV_ID_LL_WRITE_ACCESS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_LL_WRITE_MISS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "write_misses"
+ , (collected_number) perf_events[EV_ID_LL_WRITE_MISS].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_DTLB_READ_ACCESS].updated || perf_events[EV_ID_DTLB_READ_MISS].updated
+ || perf_events[EV_ID_DTLB_WRITE_ACCESS].updated || perf_events[EV_ID_DTLB_WRITE_MISS].updated)) {
+ if(unlikely(!DTLB_chart_generated)) {
+ DTLB_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'DTLB cache operations' 'events/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "dtlb_cache"
+ , RRD_FAMILY_CACHE
+ , NETDATA_CHART_PRIO_PERF_DTLB
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "read_access");
+ printf("DIMENSION %s '' absolute 1 1\n", "read_misses");
+ printf("DIMENSION %s '' absolute -1 1\n", "write_access");
+ printf("DIMENSION %s '' absolute -1 1\n", "write_misses");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "dtlb_cache"
+ );
+ if(likely(perf_events[EV_ID_DTLB_READ_ACCESS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_access"
+ , (collected_number) perf_events[EV_ID_DTLB_READ_ACCESS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_DTLB_READ_MISS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_misses"
+ , (collected_number) perf_events[EV_ID_DTLB_READ_MISS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_DTLB_WRITE_ACCESS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "write_access"
+ , (collected_number) perf_events[EV_ID_DTLB_WRITE_ACCESS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_DTLB_WRITE_MISS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "write_misses"
+ , (collected_number) perf_events[EV_ID_DTLB_WRITE_MISS].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_ITLB_READ_ACCESS].updated || perf_events[EV_ID_ITLB_READ_MISS].updated)) {
+ if(unlikely(!ITLB_chart_generated)) {
+ ITLB_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'ITLB cache operations' 'events/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "itlb_cache"
+ , RRD_FAMILY_CACHE
+ , NETDATA_CHART_PRIO_PERF_ITLB
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "read_access");
+ printf("DIMENSION %s '' absolute 1 1\n", "read_misses");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "itlb_cache"
+ );
+ if(likely(perf_events[EV_ID_ITLB_READ_ACCESS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_access"
+ , (collected_number) perf_events[EV_ID_ITLB_READ_ACCESS].value
+ );
+ }
+ if(likely(perf_events[EV_ID_ITLB_READ_MISS].updated)) {
+ printf(
+ "SET %s = %lld\n"
+ , "read_misses"
+ , (collected_number) perf_events[EV_ID_ITLB_READ_MISS].value
+ );
+ }
+ printf("END\n");
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(likely(perf_events[EV_ID_PBU_READ_ACCESS].updated)) {
+ if(unlikely(!PBU_chart_generated)) {
+ PBU_chart_generated = 1;
+
+ printf("CHART %s.%s '' 'PBU cache operations' 'events/s' %s '' line %d %d '' %s\n"
+ , RRD_TYPE_PERF
+ , "pbu_cache"
+ , RRD_FAMILY_CACHE
+ , NETDATA_CHART_PRIO_PERF_PBU
+ , update_every
+ , PLUGIN_PERF_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 1\n", "read_access");
+ }
+
+ printf(
+ "BEGIN %s.%s\n"
+ , RRD_TYPE_PERF
+ , "pbu_cache"
+ );
+ printf(
+ "SET %s = %lld\n"
+ , "read_access"
+ , (collected_number) perf_events[EV_ID_PBU_READ_ACCESS].value
+ );
+ printf("END\n");
+ }
+}
+
+void parse_command_line(int argc, char **argv) {
+ int i, plugin_enabled = 0;
+
+ for(i = 1; i < argc ; i++) {
+ if(isdigit(*argv[i]) && !freq) {
+ int n = str2i(argv[i]);
+ if(n > 0 && n < 86400) {
+ freq = n;
+ continue;
+ }
+ }
+ else if(strcmp("version", argv[i]) == 0 || strcmp("-version", argv[i]) == 0 || strcmp("--version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) {
+ printf("perf.plugin %s\n", VERSION);
+ exit(0);
+ }
+ else if(strcmp("all", argv[i]) == 0) {
+ struct perf_event *current_event = NULL;
+
+ for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++)
+ current_event->disabled = 0;
+
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("cycles", argv[i]) == 0) {
+ perf_events[EV_ID_CPU_CYCLES].disabled = 0;
+ perf_events[EV_ID_REF_CPU_CYCLES].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("instructions", argv[i]) == 0) {
+ perf_events[EV_ID_INSTRUCTIONS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("branch", argv[i]) == 0) {
+ perf_events[EV_ID_BRANCH_INSTRUCTIONS].disabled = 0;
+ perf_events[EV_ID_BRANCH_MISSES].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("cache", argv[i]) == 0) {
+ perf_events[EV_ID_CACHE_REFERENCES].disabled = 0;
+ perf_events[EV_ID_CACHE_MISSES].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("bus", argv[i]) == 0) {
+ perf_events[EV_ID_BUS_CYCLES].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("stalled", argv[i]) == 0) {
+ perf_events[EV_ID_STALLED_CYCLES_FRONTEND].disabled = 0;
+ perf_events[EV_ID_STALLED_CYCLES_BACKEND].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("migrations", argv[i]) == 0) {
+ perf_events[EV_ID_CPU_MIGRATIONS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("alignment", argv[i]) == 0) {
+ perf_events[EV_ID_ALIGNMENT_FAULTS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("emulation", argv[i]) == 0) {
+ perf_events[EV_ID_EMULATION_FAULTS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("L1D", argv[i]) == 0) {
+ perf_events[EV_ID_L1D_READ_ACCESS].disabled = 0;
+ perf_events[EV_ID_L1D_READ_MISS].disabled = 0;
+ perf_events[EV_ID_L1D_WRITE_ACCESS].disabled = 0;
+ perf_events[EV_ID_L1D_WRITE_MISS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("L1D-prefetch", argv[i]) == 0) {
+ perf_events[EV_ID_L1D_PREFETCH_ACCESS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("L1I", argv[i]) == 0) {
+ perf_events[EV_ID_L1I_READ_ACCESS].disabled = 0;
+ perf_events[EV_ID_L1I_READ_MISS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("LL", argv[i]) == 0) {
+ perf_events[EV_ID_LL_READ_ACCESS].disabled = 0;
+ perf_events[EV_ID_LL_READ_MISS].disabled = 0;
+ perf_events[EV_ID_LL_WRITE_ACCESS].disabled = 0;
+ perf_events[EV_ID_LL_WRITE_MISS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("DTLB", argv[i]) == 0) {
+ perf_events[EV_ID_DTLB_READ_ACCESS].disabled = 0;
+ perf_events[EV_ID_DTLB_READ_MISS].disabled = 0;
+ perf_events[EV_ID_DTLB_WRITE_ACCESS].disabled = 0;
+ perf_events[EV_ID_DTLB_WRITE_MISS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("ITLB", argv[i]) == 0) {
+ perf_events[EV_ID_ITLB_READ_ACCESS].disabled = 0;
+ perf_events[EV_ID_ITLB_READ_MISS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("PBU", argv[i]) == 0) {
+ perf_events[EV_ID_PBU_READ_ACCESS].disabled = 0;
+ plugin_enabled = 1;
+ continue;
+ }
+ else if(strcmp("debug", argv[i]) == 0) {
+ debug = 1;
+ continue;
+ }
+ else if(strcmp("-h", argv[i]) == 0 || strcmp("--help", argv[i]) == 0) {
+ fprintf(stderr,
+ "\n"
+ " netdata perf.plugin %s\n"
+ " Copyright (C) 2019 Netdata Inc.\n"
+ " Released under GNU General Public License v3 or later.\n"
+ " All rights reserved.\n"
+ "\n"
+ " This program is a data collector plugin for netdata.\n"
+ "\n"
+ " Available command line options:\n"
+ "\n"
+ " COLLECTION_FREQUENCY data collection frequency in seconds\n"
+ " minimum: %d\n"
+ "\n"
+ " all enable all charts\n"
+ "\n"
+ " cycles enable CPU cycles chart\n"
+ "\n"
+ " instructions enable Instructions chart\n"
+ "\n"
+ " branch enable Branch instructions chart\n"
+ "\n"
+ " cache enable Cache operations chart\n"
+ "\n"
+ " bus enable Bus cycles chart\n"
+ "\n"
+ " stalled enable Stalled frontend and backend cycles chart\n"
+ "\n"
+ " migrations enable CPU migrations chart\n"
+ "\n"
+ " alignment enable Alignment faults chart\n"
+ "\n"
+ " emulation enable Emulation faults chart\n"
+ "\n"
+ " L1D enable L1D cache operations chart\n"
+ "\n"
+ " L1D-prefetch enable L1D prefetch cache operations chart\n"
+ "\n"
+ " L1I enable L1I cache operations chart\n"
+ "\n"
+ " LL enable LL cache operations chart\n"
+ "\n"
+ " DTLB enable DTLB cache operations chart\n"
+ "\n"
+ " ITLB enable ITLB cache operations chart\n"
+ "\n"
+ " PBU enable PBU cache operations chart\n"
+ "\n"
+ " debug enable verbose output\n"
+ " default: disabled\n"
+ "\n"
+ " -v\n"
+ " -V\n"
+ " --version print version and exit\n"
+ "\n"
+ " -h\n"
+ " --help print this message and exit\n"
+ "\n"
+ " For more information:\n"
+ " https://github.com/netdata/netdata/tree/master/collectors/perf.plugin\n"
+ "\n"
+ , VERSION
+ , update_every
+ );
+ exit(1);
+ }
+
+ error("ignoring parameter '%s'", argv[i]);
+ }
+
+ if(!plugin_enabled){
+ info("no charts enabled - nothing to do.");
+ printf("DISABLE\n");
+ exit(1);
+ }
+}
+
+int main(int argc, char **argv) {
+ clocks_init();
+
+ // ------------------------------------------------------------------------
+ // initialization of netdata plugin
+
+ program_name = "perf.plugin";
+
+ // disable syslog
+ error_log_syslog = 0;
+
+ // set errors flood protection to 100 logs per hour
+ error_log_errors_per_period = 100;
+ error_log_throttle_period = 3600;
+
+ parse_command_line(argc, argv);
+
+ errno = 0;
+
+ if(freq >= update_every)
+ update_every = freq;
+ else if(freq)
+ error("update frequency %d seconds is too small for PERF. Using %d.", freq, update_every);
+
+ if(unlikely(debug)) fprintf(stderr, "perf.plugin: calling perf_init()\n");
+ int perf = !perf_init();
+
+ // ------------------------------------------------------------------------
+ // the main loop
+
+ if(unlikely(debug)) fprintf(stderr, "perf.plugin: starting data collection\n");
+
+ time_t started_t = now_monotonic_sec();
+
+ size_t iteration;
+ usec_t step = update_every * USEC_PER_SEC;
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ for(iteration = 0; 1; iteration++) {
+ usec_t dt = heartbeat_next(&hb, step);
+
+ if(unlikely(netdata_exit)) break;
+
+ if(unlikely(debug && iteration))
+ fprintf(stderr, "perf.plugin: iteration %zu, dt %llu usec\n"
+ , iteration
+ , dt
+ );
+
+ if(likely(perf)) {
+ if(unlikely(debug)) fprintf(stderr, "perf.plugin: calling perf_collect()\n");
+ perf = !perf_collect();
+
+ if(likely(perf)) {
+ if(unlikely(debug)) fprintf(stderr, "perf.plugin: calling perf_send_metrics()\n");
+ perf_send_metrics();
+ }
+ }
+
+ fflush(stdout);
+
+ // restart check (14400 seconds)
+ if(now_monotonic_sec() - started_t > 14400) break;
+ }
+
+ info("process exiting");
+ perf_free();
+}
diff --git a/collectors/plugins.d/Makefile.am b/collectors/plugins.d/Makefile.am
new file mode 100644
index 0000000..59250a9
--- /dev/null
+++ b/collectors/plugins.d/Makefile.am
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+SUBDIRS = \
+ $(NULL)
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/plugins.d/README.md b/collectors/plugins.d/README.md
new file mode 100644
index 0000000..2ecf233
--- /dev/null
+++ b/collectors/plugins.d/README.md
@@ -0,0 +1,599 @@
+<!--
+title: "External plugins overview"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/plugins.d/README.md
+-->
+
+# External plugins overview
+
+`plugins.d` is the Netdata internal plugin that collects metrics
+from external processes, thus allowing Netdata to use **external plugins**.
+
+## Provided External Plugins
+
+|plugin|language|O/S|description|
+|:----:|:------:|:-:|:----------|
+|[apps.plugin](/collectors/apps.plugin/README.md)|`C`|linux, freebsd|monitors the whole process tree on Linux and FreeBSD and breaks down system resource usage by **process**, **user** and **user group**.|
+|[charts.d.plugin](/collectors/charts.d.plugin/README.md)|`BASH`|all|a **plugin orchestrator** for data collection modules written in `BASH` v4+.|
+|[cups.plugin](/collectors/cups.plugin/README.md)|`C`|all|monitors **CUPS**|
+|[fping.plugin](/collectors/fping.plugin/README.md)|`C`|all|measures network latency, jitter and packet loss between the monitored node and any number of remote network end points.|
+|[ioping.plugin](/collectors/ioping.plugin/README.md)|`C`|all|measures disk latency.|
+|[freeipmi.plugin](/collectors/freeipmi.plugin/README.md)|`C`|linux|collects metrics from enterprise hardware sensors, on Linux servers.|
+|[nfacct.plugin](/collectors/nfacct.plugin/README.md)|`C`|linux|collects netfilter firewall, connection tracker and accounting metrics using `libmnl` and `libnetfilter_acct`.|
+|[xenstat.plugin](/collectors/xenstat.plugin/README.md)|`C`|linux|collects XenServer and XCP-ng metrics using `lxenstat`.|
+|[perf.plugin](/collectors/perf.plugin/README.md)|`C`|linux|collects CPU performance metrics using performance monitoring units (PMU).|
+|[python.d.plugin](/collectors/python.d.plugin/README.md)|`python`|all|a **plugin orchestrator** for data collection modules written in `python` v2 or v3 (both are supported).|
+|[slabinfo.plugin](/collectors/slabinfo.plugin/README.md)|`C`|linux|collects kernel internal cache objects (SLAB) metrics.|
+
+Plugin orchestrators may also be described as **modular plugins**. They are modular since they accept custom made modules to be included. Writing modules for these plugins is easier than accessing the native Netdata API directly. You will find modules already available for each orchestrator under the directory of the particular modular plugin (e.g. under python.d.plugin for the python orchestrator).
+Each of these modular plugins has each own methods for defining modules. Please check the examples and their documentation.
+
+## Motivation
+
+This plugin allows Netdata to use **external plugins** for data collection:
+
+1. external data collection plugins may be written in any computer language.
+
+2. external data collection plugins may use O/S capabilities or `setuid` to
+ run with escalated privileges (compared to the `netdata` daemon).
+ The communication between the external plugin and Netdata is unidirectional
+ (from the plugin to Netdata), so that Netdata cannot manipulate an external
+ plugin running with escalated privileges.
+
+## Operation
+
+Each of the external plugins is expected to run forever.
+Netdata will start it when it starts and stop it when it exits.
+
+If the external plugin exits or crashes, Netdata will log an error.
+If the external plugin exits or crashes without pushing metrics to Netdata, Netdata will not start it again.
+
+- Plugins that exit with any value other than zero, will be disabled. Plugins that exit with zero, will be restarted after some time.
+- Plugins may also be disabled by Netdata if they output things that Netdata does not understand.
+
+The `stdout` of external plugins is connected to Netdata to receive metrics,
+with the API defined below.
+
+The `stderr` of external plugins is connected to Netdata's `error.log`.
+
+Plugins can create any number of charts with any number of dimensions each. Each chart can have its own characteristics independently of the others generated by the same plugin. For example, one chart may have an update frequency of 1 second, another may have 5 seconds and a third may have 10 seconds.
+
+## Configuration
+
+Netdata will supply the environment variables `NETDATA_USER_CONFIG_DIR` (for user supplied) and `NETDATA_STOCK_CONFIG_DIR` (for Netdata supplied) configuration files to identify the directory where configuration files are stored. It is up to the plugin to read the configuration it needs.
+
+The `netdata.conf` section `[plugins]` section contains a list of all the plugins found at the system where Netdata runs, with a boolean setting to enable them or not.
+
+Example:
+
+```
+[plugins]
+ # enable running new plugins = yes
+ # check for new plugins every = 60
+
+ # charts.d = yes
+ # fping = yes
+ # ioping = yes
+ # python.d = yes
+```
+
+The setting `enable running new plugins` sets the default behavior for all external plugins. It can be
+overridden for distinct plugins by modifying the appropriate plugin value configuration to either `yes` or `no`.
+
+The setting `check for new plugins every` sets the interval between scans of the directory
+`/usr/libexec/netdata/plugins.d`. New plugins can be added any time, and Netdata will detect them in a timely manner.
+
+For each of the external plugins enabled, another `netdata.conf` section
+is created, in the form of `[plugin:NAME]`, where `NAME` is the name of the external plugin.
+This section allows controlling the update frequency of the plugin and provide
+additional command line arguments to it.
+
+For example, for `apps.plugin` the following section is available:
+
+```
+[plugin:apps]
+ # update every = 1
+ # command options =
+```
+
+- `update every` controls the granularity of the external plugin.
+- `command options` allows giving additional command line options to the plugin.
+
+Netdata will provide to the external plugins the environment variable `NETDATA_UPDATE_EVERY`, in seconds (the default is 1). This is the **minimum update frequency** for all charts. A plugin that is updating values more frequently than this, is just wasting resources.
+
+Netdata will call the plugin with just one command line parameter: the number of seconds the user requested this plugin to update its data (by default is also 1).
+
+Other than the above, the plugin configuration is up to the plugin.
+
+Keep in mind, that the user may use Netdata configuration to overwrite chart and dimension parameters. This is transparent to the plugin.
+
+### Autoconfiguration
+
+Plugins should attempt to autoconfigure themselves when possible.
+
+For example, if your plugin wants to monitor `squid`, you can search for it on port `3128` or `8080`. If any succeeds, you can proceed. If it fails you can output an error (on stderr) saying that you cannot find `squid` running and giving instructions about the plugin configuration. Then you can stop (exit with non-zero value), so that Netdata will not attempt to start the plugin again.
+
+## External Plugins API
+
+Any program that can print a few values to its standard output can become a Netdata external plugin.
+
+Netdata parses lines starting with:
+
+- `CHART` - create or update a chart
+- `DIMENSION` - add or update a dimension to the chart just created
+- `VARIABLE` - define a variable (to be used in health calculations)
+- `CLABEL` - add a label to a chart
+- `CLABEL_COMMIT` - commit added labels to the chart
+- `FUNCTION` - define a function that can be called later to execute it
+- `BEGIN` - initialize data collection for a chart
+- `SET` - set the value of a dimension for the initialized chart
+- `END` - complete data collection for the initialized chart
+- `FLUSH` - ignore the last collected values
+- `DISABLE` - disable this plugin
+
+a single program can produce any number of charts with any number of dimensions each.
+
+Charts can be added any time (not just the beginning).
+
+### command line parameters
+
+The plugin **MUST** accept just **one** parameter: **the number of seconds it is
+expected to update the values for its charts**. The value passed by Netdata
+to the plugin is controlled via its configuration file (so there is no need
+for the plugin to handle this configuration option).
+
+The external plugin can overwrite the update frequency. For example, the server may
+request per second updates, but the plugin may ignore it and update its charts
+every 5 seconds.
+
+### environment variables
+
+There are a few environment variables that are set by `netdata` and are
+available for the plugin to use.
+
+|variable|description|
+|:------:|:----------|
+|`NETDATA_USER_CONFIG_DIR`|The directory where all Netdata-related user configuration should be stored. If the plugin requires custom user configuration, this is the place the user has saved it (normally under `/etc/netdata`).|
+|`NETDATA_STOCK_CONFIG_DIR`|The directory where all Netdata -related stock configuration should be stored. If the plugin is shipped with configuration files, this is the place they can be found (normally under `/usr/lib/netdata/conf.d`).|
+|`NETDATA_PLUGINS_DIR`|The directory where all Netdata plugins are stored.|
+|`NETDATA_USER_PLUGINS_DIRS`|The list of directories where custom plugins are stored.|
+|`NETDATA_WEB_DIR`|The directory where the web files of Netdata are saved.|
+|`NETDATA_CACHE_DIR`|The directory where the cache files of Netdata are stored. Use this directory if the plugin requires a place to store data. A new directory should be created for the plugin for this purpose, inside this directory.|
+|`NETDATA_LOG_DIR`|The directory where the log files are stored. By default the `stderr` output of the plugin will be saved in the `error.log` file of Netdata.|
+|`NETDATA_HOST_PREFIX`|This is used in environments where system directories like `/sys` and `/proc` have to be accessed at a different path.|
+|`NETDATA_DEBUG_FLAGS`|This is a number (probably in hex starting with `0x`), that enables certain Netdata debugging features. Check **\[[Tracing Options]]** for more information.|
+|`NETDATA_UPDATE_EVERY`|The minimum number of seconds between chart refreshes. This is like the **internal clock** of Netdata (it is user configurable, defaulting to `1`). There is no meaning for a plugin to update its values more frequently than this number of seconds.|
+
+### The output of the plugin
+
+The plugin should output instructions for Netdata to its output (`stdout`). Since this uses pipes, please make sure you flush stdout after every iteration.
+
+#### DISABLE
+
+`DISABLE` will disable this plugin. This will prevent Netdata from restarting the plugin. You can also exit with the value `1` to have the same effect.
+
+#### CHART
+
+`CHART` defines a new chart.
+
+the template is:
+
+> CHART type.id name title units \[family \[context \[charttype \[priority \[update_every \[options \[plugin [module]]]]]]]]
+
+ where:
+
+- `type.id`
+
+ uniquely identifies the chart,
+ this is what will be needed to add values to the chart
+
+ the `type` part controls the menu the charts will appear in
+
+- `name`
+
+ is the name that will be presented to the user instead of `id` in `type.id`. This means that only the `id` part of
+ `type.id` is changed. When a name has been given, the chart is indexed (and can be referred) as both `type.id` and
+ `type.name`. You can set name to `''`, or `null`, or `(null)` to disable it. If a chart with the same name already
+ exists, a serial number is automatically attached to the name to avoid naming collisions.
+
+- `title`
+
+ the text above the chart
+
+- `units`
+
+ the label of the vertical axis of the chart,
+ all dimensions added to a chart should have the same units
+ of measurement
+
+- `family`
+
+ is used to group charts together
+ (for example all eth0 charts should say: eth0),
+ if empty or missing, the `id` part of `type.id` will be used
+
+ this controls the sub-menu on the dashboard
+
+- `context`
+
+ the context is giving the template of the chart. For example, if multiple charts present the same information for a different family, they should have the same `context`
+
+ this is used for looking up rendering information for the chart (colors, sizes, informational texts) and also apply alarms to it
+
+- `charttype`
+
+ one of `line`, `area` or `stacked`,
+ if empty or missing, the `line` will be used
+
+- `priority`
+
+ is the relative priority of the charts as rendered on the web page,
+ lower numbers make the charts appear before the ones with higher numbers,
+ if empty or missing, `1000` will be used
+
+- `update_every`
+
+ overwrite the update frequency set by the server,
+ if empty or missing, the user configured value will be used
+
+- `options`
+
+ a space separated list of options, enclosed in quotes. 4 options are currently supported: `obsolete` to mark a chart as obsolete (Netdata will hide it and delete it after some time), `detail` to mark a chart as insignificant (this may be used by dashboards to make the charts smaller, or somehow visualize properly a less important chart), `store_first` to make Netdata store the first collected value, assuming there was an invisible previous value set to zero (this is used by statsd charts - if the first data collected value of incremental dimensions is not zero based, unrealistic spikes will appear with this option set) and `hidden` to perform all operations on a chart, but do not offer it on dashboards (the chart will be send to external databases). `CHART` options have been added in Netdata v1.7 and the `hidden` option was added in 1.10.
+
+- `plugin` and `module`
+
+ both are just names that are used to let the user identify the plugin and the module that generated the chart. If `plugin` is unset or empty, Netdata will automatically set the filename of the plugin that generated the chart. `module` has not default.
+
+#### DIMENSION
+
+`DIMENSION` defines a new dimension for the chart
+
+the template is:
+
+> DIMENSION id \[name \[algorithm \[multiplier \[divisor [options]]]]]
+
+ where:
+
+- `id`
+
+ the `id` of this dimension (it is a text value, not numeric),
+ this will be needed later to add values to the dimension
+
+ We suggest to avoid using `.` in dimension ids. External databases expect metrics to be `.` separated and people will get confused if a dimension id contains a dot.
+
+- `name`
+
+ the name of the dimension as it will appear at the legend of the chart,
+ if empty or missing the `id` will be used
+
+- `algorithm`
+
+ one of:
+
+ - `absolute`
+
+ the value is to drawn as-is (interpolated to second boundary),
+ if `algorithm` is empty, invalid or missing, `absolute` is used
+
+ - `incremental`
+
+ the value increases over time,
+ the difference from the last value is presented in the chart,
+ the server interpolates the value and calculates a per second figure
+
+ - `percentage-of-absolute-row`
+
+ the % of this value compared to the total of all dimensions
+
+ - `percentage-of-incremental-row`
+
+ the % of this value compared to the incremental total of
+ all dimensions
+
+- `multiplier`
+
+ an integer value to multiply the collected value,
+ if empty or missing, `1` is used
+
+- `divisor`
+
+ an integer value to divide the collected value,
+ if empty or missing, `1` is used
+
+- `options`
+
+ a space separated list of options, enclosed in quotes. Options supported: `obsolete` to mark a dimension as obsolete (Netdata will delete it after some time) and `hidden` to make this dimension hidden, it will take part in the calculations but will not be presented in the chart.
+
+#### VARIABLE
+
+> VARIABLE [SCOPE] name = value
+
+`VARIABLE` defines a variable that can be used in alarms. This is to used for setting constants (like the max connections a server may accept).
+
+Variables support 2 scopes:
+
+- `GLOBAL` or `HOST` to define the variable at the host level.
+- `LOCAL` or `CHART` to define the variable at the chart level. Use chart-local variables when the same variable may exist for different charts (i.e. Netdata monitors 2 mysql servers, and you need to set the `max_connections` each server accepts). Using chart-local variables is the ideal to build alarm templates.
+
+The position of the `VARIABLE` line, sets its default scope (in case you do not specify a scope). So, defining a `VARIABLE` before any `CHART`, or between `END` and `BEGIN` (outside any chart), sets `GLOBAL` scope, while defining a `VARIABLE` just after a `CHART` or a `DIMENSION`, or within the `BEGIN` - `END` block of a chart, sets `LOCAL` scope.
+
+These variables can be set and updated at any point.
+
+Variable names should use alphanumeric characters, the `.` and the `_`.
+
+The `value` is floating point (Netdata used `long double`).
+
+Variables are transferred to upstream Netdata servers (streaming and database replication).
+
+#### CLABEL
+
+> CLABEL name value source
+
+`CLABEL` defines a label used to organize and identify a chart.
+
+Name and value accept characters according to the following table:
+
+| Character | Symbol | Label Name | Label Value |
+|---------------------|:------:|:----------:|:-----------:|
+| UTF-8 character | UTF-8 | _ | keep |
+| Lower case letter | [a-z] | keep | keep |
+| Upper case letter | [A-Z] | keep | [a-z] |
+| Digit | [0-9] | keep | keep |
+| Underscore | _ | keep | keep |
+| Minus | - | keep | keep |
+| Plus | + | _ | keep |
+| Colon | : | _ | keep |
+| Semicolon | ; | _ | : |
+| Equal | = | _ | : |
+| Period | . | keep | keep |
+| Comma | , | . | . |
+| Slash | / | keep | keep |
+| Backslash | \ | / | / |
+| At | @ | _ | keep |
+| Space | ' ' | _ | keep |
+| Opening parenthesis | ( | _ | keep |
+| Closing parenthesis | ) | _ | keep |
+| Anything else | | _ | _ |
+
+The `source` is an integer field that can have the following values:
+- `1`: The value was set automatically.
+- `2`: The value was set manually.
+- `4`: This is a K8 label.
+- `8`: This is a label defined using `netdata` agent cloud link.
+
+#### CLABEL_COMMIT
+
+`CLABEL_COMMIT` indicates that all labels were defined and the chart can be updated.
+
+#### FUNCTION
+
+> FUNCTION [GLOBAL] "name and parameters of the function" timeout "help string for users"
+
+A function can be used by users to ask for more information from the collector. Netdata maintains a registry of functions in 2 levels:
+
+- per node
+- per chart
+
+Both node and chart functions are exactly the same, but chart functions allow Netdata to relate functions with charts and therefore present a context sensitive menu of functions related to the chart the user is using.
+
+A function is identified by a string. The allowed characters in the function definition are:
+
+| Character | Symbol | In Functions |
+|-------------------|:------:|:------------:|
+| UTF-8 character | UTF-8 | keep |
+| Lower case letter | [a-z] | keep |
+| Upper case letter | [A-Z] | keep |
+| Digit | [0-9] | keep |
+| Underscore | _ | keep |
+| Comma | , | keep |
+| Minus | - | keep |
+| Period | . | keep |
+| Colon | : | keep |
+| Slash | / | keep |
+| Space | ' ' | keep |
+| Semicolon | ; | : |
+| Equal | = | : |
+| Backslash | \ | / |
+| Anything else | | _ |
+
+Uses can get a list of all the registered functions using the `/api/v1/functions` end point of Netdata.
+
+Users can call functions using the `/api/v1/function` end point of Netdata.
+Once a function is called, the plugin will receive at its standard input a command that looks like this:
+
+> FUNCTION transaction_id timeout "name and parameters of the function"
+
+The plugin is expected to parse and validate `name and parameters of the function`. Netdata allows users to edit this string, append more parameters or even change the ones the plugin originally exposed. To minimize the security risk, Netdata guarantees that only the characters shown above are accepted in function definitions, but still the plugin should carefully inspect the `name and parameters of the function` to ensure that it is valid and not harmful.
+
+If the plugin rejects the request, it should respond with this:
+
+```
+FUNCTION_RESULT_BEGIN transaction_id 400 application/json
+{
+ "status": 400,
+ "error_message": "description of the rejection reasons"
+}
+FUNCTION_RESULT_END
+```
+
+If the plugin prepares a response, it should send (via its standard output, together with the collected data, but not interleaved with them):
+
+> FUNCTION_RESULT_BEGIN transaction_id http_error_code content_type expiration
+
+Where:
+
+ - `transaction_id` is the transaction id that Netdata sent for this function execution
+ - `http_error` is the http error code Netdata should respond with, 200 is the "ok" response
+ - `content_type` is the content type of the response
+ - `expiration` is the absolute timestamp (number, unix epoch) this response expires
+
+Immediately after this, all text is assumed to be the response content.
+The content is text and line oriented. The maximum line length accepted is 15kb. Longer lines will be truncated.
+The type of the context itself depends on the plugin and the UI.
+
+To terminate the message, Netdata seeks a line with just this:
+
+> FUNCTION_RESULT_END
+
+This defines the end of the message. `FUNCTION_RESULT_END` should appear in a line alone, without any other text, so it is wise to add `\n` before and after it.
+
+After this line, Netdata resumes processing collected metrics from the plugin.
+
+## Data collection
+
+data collection is defined as a series of `BEGIN` -> `SET` -> `END` lines
+
+> BEGIN type.id [microseconds]
+
+- `type.id`
+
+ is the unique identification of the chart (as given in `CHART`)
+
+- `microseconds`
+
+ is the number of microseconds since the last update of the chart. It is optional.
+
+ Under heavy system load, the system may have some latency transferring
+ data from the plugins to Netdata via the pipe. This number improves
+ accuracy significantly, since the plugin is able to calculate the
+ duration between its iterations better than Netdata.
+
+ The first time the plugin is started, no microseconds should be given
+ to Netdata.
+
+> SET id = value
+
+- `id`
+
+ is the unique identification of the dimension (of the chart just began)
+
+- `value`
+
+ is the collected value, only integer values are collected. If you want to push fractional values, multiply this value by 100 or 1000 and set the `DIMENSION` divider to 1000.
+
+> END
+
+ END does not take any parameters, it commits the collected values for all dimensions to the chart. If a dimensions was not `SET`, its value will be empty for this commit.
+
+More `SET` lines may appear to update all the dimensions of the chart.
+All of them in one `BEGIN` -> `END` block.
+
+All `SET` lines within a single `BEGIN` -> `END` block have to refer to the
+same chart.
+
+If more charts need to be updated, each chart should have its own
+`BEGIN` -> `SET` -> `END` block.
+
+If, for any reason, a plugin has issued a `BEGIN` but wants to cancel it,
+it can issue a `FLUSH`. The `FLUSH` command will instruct Netdata to ignore
+all the values collected since the last `BEGIN` command.
+
+If a plugin does not behave properly (outputs invalid lines, or does not
+follow these guidelines), will be disabled by Netdata.
+
+### collected values
+
+Netdata will collect any **signed** value in the 64bit range:
+`-9.223.372.036.854.775.808` to `+9.223.372.036.854.775.807`
+
+If a value is not collected, leave it empty, like this:
+
+`SET id =`
+
+or do not output the line at all.
+
+## Modular Plugins
+
+1. **python**, use `python.d.plugin`, there are many examples in the [python.d
+ directory](/collectors/python.d.plugin/README.md)
+
+ python is ideal for Netdata plugins. It is a simple, yet powerful way to collect data, it has a very small memory footprint, although it is not the most CPU efficient way to do it.
+
+2. **BASH**, use `charts.d.plugin`, there are many examples in the [charts.d
+ directory](/collectors/charts.d.plugin/README.md)
+
+ BASH is the simplest scripting language for collecting values. It is the less efficient though in terms of CPU resources. You can use it to collect data quickly, but extensive use of it might use a lot of system resources.
+
+3. **C**
+
+ Of course, C is the most efficient way of collecting data. This is why Netdata itself is written in C.
+
+## Writing Plugins Properly
+
+There are a few rules for writing plugins properly:
+
+1. Respect system resources
+
+ Pay special attention to efficiency:
+
+ - Initialize everything once, at the beginning. Initialization is not an expensive operation. Your plugin will most probably be started once and run forever. So, do whatever heavy operation is needed at the beginning, just once.
+ - Do the absolutely minimum while iterating to collect values repeatedly.
+ - If you need to connect to another server to collect values, avoid re-connects if possible. Connect just once, with keep-alive (for HTTP) enabled and collect values using the same connection.
+ - Avoid any CPU or memory heavy operation while collecting data. If you control memory allocation, avoid any memory allocation while iterating to collect values.
+ - Avoid running external commands when possible. If you are writing shell scripts avoid especially pipes (each pipe is another fork, a very expensive operation).
+
+2. The best way to iterate at a constant pace is this pseudo code:
+
+```js
+ var update_every = argv[1] * 1000; /* seconds * 1000 = milliseconds */
+
+ readConfiguration();
+
+ if(!verifyWeCanCollectValues()) {
+ print("DISABLE");
+ exit(1);
+ }
+
+ createCharts(); /* print CHART and DIMENSION statements */
+
+ var loops = 0;
+ var last_run = 0;
+ var next_run = 0;
+ var dt_since_last_run = 0;
+ var now = 0;
+
+ while(true) {
+ /* find the current time in milliseconds */
+ now = currentTimeStampInMilliseconds();
+
+ /*
+ * find the time of the next loop
+ * this makes sure we are always aligned
+ * with the Netdata daemon
+ */
+ next_run = now - (now % update_every) + update_every;
+
+ /*
+ * wait until it is time
+ * it is important to do it in a loop
+ * since many wait functions can be interrupted
+ */
+ while( now < next_run ) {
+ sleepMilliseconds(next_run - now);
+ now = currentTimeStampInMilliseconds();
+ }
+
+ /* calculate the time passed since the last run */
+ if ( loops > 0 )
+ dt_since_last_run = (now - last_run) * 1000; /* in microseconds */
+
+ /* prepare for the next loop */
+ last_run = now;
+ loops++;
+
+ /* do your magic here to collect values */
+ collectValues();
+
+ /* send the collected data to Netdata */
+ printValues(dt_since_last_run); /* print BEGIN, SET, END statements */
+ }
+```
+
+ Using the above procedure, your plugin will be synchronized to start data collection on steps of `update_every`. There will be no need to keep track of latencies in data collection.
+
+ Netdata interpolates values to second boundaries, so even if your plugin is not perfectly aligned it does not matter. Netdata will find out. When your plugin works in increments of `update_every`, there will be no gaps in the charts due to the possible cumulative micro-delays in data collection. Gaps will only appear if the data collection is really delayed.
+
+3. If you are not sure of memory leaks, exit every one hour. Netdata will re-start your process.
+
+4. If possible, try to autodetect if your plugin should be enabled, without any configuration.
+
+
diff --git a/collectors/plugins.d/plugins_d.c b/collectors/plugins.d/plugins_d.c
new file mode 100644
index 0000000..79abc70
--- /dev/null
+++ b/collectors/plugins.d/plugins_d.c
@@ -0,0 +1,290 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugins_d.h"
+#include "pluginsd_parser.h"
+
+char *plugin_directories[PLUGINSD_MAX_DIRECTORIES] = { NULL };
+struct plugind *pluginsd_root = NULL;
+
+inline size_t pluginsd_initialize_plugin_directories()
+{
+ char plugins_dirs[(FILENAME_MAX * 2) + 1];
+ static char *plugins_dir_list = NULL;
+
+ // Get the configuration entry
+ if (likely(!plugins_dir_list)) {
+ snprintfz(plugins_dirs, FILENAME_MAX * 2, "\"%s\" \"%s/custom-plugins.d\"", PLUGINS_DIR, CONFIG_DIR);
+ plugins_dir_list = strdupz(config_get(CONFIG_SECTION_DIRECTORIES, "plugins", plugins_dirs));
+ }
+
+ // Parse it and store it to plugin directories
+ return quoted_strings_splitter(plugins_dir_list, plugin_directories, PLUGINSD_MAX_DIRECTORIES, config_isspace, NULL, NULL, 0);
+}
+
+static void pluginsd_worker_thread_cleanup(void *arg)
+{
+ struct plugind *cd = (struct plugind *)arg;
+
+ if (cd->enabled && !cd->obsolete) {
+ cd->obsolete = 1;
+
+ info("data collection thread exiting");
+
+ if (cd->pid) {
+ siginfo_t info;
+ info("killing child process pid %d", cd->pid);
+ if (killpid(cd->pid) != -1) {
+ info("waiting for child process pid %d to exit...", cd->pid);
+ waitid(P_PID, (id_t)cd->pid, &info, WEXITED);
+ }
+ cd->pid = 0;
+ }
+ }
+}
+
+#define SERIAL_FAILURES_THRESHOLD 10
+static void pluginsd_worker_thread_handle_success(struct plugind *cd)
+{
+ if (likely(cd->successful_collections)) {
+ sleep((unsigned int)cd->update_every);
+ return;
+ }
+
+ if (likely(cd->serial_failures <= SERIAL_FAILURES_THRESHOLD)) {
+ info(
+ "'%s' (pid %d) does not generate useful output but it reports success (exits with 0). %s.",
+ cd->fullfilename, cd->pid,
+ cd->enabled ? "Waiting a bit before starting it again." : "Will not start it again - it is now disabled.");
+ sleep((unsigned int)(cd->update_every * 10));
+ return;
+ }
+
+ if (cd->serial_failures > SERIAL_FAILURES_THRESHOLD) {
+ error(
+ "'%s' (pid %d) does not generate useful output, although it reports success (exits with 0)."
+ "We have tried to collect something %zu times - unsuccessfully. Disabling it.",
+ cd->fullfilename, cd->pid, cd->serial_failures);
+ cd->enabled = 0;
+ return;
+ }
+
+ return;
+}
+
+static void pluginsd_worker_thread_handle_error(struct plugind *cd, int worker_ret_code)
+{
+ if (worker_ret_code == -1) {
+ info("'%s' (pid %d) was killed with SIGTERM. Disabling it.", cd->fullfilename, cd->pid);
+ cd->enabled = 0;
+ return;
+ }
+
+ if (!cd->successful_collections) {
+ error(
+ "'%s' (pid %d) exited with error code %d and haven't collected any data. Disabling it.", cd->fullfilename,
+ cd->pid, worker_ret_code);
+ cd->enabled = 0;
+ return;
+ }
+
+ if (cd->serial_failures <= SERIAL_FAILURES_THRESHOLD) {
+ error(
+ "'%s' (pid %d) exited with error code %d, but has given useful output in the past (%zu times). %s",
+ cd->fullfilename, cd->pid, worker_ret_code, cd->successful_collections,
+ cd->enabled ? "Waiting a bit before starting it again." : "Will not start it again - it is disabled.");
+ sleep((unsigned int)(cd->update_every * 10));
+ return;
+ }
+
+ if (cd->serial_failures > SERIAL_FAILURES_THRESHOLD) {
+ error(
+ "'%s' (pid %d) exited with error code %d, but has given useful output in the past (%zu times)."
+ "We tried to restart it %zu times, but it failed to generate data. Disabling it.",
+ cd->fullfilename, cd->pid, worker_ret_code, cd->successful_collections, cd->serial_failures);
+ cd->enabled = 0;
+ return;
+ }
+
+ return;
+}
+#undef SERIAL_FAILURES_THRESHOLD
+
+void *pluginsd_worker_thread(void *arg)
+{
+ worker_register("PLUGINSD");
+
+ netdata_thread_cleanup_push(pluginsd_worker_thread_cleanup, arg);
+
+ struct plugind *cd = (struct plugind *)arg;
+
+ cd->obsolete = 0;
+ size_t count = 0;
+
+ while (!netdata_exit) {
+ FILE *fp_child_input = NULL;
+ FILE *fp_child_output = netdata_popen(cd->cmd, &cd->pid, &fp_child_input);
+ if (unlikely(!fp_child_input || !fp_child_output)) {
+ error("Cannot popen(\"%s\", \"r\").", cd->cmd);
+ break;
+ }
+
+ info("connected to '%s' running on pid %d", cd->fullfilename, cd->pid);
+ count = pluginsd_process(localhost, cd, fp_child_input, fp_child_output, 0);
+ error("'%s' (pid %d) disconnected after %zu successful data collections (ENDs).", cd->fullfilename, cd->pid, count);
+ killpid(cd->pid);
+
+ int worker_ret_code = netdata_pclose(fp_child_input, fp_child_output, cd->pid);
+
+ if (likely(worker_ret_code == 0))
+ pluginsd_worker_thread_handle_success(cd);
+ else
+ pluginsd_worker_thread_handle_error(cd, worker_ret_code);
+
+ cd->pid = 0;
+ if (unlikely(!cd->enabled))
+ break;
+ }
+ worker_unregister();
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+static void pluginsd_main_cleanup(void *data)
+{
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)data;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+ info("cleaning up...");
+
+ struct plugind *cd;
+ for (cd = pluginsd_root; cd; cd = cd->next) {
+ if (cd->enabled && !cd->obsolete) {
+ info("stopping plugin thread: %s", cd->id);
+ netdata_thread_cancel(cd->thread);
+ }
+ }
+
+ info("cleanup completed.");
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+
+ worker_unregister();
+}
+
+void *pluginsd_main(void *ptr)
+{
+ netdata_thread_cleanup_push(pluginsd_main_cleanup, ptr);
+
+ int automatic_run = config_get_boolean(CONFIG_SECTION_PLUGINS, "enable running new plugins", 1);
+ int scan_frequency = (int)config_get_number(CONFIG_SECTION_PLUGINS, "check for new plugins every", 60);
+ if (scan_frequency < 1)
+ scan_frequency = 1;
+
+ // disable some plugins by default
+ config_get_boolean(CONFIG_SECTION_PLUGINS, "slabinfo", CONFIG_BOOLEAN_NO);
+
+ // store the errno for each plugins directory
+ // so that we don't log broken directories on each loop
+ int directory_errors[PLUGINSD_MAX_DIRECTORIES] = { 0 };
+
+ while (!netdata_exit) {
+ int idx;
+ const char *directory_name;
+
+ for (idx = 0; idx < PLUGINSD_MAX_DIRECTORIES && (directory_name = plugin_directories[idx]); idx++) {
+ if (unlikely(netdata_exit))
+ break;
+
+ errno = 0;
+ DIR *dir = opendir(directory_name);
+ if (unlikely(!dir)) {
+ if (directory_errors[idx] != errno) {
+ directory_errors[idx] = errno;
+ error("cannot open plugins directory '%s'", directory_name);
+ }
+ continue;
+ }
+
+ struct dirent *file = NULL;
+ while (likely((file = readdir(dir)))) {
+ if (unlikely(netdata_exit))
+ break;
+
+ debug(D_PLUGINSD, "examining file '%s'", file->d_name);
+
+ if (unlikely(strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0))
+ continue;
+
+ int len = (int)strlen(file->d_name);
+ if (unlikely(len <= (int)PLUGINSD_FILE_SUFFIX_LEN))
+ continue;
+ if (unlikely(strcmp(PLUGINSD_FILE_SUFFIX, &file->d_name[len - (int)PLUGINSD_FILE_SUFFIX_LEN]) != 0)) {
+ debug(D_PLUGINSD, "file '%s' does not end in '%s'", file->d_name, PLUGINSD_FILE_SUFFIX);
+ continue;
+ }
+
+ char pluginname[CONFIG_MAX_NAME + 1];
+ snprintfz(pluginname, CONFIG_MAX_NAME, "%.*s", (int)(len - PLUGINSD_FILE_SUFFIX_LEN), file->d_name);
+ int enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, pluginname, automatic_run);
+
+ if (unlikely(!enabled)) {
+ debug(D_PLUGINSD, "plugin '%s' is not enabled", file->d_name);
+ continue;
+ }
+
+ // check if it runs already
+ struct plugind *cd;
+ for (cd = pluginsd_root; cd; cd = cd->next)
+ if (unlikely(strcmp(cd->filename, file->d_name) == 0))
+ break;
+
+ if (likely(cd && !cd->obsolete)) {
+ debug(D_PLUGINSD, "plugin '%s' is already running", cd->filename);
+ continue;
+ }
+
+ // it is not running
+ // allocate a new one, or use the obsolete one
+ if (unlikely(!cd)) {
+ cd = callocz(sizeof(struct plugind), 1);
+
+ snprintfz(cd->id, CONFIG_MAX_NAME, "plugin:%s", pluginname);
+
+ strncpyz(cd->filename, file->d_name, FILENAME_MAX);
+ snprintfz(cd->fullfilename, FILENAME_MAX, "%s/%s", directory_name, cd->filename);
+
+ cd->enabled = enabled;
+ cd->update_every = (int)config_get_number(cd->id, "update every", localhost->rrd_update_every);
+ cd->started_t = now_realtime_sec();
+
+ char *def = "";
+ snprintfz(
+ cd->cmd, PLUGINSD_CMD_MAX, "exec %s %d %s", cd->fullfilename, cd->update_every,
+ config_get(cd->id, "command options", def));
+
+ // link it
+ if (likely(pluginsd_root))
+ cd->next = pluginsd_root;
+ pluginsd_root = cd;
+
+ // it is not currently running
+ cd->obsolete = 1;
+
+ if (cd->enabled) {
+ char tag[NETDATA_THREAD_TAG_MAX + 1];
+ snprintfz(tag, NETDATA_THREAD_TAG_MAX, "PLUGINSD[%s]", pluginname);
+ // spawn a new thread for it
+ netdata_thread_create(
+ &cd->thread, tag, NETDATA_THREAD_OPTION_DEFAULT, pluginsd_worker_thread, cd);
+ }
+ }
+ }
+
+ closedir(dir);
+ }
+
+ sleep((unsigned int)scan_frequency);
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h
new file mode 100644
index 0000000..a8acf03
--- /dev/null
+++ b/collectors/plugins.d/plugins_d.h
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PLUGINS_D_H
+#define NETDATA_PLUGINS_D_H 1
+
+#include "daemon/common.h"
+
+#define PLUGINSD_FILE_SUFFIX ".plugin"
+#define PLUGINSD_FILE_SUFFIX_LEN strlen(PLUGINSD_FILE_SUFFIX)
+#define PLUGINSD_CMD_MAX (FILENAME_MAX*2)
+#define PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH 0
+
+#define PLUGINSD_KEYWORD_CHART "CHART"
+#define PLUGINSD_KEYWORD_CHART_DEFINITION_END "CHART_DEFINITION_END"
+#define PLUGINSD_KEYWORD_DIMENSION "DIMENSION"
+#define PLUGINSD_KEYWORD_BEGIN "BEGIN"
+#define PLUGINSD_KEYWORD_SET "SET"
+#define PLUGINSD_KEYWORD_END "END"
+#define PLUGINSD_KEYWORD_FLUSH "FLUSH"
+#define PLUGINSD_KEYWORD_DISABLE "DISABLE"
+#define PLUGINSD_KEYWORD_VARIABLE "VARIABLE"
+#define PLUGINSD_KEYWORD_LABEL "LABEL"
+#define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
+#define PLUGINSD_KEYWORD_CLABEL "CLABEL"
+#define PLUGINSD_KEYWORD_CLABEL_COMMIT "CLABEL_COMMIT"
+#define PLUGINSD_KEYWORD_FUNCTION "FUNCTION"
+#define PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN "FUNCTION_RESULT_BEGIN"
+#define PLUGINSD_KEYWORD_FUNCTION_RESULT_END "FUNCTION_RESULT_END"
+
+#define PLUGINSD_KEYWORD_REPLAY_CHART "REPLAY_CHART"
+#define PLUGINSD_KEYWORD_REPLAY_BEGIN "RBEGIN"
+#define PLUGINSD_KEYWORD_REPLAY_SET "RSET"
+#define PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE "RDSTATE"
+#define PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE "RSSTATE"
+#define PLUGINSD_KEYWORD_REPLAY_END "REND"
+
+#define PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT 10 // seconds
+
+#define PLUGINSD_LINE_MAX_SSL_READ 512
+
+#define PLUGINSD_MAX_WORDS 20
+
+#define PLUGINSD_MAX_DIRECTORIES 20
+extern char *plugin_directories[PLUGINSD_MAX_DIRECTORIES];
+
+struct plugind {
+ char id[CONFIG_MAX_NAME+1]; // config node id
+
+ char filename[FILENAME_MAX+1]; // just the filename
+ char fullfilename[FILENAME_MAX+1]; // with path
+ char cmd[PLUGINSD_CMD_MAX+1]; // the command that it executes
+
+ volatile pid_t pid;
+ netdata_thread_t thread;
+
+ size_t successful_collections; // the number of times we have seen
+ // values collected from this plugin
+
+ size_t serial_failures; // the number of times the plugin started
+ // without collecting values
+
+ int update_every; // the plugin default data collection frequency
+ volatile sig_atomic_t obsolete; // do not touch this structure after setting this to 1
+ volatile sig_atomic_t enabled; // if this is enabled or not
+
+ time_t started_t;
+ uint32_t capabilities; // follows the same principles as streaming capabilities
+ struct plugind *next;
+};
+
+extern struct plugind *pluginsd_root;
+
+size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugin_input, FILE *fp_plugin_output, int trust_durations);
+
+size_t pluginsd_initialize_plugin_directories();
+
+
+
+#define pluginsd_function_result_begin_to_buffer(wb, transaction, code, content_type, expires) \
+ buffer_sprintf(wb \
+ , PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \
+ , (transaction) ? (transaction) : "" \
+ , (int)(code) \
+ , (content_type) ? (content_type) : "" \
+ , (long int)(expires) \
+ )
+
+#define pluginsd_function_result_end_to_buffer(wb) \
+ buffer_strcat(wb, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")
+
+#define pluginsd_function_result_begin_to_stdout(transaction, code, content_type, expires) \
+ fprintf(stdout \
+ , PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \
+ , (transaction) ? (transaction) : "" \
+ , (int)(code) \
+ , (content_type) ? (content_type) : "" \
+ , (long int)(expires) \
+ )
+
+#define pluginsd_function_result_end_to_stdout() \
+ fprintf(stdout, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")
+
+#endif /* NETDATA_PLUGINS_D_H */
diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c
new file mode 100644
index 0000000..5501c12
--- /dev/null
+++ b/collectors/plugins.d/pluginsd_parser.c
@@ -0,0 +1,1360 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "pluginsd_parser.h"
+
+#define LOG_FUNCTIONS false
+
+static int send_to_plugin(const char *txt, void *data) {
+ PARSER *parser = data;
+
+ if(!txt || !*txt)
+ return 0;
+
+#ifdef ENABLE_HTTPS
+ struct netdata_ssl *ssl = parser->ssl_output;
+ if(ssl) {
+ if(ssl->conn && ssl->flags == NETDATA_SSL_HANDSHAKE_COMPLETE)
+ return (int)netdata_ssl_write(ssl->conn, (void *)txt, strlen(txt));
+
+ error("PLUGINSD: cannot send command (SSL)");
+ return -1;
+ }
+#endif
+
+ if(parser->fp_output) {
+ int bytes = fprintf(parser->fp_output, "%s", txt);
+ if(bytes <= 0) {
+ error("PLUGINSD: cannot send command (FILE)");
+ return -2;
+ }
+ fflush(parser->fp_output);
+ return bytes;
+ }
+
+ if(parser->fd != -1) {
+ size_t bytes = 0;
+ size_t total = strlen(txt);
+ ssize_t sent;
+
+ do {
+ sent = write(parser->fd, &txt[bytes], total - bytes);
+ if(sent <= 0) {
+ error("PLUGINSD: cannot send command (fd)");
+ return -3;
+ }
+ bytes += sent;
+ }
+ while(bytes < total);
+
+ return (int)bytes;
+ }
+
+ error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)");
+ return -4;
+}
+
+static inline RRDHOST *pluginsd_require_host_from_parent(void *user, const char *cmd) {
+ RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
+
+ if(unlikely(!host))
+ error("PLUGINSD: command %s requires a host, but is not set.", cmd);
+
+ return host;
+}
+
+static inline RRDSET *pluginsd_require_chart_from_parent(void *user, const char *cmd, const char *parent_cmd) {
+ RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
+
+ if(unlikely(!st))
+ error("PLUGINSD: command %s requires a chart defined via command %s, but is not set.", cmd, parent_cmd);
+
+ return st;
+}
+
+static inline RRDDIM_ACQUIRED *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *st, const char *dimension, const char *cmd) {
+ if (unlikely(!dimension || !*dimension)) {
+ error("PLUGINSD: 'host:%s/chart:%s' got a %s, without a dimension.",
+ rrdhost_hostname(host), rrdset_id(st), cmd);
+ return NULL;
+ }
+
+ RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(st, dimension);
+
+ if (unlikely(!rda))
+ error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s but dimension does not exist.",
+ rrdhost_hostname(host), rrdset_id(st), dimension, cmd);
+
+ return rda;
+}
+
+static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, const char *cmd) {
+ if (unlikely(!chart || !*chart)) {
+ error("PLUGINSD: 'host:%s' got a %s without a chart id.",
+ rrdhost_hostname(host), cmd);
+ return NULL;
+ }
+
+ RRDSET *st = rrdset_find(host, chart);
+ if (unlikely(!st))
+ error("PLUGINSD: 'host:%s/chart:%s' got a %s but chart does not exist.",
+ rrdhost_hostname(host), chart, cmd);
+
+ return st;
+}
+
+static inline PARSER_RC PLUGINSD_DISABLE_PLUGIN(void *user) {
+ ((PARSER_USER_OBJECT *) user)->enabled = 0;
+ return PARSER_RC_ERROR;
+}
+
+PARSER_RC pluginsd_set(char **words, size_t num_words, void *user)
+{
+ char *dimension = get_word(words, num_words, 1);
+ char *value = get_word(words, num_words, 2);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_SET);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_SET, PLUGINSD_KEYWORD_CHART);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDDIM_ACQUIRED *rda = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_SET);
+ if(!rda) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDDIM *rd = rrddim_acquired_to_rrddim(rda);
+
+ if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
+ debug(D_PLUGINSD, "PLUGINSD: 'host:%s/chart:%s/dim:%s' SET is setting value to '%s'",
+ rrdhost_hostname(host), rrdset_id(st), dimension, value && *value ? value : "UNSET");
+
+ if (value && *value)
+ rrddim_set_by_pointer(st, rd, strtoll(value, NULL, 0));
+
+ rrddim_acquired_release(rda);
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_begin(char **words, size_t num_words, void *user)
+{
+ char *id = get_word(words, num_words, 1);
+ char *microseconds_txt = get_word(words, num_words, 2);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_BEGIN);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_find_chart(host, id, PLUGINSD_KEYWORD_BEGIN);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ ((PARSER_USER_OBJECT *)user)->st = st;
+
+ usec_t microseconds = 0;
+ if (microseconds_txt && *microseconds_txt)
+ microseconds = str2ull(microseconds_txt);
+
+#ifdef NETDATA_LOG_REPLICATION_REQUESTS
+ if(st->replay.log_next_data_collection) {
+ st->replay.log_next_data_collection = false;
+
+ internal_error(true,
+ "REPLAY: 'host:%s/chart:%s' first BEGIN after replication, last collected %llu, last updated %llu, microseconds %llu",
+ rrdhost_hostname(host), rrdset_id(st),
+ st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec,
+ st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec,
+ microseconds
+ );
+ }
+#endif
+
+ if (likely(st->counter_done)) {
+ if (likely(microseconds)) {
+ if (((PARSER_USER_OBJECT *)user)->trust_durations)
+ rrdset_next_usec_unfiltered(st, microseconds);
+ else
+ rrdset_next_usec(st, microseconds);
+ }
+ else
+ rrdset_next(st);
+ }
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_end(char **words, size_t num_words, void *user)
+{
+ UNUSED(words);
+ UNUSED(num_words);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_END);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_END, PLUGINSD_KEYWORD_BEGIN);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
+ debug(D_PLUGINSD, "requested an END on chart '%s'", rrdset_id(st));
+
+ ((PARSER_USER_OBJECT *) user)->st = NULL;
+ ((PARSER_USER_OBJECT *) user)->count++;
+
+ struct timeval now;
+ now_realtime_timeval(&now);
+ rrdset_timed_done(st, now, /* pending_rrdset_next = */ false);
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user)
+{
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_CHART);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ char *type = get_word(words, num_words, 1);
+ char *name = get_word(words, num_words, 2);
+ char *title = get_word(words, num_words, 3);
+ char *units = get_word(words, num_words, 4);
+ char *family = get_word(words, num_words, 5);
+ char *context = get_word(words, num_words, 6);
+ char *chart = get_word(words, num_words, 7);
+ char *priority_s = get_word(words, num_words, 8);
+ char *update_every_s = get_word(words, num_words, 9);
+ char *options = get_word(words, num_words, 10);
+ char *plugin = get_word(words, num_words, 11);
+ char *module = get_word(words, num_words, 12);
+
+ // parse the id from type
+ char *id = NULL;
+ if (likely(type && (id = strchr(type, '.')))) {
+ *id = '\0';
+ id++;
+ }
+
+ // make sure we have the required variables
+ if (unlikely((!type || !*type || !id || !*id))) {
+ error("PLUGINSD: 'host:%s' requested a CHART, without a type.id. Disabling it.",
+ rrdhost_hostname(host));
+
+ ((PARSER_USER_OBJECT *) user)->enabled = 0;
+ return PARSER_RC_ERROR;
+ }
+
+ // parse the name, and make sure it does not include 'type.'
+ if (unlikely(name && *name)) {
+ // when data are streamed from child nodes
+ // name will be type.name
+ // so we have to remove 'type.' from name too
+ size_t len = strlen(type);
+ if (strncmp(type, name, len) == 0 && name[len] == '.')
+ name = &name[len + 1];
+
+ // if the name is the same with the id,
+ // or is just 'NULL', clear it.
+ if (unlikely(strcmp(name, id) == 0 || strcasecmp(name, "NULL") == 0 || strcasecmp(name, "(NULL)") == 0))
+ name = NULL;
+ }
+
+ int priority = 1000;
+ if (likely(priority_s && *priority_s))
+ priority = str2i(priority_s);
+
+ int update_every = ((PARSER_USER_OBJECT *) user)->cd->update_every;
+ if (likely(update_every_s && *update_every_s))
+ update_every = str2i(update_every_s);
+ if (unlikely(!update_every))
+ update_every = ((PARSER_USER_OBJECT *) user)->cd->update_every;
+
+ RRDSET_TYPE chart_type = RRDSET_TYPE_LINE;
+ if (unlikely(chart))
+ chart_type = rrdset_type_id(chart);
+
+ if (unlikely(name && !*name))
+ name = NULL;
+ if (unlikely(family && !*family))
+ family = NULL;
+ if (unlikely(context && !*context))
+ context = NULL;
+ if (unlikely(!title))
+ title = "";
+ if (unlikely(!units))
+ units = "unknown";
+
+ debug(
+ D_PLUGINSD,
+ "creating chart type='%s', id='%s', name='%s', family='%s', context='%s', chart='%s', priority=%d, update_every=%d",
+ type, id, name ? name : "", family ? family : "", context ? context : "", rrdset_type_name(chart_type),
+ priority, update_every);
+
+ RRDSET *st = NULL;
+
+ st = rrdset_create(
+ host, type, id, name, family, context, title, units,
+ (plugin && *plugin) ? plugin : ((PARSER_USER_OBJECT *)user)->cd->filename,
+ module, priority, update_every,
+ chart_type);
+
+ if (likely(st)) {
+ if (options && *options) {
+ if (strstr(options, "obsolete"))
+ rrdset_is_obsolete(st);
+ else
+ rrdset_isnot_obsolete(st);
+
+ if (strstr(options, "detail"))
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+ else
+ rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
+
+ if (strstr(options, "hidden"))
+ rrdset_flag_set(st, RRDSET_FLAG_HIDDEN);
+ else
+ rrdset_flag_clear(st, RRDSET_FLAG_HIDDEN);
+
+ if (strstr(options, "store_first"))
+ rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
+ else
+ rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
+ } else {
+ rrdset_isnot_obsolete(st);
+ rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
+ rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
+ }
+ }
+ ((PARSER_USER_OBJECT *)user)->st = st;
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, void *user)
+{
+ const char *first_entry_txt = get_word(words, num_words, 1);
+ const char *last_entry_txt = get_word(words, num_words, 2);
+ const char *world_time_txt = get_word(words, num_words, 3);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_CHART_DEFINITION_END);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_CHART_DEFINITION_END, PLUGINSD_KEYWORD_CHART);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ time_t first_entry_child = (first_entry_txt && *first_entry_txt) ? (time_t)str2ul(first_entry_txt) : 0;
+ time_t last_entry_child = (last_entry_txt && *last_entry_txt) ? (time_t)str2ul(last_entry_txt) : 0;
+ time_t child_world_time = (world_time_txt && *world_time_txt) ? (time_t)str2ul(world_time_txt) : now_realtime_sec();
+
+ if((first_entry_child != 0 || last_entry_child != 0) && (first_entry_child == 0 || last_entry_child == 0))
+ error("PLUGINSD REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_CHART_DEFINITION_END " with malformed timings (first time %ld, last time %ld, world time %ld).",
+ rrdhost_hostname(host), rrdset_id(st),
+ first_entry_child, last_entry_child, child_world_time);
+
+ bool ok = true;
+ if(!rrdset_flag_check(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS)) {
+
+#ifdef NETDATA_LOG_REPLICATION_REQUESTS
+ st->replay.start_streaming = false;
+ st->replay.after = 0;
+ st->replay.before = 0;
+#endif
+
+ rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS);
+ rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
+ rrdhost_receiver_replicating_charts_plus_one(st->rrdhost);
+
+ PARSER *parser = ((PARSER_USER_OBJECT *)user)->parser;
+ ok = replicate_chart_request(send_to_plugin, parser, host, st,
+ first_entry_child, last_entry_child, child_world_time,
+ 0, 0);
+ }
+#ifdef NETDATA_LOG_REPLICATION_REQUESTS
+ else {
+ internal_error(true, "REPLAY: 'host:%s/chart:%s' not sending duplicate replication request",
+ rrdhost_hostname(st->rrdhost), rrdset_id(st));
+ }
+#endif
+
+ return ok ? PARSER_RC_OK : PARSER_RC_ERROR;
+}
+
+PARSER_RC pluginsd_dimension(char **words, size_t num_words, void *user)
+{
+ char *id = get_word(words, num_words, 1);
+ char *name = get_word(words, num_words, 2);
+ char *algorithm = get_word(words, num_words, 3);
+ char *multiplier_s = get_word(words, num_words, 4);
+ char *divisor_s = get_word(words, num_words, 5);
+ char *options = get_word(words, num_words, 6);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_DIMENSION);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_DIMENSION, PLUGINSD_KEYWORD_CHART);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ if (unlikely(!id)) {
+ error("PLUGINSD: 'host:%s/chart:%s' got a DIMENSION, without an id. Disabling it.",
+ rrdhost_hostname(host), st ? rrdset_id(st) : "UNSET");
+ return PLUGINSD_DISABLE_PLUGIN(user);
+ }
+
+ if (unlikely(!st && !((PARSER_USER_OBJECT *) user)->st_exists)) {
+ error("PLUGINSD: 'host:%s' got a DIMENSION, without a CHART. Disabling it.",
+ rrdhost_hostname(host));
+ return PLUGINSD_DISABLE_PLUGIN(user);
+ }
+
+ long multiplier = 1;
+ if (multiplier_s && *multiplier_s) {
+ multiplier = strtol(multiplier_s, NULL, 0);
+ if (unlikely(!multiplier))
+ multiplier = 1;
+ }
+
+ long divisor = 1;
+ if (likely(divisor_s && *divisor_s)) {
+ divisor = strtol(divisor_s, NULL, 0);
+ if (unlikely(!divisor))
+ divisor = 1;
+ }
+
+ if (unlikely(!algorithm || !*algorithm))
+ algorithm = "absolute";
+
+ if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
+ debug(
+ D_PLUGINSD,
+ "creating dimension in chart %s, id='%s', name='%s', algorithm='%s', multiplier=%ld, divisor=%ld, hidden='%s'",
+ rrdset_id(st), id, name ? name : "", rrd_algorithm_name(rrd_algorithm_id(algorithm)), multiplier, divisor,
+ options ? options : "");
+
+ RRDDIM *rd = rrddim_add(st, id, name, multiplier, divisor, rrd_algorithm_id(algorithm));
+ int unhide_dimension = 1;
+
+ rrddim_option_clear(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS);
+ if (options && *options) {
+ if (strstr(options, "obsolete") != NULL)
+ rrddim_is_obsolete(st, rd);
+ else
+ rrddim_isnot_obsolete(st, rd);
+
+ unhide_dimension = !strstr(options, "hidden");
+
+ if (strstr(options, "noreset") != NULL)
+ rrddim_option_set(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS);
+ if (strstr(options, "nooverflow") != NULL)
+ rrddim_option_set(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS);
+ } else
+ rrddim_isnot_obsolete(st, rd);
+
+ if (likely(unhide_dimension)) {
+ rrddim_option_clear(rd, RRDDIM_OPTION_HIDDEN);
+ if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
+ rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
+ metaqueue_dimension_update_flags(rd);
+ }
+ }
+ else {
+ rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN);
+ if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
+ rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
+ metaqueue_dimension_update_flags(rd);
+ }
+ }
+
+ return PARSER_RC_OK;
+}
+
+// ----------------------------------------------------------------------------
+// execution of functions
+
+struct inflight_function {
+ int code;
+ int timeout;
+ BUFFER *destination_wb;
+ STRING *function;
+ void (*callback)(BUFFER *wb, int code, void *callback_data);
+ void *callback_data;
+ usec_t timeout_ut;
+ usec_t started_ut;
+ usec_t sent_ut;
+};
+
+static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void *func, void *parser_ptr) {
+ struct inflight_function *pf = func;
+
+ PARSER *parser = parser_ptr;
+
+ // leave this code as default, so that when the dictionary is destroyed this will be sent back to the caller
+ pf->code = HTTP_RESP_GATEWAY_TIMEOUT;
+
+ char buffer[2048 + 1];
+ snprintfz(buffer, 2048, "FUNCTION %s %d \"%s\"\n",
+ dictionary_acquired_item_name(item),
+ pf->timeout,
+ string2str(pf->function));
+
+ // send the command to the plugin
+ int ret = send_to_plugin(buffer, parser);
+
+ pf->sent_ut = now_realtime_usec();
+
+ if(ret < 0) {
+ error("FUNCTION: failed to send function to plugin, error %d", ret);
+ rrd_call_function_error(pf->destination_wb, "Failed to communicate with collector", HTTP_RESP_BACKEND_FETCH_FAILED);
+ }
+ else {
+ internal_error(LOG_FUNCTIONS,
+ "FUNCTION '%s' with transaction '%s' sent to collector (%d bytes, in %llu usec)",
+ string2str(pf->function), dictionary_acquired_item_name(item), ret,
+ pf->sent_ut - pf->started_ut);
+ }
+}
+
+static bool inflight_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused, void *new_func, void *parser_ptr __maybe_unused) {
+ struct inflight_function *pf = new_func;
+
+ error("PLUGINSD_PARSER: duplicate UUID on pending function '%s' detected. Ignoring the second one.", string2str(pf->function));
+ pf->code = rrd_call_function_error(pf->destination_wb, "This request is already in progress", HTTP_RESP_BAD_REQUEST);
+ pf->callback(pf->destination_wb, pf->code, pf->callback_data);
+ string_freez(pf->function);
+
+ return false;
+}
+
+static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *parser_ptr __maybe_unused) {
+ struct inflight_function *pf = func;
+
+ internal_error(LOG_FUNCTIONS,
+ "FUNCTION '%s' result of transaction '%s' received from collector (%zu bytes, request %llu usec, response %llu usec)",
+ string2str(pf->function), dictionary_acquired_item_name(item),
+ buffer_strlen(pf->destination_wb), pf->sent_ut - pf->started_ut, now_realtime_usec() - pf->sent_ut);
+
+ pf->callback(pf->destination_wb, pf->code, pf->callback_data);
+ string_freez(pf->function);
+}
+
+void inflight_functions_init(PARSER *parser) {
+ parser->inflight.functions = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
+ dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser);
+ dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser);
+ dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser);
+}
+
+static void inflight_functions_garbage_collect(PARSER *parser, usec_t now) {
+ parser->inflight.smaller_timeout = 0;
+ struct inflight_function *pf;
+ dfe_start_write(parser->inflight.functions, pf) {
+ if (pf->timeout_ut < now) {
+ internal_error(true,
+ "FUNCTION '%s' removing expired transaction '%s', after %llu usec.",
+ string2str(pf->function), pf_dfe.name, now - pf->started_ut);
+
+ if(!buffer_strlen(pf->destination_wb) || pf->code == HTTP_RESP_OK)
+ pf->code = rrd_call_function_error(pf->destination_wb,
+ "Timeout waiting for collector response.",
+ HTTP_RESP_GATEWAY_TIMEOUT);
+
+ dictionary_del(parser->inflight.functions, pf_dfe.name);
+ }
+
+ else if(!parser->inflight.smaller_timeout || pf->timeout_ut < parser->inflight.smaller_timeout)
+ parser->inflight.smaller_timeout = pf->timeout_ut;
+ }
+ dfe_done(pf);
+}
+
+// this is the function that is called from
+// rrd_call_function_and_wait() and rrd_call_function_async()
+static int pluginsd_execute_function_callback(BUFFER *destination_wb, int timeout, const char *function, void *collector_data, void (*callback)(BUFFER *wb, int code, void *callback_data), void *callback_data) {
+ PARSER *parser = collector_data;
+
+ usec_t now = now_realtime_usec();
+
+ struct inflight_function tmp = {
+ .started_ut = now,
+ .timeout_ut = now + timeout * USEC_PER_SEC,
+ .destination_wb = destination_wb,
+ .timeout = timeout,
+ .function = string_strdupz(function),
+ .callback = callback,
+ .callback_data = callback_data,
+ };
+
+ uuid_t uuid;
+ uuid_generate_time(uuid);
+
+ char key[UUID_STR_LEN];
+ uuid_unparse_lower(uuid, key);
+
+ dictionary_write_lock(parser->inflight.functions);
+
+ // if there is any error, our dictionary callbacks will call the caller callback to notify
+ // the caller about the error - no need for error handling here.
+ dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function));
+
+ if(!parser->inflight.smaller_timeout || tmp.timeout_ut < parser->inflight.smaller_timeout)
+ parser->inflight.smaller_timeout = tmp.timeout_ut;
+
+ // garbage collect stale inflight functions
+ if(parser->inflight.smaller_timeout < now)
+ inflight_functions_garbage_collect(parser, now);
+
+ dictionary_write_unlock(parser->inflight.functions);
+
+ return HTTP_RESP_OK;
+}
+
+PARSER_RC pluginsd_function(char **words, size_t num_words, void *user)
+{
+ bool global = false;
+ size_t i = 1;
+ if(num_words >= 2 && strcmp(get_word(words, num_words, 1), "GLOBAL") == 0) {
+ i++;
+ global = true;
+ }
+
+ char *name = get_word(words, num_words, i++);
+ char *timeout_s = get_word(words, num_words, i++);
+ char *help = get_word(words, num_words, i++);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_FUNCTION);
+ if(!host) return PARSER_RC_ERROR;
+
+ RRDSET *st = (global)?NULL:pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_FUNCTION, PLUGINSD_KEYWORD_CHART);
+ if(!st) global = true;
+
+ if (unlikely(!timeout_s || !name || !help || (!global && !st))) {
+ error("PLUGINSD: 'host:%s/chart:%s' got a FUNCTION, without providing the required data (global = '%s', name = '%s', timeout = '%s', help = '%s'). Ignoring it.",
+ rrdhost_hostname(host),
+ st?rrdset_id(st):"(unset)",
+ global?"yes":"no",
+ name?name:"(unset)",
+ timeout_s?timeout_s:"(unset)",
+ help?help:"(unset)"
+ );
+ return PARSER_RC_ERROR;
+ }
+
+ int timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
+ if (timeout_s && *timeout_s) {
+ timeout = str2i(timeout_s);
+ if (unlikely(timeout <= 0))
+ timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
+ }
+
+ PARSER *parser = ((PARSER_USER_OBJECT *) user)->parser;
+ rrd_collector_add_function(host, st, name, timeout, help, false, pluginsd_execute_function_callback, parser);
+
+ return PARSER_RC_OK;
+}
+
+static void pluginsd_function_result_end(struct parser *parser, void *action_data) {
+ STRING *key = action_data;
+ if(key)
+ dictionary_del(parser->inflight.functions, string2str(key));
+ string_freez(key);
+}
+
+PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, void *user)
+{
+ char *key = get_word(words, num_words, 1);
+ char *status = get_word(words, num_words, 2);
+ char *format = get_word(words, num_words, 3);
+ char *expires = get_word(words, num_words, 4);
+
+ if (unlikely(!key || !*key || !status || !*status || !format || !*format || !expires || !*expires)) {
+ error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " without providing the required data (key = '%s', status = '%s', format = '%s', expires = '%s')."
+ , key ? key : "(unset)"
+ , status ? status : "(unset)"
+ , format ? format : "(unset)"
+ , expires ? expires : "(unset)"
+ );
+ }
+
+ int code = (status && *status) ? str2i(status) : 0;
+ if (code <= 0)
+ code = HTTP_RESP_BACKEND_RESPONSE_INVALID;
+
+ time_t expiration = (expires && *expires) ? str2l(expires) : 0;
+
+ PARSER *parser = ((PARSER_USER_OBJECT *) user)->parser;
+
+ struct inflight_function *pf = NULL;
+
+ if(key && *key)
+ pf = (struct inflight_function *)dictionary_get(parser->inflight.functions, key);
+
+ if(!pf) {
+ error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " for transaction '%s', but the transaction is not found.", key?key:"(unset)");
+ }
+ else {
+ if(format && *format)
+ pf->destination_wb->contenttype = functions_format_to_content_type(format);
+
+ pf->code = code;
+
+ pf->destination_wb->expires = expiration;
+ if(expiration <= now_realtime_sec())
+ buffer_no_cacheable(pf->destination_wb);
+ else
+ buffer_cacheable(pf->destination_wb);
+ }
+
+ parser->defer.response = (pf) ? pf->destination_wb : NULL;
+ parser->defer.end_keyword = PLUGINSD_KEYWORD_FUNCTION_RESULT_END;
+ parser->defer.action = pluginsd_function_result_end;
+ parser->defer.action_data = string_strdupz(key); // it is ok is key is NULL
+ parser->flags |= PARSER_DEFER_UNTIL_KEYWORD;
+
+ return PARSER_RC_OK;
+}
+
+// ----------------------------------------------------------------------------
+
+PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user)
+{
+ char *name = get_word(words, num_words, 1);
+ char *value = get_word(words, num_words, 2);
+ NETDATA_DOUBLE v;
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_VARIABLE);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
+
+ int global = (st) ? 0 : 1;
+
+ if (name && *name) {
+ if ((strcmp(name, "GLOBAL") == 0 || strcmp(name, "HOST") == 0)) {
+ global = 1;
+ name = get_word(words, num_words, 2);
+ value = get_word(words, num_words, 3);
+ } else if ((strcmp(name, "LOCAL") == 0 || strcmp(name, "CHART") == 0)) {
+ global = 0;
+ name = get_word(words, num_words, 2);
+ value = get_word(words, num_words, 3);
+ }
+ }
+
+ if (unlikely(!name || !*name)) {
+ error("PLUGINSD: 'host:%s/chart:%s' got a VARIABLE without a variable name. Disabling it.",
+ rrdhost_hostname(host), st ? rrdset_id(st):"UNSET");
+
+ ((PARSER_USER_OBJECT *)user)->enabled = 0;
+ return PLUGINSD_DISABLE_PLUGIN(user);
+ }
+
+ if (unlikely(!value || !*value))
+ value = NULL;
+
+ if (unlikely(!value)) {
+ error("PLUGINSD: 'host:%s/chart:%s' cannot set %s VARIABLE '%s' to an empty value",
+ rrdhost_hostname(host),
+ st ? rrdset_id(st):"UNSET",
+ (global) ? "HOST" : "CHART",
+ name);
+ return PARSER_RC_OK;
+ }
+
+ if (!global && !st) {
+ error("PLUGINSD: 'host:%s/chart:%s' cannot update CHART VARIABLE '%s' without a chart",
+ rrdhost_hostname(host),
+ st ? rrdset_id(st):"UNSET",
+ name
+ );
+ return PLUGINSD_DISABLE_PLUGIN(user);
+ }
+
+ char *endptr = NULL;
+ v = (NETDATA_DOUBLE)str2ndd(value, &endptr);
+ if (unlikely(endptr && *endptr)) {
+ if (endptr == value)
+ error("PLUGINSD: 'host:%s/chart:%s' the value '%s' of VARIABLE '%s' cannot be parsed as a number",
+ rrdhost_hostname(host),
+ st ? rrdset_id(st):"UNSET",
+ value,
+ name);
+ else
+ error("PLUGINSD: 'host:%s/chart:%s' the value '%s' of VARIABLE '%s' has leftovers: '%s'",
+ rrdhost_hostname(host),
+ st ? rrdset_id(st):"UNSET",
+ value,
+ name,
+ endptr);
+ }
+
+ if (global) {
+ const RRDVAR_ACQUIRED *rva = rrdvar_custom_host_variable_add_and_acquire(host, name);
+ if (rva) {
+ rrdvar_custom_host_variable_set(host, rva, v);
+ rrdvar_custom_host_variable_release(host, rva);
+ }
+ else
+ error("PLUGINSD: 'host:%s' cannot find/create HOST VARIABLE '%s'",
+ rrdhost_hostname(host),
+ name);
+ } else {
+ const RRDSETVAR_ACQUIRED *rsa = rrdsetvar_custom_chart_variable_add_and_acquire(st, name);
+ if (rsa) {
+ rrdsetvar_custom_chart_variable_set(st, rsa, v);
+ rrdsetvar_custom_chart_variable_release(st, rsa);
+ }
+ else
+ error("PLUGINSD: 'host:%s/chart:%s' cannot find/create CHART VARIABLE '%s'",
+ rrdhost_hostname(host), rrdset_id(st), name);
+ }
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_flush(char **words __maybe_unused, size_t num_words __maybe_unused, void *user)
+{
+ debug(D_PLUGINSD, "requested a FLUSH");
+ ((PARSER_USER_OBJECT *) user)->st = NULL;
+ ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_disable(char **words __maybe_unused, size_t num_words __maybe_unused, void *user __maybe_unused)
+{
+ info("PLUGINSD: plugin called DISABLE. Disabling it.");
+ ((PARSER_USER_OBJECT *) user)->enabled = 0;
+ return PARSER_RC_ERROR;
+}
+
+PARSER_RC pluginsd_label(char **words, size_t num_words, void *user)
+{
+ const char *name = get_word(words, num_words, 1);
+ const char *label_source = get_word(words, num_words, 2);
+ const char *value = get_word(words, num_words, 3);
+
+ if (!name || !label_source || !value) {
+ error("PLUGINSD: ignoring malformed or empty LABEL command.");
+ return PLUGINSD_DISABLE_PLUGIN(user);
+ }
+
+ char *store = (char *)value;
+ bool allocated_store = false;
+
+ if(unlikely(num_words > 4)) {
+ allocated_store = true;
+ store = mallocz(PLUGINSD_LINE_MAX + 1);
+ size_t remaining = PLUGINSD_LINE_MAX;
+ char *move = store;
+ char *word;
+ for(size_t i = 3; i < num_words && remaining > 2 && (word = get_word(words, num_words, i)) ;i++) {
+ if(i > 3) {
+ *move++ = ' ';
+ *move = '\0';
+ remaining--;
+ }
+
+ size_t length = strlen(word);
+ if (length > remaining)
+ length = remaining;
+
+ remaining -= length;
+ memcpy(move, word, length);
+ move += length;
+ *move = '\0';
+ }
+ }
+
+ if(unlikely(!((PARSER_USER_OBJECT *) user)->new_host_labels))
+ ((PARSER_USER_OBJECT *) user)->new_host_labels = rrdlabels_create();
+
+ rrdlabels_add(((PARSER_USER_OBJECT *)user)->new_host_labels,
+ name,
+ store,
+ str2l(label_source));
+
+ if (allocated_store)
+ freez(store);
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_overwrite(char **words __maybe_unused, size_t num_words __maybe_unused, void *user)
+{
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_OVERWRITE);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ debug(D_PLUGINSD, "requested to OVERWRITE host labels");
+
+ if(unlikely(!host->rrdlabels))
+ host->rrdlabels = rrdlabels_create();
+
+ rrdlabels_migrate_to_these(host->rrdlabels, (DICTIONARY *) (((PARSER_USER_OBJECT *)user)->new_host_labels));
+ metaqueue_store_host_labels(host->machine_guid);
+
+ rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_host_labels);
+ ((PARSER_USER_OBJECT *)user)->new_host_labels = NULL;
+ return PARSER_RC_OK;
+}
+
+
+PARSER_RC pluginsd_clabel(char **words, size_t num_words, void *user)
+{
+ const char *name = get_word(words, num_words, 1);
+ const char *value = get_word(words, num_words, 2);
+ const char *label_source = get_word(words, num_words, 3);
+
+ if (!name || !value || !*label_source) {
+ error("Ignoring malformed or empty CHART LABEL command.");
+ return PLUGINSD_DISABLE_PLUGIN(user);
+ }
+
+ if(unlikely(!((PARSER_USER_OBJECT *) user)->chart_rrdlabels_linked_temporarily)) {
+ ((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily = ((PARSER_USER_OBJECT *)user)->st->rrdlabels;
+ rrdlabels_unmark_all(((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily);
+ }
+
+ rrdlabels_add(((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily,
+ name, value, str2l(label_source));
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_clabel_commit(char **words __maybe_unused, size_t num_words __maybe_unused, void *user)
+{
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_CLABEL_COMMIT);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_CLABEL_COMMIT, PLUGINSD_KEYWORD_BEGIN);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ debug(D_PLUGINSD, "requested to commit chart labels");
+
+ if(!((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily) {
+ error("PLUGINSD: 'host:%s' got CLABEL_COMMIT, without a CHART or BEGIN. Ignoring it.",
+ rrdhost_hostname(host));
+ return PLUGINSD_DISABLE_PLUGIN(user);
+ }
+
+ rrdlabels_remove_all_unmarked(((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily);
+
+ rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
+ rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
+
+ ((PARSER_USER_OBJECT *)user)->chart_rrdlabels_linked_temporarily = NULL;
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_replay_rrdset_begin(char **words, size_t num_words, void *user)
+{
+ char *id = get_word(words, num_words, 1);
+ char *start_time_str = get_word(words, num_words, 2);
+ char *end_time_str = get_word(words, num_words, 3);
+ char *child_now_str = get_word(words, num_words, 4);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_BEGIN);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st;
+ if (likely(!id || !*id))
+ st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_BEGIN, PLUGINSD_KEYWORD_REPLAY_BEGIN);
+ else
+ st = pluginsd_find_chart(host, id, PLUGINSD_KEYWORD_REPLAY_BEGIN);
+
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+ ((PARSER_USER_OBJECT *) user)->st = st;
+
+ if(start_time_str && end_time_str) {
+ time_t start_time = (time_t)str2ul(start_time_str);
+ time_t end_time = (time_t)str2ul(end_time_str);
+
+ time_t wall_clock_time = 0, tolerance;
+ bool wall_clock_comes_from_child; (void)wall_clock_comes_from_child;
+ if(child_now_str) {
+ wall_clock_time = (time_t)str2ul(child_now_str);
+ tolerance = st->update_every + 1;
+ wall_clock_comes_from_child = true;
+ }
+
+ if(wall_clock_time <= 0) {
+ wall_clock_time = now_realtime_sec();
+ tolerance = st->update_every + 5;
+ wall_clock_comes_from_child = false;
+ }
+
+#ifdef NETDATA_LOG_REPLICATION_REQUESTS
+ internal_error(
+ (!st->replay.start_streaming && (end_time < st->replay.after || start_time > st->replay.before)),
+ "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, which does not match our request (%ld to %ld).",
+ rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time, st->replay.after, st->replay.before);
+
+ internal_error(
+ true,
+ "REPLAY: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, child wall clock is %ld (%s), had requested %ld to %ld",
+ rrdhost_hostname(st->rrdhost), rrdset_id(st),
+ start_time, end_time, wall_clock_time, wall_clock_comes_from_child ? "from child" : "parent time",
+ st->replay.after, st->replay.before);
+#endif
+
+ if(start_time && end_time && start_time < wall_clock_time + tolerance && end_time < wall_clock_time + tolerance && start_time < end_time) {
+ if (unlikely(end_time - start_time != st->update_every))
+ rrdset_set_update_every(st, end_time - start_time);
+
+ st->last_collected_time.tv_sec = end_time;
+ st->last_collected_time.tv_usec = 0;
+
+ st->last_updated.tv_sec = end_time;
+ st->last_updated.tv_usec = 0;
+
+ st->counter++;
+ st->counter_done++;
+
+ // these are only needed for db mode RAM, SAVE, MAP, ALLOC
+ st->current_entry++;
+ if(st->current_entry >= st->entries)
+ st->current_entry -= st->entries;
+
+ ((PARSER_USER_OBJECT *) user)->replay.start_time = start_time;
+ ((PARSER_USER_OBJECT *) user)->replay.end_time = end_time;
+ ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC;
+ ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC;
+ ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = wall_clock_time;
+ ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = true;
+
+ return PARSER_RC_OK;
+ }
+
+ error("PLUGINSD REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, but timestamps are invalid (now is %ld [%s], tolerance %ld). Ignoring " PLUGINSD_KEYWORD_REPLAY_SET,
+ rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time,
+ wall_clock_time, wall_clock_comes_from_child ? "child wall clock" : "parent wall clock", tolerance);
+ }
+
+ // the child sends an RBEGIN without any parameters initially
+ // setting rset_enabled to false, means the RSET should not store any metrics
+ // to store metrics, the RBEGIN needs to have timestamps
+ ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = false;
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
+{
+ char *dimension = get_word(words, num_words, 1);
+ char *value_str = get_word(words, num_words, 2);
+ char *flags_str = get_word(words, num_words, 3);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_SET);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ if(!((PARSER_USER_OBJECT *) user)->replay.rset_enabled) {
+ error_limit_static_thread_var(erl, 1, 0);
+ error_limit(&erl, "PLUGINSD: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_SET " but it is disabled by " PLUGINSD_KEYWORD_REPLAY_BEGIN " errors",
+ rrdhost_hostname(host), rrdset_id(st));
+
+ // we have to return OK here
+ return PARSER_RC_OK;
+ }
+
+ RRDDIM_ACQUIRED *rda = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_REPLAY_SET);
+ if(!rda) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ if (unlikely(!((PARSER_USER_OBJECT *) user)->replay.start_time || !((PARSER_USER_OBJECT *) user)->replay.end_time)) {
+ error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a " PLUGINSD_KEYWORD_REPLAY_SET " with invalid timestamps %ld to %ld from a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
+ rrdhost_hostname(host),
+ rrdset_id(st),
+ dimension,
+ ((PARSER_USER_OBJECT *) user)->replay.start_time,
+ ((PARSER_USER_OBJECT *) user)->replay.end_time);
+ return PLUGINSD_DISABLE_PLUGIN(user);
+ }
+
+ if (unlikely(!value_str || !*value_str))
+ value_str = "NAN";
+
+ if(unlikely(!flags_str))
+ flags_str = "";
+
+ if (likely(value_str)) {
+ RRDDIM *rd = rrddim_acquired_to_rrddim(rda);
+
+ RRDDIM_FLAGS rd_flags = rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED);
+
+ if(!(rd_flags & RRDDIM_FLAG_ARCHIVED)) {
+ NETDATA_DOUBLE value = strtondd(value_str, NULL);
+ SN_FLAGS flags = SN_FLAG_NONE;
+
+ char c;
+ while ((c = *flags_str++)) {
+ switch (c) {
+ case 'R':
+ flags |= SN_FLAG_RESET;
+ break;
+
+ case 'E':
+ flags |= SN_EMPTY_SLOT;
+ value = NAN;
+ break;
+
+ default:
+ error("unknown flag '%c'", c);
+ break;
+ }
+ }
+
+ if (!netdata_double_isnumber(value)) {
+ value = NAN;
+ flags = SN_EMPTY_SLOT;
+ }
+
+ rrddim_store_metric(rd, ((PARSER_USER_OBJECT *) user)->replay.end_time_ut, value, flags);
+ rd->last_collected_time.tv_sec = ((PARSER_USER_OBJECT *) user)->replay.end_time;
+ rd->last_collected_time.tv_usec = 0;
+ rd->collections_counter++;
+ }
+ else {
+ error_limit_static_global_var(erl, 1, 0);
+ error_limit(&erl, "PLUGINSD: 'host:%s/chart:%s/dim:%s' has the ARCHIVED flag set, but it is replicated. Ignoring data.",
+ rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_name(rd));
+ }
+ }
+
+ rrddim_acquired_release(rda);
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, void *user)
+{
+ char *dimension = get_word(words, num_words, 1);
+ char *last_collected_ut_str = get_word(words, num_words, 2);
+ char *last_collected_value_str = get_word(words, num_words, 3);
+ char *last_calculated_value_str = get_word(words, num_words, 4);
+ char *last_stored_value_str = get_word(words, num_words, 5);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDDIM_ACQUIRED *rda = pluginsd_acquire_dimension(host, st, dimension, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
+ if(!rda) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDDIM *rd = rrddim_acquired_to_rrddim(rda);
+ usec_t dim_last_collected_ut = (usec_t)rd->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->last_collected_time.tv_usec;
+ usec_t last_collected_ut = last_collected_ut_str ? str2ull(last_collected_ut_str) : 0;
+ if(last_collected_ut > dim_last_collected_ut) {
+ rd->last_collected_time.tv_sec = last_collected_ut / USEC_PER_SEC;
+ rd->last_collected_time.tv_usec = last_collected_ut % USEC_PER_SEC;
+ }
+
+ rd->last_collected_value = last_collected_value_str ? str2ll(last_collected_value_str, NULL) : 0;
+ rd->last_calculated_value = last_calculated_value_str ? str2ndd(last_calculated_value_str, NULL) : 0;
+ rd->last_stored_value = last_stored_value_str ? str2ndd(last_stored_value_str, NULL) : 0.0;
+ rrddim_acquired_release(rda);
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, void *user)
+{
+ char *last_collected_ut_str = get_word(words, num_words, 1);
+ char *last_updated_ut_str = get_word(words, num_words, 2);
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ usec_t chart_last_collected_ut = (usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec;
+ usec_t last_collected_ut = last_collected_ut_str ? str2ull(last_collected_ut_str) : 0;
+ if(last_collected_ut > chart_last_collected_ut) {
+ st->last_collected_time.tv_sec = last_collected_ut / USEC_PER_SEC;
+ st->last_collected_time.tv_usec = last_collected_ut % USEC_PER_SEC;
+ }
+
+ usec_t chart_last_updated_ut = (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec;
+ usec_t last_updated_ut = last_updated_ut_str ? str2ull(last_updated_ut_str) : 0;
+ if(last_updated_ut > chart_last_updated_ut) {
+ st->last_updated.tv_sec = last_updated_ut / USEC_PER_SEC;
+ st->last_updated.tv_usec = last_updated_ut % USEC_PER_SEC;
+ }
+
+ st->counter++;
+ st->counter_done++;
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
+{
+ if (num_words < 7) { // accepts 7, but the 7th is optional
+ error("REPLAY: malformed " PLUGINSD_KEYWORD_REPLAY_END " command");
+ return PARSER_RC_ERROR;
+ }
+
+ const char *update_every_child_txt = get_word(words, num_words, 1);
+ const char *first_entry_child_txt = get_word(words, num_words, 2);
+ const char *last_entry_child_txt = get_word(words, num_words, 3);
+ const char *start_streaming_txt = get_word(words, num_words, 4);
+ const char *first_entry_requested_txt = get_word(words, num_words, 5);
+ const char *last_entry_requested_txt = get_word(words, num_words, 6);
+ const char *child_world_time_txt = get_word(words, num_words, 7); // optional
+
+ time_t update_every_child = (time_t)str2ul(update_every_child_txt);
+ time_t first_entry_child = (time_t)str2ul(first_entry_child_txt);
+ time_t last_entry_child = (time_t)str2ul(last_entry_child_txt);
+
+ bool start_streaming = (strcmp(start_streaming_txt, "true") == 0);
+ time_t first_entry_requested = (time_t)str2ul(first_entry_requested_txt);
+ time_t last_entry_requested = (time_t)str2ul(last_entry_requested_txt);
+
+ // the optional child world time
+ time_t child_world_time = (child_world_time_txt && *child_world_time_txt) ? (time_t)str2ul(child_world_time_txt) : now_realtime_sec();
+
+ PARSER_USER_OBJECT *user_object = user;
+
+ RRDHOST *host = pluginsd_require_host_from_parent(user, PLUGINSD_KEYWORD_REPLAY_END);
+ if(!host) return PLUGINSD_DISABLE_PLUGIN(user);
+
+ RRDSET *st = pluginsd_require_chart_from_parent(user, PLUGINSD_KEYWORD_REPLAY_END, PLUGINSD_KEYWORD_REPLAY_BEGIN);
+ if(!st) return PLUGINSD_DISABLE_PLUGIN(user);
+
+#ifdef NETDATA_LOG_REPLICATION_REQUESTS
+ internal_error(true,
+ "PLUGINSD REPLAY: 'host:%s/chart:%s': got a " PLUGINSD_KEYWORD_REPLAY_END " child db from %llu to %llu, start_streaming %s, had requested from %llu to %llu, wall clock %llu",
+ rrdhost_hostname(host), rrdset_id(st),
+ (unsigned long long)first_entry_child, (unsigned long long)last_entry_child,
+ start_streaming?"true":"false",
+ (unsigned long long)first_entry_requested, (unsigned long long)last_entry_requested,
+ (unsigned long long)child_world_time
+ );
+#endif
+
+ ((PARSER_USER_OBJECT *) user)->st = NULL;
+ ((PARSER_USER_OBJECT *) user)->count++;
+
+ if(((PARSER_USER_OBJECT *) user)->replay.rset_enabled && st->rrdhost->receiver) {
+ time_t now = now_realtime_sec();
+ time_t started = st->rrdhost->receiver->replication_first_time_t;
+ time_t current = ((PARSER_USER_OBJECT *) user)->replay.end_time;
+
+ worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION,
+ (NETDATA_DOUBLE)(current - started) * 100.0 / (NETDATA_DOUBLE)(now - started));
+ }
+
+ ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = 0;
+ ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = false;
+
+ st->counter++;
+ st->counter_done++;
+
+#ifdef NETDATA_LOG_REPLICATION_REQUESTS
+ st->replay.start_streaming = false;
+ st->replay.after = 0;
+ st->replay.before = 0;
+ if(start_streaming)
+ st->replay.log_next_data_collection = true;
+#endif
+
+ if (start_streaming) {
+ if (st->update_every != update_every_child)
+ rrdset_set_update_every(st, update_every_child);
+
+ if(rrdset_flag_check(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS)) {
+ rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
+ rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS);
+ rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK);
+ rrdhost_receiver_replicating_charts_minus_one(st->rrdhost);
+ }
+#ifdef NETDATA_LOG_REPLICATION_REQUESTS
+ else
+ internal_error(true, "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_END " with enable_streaming = true, but there is no replication in progress for this chart.",
+ rrdhost_hostname(host), rrdset_id(st));
+#endif
+ worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, 100.0);
+
+ return PARSER_RC_OK;
+ }
+
+ rrdcontext_updated_retention_rrdset(st);
+
+ bool ok = replicate_chart_request(send_to_plugin, user_object->parser, host, st,
+ first_entry_child, last_entry_child, child_world_time,
+ first_entry_requested, last_entry_requested);
+ return ok ? PARSER_RC_OK : PARSER_RC_ERROR;
+}
+
+static void pluginsd_process_thread_cleanup(void *ptr) {
+ PARSER *parser = (PARSER *)ptr;
+ rrd_collector_finished();
+ parser_destroy(parser);
+}
+
+// New plugins.d parser
+
+inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugin_input, FILE *fp_plugin_output, int trust_durations)
+{
+ int enabled = cd->enabled;
+
+ if (!fp_plugin_input || !fp_plugin_output || !enabled) {
+ cd->enabled = 0;
+ return 0;
+ }
+
+ if (unlikely(fileno(fp_plugin_input) == -1)) {
+ error("input file descriptor given is not a valid stream");
+ cd->serial_failures++;
+ return 0;
+ }
+
+ if (unlikely(fileno(fp_plugin_output) == -1)) {
+ error("output file descriptor given is not a valid stream");
+ cd->serial_failures++;
+ return 0;
+ }
+
+ clearerr(fp_plugin_input);
+ clearerr(fp_plugin_output);
+
+ PARSER_USER_OBJECT user = {
+ .enabled = cd->enabled,
+ .host = host,
+ .cd = cd,
+ .trust_durations = trust_durations
+ };
+
+ // fp_plugin_output = our input; fp_plugin_input = our output
+ PARSER *parser = parser_init(host, &user, fp_plugin_output, fp_plugin_input, -1, PARSER_INPUT_SPLIT, NULL);
+
+ rrd_collector_started();
+
+ // this keeps the parser with its current value
+ // so, parser needs to be allocated before pushing it
+ netdata_thread_cleanup_push(pluginsd_process_thread_cleanup, parser);
+
+ user.parser = parser;
+
+ while (likely(!parser_next(parser))) {
+ if (unlikely(netdata_exit || parser_action(parser, NULL)))
+ break;
+ }
+
+ // free parser with the pop function
+ netdata_thread_cleanup_pop(1);
+
+ cd->enabled = user.enabled;
+ size_t count = user.count;
+
+ if (likely(count)) {
+ cd->successful_collections += count;
+ cd->serial_failures = 0;
+ }
+ else
+ cd->serial_failures++;
+
+ return count;
+}
diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h
new file mode 100644
index 0000000..e18b43e
--- /dev/null
+++ b/collectors/plugins.d/pluginsd_parser.h
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PLUGINSD_PARSER_H
+#define NETDATA_PLUGINSD_PARSER_H
+
+#include "parser/parser.h"
+
+typedef struct parser_user_object {
+ PARSER *parser;
+ RRDSET *st;
+ RRDHOST *host;
+ void *opaque;
+ struct plugind *cd;
+ int trust_durations;
+ DICTIONARY *new_host_labels;
+ DICTIONARY *chart_rrdlabels_linked_temporarily;
+ size_t count;
+ int enabled;
+ uint8_t st_exists;
+ uint8_t host_exists;
+ void *private; // the user can set this for private use
+
+ struct {
+ time_t start_time;
+ time_t end_time;
+
+ usec_t start_time_ut;
+ usec_t end_time_ut;
+
+ time_t wall_clock_time;
+
+ bool rset_enabled;
+ } replay;
+} PARSER_USER_OBJECT;
+
+PARSER_RC pluginsd_function(char **words, size_t num_words, void *user);
+PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, void *user);
+void inflight_functions_init(PARSER *parser);
+#endif //NETDATA_PLUGINSD_PARSER_H
diff --git a/collectors/proc.plugin/Makefile.am b/collectors/proc.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/proc.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/proc.plugin/README.md b/collectors/proc.plugin/README.md
new file mode 100644
index 0000000..32e2112
--- /dev/null
+++ b/collectors/proc.plugin/README.md
@@ -0,0 +1,607 @@
+<!--
+title: "proc.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/proc.plugin/README.md
+-->
+
+# proc.plugin
+
+- `/proc/net/dev` (all network interfaces for all their values)
+- `/proc/diskstats` (all disks for all their values)
+- `/proc/mdstat` (status of RAID arrays)
+- `/proc/net/snmp` (total IPv4, TCP and UDP usage)
+- `/proc/net/snmp6` (total IPv6 usage)
+- `/proc/net/netstat` (more IPv4 usage)
+- `/proc/net/wireless` (wireless extension)
+- `/proc/net/stat/nf_conntrack` (connection tracking performance)
+- `/proc/net/stat/synproxy` (synproxy performance)
+- `/proc/net/ip_vs/stats` (IPVS connection statistics)
+- `/proc/stat` (CPU utilization and attributes)
+- `/proc/meminfo` (memory information)
+- `/proc/vmstat` (system performance)
+- `/proc/net/rpc/nfsd` (NFS server statistics for both v3 and v4 NFS servers)
+- `/sys/fs/cgroup` (Control Groups - Linux Containers)
+- `/proc/self/mountinfo` (mount points)
+- `/proc/interrupts` (total and per core hardware interrupts)
+- `/proc/softirqs` (total and per core software interrupts)
+- `/proc/loadavg` (system load and total processes running)
+- `/proc/pressure/{cpu,memory,io}` (pressure stall information)
+- `/proc/sys/kernel/random/entropy_avail` (random numbers pool availability - used in cryptography)
+- `/proc/spl/kstat/zfs/arcstats` (status of ZFS adaptive replacement cache)
+- `/proc/spl/kstat/zfs/pool/state` (state of ZFS pools)
+- `/sys/class/power_supply` (power supply properties)
+- `/sys/class/infiniband` (infiniband interconnect)
+- `ipc` (IPC semaphores and message queues)
+- `ksm` Kernel Same-Page Merging performance (several files under `/sys/kernel/mm/ksm`).
+- `netdata` (internal Netdata resources utilization)
+
+- - -
+
+## Monitoring Disks
+
+> Live demo of disk monitoring at: **[http://london.netdata.rocks](https://registry.my-netdata.io/#menu_disk)**
+
+Performance monitoring for Linux disks is quite complicated. The main reason is the plethora of disk technologies available. There are many different hardware disk technologies, but there are even more **virtual disk** technologies that can provide additional storage features.
+
+Hopefully, the Linux kernel provides many metrics that can provide deep insights of what our disks our doing. The kernel measures all these metrics on all layers of storage: **virtual disks**, **physical disks** and **partitions of disks**.
+
+### Monitored disk metrics
+
+- **I/O bandwidth/s (kb/s)**
+ The amount of data transferred from and to the disk.
+- **Amount of discarded data (kb/s)**
+- **I/O operations/s**
+ The number of I/O operations completed.
+- **Extended I/O operations/s**
+ The number of extended I/O operations completed.
+- **Queued I/O operations**
+ The number of currently queued I/O operations. For traditional disks that execute commands one after another, one of them is being run by the disk and the rest are just waiting in a queue.
+- **Backlog size (time in ms)**
+ The expected duration of the currently queued I/O operations.
+- **Utilization (time percentage)**
+ The percentage of time the disk was busy with something. This is a very interesting metric, since for most disks, that execute commands sequentially, **this is the key indication of congestion**. A sequential disk that is 100% of the available time busy, has no time to do anything more, so even if the bandwidth or the number of operations executed by the disk is low, its capacity has been reached.
+ Of course, for newer disk technologies (like fusion cards) that are capable to execute multiple commands in parallel, this metric is just meaningless.
+- **Average I/O operation time (ms)**
+ The average time for I/O requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
+- **Average I/O operation time for extended operations (ms)**
+ The average time for extended I/O requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
+- **Average I/O operation size (kb)**
+ The average amount of data of the completed I/O operations.
+- **Average amount of discarded data (kb)**
+ The average amount of data of the completed discard operations.
+- **Average Service Time (ms)**
+ The average service time for completed I/O operations. This metric is calculated using the total busy time of the disk and the number of completed operations. If the disk is able to execute multiple parallel operations the reporting average service time will be misleading.
+- **Average Service Time for extended I/O operations (ms)**
+ The average service time for completed extended I/O operations.
+- **Merged I/O operations/s**
+ The Linux kernel is capable of merging I/O operations. So, if two requests to read data from the disk are adjacent, the Linux kernel may merge them to one before giving them to disk. This metric measures the number of operations that have been merged by the Linux kernel.
+- **Merged discard operations/s**
+- **Total I/O time**
+ The sum of the duration of all completed I/O operations. This number can exceed the interval if the disk is able to execute multiple I/O operations in parallel.
+- **Space usage**
+ For mounted disks, Netdata will provide a chart for their space, with 3 dimensions:
+ 1. free
+ 2. used
+ 3. reserved for root
+- **inode usage**
+ For mounted disks, Netdata will provide a chart for their inodes (number of file and directories), with 3 dimensions:
+ 1. free
+ 2. used
+ 3. reserved for root
+
+### disk names
+
+Netdata will automatically set the name of disks on the dashboard, from the mount point they are mounted, of course only when they are mounted. Changes in mount points are not currently detected (you will have to restart Netdata to change the name of the disk). To use disk IDs provided by `/dev/disk/by-id`, the `name disks by id` option should be enabled. The `preferred disk ids` simple pattern allows choosing disk IDs to be used in the first place.
+
+### performance metrics
+
+By default, Netdata will enable monitoring metrics only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). Set `yes` for a chart instead of `auto` to enable it permanently. You can also set the `enable zero metrics` option to `yes` in the `[global]` section which enables charts with zero metrics for all internal Netdata plugins.
+
+Netdata categorizes all block devices in 3 categories:
+
+1. physical disks (i.e. block devices that do not have child devices and are not partitions)
+2. virtual disks (i.e. block devices that have child devices - like RAID devices)
+3. disk partitions (i.e. block devices that are part of a physical disk)
+
+Performance metrics are enabled by default for all disk devices, except partitions and not-mounted virtual disks. Of course, you can enable/disable monitoring any block device by editing the Netdata configuration file.
+
+### Netdata configuration
+
+You can get the running Netdata configuration using this:
+
+```sh
+cd /etc/netdata
+curl "http://localhost:19999/netdata.conf" >netdata.conf.new
+mv netdata.conf.new netdata.conf
+```
+
+Then edit `netdata.conf` and find the following section. This is the basic plugin configuration.
+
+```
+[plugin:proc:/proc/diskstats]
+ # enable new disks detected at runtime = yes
+ # performance metrics for physical disks = auto
+ # performance metrics for virtual disks = auto
+ # performance metrics for partitions = no
+ # bandwidth for all disks = auto
+ # operations for all disks = auto
+ # merged operations for all disks = auto
+ # i/o time for all disks = auto
+ # queued operations for all disks = auto
+ # utilization percentage for all disks = auto
+ # extended operations for all disks = auto
+ # backlog for all disks = auto
+ # bcache for all disks = auto
+ # bcache priority stats update every = 0
+ # remove charts of removed disks = yes
+ # path to get block device = /sys/block/%s
+ # path to get block device bcache = /sys/block/%s/bcache
+ # path to get virtual block device = /sys/devices/virtual/block/%s
+ # path to get block device infos = /sys/dev/block/%lu:%lu/%s
+ # path to device mapper = /dev/mapper
+ # path to /dev/disk/by-label = /dev/disk/by-label
+ # path to /dev/disk/by-id = /dev/disk/by-id
+ # path to /dev/vx/dsk = /dev/vx/dsk
+ # name disks by id = no
+ # preferred disk ids = *
+ # exclude disks = loop* ram*
+ # filename to monitor = /proc/diskstats
+ # performance metrics for disks with major 8 = yes
+```
+
+For each virtual disk, physical disk and partition you will have a section like this:
+
+```
+[plugin:proc:/proc/diskstats:sda]
+ # enable = yes
+ # enable performance metrics = auto
+ # bandwidth = auto
+ # operations = auto
+ # merged operations = auto
+ # i/o time = auto
+ # queued operations = auto
+ # utilization percentage = auto
+ # extended operations = auto
+ # backlog = auto
+```
+
+For all configuration options:
+
+- `auto` = enable monitoring if the collected values are not zero
+- `yes` = enable monitoring
+- `no` = disable monitoring
+
+Of course, to set options, you will have to uncomment them. The comments show the internal defaults.
+
+After saving `/etc/netdata/netdata.conf`, restart your Netdata to apply them.
+
+#### Disabling performance metrics for individual device and to multiple devices by device type
+
+You can pretty easy disable performance metrics for individual device, for ex.:
+
+```
+[plugin:proc:/proc/diskstats:sda]
+ enable performance metrics = no
+```
+
+But sometimes you need disable performance metrics for all devices with the same type, to do it you need to figure out device type from `/proc/diskstats` for ex.:
+
+```
+ 7 0 loop0 1651 0 3452 168 0 0 0 0 0 8 168
+ 7 1 loop1 4955 0 11924 880 0 0 0 0 0 64 880
+ 7 2 loop2 36 0 216 4 0 0 0 0 0 4 4
+ 7 6 loop6 0 0 0 0 0 0 0 0 0 0 0
+ 7 7 loop7 0 0 0 0 0 0 0 0 0 0 0
+ 251 2 zram2 27487 0 219896 188 79953 0 639624 1640 0 1828 1828
+ 251 3 zram3 27348 0 218784 152 79952 0 639616 1960 0 2060 2104
+```
+
+All zram devices starts with `251` number and all loop devices starts with `7`.
+So, to disable performance metrics for all loop devices you could add `performance metrics for disks with major 7 = no` to `[plugin:proc:/proc/diskstats]` section.
+
+```
+[plugin:proc:/proc/diskstats]
+ performance metrics for disks with major 7 = no
+```
+
+## Monitoring RAID arrays
+
+### Monitored RAID array metrics
+
+1. **Health** Number of failed disks in every array (aggregate chart).
+
+2. **Disks stats**
+
+- total (number of devices array ideally would have)
+- inuse (number of devices currently are in use)
+
+3. **Mismatch count**
+
+- unsynchronized blocks
+
+4. **Current status**
+
+- resync in percent
+- recovery in percent
+- reshape in percent
+- check in percent
+
+5. **Operation status** (if resync/recovery/reshape/check is active)
+
+- finish in minutes
+- speed in megabytes/s
+
+6. **Nonredundant array availability**
+
+#### configuration
+
+```
+[plugin:proc:/proc/mdstat]
+ # faulty devices = yes
+ # nonredundant arrays availability = yes
+ # mismatch count = auto
+ # disk stats = yes
+ # operation status = yes
+ # make charts obsolete = yes
+ # filename to monitor = /proc/mdstat
+ # mismatch_cnt filename to monitor = /sys/block/%s/md/mismatch_cnt
+```
+
+## Monitoring CPUs
+
+The `/proc/stat` module monitors CPU utilization, interrupts, context switches, processes started/running, thermal
+throttling, frequency, and idle states. It gathers this information from multiple files.
+
+If your system has more than 50 processors (`physical processors * cores per processor * threads per core`), the Agent
+automatically disables CPU thermal throttling, frequency, and idle state charts. To override this default, see the next
+section on configuration.
+
+### Configuration
+
+The settings for monitoring CPUs is in the `[plugin:proc:/proc/stat]` of your `netdata.conf` file.
+
+The `keep per core files open` option lets you reduce the number of file operations on multiple files.
+
+If your system has more than 50 processors and you would like to see the CPU thermal throttling, frequency, and idle
+state charts that are automatically disabled, you can set the following boolean options in the
+`[plugin:proc:/proc/stat]` section.
+
+```conf
+ keep per core files open = yes
+ keep cpuidle files open = yes
+ core_throttle_count = yes
+ package_throttle_count = yes
+ cpu frequency = yes
+ cpu idle states = yes
+```
+
+### CPU frequency
+
+The module shows the current CPU frequency as set by the `cpufreq` kernel
+module.
+
+**Requirement:**
+You need to have `CONFIG_CPU_FREQ` and (optionally) `CONFIG_CPU_FREQ_STAT`
+enabled in your kernel.
+
+`cpufreq` interface provides two different ways of getting the information through `/sys/devices/system/cpu/cpu*/cpufreq/scaling_cur_freq` and `/sys/devices/system/cpu/cpu*/cpufreq/stats/time_in_state` files. The latter is more accurate so it is preferred in the module. `scaling_cur_freq` represents only the current CPU frequency, and doesn't account for any state changes which happen between updates. The module switches back and forth between these two methods if governor is changed.
+
+It produces one chart with multiple lines (one line per core).
+
+#### configuration
+
+`scaling_cur_freq filename to monitor` and `time_in_state filename to monitor` in the `[plugin:proc:/proc/stat]` configuration section
+
+### CPU idle states
+
+The module monitors the usage of CPU idle states.
+
+**Requirement:**
+Your kernel needs to have `CONFIG_CPU_IDLE` enabled.
+
+It produces one stacked chart per CPU, showing the percentage of time spent in
+each state.
+
+#### configuration
+
+`schedstat filename to monitor`, `cpuidle name filename to monitor`, and `cpuidle time filename to monitor` in the `[plugin:proc:/proc/stat]` configuration section
+
+## Monitoring memory
+
+### Monitored memory metrics
+
+- Amount of memory swapped in/out
+- Amount of memory paged from/to disk
+- Number of memory page faults
+- Number of out of memory kills
+- Number of NUMA events
+
+### Configuration
+
+```conf
+[plugin:proc:/proc/vmstat]
+ filename to monitor = /proc/vmstat
+ swap i/o = auto
+ disk i/o = yes
+ memory page faults = yes
+ out of memory kills = yes
+ system-wide numa metric summary = auto
+```
+
+## Monitoring Network Interfaces
+
+### Monitored network interface metrics
+
+- **Physical Network Interfaces Aggregated Bandwidth (kilobits/s)**
+ The amount of data received and sent through all physical interfaces in the system. This is the source of data for the Net Inbound and Net Outbound dials in the System Overview section.
+
+- **Bandwidth (kilobits/s)**
+ The amount of data received and sent through the interface.
+
+- **Packets (packets/s)**
+ The number of packets received, packets sent, and multicast packets transmitted through the interface.
+
+- **Interface Errors (errors/s)**
+ The number of errors for the inbound and outbound traffic on the interface.
+
+- **Interface Drops (drops/s)**
+ The number of packets dropped for the inbound and outbound traffic on the interface.
+
+- **Interface FIFO Buffer Errors (errors/s)**
+ The number of FIFO buffer errors encountered while receiving and transmitting data through the interface.
+
+- **Compressed Packets (packets/s)**
+ The number of compressed packets transmitted or received by the device driver.
+
+- **Network Interface Events (events/s)**
+ The number of packet framing errors, collisions detected on the interface, and carrier losses detected by the device driver.
+
+By default Netdata will enable monitoring metrics only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though).
+
+### Monitoring wireless network interfaces
+
+The settings for monitoring wireless is in the `[plugin:proc:/proc/net/wireless]` section of your `netdata.conf` file.
+
+```conf
+ status for all interfaces = yes
+ quality for all interfaces = yes
+ discarded packets for all interfaces = yes
+ missed beacon for all interface = yes
+```
+
+You can set the following values for each configuration option:
+
+- `auto` = enable monitoring if the collected values are not zero
+- `yes` = enable monitoring
+- `no` = disable monitoring
+
+#### Monitored wireless interface metrics
+
+- **Status**
+ The current state of the interface. This is a device-dependent option.
+
+- **Link**
+ Overall quality of the link.
+
+- **Level**
+ Received signal strength (RSSI), which indicates how strong the received signal is.
+
+- **Noise**
+ Background noise level.
+
+- **Discarded packets**
+ Discarded packets for: Number of packets received with a different NWID or ESSID (`nwid`), unable to decrypt (`crypt`), hardware was not able to properly re-assemble the link layer fragments (`frag`), packets failed to deliver (`retry`), and packets lost in relation with specific wireless operations (`misc`).
+
+- **Missed beacon**
+ Number of periodic beacons from the cell or the access point the interface has missed.
+
+#### Wireless configuration
+
+#### alarms
+
+There are several alarms defined in `health.d/net.conf`.
+
+The tricky ones are `inbound packets dropped` and `inbound packets dropped ratio`. They have quite a strict policy so that they warn users about possible issues. These alarms can be annoying for some network configurations. It is especially true for some bonding configurations if an interface is a child or a bonding interface itself. If it is expected to have a certain number of drops on an interface for a certain network configuration, a separate alarm with different triggering thresholds can be created or the existing one can be disabled for this specific interface. It can be done with the help of the [families](/health/REFERENCE.md#alarm-line-families) line in the alarm configuration. For example, if you want to disable the `inbound packets dropped` alarm for `eth0`, set `families: !eth0 *` in the alarm definition for `template: inbound_packets_dropped`.
+
+#### configuration
+
+Module configuration:
+
+```
+[plugin:proc:/proc/net/dev]
+ # filename to monitor = /proc/net/dev
+ # path to get virtual interfaces = /sys/devices/virtual/net/%s
+ # path to get net device speed = /sys/class/net/%s/speed
+ # enable new interfaces detected at runtime = auto
+ # bandwidth for all interfaces = auto
+ # packets for all interfaces = auto
+ # errors for all interfaces = auto
+ # drops for all interfaces = auto
+ # fifo for all interfaces = auto
+ # compressed packets for all interfaces = auto
+ # frames, collisions, carrier counters for all interfaces = auto
+ # disable by default interfaces matching = lo fireqos* *-ifb
+ # refresh interface speed every seconds = 10
+```
+
+Per interface configuration:
+
+```
+[plugin:proc:/proc/net/dev:enp0s3]
+ # enabled = yes
+ # virtual = no
+ # bandwidth = auto
+ # packets = auto
+ # errors = auto
+ # drops = auto
+ # fifo = auto
+ # compressed = auto
+ # events = auto
+```
+
+## Linux Anti-DDoS
+
+![image6](https://cloud.githubusercontent.com/assets/2662304/14253733/53550b16-fa95-11e5-8d9d-4ed171df4735.gif)
+
+---
+
+SYNPROXY is a TCP SYN packets proxy. It can be used to protect any TCP server (like a web server) from SYN floods and similar DDos attacks.
+
+SYNPROXY is a netfilter module, in the Linux kernel (since version 3.12). It is optimized to handle millions of packets per second utilizing all CPUs available without any concurrency locking between the connections.
+
+The net effect of this, is that the real servers will not notice any change during the attack. The valid TCP connections will pass through and served, while the attack will be stopped at the firewall.
+
+Netdata does not enable SYNPROXY. It just uses the SYNPROXY metrics exposed by your kernel, so you will first need to configure it. The hard way is to run iptables SYNPROXY commands directly on the console. An easier way is to use [FireHOL](https://firehol.org/), which, is a firewall manager for iptables. FireHOL can configure SYNPROXY using the following setup guides:
+
+- **[Working with SYNPROXY](https://github.com/firehol/firehol/wiki/Working-with-SYNPROXY)**
+- **[Working with SYNPROXY and traps](https://github.com/firehol/firehol/wiki/Working-with-SYNPROXY-and-traps)**
+
+### Real-time monitoring of Linux Anti-DDoS
+
+Netdata is able to monitor in real-time (per second updates) the operation of the Linux Anti-DDoS protection.
+
+It visualizes 4 charts:
+
+1. TCP SYN Packets received on ports operated by SYNPROXY
+2. TCP Cookies (valid, invalid, retransmits)
+3. Connections Reopened
+4. Entries used
+
+Example image:
+
+![ddos](https://cloud.githubusercontent.com/assets/2662304/14398891/6016e3fc-fdf0-11e5-942b-55de6a52cb66.gif)
+
+See Linux Anti-DDoS in action at: **[Netdata demo site (with SYNPROXY enabled)](https://registry.my-netdata.io/#menu_netfilter_submenu_synproxy)**
+
+## Linux power supply
+
+This module monitors various metrics reported by power supply drivers
+on Linux. This allows tracking and alerting on things like remaining
+battery capacity.
+
+Depending on the underlying driver, it may provide the following charts
+and metrics:
+
+1. Capacity: The power supply capacity expressed as a percentage.
+
+ - capacity_now
+
+2. Charge: The charge for the power supply, expressed as amphours.
+
+ - charge_full_design
+ - charge_full
+ - charge_now
+ - charge_empty
+ - charge_empty_design
+
+3. Energy: The energy for the power supply, expressed as watthours.
+
+ - energy_full_design
+ - energy_full
+ - energy_now
+ - energy_empty
+ - energy_empty_design
+
+4. Voltage: The voltage for the power supply, expressed as volts.
+
+ - voltage_max_design
+ - voltage_max
+ - voltage_now
+ - voltage_min
+ - voltage_min_design
+
+#### configuration
+
+```
+[plugin:proc:/sys/class/power_supply]
+ # battery capacity = yes
+ # battery charge = no
+ # battery energy = no
+ # power supply voltage = no
+ # keep files open = auto
+ # directory to monitor = /sys/class/power_supply
+```
+
+#### notes
+
+- Most drivers provide at least the first chart. Battery powered ACPI
+ compliant systems (like most laptops) provide all but the third, but do
+ not provide all of the metrics for each chart.
+
+- Current, energy, and voltages are reported with a *very* high precision
+ by the power_supply framework. Usually, this is far higher than the
+ actual hardware supports reporting, so expect to see changes in these
+ charts jump instead of scaling smoothly.
+
+- If `max` or `full` attribute is defined by the driver, but not a
+ corresponding `min` or `empty` attribute, then Netdata will still provide
+ the corresponding `min` or `empty`, which will then always read as zero.
+ This way, alerts which match on these will still work.
+
+## Infiniband interconnect
+
+This module monitors every active Infiniband port. It provides generic counters statistics, and per-vendor hw-counters (if vendor is supported).
+
+### Monitored interface metrics
+
+Each port will have its counters metrics monitored, grouped in the following charts:
+
+- **Bandwidth usage**
+ Sent/Received data, in KB/s
+
+- **Packets Statistics**
+ Sent/Received packets, in 3 categories: total, unicast and multicast.
+
+- **Errors Statistics**
+ Many errors counters are provided, presenting statistics for:
+ - Packets: malformed, sent/received discarded by card/switch, missing resource
+ - Link: downed, recovered, integrity error, minor error
+ - Other events: Tick Wait to send, buffer overrun
+
+If your vendor is supported, you'll also get HW-Counters statistics. These being vendor specific, please refer to their documentation.
+
+- Mellanox: [see statistics documentation](https://community.mellanox.com/s/article/understanding-mlx5-linux-counters-and-status-parameters)
+
+### configuration
+
+Default configuration will monitor only enabled infiniband ports, and refresh newly activated or created ports every 30 seconds
+
+```
+[plugin:proc:/sys/class/infiniband]
+ # dirname to monitor = /sys/class/infiniband
+ # bandwidth counters = yes
+ # packets counters = yes
+ # errors counters = yes
+ # hardware packets counters = auto
+ # hardware errors counters = auto
+ # monitor only ports being active = auto
+ # disable by default interfaces matching =
+ # refresh ports state every seconds = 30
+```
+
+
+## IPC
+
+### Monitored IPC metrics
+
+- **number of messages in message queues**
+- **amount of memory used by message queues**
+- **number of semaphores**
+- **number of semaphore arrays**
+- **number of shared memory segments**
+- **amount of memory used by shared memory segments**
+
+As far as the message queue charts are dynamic, sane limits are applied for the number of dimensions per chart (the limit is configurable).
+
+### configuration
+
+```
+[plugin:proc:ipc]
+ # message queues = yes
+ # semaphore totals = yes
+ # shared memory totals = yes
+ # msg filename to monitor = /proc/sysvipc/msg
+ # shm filename to monitor = /proc/sysvipc/shm
+ # max dimensions in memory allowed = 50
+```
+
+
diff --git a/collectors/proc.plugin/ipc.c b/collectors/proc.plugin/ipc.c
new file mode 100644
index 0000000..7d3d2ec
--- /dev/null
+++ b/collectors/proc.plugin/ipc.c
@@ -0,0 +1,554 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#include <sys/sem.h>
+#include <sys/msg.h>
+#include <sys/shm.h>
+
+
+#ifndef SEMVMX
+#define SEMVMX 32767 /* <= 32767 semaphore maximum value */
+#endif
+
+/* Some versions of libc only define IPC_INFO when __USE_GNU is defined. */
+#ifndef IPC_INFO
+#define IPC_INFO 3
+#endif
+
+struct ipc_limits {
+ uint64_t shmmni; /* max number of segments */
+ uint64_t shmmax; /* max segment size */
+ uint64_t shmall; /* max total shared memory */
+ uint64_t shmmin; /* min segment size */
+
+ int semmni; /* max number of arrays */
+ int semmsl; /* max semaphores per array */
+ int semmns; /* max semaphores system wide */
+ int semopm; /* max ops per semop call */
+ unsigned int semvmx; /* semaphore max value (constant) */
+
+ int msgmni; /* max queues system wide */
+ size_t msgmax; /* max size of message */
+ int msgmnb; /* default max size of queue */
+};
+
+struct ipc_status {
+ int semusz; /* current number of arrays */
+ int semaem; /* current semaphores system wide */
+};
+
+/*
+ * The last arg of semctl is a union semun, but where is it defined? X/OPEN
+ * tells us to define it ourselves, but until recently Linux include files
+ * would also define it.
+ */
+#ifndef HAVE_UNION_SEMUN
+/* according to X/OPEN we have to define it ourselves */
+union semun {
+ int val;
+ struct semid_ds *buf;
+ unsigned short int *array;
+ struct seminfo *__buf;
+};
+#endif
+
+struct message_queue {
+ unsigned long long id;
+ int found;
+
+ RRDDIM *rd_messages;
+ RRDDIM *rd_bytes;
+ unsigned long long messages;
+ unsigned long long bytes;
+
+ struct message_queue * next;
+};
+
+struct shm_stats {
+ unsigned long long segments;
+ unsigned long long bytes;
+};
+
+static inline int ipc_sem_get_limits(struct ipc_limits *lim) {
+ static procfile *ff = NULL;
+ static int error_shown = 0;
+ static char filename[FILENAME_MAX + 1] = "";
+
+ if(unlikely(!filename[0]))
+ snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/sem", netdata_configured_host_prefix);
+
+ if(unlikely(!ff)) {
+ ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) {
+ if(unlikely(!error_shown)) {
+ error("IPC: Cannot open file '%s'.", filename);
+ error_shown = 1;
+ }
+ goto ipc;
+ }
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ if(unlikely(!error_shown)) {
+ error("IPC: Cannot read file '%s'.", filename);
+ error_shown = 1;
+ }
+ goto ipc;
+ }
+
+ if(procfile_lines(ff) >= 1 && procfile_linewords(ff, 0) >= 4) {
+ lim->semvmx = SEMVMX;
+ lim->semmsl = str2i(procfile_lineword(ff, 0, 0));
+ lim->semmns = str2i(procfile_lineword(ff, 0, 1));
+ lim->semopm = str2i(procfile_lineword(ff, 0, 2));
+ lim->semmni = str2i(procfile_lineword(ff, 0, 3));
+ return 0;
+ }
+ else {
+ if(unlikely(!error_shown)) {
+ error("IPC: Invalid content in file '%s'.", filename);
+ error_shown = 1;
+ }
+ goto ipc;
+ }
+
+ipc:
+ // cannot do it from the file
+ // query IPC
+ {
+ struct seminfo seminfo = {.semmni = 0};
+ union semun arg = {.array = (ushort *) &seminfo};
+
+ if(unlikely(semctl(0, 0, IPC_INFO, arg) < 0)) {
+ error("IPC: Failed to read '%s' and request IPC_INFO with semctl().", filename);
+ goto error;
+ }
+
+ lim->semvmx = SEMVMX;
+ lim->semmni = seminfo.semmni;
+ lim->semmsl = seminfo.semmsl;
+ lim->semmns = seminfo.semmns;
+ lim->semopm = seminfo.semopm;
+ return 0;
+ }
+
+error:
+ lim->semvmx = 0;
+ lim->semmni = 0;
+ lim->semmsl = 0;
+ lim->semmns = 0;
+ lim->semopm = 0;
+ return -1;
+}
+
+/*
+printf ("------ Semaphore Limits --------\n");
+printf ("max number of arrays = %d\n", limits.semmni);
+printf ("max semaphores per array = %d\n", limits.semmsl);
+printf ("max semaphores system wide = %d\n", limits.semmns);
+printf ("max ops per semop call = %d\n", limits.semopm);
+printf ("semaphore max value = %u\n", limits.semvmx);
+
+printf ("------ Semaphore Status --------\n");
+printf ("used arrays = %d\n", status.semusz);
+printf ("allocated semaphores = %d\n", status.semaem);
+*/
+
+static inline int ipc_sem_get_status(struct ipc_status *st) {
+ struct seminfo seminfo;
+ union semun arg;
+
+ arg.array = (ushort *) (void *) &seminfo;
+
+ if(unlikely(semctl (0, 0, SEM_INFO, arg) < 0)) {
+ /* kernel not configured for semaphores */
+ static int error_shown = 0;
+ if(unlikely(!error_shown)) {
+ error("IPC: kernel is not configured for semaphores");
+ error_shown = 1;
+ }
+ st->semusz = 0;
+ st->semaem = 0;
+ return -1;
+ }
+
+ st->semusz = seminfo.semusz;
+ st->semaem = seminfo.semaem;
+ return 0;
+}
+
+int ipc_msq_get_info(char *msg_filename, struct message_queue **message_queue_root) {
+ static procfile *ff;
+ struct message_queue *msq;
+
+ if(unlikely(!ff)) {
+ ff = procfile_open(msg_filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 1;
+
+ size_t lines = procfile_lines(ff);
+ size_t words = 0;
+
+ if(unlikely(lines < 2)) {
+ error("Cannot read %s. Expected 2 or more lines, read %zu.", procfile_filename(ff), lines);
+ return 1;
+ }
+
+ // loop through all lines except the first and the last ones
+ size_t l;
+ for(l = 1; l < lines - 1; l++) {
+ words = procfile_linewords(ff, l);
+ if(unlikely(words < 2)) continue;
+ if(unlikely(words < 14)) {
+ error("Cannot read %s line. Expected 14 params, read %zu.", procfile_filename(ff), words);
+ continue;
+ }
+
+ // find the id in the linked list or create a new structure
+ int found = 0;
+
+ unsigned long long id = str2ull(procfile_lineword(ff, l, 1));
+ for(msq = *message_queue_root; msq ; msq = msq->next) {
+ if(unlikely(id == msq->id)) {
+ found = 1;
+ break;
+ }
+ }
+
+ if(unlikely(!found)) {
+ msq = callocz(1, sizeof(struct message_queue));
+ msq->next = *message_queue_root;
+ *message_queue_root = msq;
+ msq->id = id;
+ }
+
+ msq->messages = str2ull(procfile_lineword(ff, l, 4));
+ msq->bytes = str2ull(procfile_lineword(ff, l, 3));
+ msq->found = 1;
+ }
+
+ return 0;
+}
+
+int ipc_shm_get_info(char *shm_filename, struct shm_stats *shm) {
+ static procfile *ff;
+
+ if(unlikely(!ff)) {
+ ff = procfile_open(shm_filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 1;
+
+ size_t lines = procfile_lines(ff);
+ size_t words = 0;
+
+ if(unlikely(lines < 2)) {
+ error("Cannot read %s. Expected 2 or more lines, read %zu.", procfile_filename(ff), lines);
+ return 1;
+ }
+
+ shm->segments = 0;
+ shm->bytes = 0;
+
+ // loop through all lines except the first and the last ones
+ size_t l;
+ for(l = 1; l < lines - 1; l++) {
+ words = procfile_linewords(ff, l);
+ if(unlikely(words < 2)) continue;
+ if(unlikely(words < 16)) {
+ error("Cannot read %s line. Expected 16 params, read %zu.", procfile_filename(ff), words);
+ continue;
+ }
+
+ shm->segments++;
+ shm->bytes += str2ull(procfile_lineword(ff, l, 3));
+ }
+
+ return 0;
+}
+
+int do_ipc(int update_every, usec_t dt) {
+ (void)dt;
+
+ static int do_sem = -1, do_msg = -1, do_shm = -1;
+ static int read_limits_next = -1;
+ static struct ipc_limits limits;
+ static struct ipc_status status;
+ static const RRDVAR_ACQUIRED *arrays_max = NULL, *semaphores_max = NULL;
+ static RRDSET *st_semaphores = NULL, *st_arrays = NULL;
+ static RRDDIM *rd_semaphores = NULL, *rd_arrays = NULL;
+ static char *msg_filename = NULL;
+ static struct message_queue *message_queue_root = NULL;
+ static long long dimensions_limit;
+ static char *shm_filename = NULL;
+
+ if(unlikely(do_sem == -1)) {
+ do_msg = config_get_boolean("plugin:proc:ipc", "message queues", CONFIG_BOOLEAN_YES);
+ do_sem = config_get_boolean("plugin:proc:ipc", "semaphore totals", CONFIG_BOOLEAN_YES);
+ do_shm = config_get_boolean("plugin:proc:ipc", "shared memory totals", CONFIG_BOOLEAN_YES);
+
+ char filename[FILENAME_MAX + 1];
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/sysvipc/msg");
+ msg_filename = config_get("plugin:proc:ipc", "msg filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/sysvipc/shm");
+ shm_filename = config_get("plugin:proc:ipc", "shm filename to monitor", filename);
+
+ dimensions_limit = config_get_number("plugin:proc:ipc", "max dimensions in memory allowed", 50);
+
+ // make sure it works
+ if(ipc_sem_get_limits(&limits) == -1) {
+ error("unable to fetch semaphore limits");
+ do_sem = CONFIG_BOOLEAN_NO;
+ }
+ else if(ipc_sem_get_status(&status) == -1) {
+ error("unable to fetch semaphore statistics");
+ do_sem = CONFIG_BOOLEAN_NO;
+ }
+ else {
+ // create the charts
+ if(unlikely(!st_semaphores)) {
+ st_semaphores = rrdset_create_localhost(
+ "system"
+ , "ipc_semaphores"
+ , NULL
+ , "ipc semaphores"
+ , NULL
+ , "IPC Semaphores"
+ , "semaphores"
+ , PLUGIN_PROC_NAME
+ , "ipc"
+ , NETDATA_CHART_PRIO_SYSTEM_IPC_SEMAPHORES
+ , localhost->rrd_update_every
+ , RRDSET_TYPE_AREA
+ );
+ rd_semaphores = rrddim_add(st_semaphores, "semaphores", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ if(unlikely(!st_arrays)) {
+ st_arrays = rrdset_create_localhost(
+ "system"
+ , "ipc_semaphore_arrays"
+ , NULL
+ , "ipc semaphores"
+ , NULL
+ , "IPC Semaphore Arrays"
+ , "arrays"
+ , PLUGIN_PROC_NAME
+ , "ipc"
+ , NETDATA_CHART_PRIO_SYSTEM_IPC_SEM_ARRAYS
+ , localhost->rrd_update_every
+ , RRDSET_TYPE_AREA
+ );
+ rd_arrays = rrddim_add(st_arrays, "arrays", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ // variables
+ semaphores_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_max");
+ arrays_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_arrays_max");
+ }
+
+ struct stat stbuf;
+ if (stat(msg_filename, &stbuf)) {
+ do_msg = CONFIG_BOOLEAN_NO;
+ }
+
+ if(unlikely(do_sem == CONFIG_BOOLEAN_NO && do_msg == CONFIG_BOOLEAN_NO)) {
+ error("ipc module disabled");
+ return 1;
+ }
+ }
+
+ if(likely(do_sem != CONFIG_BOOLEAN_NO)) {
+ if(unlikely(read_limits_next < 0)) {
+ if(unlikely(ipc_sem_get_limits(&limits) == -1)) {
+ error("Unable to fetch semaphore limits.");
+ }
+ else {
+ if(semaphores_max) rrdvar_custom_host_variable_set(localhost, semaphores_max, limits.semmns);
+ if(arrays_max) rrdvar_custom_host_variable_set(localhost, arrays_max, limits.semmni);
+
+ st_arrays->red = limits.semmni;
+ st_semaphores->red = limits.semmns;
+
+ read_limits_next = 60 / update_every;
+ }
+ }
+ else
+ read_limits_next--;
+
+ if(unlikely(ipc_sem_get_status(&status) == -1)) {
+ error("Unable to get semaphore statistics");
+ return 0;
+ }
+
+ rrddim_set_by_pointer(st_semaphores, rd_semaphores, status.semaem);
+ rrdset_done(st_semaphores);
+
+ rrddim_set_by_pointer(st_arrays, rd_arrays, status.semusz);
+ rrdset_done(st_arrays);
+ }
+
+ if(likely(do_msg != CONFIG_BOOLEAN_NO)) {
+ static RRDSET *st_msq_messages = NULL, *st_msq_bytes = NULL;
+
+ int ret = ipc_msq_get_info(msg_filename, &message_queue_root);
+
+ if(!ret && message_queue_root) {
+ if(unlikely(!st_msq_messages))
+ st_msq_messages = rrdset_create_localhost(
+ "system"
+ , "message_queue_messages"
+ , NULL
+ , "ipc message queues"
+ , NULL
+ , "IPC Message Queue Number of Messages"
+ , "messages"
+ , PLUGIN_PROC_NAME
+ , "ipc"
+ , NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_MESSAGES
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ if(unlikely(!st_msq_bytes))
+ st_msq_bytes = rrdset_create_localhost(
+ "system"
+ , "message_queue_bytes"
+ , NULL
+ , "ipc message queues"
+ , NULL
+ , "IPC Message Queue Used Bytes"
+ , "bytes"
+ , PLUGIN_PROC_NAME
+ , "ipc"
+ , NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_SIZE
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ struct message_queue *msq = message_queue_root, *msq_prev = NULL;
+ while(likely(msq)){
+ if(likely(msq->found)) {
+ if(unlikely(!msq->rd_messages || !msq->rd_bytes)) {
+ char id[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(id, RRD_ID_LENGTH_MAX, "%llu", msq->id);
+ if(likely(!msq->rd_messages)) msq->rd_messages = rrddim_add(st_msq_messages, id, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ if(likely(!msq->rd_bytes)) msq->rd_bytes = rrddim_add(st_msq_bytes, id, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_msq_messages, msq->rd_messages, msq->messages);
+ rrddim_set_by_pointer(st_msq_bytes, msq->rd_bytes, msq->bytes);
+
+ msq->found = 0;
+ }
+ else {
+ rrddim_is_obsolete(st_msq_messages, msq->rd_messages);
+ rrddim_is_obsolete(st_msq_bytes, msq->rd_bytes);
+
+ // remove message queue from the linked list
+ if(!msq_prev)
+ message_queue_root = msq->next;
+ else
+ msq_prev->next = msq->next;
+ freez(msq);
+ msq = NULL;
+ }
+ if(likely(msq)) {
+ msq_prev = msq;
+ msq = msq->next;
+ }
+ else if(!msq_prev)
+ msq = message_queue_root;
+ else
+ msq = msq_prev->next;
+ }
+
+ rrdset_done(st_msq_messages);
+ rrdset_done(st_msq_bytes);
+
+ long long dimensions_num = rrdset_number_of_dimensions(st_msq_messages);
+
+ if(unlikely(dimensions_num > dimensions_limit)) {
+ info("Message queue statistics has been disabled");
+ info("There are %lld dimensions in memory but limit was set to %lld", dimensions_num, dimensions_limit);
+ rrdset_is_obsolete(st_msq_messages);
+ rrdset_is_obsolete(st_msq_bytes);
+ st_msq_messages = NULL;
+ st_msq_bytes = NULL;
+ do_msg = CONFIG_BOOLEAN_NO;
+ }
+ else if(unlikely(!message_queue_root)) {
+ info("Making chart %s (%s) obsolete since it does not have any dimensions", rrdset_name(st_msq_messages), rrdset_id(st_msq_messages));
+ rrdset_is_obsolete(st_msq_messages);
+ st_msq_messages = NULL;
+
+ info("Making chart %s (%s) obsolete since it does not have any dimensions", rrdset_name(st_msq_bytes), rrdset_id(st_msq_bytes));
+ rrdset_is_obsolete(st_msq_bytes);
+ st_msq_bytes = NULL;
+ }
+ }
+ }
+
+ if(likely(do_shm != CONFIG_BOOLEAN_NO)) {
+ static RRDSET *st_shm_segments = NULL, *st_shm_bytes = NULL;
+ static RRDDIM *rd_shm_segments = NULL, *rd_shm_bytes = NULL;
+ struct shm_stats shm;
+
+ if(!ipc_shm_get_info(shm_filename, &shm)) {
+ if(unlikely(!st_shm_segments)) {
+ st_shm_segments = rrdset_create_localhost(
+ "system"
+ , "shared_memory_segments"
+ , NULL
+ , "ipc shared memory"
+ , NULL
+ , "IPC Shared Memory Number of Segments"
+ , "segments"
+ , PLUGIN_PROC_NAME
+ , "ipc"
+ , NETDATA_CHART_PRIO_SYSTEM_IPC_SHARED_MEM_SEGS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_shm_segments = rrddim_add(st_shm_segments, "segments", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_shm_segments, rd_shm_segments, shm.segments);
+ rrdset_done(st_shm_segments);
+
+ if(unlikely(!st_shm_bytes)) {
+ st_shm_bytes = rrdset_create_localhost(
+ "system"
+ , "shared_memory_bytes"
+ , NULL
+ , "ipc shared memory"
+ , NULL
+ , "IPC Shared Memory Used Bytes"
+ , "bytes"
+ , PLUGIN_PROC_NAME
+ , "ipc"
+ , NETDATA_CHART_PRIO_SYSTEM_IPC_SHARED_MEM_SIZE
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_shm_bytes = rrddim_add(st_shm_bytes, "bytes", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_shm_bytes, rd_shm_bytes, shm.bytes);
+ rrdset_done(st_shm_bytes);
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/plugin_proc.c b/collectors/proc.plugin/plugin_proc.c
new file mode 100644
index 0000000..1b24df4
--- /dev/null
+++ b/collectors/proc.plugin/plugin_proc.c
@@ -0,0 +1,189 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+static struct proc_module {
+ const char *name;
+ const char *dim;
+
+ int enabled;
+
+ int (*func)(int update_every, usec_t dt);
+
+ RRDDIM *rd;
+
+} proc_modules[] = {
+
+ // system metrics
+ {.name = "/proc/stat", .dim = "stat", .func = do_proc_stat},
+ {.name = "/proc/uptime", .dim = "uptime", .func = do_proc_uptime},
+ {.name = "/proc/loadavg", .dim = "loadavg", .func = do_proc_loadavg},
+ {.name = "/proc/sys/kernel/random/entropy_avail", .dim = "entropy", .func = do_proc_sys_kernel_random_entropy_avail},
+
+ // pressure metrics
+ {.name = "/proc/pressure", .dim = "pressure", .func = do_proc_pressure},
+
+ // CPU metrics
+ {.name = "/proc/interrupts", .dim = "interrupts", .func = do_proc_interrupts},
+ {.name = "/proc/softirqs", .dim = "softirqs", .func = do_proc_softirqs},
+
+ // memory metrics
+ {.name = "/proc/vmstat", .dim = "vmstat", .func = do_proc_vmstat},
+ {.name = "/proc/meminfo", .dim = "meminfo", .func = do_proc_meminfo},
+ {.name = "/sys/kernel/mm/ksm", .dim = "ksm", .func = do_sys_kernel_mm_ksm},
+ {.name = "/sys/block/zram", .dim = "zram", .func = do_sys_block_zram},
+ {.name = "/sys/devices/system/edac/mc", .dim = "ecc", .func = do_proc_sys_devices_system_edac_mc},
+ {.name = "/sys/devices/system/node", .dim = "numa", .func = do_proc_sys_devices_system_node},
+ {.name = "/proc/pagetypeinfo", .dim = "pagetypeinfo", .func = do_proc_pagetypeinfo},
+
+ // network metrics
+ {.name = "/proc/net/wireless", .dim = "netwireless", .func = do_proc_net_wireless},
+ {.name = "/proc/net/sockstat", .dim = "sockstat", .func = do_proc_net_sockstat},
+ {.name = "/proc/net/sockstat6", .dim = "sockstat6", .func = do_proc_net_sockstat6},
+ {.name = "/proc/net/netstat", .dim = "netstat", .func = do_proc_net_netstat},
+ {.name = "/proc/net/sctp/snmp", .dim = "sctp", .func = do_proc_net_sctp_snmp},
+ {.name = "/proc/net/softnet_stat", .dim = "softnet", .func = do_proc_net_softnet_stat},
+ {.name = "/proc/net/ip_vs/stats", .dim = "ipvs", .func = do_proc_net_ip_vs_stats},
+ {.name = "/sys/class/infiniband", .dim = "infiniband", .func = do_sys_class_infiniband},
+
+ // firewall metrics
+ {.name = "/proc/net/stat/conntrack", .dim = "conntrack", .func = do_proc_net_stat_conntrack},
+ {.name = "/proc/net/stat/synproxy", .dim = "synproxy", .func = do_proc_net_stat_synproxy},
+
+ // disk metrics
+ {.name = "/proc/diskstats", .dim = "diskstats", .func = do_proc_diskstats},
+ {.name = "/proc/mdstat", .dim = "mdstat", .func = do_proc_mdstat},
+
+ // NFS metrics
+ {.name = "/proc/net/rpc/nfsd", .dim = "nfsd", .func = do_proc_net_rpc_nfsd},
+ {.name = "/proc/net/rpc/nfs", .dim = "nfs", .func = do_proc_net_rpc_nfs},
+
+ // ZFS metrics
+ {.name = "/proc/spl/kstat/zfs/arcstats", .dim = "zfs_arcstats", .func = do_proc_spl_kstat_zfs_arcstats},
+ {.name = "/proc/spl/kstat/zfs/pool/state",.dim = "zfs_pool_state",.func = do_proc_spl_kstat_zfs_pool_state},
+
+ // BTRFS metrics
+ {.name = "/sys/fs/btrfs", .dim = "btrfs", .func = do_sys_fs_btrfs},
+
+ // IPC metrics
+ {.name = "ipc", .dim = "ipc", .func = do_ipc},
+
+ {.name = "/sys/class/power_supply", .dim = "power_supply", .func = do_sys_class_power_supply},
+ // linux power supply metrics
+
+ // the terminator of this array
+ {.name = NULL, .dim = NULL, .func = NULL}
+};
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 36
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 36
+#endif
+
+static netdata_thread_t *netdev_thread = NULL;
+
+static void proc_main_cleanup(void *ptr)
+{
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ info("cleaning up...");
+
+ if (netdev_thread) {
+ netdata_thread_join(*netdev_thread, NULL);
+ freez(netdev_thread);
+ }
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+
+ worker_unregister();
+}
+
+void *proc_main(void *ptr)
+{
+ worker_register("PROC");
+
+ if (config_get_boolean("plugin:proc", "/proc/net/dev", CONFIG_BOOLEAN_YES)) {
+ netdev_thread = mallocz(sizeof(netdata_thread_t));
+ debug(D_SYSTEM, "Starting thread %s.", THREAD_NETDEV_NAME);
+ netdata_thread_create(
+ netdev_thread, THREAD_NETDEV_NAME, NETDATA_THREAD_OPTION_JOINABLE, netdev_main, netdev_thread);
+ }
+
+ netdata_thread_cleanup_push(proc_main_cleanup, ptr);
+
+ config_get_boolean("plugin:proc", "/proc/pagetypeinfo", CONFIG_BOOLEAN_NO);
+
+ // check the enabled status for each module
+ int i;
+ for (i = 0; proc_modules[i].name; i++) {
+ struct proc_module *pm = &proc_modules[i];
+
+ pm->enabled = config_get_boolean("plugin:proc", pm->name, CONFIG_BOOLEAN_YES);
+ pm->rd = NULL;
+
+ worker_register_job_name(i, proc_modules[i].dim);
+ }
+
+ usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ while (!netdata_exit) {
+ worker_is_idle();
+ usec_t hb_dt = heartbeat_next(&hb, step);
+
+ if (unlikely(netdata_exit))
+ break;
+
+ for (i = 0; proc_modules[i].name; i++) {
+ if (unlikely(netdata_exit))
+ break;
+
+ struct proc_module *pm = &proc_modules[i];
+ if (unlikely(!pm->enabled))
+ continue;
+
+ debug(D_PROCNETDEV_LOOP, "PROC calling %s.", pm->name);
+
+ worker_is_busy(i);
+ pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
+ }
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+int get_numa_node_count(void)
+{
+ static int numa_node_count = -1;
+
+ if (numa_node_count != -1)
+ return numa_node_count;
+
+ numa_node_count = 0;
+
+ char name[FILENAME_MAX + 1];
+ snprintfz(name, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/node");
+ char *dirname = config_get("plugin:proc:/sys/devices/system/node", "directory to monitor", name);
+
+ DIR *dir = opendir(dirname);
+ if (dir) {
+ struct dirent *de = NULL;
+ while ((de = readdir(dir))) {
+ if (de->d_type != DT_DIR)
+ continue;
+
+ if (strncmp(de->d_name, "node", 4) != 0)
+ continue;
+
+ if (!isdigit(de->d_name[4]))
+ continue;
+
+ numa_node_count++;
+ }
+ closedir(dir);
+ }
+
+ return numa_node_count;
+}
diff --git a/collectors/proc.plugin/plugin_proc.h b/collectors/proc.plugin/plugin_proc.h
new file mode 100644
index 0000000..d67ccd6
--- /dev/null
+++ b/collectors/proc.plugin/plugin_proc.h
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PLUGIN_PROC_H
+#define NETDATA_PLUGIN_PROC_H 1
+
+#include "daemon/common.h"
+
+#define PLUGIN_PROC_CONFIG_NAME "proc"
+#define PLUGIN_PROC_NAME PLUGIN_PROC_CONFIG_NAME ".plugin"
+
+#define THREAD_NETDEV_NAME "PLUGIN[proc netdev]"
+void *netdev_main(void *ptr);
+
+int do_proc_net_wireless(int update_every, usec_t dt);
+int do_proc_diskstats(int update_every, usec_t dt);
+int do_proc_mdstat(int update_every, usec_t dt);
+int do_proc_net_netstat(int update_every, usec_t dt);
+int do_proc_net_stat_conntrack(int update_every, usec_t dt);
+int do_proc_net_ip_vs_stats(int update_every, usec_t dt);
+int do_proc_stat(int update_every, usec_t dt);
+int do_proc_meminfo(int update_every, usec_t dt);
+int do_proc_vmstat(int update_every, usec_t dt);
+int do_proc_net_rpc_nfs(int update_every, usec_t dt);
+int do_proc_net_rpc_nfsd(int update_every, usec_t dt);
+int do_proc_sys_kernel_random_entropy_avail(int update_every, usec_t dt);
+int do_proc_interrupts(int update_every, usec_t dt);
+int do_proc_softirqs(int update_every, usec_t dt);
+int do_proc_pressure(int update_every, usec_t dt);
+int do_sys_kernel_mm_ksm(int update_every, usec_t dt);
+int do_sys_block_zram(int update_every, usec_t dt);
+int do_proc_loadavg(int update_every, usec_t dt);
+int do_proc_net_stat_synproxy(int update_every, usec_t dt);
+int do_proc_net_softnet_stat(int update_every, usec_t dt);
+int do_proc_uptime(int update_every, usec_t dt);
+int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt);
+int do_proc_sys_devices_system_node(int update_every, usec_t dt);
+int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt);
+int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt);
+int do_sys_fs_btrfs(int update_every, usec_t dt);
+int do_proc_net_sockstat(int update_every, usec_t dt);
+int do_proc_net_sockstat6(int update_every, usec_t dt);
+int do_proc_net_sctp_snmp(int update_every, usec_t dt);
+int do_ipc(int update_every, usec_t dt);
+int do_sys_class_power_supply(int update_every, usec_t dt);
+int do_proc_pagetypeinfo(int update_every, usec_t dt);
+int do_sys_class_infiniband(int update_every, usec_t dt);
+int get_numa_node_count(void);
+
+// metrics that need to be shared among data collectors
+extern unsigned long long zfs_arcstats_shrinkable_cache_size_bytes;
+
+// netdev renames
+void netdev_rename_device_add(
+ const char *host_device,
+ const char *container_device,
+ const char *container_name,
+ DICTIONARY *labels,
+ const char *ctx_prefix);
+
+void netdev_rename_device_del(const char *host_device);
+
+#include "proc_self_mountinfo.h"
+#include "proc_pressure.h"
+#include "zfs_common.h"
+
+#endif /* NETDATA_PLUGIN_PROC_H */
diff --git a/collectors/proc.plugin/proc_diskstats.c b/collectors/proc.plugin/proc_diskstats.c
new file mode 100644
index 0000000..28d0e75
--- /dev/null
+++ b/collectors/proc.plugin/proc_diskstats.c
@@ -0,0 +1,2045 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define RRD_TYPE_DISK "disk"
+#define PLUGIN_PROC_MODULE_DISKSTATS_NAME "/proc/diskstats"
+#define CONFIG_SECTION_PLUGIN_PROC_DISKSTATS "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_DISKSTATS_NAME
+
+#define DISK_TYPE_UNKNOWN 0
+#define DISK_TYPE_PHYSICAL 1
+#define DISK_TYPE_PARTITION 2
+#define DISK_TYPE_VIRTUAL 3
+
+#define DEFAULT_PREFERRED_IDS "*"
+#define DEFAULT_EXCLUDED_DISKS "loop* ram*"
+
+static struct disk {
+ char *disk; // the name of the disk (sda, sdb, etc, after being looked up)
+ char *device; // the device of the disk (before being looked up)
+ uint32_t hash;
+ unsigned long major;
+ unsigned long minor;
+ int sector_size;
+ int type;
+
+ char *mount_point;
+
+ char *chart_id;
+
+ // disk options caching
+ int do_io;
+ int do_ops;
+ int do_mops;
+ int do_iotime;
+ int do_qops;
+ int do_util;
+ int do_ext;
+ int do_backlog;
+ int do_bcache;
+
+ int updated;
+
+ int device_is_bcache;
+
+ char *bcache_filename_dirty_data;
+ char *bcache_filename_writeback_rate;
+ char *bcache_filename_cache_congested;
+ char *bcache_filename_cache_available_percent;
+ char *bcache_filename_stats_five_minute_cache_hit_ratio;
+ char *bcache_filename_stats_hour_cache_hit_ratio;
+ char *bcache_filename_stats_day_cache_hit_ratio;
+ char *bcache_filename_stats_total_cache_hit_ratio;
+ char *bcache_filename_stats_total_cache_hits;
+ char *bcache_filename_stats_total_cache_misses;
+ char *bcache_filename_stats_total_cache_miss_collisions;
+ char *bcache_filename_stats_total_cache_bypass_hits;
+ char *bcache_filename_stats_total_cache_bypass_misses;
+ char *bcache_filename_stats_total_cache_readaheads;
+ char *bcache_filename_cache_read_races;
+ char *bcache_filename_cache_io_errors;
+ char *bcache_filename_priority_stats;
+
+ usec_t bcache_priority_stats_update_every_usec;
+ usec_t bcache_priority_stats_elapsed_usec;
+
+ RRDSET *st_io;
+ RRDDIM *rd_io_reads;
+ RRDDIM *rd_io_writes;
+
+ RRDSET *st_ext_io;
+ RRDDIM *rd_io_discards;
+
+ RRDSET *st_ops;
+ RRDDIM *rd_ops_reads;
+ RRDDIM *rd_ops_writes;
+
+ RRDSET *st_ext_ops;
+ RRDDIM *rd_ops_discards;
+ RRDDIM *rd_ops_flushes;
+
+ RRDSET *st_qops;
+ RRDDIM *rd_qops_operations;
+
+ RRDSET *st_backlog;
+ RRDDIM *rd_backlog_backlog;
+
+ RRDSET *st_busy;
+ RRDDIM *rd_busy_busy;
+
+ RRDSET *st_util;
+ RRDDIM *rd_util_utilization;
+
+ RRDSET *st_mops;
+ RRDDIM *rd_mops_reads;
+ RRDDIM *rd_mops_writes;
+
+ RRDSET *st_ext_mops;
+ RRDDIM *rd_mops_discards;
+
+ RRDSET *st_iotime;
+ RRDDIM *rd_iotime_reads;
+ RRDDIM *rd_iotime_writes;
+
+ RRDSET *st_ext_iotime;
+ RRDDIM *rd_iotime_discards;
+ RRDDIM *rd_iotime_flushes;
+
+ RRDSET *st_await;
+ RRDDIM *rd_await_reads;
+ RRDDIM *rd_await_writes;
+
+ RRDSET *st_ext_await;
+ RRDDIM *rd_await_discards;
+ RRDDIM *rd_await_flushes;
+
+ RRDSET *st_avgsz;
+ RRDDIM *rd_avgsz_reads;
+ RRDDIM *rd_avgsz_writes;
+
+ RRDSET *st_ext_avgsz;
+ RRDDIM *rd_avgsz_discards;
+
+ RRDSET *st_svctm;
+ RRDDIM *rd_svctm_svctm;
+
+ RRDSET *st_bcache_size;
+ RRDDIM *rd_bcache_dirty_size;
+
+ RRDSET *st_bcache_usage;
+ RRDDIM *rd_bcache_available_percent;
+
+ RRDSET *st_bcache_hit_ratio;
+ RRDDIM *rd_bcache_hit_ratio_5min;
+ RRDDIM *rd_bcache_hit_ratio_1hour;
+ RRDDIM *rd_bcache_hit_ratio_1day;
+ RRDDIM *rd_bcache_hit_ratio_total;
+
+ RRDSET *st_bcache;
+ RRDDIM *rd_bcache_hits;
+ RRDDIM *rd_bcache_misses;
+ RRDDIM *rd_bcache_miss_collisions;
+
+ RRDSET *st_bcache_bypass;
+ RRDDIM *rd_bcache_bypass_hits;
+ RRDDIM *rd_bcache_bypass_misses;
+
+ RRDSET *st_bcache_rates;
+ RRDDIM *rd_bcache_rate_congested;
+ RRDDIM *rd_bcache_readaheads;
+ RRDDIM *rd_bcache_rate_writeback;
+
+ RRDSET *st_bcache_cache_allocations;
+ RRDDIM *rd_bcache_cache_allocations_unused;
+ RRDDIM *rd_bcache_cache_allocations_clean;
+ RRDDIM *rd_bcache_cache_allocations_dirty;
+ RRDDIM *rd_bcache_cache_allocations_metadata;
+ RRDDIM *rd_bcache_cache_allocations_unknown;
+
+ RRDSET *st_bcache_cache_read_races;
+ RRDDIM *rd_bcache_cache_read_races;
+ RRDDIM *rd_bcache_cache_io_errors;
+
+ struct disk *next;
+} *disk_root = NULL;
+
+#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); (st) = NULL; } } while(st)
+
+// static char *path_to_get_hw_sector_size = NULL;
+// static char *path_to_get_hw_sector_size_partitions = NULL;
+static char *path_to_sys_dev_block_major_minor_string = NULL;
+static char *path_to_sys_block_device = NULL;
+static char *path_to_sys_block_device_bcache = NULL;
+static char *path_to_sys_devices_virtual_block_device = NULL;
+static char *path_to_device_mapper = NULL;
+static char *path_to_device_label = NULL;
+static char *path_to_device_id = NULL;
+static char *path_to_veritas_volume_groups = NULL;
+static int name_disks_by_id = CONFIG_BOOLEAN_NO;
+static int global_bcache_priority_stats_update_every = 0; // disabled by default
+
+static int global_enable_new_disks_detected_at_runtime = CONFIG_BOOLEAN_YES,
+ global_enable_performance_for_physical_disks = CONFIG_BOOLEAN_AUTO,
+ global_enable_performance_for_virtual_disks = CONFIG_BOOLEAN_AUTO,
+ global_enable_performance_for_partitions = CONFIG_BOOLEAN_NO,
+ global_do_io = CONFIG_BOOLEAN_AUTO,
+ global_do_ops = CONFIG_BOOLEAN_AUTO,
+ global_do_mops = CONFIG_BOOLEAN_AUTO,
+ global_do_iotime = CONFIG_BOOLEAN_AUTO,
+ global_do_qops = CONFIG_BOOLEAN_AUTO,
+ global_do_util = CONFIG_BOOLEAN_AUTO,
+ global_do_ext = CONFIG_BOOLEAN_AUTO,
+ global_do_backlog = CONFIG_BOOLEAN_AUTO,
+ global_do_bcache = CONFIG_BOOLEAN_AUTO,
+ globals_initialized = 0,
+ global_cleanup_removed_disks = 1;
+
+static SIMPLE_PATTERN *preferred_ids = NULL;
+static SIMPLE_PATTERN *excluded_disks = NULL;
+
+static unsigned long long int bcache_read_number_with_units(const char *filename) {
+ char buffer[50 + 1];
+ if(read_file(filename, buffer, 50) == 0) {
+ static int unknown_units_error = 10;
+
+ char *end = NULL;
+ NETDATA_DOUBLE value = str2ndd(buffer, &end);
+ if(end && *end) {
+ if(*end == 'k')
+ return (unsigned long long int)(value * 1024.0);
+ else if(*end == 'M')
+ return (unsigned long long int)(value * 1024.0 * 1024.0);
+ else if(*end == 'G')
+ return (unsigned long long int)(value * 1024.0 * 1024.0 * 1024.0);
+ else if(*end == 'T')
+ return (unsigned long long int)(value * 1024.0 * 1024.0 * 1024.0 * 1024.0);
+ else if(unknown_units_error > 0) {
+ error("bcache file '%s' provides value '%s' with unknown units '%s'", filename, buffer, end);
+ unknown_units_error--;
+ }
+ }
+
+ return (unsigned long long int)value;
+ }
+
+ return 0;
+}
+
+void bcache_read_priority_stats(struct disk *d, const char *family, int update_every, usec_t dt) {
+ static procfile *ff = NULL;
+ static char *separators = " \t:%[]";
+
+ static ARL_BASE *arl_base = NULL;
+
+ static unsigned long long unused;
+ static unsigned long long clean;
+ static unsigned long long dirty;
+ static unsigned long long metadata;
+ static unsigned long long unknown;
+
+ // check if it is time to update this metric
+ d->bcache_priority_stats_elapsed_usec += dt;
+ if(likely(d->bcache_priority_stats_elapsed_usec < d->bcache_priority_stats_update_every_usec)) return;
+ d->bcache_priority_stats_elapsed_usec = 0;
+
+ // initialize ARL
+ if(unlikely(!arl_base)) {
+ arl_base = arl_create("bcache/priority_stats", NULL, 60);
+ arl_expect(arl_base, "Unused", &unused);
+ arl_expect(arl_base, "Clean", &clean);
+ arl_expect(arl_base, "Dirty", &dirty);
+ arl_expect(arl_base, "Metadata", &metadata);
+ }
+
+ ff = procfile_reopen(ff, d->bcache_filename_priority_stats, separators, PROCFILE_FLAG_DEFAULT);
+ if(likely(ff)) ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ separators = " \t:%[]";
+ return;
+ }
+
+ // do not reset the separators on every iteration
+ separators = NULL;
+
+ arl_begin(arl_base);
+ unused = clean = dirty = metadata = unknown = 0;
+
+ size_t lines = procfile_lines(ff), l;
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 2)) {
+ if(unlikely(words)) error("Cannot read '%s' line %zu. Expected 2 params, read %zu.", d->bcache_filename_priority_stats, l, words);
+ continue;
+ }
+
+ if(unlikely(arl_check(arl_base,
+ procfile_lineword(ff, l, 0),
+ procfile_lineword(ff, l, 1)))) break;
+ }
+
+ unknown = 100 - unused - clean - dirty - metadata;
+
+ // create / update the cache allocations chart
+ {
+ if(unlikely(!d->st_bcache_cache_allocations)) {
+ d->st_bcache_cache_allocations = rrdset_create_localhost(
+ "disk_bcache_cache_alloc"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.bcache_cache_alloc"
+ , "BCache Cache Allocations"
+ , "percentage"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_BCACHE_CACHE_ALLOC
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ d->rd_bcache_cache_allocations_unused = rrddim_add(d->st_bcache_cache_allocations, "unused", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_bcache_cache_allocations_dirty = rrddim_add(d->st_bcache_cache_allocations, "dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_bcache_cache_allocations_clean = rrddim_add(d->st_bcache_cache_allocations, "clean", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_bcache_cache_allocations_metadata = rrddim_add(d->st_bcache_cache_allocations, "metadata", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_bcache_cache_allocations_unknown = rrddim_add(d->st_bcache_cache_allocations, "undefined", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ d->bcache_priority_stats_update_every_usec = update_every * USEC_PER_SEC;
+ }
+
+ rrddim_set_by_pointer(d->st_bcache_cache_allocations, d->rd_bcache_cache_allocations_unused, unused);
+ rrddim_set_by_pointer(d->st_bcache_cache_allocations, d->rd_bcache_cache_allocations_dirty, dirty);
+ rrddim_set_by_pointer(d->st_bcache_cache_allocations, d->rd_bcache_cache_allocations_clean, clean);
+ rrddim_set_by_pointer(d->st_bcache_cache_allocations, d->rd_bcache_cache_allocations_metadata, metadata);
+ rrddim_set_by_pointer(d->st_bcache_cache_allocations, d->rd_bcache_cache_allocations_unknown, unknown);
+ rrdset_done(d->st_bcache_cache_allocations);
+ }
+}
+
+static inline int is_major_enabled(int major) {
+ static int8_t *major_configs = NULL;
+ static size_t major_size = 0;
+
+ if(major < 0) return 1;
+
+ size_t wanted_size = (size_t)major + 1;
+
+ if(major_size < wanted_size) {
+ major_configs = reallocz(major_configs, wanted_size * sizeof(int8_t));
+
+ size_t i;
+ for(i = major_size; i < wanted_size ; i++)
+ major_configs[i] = -1;
+
+ major_size = wanted_size;
+ }
+
+ if(major_configs[major] == -1) {
+ char buffer[CONFIG_MAX_NAME + 1];
+ snprintfz(buffer, CONFIG_MAX_NAME, "performance metrics for disks with major %d", major);
+ major_configs[major] = (char)config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, buffer, 1);
+ }
+
+ return (int)major_configs[major];
+}
+
+static inline int get_disk_name_from_path(const char *path, char *result, size_t result_size, unsigned long major, unsigned long minor, char *disk, char *prefix, int depth) {
+ //info("DEVICE-MAPPER ('%s', %lu:%lu): examining directory '%s' (allowed depth %d).", disk, major, minor, path, depth);
+
+ int found = 0, preferred = 0;
+
+ char *first_result = mallocz(result_size);
+
+ DIR *dir = opendir(path);
+ if (!dir) {
+ error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot open directory '%s'.", disk, major, minor, path);
+ goto failed;
+ }
+
+ struct dirent *de = NULL;
+ while ((de = readdir(dir))) {
+ if(de->d_type == DT_DIR) {
+ if((de->d_name[0] == '.' && de->d_name[1] == '\0') || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0'))
+ continue;
+
+ if(depth <= 0) {
+ error("DEVICE-MAPPER ('%s', %lu:%lu): Depth limit reached for path '%s/%s'. Ignoring path.", disk, major, minor, path, de->d_name);
+ break;
+ }
+ else {
+ char *path_nested = NULL;
+ char *prefix_nested = NULL;
+
+ {
+ char buffer[FILENAME_MAX + 1];
+ snprintfz(buffer, FILENAME_MAX, "%s/%s", path, de->d_name);
+ path_nested = strdupz(buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s%s%s", (prefix)?prefix:"", (prefix)?"_":"", de->d_name);
+ prefix_nested = strdupz(buffer);
+ }
+
+ found = get_disk_name_from_path(path_nested, result, result_size, major, minor, disk, prefix_nested, depth - 1);
+ freez(path_nested);
+ freez(prefix_nested);
+
+ if(found) break;
+ }
+ }
+ else if(de->d_type == DT_LNK || de->d_type == DT_BLK) {
+ char filename[FILENAME_MAX + 1];
+
+ if(de->d_type == DT_LNK) {
+ snprintfz(filename, FILENAME_MAX, "%s/%s", path, de->d_name);
+ ssize_t len = readlink(filename, result, result_size - 1);
+ if(len <= 0) {
+ error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot read link '%s'.", disk, major, minor, filename);
+ continue;
+ }
+
+ result[len] = '\0';
+ if(result[0] != '/')
+ snprintfz(filename, FILENAME_MAX, "%s/%s", path, result);
+ else
+ strncpyz(filename, result, FILENAME_MAX);
+ }
+ else {
+ snprintfz(filename, FILENAME_MAX, "%s/%s", path, de->d_name);
+ }
+
+ struct stat sb;
+ if(stat(filename, &sb) == -1) {
+ error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot stat() file '%s'.", disk, major, minor, filename);
+ continue;
+ }
+
+ if((sb.st_mode & S_IFMT) != S_IFBLK) {
+ //info("DEVICE-MAPPER ('%s', %lu:%lu): file '%s' is not a block device.", disk, major, minor, filename);
+ continue;
+ }
+
+ if(major(sb.st_rdev) != major || minor(sb.st_rdev) != minor || strcmp(basename(filename), disk)) {
+ //info("DEVICE-MAPPER ('%s', %lu:%lu): filename '%s' does not match %lu:%lu.", disk, major, minor, filename, (unsigned long)major(sb.st_rdev), (unsigned long)minor(sb.st_rdev));
+ continue;
+ }
+
+ //info("DEVICE-MAPPER ('%s', %lu:%lu): filename '%s' matches.", disk, major, minor, filename);
+
+ snprintfz(result, result_size - 1, "%s%s%s", (prefix)?prefix:"", (prefix)?"_":"", de->d_name);
+
+ if(!found) {
+ strncpyz(first_result, result, result_size);
+ found = 1;
+ }
+
+ if(simple_pattern_matches(preferred_ids, result)) {
+ preferred = 1;
+ break;
+ }
+ }
+ }
+ closedir(dir);
+
+
+failed:
+
+ if(!found)
+ result[0] = '\0';
+ else if(!preferred)
+ strncpyz(result, first_result, result_size);
+
+ freez(first_result);
+
+ return found;
+}
+
+static inline char *get_disk_name(unsigned long major, unsigned long minor, char *disk) {
+ char result[FILENAME_MAX + 1] = "";
+
+ if(!path_to_device_mapper || !*path_to_device_mapper || !get_disk_name_from_path(path_to_device_mapper, result, FILENAME_MAX + 1, major, minor, disk, NULL, 0))
+ if(!path_to_device_label || !*path_to_device_label || !get_disk_name_from_path(path_to_device_label, result, FILENAME_MAX + 1, major, minor, disk, NULL, 0))
+ if(!path_to_veritas_volume_groups || !*path_to_veritas_volume_groups || !get_disk_name_from_path(path_to_veritas_volume_groups, result, FILENAME_MAX + 1, major, minor, disk, "vx", 2))
+ if(name_disks_by_id != CONFIG_BOOLEAN_YES || !path_to_device_id || !*path_to_device_id || !get_disk_name_from_path(path_to_device_id, result, FILENAME_MAX + 1, major, minor, disk, NULL, 0))
+ strncpy(result, disk, FILENAME_MAX);
+
+ if(!result[0])
+ strncpy(result, disk, FILENAME_MAX);
+
+ netdata_fix_chart_name(result);
+ return strdup(result);
+}
+
+static void get_disk_config(struct disk *d) {
+ int def_enable = global_enable_new_disks_detected_at_runtime;
+
+ if(def_enable != CONFIG_BOOLEAN_NO && (simple_pattern_matches(excluded_disks, d->device) || simple_pattern_matches(excluded_disks, d->disk)))
+ def_enable = CONFIG_BOOLEAN_NO;
+
+ char var_name[4096 + 1];
+ snprintfz(var_name, 4096, CONFIG_SECTION_PLUGIN_PROC_DISKSTATS ":%s", d->disk);
+
+ def_enable = config_get_boolean_ondemand(var_name, "enable", def_enable);
+ if(unlikely(def_enable == CONFIG_BOOLEAN_NO)) {
+ // the user does not want any metrics for this disk
+ d->do_io = CONFIG_BOOLEAN_NO;
+ d->do_ops = CONFIG_BOOLEAN_NO;
+ d->do_mops = CONFIG_BOOLEAN_NO;
+ d->do_iotime = CONFIG_BOOLEAN_NO;
+ d->do_qops = CONFIG_BOOLEAN_NO;
+ d->do_util = CONFIG_BOOLEAN_NO;
+ d->do_ext = CONFIG_BOOLEAN_NO;
+ d->do_backlog = CONFIG_BOOLEAN_NO;
+ d->do_bcache = CONFIG_BOOLEAN_NO;
+ }
+ else {
+ // this disk is enabled
+ // check its direct settings
+
+ int def_performance = CONFIG_BOOLEAN_AUTO;
+
+ // since this is 'on demand' we can figure the performance settings
+ // based on the type of disk
+
+ if(!d->device_is_bcache) {
+ switch(d->type) {
+ default:
+ case DISK_TYPE_UNKNOWN:
+ break;
+
+ case DISK_TYPE_PHYSICAL:
+ def_performance = global_enable_performance_for_physical_disks;
+ break;
+
+ case DISK_TYPE_PARTITION:
+ def_performance = global_enable_performance_for_partitions;
+ break;
+
+ case DISK_TYPE_VIRTUAL:
+ def_performance = global_enable_performance_for_virtual_disks;
+ break;
+ }
+ }
+
+ // check if we have to disable performance for this disk
+ if(def_performance)
+ def_performance = is_major_enabled((int)d->major);
+
+ // ------------------------------------------------------------
+ // now we have def_performance and def_space
+ // to work further
+
+ // def_performance
+ // check the user configuration (this will also show our 'on demand' decision)
+ def_performance = config_get_boolean_ondemand(var_name, "enable performance metrics", def_performance);
+
+ int ddo_io = CONFIG_BOOLEAN_NO,
+ ddo_ops = CONFIG_BOOLEAN_NO,
+ ddo_mops = CONFIG_BOOLEAN_NO,
+ ddo_iotime = CONFIG_BOOLEAN_NO,
+ ddo_qops = CONFIG_BOOLEAN_NO,
+ ddo_util = CONFIG_BOOLEAN_NO,
+ ddo_ext = CONFIG_BOOLEAN_NO,
+ ddo_backlog = CONFIG_BOOLEAN_NO,
+ ddo_bcache = CONFIG_BOOLEAN_NO;
+
+ // we enable individual performance charts only when def_performance is not disabled
+ if(unlikely(def_performance != CONFIG_BOOLEAN_NO)) {
+ ddo_io = global_do_io,
+ ddo_ops = global_do_ops,
+ ddo_mops = global_do_mops,
+ ddo_iotime = global_do_iotime,
+ ddo_qops = global_do_qops,
+ ddo_util = global_do_util,
+ ddo_ext = global_do_ext,
+ ddo_backlog = global_do_backlog,
+ ddo_bcache = global_do_bcache;
+ }
+
+ d->do_io = config_get_boolean_ondemand(var_name, "bandwidth", ddo_io);
+ d->do_ops = config_get_boolean_ondemand(var_name, "operations", ddo_ops);
+ d->do_mops = config_get_boolean_ondemand(var_name, "merged operations", ddo_mops);
+ d->do_iotime = config_get_boolean_ondemand(var_name, "i/o time", ddo_iotime);
+ d->do_qops = config_get_boolean_ondemand(var_name, "queued operations", ddo_qops);
+ d->do_util = config_get_boolean_ondemand(var_name, "utilization percentage", ddo_util);
+ d->do_ext = config_get_boolean_ondemand(var_name, "extended operations", ddo_ext);
+ d->do_backlog = config_get_boolean_ondemand(var_name, "backlog", ddo_backlog);
+
+ if(d->device_is_bcache)
+ d->do_bcache = config_get_boolean_ondemand(var_name, "bcache", ddo_bcache);
+ else
+ d->do_bcache = 0;
+ }
+}
+
+static struct disk *get_disk(unsigned long major, unsigned long minor, char *disk) {
+ static struct mountinfo *disk_mountinfo_root = NULL;
+
+ struct disk *d;
+
+ uint32_t hash = simple_hash(disk);
+
+ // search for it in our RAM list.
+ // this is sequential, but since we just walk through
+ // and the number of disks / partitions in a system
+ // should not be that many, it should be acceptable
+ for(d = disk_root; d ; d = d->next){
+ if (unlikely(
+ d->major == major && d->minor == minor && d->hash == hash && !strcmp(d->device, disk)))
+ return d;
+ }
+
+ // not found
+ // create a new disk structure
+ d = (struct disk *)callocz(1, sizeof(struct disk));
+
+ d->disk = get_disk_name(major, minor, disk);
+ d->device = strdupz(disk);
+ d->hash = simple_hash(d->device);
+ d->major = major;
+ d->minor = minor;
+ d->type = DISK_TYPE_UNKNOWN; // Default type. Changed later if not correct.
+ d->sector_size = 512; // the default, will be changed below
+ d->next = NULL;
+
+ // append it to the list
+ if(unlikely(!disk_root))
+ disk_root = d;
+ else {
+ struct disk *last;
+ for(last = disk_root; last->next ;last = last->next);
+ last->next = d;
+ }
+
+ d->chart_id = strdupz(d->device);
+
+ // read device uuid if it is an LVM volume
+ if (!strncmp(d->device, "dm-", 3)) {
+ char uuid_filename[FILENAME_MAX + 1];
+ snprintfz(uuid_filename, FILENAME_MAX, path_to_sys_devices_virtual_block_device, disk);
+ strncat(uuid_filename, "/dm/uuid", FILENAME_MAX);
+
+ char device_uuid[RRD_ID_LENGTH_MAX + 1];
+ if (!read_file(uuid_filename, device_uuid, RRD_ID_LENGTH_MAX) && !strncmp(device_uuid, "LVM-", 4)) {
+ trim(device_uuid);
+
+ char chart_id[RRD_ID_LENGTH_MAX + 1];
+ snprintf(chart_id, RRD_ID_LENGTH_MAX, "%s-%s", d->device, device_uuid + 4);
+
+ freez(d->chart_id);
+ d->chart_id = strdupz(chart_id);
+ }
+ }
+
+ char buffer[FILENAME_MAX + 1];
+
+ // find if it is a physical disk
+ // by checking if /sys/block/DISK is readable.
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_block_device, disk);
+ if(likely(access(buffer, R_OK) == 0)) {
+ // assign it here, but it will be overwritten if it is not a physical disk
+ d->type = DISK_TYPE_PHYSICAL;
+ }
+
+ // find if it is a partition
+ // by checking if /sys/dev/block/MAJOR:MINOR/partition is readable.
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_dev_block_major_minor_string, major, minor, "partition");
+ if(likely(access(buffer, R_OK) == 0)) {
+ d->type = DISK_TYPE_PARTITION;
+ }
+ else {
+ // find if it is a virtual disk
+ // by checking if /sys/devices/virtual/block/DISK is readable.
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_devices_virtual_block_device, disk);
+ if(likely(access(buffer, R_OK) == 0)) {
+ d->type = DISK_TYPE_VIRTUAL;
+ }
+ else {
+ // find if it is a virtual device
+ // by checking if /sys/dev/block/MAJOR:MINOR/slaves has entries
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_dev_block_major_minor_string, major, minor, "slaves/");
+ DIR *dirp = opendir(buffer);
+ if (likely(dirp != NULL)) {
+ struct dirent *dp;
+ while ((dp = readdir(dirp))) {
+ // . and .. are also files in empty folders.
+ if (unlikely(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0)) {
+ continue;
+ }
+
+ d->type = DISK_TYPE_VIRTUAL;
+
+ // Stop the loop after we found one file.
+ break;
+ }
+ if (unlikely(closedir(dirp) == -1))
+ error("Unable to close dir %s", buffer);
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // check if we can find its mount point
+
+ // mountinfo_find() can be called with NULL disk_mountinfo_root
+ struct mountinfo *mi = mountinfo_find(disk_mountinfo_root, d->major, d->minor, d->device);
+ if(unlikely(!mi)) {
+ // mountinfo_free_all can be called with NULL
+ mountinfo_free_all(disk_mountinfo_root);
+ disk_mountinfo_root = mountinfo_read(0);
+ mi = mountinfo_find(disk_mountinfo_root, d->major, d->minor, d->device);
+ }
+
+ if(unlikely(mi))
+ d->mount_point = strdupz(mi->mount_point);
+ else
+ d->mount_point = NULL;
+
+ // ------------------------------------------------------------------------
+ // find the disk sector size
+
+ /*
+ * sector size is always 512 bytes inside the kernel #3481
+ *
+ {
+ char tf[FILENAME_MAX + 1], *t;
+ strncpyz(tf, d->device, FILENAME_MAX);
+
+ // replace all / with !
+ for(t = tf; *t ;t++)
+ if(unlikely(*t == '/')) *t = '!';
+
+ if(likely(d->type == DISK_TYPE_PARTITION))
+ snprintfz(buffer, FILENAME_MAX, path_to_get_hw_sector_size_partitions, d->major, d->minor, tf);
+ else
+ snprintfz(buffer, FILENAME_MAX, path_to_get_hw_sector_size, tf);
+
+ FILE *fpss = fopen(buffer, "r");
+ if(likely(fpss)) {
+ char buffer2[1024 + 1];
+ char *tmp = fgets(buffer2, 1024, fpss);
+
+ if(likely(tmp)) {
+ d->sector_size = str2i(tmp);
+ if(unlikely(d->sector_size <= 0)) {
+ error("Invalid sector size %d for device %s in %s. Assuming 512.", d->sector_size, d->device, buffer);
+ d->sector_size = 512;
+ }
+ }
+ else error("Cannot read data for sector size for device %s from %s. Assuming 512.", d->device, buffer);
+
+ fclose(fpss);
+ }
+ else error("Cannot read sector size for device %s from %s. Assuming 512.", d->device, buffer);
+ }
+ */
+
+ // ------------------------------------------------------------------------
+ // check if the device is a bcache
+
+ struct stat bcache;
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_block_device_bcache, disk);
+ if(unlikely(stat(buffer, &bcache) == 0 && (bcache.st_mode & S_IFMT) == S_IFDIR)) {
+ // we have the 'bcache' directory
+ d->device_is_bcache = 1;
+
+ char buffer2[FILENAME_MAX + 1];
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/cache/congested", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_cache_congested = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/readahead", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_total_cache_readaheads = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/cache/cache0/priority_stats", buffer); // only one cache is supported by bcache
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_priority_stats = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/cache/internal/cache_read_races", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_cache_read_races = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/cache/cache0/io_errors", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_cache_io_errors = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/dirty_data", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_dirty_data = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/writeback_rate", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_writeback_rate = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/cache/cache_available_percent", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_cache_available_percent = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_total/cache_hits", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_total_cache_hits = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_five_minute/cache_hit_ratio", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_five_minute_cache_hit_ratio = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_hour/cache_hit_ratio", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_hour_cache_hit_ratio = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_day/cache_hit_ratio", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_day_cache_hit_ratio = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_total/cache_hit_ratio", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_total_cache_hit_ratio = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_total/cache_misses", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_total_cache_misses = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_total/cache_bypass_hits", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_total_cache_bypass_hits = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_total/cache_bypass_misses", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_total_cache_bypass_misses = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+
+ snprintfz(buffer2, FILENAME_MAX, "%s/stats_total/cache_miss_collisions", buffer);
+ if(access(buffer2, R_OK) == 0)
+ d->bcache_filename_stats_total_cache_miss_collisions = strdupz(buffer2);
+ else
+ error("bcache file '%s' cannot be read.", buffer2);
+ }
+
+ get_disk_config(d);
+ return d;
+}
+
+static void add_labels_to_disk(struct disk *d, RRDSET *st) {
+ rrdlabels_add(st->rrdlabels, "device", d->disk, RRDLABEL_SRC_AUTO);
+ rrdlabels_add(st->rrdlabels, "mount_point", d->mount_point, RRDLABEL_SRC_AUTO);
+
+ switch (d->type) {
+ default:
+ case DISK_TYPE_UNKNOWN:
+ rrdlabels_add(st->rrdlabels, "device_type", "unknown", RRDLABEL_SRC_AUTO);
+ break;
+
+ case DISK_TYPE_PHYSICAL:
+ rrdlabels_add(st->rrdlabels, "device_type", "physical", RRDLABEL_SRC_AUTO);
+ break;
+
+ case DISK_TYPE_PARTITION:
+ rrdlabels_add(st->rrdlabels, "device_type", "partition", RRDLABEL_SRC_AUTO);
+ break;
+
+ case DISK_TYPE_VIRTUAL:
+ rrdlabels_add(st->rrdlabels, "device_type", "virtual", RRDLABEL_SRC_AUTO);
+ break;
+ }
+}
+
+int do_proc_diskstats(int update_every, usec_t dt) {
+ static procfile *ff = NULL;
+
+ if(unlikely(!globals_initialized)) {
+ globals_initialized = 1;
+
+ global_enable_new_disks_detected_at_runtime = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "enable new disks detected at runtime", global_enable_new_disks_detected_at_runtime);
+ global_enable_performance_for_physical_disks = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "performance metrics for physical disks", global_enable_performance_for_physical_disks);
+ global_enable_performance_for_virtual_disks = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "performance metrics for virtual disks", global_enable_performance_for_virtual_disks);
+ global_enable_performance_for_partitions = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "performance metrics for partitions", global_enable_performance_for_partitions);
+
+ global_do_io = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "bandwidth for all disks", global_do_io);
+ global_do_ops = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "operations for all disks", global_do_ops);
+ global_do_mops = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "merged operations for all disks", global_do_mops);
+ global_do_iotime = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "i/o time for all disks", global_do_iotime);
+ global_do_qops = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "queued operations for all disks", global_do_qops);
+ global_do_util = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "utilization percentage for all disks", global_do_util);
+ global_do_ext = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "extended operations for all disks", global_do_ext);
+ global_do_backlog = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "backlog for all disks", global_do_backlog);
+ global_do_bcache = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "bcache for all disks", global_do_bcache);
+ global_bcache_priority_stats_update_every = (int)config_get_number(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "bcache priority stats update every", global_bcache_priority_stats_update_every);
+
+ global_cleanup_removed_disks = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "remove charts of removed disks" , global_cleanup_removed_disks);
+
+ char buffer[FILENAME_MAX + 1];
+
+ snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s");
+ path_to_sys_block_device = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to get block device", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/bcache");
+ path_to_sys_block_device_bcache = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to get block device bcache", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/virtual/block/%s");
+ path_to_sys_devices_virtual_block_device = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to get virtual block device", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/%s");
+ path_to_sys_dev_block_major_minor_string = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to get block device infos", buffer);
+
+ //snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/queue/hw_sector_size");
+ //path_to_get_hw_sector_size = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to get h/w sector size", buffer);
+
+ //snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/subsystem/%s/../queue/hw_sector_size");
+ //path_to_get_hw_sector_size_partitions = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to get h/w sector size for partitions", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s/dev/mapper", netdata_configured_host_prefix);
+ path_to_device_mapper = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to device mapper", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s/dev/disk/by-label", netdata_configured_host_prefix);
+ path_to_device_label = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to /dev/disk/by-label", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s/dev/disk/by-id", netdata_configured_host_prefix);
+ path_to_device_id = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to /dev/disk/by-id", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s/dev/vx/dsk", netdata_configured_host_prefix);
+ path_to_veritas_volume_groups = config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "path to /dev/vx/dsk", buffer);
+
+ name_disks_by_id = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "name disks by id", name_disks_by_id);
+
+ preferred_ids = simple_pattern_create(
+ config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "preferred disk ids", DEFAULT_PREFERRED_IDS)
+ , NULL
+ , SIMPLE_PATTERN_EXACT
+ );
+
+ excluded_disks = simple_pattern_create(
+ config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "exclude disks", DEFAULT_EXCLUDED_DISKS)
+ , NULL
+ , SIMPLE_PATTERN_EXACT
+ );
+ }
+
+ // --------------------------------------------------------------------------
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/diskstats");
+ ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT);
+ }
+ if(unlikely(!ff)) return 0;
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ collected_number system_read_kb = 0, system_write_kb = 0;
+
+ int do_dc_stats = 0, do_fl_stats = 0;
+
+ for(l = 0; l < lines ;l++) {
+ // --------------------------------------------------------------------------
+ // Read parameters
+
+ char *disk;
+ unsigned long major = 0, minor = 0;
+
+ collected_number reads = 0, mreads = 0, readsectors = 0, readms = 0,
+ writes = 0, mwrites = 0, writesectors = 0, writems = 0,
+ queued_ios = 0, busy_ms = 0, backlog_ms = 0,
+ discards = 0, mdiscards = 0, discardsectors = 0, discardms = 0,
+ flushes = 0, flushms = 0;
+
+
+ collected_number last_reads = 0, last_readsectors = 0, last_readms = 0,
+ last_writes = 0, last_writesectors = 0, last_writems = 0,
+ last_busy_ms = 0,
+ last_discards = 0, last_discardsectors = 0, last_discardms = 0,
+ last_flushes = 0, last_flushms = 0;
+
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 14)) continue;
+
+ major = str2ul(procfile_lineword(ff, l, 0));
+ minor = str2ul(procfile_lineword(ff, l, 1));
+ disk = procfile_lineword(ff, l, 2);
+
+ // # of reads completed # of writes completed
+ // This is the total number of reads or writes completed successfully.
+ reads = str2ull(procfile_lineword(ff, l, 3)); // rd_ios
+ writes = str2ull(procfile_lineword(ff, l, 7)); // wr_ios
+
+ // # of reads merged # of writes merged
+ // Reads and writes which are adjacent to each other may be merged for
+ // efficiency. Thus two 4K reads may become one 8K read before it is
+ // ultimately handed to the disk, and so it will be counted (and queued)
+ mreads = str2ull(procfile_lineword(ff, l, 4)); // rd_merges_or_rd_sec
+ mwrites = str2ull(procfile_lineword(ff, l, 8)); // wr_merges
+
+ // # of sectors read # of sectors written
+ // This is the total number of sectors read or written successfully.
+ readsectors = str2ull(procfile_lineword(ff, l, 5)); // rd_sec_or_wr_ios
+ writesectors = str2ull(procfile_lineword(ff, l, 9)); // wr_sec
+
+ // # of milliseconds spent reading # of milliseconds spent writing
+ // This is the total number of milliseconds spent by all reads or writes (as
+ // measured from __make_request() to end_that_request_last()).
+ readms = str2ull(procfile_lineword(ff, l, 6)); // rd_ticks_or_wr_sec
+ writems = str2ull(procfile_lineword(ff, l, 10)); // wr_ticks
+
+ // # of I/Os currently in progress
+ // The only field that should go to zero. Incremented as requests are
+ // given to appropriate struct request_queue and decremented as they finish.
+ queued_ios = str2ull(procfile_lineword(ff, l, 11)); // ios_pgr
+
+ // # of milliseconds spent doing I/Os
+ // This field increases so long as field queued_ios is nonzero.
+ busy_ms = str2ull(procfile_lineword(ff, l, 12)); // tot_ticks
+
+ // weighted # of milliseconds spent doing I/Os
+ // This field is incremented at each I/O start, I/O completion, I/O
+ // merge, or read of these stats by the number of I/Os in progress
+ // (field queued_ios) times the number of milliseconds spent doing I/O since the
+ // last update of this field. This can provide an easy measure of both
+ // I/O completion time and the backlog that may be accumulating.
+ backlog_ms = str2ull(procfile_lineword(ff, l, 13)); // rq_ticks
+
+ if (unlikely(words > 13)) {
+ do_dc_stats = 1;
+
+ // # of discards completed
+ // This is the total number of discards completed successfully.
+ discards = str2ull(procfile_lineword(ff, l, 14)); // dc_ios
+
+ // # of discards merged
+ // See the description of mreads/mwrites
+ mdiscards = str2ull(procfile_lineword(ff, l, 15)); // dc_merges
+
+ // # of sectors discarded
+ // This is the total number of sectors discarded successfully.
+ discardsectors = str2ull(procfile_lineword(ff, l, 16)); // dc_sec
+
+ // # of milliseconds spent discarding
+ // This is the total number of milliseconds spent by all discards (as
+ // measured from __make_request() to end_that_request_last()).
+ discardms = str2ull(procfile_lineword(ff, l, 17)); // dc_ticks
+ }
+
+ if (unlikely(words > 17)) {
+ do_fl_stats = 1;
+
+ // number of flush I/Os processed
+ // These values increment when an flush I/O request completes.
+ // Block layer combines flush requests and executes at most one at a time.
+ // This counts flush requests executed by disk. Not tracked for partitions.
+ flushes = str2ull(procfile_lineword(ff, l, 18)); // fl_ios
+
+ // total wait time for flush requests
+ flushms = str2ull(procfile_lineword(ff, l, 19)); // fl_ticks
+ }
+
+ // --------------------------------------------------------------------------
+ // get a disk structure for the disk
+
+ struct disk *d = get_disk(major, minor, disk);
+ d->updated = 1;
+
+ // --------------------------------------------------------------------------
+ // count the global system disk I/O of physical disks
+
+ if(unlikely(d->type == DISK_TYPE_PHYSICAL)) {
+ system_read_kb += readsectors * d->sector_size / 1024;
+ system_write_kb += writesectors * d->sector_size / 1024;
+ }
+
+ // --------------------------------------------------------------------------
+ // Set its family based on mount point
+
+ char *family = d->mount_point;
+ if(!family) family = d->disk;
+
+
+ // --------------------------------------------------------------------------
+ // Do performance metrics
+
+ if(d->do_io == CONFIG_BOOLEAN_YES || (d->do_io == CONFIG_BOOLEAN_AUTO &&
+ (readsectors || writesectors || discardsectors ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->do_io = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_io)) {
+ d->st_io = rrdset_create_localhost(
+ RRD_TYPE_DISK
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.io"
+ , "Disk I/O Bandwidth"
+ , "KiB/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_IO
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ d->rd_io_reads = rrddim_add(d->st_io, "reads", NULL, d->sector_size, 1024, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_io_writes = rrddim_add(d->st_io, "writes", NULL, d->sector_size * -1, 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_io);
+ }
+
+ last_readsectors = rrddim_set_by_pointer(d->st_io, d->rd_io_reads, readsectors);
+ last_writesectors = rrddim_set_by_pointer(d->st_io, d->rd_io_writes, writesectors);
+ rrdset_done(d->st_io);
+ }
+
+ if (do_dc_stats && d->do_io == CONFIG_BOOLEAN_YES && d->do_ext != CONFIG_BOOLEAN_NO) {
+ if (unlikely(!d->st_ext_io)) {
+ d->st_ext_io = rrdset_create_localhost(
+ "disk_ext"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk_ext.io"
+ , "Amount of Discarded Data"
+ , "KiB/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_IO + 1
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ d->rd_io_discards = rrddim_add(d->st_ext_io, "discards", NULL, d->sector_size, 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_ext_io);
+ }
+
+ last_discardsectors = rrddim_set_by_pointer(d->st_ext_io, d->rd_io_discards, discardsectors);
+ rrdset_done(d->st_ext_io);
+ }
+
+ if(d->do_ops == CONFIG_BOOLEAN_YES || (d->do_ops == CONFIG_BOOLEAN_AUTO &&
+ (reads || writes || discards || flushes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->do_ops = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_ops)) {
+ d->st_ops = rrdset_create_localhost(
+ "disk_ops"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.ops"
+ , "Disk Completed I/O Operations"
+ , "operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_OPS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_ops, RRDSET_FLAG_DETAIL);
+
+ d->rd_ops_reads = rrddim_add(d->st_ops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_ops_writes = rrddim_add(d->st_ops, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_ops);
+ }
+
+ last_reads = rrddim_set_by_pointer(d->st_ops, d->rd_ops_reads, reads);
+ last_writes = rrddim_set_by_pointer(d->st_ops, d->rd_ops_writes, writes);
+ rrdset_done(d->st_ops);
+ }
+
+ if (do_dc_stats && d->do_ops == CONFIG_BOOLEAN_YES && d->do_ext != CONFIG_BOOLEAN_NO) {
+ if (unlikely(!d->st_ext_ops)) {
+ d->st_ext_ops = rrdset_create_localhost(
+ "disk_ext_ops"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk_ext.ops"
+ , "Disk Completed Extended I/O Operations"
+ , "operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_OPS + 1
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_ext_ops, RRDSET_FLAG_DETAIL);
+
+ d->rd_ops_discards = rrddim_add(d->st_ext_ops, "discards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ if (do_fl_stats)
+ d->rd_ops_flushes = rrddim_add(d->st_ext_ops, "flushes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_ext_ops);
+ }
+
+ last_discards = rrddim_set_by_pointer(d->st_ext_ops, d->rd_ops_discards, discards);
+ if (do_fl_stats)
+ last_flushes = rrddim_set_by_pointer(d->st_ext_ops, d->rd_ops_flushes, flushes);
+ rrdset_done(d->st_ext_ops);
+ }
+
+ if(d->do_qops == CONFIG_BOOLEAN_YES || (d->do_qops == CONFIG_BOOLEAN_AUTO &&
+ (queued_ios || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->do_qops = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_qops)) {
+ d->st_qops = rrdset_create_localhost(
+ "disk_qops"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.qops"
+ , "Disk Current I/O Operations"
+ , "operations"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_QOPS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_qops, RRDSET_FLAG_DETAIL);
+
+ d->rd_qops_operations = rrddim_add(d->st_qops, "operations", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_qops);
+ }
+
+ rrddim_set_by_pointer(d->st_qops, d->rd_qops_operations, queued_ios);
+ rrdset_done(d->st_qops);
+ }
+
+ if(d->do_backlog == CONFIG_BOOLEAN_YES || (d->do_backlog == CONFIG_BOOLEAN_AUTO &&
+ (backlog_ms || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->do_backlog = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_backlog)) {
+ d->st_backlog = rrdset_create_localhost(
+ "disk_backlog"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.backlog"
+ , "Disk Backlog"
+ , "milliseconds"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_BACKLOG
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(d->st_backlog, RRDSET_FLAG_DETAIL);
+
+ d->rd_backlog_backlog = rrddim_add(d->st_backlog, "backlog", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_backlog);
+ }
+
+ rrddim_set_by_pointer(d->st_backlog, d->rd_backlog_backlog, backlog_ms);
+ rrdset_done(d->st_backlog);
+ }
+
+ if(d->do_util == CONFIG_BOOLEAN_YES || (d->do_util == CONFIG_BOOLEAN_AUTO &&
+ (busy_ms || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->do_util = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_busy)) {
+ d->st_busy = rrdset_create_localhost(
+ "disk_busy"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.busy"
+ , "Disk Busy Time"
+ , "milliseconds"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_BUSY
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(d->st_busy, RRDSET_FLAG_DETAIL);
+
+ d->rd_busy_busy = rrddim_add(d->st_busy, "busy", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_busy);
+ }
+
+ last_busy_ms = rrddim_set_by_pointer(d->st_busy, d->rd_busy_busy, busy_ms);
+ rrdset_done(d->st_busy);
+
+ if(unlikely(!d->st_util)) {
+ d->st_util = rrdset_create_localhost(
+ "disk_util"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.util"
+ , "Disk Utilization Time"
+ , "% of time working"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_UTIL
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(d->st_util, RRDSET_FLAG_DETAIL);
+
+ d->rd_util_utilization = rrddim_add(d->st_util, "utilization", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_util);
+ }
+
+ collected_number disk_utilization = (busy_ms - last_busy_ms) / (10 * update_every);
+ if (disk_utilization > 100)
+ disk_utilization = 100;
+
+ rrddim_set_by_pointer(d->st_util, d->rd_util_utilization, disk_utilization);
+ rrdset_done(d->st_util);
+ }
+
+ if(d->do_mops == CONFIG_BOOLEAN_YES || (d->do_mops == CONFIG_BOOLEAN_AUTO &&
+ (mreads || mwrites || mdiscards ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->do_mops = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_mops)) {
+ d->st_mops = rrdset_create_localhost(
+ "disk_mops"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.mops"
+ , "Disk Merged Operations"
+ , "merged operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_MOPS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_mops, RRDSET_FLAG_DETAIL);
+
+ d->rd_mops_reads = rrddim_add(d->st_mops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_mops_writes = rrddim_add(d->st_mops, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_mops);
+ }
+
+ rrddim_set_by_pointer(d->st_mops, d->rd_mops_reads, mreads);
+ rrddim_set_by_pointer(d->st_mops, d->rd_mops_writes, mwrites);
+ rrdset_done(d->st_mops);
+ }
+
+ if(do_dc_stats && d->do_mops == CONFIG_BOOLEAN_YES && d->do_ext != CONFIG_BOOLEAN_NO) {
+ d->do_mops = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_ext_mops)) {
+ d->st_ext_mops = rrdset_create_localhost(
+ "disk_ext_mops"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk_ext.mops"
+ , "Disk Merged Discard Operations"
+ , "merged operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_MOPS + 1
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_ext_mops, RRDSET_FLAG_DETAIL);
+
+ d->rd_mops_discards = rrddim_add(d->st_ext_mops, "discards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_ext_mops);
+ }
+
+ rrddim_set_by_pointer(d->st_ext_mops, d->rd_mops_discards, mdiscards);
+ rrdset_done(d->st_ext_mops);
+ }
+
+ if(d->do_iotime == CONFIG_BOOLEAN_YES || (d->do_iotime == CONFIG_BOOLEAN_AUTO &&
+ (readms || writems || discardms || flushms || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->do_iotime = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_iotime)) {
+ d->st_iotime = rrdset_create_localhost(
+ "disk_iotime"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.iotime"
+ , "Disk Total I/O Time"
+ , "milliseconds/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_IOTIME
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_iotime, RRDSET_FLAG_DETAIL);
+
+ d->rd_iotime_reads = rrddim_add(d->st_iotime, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_iotime_writes = rrddim_add(d->st_iotime, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_iotime);
+ }
+
+ last_readms = rrddim_set_by_pointer(d->st_iotime, d->rd_iotime_reads, readms);
+ last_writems = rrddim_set_by_pointer(d->st_iotime, d->rd_iotime_writes, writems);
+ rrdset_done(d->st_iotime);
+ }
+
+ if(do_dc_stats && d->do_iotime == CONFIG_BOOLEAN_YES && d->do_ext != CONFIG_BOOLEAN_NO) {
+ if(unlikely(!d->st_ext_iotime)) {
+ d->st_ext_iotime = rrdset_create_localhost(
+ "disk_ext_iotime"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk_ext.iotime"
+ , "Disk Total I/O Time for Extended Operations"
+ , "milliseconds/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_IOTIME + 1
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_ext_iotime, RRDSET_FLAG_DETAIL);
+
+ d->rd_iotime_discards = rrddim_add(d->st_ext_iotime, "discards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ if (do_fl_stats)
+ d->rd_iotime_flushes = rrddim_add(d->st_ext_iotime, "flushes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_ext_iotime);
+ }
+
+ last_discardms = rrddim_set_by_pointer(d->st_ext_iotime, d->rd_iotime_discards, discardms);
+ if (do_fl_stats)
+ last_flushms = rrddim_set_by_pointer(d->st_ext_iotime, d->rd_iotime_flushes, flushms);
+ rrdset_done(d->st_ext_iotime);
+ }
+
+ // calculate differential charts
+ // only if this is not the first time we run
+
+ if(likely(dt)) {
+ if( (d->do_iotime == CONFIG_BOOLEAN_YES || (d->do_iotime == CONFIG_BOOLEAN_AUTO &&
+ (readms || writems ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) &&
+ (d->do_ops == CONFIG_BOOLEAN_YES || (d->do_ops == CONFIG_BOOLEAN_AUTO &&
+ (reads || writes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))) {
+
+ if(unlikely(!d->st_await)) {
+ d->st_await = rrdset_create_localhost(
+ "disk_await"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.await"
+ , "Average Completed I/O Operation Time"
+ , "milliseconds/operation"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_AWAIT
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_await, RRDSET_FLAG_DETAIL);
+
+ d->rd_await_reads = rrddim_add(d->st_await, "reads", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_await_writes = rrddim_add(d->st_await, "writes", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_await);
+ }
+
+ rrddim_set_by_pointer(d->st_await, d->rd_await_reads, (reads - last_reads) ? (readms - last_readms) / (reads - last_reads) : 0);
+ rrddim_set_by_pointer(d->st_await, d->rd_await_writes, (writes - last_writes) ? (writems - last_writems) / (writes - last_writes) : 0);
+ rrdset_done(d->st_await);
+ }
+
+ if (do_dc_stats && d->do_iotime == CONFIG_BOOLEAN_YES && d->do_ops == CONFIG_BOOLEAN_YES && d->do_ext != CONFIG_BOOLEAN_NO) {
+ if(unlikely(!d->st_ext_await)) {
+ d->st_ext_await = rrdset_create_localhost(
+ "disk_ext_await"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk_ext.await"
+ , "Average Completed Extended I/O Operation Time"
+ , "milliseconds/operation"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_AWAIT + 1
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_ext_await, RRDSET_FLAG_DETAIL);
+
+ d->rd_await_discards = rrddim_add(d->st_ext_await, "discards", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ if (do_fl_stats)
+ d->rd_await_flushes = rrddim_add(d->st_ext_await, "flushes", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_ext_await);
+ }
+
+ rrddim_set_by_pointer(
+ d->st_ext_await, d->rd_await_discards,
+ (discards - last_discards) ? (discardms - last_discardms) / (discards - last_discards) : 0);
+
+ if (do_fl_stats)
+ rrddim_set_by_pointer(
+ d->st_ext_await, d->rd_await_flushes,
+ (flushes - last_flushes) ? (flushms - last_flushms) / (flushes - last_flushes) : 0);
+
+ rrdset_done(d->st_ext_await);
+ }
+
+ if( (d->do_io == CONFIG_BOOLEAN_YES || (d->do_io == CONFIG_BOOLEAN_AUTO &&
+ (readsectors || writesectors || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) &&
+ (d->do_ops == CONFIG_BOOLEAN_YES || (d->do_ops == CONFIG_BOOLEAN_AUTO &&
+ (reads || writes || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))) {
+
+ if(unlikely(!d->st_avgsz)) {
+ d->st_avgsz = rrdset_create_localhost(
+ "disk_avgsz"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.avgsz"
+ , "Average Completed I/O Operation Bandwidth"
+ , "KiB/operation"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_AVGSZ
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(d->st_avgsz, RRDSET_FLAG_DETAIL);
+
+ d->rd_avgsz_reads = rrddim_add(d->st_avgsz, "reads", NULL, d->sector_size, 1024, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_avgsz_writes = rrddim_add(d->st_avgsz, "writes", NULL, d->sector_size * -1, 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_avgsz);
+ }
+
+ rrddim_set_by_pointer(d->st_avgsz, d->rd_avgsz_reads, (reads - last_reads) ? (readsectors - last_readsectors) / (reads - last_reads) : 0);
+ rrddim_set_by_pointer(d->st_avgsz, d->rd_avgsz_writes, (writes - last_writes) ? (writesectors - last_writesectors) / (writes - last_writes) : 0);
+ rrdset_done(d->st_avgsz);
+ }
+
+ if(do_dc_stats && d->do_io == CONFIG_BOOLEAN_YES && d->do_ops == CONFIG_BOOLEAN_YES && d->do_ext != CONFIG_BOOLEAN_NO) {
+ if(unlikely(!d->st_ext_avgsz)) {
+ d->st_ext_avgsz = rrdset_create_localhost(
+ "disk_ext_avgsz"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk_ext.avgsz"
+ , "Average Amount of Discarded Data"
+ , "KiB/operation"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_AVGSZ
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(d->st_ext_avgsz, RRDSET_FLAG_DETAIL);
+
+ d->rd_avgsz_discards = rrddim_add(d->st_ext_avgsz, "discards", NULL, d->sector_size, 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_ext_avgsz);
+ }
+
+ rrddim_set_by_pointer(
+ d->st_ext_avgsz, d->rd_avgsz_discards,
+ (discards - last_discards) ? (discardsectors - last_discardsectors) / (discards - last_discards) :
+ 0);
+ rrdset_done(d->st_ext_avgsz);
+ }
+
+ if( (d->do_util == CONFIG_BOOLEAN_YES || (d->do_util == CONFIG_BOOLEAN_AUTO &&
+ (busy_ms ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) &&
+ (d->do_ops == CONFIG_BOOLEAN_YES || (d->do_ops == CONFIG_BOOLEAN_AUTO &&
+ (reads || writes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))) {
+
+ if(unlikely(!d->st_svctm)) {
+ d->st_svctm = rrdset_create_localhost(
+ "disk_svctm"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.svctm"
+ , "Average Service Time"
+ , "milliseconds/operation"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_DISK_SVCTM
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_svctm, RRDSET_FLAG_DETAIL);
+
+ d->rd_svctm_svctm = rrddim_add(d->st_svctm, "svctm", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_svctm);
+ }
+
+ rrddim_set_by_pointer(d->st_svctm, d->rd_svctm_svctm, ((reads - last_reads) + (writes - last_writes)) ? (busy_ms - last_busy_ms) / ((reads - last_reads) + (writes - last_writes)) : 0);
+ rrdset_done(d->st_svctm);
+ }
+ }
+
+ // read bcache metrics and generate the bcache charts
+
+ if(d->device_is_bcache && d->do_bcache != CONFIG_BOOLEAN_NO) {
+ unsigned long long int
+ stats_total_cache_bypass_hits = 0,
+ stats_total_cache_bypass_misses = 0,
+ stats_total_cache_hits = 0,
+ stats_total_cache_miss_collisions = 0,
+ stats_total_cache_misses = 0,
+ stats_five_minute_cache_hit_ratio = 0,
+ stats_hour_cache_hit_ratio = 0,
+ stats_day_cache_hit_ratio = 0,
+ stats_total_cache_hit_ratio = 0,
+ cache_available_percent = 0,
+ cache_readaheads = 0,
+ cache_read_races = 0,
+ cache_io_errors = 0,
+ cache_congested = 0,
+ dirty_data = 0,
+ writeback_rate = 0;
+
+ // read the bcache values
+
+ if(d->bcache_filename_dirty_data)
+ dirty_data = bcache_read_number_with_units(d->bcache_filename_dirty_data);
+
+ if(d->bcache_filename_writeback_rate)
+ writeback_rate = bcache_read_number_with_units(d->bcache_filename_writeback_rate);
+
+ if(d->bcache_filename_cache_congested)
+ cache_congested = bcache_read_number_with_units(d->bcache_filename_cache_congested);
+
+ if(d->bcache_filename_cache_available_percent)
+ read_single_number_file(d->bcache_filename_cache_available_percent, &cache_available_percent);
+
+ if(d->bcache_filename_stats_five_minute_cache_hit_ratio)
+ read_single_number_file(d->bcache_filename_stats_five_minute_cache_hit_ratio, &stats_five_minute_cache_hit_ratio);
+
+ if(d->bcache_filename_stats_hour_cache_hit_ratio)
+ read_single_number_file(d->bcache_filename_stats_hour_cache_hit_ratio, &stats_hour_cache_hit_ratio);
+
+ if(d->bcache_filename_stats_day_cache_hit_ratio)
+ read_single_number_file(d->bcache_filename_stats_day_cache_hit_ratio, &stats_day_cache_hit_ratio);
+
+ if(d->bcache_filename_stats_total_cache_hit_ratio)
+ read_single_number_file(d->bcache_filename_stats_total_cache_hit_ratio, &stats_total_cache_hit_ratio);
+
+ if(d->bcache_filename_stats_total_cache_hits)
+ read_single_number_file(d->bcache_filename_stats_total_cache_hits, &stats_total_cache_hits);
+
+ if(d->bcache_filename_stats_total_cache_misses)
+ read_single_number_file(d->bcache_filename_stats_total_cache_misses, &stats_total_cache_misses);
+
+ if(d->bcache_filename_stats_total_cache_miss_collisions)
+ read_single_number_file(d->bcache_filename_stats_total_cache_miss_collisions, &stats_total_cache_miss_collisions);
+
+ if(d->bcache_filename_stats_total_cache_bypass_hits)
+ read_single_number_file(d->bcache_filename_stats_total_cache_bypass_hits, &stats_total_cache_bypass_hits);
+
+ if(d->bcache_filename_stats_total_cache_bypass_misses)
+ read_single_number_file(d->bcache_filename_stats_total_cache_bypass_misses, &stats_total_cache_bypass_misses);
+
+ if(d->bcache_filename_stats_total_cache_readaheads)
+ cache_readaheads = bcache_read_number_with_units(d->bcache_filename_stats_total_cache_readaheads);
+
+ if(d->bcache_filename_cache_read_races)
+ read_single_number_file(d->bcache_filename_cache_read_races, &cache_read_races);
+
+ if(d->bcache_filename_cache_io_errors)
+ read_single_number_file(d->bcache_filename_cache_io_errors, &cache_io_errors);
+
+ if(d->bcache_filename_priority_stats && global_bcache_priority_stats_update_every >= 1)
+ bcache_read_priority_stats(d, family, global_bcache_priority_stats_update_every, dt);
+
+ // update the charts
+
+ {
+ if(unlikely(!d->st_bcache_hit_ratio)) {
+ d->st_bcache_hit_ratio = rrdset_create_localhost(
+ "disk_bcache_hit_ratio"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.bcache_hit_ratio"
+ , "BCache Cache Hit Ratio"
+ , "percentage"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_BCACHE_HIT_RATIO
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ d->rd_bcache_hit_ratio_5min = rrddim_add(d->st_bcache_hit_ratio, "5min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_bcache_hit_ratio_1hour = rrddim_add(d->st_bcache_hit_ratio, "1hour", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_bcache_hit_ratio_1day = rrddim_add(d->st_bcache_hit_ratio, "1day", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_bcache_hit_ratio_total = rrddim_add(d->st_bcache_hit_ratio, "ever", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_bcache_hit_ratio);
+ }
+
+ rrddim_set_by_pointer(d->st_bcache_hit_ratio, d->rd_bcache_hit_ratio_5min, stats_five_minute_cache_hit_ratio);
+ rrddim_set_by_pointer(d->st_bcache_hit_ratio, d->rd_bcache_hit_ratio_1hour, stats_hour_cache_hit_ratio);
+ rrddim_set_by_pointer(d->st_bcache_hit_ratio, d->rd_bcache_hit_ratio_1day, stats_day_cache_hit_ratio);
+ rrddim_set_by_pointer(d->st_bcache_hit_ratio, d->rd_bcache_hit_ratio_total, stats_total_cache_hit_ratio);
+ rrdset_done(d->st_bcache_hit_ratio);
+ }
+
+ {
+
+ if(unlikely(!d->st_bcache_rates)) {
+ d->st_bcache_rates = rrdset_create_localhost(
+ "disk_bcache_rates"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.bcache_rates"
+ , "BCache Rates"
+ , "KiB/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_BCACHE_RATES
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ d->rd_bcache_rate_congested = rrddim_add(d->st_bcache_rates, "congested", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_bcache_rate_writeback = rrddim_add(d->st_bcache_rates, "writeback", NULL, -1, 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_bcache_rates);
+ }
+
+ rrddim_set_by_pointer(d->st_bcache_rates, d->rd_bcache_rate_writeback, writeback_rate);
+ rrddim_set_by_pointer(d->st_bcache_rates, d->rd_bcache_rate_congested, cache_congested);
+ rrdset_done(d->st_bcache_rates);
+ }
+
+ {
+ if(unlikely(!d->st_bcache_size)) {
+ d->st_bcache_size = rrdset_create_localhost(
+ "disk_bcache_size"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.bcache_size"
+ , "BCache Cache Sizes"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_BCACHE_SIZE
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ d->rd_bcache_dirty_size = rrddim_add(d->st_bcache_size, "dirty", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_bcache_size);
+ }
+
+ rrddim_set_by_pointer(d->st_bcache_size, d->rd_bcache_dirty_size, dirty_data);
+ rrdset_done(d->st_bcache_size);
+ }
+
+ {
+ if(unlikely(!d->st_bcache_usage)) {
+ d->st_bcache_usage = rrdset_create_localhost(
+ "disk_bcache_usage"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.bcache_usage"
+ , "BCache Cache Usage"
+ , "percentage"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_BCACHE_USAGE
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ d->rd_bcache_available_percent = rrddim_add(d->st_bcache_usage, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_disk(d, d->st_bcache_usage);
+ }
+
+ rrddim_set_by_pointer(d->st_bcache_usage, d->rd_bcache_available_percent, cache_available_percent);
+ rrdset_done(d->st_bcache_usage);
+ }
+
+ {
+
+ if(unlikely(!d->st_bcache_cache_read_races)) {
+ d->st_bcache_cache_read_races = rrdset_create_localhost(
+ "disk_bcache_cache_read_races"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.bcache_cache_read_races"
+ , "BCache Cache Read Races"
+ , "operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_BCACHE_CACHE_READ_RACES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ d->rd_bcache_cache_read_races = rrddim_add(d->st_bcache_cache_read_races, "races", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_bcache_cache_io_errors = rrddim_add(d->st_bcache_cache_read_races, "errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_bcache_cache_read_races);
+ }
+
+ rrddim_set_by_pointer(d->st_bcache_cache_read_races, d->rd_bcache_cache_read_races, cache_read_races);
+ rrddim_set_by_pointer(d->st_bcache_cache_read_races, d->rd_bcache_cache_io_errors, cache_io_errors);
+ rrdset_done(d->st_bcache_cache_read_races);
+ }
+
+ if(d->do_bcache == CONFIG_BOOLEAN_YES || (d->do_bcache == CONFIG_BOOLEAN_AUTO &&
+ (stats_total_cache_hits ||
+ stats_total_cache_misses ||
+ stats_total_cache_miss_collisions ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+
+ if(unlikely(!d->st_bcache)) {
+ d->st_bcache = rrdset_create_localhost(
+ "disk_bcache"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.bcache"
+ , "BCache Cache I/O Operations"
+ , "operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_BCACHE_OPS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_bcache, RRDSET_FLAG_DETAIL);
+
+ d->rd_bcache_hits = rrddim_add(d->st_bcache, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_bcache_misses = rrddim_add(d->st_bcache, "misses", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_bcache_miss_collisions = rrddim_add(d->st_bcache, "collisions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_bcache_readaheads = rrddim_add(d->st_bcache, "readaheads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_bcache);
+ }
+
+ rrddim_set_by_pointer(d->st_bcache, d->rd_bcache_hits, stats_total_cache_hits);
+ rrddim_set_by_pointer(d->st_bcache, d->rd_bcache_misses, stats_total_cache_misses);
+ rrddim_set_by_pointer(d->st_bcache, d->rd_bcache_miss_collisions, stats_total_cache_miss_collisions);
+ rrddim_set_by_pointer(d->st_bcache, d->rd_bcache_readaheads, cache_readaheads);
+ rrdset_done(d->st_bcache);
+ }
+
+ if(d->do_bcache == CONFIG_BOOLEAN_YES || (d->do_bcache == CONFIG_BOOLEAN_AUTO &&
+ (stats_total_cache_bypass_hits ||
+ stats_total_cache_bypass_misses ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+
+ if(unlikely(!d->st_bcache_bypass)) {
+ d->st_bcache_bypass = rrdset_create_localhost(
+ "disk_bcache_bypass"
+ , d->chart_id
+ , d->disk
+ , family
+ , "disk.bcache_bypass"
+ , "BCache Cache Bypass I/O Operations"
+ , "operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_BCACHE_BYPASS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_bcache_bypass, RRDSET_FLAG_DETAIL);
+
+ d->rd_bcache_bypass_hits = rrddim_add(d->st_bcache_bypass, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_bcache_bypass_misses = rrddim_add(d->st_bcache_bypass, "misses", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_disk(d, d->st_bcache_bypass);
+ }
+
+ rrddim_set_by_pointer(d->st_bcache_bypass, d->rd_bcache_bypass_hits, stats_total_cache_bypass_hits);
+ rrddim_set_by_pointer(d->st_bcache_bypass, d->rd_bcache_bypass_misses, stats_total_cache_bypass_misses);
+ rrdset_done(d->st_bcache_bypass);
+ }
+ }
+ }
+
+ // update the system total I/O
+
+ if(global_do_io == CONFIG_BOOLEAN_YES || (global_do_io == CONFIG_BOOLEAN_AUTO &&
+ (system_read_kb || system_write_kb ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ static RRDSET *st_io = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_io)) {
+ st_io = rrdset_create_localhost(
+ "system"
+ , "io"
+ , NULL
+ , "disk"
+ , NULL
+ , "Disk I/O"
+ , "KiB/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_DISKSTATS_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_IO
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st_io, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_io, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_io, rd_in, system_read_kb);
+ rrddim_set_by_pointer(st_io, rd_out, system_write_kb);
+ rrdset_done(st_io);
+ }
+
+ // cleanup removed disks
+
+ struct disk *d = disk_root, *last = NULL;
+ while(d) {
+ if(unlikely(global_cleanup_removed_disks && !d->updated)) {
+ struct disk *t = d;
+
+ rrdset_obsolete_and_pointer_null(d->st_avgsz);
+ rrdset_obsolete_and_pointer_null(d->st_ext_avgsz);
+ rrdset_obsolete_and_pointer_null(d->st_await);
+ rrdset_obsolete_and_pointer_null(d->st_ext_await);
+ rrdset_obsolete_and_pointer_null(d->st_backlog);
+ rrdset_obsolete_and_pointer_null(d->st_busy);
+ rrdset_obsolete_and_pointer_null(d->st_io);
+ rrdset_obsolete_and_pointer_null(d->st_ext_io);
+ rrdset_obsolete_and_pointer_null(d->st_iotime);
+ rrdset_obsolete_and_pointer_null(d->st_ext_iotime);
+ rrdset_obsolete_and_pointer_null(d->st_mops);
+ rrdset_obsolete_and_pointer_null(d->st_ext_mops);
+ rrdset_obsolete_and_pointer_null(d->st_ops);
+ rrdset_obsolete_and_pointer_null(d->st_ext_ops);
+ rrdset_obsolete_and_pointer_null(d->st_qops);
+ rrdset_obsolete_and_pointer_null(d->st_svctm);
+ rrdset_obsolete_and_pointer_null(d->st_util);
+ rrdset_obsolete_and_pointer_null(d->st_bcache);
+ rrdset_obsolete_and_pointer_null(d->st_bcache_bypass);
+ rrdset_obsolete_and_pointer_null(d->st_bcache_rates);
+ rrdset_obsolete_and_pointer_null(d->st_bcache_size);
+ rrdset_obsolete_and_pointer_null(d->st_bcache_usage);
+ rrdset_obsolete_and_pointer_null(d->st_bcache_hit_ratio);
+ rrdset_obsolete_and_pointer_null(d->st_bcache_cache_allocations);
+ rrdset_obsolete_and_pointer_null(d->st_bcache_cache_read_races);
+
+ if(d == disk_root) {
+ disk_root = d = d->next;
+ last = NULL;
+ }
+ else if(last) {
+ last->next = d = d->next;
+ }
+
+ freez(t->bcache_filename_dirty_data);
+ freez(t->bcache_filename_writeback_rate);
+ freez(t->bcache_filename_cache_congested);
+ freez(t->bcache_filename_cache_available_percent);
+ freez(t->bcache_filename_stats_five_minute_cache_hit_ratio);
+ freez(t->bcache_filename_stats_hour_cache_hit_ratio);
+ freez(t->bcache_filename_stats_day_cache_hit_ratio);
+ freez(t->bcache_filename_stats_total_cache_hit_ratio);
+ freez(t->bcache_filename_stats_total_cache_hits);
+ freez(t->bcache_filename_stats_total_cache_misses);
+ freez(t->bcache_filename_stats_total_cache_miss_collisions);
+ freez(t->bcache_filename_stats_total_cache_bypass_hits);
+ freez(t->bcache_filename_stats_total_cache_bypass_misses);
+ freez(t->bcache_filename_stats_total_cache_readaheads);
+ freez(t->bcache_filename_cache_read_races);
+ freez(t->bcache_filename_cache_io_errors);
+ freez(t->bcache_filename_priority_stats);
+
+ freez(t->disk);
+ freez(t->device);
+ freez(t->mount_point);
+ freez(t->chart_id);
+ freez(t);
+ }
+ else {
+ d->updated = 0;
+ last = d;
+ d = d->next;
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_interrupts.c b/collectors/proc.plugin/proc_interrupts.c
new file mode 100644
index 0000000..f876847
--- /dev/null
+++ b/collectors/proc.plugin/proc_interrupts.c
@@ -0,0 +1,245 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_INTERRUPTS_NAME "/proc/interrupts"
+#define CONFIG_SECTION_PLUGIN_PROC_INTERRUPTS "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_INTERRUPTS_NAME
+
+#define MAX_INTERRUPT_NAME 50
+
+struct cpu_interrupt {
+ unsigned long long value;
+ RRDDIM *rd;
+};
+
+struct interrupt {
+ int used;
+ char *id;
+ char name[MAX_INTERRUPT_NAME + 1];
+ RRDDIM *rd;
+ unsigned long long total;
+ struct cpu_interrupt cpu[];
+};
+
+// since each interrupt is variable in size
+// we use this to calculate its record size
+#define recordsize(cpus) (sizeof(struct interrupt) + ((cpus) * sizeof(struct cpu_interrupt)))
+
+// given a base, get a pointer to each record
+#define irrindex(base, line, cpus) ((struct interrupt *)&((char *)(base))[(line) * recordsize(cpus)])
+
+static inline struct interrupt *get_interrupts_array(size_t lines, int cpus) {
+ static struct interrupt *irrs = NULL;
+ static size_t allocated = 0;
+
+ if(unlikely(lines != allocated)) {
+ size_t l;
+ int c;
+
+ irrs = (struct interrupt *)reallocz(irrs, lines * recordsize(cpus));
+
+ // reset all interrupt RRDDIM pointers as any line could have shifted
+ for(l = 0; l < lines ;l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+ irr->rd = NULL;
+ irr->name[0] = '\0';
+ for(c = 0; c < cpus ;c++)
+ irr->cpu[c].rd = NULL;
+ }
+
+ allocated = lines;
+ }
+
+ return irrs;
+}
+
+int do_proc_interrupts(int update_every, usec_t dt) {
+ (void)dt;
+ static procfile *ff = NULL;
+ static int cpus = -1, do_per_core = CONFIG_BOOLEAN_INVALID;
+ struct interrupt *irrs = NULL;
+
+ if(unlikely(do_per_core == CONFIG_BOOLEAN_INVALID))
+ do_per_core = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_INTERRUPTS, "interrupts per core", CONFIG_BOOLEAN_AUTO);
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/interrupts");
+ ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_INTERRUPTS, "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ }
+ if(unlikely(!ff))
+ return 1;
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+ size_t words = procfile_linewords(ff, 0);
+
+ if(unlikely(!lines)) {
+ error("Cannot read /proc/interrupts, zero lines reported.");
+ return 1;
+ }
+
+ // find how many CPUs are there
+ if(unlikely(cpus == -1)) {
+ uint32_t w;
+ cpus = 0;
+ for(w = 0; w < words ; w++) {
+ if(likely(strncmp(procfile_lineword(ff, 0, w), "CPU", 3) == 0))
+ cpus++;
+ }
+ }
+
+ if(unlikely(!cpus)) {
+ error("PLUGIN: PROC_INTERRUPTS: Cannot find the number of CPUs in /proc/interrupts");
+ return 1;
+ }
+
+ // allocate the size we need;
+ irrs = get_interrupts_array(lines, cpus);
+ irrs[0].used = 0;
+
+ // loop through all lines
+ for(l = 1; l < lines ;l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+ irr->used = 0;
+ irr->total = 0;
+
+ words = procfile_linewords(ff, l);
+ if(unlikely(!words)) continue;
+
+ irr->id = procfile_lineword(ff, l, 0);
+ if(unlikely(!irr->id || !irr->id[0])) continue;
+
+ size_t idlen = strlen(irr->id);
+ if(irr->id[idlen - 1] == ':')
+ irr->id[--idlen] = '\0';
+
+ int c;
+ for(c = 0; c < cpus ;c++) {
+ if(likely((c + 1) < (int)words))
+ irr->cpu[c].value = str2ull(procfile_lineword(ff, l, (uint32_t)(c + 1)));
+ else
+ irr->cpu[c].value = 0;
+
+ irr->total += irr->cpu[c].value;
+ }
+
+ if(unlikely(isdigit(irr->id[0]) && (uint32_t)(cpus + 2) < words)) {
+ strncpyz(irr->name, procfile_lineword(ff, l, words - 1), MAX_INTERRUPT_NAME);
+ size_t nlen = strlen(irr->name);
+ if(likely(nlen + 1 + idlen <= MAX_INTERRUPT_NAME)) {
+ irr->name[nlen] = '_';
+ strncpyz(&irr->name[nlen + 1], irr->id, MAX_INTERRUPT_NAME - nlen - 1);
+ }
+ else {
+ irr->name[MAX_INTERRUPT_NAME - idlen - 1] = '_';
+ strncpyz(&irr->name[MAX_INTERRUPT_NAME - idlen], irr->id, idlen);
+ }
+ }
+ else {
+ strncpyz(irr->name, irr->id, MAX_INTERRUPT_NAME);
+ }
+
+ irr->used = 1;
+ }
+
+ static RRDSET *st_system_interrupts = NULL;
+ if(unlikely(!st_system_interrupts))
+ st_system_interrupts = rrdset_create_localhost(
+ "system"
+ , "interrupts"
+ , NULL
+ , "interrupts"
+ , NULL
+ , "System interrupts"
+ , "interrupts/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_INTERRUPTS_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_INTERRUPTS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ for(l = 0; l < lines ;l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+ if(irr->used && irr->total) {
+ // some interrupt may have changed without changing the total number of lines
+ // if the same number of interrupts have been added and removed between two
+ // calls of this function.
+ if(unlikely(!irr->rd || strncmp(rrddim_name(irr->rd), irr->name, MAX_INTERRUPT_NAME) != 0)) {
+ irr->rd = rrddim_add(st_system_interrupts, irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_reset_name(st_system_interrupts, irr->rd, irr->name);
+
+ // also reset per cpu RRDDIMs to avoid repeating strncmp() in the per core loop
+ if(likely(do_per_core != CONFIG_BOOLEAN_NO)) {
+ int c;
+ for(c = 0; c < cpus; c++) irr->cpu[c].rd = NULL;
+ }
+ }
+
+ rrddim_set_by_pointer(st_system_interrupts, irr->rd, irr->total);
+ }
+ }
+
+ rrdset_done(st_system_interrupts);
+
+ if(likely(do_per_core != CONFIG_BOOLEAN_NO)) {
+ static RRDSET **core_st = NULL;
+ static int old_cpus = 0;
+
+ if(old_cpus < cpus) {
+ core_st = reallocz(core_st, sizeof(RRDSET *) * cpus);
+ memset(&core_st[old_cpus], 0, sizeof(RRDSET *) * (cpus - old_cpus));
+ old_cpus = cpus;
+ }
+
+ int c;
+
+ for(c = 0; c < cpus ;c++) {
+ if(unlikely(!core_st[c])) {
+ char id[50+1];
+ snprintfz(id, 50, "cpu%d_interrupts", c);
+
+ char title[100+1];
+ snprintfz(title, 100, "CPU Interrupts");
+ core_st[c] = rrdset_create_localhost(
+ "cpu"
+ , id
+ , NULL
+ , "interrupts"
+ , "cpu.interrupts"
+ , title
+ , "interrupts/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_INTERRUPTS_NAME
+ , NETDATA_CHART_PRIO_INTERRUPTS_PER_CORE + c
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ char core[50+1];
+ snprintfz(core, 50, "cpu%d", c);
+ rrdlabels_add(core_st[c]->rrdlabels, "cpu", core, RRDLABEL_SRC_AUTO);
+ }
+
+ for(l = 0; l < lines ;l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+ if(irr->used && (do_per_core == CONFIG_BOOLEAN_YES || irr->cpu[c].value)) {
+ if(unlikely(!irr->cpu[c].rd)) {
+ irr->cpu[c].rd = rrddim_add(core_st[c], irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_reset_name(core_st[c], irr->cpu[c].rd, irr->name);
+ }
+
+ rrddim_set_by_pointer(core_st[c], irr->cpu[c].rd, irr->cpu[c].value);
+ }
+ }
+
+ rrdset_done(core_st[c]);
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_loadavg.c b/collectors/proc.plugin/proc_loadavg.c
new file mode 100644
index 0000000..d928c86
--- /dev/null
+++ b/collectors/proc.plugin/proc_loadavg.c
@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_LOADAVG_NAME "/proc/loadavg"
+#define CONFIG_SECTION_PLUGIN_PROC_LOADAVG "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_LOADAVG_NAME
+
+// linux calculates this once every 5 seconds
+#define MIN_LOADAVG_UPDATE_EVERY 5
+
+int do_proc_loadavg(int update_every, usec_t dt) {
+ static procfile *ff = NULL;
+ static int do_loadavg = -1, do_all_processes = -1;
+ static usec_t next_loadavg_dt = 0;
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/loadavg");
+
+ ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_LOADAVG, "filename to monitor", filename), " \t,:|/", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff))
+ return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ if(unlikely(do_loadavg == -1)) {
+ do_loadavg = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_LOADAVG, "enable load average", 1);
+ do_all_processes = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_LOADAVG, "enable total processes", 1);
+ }
+
+ if(unlikely(procfile_lines(ff) < 1)) {
+ error("/proc/loadavg has no lines.");
+ return 1;
+ }
+ if(unlikely(procfile_linewords(ff, 0) < 6)) {
+ error("/proc/loadavg has less than 6 words in it.");
+ return 1;
+ }
+
+ double load1 = strtod(procfile_lineword(ff, 0, 0), NULL);
+ double load5 = strtod(procfile_lineword(ff, 0, 1), NULL);
+ double load15 = strtod(procfile_lineword(ff, 0, 2), NULL);
+
+ //unsigned long long running_processes = str2ull(procfile_lineword(ff, 0, 3));
+ unsigned long long active_processes = str2ull(procfile_lineword(ff, 0, 4));
+
+ //get system pid_max
+ unsigned long long max_processes = get_system_pid_max();
+ //
+ //unsigned long long next_pid = str2ull(procfile_lineword(ff, 0, 5));
+
+ if(next_loadavg_dt <= dt) {
+ if(likely(do_loadavg)) {
+ static RRDSET *load_chart = NULL;
+ static RRDDIM *rd_load1 = NULL, *rd_load5 = NULL, *rd_load15 = NULL;
+
+ if(unlikely(!load_chart)) {
+ load_chart = rrdset_create_localhost(
+ "system"
+ , "load"
+ , NULL
+ , "load"
+ , NULL
+ , "System Load Average"
+ , "load"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_LOADAVG_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_LOAD
+ , (update_every < MIN_LOADAVG_UPDATE_EVERY) ? MIN_LOADAVG_UPDATE_EVERY : update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_load1 = rrddim_add(load_chart, "load1", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ rd_load5 = rrddim_add(load_chart, "load5", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ rd_load15 = rrddim_add(load_chart, "load15", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(load_chart, rd_load1, (collected_number) (load1 * 1000));
+ rrddim_set_by_pointer(load_chart, rd_load5, (collected_number) (load5 * 1000));
+ rrddim_set_by_pointer(load_chart, rd_load15, (collected_number) (load15 * 1000));
+ rrdset_done(load_chart);
+
+ next_loadavg_dt = load_chart->update_every * USEC_PER_SEC;
+ }
+ else
+ next_loadavg_dt = MIN_LOADAVG_UPDATE_EVERY * USEC_PER_SEC;
+ }
+ else
+ next_loadavg_dt -= dt;
+
+
+ if(likely(do_all_processes)) {
+ static RRDSET *processes_chart = NULL;
+ static RRDDIM *rd_active = NULL;
+ static const RRDSETVAR_ACQUIRED *rd_pidmax;
+
+ if(unlikely(!processes_chart)) {
+ processes_chart = rrdset_create_localhost(
+ "system"
+ , "active_processes"
+ , NULL
+ , "processes"
+ , NULL
+ , "System Active Processes"
+ , "processes"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_LOADAVG_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_ACTIVE_PROCESSES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_active = rrddim_add(processes_chart, "active", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_pidmax = rrdsetvar_custom_chart_variable_add_and_acquire(processes_chart, "pidmax");
+ }
+
+ rrddim_set_by_pointer(processes_chart, rd_active, active_processes);
+ rrdsetvar_custom_chart_variable_set(processes_chart, rd_pidmax, max_processes);
+ rrdset_done(processes_chart);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_mdstat.c b/collectors/proc.plugin/proc_mdstat.c
new file mode 100644
index 0000000..63e0c68
--- /dev/null
+++ b/collectors/proc.plugin/proc_mdstat.c
@@ -0,0 +1,640 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_MDSTAT_NAME "/proc/mdstat"
+
+struct raid {
+ int redundant;
+ char *name;
+ uint32_t hash;
+ char *level;
+
+ RRDDIM *rd_health;
+ unsigned long long failed_disks;
+
+ RRDSET *st_disks;
+ RRDDIM *rd_down;
+ RRDDIM *rd_inuse;
+ unsigned long long total_disks;
+ unsigned long long inuse_disks;
+
+ RRDSET *st_operation;
+ RRDDIM *rd_check;
+ RRDDIM *rd_resync;
+ RRDDIM *rd_recovery;
+ RRDDIM *rd_reshape;
+ unsigned long long check;
+ unsigned long long resync;
+ unsigned long long recovery;
+ unsigned long long reshape;
+
+ RRDSET *st_finish;
+ RRDDIM *rd_finish_in;
+ unsigned long long finish_in;
+
+ RRDSET *st_speed;
+ RRDDIM *rd_speed;
+ unsigned long long speed;
+
+ char *mismatch_cnt_filename;
+ RRDSET *st_mismatch_cnt;
+ RRDDIM *rd_mismatch_cnt;
+ unsigned long long mismatch_cnt;
+
+ RRDSET *st_nonredundant;
+ RRDDIM *rd_nonredundant;
+};
+
+struct old_raid {
+ int redundant;
+ char *name;
+ uint32_t hash;
+ int found;
+};
+
+static inline char *remove_trailing_chars(char *s, char c)
+{
+ while (*s) {
+ if (unlikely(*s == c)) {
+ *s = '\0';
+ }
+ s++;
+ }
+ return s;
+}
+
+static inline void make_chart_obsolete(char *name, const char *id_modifier)
+{
+ char id[50 + 1];
+ RRDSET *st = NULL;
+
+ if (likely(name && id_modifier)) {
+ snprintfz(id, 50, "mdstat.%s_%s", name, id_modifier);
+ st = rrdset_find_active_byname_localhost(id);
+ if (likely(st))
+ rrdset_is_obsolete(st);
+ }
+}
+
+static void add_labels_to_mdstat(struct raid *raid, RRDSET *st) {
+ rrdlabels_add(st->rrdlabels, "device", raid->name, RRDLABEL_SRC_AUTO);
+ rrdlabels_add(st->rrdlabels, "raid_level", raid->level, RRDLABEL_SRC_AUTO);
+}
+
+int do_proc_mdstat(int update_every, usec_t dt)
+{
+ (void)dt;
+ static procfile *ff = NULL;
+ static int do_health = -1, do_nonredundant = -1, do_disks = -1, do_operations = -1, do_mismatch = -1,
+ do_mismatch_config = -1;
+ static int make_charts_obsolete = -1;
+ static char *mdstat_filename = NULL, *mismatch_cnt_filename = NULL;
+ static struct raid *raids = NULL;
+ static size_t raids_allocated = 0;
+ size_t raids_num = 0, raid_idx = 0, redundant_num = 0;
+ static struct old_raid *old_raids = NULL;
+ static size_t old_raids_allocated = 0;
+ size_t old_raid_idx = 0;
+
+ if (unlikely(do_health == -1)) {
+ do_health =
+ config_get_boolean("plugin:proc:/proc/mdstat", "faulty devices", CONFIG_BOOLEAN_YES);
+ do_nonredundant =
+ config_get_boolean("plugin:proc:/proc/mdstat", "nonredundant arrays availability", CONFIG_BOOLEAN_YES);
+ do_mismatch_config =
+ config_get_boolean_ondemand("plugin:proc:/proc/mdstat", "mismatch count", CONFIG_BOOLEAN_AUTO);
+ do_disks =
+ config_get_boolean("plugin:proc:/proc/mdstat", "disk stats", CONFIG_BOOLEAN_YES);
+ do_operations =
+ config_get_boolean("plugin:proc:/proc/mdstat", "operation status", CONFIG_BOOLEAN_YES);
+
+ make_charts_obsolete =
+ config_get_boolean("plugin:proc:/proc/mdstat", "make charts obsolete", CONFIG_BOOLEAN_YES);
+
+ char filename[FILENAME_MAX + 1];
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/mdstat");
+ mdstat_filename = config_get("plugin:proc:/proc/mdstat", "filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/md/mismatch_cnt");
+ mismatch_cnt_filename = config_get("plugin:proc:/proc/mdstat", "mismatch_cnt filename to monitor", filename);
+ }
+
+ if (unlikely(!ff)) {
+ ff = procfile_open(mdstat_filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if (unlikely(!ff))
+ return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if (unlikely(!ff))
+ return 0; // we return 0, so that we will retry opening it next time
+
+ size_t lines = procfile_lines(ff);
+ size_t words = 0;
+
+ if (unlikely(lines < 2)) {
+ error("Cannot read /proc/mdstat. Expected 2 or more lines, read %zu.", lines);
+ return 1;
+ }
+
+ // find how many raids are there
+ size_t l;
+ raids_num = 0;
+ for (l = 1; l < lines - 2; l++) {
+ if (unlikely(procfile_lineword(ff, l, 1)[0] == 'a')) // check if the raid is active
+ raids_num++;
+ }
+
+ if (unlikely(!raids_num && !old_raids_allocated))
+ return 0; // we return 0, so that we will retry searching for raids next time
+
+ // allocate the memory we need;
+ if (unlikely(raids_num != raids_allocated)) {
+ for (raid_idx = 0; raid_idx < raids_allocated; raid_idx++) {
+ struct raid *raid = &raids[raid_idx];
+ freez(raid->name);
+ freez(raid->level);
+ freez(raid->mismatch_cnt_filename);
+ }
+ if (raids_num) {
+ raids = (struct raid *)reallocz(raids, raids_num * sizeof(struct raid));
+ memset(raids, 0, raids_num * sizeof(struct raid));
+ } else {
+ freez(raids);
+ raids = NULL;
+ }
+ raids_allocated = raids_num;
+ }
+
+ // loop through all lines except the first and the last ones
+ for (l = 1, raid_idx = 0; l < (lines - 2) && raid_idx < raids_num; l++) {
+ struct raid *raid = &raids[raid_idx];
+ raid->redundant = 0;
+
+ words = procfile_linewords(ff, l);
+
+ if (unlikely(words < 3))
+ continue;
+
+ if (unlikely(procfile_lineword(ff, l, 1)[0] != 'a'))
+ continue;
+
+ if (unlikely(!raid->name)) {
+ raid->name = strdupz(procfile_lineword(ff, l, 0));
+ raid->hash = simple_hash(raid->name);
+ raid->level = strdupz(procfile_lineword(ff, l, 2));
+ } else if (unlikely(strcmp(raid->name, procfile_lineword(ff, l, 0)))) {
+ freez(raid->name);
+ freez(raid->mismatch_cnt_filename);
+ freez(raid->level);
+ memset(raid, 0, sizeof(struct raid));
+ raid->name = strdupz(procfile_lineword(ff, l, 0));
+ raid->hash = simple_hash(raid->name);
+ raid->level = strdupz(procfile_lineword(ff, l, 2));
+ }
+
+ if (unlikely(!raid->name || !raid->name[0]))
+ continue;
+
+ raid_idx++;
+
+ // check if raid has disk status
+ l++;
+ words = procfile_linewords(ff, l);
+ if (words < 2 || procfile_lineword(ff, l, words - 1)[0] != '[')
+ continue;
+
+ // split inuse and total number of disks
+ if (likely(do_health || do_disks)) {
+ char *s = NULL, *str_total = NULL, *str_inuse = NULL;
+
+ s = procfile_lineword(ff, l, words - 2);
+ if (unlikely(s[0] != '[')) {
+ error("Cannot read /proc/mdstat raid health status. Unexpected format: missing opening bracket.");
+ continue;
+ }
+ str_total = ++s;
+ while (*s) {
+ if (unlikely(*s == '/')) {
+ *s = '\0';
+ str_inuse = s + 1;
+ } else if (unlikely(*s == ']')) {
+ *s = '\0';
+ break;
+ }
+ s++;
+ }
+ if (unlikely(str_total[0] == '\0' || !str_inuse || str_inuse[0] == '\0')) {
+ error("Cannot read /proc/mdstat raid health status. Unexpected format.");
+ continue;
+ }
+
+ raid->inuse_disks = str2ull(str_inuse);
+ raid->total_disks = str2ull(str_total);
+ raid->failed_disks = raid->total_disks - raid->inuse_disks;
+ }
+
+ raid->redundant = 1;
+ redundant_num++;
+ l++;
+
+ // check if any operation is performed on the raid
+ if (likely(do_operations)) {
+ char *s = NULL;
+
+ raid->check = 0;
+ raid->resync = 0;
+ raid->recovery = 0;
+ raid->reshape = 0;
+ raid->finish_in = 0;
+ raid->speed = 0;
+
+ words = procfile_linewords(ff, l);
+
+ if (likely(words < 2))
+ continue;
+
+ if (unlikely(procfile_lineword(ff, l, 0)[0] != '['))
+ continue;
+
+ if (unlikely(words < 7)) {
+ error("Cannot read /proc/mdstat line. Expected 7 params, read %zu.", words);
+ continue;
+ }
+
+ char *word;
+ word = procfile_lineword(ff, l, 3);
+ remove_trailing_chars(word, '%');
+
+ unsigned long long percentage = (unsigned long long)(str2ndd(word, NULL) * 100);
+ // possible operations: check, resync, recovery, reshape
+ // 4-th character is unique for each operation so it is checked
+ switch (procfile_lineword(ff, l, 1)[3]) {
+ case 'c': // check
+ raid->check = percentage;
+ break;
+ case 'y': // resync
+ raid->resync = percentage;
+ break;
+ case 'o': // recovery
+ raid->recovery = percentage;
+ break;
+ case 'h': // reshape
+ raid->reshape = percentage;
+ break;
+ }
+
+ word = procfile_lineword(ff, l, 5);
+ s = remove_trailing_chars(word, 'm'); // remove trailing "min"
+
+ word += 7; // skip leading "finish="
+
+ if (likely(s > word))
+ raid->finish_in = (unsigned long long)(str2ndd(word, NULL) * 60);
+
+ word = procfile_lineword(ff, l, 6);
+ s = remove_trailing_chars(word, 'K'); // remove trailing "K/sec"
+
+ word += 6; // skip leading "speed="
+
+ if (likely(s > word))
+ raid->speed = str2ull(word);
+ }
+ }
+
+ // read mismatch_cnt files
+ if (do_mismatch == -1) {
+ if (do_mismatch_config == CONFIG_BOOLEAN_AUTO) {
+ if (raids_num > 50)
+ do_mismatch = CONFIG_BOOLEAN_NO;
+ else
+ do_mismatch = CONFIG_BOOLEAN_YES;
+ } else
+ do_mismatch = do_mismatch_config;
+ }
+
+ if (likely(do_mismatch)) {
+ for (raid_idx = 0; raid_idx < raids_num; raid_idx++) {
+ char filename[FILENAME_MAX + 1];
+ struct raid *raid = &raids[raid_idx];
+
+ if (likely(raid->redundant)) {
+ if (unlikely(!raid->mismatch_cnt_filename)) {
+ snprintfz(filename, FILENAME_MAX, mismatch_cnt_filename, raid->name);
+ raid->mismatch_cnt_filename = strdupz(filename);
+ }
+ if (unlikely(read_single_number_file(raid->mismatch_cnt_filename, &raid->mismatch_cnt))) {
+ error("Cannot read file '%s'", raid->mismatch_cnt_filename);
+ do_mismatch = CONFIG_BOOLEAN_NO;
+ error("Monitoring for mismatch count has been disabled");
+ break;
+ }
+ }
+ }
+ }
+
+ // check for disappeared raids
+ for (old_raid_idx = 0; old_raid_idx < old_raids_allocated; old_raid_idx++) {
+ struct old_raid *old_raid = &old_raids[old_raid_idx];
+ int found = 0;
+
+ for (raid_idx = 0; raid_idx < raids_num; raid_idx++) {
+ struct raid *raid = &raids[raid_idx];
+
+ if (unlikely(
+ raid->hash == old_raid->hash && !strcmp(raid->name, old_raid->name) &&
+ raid->redundant == old_raid->redundant))
+ found = 1;
+ }
+
+ old_raid->found = found;
+ }
+
+ int raid_disappeared = 0;
+ for (old_raid_idx = 0; old_raid_idx < old_raids_allocated; old_raid_idx++) {
+ struct old_raid *old_raid = &old_raids[old_raid_idx];
+
+ if (unlikely(!old_raid->found)) {
+ if (likely(make_charts_obsolete)) {
+ make_chart_obsolete(old_raid->name, "disks");
+ make_chart_obsolete(old_raid->name, "mismatch");
+ make_chart_obsolete(old_raid->name, "operation");
+ make_chart_obsolete(old_raid->name, "finish");
+ make_chart_obsolete(old_raid->name, "speed");
+ make_chart_obsolete(old_raid->name, "availability");
+ }
+ raid_disappeared = 1;
+ }
+ }
+
+ // allocate memory for nonredundant arrays
+ if (unlikely(raid_disappeared || old_raids_allocated != raids_num)) {
+ for (old_raid_idx = 0; old_raid_idx < old_raids_allocated; old_raid_idx++) {
+ freez(old_raids[old_raid_idx].name);
+ }
+ if (likely(raids_num)) {
+ old_raids = reallocz(old_raids, sizeof(struct old_raid) * raids_num);
+ memset(old_raids, 0, sizeof(struct old_raid) * raids_num);
+ } else {
+ freez(old_raids);
+ old_raids = NULL;
+ }
+ old_raids_allocated = raids_num;
+ for (old_raid_idx = 0; old_raid_idx < old_raids_allocated; old_raid_idx++) {
+ struct old_raid *old_raid = &old_raids[old_raid_idx];
+ struct raid *raid = &raids[old_raid_idx];
+
+ old_raid->name = strdupz(raid->name);
+ old_raid->hash = raid->hash;
+ old_raid->redundant = raid->redundant;
+ }
+ }
+
+ if (likely(do_health && redundant_num)) {
+ static RRDSET *st_mdstat_health = NULL;
+ if (unlikely(!st_mdstat_health)) {
+ st_mdstat_health = rrdset_create_localhost(
+ "mdstat",
+ "mdstat_health",
+ NULL,
+ "health",
+ "md.health",
+ "Faulty Devices In MD",
+ "failed disks",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_MDSTAT_NAME,
+ NETDATA_CHART_PRIO_MDSTAT_HEALTH,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_isnot_obsolete(st_mdstat_health);
+ }
+
+ if (!redundant_num) {
+ if (likely(make_charts_obsolete))
+ make_chart_obsolete("mdstat", "health");
+ } else {
+ for (raid_idx = 0; raid_idx < raids_num; raid_idx++) {
+ struct raid *raid = &raids[raid_idx];
+
+ if (likely(raid->redundant)) {
+ if (unlikely(!raid->rd_health && !(raid->rd_health = rrddim_find_active(st_mdstat_health, raid->name))))
+ raid->rd_health = rrddim_add(st_mdstat_health, raid->name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(st_mdstat_health, raid->rd_health, raid->failed_disks);
+ }
+ }
+
+ rrdset_done(st_mdstat_health);
+ }
+ }
+
+ for (raid_idx = 0; raid_idx < raids_num; raid_idx++) {
+ struct raid *raid = &raids[raid_idx];
+ char id[50 + 1];
+ char family[50 + 1];
+
+ if (likely(raid->redundant)) {
+ if (likely(do_disks)) {
+ snprintfz(id, 50, "%s_disks", raid->name);
+
+ if (unlikely(!raid->st_disks && !(raid->st_disks = rrdset_find_active_byname_localhost(id)))) {
+ snprintfz(family, 50, "%s (%s)", raid->name, raid->level);
+
+ raid->st_disks = rrdset_create_localhost(
+ "mdstat",
+ id,
+ NULL,
+ family,
+ "md.disks",
+ "Disks Stats",
+ "disks",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_MDSTAT_NAME,
+ NETDATA_CHART_PRIO_MDSTAT_DISKS + raid_idx * 10,
+ update_every,
+ RRDSET_TYPE_STACKED);
+
+ rrdset_isnot_obsolete(raid->st_disks);
+
+ add_labels_to_mdstat(raid, raid->st_disks);
+ }
+
+ if (unlikely(!raid->rd_inuse && !(raid->rd_inuse = rrddim_find_active(raid->st_disks, "inuse"))))
+ raid->rd_inuse = rrddim_add(raid->st_disks, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ if (unlikely(!raid->rd_down && !(raid->rd_down = rrddim_find_active(raid->st_disks, "down"))))
+ raid->rd_down = rrddim_add(raid->st_disks, "down", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(raid->st_disks, raid->rd_inuse, raid->inuse_disks);
+ rrddim_set_by_pointer(raid->st_disks, raid->rd_down, raid->failed_disks);
+ rrdset_done(raid->st_disks);
+ }
+
+ if (likely(do_mismatch)) {
+ snprintfz(id, 50, "%s_mismatch", raid->name);
+
+ if (unlikely(!raid->st_mismatch_cnt && !(raid->st_mismatch_cnt = rrdset_find_active_byname_localhost(id)))) {
+ snprintfz(family, 50, "%s (%s)", raid->name, raid->level);
+
+ raid->st_mismatch_cnt = rrdset_create_localhost(
+ "mdstat",
+ id,
+ NULL,
+ family,
+ "md.mismatch_cnt",
+ "Mismatch Count",
+ "unsynchronized blocks",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_MDSTAT_NAME,
+ NETDATA_CHART_PRIO_MDSTAT_MISMATCH + raid_idx * 10,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_isnot_obsolete(raid->st_mismatch_cnt);
+
+ add_labels_to_mdstat(raid, raid->st_mismatch_cnt);
+ }
+
+ if (unlikely(!raid->rd_mismatch_cnt && !(raid->rd_mismatch_cnt = rrddim_find_active(raid->st_mismatch_cnt, "count"))))
+ raid->rd_mismatch_cnt = rrddim_add(raid->st_mismatch_cnt, "count", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(raid->st_mismatch_cnt, raid->rd_mismatch_cnt, raid->mismatch_cnt);
+ rrdset_done(raid->st_mismatch_cnt);
+ }
+
+ if (likely(do_operations)) {
+ snprintfz(id, 50, "%s_operation", raid->name);
+
+ if (unlikely(!raid->st_operation && !(raid->st_operation = rrdset_find_active_byname_localhost(id)))) {
+ snprintfz(family, 50, "%s (%s)", raid->name, raid->level);
+
+ raid->st_operation = rrdset_create_localhost(
+ "mdstat",
+ id,
+ NULL,
+ family,
+ "md.status",
+ "Current Status",
+ "percent",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_MDSTAT_NAME,
+ NETDATA_CHART_PRIO_MDSTAT_OPERATION + raid_idx * 10,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_isnot_obsolete(raid->st_operation);
+
+ add_labels_to_mdstat(raid, raid->st_operation);
+ }
+
+ if(unlikely(!raid->rd_check && !(raid->rd_check = rrddim_find_active(raid->st_operation, "check"))))
+ raid->rd_check = rrddim_add(raid->st_operation, "check", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ if(unlikely(!raid->rd_resync && !(raid->rd_resync = rrddim_find_active(raid->st_operation, "resync"))))
+ raid->rd_resync = rrddim_add(raid->st_operation, "resync", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ if(unlikely(!raid->rd_recovery && !(raid->rd_recovery = rrddim_find_active(raid->st_operation, "recovery"))))
+ raid->rd_recovery = rrddim_add(raid->st_operation, "recovery", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ if(unlikely(!raid->rd_reshape && !(raid->rd_reshape = rrddim_find_active(raid->st_operation, "reshape"))))
+ raid->rd_reshape = rrddim_add(raid->st_operation, "reshape", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(raid->st_operation, raid->rd_check, raid->check);
+ rrddim_set_by_pointer(raid->st_operation, raid->rd_resync, raid->resync);
+ rrddim_set_by_pointer(raid->st_operation, raid->rd_recovery, raid->recovery);
+ rrddim_set_by_pointer(raid->st_operation, raid->rd_reshape, raid->reshape);
+ rrdset_done(raid->st_operation);
+
+ snprintfz(id, 50, "%s_finish", raid->name);
+ if (unlikely(!raid->st_finish && !(raid->st_finish = rrdset_find_active_byname_localhost(id)))) {
+ snprintfz(family, 50, "%s (%s)", raid->name, raid->level);
+
+ raid->st_finish = rrdset_create_localhost(
+ "mdstat",
+ id,
+ NULL,
+ family,
+ "md.expected_time_until_operation_finish",
+ "Approximate Time Until Finish",
+ "seconds",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_MDSTAT_NAME,
+ NETDATA_CHART_PRIO_MDSTAT_FINISH + raid_idx * 10,
+ update_every, RRDSET_TYPE_LINE);
+
+ rrdset_isnot_obsolete(raid->st_finish);
+
+ add_labels_to_mdstat(raid, raid->st_finish);
+ }
+
+ if(unlikely(!raid->rd_finish_in && !(raid->rd_finish_in = rrddim_find_active(raid->st_finish, "finish_in"))))
+ raid->rd_finish_in = rrddim_add(raid->st_finish, "finish_in", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(raid->st_finish, raid->rd_finish_in, raid->finish_in);
+ rrdset_done(raid->st_finish);
+
+ snprintfz(id, 50, "%s_speed", raid->name);
+ if (unlikely(!raid->st_speed && !(raid->st_speed = rrdset_find_active_byname_localhost(id)))) {
+ snprintfz(family, 50, "%s (%s)", raid->name, raid->level);
+
+ raid->st_speed = rrdset_create_localhost(
+ "mdstat",
+ id,
+ NULL,
+ family,
+ "md.operation_speed",
+ "Operation Speed",
+ "KiB/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_MDSTAT_NAME,
+ NETDATA_CHART_PRIO_MDSTAT_SPEED + raid_idx * 10,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_isnot_obsolete(raid->st_speed);
+
+ add_labels_to_mdstat(raid, raid->st_speed);
+ }
+
+ if (unlikely(!raid->rd_speed && !(raid->rd_speed = rrddim_find_active(raid->st_speed, "speed"))))
+ raid->rd_speed = rrddim_add(raid->st_speed, "speed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(raid->st_speed, raid->rd_speed, raid->speed);
+ rrdset_done(raid->st_speed);
+ }
+ } else {
+ if (likely(do_nonredundant)) {
+ snprintfz(id, 50, "%s_availability", raid->name);
+
+ if (unlikely(!raid->st_nonredundant && !(raid->st_nonredundant = rrdset_find_active_localhost(id)))) {
+ snprintfz(family, 50, "%s (%s)", raid->name, raid->level);
+
+ raid->st_nonredundant = rrdset_create_localhost(
+ "mdstat",
+ id,
+ NULL,
+ family,
+ "md.nonredundant",
+ "Nonredundant Array Availability",
+ "boolean",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_MDSTAT_NAME,
+ NETDATA_CHART_PRIO_MDSTAT_NONREDUNDANT + raid_idx * 10,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_isnot_obsolete(raid->st_nonredundant);
+
+ add_labels_to_mdstat(raid, raid->st_nonredundant);
+ }
+
+ if (unlikely(!raid->rd_nonredundant && !(raid->rd_nonredundant = rrddim_find_active(raid->st_nonredundant, "available"))))
+ raid->rd_nonredundant = rrddim_add(raid->st_nonredundant, "available", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ rrddim_set_by_pointer(raid->st_nonredundant, raid->rd_nonredundant, 1);
+ rrdset_done(raid->st_nonredundant);
+ }
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_meminfo.c b/collectors/proc.plugin/proc_meminfo.c
new file mode 100644
index 0000000..2f390c6
--- /dev/null
+++ b/collectors/proc.plugin/proc_meminfo.c
@@ -0,0 +1,513 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_MEMINFO_NAME "/proc/meminfo"
+#define CONFIG_SECTION_PLUGIN_PROC_MEMINFO "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_MEMINFO_NAME
+
+int do_proc_meminfo(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+ static int do_ram = -1, do_swap = -1, do_hwcorrupt = -1, do_committed = -1, do_writeback = -1, do_kernel = -1, do_slab = -1, do_hugepages = -1, do_transparent_hugepages = -1;
+ static int do_percpu = 0;
+
+ static ARL_BASE *arl_base = NULL;
+ static ARL_ENTRY *arl_hwcorrupted = NULL, *arl_memavailable = NULL;
+
+ static unsigned long long
+ MemTotal = 0,
+ MemFree = 0,
+ MemAvailable = 0,
+ Buffers = 0,
+ Cached = 0,
+ //SwapCached = 0,
+ //Active = 0,
+ //Inactive = 0,
+ //ActiveAnon = 0,
+ //InactiveAnon = 0,
+ //ActiveFile = 0,
+ //InactiveFile = 0,
+ //Unevictable = 0,
+ //Mlocked = 0,
+ SwapTotal = 0,
+ SwapFree = 0,
+ Dirty = 0,
+ Writeback = 0,
+ //AnonPages = 0,
+ //Mapped = 0,
+ Shmem = 0,
+ Slab = 0,
+ SReclaimable = 0,
+ SUnreclaim = 0,
+ KernelStack = 0,
+ PageTables = 0,
+ NFS_Unstable = 0,
+ Bounce = 0,
+ WritebackTmp = 0,
+ //CommitLimit = 0,
+ Committed_AS = 0,
+ //VmallocTotal = 0,
+ VmallocUsed = 0,
+ //VmallocChunk = 0,
+ Percpu = 0,
+ AnonHugePages = 0,
+ ShmemHugePages = 0,
+ HugePages_Total = 0,
+ HugePages_Free = 0,
+ HugePages_Rsvd = 0,
+ HugePages_Surp = 0,
+ Hugepagesize = 0,
+ //DirectMap4k = 0,
+ //DirectMap2M = 0,
+ HardwareCorrupted = 0;
+
+ if(unlikely(!arl_base)) {
+ do_ram = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "system ram", 1);
+ do_swap = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "system swap", CONFIG_BOOLEAN_AUTO);
+ do_hwcorrupt = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "hardware corrupted ECC", CONFIG_BOOLEAN_AUTO);
+ do_committed = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "committed memory", 1);
+ do_writeback = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "writeback memory", 1);
+ do_kernel = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "kernel memory", 1);
+ do_slab = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "slab memory", 1);
+ do_hugepages = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "hugepages", CONFIG_BOOLEAN_AUTO);
+ do_transparent_hugepages = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "transparent hugepages", CONFIG_BOOLEAN_AUTO);
+
+ arl_base = arl_create("meminfo", NULL, 60);
+ arl_expect(arl_base, "MemTotal", &MemTotal);
+ arl_expect(arl_base, "MemFree", &MemFree);
+ arl_memavailable = arl_expect(arl_base, "MemAvailable", &MemAvailable);
+ arl_expect(arl_base, "Buffers", &Buffers);
+ arl_expect(arl_base, "Cached", &Cached);
+ //arl_expect(arl_base, "SwapCached", &SwapCached);
+ //arl_expect(arl_base, "Active", &Active);
+ //arl_expect(arl_base, "Inactive", &Inactive);
+ //arl_expect(arl_base, "ActiveAnon", &ActiveAnon);
+ //arl_expect(arl_base, "InactiveAnon", &InactiveAnon);
+ //arl_expect(arl_base, "ActiveFile", &ActiveFile);
+ //arl_expect(arl_base, "InactiveFile", &InactiveFile);
+ //arl_expect(arl_base, "Unevictable", &Unevictable);
+ //arl_expect(arl_base, "Mlocked", &Mlocked);
+ arl_expect(arl_base, "SwapTotal", &SwapTotal);
+ arl_expect(arl_base, "SwapFree", &SwapFree);
+ arl_expect(arl_base, "Dirty", &Dirty);
+ arl_expect(arl_base, "Writeback", &Writeback);
+ //arl_expect(arl_base, "AnonPages", &AnonPages);
+ //arl_expect(arl_base, "Mapped", &Mapped);
+ arl_expect(arl_base, "Shmem", &Shmem);
+ arl_expect(arl_base, "Slab", &Slab);
+ arl_expect(arl_base, "SReclaimable", &SReclaimable);
+ arl_expect(arl_base, "SUnreclaim", &SUnreclaim);
+ arl_expect(arl_base, "KernelStack", &KernelStack);
+ arl_expect(arl_base, "PageTables", &PageTables);
+ arl_expect(arl_base, "NFS_Unstable", &NFS_Unstable);
+ arl_expect(arl_base, "Bounce", &Bounce);
+ arl_expect(arl_base, "WritebackTmp", &WritebackTmp);
+ //arl_expect(arl_base, "CommitLimit", &CommitLimit);
+ arl_expect(arl_base, "Committed_AS", &Committed_AS);
+ //arl_expect(arl_base, "VmallocTotal", &VmallocTotal);
+ arl_expect(arl_base, "VmallocUsed", &VmallocUsed);
+ //arl_expect(arl_base, "VmallocChunk", &VmallocChunk);
+ arl_expect(arl_base, "Percpu", &Percpu);
+ arl_hwcorrupted = arl_expect(arl_base, "HardwareCorrupted", &HardwareCorrupted);
+ arl_expect(arl_base, "AnonHugePages", &AnonHugePages);
+ arl_expect(arl_base, "ShmemHugePages", &ShmemHugePages);
+ arl_expect(arl_base, "HugePages_Total", &HugePages_Total);
+ arl_expect(arl_base, "HugePages_Free", &HugePages_Free);
+ arl_expect(arl_base, "HugePages_Rsvd", &HugePages_Rsvd);
+ arl_expect(arl_base, "HugePages_Surp", &HugePages_Surp);
+ arl_expect(arl_base, "Hugepagesize", &Hugepagesize);
+ //arl_expect(arl_base, "DirectMap4k", &DirectMap4k);
+ //arl_expect(arl_base, "DirectMap2M", &DirectMap2M);
+ }
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/meminfo");
+ ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_MEMINFO, "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff))
+ return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ arl_begin(arl_base);
+
+ static int first_ff_read = 1;
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 2)) continue;
+
+ if (first_ff_read && !strcmp(procfile_lineword(ff, l, 0), "Percpu"))
+ do_percpu = 1;
+
+ if(unlikely(arl_check(arl_base,
+ procfile_lineword(ff, l, 0),
+ procfile_lineword(ff, l, 1)))) break;
+ }
+
+ if (first_ff_read)
+ first_ff_read = 0;
+
+ // http://calimeroteknik.free.fr/blag/?article20/really-used-memory-on-gnu-linux
+ unsigned long long MemCached = Cached + SReclaimable - Shmem;
+ unsigned long long MemUsed = MemTotal - MemFree - MemCached - Buffers;
+ // The Linux kernel doesn't report ZFS ARC usage as cache memory (the ARC is included in the total used system memory)
+ MemCached += (zfs_arcstats_shrinkable_cache_size_bytes / 1024);
+ MemUsed -= (zfs_arcstats_shrinkable_cache_size_bytes / 1024);
+ MemAvailable += (zfs_arcstats_shrinkable_cache_size_bytes / 1024);
+
+ if(do_ram) {
+ {
+ static RRDSET *st_system_ram = NULL;
+ static RRDDIM *rd_free = NULL, *rd_used = NULL, *rd_cached = NULL, *rd_buffers = NULL;
+
+ if(unlikely(!st_system_ram)) {
+ st_system_ram = rrdset_create_localhost(
+ "system"
+ , "ram"
+ , NULL
+ , "ram"
+ , NULL
+ , "System RAM"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_RAM
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_free = rrddim_add(st_system_ram, "free", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_used = rrddim_add(st_system_ram, "used", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_cached = rrddim_add(st_system_ram, "cached", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_buffers = rrddim_add(st_system_ram, "buffers", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_system_ram, rd_free, MemFree);
+ rrddim_set_by_pointer(st_system_ram, rd_used, MemUsed);
+ rrddim_set_by_pointer(st_system_ram, rd_cached, MemCached);
+ rrddim_set_by_pointer(st_system_ram, rd_buffers, Buffers);
+ rrdset_done(st_system_ram);
+ }
+
+ if(arl_memavailable->flags & ARL_ENTRY_FLAG_FOUND) {
+ static RRDSET *st_mem_available = NULL;
+ static RRDDIM *rd_avail = NULL;
+
+ if(unlikely(!st_mem_available)) {
+ st_mem_available = rrdset_create_localhost(
+ "mem"
+ , "available"
+ , NULL
+ , "system"
+ , NULL
+ , "Available RAM for applications"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_SYSTEM_AVAILABLE
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_avail = rrddim_add(st_mem_available, "MemAvailable", "avail", 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_available, rd_avail, MemAvailable);
+ rrdset_done(st_mem_available);
+ }
+ }
+
+ unsigned long long SwapUsed = SwapTotal - SwapFree;
+
+ if(do_swap == CONFIG_BOOLEAN_YES || (do_swap == CONFIG_BOOLEAN_AUTO &&
+ (SwapTotal || SwapUsed || SwapFree ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_swap = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_system_swap = NULL;
+ static RRDDIM *rd_free = NULL, *rd_used = NULL;
+
+ if(unlikely(!st_system_swap)) {
+ st_system_swap = rrdset_create_localhost(
+ "system"
+ , "swap"
+ , NULL
+ , "swap"
+ , NULL
+ , "System Swap"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_SWAP
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_flag_set(st_system_swap, RRDSET_FLAG_DETAIL);
+
+ rd_free = rrddim_add(st_system_swap, "free", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_used = rrddim_add(st_system_swap, "used", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_system_swap, rd_used, SwapUsed);
+ rrddim_set_by_pointer(st_system_swap, rd_free, SwapFree);
+ rrdset_done(st_system_swap);
+ }
+
+ if(arl_hwcorrupted->flags & ARL_ENTRY_FLAG_FOUND &&
+ (do_hwcorrupt == CONFIG_BOOLEAN_YES || (do_hwcorrupt == CONFIG_BOOLEAN_AUTO &&
+ (HardwareCorrupted > 0 ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))) {
+ do_hwcorrupt = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_mem_hwcorrupt = NULL;
+ static RRDDIM *rd_corrupted = NULL;
+
+ if(unlikely(!st_mem_hwcorrupt)) {
+ st_mem_hwcorrupt = rrdset_create_localhost(
+ "mem"
+ , "hwcorrupt"
+ , NULL
+ , "ecc"
+ , NULL
+ , "Corrupted Memory, detected by ECC"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_HW
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_mem_hwcorrupt, RRDSET_FLAG_DETAIL);
+
+ rd_corrupted = rrddim_add(st_mem_hwcorrupt, "HardwareCorrupted", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_hwcorrupt, rd_corrupted, HardwareCorrupted);
+ rrdset_done(st_mem_hwcorrupt);
+ }
+
+ if(do_committed) {
+ static RRDSET *st_mem_committed = NULL;
+ static RRDDIM *rd_committed = NULL;
+
+ if(unlikely(!st_mem_committed)) {
+ st_mem_committed = rrdset_create_localhost(
+ "mem"
+ , "committed"
+ , NULL
+ , "system"
+ , NULL
+ , "Committed (Allocated) Memory"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_SYSTEM_COMMITTED
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(st_mem_committed, RRDSET_FLAG_DETAIL);
+
+ rd_committed = rrddim_add(st_mem_committed, "Committed_AS", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_committed, rd_committed, Committed_AS);
+ rrdset_done(st_mem_committed);
+ }
+
+ if(do_writeback) {
+ static RRDSET *st_mem_writeback = NULL;
+ static RRDDIM *rd_dirty = NULL, *rd_writeback = NULL, *rd_fusewriteback = NULL, *rd_nfs_writeback = NULL, *rd_bounce = NULL;
+
+ if(unlikely(!st_mem_writeback)) {
+ st_mem_writeback = rrdset_create_localhost(
+ "mem"
+ , "writeback"
+ , NULL
+ , "kernel"
+ , NULL
+ , "Writeback Memory"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_KERNEL
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st_mem_writeback, RRDSET_FLAG_DETAIL);
+
+ rd_dirty = rrddim_add(st_mem_writeback, "Dirty", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_writeback = rrddim_add(st_mem_writeback, "Writeback", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_fusewriteback = rrddim_add(st_mem_writeback, "FuseWriteback", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_nfs_writeback = rrddim_add(st_mem_writeback, "NfsWriteback", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_bounce = rrddim_add(st_mem_writeback, "Bounce", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_writeback, rd_dirty, Dirty);
+ rrddim_set_by_pointer(st_mem_writeback, rd_writeback, Writeback);
+ rrddim_set_by_pointer(st_mem_writeback, rd_fusewriteback, WritebackTmp);
+ rrddim_set_by_pointer(st_mem_writeback, rd_nfs_writeback, NFS_Unstable);
+ rrddim_set_by_pointer(st_mem_writeback, rd_bounce, Bounce);
+ rrdset_done(st_mem_writeback);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_kernel) {
+ static RRDSET *st_mem_kernel = NULL;
+ static RRDDIM *rd_slab = NULL, *rd_kernelstack = NULL, *rd_pagetables = NULL, *rd_vmallocused = NULL,
+ *rd_percpu = NULL;
+
+ if(unlikely(!st_mem_kernel)) {
+ st_mem_kernel = rrdset_create_localhost(
+ "mem"
+ , "kernel"
+ , NULL
+ , "kernel"
+ , NULL
+ , "Memory Used by Kernel"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_KERNEL + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_flag_set(st_mem_kernel, RRDSET_FLAG_DETAIL);
+
+ rd_slab = rrddim_add(st_mem_kernel, "Slab", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_kernelstack = rrddim_add(st_mem_kernel, "KernelStack", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_pagetables = rrddim_add(st_mem_kernel, "PageTables", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_vmallocused = rrddim_add(st_mem_kernel, "VmallocUsed", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ if (do_percpu)
+ rd_percpu = rrddim_add(st_mem_kernel, "Percpu", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_kernel, rd_slab, Slab);
+ rrddim_set_by_pointer(st_mem_kernel, rd_kernelstack, KernelStack);
+ rrddim_set_by_pointer(st_mem_kernel, rd_pagetables, PageTables);
+ rrddim_set_by_pointer(st_mem_kernel, rd_vmallocused, VmallocUsed);
+ if (do_percpu)
+ rrddim_set_by_pointer(st_mem_kernel, rd_percpu, Percpu);
+
+ rrdset_done(st_mem_kernel);
+ }
+
+ if(do_slab) {
+ static RRDSET *st_mem_slab = NULL;
+ static RRDDIM *rd_reclaimable = NULL, *rd_unreclaimable = NULL;
+
+ if(unlikely(!st_mem_slab)) {
+ st_mem_slab = rrdset_create_localhost(
+ "mem"
+ , "slab"
+ , NULL
+ , "slab"
+ , NULL
+ , "Reclaimable Kernel Memory"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_SLAB
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_flag_set(st_mem_slab, RRDSET_FLAG_DETAIL);
+
+ rd_reclaimable = rrddim_add(st_mem_slab, "reclaimable", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_unreclaimable = rrddim_add(st_mem_slab, "unreclaimable", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_slab, rd_reclaimable, SReclaimable);
+ rrddim_set_by_pointer(st_mem_slab, rd_unreclaimable, SUnreclaim);
+ rrdset_done(st_mem_slab);
+ }
+
+ if(do_hugepages == CONFIG_BOOLEAN_YES || (do_hugepages == CONFIG_BOOLEAN_AUTO &&
+ ((Hugepagesize && HugePages_Total) ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_hugepages = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_mem_hugepages = NULL;
+ static RRDDIM *rd_used = NULL, *rd_free = NULL, *rd_rsvd = NULL, *rd_surp = NULL;
+
+ if(unlikely(!st_mem_hugepages)) {
+ st_mem_hugepages = rrdset_create_localhost(
+ "mem"
+ , "hugepages"
+ , NULL
+ , "hugepages"
+ , NULL
+ , "Dedicated HugePages Memory"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_HUGEPAGES + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_flag_set(st_mem_hugepages, RRDSET_FLAG_DETAIL);
+
+ rd_free = rrddim_add(st_mem_hugepages, "free", NULL, Hugepagesize, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_used = rrddim_add(st_mem_hugepages, "used", NULL, Hugepagesize, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_surp = rrddim_add(st_mem_hugepages, "surplus", NULL, Hugepagesize, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_rsvd = rrddim_add(st_mem_hugepages, "reserved", NULL, Hugepagesize, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_hugepages, rd_used, HugePages_Total - HugePages_Free - HugePages_Rsvd);
+ rrddim_set_by_pointer(st_mem_hugepages, rd_free, HugePages_Free);
+ rrddim_set_by_pointer(st_mem_hugepages, rd_rsvd, HugePages_Rsvd);
+ rrddim_set_by_pointer(st_mem_hugepages, rd_surp, HugePages_Surp);
+ rrdset_done(st_mem_hugepages);
+ }
+
+ if(do_transparent_hugepages == CONFIG_BOOLEAN_YES || (do_transparent_hugepages == CONFIG_BOOLEAN_AUTO &&
+ (AnonHugePages ||
+ ShmemHugePages ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_transparent_hugepages = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_mem_transparent_hugepages = NULL;
+ static RRDDIM *rd_anonymous = NULL, *rd_shared = NULL;
+
+ if(unlikely(!st_mem_transparent_hugepages)) {
+ st_mem_transparent_hugepages = rrdset_create_localhost(
+ "mem"
+ , "transparent_hugepages"
+ , NULL
+ , "hugepages"
+ , NULL
+ , "Transparent HugePages Memory"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_MEMINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_HUGEPAGES
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_flag_set(st_mem_transparent_hugepages, RRDSET_FLAG_DETAIL);
+
+ rd_anonymous = rrddim_add(st_mem_transparent_hugepages, "anonymous", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_shared = rrddim_add(st_mem_transparent_hugepages, "shmem", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_transparent_hugepages, rd_anonymous, AnonHugePages);
+ rrddim_set_by_pointer(st_mem_transparent_hugepages, rd_shared, ShmemHugePages);
+ rrdset_done(st_mem_transparent_hugepages);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_dev.c b/collectors/proc.plugin/proc_net_dev.c
new file mode 100644
index 0000000..e124f63
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_dev.c
@@ -0,0 +1,1522 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_NETDEV_NAME "/proc/net/dev"
+#define CONFIG_SECTION_PLUGIN_PROC_NETDEV "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_NETDEV_NAME
+
+#define STATE_LENGTH_MAX 32
+
+#define READ_RETRY_PERIOD 60 // seconds
+
+enum {
+ NETDEV_DUPLEX_UNKNOWN,
+ NETDEV_DUPLEX_HALF,
+ NETDEV_DUPLEX_FULL
+};
+
+enum {
+ NETDEV_OPERSTATE_UNKNOWN,
+ NETDEV_OPERSTATE_NOTPRESENT,
+ NETDEV_OPERSTATE_DOWN,
+ NETDEV_OPERSTATE_LOWERLAYERDOWN,
+ NETDEV_OPERSTATE_TESTING,
+ NETDEV_OPERSTATE_DORMANT,
+ NETDEV_OPERSTATE_UP
+};
+
+static inline int get_operstate(char *operstate)
+{
+ // As defined in https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-net
+ if (!strcmp(operstate, "up"))
+ return NETDEV_OPERSTATE_UP;
+ if (!strcmp(operstate, "down"))
+ return NETDEV_OPERSTATE_DOWN;
+ if (!strcmp(operstate, "notpresent"))
+ return NETDEV_OPERSTATE_NOTPRESENT;
+ if (!strcmp(operstate, "lowerlayerdown"))
+ return NETDEV_OPERSTATE_LOWERLAYERDOWN;
+ if (!strcmp(operstate, "testing"))
+ return NETDEV_OPERSTATE_TESTING;
+ if (!strcmp(operstate, "dormant"))
+ return NETDEV_OPERSTATE_DORMANT;
+
+ return NETDEV_OPERSTATE_UNKNOWN;
+}
+
+// ----------------------------------------------------------------------------
+// netdev list
+
+static struct netdev {
+ char *name;
+ uint32_t hash;
+ size_t len;
+
+ // flags
+ int virtual;
+ int configured;
+ int enabled;
+ int updated;
+
+ int carrier_file_exists;
+ time_t carrier_file_lost_time;
+
+ int duplex_file_exists;
+ time_t duplex_file_lost_time;
+
+ int speed_file_exists;
+ time_t speed_file_lost_time;
+
+ int do_bandwidth;
+ int do_packets;
+ int do_errors;
+ int do_drops;
+ int do_fifo;
+ int do_compressed;
+ int do_events;
+ int do_speed;
+ int do_duplex;
+ int do_operstate;
+ int do_carrier;
+ int do_mtu;
+
+ const char *chart_type_net_bytes;
+ const char *chart_type_net_packets;
+ const char *chart_type_net_errors;
+ const char *chart_type_net_fifo;
+ const char *chart_type_net_events;
+ const char *chart_type_net_drops;
+ const char *chart_type_net_compressed;
+ const char *chart_type_net_speed;
+ const char *chart_type_net_duplex;
+ const char *chart_type_net_operstate;
+ const char *chart_type_net_carrier;
+ const char *chart_type_net_mtu;
+
+ const char *chart_id_net_bytes;
+ const char *chart_id_net_packets;
+ const char *chart_id_net_errors;
+ const char *chart_id_net_fifo;
+ const char *chart_id_net_events;
+ const char *chart_id_net_drops;
+ const char *chart_id_net_compressed;
+ const char *chart_id_net_speed;
+ const char *chart_id_net_duplex;
+ const char *chart_id_net_operstate;
+ const char *chart_id_net_carrier;
+ const char *chart_id_net_mtu;
+
+ const char *chart_ctx_net_bytes;
+ const char *chart_ctx_net_packets;
+ const char *chart_ctx_net_errors;
+ const char *chart_ctx_net_fifo;
+ const char *chart_ctx_net_events;
+ const char *chart_ctx_net_drops;
+ const char *chart_ctx_net_compressed;
+ const char *chart_ctx_net_speed;
+ const char *chart_ctx_net_duplex;
+ const char *chart_ctx_net_operstate;
+ const char *chart_ctx_net_carrier;
+ const char *chart_ctx_net_mtu;
+
+ const char *chart_family;
+
+ DICTIONARY *chart_labels;
+
+ int flipped;
+ unsigned long priority;
+
+ // data collected
+ kernel_uint_t rbytes;
+ kernel_uint_t rpackets;
+ kernel_uint_t rerrors;
+ kernel_uint_t rdrops;
+ kernel_uint_t rfifo;
+ kernel_uint_t rframe;
+ kernel_uint_t rcompressed;
+ kernel_uint_t rmulticast;
+
+ kernel_uint_t tbytes;
+ kernel_uint_t tpackets;
+ kernel_uint_t terrors;
+ kernel_uint_t tdrops;
+ kernel_uint_t tfifo;
+ kernel_uint_t tcollisions;
+ kernel_uint_t tcarrier;
+ kernel_uint_t tcompressed;
+ kernel_uint_t speed;
+ kernel_uint_t duplex;
+ kernel_uint_t operstate;
+ unsigned long long carrier;
+ unsigned long long mtu;
+
+ // charts
+ RRDSET *st_bandwidth;
+ RRDSET *st_packets;
+ RRDSET *st_errors;
+ RRDSET *st_drops;
+ RRDSET *st_fifo;
+ RRDSET *st_compressed;
+ RRDSET *st_events;
+ RRDSET *st_speed;
+ RRDSET *st_duplex;
+ RRDSET *st_operstate;
+ RRDSET *st_carrier;
+ RRDSET *st_mtu;
+
+ // dimensions
+ RRDDIM *rd_rbytes;
+ RRDDIM *rd_rpackets;
+ RRDDIM *rd_rerrors;
+ RRDDIM *rd_rdrops;
+ RRDDIM *rd_rfifo;
+ RRDDIM *rd_rframe;
+ RRDDIM *rd_rcompressed;
+ RRDDIM *rd_rmulticast;
+
+ RRDDIM *rd_tbytes;
+ RRDDIM *rd_tpackets;
+ RRDDIM *rd_terrors;
+ RRDDIM *rd_tdrops;
+ RRDDIM *rd_tfifo;
+ RRDDIM *rd_tcollisions;
+ RRDDIM *rd_tcarrier;
+ RRDDIM *rd_tcompressed;
+
+ RRDDIM *rd_speed;
+ RRDDIM *rd_duplex_full;
+ RRDDIM *rd_duplex_half;
+ RRDDIM *rd_duplex_unknown;
+ RRDDIM *rd_operstate_unknown;
+ RRDDIM *rd_operstate_notpresent;
+ RRDDIM *rd_operstate_down;
+ RRDDIM *rd_operstate_lowerlayerdown;
+ RRDDIM *rd_operstate_testing;
+ RRDDIM *rd_operstate_dormant;
+ RRDDIM *rd_operstate_up;
+ RRDDIM *rd_carrier_up;
+ RRDDIM *rd_carrier_down;
+ RRDDIM *rd_mtu;
+
+ char *filename_speed;
+ const RRDSETVAR_ACQUIRED *chart_var_speed;
+
+ char *filename_duplex;
+ char *filename_operstate;
+ char *filename_carrier;
+ char *filename_mtu;
+
+ struct netdev *next;
+} *netdev_root = NULL, *netdev_last_used = NULL;
+
+static size_t netdev_added = 0, netdev_found = 0;
+
+// ----------------------------------------------------------------------------
+
+static void netdev_charts_release(struct netdev *d) {
+ if(d->st_bandwidth) rrdset_is_obsolete(d->st_bandwidth);
+ if(d->st_packets) rrdset_is_obsolete(d->st_packets);
+ if(d->st_errors) rrdset_is_obsolete(d->st_errors);
+ if(d->st_drops) rrdset_is_obsolete(d->st_drops);
+ if(d->st_fifo) rrdset_is_obsolete(d->st_fifo);
+ if(d->st_compressed) rrdset_is_obsolete(d->st_compressed);
+ if(d->st_events) rrdset_is_obsolete(d->st_events);
+ if(d->st_speed) rrdset_is_obsolete(d->st_speed);
+ if(d->st_duplex) rrdset_is_obsolete(d->st_duplex);
+ if(d->st_operstate) rrdset_is_obsolete(d->st_operstate);
+ if(d->st_carrier) rrdset_is_obsolete(d->st_carrier);
+ if(d->st_mtu) rrdset_is_obsolete(d->st_mtu);
+
+ d->st_bandwidth = NULL;
+ d->st_compressed = NULL;
+ d->st_drops = NULL;
+ d->st_errors = NULL;
+ d->st_events = NULL;
+ d->st_fifo = NULL;
+ d->st_packets = NULL;
+ d->st_speed = NULL;
+ d->st_duplex = NULL;
+ d->st_operstate = NULL;
+ d->st_carrier = NULL;
+ d->st_mtu = NULL;
+
+ d->rd_rbytes = NULL;
+ d->rd_rpackets = NULL;
+ d->rd_rerrors = NULL;
+ d->rd_rdrops = NULL;
+ d->rd_rfifo = NULL;
+ d->rd_rframe = NULL;
+ d->rd_rcompressed = NULL;
+ d->rd_rmulticast = NULL;
+
+ d->rd_tbytes = NULL;
+ d->rd_tpackets = NULL;
+ d->rd_terrors = NULL;
+ d->rd_tdrops = NULL;
+ d->rd_tfifo = NULL;
+ d->rd_tcollisions = NULL;
+ d->rd_tcarrier = NULL;
+ d->rd_tcompressed = NULL;
+
+ d->rd_speed = NULL;
+ d->rd_duplex_full = NULL;
+ d->rd_duplex_half = NULL;
+ d->rd_duplex_unknown = NULL;
+ d->rd_carrier_up = NULL;
+ d->rd_carrier_down = NULL;
+ d->rd_mtu = NULL;
+
+ d->rd_operstate_unknown = NULL;
+ d->rd_operstate_notpresent = NULL;
+ d->rd_operstate_down = NULL;
+ d->rd_operstate_lowerlayerdown = NULL;
+ d->rd_operstate_testing = NULL;
+ d->rd_operstate_dormant = NULL;
+ d->rd_operstate_up = NULL;
+
+ d->chart_var_speed = NULL;
+}
+
+static void netdev_free_chart_strings(struct netdev *d) {
+ freez((void *)d->chart_type_net_bytes);
+ freez((void *)d->chart_type_net_compressed);
+ freez((void *)d->chart_type_net_drops);
+ freez((void *)d->chart_type_net_errors);
+ freez((void *)d->chart_type_net_events);
+ freez((void *)d->chart_type_net_fifo);
+ freez((void *)d->chart_type_net_packets);
+ freez((void *)d->chart_type_net_speed);
+ freez((void *)d->chart_type_net_duplex);
+ freez((void *)d->chart_type_net_operstate);
+ freez((void *)d->chart_type_net_carrier);
+ freez((void *)d->chart_type_net_mtu);
+
+ freez((void *)d->chart_id_net_bytes);
+ freez((void *)d->chart_id_net_compressed);
+ freez((void *)d->chart_id_net_drops);
+ freez((void *)d->chart_id_net_errors);
+ freez((void *)d->chart_id_net_events);
+ freez((void *)d->chart_id_net_fifo);
+ freez((void *)d->chart_id_net_packets);
+ freez((void *)d->chart_id_net_speed);
+ freez((void *)d->chart_id_net_duplex);
+ freez((void *)d->chart_id_net_operstate);
+ freez((void *)d->chart_id_net_carrier);
+ freez((void *)d->chart_id_net_mtu);
+
+ freez((void *)d->chart_ctx_net_bytes);
+ freez((void *)d->chart_ctx_net_compressed);
+ freez((void *)d->chart_ctx_net_drops);
+ freez((void *)d->chart_ctx_net_errors);
+ freez((void *)d->chart_ctx_net_events);
+ freez((void *)d->chart_ctx_net_fifo);
+ freez((void *)d->chart_ctx_net_packets);
+ freez((void *)d->chart_ctx_net_speed);
+ freez((void *)d->chart_ctx_net_duplex);
+ freez((void *)d->chart_ctx_net_operstate);
+ freez((void *)d->chart_ctx_net_carrier);
+ freez((void *)d->chart_ctx_net_mtu);
+
+ freez((void *)d->chart_family);
+}
+
+static void netdev_free(struct netdev *d) {
+ netdev_charts_release(d);
+ netdev_free_chart_strings(d);
+ rrdlabels_destroy(d->chart_labels);
+
+ freez((void *)d->name);
+ freez((void *)d->filename_speed);
+ freez((void *)d->filename_duplex);
+ freez((void *)d->filename_operstate);
+ freez((void *)d->filename_carrier);
+ freez((void *)d->filename_mtu);
+ freez((void *)d);
+ netdev_added--;
+}
+
+// ----------------------------------------------------------------------------
+// netdev renames
+
+static struct netdev_rename {
+ const char *host_device;
+ uint32_t hash;
+
+ const char *container_device;
+ const char *container_name;
+ const char *ctx_prefix;
+
+ DICTIONARY *chart_labels;
+
+ int processed;
+
+ struct netdev_rename *next;
+} *netdev_rename_root = NULL;
+
+static int netdev_pending_renames = 0;
+static netdata_mutex_t netdev_rename_mutex = NETDATA_MUTEX_INITIALIZER;
+
+static struct netdev_rename *netdev_rename_find(const char *host_device, uint32_t hash) {
+ struct netdev_rename *r;
+
+ for(r = netdev_rename_root; r ; r = r->next)
+ if(r->hash == hash && !strcmp(host_device, r->host_device))
+ return r;
+
+ return NULL;
+}
+
+// other threads can call this function to register a rename to a netdev
+void netdev_rename_device_add(
+ const char *host_device,
+ const char *container_device,
+ const char *container_name,
+ DICTIONARY *labels,
+ const char *ctx_prefix)
+{
+ netdata_mutex_lock(&netdev_rename_mutex);
+
+ uint32_t hash = simple_hash(host_device);
+ struct netdev_rename *r = netdev_rename_find(host_device, hash);
+ if(!r) {
+ r = callocz(1, sizeof(struct netdev_rename));
+ r->host_device = strdupz(host_device);
+ r->container_device = strdupz(container_device);
+ r->container_name = strdupz(container_name);
+ r->ctx_prefix = strdupz(ctx_prefix);
+ r->chart_labels = rrdlabels_create();
+ rrdlabels_migrate_to_these(r->chart_labels, labels);
+ r->hash = hash;
+ r->next = netdev_rename_root;
+ r->processed = 0;
+ netdev_rename_root = r;
+ netdev_pending_renames++;
+ info("CGROUP: registered network interface rename for '%s' as '%s' under '%s'", r->host_device, r->container_device, r->container_name);
+ }
+ else {
+ if(strcmp(r->container_device, container_device) != 0 || strcmp(r->container_name, container_name) != 0) {
+ freez((void *) r->container_device);
+ freez((void *) r->container_name);
+
+ r->container_device = strdupz(container_device);
+ r->container_name = strdupz(container_name);
+
+ rrdlabels_migrate_to_these(r->chart_labels, labels);
+
+ r->processed = 0;
+ netdev_pending_renames++;
+ info("CGROUP: altered network interface rename for '%s' as '%s' under '%s'", r->host_device, r->container_device, r->container_name);
+ }
+ }
+
+ netdata_mutex_unlock(&netdev_rename_mutex);
+}
+
+// other threads can call this function to delete a rename to a netdev
+void netdev_rename_device_del(const char *host_device) {
+ netdata_mutex_lock(&netdev_rename_mutex);
+
+ struct netdev_rename *r, *last = NULL;
+
+ uint32_t hash = simple_hash(host_device);
+ for(r = netdev_rename_root; r ; last = r, r = r->next) {
+ if (r->hash == hash && !strcmp(host_device, r->host_device)) {
+ if (netdev_rename_root == r)
+ netdev_rename_root = r->next;
+ else if (last)
+ last->next = r->next;
+
+ if(!r->processed)
+ netdev_pending_renames--;
+
+ info("CGROUP: unregistered network interface rename for '%s' as '%s' under '%s'", r->host_device, r->container_device, r->container_name);
+
+ freez((void *) r->host_device);
+ freez((void *) r->container_name);
+ freez((void *) r->container_device);
+ freez((void *) r->ctx_prefix);
+ rrdlabels_destroy(r->chart_labels);
+ freez((void *) r);
+ break;
+ }
+ }
+
+ netdata_mutex_unlock(&netdev_rename_mutex);
+}
+
+static inline void netdev_rename_cgroup(struct netdev *d, struct netdev_rename *r) {
+ info("CGROUP: renaming network interface '%s' as '%s' under '%s'", r->host_device, r->container_device, r->container_name);
+
+ netdev_charts_release(d);
+ netdev_free_chart_strings(d);
+
+ char buffer[RRD_ID_LENGTH_MAX + 1];
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "cgroup_%s", r->container_name);
+ d->chart_type_net_bytes = strdupz(buffer);
+ d->chart_type_net_compressed = strdupz(buffer);
+ d->chart_type_net_drops = strdupz(buffer);
+ d->chart_type_net_errors = strdupz(buffer);
+ d->chart_type_net_events = strdupz(buffer);
+ d->chart_type_net_fifo = strdupz(buffer);
+ d->chart_type_net_packets = strdupz(buffer);
+ d->chart_type_net_speed = strdupz(buffer);
+ d->chart_type_net_duplex = strdupz(buffer);
+ d->chart_type_net_operstate = strdupz(buffer);
+ d->chart_type_net_carrier = strdupz(buffer);
+ d->chart_type_net_mtu = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_%s", r->container_device);
+ d->chart_id_net_bytes = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_compressed_%s", r->container_device);
+ d->chart_id_net_compressed = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_drops_%s", r->container_device);
+ d->chart_id_net_drops = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_errors_%s", r->container_device);
+ d->chart_id_net_errors = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_events_%s", r->container_device);
+ d->chart_id_net_events = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_fifo_%s", r->container_device);
+ d->chart_id_net_fifo = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_packets_%s", r->container_device);
+ d->chart_id_net_packets = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_speed_%s", r->container_device);
+ d->chart_id_net_speed = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_duplex_%s", r->container_device);
+ d->chart_id_net_duplex = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_operstate_%s", r->container_device);
+ d->chart_id_net_operstate = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_carrier_%s", r->container_device);
+ d->chart_id_net_carrier = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net_mtu_%s", r->container_device);
+ d->chart_id_net_mtu = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_net", r->ctx_prefix);
+ d->chart_ctx_net_bytes = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_compressed", r->ctx_prefix);
+ d->chart_ctx_net_compressed = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_drops", r->ctx_prefix);
+ d->chart_ctx_net_drops = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_errors", r->ctx_prefix);
+ d->chart_ctx_net_errors = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_events", r->ctx_prefix);
+ d->chart_ctx_net_events = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_fifo", r->ctx_prefix);
+ d->chart_ctx_net_fifo = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_packets", r->ctx_prefix);
+ d->chart_ctx_net_packets = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_speed", r->ctx_prefix);
+ d->chart_ctx_net_speed = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_duplex", r->ctx_prefix);
+ d->chart_ctx_net_duplex = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_operstate", r->ctx_prefix);
+ d->chart_ctx_net_operstate = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_carrier", r->ctx_prefix);
+ d->chart_ctx_net_carrier = strdupz(buffer);
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%scgroup.net_mtu", r->ctx_prefix);
+ d->chart_ctx_net_mtu = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "net %s", r->container_device);
+ d->chart_family = strdupz(buffer);
+
+ rrdlabels_copy(d->chart_labels, r->chart_labels);
+
+ d->priority = NETDATA_CHART_PRIO_CGROUP_NET_IFACE;
+ d->flipped = 1;
+}
+
+static inline void netdev_rename(struct netdev *d) {
+ struct netdev_rename *r = netdev_rename_find(d->name, d->hash);
+ if(unlikely(r && !r->processed)) {
+ netdev_rename_cgroup(d, r);
+ r->processed = 1;
+ netdev_pending_renames--;
+ }
+}
+
+static inline void netdev_rename_lock(struct netdev *d) {
+ netdata_mutex_lock(&netdev_rename_mutex);
+ netdev_rename(d);
+ netdata_mutex_unlock(&netdev_rename_mutex);
+}
+
+static inline void netdev_rename_all_lock(void) {
+ netdata_mutex_lock(&netdev_rename_mutex);
+
+ struct netdev *d;
+ for(d = netdev_root; d ; d = d->next)
+ netdev_rename(d);
+
+ netdev_pending_renames = 0;
+ netdata_mutex_unlock(&netdev_rename_mutex);
+}
+
+// ----------------------------------------------------------------------------
+// netdev data collection
+
+static void netdev_cleanup() {
+ if(likely(netdev_found == netdev_added)) return;
+
+ netdev_added = 0;
+ struct netdev *d = netdev_root, *last = NULL;
+ while(d) {
+ if(unlikely(!d->updated)) {
+ // info("Removing network device '%s', linked after '%s'", d->name, last?last->name:"ROOT");
+
+ if(netdev_last_used == d)
+ netdev_last_used = last;
+
+ struct netdev *t = d;
+
+ if(d == netdev_root || !last)
+ netdev_root = d = d->next;
+
+ else
+ last->next = d = d->next;
+
+ t->next = NULL;
+ netdev_free(t);
+ }
+ else {
+ netdev_added++;
+ last = d;
+ d->updated = 0;
+ d = d->next;
+ }
+ }
+}
+
+static struct netdev *get_netdev(const char *name) {
+ struct netdev *d;
+
+ uint32_t hash = simple_hash(name);
+
+ // search it, from the last position to the end
+ for(d = netdev_last_used ; d ; d = d->next) {
+ if(unlikely(hash == d->hash && !strcmp(name, d->name))) {
+ netdev_last_used = d->next;
+ return d;
+ }
+ }
+
+ // search it from the beginning to the last position we used
+ for(d = netdev_root ; d != netdev_last_used ; d = d->next) {
+ if(unlikely(hash == d->hash && !strcmp(name, d->name))) {
+ netdev_last_used = d->next;
+ return d;
+ }
+ }
+
+ // create a new one
+ d = callocz(1, sizeof(struct netdev));
+ d->name = strdupz(name);
+ d->hash = simple_hash(d->name);
+ d->len = strlen(d->name);
+ d->chart_labels = rrdlabels_create();
+
+ d->chart_type_net_bytes = strdupz("net");
+ d->chart_type_net_compressed = strdupz("net_compressed");
+ d->chart_type_net_drops = strdupz("net_drops");
+ d->chart_type_net_errors = strdupz("net_errors");
+ d->chart_type_net_events = strdupz("net_events");
+ d->chart_type_net_fifo = strdupz("net_fifo");
+ d->chart_type_net_packets = strdupz("net_packets");
+ d->chart_type_net_speed = strdupz("net_speed");
+ d->chart_type_net_duplex = strdupz("net_duplex");
+ d->chart_type_net_operstate = strdupz("net_operstate");
+ d->chart_type_net_carrier = strdupz("net_carrier");
+ d->chart_type_net_mtu = strdupz("net_mtu");
+
+ d->chart_id_net_bytes = strdupz(d->name);
+ d->chart_id_net_compressed = strdupz(d->name);
+ d->chart_id_net_drops = strdupz(d->name);
+ d->chart_id_net_errors = strdupz(d->name);
+ d->chart_id_net_events = strdupz(d->name);
+ d->chart_id_net_fifo = strdupz(d->name);
+ d->chart_id_net_packets = strdupz(d->name);
+ d->chart_id_net_speed = strdupz(d->name);
+ d->chart_id_net_duplex = strdupz(d->name);
+ d->chart_id_net_operstate = strdupz(d->name);
+ d->chart_id_net_carrier = strdupz(d->name);
+ d->chart_id_net_mtu = strdupz(d->name);
+
+ d->chart_ctx_net_bytes = strdupz("net.net");
+ d->chart_ctx_net_compressed = strdupz("net.compressed");
+ d->chart_ctx_net_drops = strdupz("net.drops");
+ d->chart_ctx_net_errors = strdupz("net.errors");
+ d->chart_ctx_net_events = strdupz("net.events");
+ d->chart_ctx_net_fifo = strdupz("net.fifo");
+ d->chart_ctx_net_packets = strdupz("net.packets");
+ d->chart_ctx_net_speed = strdupz("net.speed");
+ d->chart_ctx_net_duplex = strdupz("net.duplex");
+ d->chart_ctx_net_operstate = strdupz("net.operstate");
+ d->chart_ctx_net_carrier = strdupz("net.carrier");
+ d->chart_ctx_net_mtu = strdupz("net.mtu");
+
+ d->chart_family = strdupz(d->name);
+ d->priority = NETDATA_CHART_PRIO_FIRST_NET_IFACE;
+
+ netdev_rename_lock(d);
+
+ netdev_added++;
+
+ // link it to the end
+ if(netdev_root) {
+ struct netdev *e;
+ for(e = netdev_root; e->next ; e = e->next) ;
+ e->next = d;
+ }
+ else
+ netdev_root = d;
+
+ return d;
+}
+
+int do_proc_net_dev(int update_every, usec_t dt) {
+ (void)dt;
+ static SIMPLE_PATTERN *disabled_list = NULL;
+ static procfile *ff = NULL;
+ static int enable_new_interfaces = -1;
+ static int do_bandwidth = -1, do_packets = -1, do_errors = -1, do_drops = -1, do_fifo = -1, do_compressed = -1,
+ do_events = -1, do_speed = -1, do_duplex = -1, do_operstate = -1, do_carrier = -1, do_mtu = -1;
+ static char *path_to_sys_devices_virtual_net = NULL, *path_to_sys_class_net_speed = NULL,
+ *proc_net_dev_filename = NULL;
+ static char *path_to_sys_class_net_duplex = NULL;
+ static char *path_to_sys_class_net_operstate = NULL;
+ static char *path_to_sys_class_net_carrier = NULL;
+ static char *path_to_sys_class_net_mtu = NULL;
+
+ if(unlikely(enable_new_interfaces == -1)) {
+ char filename[FILENAME_MAX + 1];
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, (*netdata_configured_host_prefix)?"/proc/1/net/dev":"/proc/net/dev");
+ proc_net_dev_filename = config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/virtual/net/%s");
+ path_to_sys_devices_virtual_net = config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "path to get virtual interfaces", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/net/%s/speed");
+ path_to_sys_class_net_speed = config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "path to get net device speed", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/net/%s/duplex");
+ path_to_sys_class_net_duplex = config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "path to get net device duplex", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/net/%s/operstate");
+ path_to_sys_class_net_operstate = config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "path to get net device operstate", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/net/%s/carrier");
+ path_to_sys_class_net_carrier = config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "path to get net device carrier", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/net/%s/mtu");
+ path_to_sys_class_net_mtu = config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "path to get net device mtu", filename);
+
+
+ enable_new_interfaces = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "enable new interfaces detected at runtime", CONFIG_BOOLEAN_AUTO);
+
+ do_bandwidth = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "bandwidth for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_packets = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "packets for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_errors = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "errors for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_drops = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "drops for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_fifo = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "fifo for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_compressed = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "compressed packets for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_events = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "frames, collisions, carrier counters for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_speed = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "speed for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_duplex = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "duplex for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_operstate = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "operstate for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_carrier = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "carrier for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_mtu = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "mtu for all interfaces", CONFIG_BOOLEAN_AUTO);
+
+ disabled_list = simple_pattern_create(config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "disable by default interfaces matching", "lo fireqos* *-ifb fwpr* fwbr* fwln*"), NULL, SIMPLE_PATTERN_EXACT);
+ }
+
+ if(unlikely(!ff)) {
+ ff = procfile_open(proc_net_dev_filename, " \t,|", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ // rename all the devices, if we have pending renames
+ if(unlikely(netdev_pending_renames))
+ netdev_rename_all_lock();
+
+ netdev_found = 0;
+
+ kernel_uint_t system_rbytes = 0;
+ kernel_uint_t system_tbytes = 0;
+
+ size_t lines = procfile_lines(ff), l;
+ for(l = 2; l < lines ;l++) {
+ // require 17 words on each line
+ if(unlikely(procfile_linewords(ff, l) < 17)) continue;
+
+ char *name = procfile_lineword(ff, l, 0);
+ size_t len = strlen(name);
+ if(name[len - 1] == ':') name[len - 1] = '\0';
+
+ struct netdev *d = get_netdev(name);
+ d->updated = 1;
+ netdev_found++;
+
+ if(unlikely(!d->configured)) {
+ // this is the first time we see this interface
+
+ // remember we configured it
+ d->configured = 1;
+
+ d->enabled = enable_new_interfaces;
+
+ if(d->enabled)
+ d->enabled = !simple_pattern_matches(disabled_list, d->name);
+
+ char buffer[FILENAME_MAX + 1];
+
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_devices_virtual_net, d->name);
+ if (likely(access(buffer, R_OK) == 0)) {
+ d->virtual = 1;
+ rrdlabels_add(d->chart_labels, "interface_type", "virtual", RRDLABEL_SRC_AUTO|RRDLABEL_FLAG_PERMANENT);
+ }
+ else {
+ d->virtual = 0;
+ rrdlabels_add(d->chart_labels, "interface_type", "real", RRDLABEL_SRC_AUTO|RRDLABEL_FLAG_PERMANENT);
+ }
+ rrdlabels_add(d->chart_labels, "device", name, RRDLABEL_SRC_AUTO|RRDLABEL_FLAG_PERMANENT);
+
+ if(likely(!d->virtual)) {
+ // set the filename to get the interface speed
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_class_net_speed, d->name);
+ d->filename_speed = strdupz(buffer);
+
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_class_net_duplex, d->name);
+ d->filename_duplex = strdupz(buffer);
+ }
+
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_class_net_operstate, d->name);
+ d->filename_operstate = strdupz(buffer);
+
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_class_net_carrier, d->name);
+ d->filename_carrier = strdupz(buffer);
+
+ snprintfz(buffer, FILENAME_MAX, path_to_sys_class_net_mtu, d->name);
+ d->filename_mtu = strdupz(buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "plugin:proc:/proc/net/dev:%s", d->name);
+ d->enabled = config_get_boolean_ondemand(buffer, "enabled", d->enabled);
+ d->virtual = config_get_boolean(buffer, "virtual", d->virtual);
+
+ if(d->enabled == CONFIG_BOOLEAN_NO)
+ continue;
+
+ d->do_bandwidth = config_get_boolean_ondemand(buffer, "bandwidth", do_bandwidth);
+ d->do_packets = config_get_boolean_ondemand(buffer, "packets", do_packets);
+ d->do_errors = config_get_boolean_ondemand(buffer, "errors", do_errors);
+ d->do_drops = config_get_boolean_ondemand(buffer, "drops", do_drops);
+ d->do_fifo = config_get_boolean_ondemand(buffer, "fifo", do_fifo);
+ d->do_compressed = config_get_boolean_ondemand(buffer, "compressed", do_compressed);
+ d->do_events = config_get_boolean_ondemand(buffer, "events", do_events);
+ d->do_speed = config_get_boolean_ondemand(buffer, "speed", do_speed);
+ d->do_duplex = config_get_boolean_ondemand(buffer, "duplex", do_duplex);
+ d->do_operstate = config_get_boolean_ondemand(buffer, "operstate", do_operstate);
+ d->do_carrier = config_get_boolean_ondemand(buffer, "carrier", do_carrier);
+ d->do_mtu = config_get_boolean_ondemand(buffer, "mtu", do_mtu);
+ }
+
+ if(unlikely(!d->enabled))
+ continue;
+
+ if(likely(d->do_bandwidth != CONFIG_BOOLEAN_NO || !d->virtual)) {
+ d->rbytes = str2kernel_uint_t(procfile_lineword(ff, l, 1));
+ d->tbytes = str2kernel_uint_t(procfile_lineword(ff, l, 9));
+
+ if(likely(!d->virtual)) {
+ system_rbytes += d->rbytes;
+ system_tbytes += d->tbytes;
+ }
+ }
+
+ if(likely(d->do_packets != CONFIG_BOOLEAN_NO)) {
+ d->rpackets = str2kernel_uint_t(procfile_lineword(ff, l, 2));
+ d->rmulticast = str2kernel_uint_t(procfile_lineword(ff, l, 8));
+ d->tpackets = str2kernel_uint_t(procfile_lineword(ff, l, 10));
+ }
+
+ if(likely(d->do_errors != CONFIG_BOOLEAN_NO)) {
+ d->rerrors = str2kernel_uint_t(procfile_lineword(ff, l, 3));
+ d->terrors = str2kernel_uint_t(procfile_lineword(ff, l, 11));
+ }
+
+ if(likely(d->do_drops != CONFIG_BOOLEAN_NO)) {
+ d->rdrops = str2kernel_uint_t(procfile_lineword(ff, l, 4));
+ d->tdrops = str2kernel_uint_t(procfile_lineword(ff, l, 12));
+ }
+
+ if(likely(d->do_fifo != CONFIG_BOOLEAN_NO)) {
+ d->rfifo = str2kernel_uint_t(procfile_lineword(ff, l, 5));
+ d->tfifo = str2kernel_uint_t(procfile_lineword(ff, l, 13));
+ }
+
+ if(likely(d->do_compressed != CONFIG_BOOLEAN_NO)) {
+ d->rcompressed = str2kernel_uint_t(procfile_lineword(ff, l, 7));
+ d->tcompressed = str2kernel_uint_t(procfile_lineword(ff, l, 16));
+ }
+
+ if(likely(d->do_events != CONFIG_BOOLEAN_NO)) {
+ d->rframe = str2kernel_uint_t(procfile_lineword(ff, l, 6));
+ d->tcollisions = str2kernel_uint_t(procfile_lineword(ff, l, 14));
+ d->tcarrier = str2kernel_uint_t(procfile_lineword(ff, l, 15));
+ }
+
+ if ((d->do_carrier != CONFIG_BOOLEAN_NO ||
+ d->do_duplex != CONFIG_BOOLEAN_NO ||
+ d->do_speed != CONFIG_BOOLEAN_NO) &&
+ d->filename_carrier &&
+ (d->carrier_file_exists ||
+ now_monotonic_sec() - d->carrier_file_lost_time > READ_RETRY_PERIOD)) {
+ if (read_single_number_file(d->filename_carrier, &d->carrier)) {
+ if (d->carrier_file_exists)
+ error(
+ "Cannot refresh interface %s carrier state by reading '%s'. Next update is in %d seconds.",
+ d->name,
+ d->filename_carrier,
+ READ_RETRY_PERIOD);
+ d->carrier_file_exists = 0;
+ d->carrier_file_lost_time = now_monotonic_sec();
+ } else {
+ d->carrier_file_exists = 1;
+ d->carrier_file_lost_time = 0;
+ }
+ }
+
+ if (d->do_duplex != CONFIG_BOOLEAN_NO &&
+ d->filename_duplex &&
+ (d->carrier || d->carrier_file_exists) &&
+ (d->duplex_file_exists ||
+ now_monotonic_sec() - d->duplex_file_lost_time > READ_RETRY_PERIOD)) {
+ char buffer[STATE_LENGTH_MAX + 1];
+
+ if (read_file(d->filename_duplex, buffer, STATE_LENGTH_MAX)) {
+ if (d->duplex_file_exists)
+ error("Cannot refresh interface %s duplex state by reading '%s'.", d->name, d->filename_duplex);
+ d->duplex_file_exists = 0;
+ d->duplex_file_lost_time = now_monotonic_sec();
+ d->duplex = NETDEV_DUPLEX_UNKNOWN;
+ } else {
+ // values can be unknown, half or full -- just check the first letter for speed
+ if (buffer[0] == 'f')
+ d->duplex = NETDEV_DUPLEX_FULL;
+ else if (buffer[0] == 'h')
+ d->duplex = NETDEV_DUPLEX_HALF;
+ else
+ d->duplex = NETDEV_DUPLEX_UNKNOWN;
+ d->duplex_file_exists = 1;
+ d->duplex_file_lost_time = 0;
+ }
+ } else {
+ d->duplex = NETDEV_DUPLEX_UNKNOWN;
+ }
+
+ if(d->do_operstate != CONFIG_BOOLEAN_NO && d->filename_operstate) {
+ char buffer[STATE_LENGTH_MAX + 1], *trimmed_buffer;
+
+ if (read_file(d->filename_operstate, buffer, STATE_LENGTH_MAX)) {
+ error(
+ "Cannot refresh %s operstate by reading '%s'. Will not update its status anymore.",
+ d->name, d->filename_operstate);
+ freez(d->filename_operstate);
+ d->filename_operstate = NULL;
+ } else {
+ trimmed_buffer = trim(buffer);
+ d->operstate = get_operstate(trimmed_buffer);
+ }
+ }
+
+ if (d->do_mtu != CONFIG_BOOLEAN_NO && d->filename_mtu) {
+ if (read_single_number_file(d->filename_mtu, &d->mtu)) {
+ error(
+ "Cannot refresh mtu for interface %s by reading '%s'. Stop updating it.", d->name, d->filename_mtu);
+ freez(d->filename_mtu);
+ d->filename_mtu = NULL;
+ }
+ }
+
+ //info("PROC_NET_DEV: %s speed %zu, bytes %zu/%zu, packets %zu/%zu/%zu, errors %zu/%zu, drops %zu/%zu, fifo %zu/%zu, compressed %zu/%zu, rframe %zu, tcollisions %zu, tcarrier %zu"
+ // , d->name, d->speed
+ // , d->rbytes, d->tbytes
+ // , d->rpackets, d->tpackets, d->rmulticast
+ // , d->rerrors, d->terrors
+ // , d->rdrops, d->tdrops
+ // , d->rfifo, d->tfifo
+ // , d->rcompressed, d->tcompressed
+ // , d->rframe, d->tcollisions, d->tcarrier
+ // );
+
+ if(unlikely(d->do_bandwidth == CONFIG_BOOLEAN_AUTO &&
+ (d->rbytes || d->tbytes || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ d->do_bandwidth = CONFIG_BOOLEAN_YES;
+
+ if(d->do_bandwidth == CONFIG_BOOLEAN_YES) {
+ if(unlikely(!d->st_bandwidth)) {
+
+ d->st_bandwidth = rrdset_create_localhost(
+ d->chart_type_net_bytes
+ , d->chart_id_net_bytes
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_bytes
+ , "Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_update_rrdlabels(d->st_bandwidth, d->chart_labels);
+
+ d->rd_rbytes = rrddim_add(d->st_bandwidth, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_tbytes = rrddim_add(d->st_bandwidth, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+
+ if(d->flipped) {
+ // flip receive/transmit
+
+ RRDDIM *td = d->rd_rbytes;
+ d->rd_rbytes = d->rd_tbytes;
+ d->rd_tbytes = td;
+ }
+ }
+
+ rrddim_set_by_pointer(d->st_bandwidth, d->rd_rbytes, (collected_number)d->rbytes);
+ rrddim_set_by_pointer(d->st_bandwidth, d->rd_tbytes, (collected_number)d->tbytes);
+ rrdset_done(d->st_bandwidth);
+
+ // update the interface speed
+ if(d->filename_speed) {
+ if(unlikely(!d->chart_var_speed)) {
+ d->chart_var_speed =
+ rrdsetvar_custom_chart_variable_add_and_acquire(d->st_bandwidth, "nic_speed_max");
+ if(!d->chart_var_speed) {
+ error(
+ "Cannot create interface %s chart variable 'nic_speed_max'. Will not update its speed anymore.",
+ d->name);
+ freez(d->filename_speed);
+ d->filename_speed = NULL;
+ }
+ }
+
+ if (d->filename_speed && d->chart_var_speed) {
+ int ret = 0;
+
+ if ((d->carrier || d->carrier_file_exists) &&
+ (d->speed_file_exists || now_monotonic_sec() - d->speed_file_lost_time > READ_RETRY_PERIOD)) {
+ ret = read_single_number_file(d->filename_speed, (unsigned long long *) &d->speed);
+ } else {
+ d->speed = 0; // TODO: this is wrong, shouldn't use 0 value, but NULL.
+ }
+
+ if(ret) {
+ if (d->speed_file_exists)
+ error("Cannot refresh interface %s speed by reading '%s'.", d->name, d->filename_speed);
+ d->speed_file_exists = 0;
+ d->speed_file_lost_time = now_monotonic_sec();
+ }
+ else {
+ if(d->do_speed != CONFIG_BOOLEAN_NO) {
+ if(unlikely(!d->st_speed)) {
+ d->st_speed = rrdset_create_localhost(
+ d->chart_type_net_speed
+ , d->chart_id_net_speed
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_speed
+ , "Interface Speed"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 7
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_speed, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_speed, d->chart_labels);
+
+ d->rd_speed = rrddim_add(d->st_speed, "speed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(d->st_speed, d->rd_speed, (collected_number)d->speed * KILOBITS_IN_A_MEGABIT);
+ rrdset_done(d->st_speed);
+ }
+
+ rrdsetvar_custom_chart_variable_set(
+ d->st_bandwidth, d->chart_var_speed, (NETDATA_DOUBLE)d->speed * KILOBITS_IN_A_MEGABIT);
+
+ if (d->speed) {
+ d->speed_file_exists = 1;
+ d->speed_file_lost_time = 0;
+ }
+ }
+ }
+ }
+ }
+
+ if(d->do_duplex != CONFIG_BOOLEAN_NO && d->filename_duplex) {
+ if(unlikely(!d->st_duplex)) {
+ d->st_duplex = rrdset_create_localhost(
+ d->chart_type_net_duplex
+ , d->chart_id_net_duplex
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_duplex
+ , "Interface Duplex State"
+ , "state"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 8
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_duplex, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_duplex, d->chart_labels);
+
+ d->rd_duplex_full = rrddim_add(d->st_duplex, "full", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_duplex_half = rrddim_add(d->st_duplex, "half", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_duplex_unknown = rrddim_add(d->st_duplex, "unknown", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(d->st_duplex, d->rd_duplex_full, (collected_number)(d->duplex == NETDEV_DUPLEX_FULL));
+ rrddim_set_by_pointer(d->st_duplex, d->rd_duplex_half, (collected_number)(d->duplex == NETDEV_DUPLEX_HALF));
+ rrddim_set_by_pointer(d->st_duplex, d->rd_duplex_unknown, (collected_number)(d->duplex == NETDEV_DUPLEX_UNKNOWN));
+ rrdset_done(d->st_duplex);
+ }
+
+ if(d->do_operstate != CONFIG_BOOLEAN_NO && d->filename_operstate) {
+ if(unlikely(!d->st_operstate)) {
+ d->st_operstate = rrdset_create_localhost(
+ d->chart_type_net_operstate
+ , d->chart_id_net_operstate
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_operstate
+ , "Interface Operational State"
+ , "state"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 9
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_operstate, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_operstate, d->chart_labels);
+
+ d->rd_operstate_up = rrddim_add(d->st_operstate, "up", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_operstate_down = rrddim_add(d->st_operstate, "down", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_operstate_notpresent = rrddim_add(d->st_operstate, "notpresent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_operstate_lowerlayerdown = rrddim_add(d->st_operstate, "lowerlayerdown", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_operstate_testing = rrddim_add(d->st_operstate, "testing", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_operstate_dormant = rrddim_add(d->st_operstate, "dormant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_operstate_unknown = rrddim_add(d->st_operstate, "unknown", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(d->st_operstate, d->rd_operstate_up, (collected_number)(d->operstate == NETDEV_OPERSTATE_UP));
+ rrddim_set_by_pointer(d->st_operstate, d->rd_operstate_down, (collected_number)(d->operstate == NETDEV_OPERSTATE_DOWN));
+ rrddim_set_by_pointer(d->st_operstate, d->rd_operstate_notpresent, (collected_number)(d->operstate == NETDEV_OPERSTATE_NOTPRESENT));
+ rrddim_set_by_pointer(d->st_operstate, d->rd_operstate_lowerlayerdown, (collected_number)(d->operstate == NETDEV_OPERSTATE_LOWERLAYERDOWN));
+ rrddim_set_by_pointer(d->st_operstate, d->rd_operstate_testing, (collected_number)(d->operstate == NETDEV_OPERSTATE_TESTING));
+ rrddim_set_by_pointer(d->st_operstate, d->rd_operstate_dormant, (collected_number)(d->operstate == NETDEV_OPERSTATE_DORMANT));
+ rrddim_set_by_pointer(d->st_operstate, d->rd_operstate_unknown, (collected_number)(d->operstate == NETDEV_OPERSTATE_UNKNOWN));
+ rrdset_done(d->st_operstate);
+ }
+
+ if(d->do_carrier != CONFIG_BOOLEAN_NO && d->carrier_file_exists) {
+ if(unlikely(!d->st_carrier)) {
+ d->st_carrier = rrdset_create_localhost(
+ d->chart_type_net_carrier
+ , d->chart_id_net_carrier
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_carrier
+ , "Interface Physical Link State"
+ , "state"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 10
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_carrier, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_carrier, d->chart_labels);
+
+ d->rd_carrier_up = rrddim_add(d->st_carrier, "up", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_carrier_down = rrddim_add(d->st_carrier, "down", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(d->st_carrier, d->rd_carrier_up, (collected_number)(d->carrier == 1));
+ rrddim_set_by_pointer(d->st_carrier, d->rd_carrier_down, (collected_number)(d->carrier != 1));
+ rrdset_done(d->st_carrier);
+ }
+
+ if(d->do_mtu != CONFIG_BOOLEAN_NO && d->filename_mtu) {
+ if(unlikely(!d->st_mtu)) {
+ d->st_mtu = rrdset_create_localhost(
+ d->chart_type_net_mtu
+ , d->chart_id_net_mtu
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_mtu
+ , "Interface MTU"
+ , "octets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 11
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_mtu, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_mtu, d->chart_labels);
+
+ d->rd_mtu = rrddim_add(d->st_mtu, "mtu", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(d->st_mtu, d->rd_mtu, (collected_number)d->mtu);
+ rrdset_done(d->st_mtu);
+ }
+
+ if(unlikely(d->do_packets == CONFIG_BOOLEAN_AUTO &&
+ (d->rpackets || d->tpackets || d->rmulticast || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ d->do_packets = CONFIG_BOOLEAN_YES;
+
+ if(d->do_packets == CONFIG_BOOLEAN_YES) {
+ if(unlikely(!d->st_packets)) {
+
+ d->st_packets = rrdset_create_localhost(
+ d->chart_type_net_packets
+ , d->chart_id_net_packets
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_packets
+ , "Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 1
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_packets, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_packets, d->chart_labels);
+
+ d->rd_rpackets = rrddim_add(d->st_packets, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_tpackets = rrddim_add(d->st_packets, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_rmulticast = rrddim_add(d->st_packets, "multicast", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(d->flipped) {
+ // flip receive/transmit
+
+ RRDDIM *td = d->rd_rpackets;
+ d->rd_rpackets = d->rd_tpackets;
+ d->rd_tpackets = td;
+ }
+ }
+
+ rrddim_set_by_pointer(d->st_packets, d->rd_rpackets, (collected_number)d->rpackets);
+ rrddim_set_by_pointer(d->st_packets, d->rd_tpackets, (collected_number)d->tpackets);
+ rrddim_set_by_pointer(d->st_packets, d->rd_rmulticast, (collected_number)d->rmulticast);
+ rrdset_done(d->st_packets);
+ }
+
+ if(unlikely(d->do_errors == CONFIG_BOOLEAN_AUTO &&
+ (d->rerrors || d->terrors || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ d->do_errors = CONFIG_BOOLEAN_YES;
+
+ if(d->do_errors == CONFIG_BOOLEAN_YES) {
+ if(unlikely(!d->st_errors)) {
+
+ d->st_errors = rrdset_create_localhost(
+ d->chart_type_net_errors
+ , d->chart_id_net_errors
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_errors
+ , "Interface Errors"
+ , "errors/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 2
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_errors, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_errors, d->chart_labels);
+
+ d->rd_rerrors = rrddim_add(d->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_terrors = rrddim_add(d->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(d->flipped) {
+ // flip receive/transmit
+
+ RRDDIM *td = d->rd_rerrors;
+ d->rd_rerrors = d->rd_terrors;
+ d->rd_terrors = td;
+ }
+ }
+
+ rrddim_set_by_pointer(d->st_errors, d->rd_rerrors, (collected_number)d->rerrors);
+ rrddim_set_by_pointer(d->st_errors, d->rd_terrors, (collected_number)d->terrors);
+ rrdset_done(d->st_errors);
+ }
+
+ if(unlikely(d->do_drops == CONFIG_BOOLEAN_AUTO &&
+ (d->rdrops || d->tdrops || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ d->do_drops = CONFIG_BOOLEAN_YES;
+
+ if(d->do_drops == CONFIG_BOOLEAN_YES) {
+ if(unlikely(!d->st_drops)) {
+
+ d->st_drops = rrdset_create_localhost(
+ d->chart_type_net_drops
+ , d->chart_id_net_drops
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_drops
+ , "Interface Drops"
+ , "drops/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 3
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_drops, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_drops, d->chart_labels);
+
+ d->rd_rdrops = rrddim_add(d->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_tdrops = rrddim_add(d->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(d->flipped) {
+ // flip receive/transmit
+
+ RRDDIM *td = d->rd_rdrops;
+ d->rd_rdrops = d->rd_tdrops;
+ d->rd_tdrops = td;
+ }
+ }
+
+ rrddim_set_by_pointer(d->st_drops, d->rd_rdrops, (collected_number)d->rdrops);
+ rrddim_set_by_pointer(d->st_drops, d->rd_tdrops, (collected_number)d->tdrops);
+ rrdset_done(d->st_drops);
+ }
+
+ if(unlikely(d->do_fifo == CONFIG_BOOLEAN_AUTO &&
+ (d->rfifo || d->tfifo || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ d->do_fifo = CONFIG_BOOLEAN_YES;
+
+ if(d->do_fifo == CONFIG_BOOLEAN_YES) {
+ if(unlikely(!d->st_fifo)) {
+
+ d->st_fifo = rrdset_create_localhost(
+ d->chart_type_net_fifo
+ , d->chart_id_net_fifo
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_fifo
+ , "Interface FIFO Buffer Errors"
+ , "errors"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 4
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_fifo, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_fifo, d->chart_labels);
+
+ d->rd_rfifo = rrddim_add(d->st_fifo, "receive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_tfifo = rrddim_add(d->st_fifo, "transmit", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(d->flipped) {
+ // flip receive/transmit
+
+ RRDDIM *td = d->rd_rfifo;
+ d->rd_rfifo = d->rd_tfifo;
+ d->rd_tfifo = td;
+ }
+ }
+
+ rrddim_set_by_pointer(d->st_fifo, d->rd_rfifo, (collected_number)d->rfifo);
+ rrddim_set_by_pointer(d->st_fifo, d->rd_tfifo, (collected_number)d->tfifo);
+ rrdset_done(d->st_fifo);
+ }
+
+ if(unlikely(d->do_compressed == CONFIG_BOOLEAN_AUTO &&
+ (d->rcompressed || d->tcompressed || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ d->do_compressed = CONFIG_BOOLEAN_YES;
+
+ if(d->do_compressed == CONFIG_BOOLEAN_YES) {
+ if(unlikely(!d->st_compressed)) {
+
+ d->st_compressed = rrdset_create_localhost(
+ d->chart_type_net_compressed
+ , d->chart_id_net_compressed
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_compressed
+ , "Compressed Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 5
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_compressed, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_compressed, d->chart_labels);
+
+ d->rd_rcompressed = rrddim_add(d->st_compressed, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_tcompressed = rrddim_add(d->st_compressed, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(d->flipped) {
+ // flip receive/transmit
+
+ RRDDIM *td = d->rd_rcompressed;
+ d->rd_rcompressed = d->rd_tcompressed;
+ d->rd_tcompressed = td;
+ }
+ }
+
+ rrddim_set_by_pointer(d->st_compressed, d->rd_rcompressed, (collected_number)d->rcompressed);
+ rrddim_set_by_pointer(d->st_compressed, d->rd_tcompressed, (collected_number)d->tcompressed);
+ rrdset_done(d->st_compressed);
+ }
+
+ if(unlikely(d->do_events == CONFIG_BOOLEAN_AUTO &&
+ (d->rframe || d->tcollisions || d->tcarrier || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+ d->do_events = CONFIG_BOOLEAN_YES;
+
+ if(d->do_events == CONFIG_BOOLEAN_YES) {
+ if(unlikely(!d->st_events)) {
+
+ d->st_events = rrdset_create_localhost(
+ d->chart_type_net_events
+ , d->chart_id_net_events
+ , NULL
+ , d->chart_family
+ , d->chart_ctx_net_events
+ , "Network Interface Events"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , d->priority + 6
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(d->st_events, RRDSET_FLAG_DETAIL);
+
+ rrdset_update_rrdlabels(d->st_events, d->chart_labels);
+
+ d->rd_rframe = rrddim_add(d->st_events, "frames", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_tcollisions = rrddim_add(d->st_events, "collisions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ d->rd_tcarrier = rrddim_add(d->st_events, "carrier", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(d->st_events, d->rd_rframe, (collected_number)d->rframe);
+ rrddim_set_by_pointer(d->st_events, d->rd_tcollisions, (collected_number)d->tcollisions);
+ rrddim_set_by_pointer(d->st_events, d->rd_tcarrier, (collected_number)d->tcarrier);
+ rrdset_done(d->st_events);
+ }
+ }
+
+ if(do_bandwidth == CONFIG_BOOLEAN_YES || (do_bandwidth == CONFIG_BOOLEAN_AUTO &&
+ (system_rbytes || system_tbytes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_bandwidth = CONFIG_BOOLEAN_YES;
+ static RRDSET *st_system_net = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_system_net)) {
+ st_system_net = rrdset_create_localhost(
+ "system"
+ , "net"
+ , NULL
+ , "network"
+ , NULL
+ , "Physical Network Interfaces Aggregated Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETDEV_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_NET
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st_system_net, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_system_net, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_system_net, rd_in, (collected_number)system_rbytes);
+ rrddim_set_by_pointer(st_system_net, rd_out, (collected_number)system_tbytes);
+
+ rrdset_done(st_system_net);
+ }
+
+ netdev_cleanup();
+
+ return 0;
+}
+
+static void netdev_main_cleanup(void *ptr)
+{
+ UNUSED(ptr);
+
+ info("cleaning up...");
+
+ worker_unregister();
+}
+
+void *netdev_main(void *ptr)
+{
+ worker_register("NETDEV");
+ worker_register_job_name(0, "netdev");
+
+ netdata_thread_cleanup_push(netdev_main_cleanup, ptr);
+
+ usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+
+ while (!netdata_exit) {
+ worker_is_idle();
+ usec_t hb_dt = heartbeat_next(&hb, step);
+
+ if (unlikely(netdata_exit))
+ break;
+
+ worker_is_busy(0);
+ if(do_proc_net_dev(localhost->rrd_update_every, hb_dt))
+ break;
+ }
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/proc.plugin/proc_net_ip_vs_stats.c b/collectors/proc.plugin/proc_net_ip_vs_stats.c
new file mode 100644
index 0000000..2b9c933
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_ip_vs_stats.c
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define RRD_TYPE_NET_IPVS "ipvs"
+#define PLUGIN_PROC_MODULE_NET_IPVS_NAME "/proc/net/ip_vs_stats"
+#define CONFIG_SECTION_PLUGIN_PROC_NET_IPVS "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_NET_IPVS_NAME
+
+int do_proc_net_ip_vs_stats(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_bandwidth = -1, do_sockets = -1, do_packets = -1;
+ static procfile *ff = NULL;
+
+ if(do_bandwidth == -1) do_bandwidth = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_NET_IPVS, "IPVS bandwidth", 1);
+ if(do_sockets == -1) do_sockets = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_NET_IPVS, "IPVS connections", 1);
+ if(do_packets == -1) do_packets = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_NET_IPVS, "IPVS packets", 1);
+
+ if(!ff) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/ip_vs_stats");
+ ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_NET_IPVS, "filename to monitor", filename), " \t,:|", PROCFILE_FLAG_DEFAULT);
+ }
+ if(!ff) return 1;
+
+ ff = procfile_readall(ff);
+ if(!ff) return 0; // we return 0, so that we will retry to open it next time
+
+ // make sure we have 3 lines
+ if(procfile_lines(ff) < 3) return 1;
+
+ // make sure we have 5 words on the 3rd line
+ if(procfile_linewords(ff, 2) < 5) return 1;
+
+ unsigned long long entries, InPackets, OutPackets, InBytes, OutBytes;
+
+ entries = strtoull(procfile_lineword(ff, 2, 0), NULL, 16);
+ InPackets = strtoull(procfile_lineword(ff, 2, 1), NULL, 16);
+ OutPackets = strtoull(procfile_lineword(ff, 2, 2), NULL, 16);
+ InBytes = strtoull(procfile_lineword(ff, 2, 3), NULL, 16);
+ OutBytes = strtoull(procfile_lineword(ff, 2, 4), NULL, 16);
+
+ if(do_sockets) {
+ static RRDSET *st = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_IPVS
+ , "sockets"
+ , NULL
+ , RRD_TYPE_NET_IPVS
+ , NULL
+ , "IPVS New Connections"
+ , "connections/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_IPVS_NAME
+ , NETDATA_CHART_PRIO_IPVS_SOCKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "connections", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "connections", entries);
+ rrdset_done(st);
+ }
+
+ if(do_packets) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_IPVS
+ , "packets"
+ , NULL
+ , RRD_TYPE_NET_IPVS
+ , NULL
+ , "IPVS Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_IPVS_NAME
+ , NETDATA_CHART_PRIO_IPVS_PACKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "received", InPackets);
+ rrddim_set(st, "sent", OutPackets);
+ rrdset_done(st);
+ }
+
+ if(do_bandwidth) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_IPVS
+ , "net"
+ , NULL
+ , RRD_TYPE_NET_IPVS
+ , NULL
+ , "IPVS Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_IPVS_NAME
+ , NETDATA_CHART_PRIO_IPVS_NET
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "received", InBytes);
+ rrddim_set(st, "sent", OutBytes);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_netstat.c b/collectors/proc.plugin/proc_net_netstat.c
new file mode 100644
index 0000000..f7635e3
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_netstat.c
@@ -0,0 +1,3110 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define RRD_TYPE_NET_NETSTAT "ip"
+#define RRD_TYPE_NET_SNMP "ipv4"
+#define RRD_TYPE_NET_SNMP6 "ipv6"
+#define PLUGIN_PROC_MODULE_NETSTAT_NAME "/proc/net/netstat"
+#define CONFIG_SECTION_PLUGIN_PROC_NETSTAT "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_NETSTAT_NAME
+
+static struct proc_net_snmp {
+ // kernel_uint_t ip_Forwarding;
+ kernel_uint_t ip_DefaultTTL;
+ kernel_uint_t ip_InReceives;
+ kernel_uint_t ip_InHdrErrors;
+ kernel_uint_t ip_InAddrErrors;
+ kernel_uint_t ip_ForwDatagrams;
+ kernel_uint_t ip_InUnknownProtos;
+ kernel_uint_t ip_InDiscards;
+ kernel_uint_t ip_InDelivers;
+ kernel_uint_t ip_OutRequests;
+ kernel_uint_t ip_OutDiscards;
+ kernel_uint_t ip_OutNoRoutes;
+ kernel_uint_t ip_ReasmTimeout;
+ kernel_uint_t ip_ReasmReqds;
+ kernel_uint_t ip_ReasmOKs;
+ kernel_uint_t ip_ReasmFails;
+ kernel_uint_t ip_FragOKs;
+ kernel_uint_t ip_FragFails;
+ kernel_uint_t ip_FragCreates;
+
+ kernel_uint_t icmp_InMsgs;
+ kernel_uint_t icmp_OutMsgs;
+ kernel_uint_t icmp_InErrors;
+ kernel_uint_t icmp_OutErrors;
+ kernel_uint_t icmp_InCsumErrors;
+
+ kernel_uint_t icmpmsg_InEchoReps;
+ kernel_uint_t icmpmsg_OutEchoReps;
+ kernel_uint_t icmpmsg_InDestUnreachs;
+ kernel_uint_t icmpmsg_OutDestUnreachs;
+ kernel_uint_t icmpmsg_InRedirects;
+ kernel_uint_t icmpmsg_OutRedirects;
+ kernel_uint_t icmpmsg_InEchos;
+ kernel_uint_t icmpmsg_OutEchos;
+ kernel_uint_t icmpmsg_InRouterAdvert;
+ kernel_uint_t icmpmsg_OutRouterAdvert;
+ kernel_uint_t icmpmsg_InRouterSelect;
+ kernel_uint_t icmpmsg_OutRouterSelect;
+ kernel_uint_t icmpmsg_InTimeExcds;
+ kernel_uint_t icmpmsg_OutTimeExcds;
+ kernel_uint_t icmpmsg_InParmProbs;
+ kernel_uint_t icmpmsg_OutParmProbs;
+ kernel_uint_t icmpmsg_InTimestamps;
+ kernel_uint_t icmpmsg_OutTimestamps;
+ kernel_uint_t icmpmsg_InTimestampReps;
+ kernel_uint_t icmpmsg_OutTimestampReps;
+
+ //kernel_uint_t tcp_RtoAlgorithm;
+ //kernel_uint_t tcp_RtoMin;
+ //kernel_uint_t tcp_RtoMax;
+ ssize_t tcp_MaxConn;
+ kernel_uint_t tcp_ActiveOpens;
+ kernel_uint_t tcp_PassiveOpens;
+ kernel_uint_t tcp_AttemptFails;
+ kernel_uint_t tcp_EstabResets;
+ kernel_uint_t tcp_CurrEstab;
+ kernel_uint_t tcp_InSegs;
+ kernel_uint_t tcp_OutSegs;
+ kernel_uint_t tcp_RetransSegs;
+ kernel_uint_t tcp_InErrs;
+ kernel_uint_t tcp_OutRsts;
+ kernel_uint_t tcp_InCsumErrors;
+
+ kernel_uint_t udp_InDatagrams;
+ kernel_uint_t udp_NoPorts;
+ kernel_uint_t udp_InErrors;
+ kernel_uint_t udp_OutDatagrams;
+ kernel_uint_t udp_RcvbufErrors;
+ kernel_uint_t udp_SndbufErrors;
+ kernel_uint_t udp_InCsumErrors;
+ kernel_uint_t udp_IgnoredMulti;
+
+ kernel_uint_t udplite_InDatagrams;
+ kernel_uint_t udplite_NoPorts;
+ kernel_uint_t udplite_InErrors;
+ kernel_uint_t udplite_OutDatagrams;
+ kernel_uint_t udplite_RcvbufErrors;
+ kernel_uint_t udplite_SndbufErrors;
+ kernel_uint_t udplite_InCsumErrors;
+ kernel_uint_t udplite_IgnoredMulti;
+} snmp_root = { 0 };
+
+static void parse_line_pair(procfile *ff_netstat, ARL_BASE *base, size_t header_line, size_t values_line) {
+ size_t hwords = procfile_linewords(ff_netstat, header_line);
+ size_t vwords = procfile_linewords(ff_netstat, values_line);
+ size_t w;
+
+ if(unlikely(vwords > hwords)) {
+ error("File /proc/net/netstat on header line %zu has %zu words, but on value line %zu has %zu words.", header_line, hwords, values_line, vwords);
+ vwords = hwords;
+ }
+
+ for(w = 1; w < vwords ;w++) {
+ if(unlikely(arl_check(base, procfile_lineword(ff_netstat, header_line, w), procfile_lineword(ff_netstat, values_line, w))))
+ break;
+ }
+}
+
+int do_proc_net_netstat(int update_every, usec_t dt) {
+ (void)dt;
+
+ static int do_bandwidth = -1, do_inerrors = -1, do_mcast = -1, do_bcast = -1, do_mcast_p = -1, do_bcast_p = -1, do_ecn = -1, \
+ do_tcpext_reorder = -1, do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, do_tcpext_memory = -1,
+ do_tcpext_syn_queue = -1, do_tcpext_accept_queue = -1;
+
+ static int do_ip_packets = -1, do_ip_fragsout = -1, do_ip_fragsin = -1, do_ip_errors = -1,
+ do_tcp_sockets = -1, do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1, do_tcp_opens = -1,
+ do_udp_packets = -1, do_udp_errors = -1, do_icmp_packets = -1, do_icmpmsg = -1, do_udplite_packets = -1;
+
+ static int do_ip6_packets = -1, do_ip6_fragsout = -1, do_ip6_fragsin = -1, do_ip6_errors = -1,
+ do_ip6_udplite_packets = -1, do_ip6_udplite_errors = -1, do_ip6_udp_packets = -1, do_ip6_udp_errors = -1,
+ do_ip6_bandwidth = -1, do_ip6_mcast = -1, do_ip6_bcast = -1, do_ip6_mcast_p = -1, do_ip6_icmp = -1,
+ do_ip6_icmp_redir = -1, do_ip6_icmp_errors = -1, do_ip6_icmp_echos = -1, do_ip6_icmp_groupmemb = -1,
+ do_ip6_icmp_router = -1, do_ip6_icmp_neighbor = -1, do_ip6_icmp_mldv2 = -1, do_ip6_icmp_types = -1,
+ do_ip6_ect = -1;
+
+ static uint32_t hash_ipext = 0, hash_tcpext = 0;
+ static uint32_t hash_ip = 0, hash_icmp = 0, hash_tcp = 0, hash_udp = 0, hash_icmpmsg = 0, hash_udplite = 0;
+
+ static procfile *ff_netstat = NULL;
+ static procfile *ff_snmp = NULL;
+ static procfile *ff_snmp6 = NULL;
+
+ static ARL_BASE *arl_tcpext = NULL;
+ static ARL_BASE *arl_ipext = NULL;
+
+ static ARL_BASE *arl_ip = NULL;
+ static ARL_BASE *arl_icmp = NULL;
+ static ARL_BASE *arl_icmpmsg = NULL;
+ static ARL_BASE *arl_tcp = NULL;
+ static ARL_BASE *arl_udp = NULL;
+ static ARL_BASE *arl_udplite = NULL;
+
+ static ARL_BASE *arl_ipv6 = NULL;
+
+ static const RRDVAR_ACQUIRED *tcp_max_connections_var = NULL;
+
+ // --------------------------------------------------------------------
+ // IP
+
+ // IP bandwidth
+ static unsigned long long ipext_InOctets = 0;
+ static unsigned long long ipext_OutOctets = 0;
+
+ // IP input errors
+ static unsigned long long ipext_InNoRoutes = 0;
+ static unsigned long long ipext_InTruncatedPkts = 0;
+ static unsigned long long ipext_InCsumErrors = 0;
+
+ // IP multicast bandwidth
+ static unsigned long long ipext_InMcastOctets = 0;
+ static unsigned long long ipext_OutMcastOctets = 0;
+
+ // IP multicast packets
+ static unsigned long long ipext_InMcastPkts = 0;
+ static unsigned long long ipext_OutMcastPkts = 0;
+
+ // IP broadcast bandwidth
+ static unsigned long long ipext_InBcastOctets = 0;
+ static unsigned long long ipext_OutBcastOctets = 0;
+
+ // IP broadcast packets
+ static unsigned long long ipext_InBcastPkts = 0;
+ static unsigned long long ipext_OutBcastPkts = 0;
+
+ // IP ECN
+ static unsigned long long ipext_InNoECTPkts = 0;
+ static unsigned long long ipext_InECT1Pkts = 0;
+ static unsigned long long ipext_InECT0Pkts = 0;
+ static unsigned long long ipext_InCEPkts = 0;
+
+ // --------------------------------------------------------------------
+ // IP TCP
+
+ // IP TCP Reordering
+ static unsigned long long tcpext_TCPRenoReorder = 0;
+ static unsigned long long tcpext_TCPFACKReorder = 0;
+ static unsigned long long tcpext_TCPSACKReorder = 0;
+ static unsigned long long tcpext_TCPTSReorder = 0;
+
+ // IP TCP SYN Cookies
+ static unsigned long long tcpext_SyncookiesSent = 0;
+ static unsigned long long tcpext_SyncookiesRecv = 0;
+ static unsigned long long tcpext_SyncookiesFailed = 0;
+
+ // IP TCP Out Of Order Queue
+ // http://www.spinics.net/lists/netdev/msg204696.html
+ static unsigned long long tcpext_TCPOFOQueue = 0; // Number of packets queued in OFO queue
+ static unsigned long long tcpext_TCPOFODrop = 0; // Number of packets meant to be queued in OFO but dropped because socket rcvbuf limit hit.
+ static unsigned long long tcpext_TCPOFOMerge = 0; // Number of packets in OFO that were merged with other packets.
+ static unsigned long long tcpext_OfoPruned = 0; // packets dropped from out-of-order queue because of socket buffer overrun
+
+ // IP TCP connection resets
+ // https://github.com/ecki/net-tools/blob/bd8bceaed2311651710331a7f8990c3e31be9840/statistics.c
+ static unsigned long long tcpext_TCPAbortOnData = 0; // connections reset due to unexpected data
+ static unsigned long long tcpext_TCPAbortOnClose = 0; // connections reset due to early user close
+ static unsigned long long tcpext_TCPAbortOnMemory = 0; // connections aborted due to memory pressure
+ static unsigned long long tcpext_TCPAbortOnTimeout = 0; // connections aborted due to timeout
+ static unsigned long long tcpext_TCPAbortOnLinger = 0; // connections aborted after user close in linger timeout
+ static unsigned long long tcpext_TCPAbortFailed = 0; // times unable to send RST due to no memory
+
+ // https://perfchron.com/2015/12/26/investigating-linux-network-issues-with-netstat-and-nstat/
+ static unsigned long long tcpext_ListenOverflows = 0; // times the listen queue of a socket overflowed
+ static unsigned long long tcpext_ListenDrops = 0; // SYNs to LISTEN sockets ignored
+
+ // IP TCP memory pressures
+ static unsigned long long tcpext_TCPMemoryPressures = 0;
+
+ static unsigned long long tcpext_TCPReqQFullDrop = 0;
+ static unsigned long long tcpext_TCPReqQFullDoCookies = 0;
+
+ static unsigned long long tcpext_TCPSynRetrans = 0;
+
+ // IPv6
+ static unsigned long long Ip6InReceives = 0ULL;
+ static unsigned long long Ip6InHdrErrors = 0ULL;
+ static unsigned long long Ip6InTooBigErrors = 0ULL;
+ static unsigned long long Ip6InNoRoutes = 0ULL;
+ static unsigned long long Ip6InAddrErrors = 0ULL;
+ static unsigned long long Ip6InUnknownProtos = 0ULL;
+ static unsigned long long Ip6InTruncatedPkts = 0ULL;
+ static unsigned long long Ip6InDiscards = 0ULL;
+ static unsigned long long Ip6InDelivers = 0ULL;
+ static unsigned long long Ip6OutForwDatagrams = 0ULL;
+ static unsigned long long Ip6OutRequests = 0ULL;
+ static unsigned long long Ip6OutDiscards = 0ULL;
+ static unsigned long long Ip6OutNoRoutes = 0ULL;
+ static unsigned long long Ip6ReasmTimeout = 0ULL;
+ static unsigned long long Ip6ReasmReqds = 0ULL;
+ static unsigned long long Ip6ReasmOKs = 0ULL;
+ static unsigned long long Ip6ReasmFails = 0ULL;
+ static unsigned long long Ip6FragOKs = 0ULL;
+ static unsigned long long Ip6FragFails = 0ULL;
+ static unsigned long long Ip6FragCreates = 0ULL;
+ static unsigned long long Ip6InMcastPkts = 0ULL;
+ static unsigned long long Ip6OutMcastPkts = 0ULL;
+ static unsigned long long Ip6InOctets = 0ULL;
+ static unsigned long long Ip6OutOctets = 0ULL;
+ static unsigned long long Ip6InMcastOctets = 0ULL;
+ static unsigned long long Ip6OutMcastOctets = 0ULL;
+ static unsigned long long Ip6InBcastOctets = 0ULL;
+ static unsigned long long Ip6OutBcastOctets = 0ULL;
+ static unsigned long long Ip6InNoECTPkts = 0ULL;
+ static unsigned long long Ip6InECT1Pkts = 0ULL;
+ static unsigned long long Ip6InECT0Pkts = 0ULL;
+ static unsigned long long Ip6InCEPkts = 0ULL;
+ static unsigned long long Icmp6InMsgs = 0ULL;
+ static unsigned long long Icmp6InErrors = 0ULL;
+ static unsigned long long Icmp6OutMsgs = 0ULL;
+ static unsigned long long Icmp6OutErrors = 0ULL;
+ static unsigned long long Icmp6InCsumErrors = 0ULL;
+ static unsigned long long Icmp6InDestUnreachs = 0ULL;
+ static unsigned long long Icmp6InPktTooBigs = 0ULL;
+ static unsigned long long Icmp6InTimeExcds = 0ULL;
+ static unsigned long long Icmp6InParmProblems = 0ULL;
+ static unsigned long long Icmp6InEchos = 0ULL;
+ static unsigned long long Icmp6InEchoReplies = 0ULL;
+ static unsigned long long Icmp6InGroupMembQueries = 0ULL;
+ static unsigned long long Icmp6InGroupMembResponses = 0ULL;
+ static unsigned long long Icmp6InGroupMembReductions = 0ULL;
+ static unsigned long long Icmp6InRouterSolicits = 0ULL;
+ static unsigned long long Icmp6InRouterAdvertisements = 0ULL;
+ static unsigned long long Icmp6InNeighborSolicits = 0ULL;
+ static unsigned long long Icmp6InNeighborAdvertisements = 0ULL;
+ static unsigned long long Icmp6InRedirects = 0ULL;
+ static unsigned long long Icmp6InMLDv2Reports = 0ULL;
+ static unsigned long long Icmp6OutDestUnreachs = 0ULL;
+ static unsigned long long Icmp6OutPktTooBigs = 0ULL;
+ static unsigned long long Icmp6OutTimeExcds = 0ULL;
+ static unsigned long long Icmp6OutParmProblems = 0ULL;
+ static unsigned long long Icmp6OutEchos = 0ULL;
+ static unsigned long long Icmp6OutEchoReplies = 0ULL;
+ static unsigned long long Icmp6OutGroupMembQueries = 0ULL;
+ static unsigned long long Icmp6OutGroupMembResponses = 0ULL;
+ static unsigned long long Icmp6OutGroupMembReductions = 0ULL;
+ static unsigned long long Icmp6OutRouterSolicits = 0ULL;
+ static unsigned long long Icmp6OutRouterAdvertisements = 0ULL;
+ static unsigned long long Icmp6OutNeighborSolicits = 0ULL;
+ static unsigned long long Icmp6OutNeighborAdvertisements = 0ULL;
+ static unsigned long long Icmp6OutRedirects = 0ULL;
+ static unsigned long long Icmp6OutMLDv2Reports = 0ULL;
+ static unsigned long long Icmp6InType1 = 0ULL;
+ static unsigned long long Icmp6InType128 = 0ULL;
+ static unsigned long long Icmp6InType129 = 0ULL;
+ static unsigned long long Icmp6InType136 = 0ULL;
+ static unsigned long long Icmp6OutType1 = 0ULL;
+ static unsigned long long Icmp6OutType128 = 0ULL;
+ static unsigned long long Icmp6OutType129 = 0ULL;
+ static unsigned long long Icmp6OutType133 = 0ULL;
+ static unsigned long long Icmp6OutType135 = 0ULL;
+ static unsigned long long Icmp6OutType143 = 0ULL;
+ static unsigned long long Udp6InDatagrams = 0ULL;
+ static unsigned long long Udp6NoPorts = 0ULL;
+ static unsigned long long Udp6InErrors = 0ULL;
+ static unsigned long long Udp6OutDatagrams = 0ULL;
+ static unsigned long long Udp6RcvbufErrors = 0ULL;
+ static unsigned long long Udp6SndbufErrors = 0ULL;
+ static unsigned long long Udp6InCsumErrors = 0ULL;
+ static unsigned long long Udp6IgnoredMulti = 0ULL;
+ static unsigned long long UdpLite6InDatagrams = 0ULL;
+ static unsigned long long UdpLite6NoPorts = 0ULL;
+ static unsigned long long UdpLite6InErrors = 0ULL;
+ static unsigned long long UdpLite6OutDatagrams = 0ULL;
+ static unsigned long long UdpLite6RcvbufErrors = 0ULL;
+ static unsigned long long UdpLite6SndbufErrors = 0ULL;
+ static unsigned long long UdpLite6InCsumErrors = 0ULL;
+
+ // prepare for /proc/net/netstat parsing
+
+ if(unlikely(!arl_ipext)) {
+ hash_ipext = simple_hash("IpExt");
+ hash_tcpext = simple_hash("TcpExt");
+
+ do_bandwidth = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "bandwidth", CONFIG_BOOLEAN_AUTO);
+ do_inerrors = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "input errors", CONFIG_BOOLEAN_AUTO);
+ do_mcast = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "multicast bandwidth", CONFIG_BOOLEAN_AUTO);
+ do_bcast = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "broadcast bandwidth", CONFIG_BOOLEAN_AUTO);
+ do_mcast_p = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "multicast packets", CONFIG_BOOLEAN_AUTO);
+ do_bcast_p = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "broadcast packets", CONFIG_BOOLEAN_AUTO);
+ do_ecn = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "ECN packets", CONFIG_BOOLEAN_AUTO);
+
+ do_tcpext_reorder = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "TCP reorders", CONFIG_BOOLEAN_AUTO);
+ do_tcpext_syscookies = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "TCP SYN cookies", CONFIG_BOOLEAN_AUTO);
+ do_tcpext_ofo = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "TCP out-of-order queue", CONFIG_BOOLEAN_AUTO);
+ do_tcpext_connaborts = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "TCP connection aborts", CONFIG_BOOLEAN_AUTO);
+ do_tcpext_memory = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "TCP memory pressures", CONFIG_BOOLEAN_AUTO);
+
+ do_tcpext_syn_queue = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "TCP SYN queue", CONFIG_BOOLEAN_AUTO);
+ do_tcpext_accept_queue = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "TCP accept queue", CONFIG_BOOLEAN_AUTO);
+
+ arl_ipext = arl_create("netstat/ipext", NULL, 60);
+ arl_tcpext = arl_create("netstat/tcpext", NULL, 60);
+
+ // --------------------------------------------------------------------
+ // IP
+
+ if(do_bandwidth != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_ipext, "InOctets", &ipext_InOctets);
+ arl_expect(arl_ipext, "OutOctets", &ipext_OutOctets);
+ }
+
+ if(do_inerrors != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_ipext, "InNoRoutes", &ipext_InNoRoutes);
+ arl_expect(arl_ipext, "InTruncatedPkts", &ipext_InTruncatedPkts);
+ arl_expect(arl_ipext, "InCsumErrors", &ipext_InCsumErrors);
+ }
+
+ if(do_mcast != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_ipext, "InMcastOctets", &ipext_InMcastOctets);
+ arl_expect(arl_ipext, "OutMcastOctets", &ipext_OutMcastOctets);
+ }
+
+ if(do_mcast_p != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_ipext, "InMcastPkts", &ipext_InMcastPkts);
+ arl_expect(arl_ipext, "OutMcastPkts", &ipext_OutMcastPkts);
+ }
+
+ if(do_bcast != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_ipext, "InBcastPkts", &ipext_InBcastPkts);
+ arl_expect(arl_ipext, "OutBcastPkts", &ipext_OutBcastPkts);
+ }
+
+ if(do_bcast_p != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_ipext, "InBcastOctets", &ipext_InBcastOctets);
+ arl_expect(arl_ipext, "OutBcastOctets", &ipext_OutBcastOctets);
+ }
+
+ if(do_ecn != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_ipext, "InNoECTPkts", &ipext_InNoECTPkts);
+ arl_expect(arl_ipext, "InECT1Pkts", &ipext_InECT1Pkts);
+ arl_expect(arl_ipext, "InECT0Pkts", &ipext_InECT0Pkts);
+ arl_expect(arl_ipext, "InCEPkts", &ipext_InCEPkts);
+ }
+
+ // --------------------------------------------------------------------
+ // IP TCP
+
+ if(do_tcpext_reorder != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_tcpext, "TCPFACKReorder", &tcpext_TCPFACKReorder);
+ arl_expect(arl_tcpext, "TCPSACKReorder", &tcpext_TCPSACKReorder);
+ arl_expect(arl_tcpext, "TCPRenoReorder", &tcpext_TCPRenoReorder);
+ arl_expect(arl_tcpext, "TCPTSReorder", &tcpext_TCPTSReorder);
+ }
+
+ if(do_tcpext_syscookies != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_tcpext, "SyncookiesSent", &tcpext_SyncookiesSent);
+ arl_expect(arl_tcpext, "SyncookiesRecv", &tcpext_SyncookiesRecv);
+ arl_expect(arl_tcpext, "SyncookiesFailed", &tcpext_SyncookiesFailed);
+ }
+
+ if(do_tcpext_ofo != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_tcpext, "TCPOFOQueue", &tcpext_TCPOFOQueue);
+ arl_expect(arl_tcpext, "TCPOFODrop", &tcpext_TCPOFODrop);
+ arl_expect(arl_tcpext, "TCPOFOMerge", &tcpext_TCPOFOMerge);
+ arl_expect(arl_tcpext, "OfoPruned", &tcpext_OfoPruned);
+ }
+
+ if(do_tcpext_connaborts != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_tcpext, "TCPAbortOnData", &tcpext_TCPAbortOnData);
+ arl_expect(arl_tcpext, "TCPAbortOnClose", &tcpext_TCPAbortOnClose);
+ arl_expect(arl_tcpext, "TCPAbortOnMemory", &tcpext_TCPAbortOnMemory);
+ arl_expect(arl_tcpext, "TCPAbortOnTimeout", &tcpext_TCPAbortOnTimeout);
+ arl_expect(arl_tcpext, "TCPAbortOnLinger", &tcpext_TCPAbortOnLinger);
+ arl_expect(arl_tcpext, "TCPAbortFailed", &tcpext_TCPAbortFailed);
+ }
+
+ if(do_tcpext_memory != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_tcpext, "TCPMemoryPressures", &tcpext_TCPMemoryPressures);
+ }
+
+ if(do_tcpext_accept_queue != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_tcpext, "ListenOverflows", &tcpext_ListenOverflows);
+ arl_expect(arl_tcpext, "ListenDrops", &tcpext_ListenDrops);
+ }
+
+ if(do_tcpext_syn_queue != CONFIG_BOOLEAN_NO) {
+ arl_expect(arl_tcpext, "TCPReqQFullDrop", &tcpext_TCPReqQFullDrop);
+ arl_expect(arl_tcpext, "TCPReqQFullDoCookies", &tcpext_TCPReqQFullDoCookies);
+ }
+
+ arl_expect(arl_tcpext, "TCPSynRetrans", &tcpext_TCPSynRetrans);
+ }
+
+ // prepare for /proc/net/snmp parsing
+
+ if(unlikely(!arl_ip)) {
+ do_ip_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 packets", CONFIG_BOOLEAN_AUTO);
+ do_ip_fragsout = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 fragments sent", CONFIG_BOOLEAN_AUTO);
+ do_ip_fragsin = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 fragments assembly", CONFIG_BOOLEAN_AUTO);
+ do_ip_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 errors", CONFIG_BOOLEAN_AUTO);
+ do_tcp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 TCP connections", CONFIG_BOOLEAN_AUTO);
+ do_tcp_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 TCP packets", CONFIG_BOOLEAN_AUTO);
+ do_tcp_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 TCP errors", CONFIG_BOOLEAN_AUTO);
+ do_tcp_opens = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 TCP opens", CONFIG_BOOLEAN_AUTO);
+ do_tcp_handshake = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 TCP handshake issues", CONFIG_BOOLEAN_AUTO);
+ do_udp_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 UDP packets", CONFIG_BOOLEAN_AUTO);
+ do_udp_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 UDP errors", CONFIG_BOOLEAN_AUTO);
+ do_icmp_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 ICMP packets", CONFIG_BOOLEAN_AUTO);
+ do_icmpmsg = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 ICMP messages", CONFIG_BOOLEAN_AUTO);
+ do_udplite_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 UDPLite packets", CONFIG_BOOLEAN_AUTO);
+
+ hash_ip = simple_hash("Ip");
+ hash_tcp = simple_hash("Tcp");
+ hash_udp = simple_hash("Udp");
+ hash_icmp = simple_hash("Icmp");
+ hash_icmpmsg = simple_hash("IcmpMsg");
+ hash_udplite = simple_hash("UdpLite");
+
+ arl_ip = arl_create("snmp/Ip", arl_callback_str2kernel_uint_t, 60);
+ // arl_expect(arl_ip, "Forwarding", &snmp_root.ip_Forwarding);
+ arl_expect(arl_ip, "DefaultTTL", &snmp_root.ip_DefaultTTL);
+ arl_expect(arl_ip, "InReceives", &snmp_root.ip_InReceives);
+ arl_expect(arl_ip, "InHdrErrors", &snmp_root.ip_InHdrErrors);
+ arl_expect(arl_ip, "InAddrErrors", &snmp_root.ip_InAddrErrors);
+ arl_expect(arl_ip, "ForwDatagrams", &snmp_root.ip_ForwDatagrams);
+ arl_expect(arl_ip, "InUnknownProtos", &snmp_root.ip_InUnknownProtos);
+ arl_expect(arl_ip, "InDiscards", &snmp_root.ip_InDiscards);
+ arl_expect(arl_ip, "InDelivers", &snmp_root.ip_InDelivers);
+ arl_expect(arl_ip, "OutRequests", &snmp_root.ip_OutRequests);
+ arl_expect(arl_ip, "OutDiscards", &snmp_root.ip_OutDiscards);
+ arl_expect(arl_ip, "OutNoRoutes", &snmp_root.ip_OutNoRoutes);
+ arl_expect(arl_ip, "ReasmTimeout", &snmp_root.ip_ReasmTimeout);
+ arl_expect(arl_ip, "ReasmReqds", &snmp_root.ip_ReasmReqds);
+ arl_expect(arl_ip, "ReasmOKs", &snmp_root.ip_ReasmOKs);
+ arl_expect(arl_ip, "ReasmFails", &snmp_root.ip_ReasmFails);
+ arl_expect(arl_ip, "FragOKs", &snmp_root.ip_FragOKs);
+ arl_expect(arl_ip, "FragFails", &snmp_root.ip_FragFails);
+ arl_expect(arl_ip, "FragCreates", &snmp_root.ip_FragCreates);
+
+ arl_icmp = arl_create("snmp/Icmp", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_icmp, "InMsgs", &snmp_root.icmp_InMsgs);
+ arl_expect(arl_icmp, "OutMsgs", &snmp_root.icmp_OutMsgs);
+ arl_expect(arl_icmp, "InErrors", &snmp_root.icmp_InErrors);
+ arl_expect(arl_icmp, "OutErrors", &snmp_root.icmp_OutErrors);
+ arl_expect(arl_icmp, "InCsumErrors", &snmp_root.icmp_InCsumErrors);
+
+ arl_icmpmsg = arl_create("snmp/Icmpmsg", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_icmpmsg, "InType0", &snmp_root.icmpmsg_InEchoReps);
+ arl_expect(arl_icmpmsg, "OutType0", &snmp_root.icmpmsg_OutEchoReps);
+ arl_expect(arl_icmpmsg, "InType3", &snmp_root.icmpmsg_InDestUnreachs);
+ arl_expect(arl_icmpmsg, "OutType3", &snmp_root.icmpmsg_OutDestUnreachs);
+ arl_expect(arl_icmpmsg, "InType5", &snmp_root.icmpmsg_InRedirects);
+ arl_expect(arl_icmpmsg, "OutType5", &snmp_root.icmpmsg_OutRedirects);
+ arl_expect(arl_icmpmsg, "InType8", &snmp_root.icmpmsg_InEchos);
+ arl_expect(arl_icmpmsg, "OutType8", &snmp_root.icmpmsg_OutEchos);
+ arl_expect(arl_icmpmsg, "InType9", &snmp_root.icmpmsg_InRouterAdvert);
+ arl_expect(arl_icmpmsg, "OutType9", &snmp_root.icmpmsg_OutRouterAdvert);
+ arl_expect(arl_icmpmsg, "InType10", &snmp_root.icmpmsg_InRouterSelect);
+ arl_expect(arl_icmpmsg, "OutType10", &snmp_root.icmpmsg_OutRouterSelect);
+ arl_expect(arl_icmpmsg, "InType11", &snmp_root.icmpmsg_InTimeExcds);
+ arl_expect(arl_icmpmsg, "OutType11", &snmp_root.icmpmsg_OutTimeExcds);
+ arl_expect(arl_icmpmsg, "InType12", &snmp_root.icmpmsg_InParmProbs);
+ arl_expect(arl_icmpmsg, "OutType12", &snmp_root.icmpmsg_OutParmProbs);
+ arl_expect(arl_icmpmsg, "InType13", &snmp_root.icmpmsg_InTimestamps);
+ arl_expect(arl_icmpmsg, "OutType13", &snmp_root.icmpmsg_OutTimestamps);
+ arl_expect(arl_icmpmsg, "InType14", &snmp_root.icmpmsg_InTimestampReps);
+ arl_expect(arl_icmpmsg, "OutType14", &snmp_root.icmpmsg_OutTimestampReps);
+
+ arl_tcp = arl_create("snmp/Tcp", arl_callback_str2kernel_uint_t, 60);
+ // arl_expect(arl_tcp, "RtoAlgorithm", &snmp_root.tcp_RtoAlgorithm);
+ // arl_expect(arl_tcp, "RtoMin", &snmp_root.tcp_RtoMin);
+ // arl_expect(arl_tcp, "RtoMax", &snmp_root.tcp_RtoMax);
+ arl_expect_custom(arl_tcp, "MaxConn", arl_callback_ssize_t, &snmp_root.tcp_MaxConn);
+ arl_expect(arl_tcp, "ActiveOpens", &snmp_root.tcp_ActiveOpens);
+ arl_expect(arl_tcp, "PassiveOpens", &snmp_root.tcp_PassiveOpens);
+ arl_expect(arl_tcp, "AttemptFails", &snmp_root.tcp_AttemptFails);
+ arl_expect(arl_tcp, "EstabResets", &snmp_root.tcp_EstabResets);
+ arl_expect(arl_tcp, "CurrEstab", &snmp_root.tcp_CurrEstab);
+ arl_expect(arl_tcp, "InSegs", &snmp_root.tcp_InSegs);
+ arl_expect(arl_tcp, "OutSegs", &snmp_root.tcp_OutSegs);
+ arl_expect(arl_tcp, "RetransSegs", &snmp_root.tcp_RetransSegs);
+ arl_expect(arl_tcp, "InErrs", &snmp_root.tcp_InErrs);
+ arl_expect(arl_tcp, "OutRsts", &snmp_root.tcp_OutRsts);
+ arl_expect(arl_tcp, "InCsumErrors", &snmp_root.tcp_InCsumErrors);
+
+ arl_udp = arl_create("snmp/Udp", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_udp, "InDatagrams", &snmp_root.udp_InDatagrams);
+ arl_expect(arl_udp, "NoPorts", &snmp_root.udp_NoPorts);
+ arl_expect(arl_udp, "InErrors", &snmp_root.udp_InErrors);
+ arl_expect(arl_udp, "OutDatagrams", &snmp_root.udp_OutDatagrams);
+ arl_expect(arl_udp, "RcvbufErrors", &snmp_root.udp_RcvbufErrors);
+ arl_expect(arl_udp, "SndbufErrors", &snmp_root.udp_SndbufErrors);
+ arl_expect(arl_udp, "InCsumErrors", &snmp_root.udp_InCsumErrors);
+ arl_expect(arl_udp, "IgnoredMulti", &snmp_root.udp_IgnoredMulti);
+
+ arl_udplite = arl_create("snmp/Udplite", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_udplite, "InDatagrams", &snmp_root.udplite_InDatagrams);
+ arl_expect(arl_udplite, "NoPorts", &snmp_root.udplite_NoPorts);
+ arl_expect(arl_udplite, "InErrors", &snmp_root.udplite_InErrors);
+ arl_expect(arl_udplite, "OutDatagrams", &snmp_root.udplite_OutDatagrams);
+ arl_expect(arl_udplite, "RcvbufErrors", &snmp_root.udplite_RcvbufErrors);
+ arl_expect(arl_udplite, "SndbufErrors", &snmp_root.udplite_SndbufErrors);
+ arl_expect(arl_udplite, "InCsumErrors", &snmp_root.udplite_InCsumErrors);
+ arl_expect(arl_udplite, "IgnoredMulti", &snmp_root.udplite_IgnoredMulti);
+
+ tcp_max_connections_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_connections");
+ }
+
+ // prepare for /proc/net/snmp6 parsing
+
+ if(unlikely(!arl_ipv6)) {
+ do_ip6_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 packets", CONFIG_BOOLEAN_AUTO);
+ do_ip6_fragsout = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 fragments sent", CONFIG_BOOLEAN_AUTO);
+ do_ip6_fragsin = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 fragments assembly", CONFIG_BOOLEAN_AUTO);
+ do_ip6_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 errors", CONFIG_BOOLEAN_AUTO);
+ do_ip6_udp_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDP packets", CONFIG_BOOLEAN_AUTO);
+ do_ip6_udp_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDP errors", CONFIG_BOOLEAN_AUTO);
+ do_ip6_udplite_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDPlite packets", CONFIG_BOOLEAN_AUTO);
+ do_ip6_udplite_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDPlite errors", CONFIG_BOOLEAN_AUTO);
+ do_ip6_bandwidth = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "bandwidth", CONFIG_BOOLEAN_AUTO);
+ do_ip6_mcast = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "multicast bandwidth", CONFIG_BOOLEAN_AUTO);
+ do_ip6_bcast = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "broadcast bandwidth", CONFIG_BOOLEAN_AUTO);
+ do_ip6_mcast_p = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "multicast packets", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp_redir = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp redirects", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp errors", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp_echos = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp echos", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp_groupmemb = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp group membership", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp_router = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp router", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp_neighbor = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp neighbor", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp_mldv2 = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp mldv2", CONFIG_BOOLEAN_AUTO);
+ do_ip6_icmp_types = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp types", CONFIG_BOOLEAN_AUTO);
+ do_ip6_ect = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ect", CONFIG_BOOLEAN_AUTO);
+
+ arl_ipv6 = arl_create("snmp6", NULL, 60);
+ arl_expect(arl_ipv6, "Ip6InReceives", &Ip6InReceives);
+ arl_expect(arl_ipv6, "Ip6InHdrErrors", &Ip6InHdrErrors);
+ arl_expect(arl_ipv6, "Ip6InTooBigErrors", &Ip6InTooBigErrors);
+ arl_expect(arl_ipv6, "Ip6InNoRoutes", &Ip6InNoRoutes);
+ arl_expect(arl_ipv6, "Ip6InAddrErrors", &Ip6InAddrErrors);
+ arl_expect(arl_ipv6, "Ip6InUnknownProtos", &Ip6InUnknownProtos);
+ arl_expect(arl_ipv6, "Ip6InTruncatedPkts", &Ip6InTruncatedPkts);
+ arl_expect(arl_ipv6, "Ip6InDiscards", &Ip6InDiscards);
+ arl_expect(arl_ipv6, "Ip6InDelivers", &Ip6InDelivers);
+ arl_expect(arl_ipv6, "Ip6OutForwDatagrams", &Ip6OutForwDatagrams);
+ arl_expect(arl_ipv6, "Ip6OutRequests", &Ip6OutRequests);
+ arl_expect(arl_ipv6, "Ip6OutDiscards", &Ip6OutDiscards);
+ arl_expect(arl_ipv6, "Ip6OutNoRoutes", &Ip6OutNoRoutes);
+ arl_expect(arl_ipv6, "Ip6ReasmTimeout", &Ip6ReasmTimeout);
+ arl_expect(arl_ipv6, "Ip6ReasmReqds", &Ip6ReasmReqds);
+ arl_expect(arl_ipv6, "Ip6ReasmOKs", &Ip6ReasmOKs);
+ arl_expect(arl_ipv6, "Ip6ReasmFails", &Ip6ReasmFails);
+ arl_expect(arl_ipv6, "Ip6FragOKs", &Ip6FragOKs);
+ arl_expect(arl_ipv6, "Ip6FragFails", &Ip6FragFails);
+ arl_expect(arl_ipv6, "Ip6FragCreates", &Ip6FragCreates);
+ arl_expect(arl_ipv6, "Ip6InMcastPkts", &Ip6InMcastPkts);
+ arl_expect(arl_ipv6, "Ip6OutMcastPkts", &Ip6OutMcastPkts);
+ arl_expect(arl_ipv6, "Ip6InOctets", &Ip6InOctets);
+ arl_expect(arl_ipv6, "Ip6OutOctets", &Ip6OutOctets);
+ arl_expect(arl_ipv6, "Ip6InMcastOctets", &Ip6InMcastOctets);
+ arl_expect(arl_ipv6, "Ip6OutMcastOctets", &Ip6OutMcastOctets);
+ arl_expect(arl_ipv6, "Ip6InBcastOctets", &Ip6InBcastOctets);
+ arl_expect(arl_ipv6, "Ip6OutBcastOctets", &Ip6OutBcastOctets);
+ arl_expect(arl_ipv6, "Ip6InNoECTPkts", &Ip6InNoECTPkts);
+ arl_expect(arl_ipv6, "Ip6InECT1Pkts", &Ip6InECT1Pkts);
+ arl_expect(arl_ipv6, "Ip6InECT0Pkts", &Ip6InECT0Pkts);
+ arl_expect(arl_ipv6, "Ip6InCEPkts", &Ip6InCEPkts);
+ arl_expect(arl_ipv6, "Icmp6InMsgs", &Icmp6InMsgs);
+ arl_expect(arl_ipv6, "Icmp6InErrors", &Icmp6InErrors);
+ arl_expect(arl_ipv6, "Icmp6OutMsgs", &Icmp6OutMsgs);
+ arl_expect(arl_ipv6, "Icmp6OutErrors", &Icmp6OutErrors);
+ arl_expect(arl_ipv6, "Icmp6InCsumErrors", &Icmp6InCsumErrors);
+ arl_expect(arl_ipv6, "Icmp6InDestUnreachs", &Icmp6InDestUnreachs);
+ arl_expect(arl_ipv6, "Icmp6InPktTooBigs", &Icmp6InPktTooBigs);
+ arl_expect(arl_ipv6, "Icmp6InTimeExcds", &Icmp6InTimeExcds);
+ arl_expect(arl_ipv6, "Icmp6InParmProblems", &Icmp6InParmProblems);
+ arl_expect(arl_ipv6, "Icmp6InEchos", &Icmp6InEchos);
+ arl_expect(arl_ipv6, "Icmp6InEchoReplies", &Icmp6InEchoReplies);
+ arl_expect(arl_ipv6, "Icmp6InGroupMembQueries", &Icmp6InGroupMembQueries);
+ arl_expect(arl_ipv6, "Icmp6InGroupMembResponses", &Icmp6InGroupMembResponses);
+ arl_expect(arl_ipv6, "Icmp6InGroupMembReductions", &Icmp6InGroupMembReductions);
+ arl_expect(arl_ipv6, "Icmp6InRouterSolicits", &Icmp6InRouterSolicits);
+ arl_expect(arl_ipv6, "Icmp6InRouterAdvertisements", &Icmp6InRouterAdvertisements);
+ arl_expect(arl_ipv6, "Icmp6InNeighborSolicits", &Icmp6InNeighborSolicits);
+ arl_expect(arl_ipv6, "Icmp6InNeighborAdvertisements", &Icmp6InNeighborAdvertisements);
+ arl_expect(arl_ipv6, "Icmp6InRedirects", &Icmp6InRedirects);
+ arl_expect(arl_ipv6, "Icmp6InMLDv2Reports", &Icmp6InMLDv2Reports);
+ arl_expect(arl_ipv6, "Icmp6OutDestUnreachs", &Icmp6OutDestUnreachs);
+ arl_expect(arl_ipv6, "Icmp6OutPktTooBigs", &Icmp6OutPktTooBigs);
+ arl_expect(arl_ipv6, "Icmp6OutTimeExcds", &Icmp6OutTimeExcds);
+ arl_expect(arl_ipv6, "Icmp6OutParmProblems", &Icmp6OutParmProblems);
+ arl_expect(arl_ipv6, "Icmp6OutEchos", &Icmp6OutEchos);
+ arl_expect(arl_ipv6, "Icmp6OutEchoReplies", &Icmp6OutEchoReplies);
+ arl_expect(arl_ipv6, "Icmp6OutGroupMembQueries", &Icmp6OutGroupMembQueries);
+ arl_expect(arl_ipv6, "Icmp6OutGroupMembResponses", &Icmp6OutGroupMembResponses);
+ arl_expect(arl_ipv6, "Icmp6OutGroupMembReductions", &Icmp6OutGroupMembReductions);
+ arl_expect(arl_ipv6, "Icmp6OutRouterSolicits", &Icmp6OutRouterSolicits);
+ arl_expect(arl_ipv6, "Icmp6OutRouterAdvertisements", &Icmp6OutRouterAdvertisements);
+ arl_expect(arl_ipv6, "Icmp6OutNeighborSolicits", &Icmp6OutNeighborSolicits);
+ arl_expect(arl_ipv6, "Icmp6OutNeighborAdvertisements", &Icmp6OutNeighborAdvertisements);
+ arl_expect(arl_ipv6, "Icmp6OutRedirects", &Icmp6OutRedirects);
+ arl_expect(arl_ipv6, "Icmp6OutMLDv2Reports", &Icmp6OutMLDv2Reports);
+ arl_expect(arl_ipv6, "Icmp6InType1", &Icmp6InType1);
+ arl_expect(arl_ipv6, "Icmp6InType128", &Icmp6InType128);
+ arl_expect(arl_ipv6, "Icmp6InType129", &Icmp6InType129);
+ arl_expect(arl_ipv6, "Icmp6InType136", &Icmp6InType136);
+ arl_expect(arl_ipv6, "Icmp6OutType1", &Icmp6OutType1);
+ arl_expect(arl_ipv6, "Icmp6OutType128", &Icmp6OutType128);
+ arl_expect(arl_ipv6, "Icmp6OutType129", &Icmp6OutType129);
+ arl_expect(arl_ipv6, "Icmp6OutType133", &Icmp6OutType133);
+ arl_expect(arl_ipv6, "Icmp6OutType135", &Icmp6OutType135);
+ arl_expect(arl_ipv6, "Icmp6OutType143", &Icmp6OutType143);
+ arl_expect(arl_ipv6, "Udp6InDatagrams", &Udp6InDatagrams);
+ arl_expect(arl_ipv6, "Udp6NoPorts", &Udp6NoPorts);
+ arl_expect(arl_ipv6, "Udp6InErrors", &Udp6InErrors);
+ arl_expect(arl_ipv6, "Udp6OutDatagrams", &Udp6OutDatagrams);
+ arl_expect(arl_ipv6, "Udp6RcvbufErrors", &Udp6RcvbufErrors);
+ arl_expect(arl_ipv6, "Udp6SndbufErrors", &Udp6SndbufErrors);
+ arl_expect(arl_ipv6, "Udp6InCsumErrors", &Udp6InCsumErrors);
+ arl_expect(arl_ipv6, "Udp6IgnoredMulti", &Udp6IgnoredMulti);
+ arl_expect(arl_ipv6, "UdpLite6InDatagrams", &UdpLite6InDatagrams);
+ arl_expect(arl_ipv6, "UdpLite6NoPorts", &UdpLite6NoPorts);
+ arl_expect(arl_ipv6, "UdpLite6InErrors", &UdpLite6InErrors);
+ arl_expect(arl_ipv6, "UdpLite6OutDatagrams", &UdpLite6OutDatagrams);
+ arl_expect(arl_ipv6, "UdpLite6RcvbufErrors", &UdpLite6RcvbufErrors);
+ arl_expect(arl_ipv6, "UdpLite6SndbufErrors", &UdpLite6SndbufErrors);
+ arl_expect(arl_ipv6, "UdpLite6InCsumErrors", &UdpLite6InCsumErrors);
+ }
+
+ size_t lines, l, words;
+
+ // parse /proc/net/netstat
+
+ if(unlikely(!ff_netstat)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/netstat");
+ ff_netstat = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_NETSTAT, "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff_netstat)) return 1;
+ }
+
+ ff_netstat = procfile_readall(ff_netstat);
+ if(unlikely(!ff_netstat)) return 0; // we return 0, so that we will retry to open it next time
+
+ lines = procfile_lines(ff_netstat);
+
+ arl_begin(arl_ipext);
+ arl_begin(arl_tcpext);
+
+ for(l = 0; l < lines ;l++) {
+ char *key = procfile_lineword(ff_netstat, l, 0);
+ uint32_t hash = simple_hash(key);
+
+ if(unlikely(hash == hash_ipext && strcmp(key, "IpExt") == 0)) {
+ size_t h = l++;
+
+ words = procfile_linewords(ff_netstat, l);
+ if(unlikely(words < 2)) {
+ error("Cannot read /proc/net/netstat IpExt line. Expected 2+ params, read %zu.", words);
+ continue;
+ }
+
+ parse_line_pair(ff_netstat, arl_ipext, h, l);
+
+ }
+ else if(unlikely(hash == hash_tcpext && strcmp(key, "TcpExt") == 0)) {
+ size_t h = l++;
+
+ words = procfile_linewords(ff_netstat, l);
+ if(unlikely(words < 2)) {
+ error("Cannot read /proc/net/netstat TcpExt line. Expected 2+ params, read %zu.", words);
+ continue;
+ }
+
+ parse_line_pair(ff_netstat, arl_tcpext, h, l);
+ }
+ }
+
+ // parse /proc/net/snmp
+
+ if(unlikely(!ff_snmp)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/snmp");
+ ff_snmp = procfile_open(config_get("plugin:proc:/proc/net/snmp", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff_snmp)) return 1;
+ }
+
+ ff_snmp = procfile_readall(ff_snmp);
+ if(unlikely(!ff_snmp)) return 0; // we return 0, so that we will retry to open it next time
+
+ lines = procfile_lines(ff_snmp);
+ size_t w;
+
+ for(l = 0; l < lines ;l++) {
+ char *key = procfile_lineword(ff_snmp, l, 0);
+ uint32_t hash = simple_hash(key);
+
+ if(unlikely(hash == hash_ip && strcmp(key, "Ip") == 0)) {
+ size_t h = l++;
+
+ if(strcmp(procfile_lineword(ff_snmp, l, 0), "Ip") != 0) {
+ error("Cannot read Ip line from /proc/net/snmp.");
+ break;
+ }
+
+ words = procfile_linewords(ff_snmp, l);
+ if(words < 3) {
+ error("Cannot read /proc/net/snmp Ip line. Expected 3+ params, read %zu.", words);
+ continue;
+ }
+
+ arl_begin(arl_ip);
+ for(w = 1; w < words ; w++) {
+ if (unlikely(arl_check(arl_ip, procfile_lineword(ff_snmp, h, w), procfile_lineword(ff_snmp, l, w)) != 0))
+ break;
+ }
+ }
+ else if(unlikely(hash == hash_icmp && strcmp(key, "Icmp") == 0)) {
+ size_t h = l++;
+
+ if(strcmp(procfile_lineword(ff_snmp, l, 0), "Icmp") != 0) {
+ error("Cannot read Icmp line from /proc/net/snmp.");
+ break;
+ }
+
+ words = procfile_linewords(ff_snmp, l);
+ if(words < 3) {
+ error("Cannot read /proc/net/snmp Icmp line. Expected 3+ params, read %zu.", words);
+ continue;
+ }
+
+ arl_begin(arl_icmp);
+ for(w = 1; w < words ; w++) {
+ if (unlikely(arl_check(arl_icmp, procfile_lineword(ff_snmp, h, w), procfile_lineword(ff_snmp, l, w)) != 0))
+ break;
+ }
+ }
+ else if(unlikely(hash == hash_icmpmsg && strcmp(key, "IcmpMsg") == 0)) {
+ size_t h = l++;
+
+ if(strcmp(procfile_lineword(ff_snmp, l, 0), "IcmpMsg") != 0) {
+ error("Cannot read IcmpMsg line from /proc/net/snmp.");
+ break;
+ }
+
+ words = procfile_linewords(ff_snmp, l);
+ if(words < 2) {
+ error("Cannot read /proc/net/snmp IcmpMsg line. Expected 2+ params, read %zu.", words);
+ continue;
+ }
+
+ arl_begin(arl_icmpmsg);
+ for(w = 1; w < words ; w++) {
+ if (unlikely(arl_check(arl_icmpmsg, procfile_lineword(ff_snmp, h, w), procfile_lineword(ff_snmp, l, w)) != 0))
+ break;
+ }
+ }
+ else if(unlikely(hash == hash_tcp && strcmp(key, "Tcp") == 0)) {
+ size_t h = l++;
+
+ if(strcmp(procfile_lineword(ff_snmp, l, 0), "Tcp") != 0) {
+ error("Cannot read Tcp line from /proc/net/snmp.");
+ break;
+ }
+
+ words = procfile_linewords(ff_snmp, l);
+ if(words < 3) {
+ error("Cannot read /proc/net/snmp Tcp line. Expected 3+ params, read %zu.", words);
+ continue;
+ }
+
+ arl_begin(arl_tcp);
+ for(w = 1; w < words ; w++) {
+ if (unlikely(arl_check(arl_tcp, procfile_lineword(ff_snmp, h, w), procfile_lineword(ff_snmp, l, w)) != 0))
+ break;
+ }
+ }
+ else if(unlikely(hash == hash_udp && strcmp(key, "Udp") == 0)) {
+ size_t h = l++;
+
+ if(strcmp(procfile_lineword(ff_snmp, l, 0), "Udp") != 0) {
+ error("Cannot read Udp line from /proc/net/snmp.");
+ break;
+ }
+
+ words = procfile_linewords(ff_snmp, l);
+ if(words < 3) {
+ error("Cannot read /proc/net/snmp Udp line. Expected 3+ params, read %zu.", words);
+ continue;
+ }
+
+ arl_begin(arl_udp);
+ for(w = 1; w < words ; w++) {
+ if (unlikely(arl_check(arl_udp, procfile_lineword(ff_snmp, h, w), procfile_lineword(ff_snmp, l, w)) != 0))
+ break;
+ }
+ }
+ else if(unlikely(hash == hash_udplite && strcmp(key, "UdpLite") == 0)) {
+ size_t h = l++;
+
+ if(strcmp(procfile_lineword(ff_snmp, l, 0), "UdpLite") != 0) {
+ error("Cannot read UdpLite line from /proc/net/snmp.");
+ break;
+ }
+
+ words = procfile_linewords(ff_snmp, l);
+ if(words < 3) {
+ error("Cannot read /proc/net/snmp UdpLite line. Expected 3+ params, read %zu.", words);
+ continue;
+ }
+
+ arl_begin(arl_udplite);
+ for(w = 1; w < words ; w++) {
+ if (unlikely(arl_check(arl_udplite, procfile_lineword(ff_snmp, h, w), procfile_lineword(ff_snmp, l, w)) != 0))
+ break;
+ }
+ }
+ }
+
+ // parse /proc/net/snmp
+
+ if(unlikely(!ff_snmp6)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/snmp6");
+ ff_snmp6 = procfile_open(config_get("plugin:proc:/proc/net/snmp6", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff_snmp6))
+ return 1;
+ }
+
+ ff_snmp6 = procfile_readall(ff_snmp6);
+ if(unlikely(!ff_snmp6))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ lines = procfile_lines(ff_snmp6);
+
+ arl_begin(arl_ipv6);
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff_snmp6, l);
+ if(unlikely(words < 2)) {
+ if(unlikely(words)) error("Cannot read /proc/net/snmp6 line %zu. Expected 2 params, read %zu.", l, words);
+ continue;
+ }
+
+ if(unlikely(arl_check(arl_ipv6,
+ procfile_lineword(ff_snmp6, l, 0),
+ procfile_lineword(ff_snmp6, l, 1)))) break;
+ }
+
+ // netstat IpExt charts
+
+ if(do_bandwidth == CONFIG_BOOLEAN_YES || (do_bandwidth == CONFIG_BOOLEAN_AUTO &&
+ (ipext_InOctets ||
+ ipext_OutOctets ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_bandwidth = CONFIG_BOOLEAN_YES;
+ static RRDSET *st_system_ip = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_system_ip)) {
+ st_system_ip = rrdset_create_localhost(
+ "system"
+ , RRD_TYPE_NET_NETSTAT
+ , NULL
+ , "network"
+ , NULL
+ , "IP Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_IP
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st_system_ip, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_system_ip, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_system_ip, rd_in, ipext_InOctets);
+ rrddim_set_by_pointer(st_system_ip, rd_out, ipext_OutOctets);
+ rrdset_done(st_system_ip);
+ }
+
+ if(do_inerrors == CONFIG_BOOLEAN_YES || (do_inerrors == CONFIG_BOOLEAN_AUTO &&
+ (ipext_InNoRoutes ||
+ ipext_InTruncatedPkts ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_inerrors = CONFIG_BOOLEAN_YES;
+ static RRDSET *st_ip_inerrors = NULL;
+ static RRDDIM *rd_noroutes = NULL, *rd_truncated = NULL, *rd_checksum = NULL;
+
+ if(unlikely(!st_ip_inerrors)) {
+ st_ip_inerrors = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "inerrors"
+ , NULL
+ , "errors"
+ , NULL
+ , "IP Input Errors"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_ERRORS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_ip_inerrors, RRDSET_FLAG_DETAIL);
+
+ rd_noroutes = rrddim_add(st_ip_inerrors, "InNoRoutes", "noroutes", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_truncated = rrddim_add(st_ip_inerrors, "InTruncatedPkts", "truncated", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_checksum = rrddim_add(st_ip_inerrors, "InCsumErrors", "checksum", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ip_inerrors, rd_noroutes, ipext_InNoRoutes);
+ rrddim_set_by_pointer(st_ip_inerrors, rd_truncated, ipext_InTruncatedPkts);
+ rrddim_set_by_pointer(st_ip_inerrors, rd_checksum, ipext_InCsumErrors);
+ rrdset_done(st_ip_inerrors);
+ }
+
+ if(do_mcast == CONFIG_BOOLEAN_YES || (do_mcast == CONFIG_BOOLEAN_AUTO &&
+ (ipext_InMcastOctets ||
+ ipext_OutMcastOctets ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_mcast = CONFIG_BOOLEAN_YES;
+ static RRDSET *st_ip_mcast = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_ip_mcast)) {
+ st_ip_mcast = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "mcast"
+ , NULL
+ , "multicast"
+ , NULL
+ , "IP Multicast Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_MCAST
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(st_ip_mcast, RRDSET_FLAG_DETAIL);
+
+ rd_in = rrddim_add(st_ip_mcast, "InMcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_ip_mcast, "OutMcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ip_mcast, rd_in, ipext_InMcastOctets);
+ rrddim_set_by_pointer(st_ip_mcast, rd_out, ipext_OutMcastOctets);
+
+ rrdset_done(st_ip_mcast);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_bcast == CONFIG_BOOLEAN_YES || (do_bcast == CONFIG_BOOLEAN_AUTO &&
+ (ipext_InBcastOctets ||
+ ipext_OutBcastOctets ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_bcast = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_ip_bcast = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_ip_bcast)) {
+ st_ip_bcast = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "bcast"
+ , NULL
+ , "broadcast"
+ , NULL
+ , "IP Broadcast Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_BCAST
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(st_ip_bcast, RRDSET_FLAG_DETAIL);
+
+ rd_in = rrddim_add(st_ip_bcast, "InBcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_ip_bcast, "OutBcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ip_bcast, rd_in, ipext_InBcastOctets);
+ rrddim_set_by_pointer(st_ip_bcast, rd_out, ipext_OutBcastOctets);
+
+ rrdset_done(st_ip_bcast);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_mcast_p == CONFIG_BOOLEAN_YES || (do_mcast_p == CONFIG_BOOLEAN_AUTO &&
+ (ipext_InMcastPkts ||
+ ipext_OutMcastPkts ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_mcast_p = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_ip_mcastpkts = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_ip_mcastpkts)) {
+ st_ip_mcastpkts = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "mcastpkts"
+ , NULL
+ , "multicast"
+ , NULL
+ , "IP Multicast Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_MCAST_PACKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_ip_mcastpkts, RRDSET_FLAG_DETAIL);
+
+ rd_in = rrddim_add(st_ip_mcastpkts, "InMcastPkts", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_ip_mcastpkts, "OutMcastPkts", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ip_mcastpkts, rd_in, ipext_InMcastPkts);
+ rrddim_set_by_pointer(st_ip_mcastpkts, rd_out, ipext_OutMcastPkts);
+ rrdset_done(st_ip_mcastpkts);
+ }
+
+ if(do_bcast_p == CONFIG_BOOLEAN_YES || (do_bcast_p == CONFIG_BOOLEAN_AUTO &&
+ (ipext_InBcastPkts ||
+ ipext_OutBcastPkts ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_bcast_p = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_ip_bcastpkts = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_ip_bcastpkts)) {
+ st_ip_bcastpkts = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "bcastpkts"
+ , NULL
+ , "broadcast"
+ , NULL
+ , "IP Broadcast Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_BCAST_PACKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_ip_bcastpkts, RRDSET_FLAG_DETAIL);
+
+ rd_in = rrddim_add(st_ip_bcastpkts, "InBcastPkts", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_ip_bcastpkts, "OutBcastPkts", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ip_bcastpkts, rd_in, ipext_InBcastPkts);
+ rrddim_set_by_pointer(st_ip_bcastpkts, rd_out, ipext_OutBcastPkts);
+ rrdset_done(st_ip_bcastpkts);
+ }
+
+ if(do_ecn == CONFIG_BOOLEAN_YES || (do_ecn == CONFIG_BOOLEAN_AUTO &&
+ (ipext_InCEPkts ||
+ ipext_InECT0Pkts ||
+ ipext_InECT1Pkts ||
+ ipext_InNoECTPkts ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ecn = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_ecnpkts = NULL;
+ static RRDDIM *rd_cep = NULL, *rd_noectp = NULL, *rd_ectp0 = NULL, *rd_ectp1 = NULL;
+
+ if(unlikely(!st_ecnpkts)) {
+ st_ecnpkts = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "ecnpkts"
+ , NULL
+ , "ecn"
+ , NULL
+ , "IP ECN Statistics"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_ECN
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_ecnpkts, RRDSET_FLAG_DETAIL);
+
+ rd_cep = rrddim_add(st_ecnpkts, "InCEPkts", "CEP", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_noectp = rrddim_add(st_ecnpkts, "InNoECTPkts", "NoECTP", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ectp0 = rrddim_add(st_ecnpkts, "InECT0Pkts", "ECTP0", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ectp1 = rrddim_add(st_ecnpkts, "InECT1Pkts", "ECTP1", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ecnpkts, rd_cep, ipext_InCEPkts);
+ rrddim_set_by_pointer(st_ecnpkts, rd_noectp, ipext_InNoECTPkts);
+ rrddim_set_by_pointer(st_ecnpkts, rd_ectp0, ipext_InECT0Pkts);
+ rrddim_set_by_pointer(st_ecnpkts, rd_ectp1, ipext_InECT1Pkts);
+ rrdset_done(st_ecnpkts);
+ }
+
+ // netstat TcpExt charts
+
+ if(do_tcpext_memory == CONFIG_BOOLEAN_YES || (do_tcpext_memory == CONFIG_BOOLEAN_AUTO &&
+ (tcpext_TCPMemoryPressures ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_memory = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_tcpmemorypressures = NULL;
+ static RRDDIM *rd_pressures = NULL;
+
+ if(unlikely(!st_tcpmemorypressures)) {
+ st_tcpmemorypressures = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "tcpmemorypressures"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP Memory Pressures"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_TCP_MEM
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_pressures = rrddim_add(st_tcpmemorypressures, "TCPMemoryPressures", "pressures", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_tcpmemorypressures, rd_pressures, tcpext_TCPMemoryPressures);
+ rrdset_done(st_tcpmemorypressures);
+ }
+
+ if(do_tcpext_connaborts == CONFIG_BOOLEAN_YES || (do_tcpext_connaborts == CONFIG_BOOLEAN_AUTO &&
+ (tcpext_TCPAbortOnData ||
+ tcpext_TCPAbortOnClose ||
+ tcpext_TCPAbortOnMemory ||
+ tcpext_TCPAbortOnTimeout ||
+ tcpext_TCPAbortOnLinger ||
+ tcpext_TCPAbortFailed ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_connaborts = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_tcpconnaborts = NULL;
+ static RRDDIM *rd_baddata = NULL, *rd_userclosed = NULL, *rd_nomemory = NULL, *rd_timeout = NULL, *rd_linger = NULL, *rd_failed = NULL;
+
+ if(unlikely(!st_tcpconnaborts)) {
+ st_tcpconnaborts = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "tcpconnaborts"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP Connection Aborts"
+ , "connections/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_TCP_CONNABORTS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_baddata = rrddim_add(st_tcpconnaborts, "TCPAbortOnData", "baddata", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_userclosed = rrddim_add(st_tcpconnaborts, "TCPAbortOnClose", "userclosed", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_nomemory = rrddim_add(st_tcpconnaborts, "TCPAbortOnMemory", "nomemory", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_timeout = rrddim_add(st_tcpconnaborts, "TCPAbortOnTimeout", "timeout", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_linger = rrddim_add(st_tcpconnaborts, "TCPAbortOnLinger", "linger", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st_tcpconnaborts, "TCPAbortFailed", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_tcpconnaborts, rd_baddata, tcpext_TCPAbortOnData);
+ rrddim_set_by_pointer(st_tcpconnaborts, rd_userclosed, tcpext_TCPAbortOnClose);
+ rrddim_set_by_pointer(st_tcpconnaborts, rd_nomemory, tcpext_TCPAbortOnMemory);
+ rrddim_set_by_pointer(st_tcpconnaborts, rd_timeout, tcpext_TCPAbortOnTimeout);
+ rrddim_set_by_pointer(st_tcpconnaborts, rd_linger, tcpext_TCPAbortOnLinger);
+ rrddim_set_by_pointer(st_tcpconnaborts, rd_failed, tcpext_TCPAbortFailed);
+ rrdset_done(st_tcpconnaborts);
+ }
+
+ if(do_tcpext_reorder == CONFIG_BOOLEAN_YES || (do_tcpext_reorder == CONFIG_BOOLEAN_AUTO &&
+ (tcpext_TCPRenoReorder ||
+ tcpext_TCPFACKReorder ||
+ tcpext_TCPSACKReorder ||
+ tcpext_TCPTSReorder ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_reorder = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_tcpreorders = NULL;
+ static RRDDIM *rd_timestamp = NULL, *rd_sack = NULL, *rd_fack = NULL, *rd_reno = NULL;
+
+ if(unlikely(!st_tcpreorders)) {
+ st_tcpreorders = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "tcpreorders"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP Reordered Packets by Detection Method"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_TCP_REORDERS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_timestamp = rrddim_add(st_tcpreorders, "TCPTSReorder", "timestamp", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sack = rrddim_add(st_tcpreorders, "TCPSACKReorder", "sack", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_fack = rrddim_add(st_tcpreorders, "TCPFACKReorder", "fack", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_reno = rrddim_add(st_tcpreorders, "TCPRenoReorder", "reno", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_tcpreorders, rd_timestamp, tcpext_TCPTSReorder);
+ rrddim_set_by_pointer(st_tcpreorders, rd_sack, tcpext_TCPSACKReorder);
+ rrddim_set_by_pointer(st_tcpreorders, rd_fack, tcpext_TCPFACKReorder);
+ rrddim_set_by_pointer(st_tcpreorders, rd_reno, tcpext_TCPRenoReorder);
+ rrdset_done(st_tcpreorders);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_tcpext_ofo == CONFIG_BOOLEAN_YES || (do_tcpext_ofo == CONFIG_BOOLEAN_AUTO &&
+ (tcpext_TCPOFOQueue ||
+ tcpext_TCPOFODrop ||
+ tcpext_TCPOFOMerge ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_ofo = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_ip_tcpofo = NULL;
+ static RRDDIM *rd_inqueue = NULL, *rd_dropped = NULL, *rd_merged = NULL, *rd_pruned = NULL;
+
+ if(unlikely(!st_ip_tcpofo)) {
+
+ st_ip_tcpofo = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "tcpofo"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP Out-Of-Order Queue"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_TCP_OFO
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inqueue = rrddim_add(st_ip_tcpofo, "TCPOFOQueue", "inqueue", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_dropped = rrddim_add(st_ip_tcpofo, "TCPOFODrop", "dropped", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_merged = rrddim_add(st_ip_tcpofo, "TCPOFOMerge", "merged", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_pruned = rrddim_add(st_ip_tcpofo, "OfoPruned", "pruned", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ip_tcpofo, rd_inqueue, tcpext_TCPOFOQueue);
+ rrddim_set_by_pointer(st_ip_tcpofo, rd_dropped, tcpext_TCPOFODrop);
+ rrddim_set_by_pointer(st_ip_tcpofo, rd_merged, tcpext_TCPOFOMerge);
+ rrddim_set_by_pointer(st_ip_tcpofo, rd_pruned, tcpext_OfoPruned);
+ rrdset_done(st_ip_tcpofo);
+ }
+
+ if(do_tcpext_syscookies == CONFIG_BOOLEAN_YES || (do_tcpext_syscookies == CONFIG_BOOLEAN_AUTO &&
+ (tcpext_SyncookiesSent ||
+ tcpext_SyncookiesRecv ||
+ tcpext_SyncookiesFailed ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_syscookies = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_syncookies = NULL;
+ static RRDDIM *rd_received = NULL, *rd_sent = NULL, *rd_failed = NULL;
+
+ if(unlikely(!st_syncookies)) {
+
+ st_syncookies = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "tcpsyncookies"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP SYN Cookies"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_TCP_SYNCOOKIES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_received = rrddim_add(st_syncookies, "SyncookiesRecv", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st_syncookies, "SyncookiesSent", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st_syncookies, "SyncookiesFailed", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_syncookies, rd_received, tcpext_SyncookiesRecv);
+ rrddim_set_by_pointer(st_syncookies, rd_sent, tcpext_SyncookiesSent);
+ rrddim_set_by_pointer(st_syncookies, rd_failed, tcpext_SyncookiesFailed);
+ rrdset_done(st_syncookies);
+ }
+
+ if(do_tcpext_syn_queue == CONFIG_BOOLEAN_YES || (do_tcpext_syn_queue == CONFIG_BOOLEAN_AUTO &&
+ (tcpext_TCPReqQFullDrop ||
+ tcpext_TCPReqQFullDoCookies ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_syn_queue = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_syn_queue = NULL;
+ static RRDDIM
+ *rd_TCPReqQFullDrop = NULL,
+ *rd_TCPReqQFullDoCookies = NULL;
+
+ if(unlikely(!st_syn_queue)) {
+
+ st_syn_queue = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "tcp_syn_queue"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP SYN Queue Issues"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_TCP_SYN_QUEUE
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_TCPReqQFullDrop = rrddim_add(st_syn_queue, "TCPReqQFullDrop", "drops", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_TCPReqQFullDoCookies = rrddim_add(st_syn_queue, "TCPReqQFullDoCookies", "cookies", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_syn_queue, rd_TCPReqQFullDrop, tcpext_TCPReqQFullDrop);
+ rrddim_set_by_pointer(st_syn_queue, rd_TCPReqQFullDoCookies, tcpext_TCPReqQFullDoCookies);
+ rrdset_done(st_syn_queue);
+ }
+
+ if(do_tcpext_accept_queue == CONFIG_BOOLEAN_YES || (do_tcpext_accept_queue == CONFIG_BOOLEAN_AUTO &&
+ (tcpext_ListenOverflows ||
+ tcpext_ListenDrops ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcpext_accept_queue = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_accept_queue = NULL;
+ static RRDDIM *rd_overflows = NULL,
+ *rd_drops = NULL;
+
+ if(unlikely(!st_accept_queue)) {
+
+ st_accept_queue = rrdset_create_localhost(
+ RRD_TYPE_NET_NETSTAT
+ , "tcp_accept_queue"
+ , NULL
+ , "tcp"
+ , NULL
+ , "TCP Accept Queue Issues"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IP_TCP_ACCEPT_QUEUE
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_overflows = rrddim_add(st_accept_queue, "ListenOverflows", "overflows", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_drops = rrddim_add(st_accept_queue, "ListenDrops", "drops", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_accept_queue, rd_overflows, tcpext_ListenOverflows);
+ rrddim_set_by_pointer(st_accept_queue, rd_drops, tcpext_ListenDrops);
+ rrdset_done(st_accept_queue);
+ }
+
+ // snmp Ip charts
+
+ if(do_ip_packets == CONFIG_BOOLEAN_YES || (do_ip_packets == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.ip_OutRequests ||
+ snmp_root.ip_InReceives ||
+ snmp_root.ip_ForwDatagrams ||
+ snmp_root.ip_InDelivers ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip_packets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InReceives = NULL,
+ *rd_OutRequests = NULL,
+ *rd_ForwDatagrams = NULL,
+ *rd_InDelivers = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "packets"
+ , NULL
+ , "packets"
+ , NULL
+ , "IPv4 Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_PACKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InReceives = rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutRequests = rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ForwDatagrams = rrddim_add(st, "ForwDatagrams", "forwarded", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InDelivers = rrddim_add(st, "InDelivers", "delivered", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_OutRequests, (collected_number)snmp_root.ip_OutRequests);
+ rrddim_set_by_pointer(st, rd_InReceives, (collected_number)snmp_root.ip_InReceives);
+ rrddim_set_by_pointer(st, rd_ForwDatagrams, (collected_number)snmp_root.ip_ForwDatagrams);
+ rrddim_set_by_pointer(st, rd_InDelivers, (collected_number)snmp_root.ip_InDelivers);
+ rrdset_done(st);
+ }
+
+ if(do_ip_fragsout == CONFIG_BOOLEAN_YES || (do_ip_fragsout == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.ip_FragOKs ||
+ snmp_root.ip_FragFails ||
+ snmp_root.ip_FragCreates ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip_fragsout = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_FragOKs = NULL,
+ *rd_FragFails = NULL,
+ *rd_FragCreates = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "fragsout"
+ , NULL
+ , "fragments"
+ , NULL
+ , "IPv4 Fragments Sent"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_FRAGMENTS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_FragOKs = rrddim_add(st, "FragOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_FragFails = rrddim_add(st, "FragFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_FragCreates = rrddim_add(st, "FragCreates", "created", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_FragOKs, (collected_number)snmp_root.ip_FragOKs);
+ rrddim_set_by_pointer(st, rd_FragFails, (collected_number)snmp_root.ip_FragFails);
+ rrddim_set_by_pointer(st, rd_FragCreates, (collected_number)snmp_root.ip_FragCreates);
+ rrdset_done(st);
+ }
+
+ if(do_ip_fragsin == CONFIG_BOOLEAN_YES || (do_ip_fragsin == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.ip_ReasmOKs ||
+ snmp_root.ip_ReasmFails ||
+ snmp_root.ip_ReasmReqds ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip_fragsin = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ReasmOKs = NULL,
+ *rd_ReasmFails = NULL,
+ *rd_ReasmReqds = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "fragsin"
+ , NULL
+ , "fragments"
+ , NULL
+ , "IPv4 Fragments Reassembly"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_FRAGMENTS + 1
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ReasmOKs = rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ReasmFails = rrddim_add(st, "ReasmFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ReasmReqds = rrddim_add(st, "ReasmReqds", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ReasmOKs, (collected_number)snmp_root.ip_ReasmOKs);
+ rrddim_set_by_pointer(st, rd_ReasmFails, (collected_number)snmp_root.ip_ReasmFails);
+ rrddim_set_by_pointer(st, rd_ReasmReqds, (collected_number)snmp_root.ip_ReasmReqds);
+ rrdset_done(st);
+ }
+
+ if(do_ip_errors == CONFIG_BOOLEAN_YES || (do_ip_errors == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.ip_InDiscards ||
+ snmp_root.ip_OutDiscards ||
+ snmp_root.ip_InHdrErrors ||
+ snmp_root.ip_InAddrErrors ||
+ snmp_root.ip_InUnknownProtos ||
+ snmp_root.ip_OutNoRoutes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip_errors = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InDiscards = NULL,
+ *rd_OutDiscards = NULL,
+ *rd_InHdrErrors = NULL,
+ *rd_OutNoRoutes = NULL,
+ *rd_InAddrErrors = NULL,
+ *rd_InUnknownProtos = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "errors"
+ , NULL
+ , "errors"
+ , NULL
+ , "IPv4 Errors"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_ERRORS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_InDiscards = rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutDiscards = rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rd_InHdrErrors = rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutNoRoutes = rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rd_InAddrErrors = rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InUnknownProtos = rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InDiscards, (collected_number)snmp_root.ip_InDiscards);
+ rrddim_set_by_pointer(st, rd_OutDiscards, (collected_number)snmp_root.ip_OutDiscards);
+ rrddim_set_by_pointer(st, rd_InHdrErrors, (collected_number)snmp_root.ip_InHdrErrors);
+ rrddim_set_by_pointer(st, rd_InAddrErrors, (collected_number)snmp_root.ip_InAddrErrors);
+ rrddim_set_by_pointer(st, rd_InUnknownProtos, (collected_number)snmp_root.ip_InUnknownProtos);
+ rrddim_set_by_pointer(st, rd_OutNoRoutes, (collected_number)snmp_root.ip_OutNoRoutes);
+ rrdset_done(st);
+ }
+
+ // snmp Icmp charts
+
+ if(do_icmp_packets == CONFIG_BOOLEAN_YES || (do_icmp_packets == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.icmp_InMsgs ||
+ snmp_root.icmp_OutMsgs ||
+ snmp_root.icmp_InErrors ||
+ snmp_root.icmp_OutErrors ||
+ snmp_root.icmp_InCsumErrors ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmp_packets = CONFIG_BOOLEAN_YES;
+
+ {
+ static RRDSET *st_packets = NULL;
+ static RRDDIM *rd_InMsgs = NULL,
+ *rd_OutMsgs = NULL;
+
+ if(unlikely(!st_packets)) {
+ st_packets = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "icmp"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_ICMP
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InMsgs = rrddim_add(st_packets, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutMsgs = rrddim_add(st_packets, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_packets, rd_InMsgs, (collected_number)snmp_root.icmp_InMsgs);
+ rrddim_set_by_pointer(st_packets, rd_OutMsgs, (collected_number)snmp_root.icmp_OutMsgs);
+ rrdset_done(st_packets);
+ }
+
+ {
+ static RRDSET *st_errors = NULL;
+ static RRDDIM *rd_InErrors = NULL,
+ *rd_OutErrors = NULL,
+ *rd_InCsumErrors = NULL;
+
+ if(unlikely(!st_errors)) {
+ st_errors = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "icmp_errors"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Errors"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_ICMP + 1
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InErrors = rrddim_add(st_errors, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutErrors = rrddim_add(st_errors, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InCsumErrors = rrddim_add(st_errors, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_errors, rd_InErrors, (collected_number)snmp_root.icmp_InErrors);
+ rrddim_set_by_pointer(st_errors, rd_OutErrors, (collected_number)snmp_root.icmp_OutErrors);
+ rrddim_set_by_pointer(st_errors, rd_InCsumErrors, (collected_number)snmp_root.icmp_InCsumErrors);
+ rrdset_done(st_errors);
+ }
+ }
+
+ // snmp IcmpMsg charts
+
+ if(do_icmpmsg == CONFIG_BOOLEAN_YES || (do_icmpmsg == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.icmpmsg_InEchoReps ||
+ snmp_root.icmpmsg_OutEchoReps ||
+ snmp_root.icmpmsg_InDestUnreachs ||
+ snmp_root.icmpmsg_OutDestUnreachs ||
+ snmp_root.icmpmsg_InRedirects ||
+ snmp_root.icmpmsg_OutRedirects ||
+ snmp_root.icmpmsg_InEchos ||
+ snmp_root.icmpmsg_OutEchos ||
+ snmp_root.icmpmsg_InRouterAdvert ||
+ snmp_root.icmpmsg_OutRouterAdvert ||
+ snmp_root.icmpmsg_InRouterSelect ||
+ snmp_root.icmpmsg_OutRouterSelect ||
+ snmp_root.icmpmsg_InTimeExcds ||
+ snmp_root.icmpmsg_OutTimeExcds ||
+ snmp_root.icmpmsg_InParmProbs ||
+ snmp_root.icmpmsg_OutParmProbs ||
+ snmp_root.icmpmsg_InTimestamps ||
+ snmp_root.icmpmsg_OutTimestamps ||
+ snmp_root.icmpmsg_InTimestampReps ||
+ snmp_root.icmpmsg_OutTimestampReps ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_icmpmsg = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InEchoReps = NULL,
+ *rd_OutEchoReps = NULL,
+ *rd_InDestUnreachs = NULL,
+ *rd_OutDestUnreachs = NULL,
+ *rd_InRedirects = NULL,
+ *rd_OutRedirects = NULL,
+ *rd_InEchos = NULL,
+ *rd_OutEchos = NULL,
+ *rd_InRouterAdvert = NULL,
+ *rd_OutRouterAdvert = NULL,
+ *rd_InRouterSelect = NULL,
+ *rd_OutRouterSelect = NULL,
+ *rd_InTimeExcds = NULL,
+ *rd_OutTimeExcds = NULL,
+ *rd_InParmProbs = NULL,
+ *rd_OutParmProbs = NULL,
+ *rd_InTimestamps = NULL,
+ *rd_OutTimestamps = NULL,
+ *rd_InTimestampReps = NULL,
+ *rd_OutTimestampReps = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "icmpmsg"
+ , NULL
+ , "icmp"
+ , NULL
+ , "IPv4 ICMP Messages"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_ICMP + 2
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InEchoReps = rrddim_add(st, "InType0", "InEchoReps", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutEchoReps = rrddim_add(st, "OutType0", "OutEchoReps", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InDestUnreachs = rrddim_add(st, "InType3", "InDestUnreachs", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutDestUnreachs = rrddim_add(st, "OutType3", "OutDestUnreachs", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InRedirects = rrddim_add(st, "InType5", "InRedirects", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutRedirects = rrddim_add(st, "OutType5", "OutRedirects", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InEchos = rrddim_add(st, "InType8", "InEchos", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutEchos = rrddim_add(st, "OutType8", "OutEchos", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InRouterAdvert = rrddim_add(st, "InType9", "InRouterAdvert", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutRouterAdvert = rrddim_add(st, "OutType9", "OutRouterAdvert", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InRouterSelect = rrddim_add(st, "InType10", "InRouterSelect", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutRouterSelect = rrddim_add(st, "OutType10", "OutRouterSelect", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InTimeExcds = rrddim_add(st, "InType11", "InTimeExcds", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutTimeExcds = rrddim_add(st, "OutType11", "OutTimeExcds", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InParmProbs = rrddim_add(st, "InType12", "InParmProbs", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutParmProbs = rrddim_add(st, "OutType12", "OutParmProbs", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InTimestamps = rrddim_add(st, "InType13", "InTimestamps", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutTimestamps = rrddim_add(st, "OutType13", "OutTimestamps", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InTimestampReps = rrddim_add(st, "InType14", "InTimestampReps", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutTimestampReps = rrddim_add(st, "OutType14", "OutTimestampReps", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InEchoReps, (collected_number)snmp_root.icmpmsg_InEchoReps);
+ rrddim_set_by_pointer(st, rd_OutEchoReps, (collected_number)snmp_root.icmpmsg_OutEchoReps);
+ rrddim_set_by_pointer(st, rd_InDestUnreachs, (collected_number)snmp_root.icmpmsg_InDestUnreachs);
+ rrddim_set_by_pointer(st, rd_OutDestUnreachs, (collected_number)snmp_root.icmpmsg_OutDestUnreachs);
+ rrddim_set_by_pointer(st, rd_InRedirects, (collected_number)snmp_root.icmpmsg_InRedirects);
+ rrddim_set_by_pointer(st, rd_OutRedirects, (collected_number)snmp_root.icmpmsg_OutRedirects);
+ rrddim_set_by_pointer(st, rd_InEchos, (collected_number)snmp_root.icmpmsg_InEchos);
+ rrddim_set_by_pointer(st, rd_OutEchos, (collected_number)snmp_root.icmpmsg_OutEchos);
+ rrddim_set_by_pointer(st, rd_InRouterAdvert, (collected_number)snmp_root.icmpmsg_InRouterAdvert);
+ rrddim_set_by_pointer(st, rd_OutRouterAdvert, (collected_number)snmp_root.icmpmsg_OutRouterAdvert);
+ rrddim_set_by_pointer(st, rd_InRouterSelect, (collected_number)snmp_root.icmpmsg_InRouterSelect);
+ rrddim_set_by_pointer(st, rd_OutRouterSelect, (collected_number)snmp_root.icmpmsg_OutRouterSelect);
+ rrddim_set_by_pointer(st, rd_InTimeExcds, (collected_number)snmp_root.icmpmsg_InTimeExcds);
+ rrddim_set_by_pointer(st, rd_OutTimeExcds, (collected_number)snmp_root.icmpmsg_OutTimeExcds);
+ rrddim_set_by_pointer(st, rd_InParmProbs, (collected_number)snmp_root.icmpmsg_InParmProbs);
+ rrddim_set_by_pointer(st, rd_OutParmProbs, (collected_number)snmp_root.icmpmsg_OutParmProbs);
+ rrddim_set_by_pointer(st, rd_InTimestamps, (collected_number)snmp_root.icmpmsg_InTimestamps);
+ rrddim_set_by_pointer(st, rd_OutTimestamps, (collected_number)snmp_root.icmpmsg_OutTimestamps);
+ rrddim_set_by_pointer(st, rd_InTimestampReps, (collected_number)snmp_root.icmpmsg_InTimestampReps);
+ rrddim_set_by_pointer(st, rd_OutTimestampReps, (collected_number)snmp_root.icmpmsg_OutTimestampReps);
+
+ rrdset_done(st);
+ }
+
+ // snmp Tcp charts
+
+ // this is smart enough to update it, only when it is changed
+ rrdvar_custom_host_variable_set(localhost, tcp_max_connections_var, snmp_root.tcp_MaxConn);
+
+ // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html
+ if(do_tcp_sockets == CONFIG_BOOLEAN_YES || (do_tcp_sockets == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.tcp_CurrEstab ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcp_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_CurrEstab = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "tcpsock"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Connections"
+ , "active connections"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_TCP
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_CurrEstab = rrddim_add(st, "CurrEstab", "connections", 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_CurrEstab, (collected_number)snmp_root.tcp_CurrEstab);
+ rrdset_done(st);
+ }
+
+ if(do_tcp_packets == CONFIG_BOOLEAN_YES || (do_tcp_packets == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.tcp_InSegs ||
+ snmp_root.tcp_OutSegs ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcp_packets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InSegs = NULL,
+ *rd_OutSegs = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "tcppackets"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_TCP + 4
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InSegs = rrddim_add(st, "InSegs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutSegs = rrddim_add(st, "OutSegs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InSegs, (collected_number)snmp_root.tcp_InSegs);
+ rrddim_set_by_pointer(st, rd_OutSegs, (collected_number)snmp_root.tcp_OutSegs);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_tcp_errors == CONFIG_BOOLEAN_YES || (do_tcp_errors == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.tcp_InErrs ||
+ snmp_root.tcp_InCsumErrors ||
+ snmp_root.tcp_RetransSegs ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcp_errors = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InErrs = NULL,
+ *rd_InCsumErrors = NULL,
+ *rd_RetransSegs = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "tcperrors"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Errors"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_TCP + 20
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_InErrs = rrddim_add(st, "InErrs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_RetransSegs = rrddim_add(st, "RetransSegs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InErrs, (collected_number)snmp_root.tcp_InErrs);
+ rrddim_set_by_pointer(st, rd_InCsumErrors, (collected_number)snmp_root.tcp_InCsumErrors);
+ rrddim_set_by_pointer(st, rd_RetransSegs, (collected_number)snmp_root.tcp_RetransSegs);
+ rrdset_done(st);
+ }
+
+ if(do_tcp_opens == CONFIG_BOOLEAN_YES || (do_tcp_opens == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.tcp_ActiveOpens ||
+ snmp_root.tcp_PassiveOpens ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcp_opens = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ActiveOpens = NULL,
+ *rd_PassiveOpens = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "tcpopens"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Opens"
+ , "connections/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_TCP + 5
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ActiveOpens = rrddim_add(st, "ActiveOpens", "active", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_PassiveOpens = rrddim_add(st, "PassiveOpens", "passive", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ActiveOpens, (collected_number)snmp_root.tcp_ActiveOpens);
+ rrddim_set_by_pointer(st, rd_PassiveOpens, (collected_number)snmp_root.tcp_PassiveOpens);
+ rrdset_done(st);
+ }
+
+ if(do_tcp_handshake == CONFIG_BOOLEAN_YES || (do_tcp_handshake == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.tcp_EstabResets ||
+ snmp_root.tcp_OutRsts ||
+ snmp_root.tcp_AttemptFails ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcp_handshake = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_EstabResets = NULL,
+ *rd_OutRsts = NULL,
+ *rd_AttemptFails = NULL,
+ *rd_TCPSynRetrans = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "tcphandshake"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Handshake Issues"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_TCP + 30
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_EstabResets = rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutRsts = rrddim_add(st, "OutRsts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_AttemptFails = rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_TCPSynRetrans = rrddim_add(st, "TCPSynRetrans", "SynRetrans", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_EstabResets, (collected_number)snmp_root.tcp_EstabResets);
+ rrddim_set_by_pointer(st, rd_OutRsts, (collected_number)snmp_root.tcp_OutRsts);
+ rrddim_set_by_pointer(st, rd_AttemptFails, (collected_number)snmp_root.tcp_AttemptFails);
+ rrddim_set_by_pointer(st, rd_TCPSynRetrans, tcpext_TCPSynRetrans);
+ rrdset_done(st);
+ }
+
+ // snmp Udp charts
+
+ // see http://net-snmp.sourceforge.net/docs/mibs/udp.html
+ if(do_udp_packets == CONFIG_BOOLEAN_YES || (do_udp_packets == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.udp_InDatagrams ||
+ snmp_root.udp_OutDatagrams ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udp_packets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InDatagrams = NULL,
+ *rd_OutDatagrams = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "udppackets"
+ , NULL
+ , "udp"
+ , NULL
+ , "IPv4 UDP Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_UDP
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InDatagrams = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutDatagrams = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InDatagrams, (collected_number)snmp_root.udp_InDatagrams);
+ rrddim_set_by_pointer(st, rd_OutDatagrams, (collected_number)snmp_root.udp_OutDatagrams);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_udp_errors == CONFIG_BOOLEAN_YES || (do_udp_errors == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.udp_InErrors ||
+ snmp_root.udp_NoPorts ||
+ snmp_root.udp_RcvbufErrors ||
+ snmp_root.udp_SndbufErrors ||
+ snmp_root.udp_InCsumErrors ||
+ snmp_root.udp_IgnoredMulti ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udp_errors = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_RcvbufErrors = NULL,
+ *rd_SndbufErrors = NULL,
+ *rd_InErrors = NULL,
+ *rd_NoPorts = NULL,
+ *rd_InCsumErrors = NULL,
+ *rd_IgnoredMulti = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "udperrors"
+ , NULL
+ , "udp"
+ , NULL
+ , "IPv4 UDP Errors"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_UDP + 10
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_RcvbufErrors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_SndbufErrors = rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_NoPorts = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_IgnoredMulti = rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InErrors, (collected_number)snmp_root.udp_InErrors);
+ rrddim_set_by_pointer(st, rd_NoPorts, (collected_number)snmp_root.udp_NoPorts);
+ rrddim_set_by_pointer(st, rd_RcvbufErrors, (collected_number)snmp_root.udp_RcvbufErrors);
+ rrddim_set_by_pointer(st, rd_SndbufErrors, (collected_number)snmp_root.udp_SndbufErrors);
+ rrddim_set_by_pointer(st, rd_InCsumErrors, (collected_number)snmp_root.udp_InCsumErrors);
+ rrddim_set_by_pointer(st, rd_IgnoredMulti, (collected_number)snmp_root.udp_IgnoredMulti);
+ rrdset_done(st);
+ }
+
+ // snmp UdpLite charts
+
+ if(do_udplite_packets == CONFIG_BOOLEAN_YES || (do_udplite_packets == CONFIG_BOOLEAN_AUTO &&
+ (snmp_root.udplite_InDatagrams ||
+ snmp_root.udplite_OutDatagrams ||
+ snmp_root.udplite_NoPorts ||
+ snmp_root.udplite_InErrors ||
+ snmp_root.udplite_InCsumErrors ||
+ snmp_root.udplite_RcvbufErrors ||
+ snmp_root.udplite_SndbufErrors ||
+ snmp_root.udplite_IgnoredMulti ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udplite_packets = CONFIG_BOOLEAN_YES;
+
+ {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InDatagrams = NULL,
+ *rd_OutDatagrams = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "udplite"
+ , NULL
+ , "udplite"
+ , NULL
+ , "IPv4 UDPLite Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_UDPLITE
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InDatagrams = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutDatagrams = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InDatagrams, (collected_number)snmp_root.udplite_InDatagrams);
+ rrddim_set_by_pointer(st, rd_OutDatagrams, (collected_number)snmp_root.udplite_OutDatagrams);
+ rrdset_done(st);
+ }
+
+ {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_RcvbufErrors = NULL,
+ *rd_SndbufErrors = NULL,
+ *rd_InErrors = NULL,
+ *rd_NoPorts = NULL,
+ *rd_InCsumErrors = NULL,
+ *rd_IgnoredMulti = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "udplite_errors"
+ , NULL
+ , "udplite"
+ , NULL
+ , "IPv4 UDPLite Errors"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_UDPLITE + 10
+ , update_every
+ , RRDSET_TYPE_LINE);
+
+ rd_RcvbufErrors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_SndbufErrors = rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_NoPorts = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_IgnoredMulti = rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_NoPorts, (collected_number)snmp_root.udplite_NoPorts);
+ rrddim_set_by_pointer(st, rd_InErrors, (collected_number)snmp_root.udplite_InErrors);
+ rrddim_set_by_pointer(st, rd_InCsumErrors, (collected_number)snmp_root.udplite_InCsumErrors);
+ rrddim_set_by_pointer(st, rd_RcvbufErrors, (collected_number)snmp_root.udplite_RcvbufErrors);
+ rrddim_set_by_pointer(st, rd_SndbufErrors, (collected_number)snmp_root.udplite_SndbufErrors);
+ rrddim_set_by_pointer(st, rd_IgnoredMulti, (collected_number)snmp_root.udplite_IgnoredMulti);
+ rrdset_done(st);
+ }
+ }
+
+ // snmp6 charts
+
+ if(do_ip6_bandwidth == CONFIG_BOOLEAN_YES || (do_ip6_bandwidth == CONFIG_BOOLEAN_AUTO &&
+ (Ip6InOctets ||
+ Ip6OutOctets ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_bandwidth = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_received = NULL,
+ *rd_sent = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "ipv6"
+ , NULL
+ , "network"
+ , NULL
+ , "IPv6 Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_IPV6
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_received = rrddim_add(st, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_received, Ip6InOctets);
+ rrddim_set_by_pointer(st, rd_sent, Ip6OutOctets);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_packets == CONFIG_BOOLEAN_YES || (do_ip6_packets == CONFIG_BOOLEAN_AUTO &&
+ (Ip6InReceives ||
+ Ip6OutRequests ||
+ Ip6InDelivers ||
+ Ip6OutForwDatagrams ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_packets = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_received = NULL,
+ *rd_sent = NULL,
+ *rd_forwarded = NULL,
+ *rd_delivers = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "packets"
+ , NULL
+ , "packets"
+ , NULL
+ , "IPv6 Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_PACKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_received = rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_forwarded = rrddim_add(st, "OutForwDatagrams", "forwarded", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_delivers = rrddim_add(st, "InDelivers", "delivers", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_received, Ip6InReceives);
+ rrddim_set_by_pointer(st, rd_sent, Ip6OutRequests);
+ rrddim_set_by_pointer(st, rd_forwarded, Ip6OutForwDatagrams);
+ rrddim_set_by_pointer(st, rd_delivers, Ip6InDelivers);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_fragsout == CONFIG_BOOLEAN_YES || (do_ip6_fragsout == CONFIG_BOOLEAN_AUTO &&
+ (Ip6FragOKs ||
+ Ip6FragFails ||
+ Ip6FragCreates ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_fragsout = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ok = NULL,
+ *rd_failed = NULL,
+ *rd_all = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "fragsout"
+ , NULL
+ , "fragments6"
+ , NULL
+ , "IPv6 Fragments Sent"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_FRAGSOUT
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ok = rrddim_add(st, "FragOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st, "FragFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_all = rrddim_add(st, "FragCreates", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ok, Ip6FragOKs);
+ rrddim_set_by_pointer(st, rd_failed, Ip6FragFails);
+ rrddim_set_by_pointer(st, rd_all, Ip6FragCreates);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_fragsin == CONFIG_BOOLEAN_YES || (do_ip6_fragsin == CONFIG_BOOLEAN_AUTO &&
+ (Ip6ReasmOKs ||
+ Ip6ReasmFails ||
+ Ip6ReasmTimeout ||
+ Ip6ReasmReqds ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_fragsin = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_ok = NULL,
+ *rd_failed = NULL,
+ *rd_timeout = NULL,
+ *rd_all = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "fragsin"
+ , NULL
+ , "fragments6"
+ , NULL
+ , "IPv6 Fragments Reassembly"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_FRAGSIN
+ , update_every
+ , RRDSET_TYPE_LINE);
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_ok = rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_failed = rrddim_add(st, "ReasmFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_timeout = rrddim_add(st, "ReasmTimeout", "timeout", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_all = rrddim_add(st, "ReasmReqds", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_ok, Ip6ReasmOKs);
+ rrddim_set_by_pointer(st, rd_failed, Ip6ReasmFails);
+ rrddim_set_by_pointer(st, rd_timeout, Ip6ReasmTimeout);
+ rrddim_set_by_pointer(st, rd_all, Ip6ReasmReqds);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_errors == CONFIG_BOOLEAN_YES || (do_ip6_errors == CONFIG_BOOLEAN_AUTO &&
+ (Ip6InDiscards ||
+ Ip6OutDiscards ||
+ Ip6InHdrErrors ||
+ Ip6InAddrErrors ||
+ Ip6InUnknownProtos ||
+ Ip6InTooBigErrors ||
+ Ip6InTruncatedPkts ||
+ Ip6InNoRoutes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_errors = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InDiscards = NULL,
+ *rd_OutDiscards = NULL,
+ *rd_InHdrErrors = NULL,
+ *rd_InAddrErrors = NULL,
+ *rd_InUnknownProtos = NULL,
+ *rd_InTooBigErrors = NULL,
+ *rd_InTruncatedPkts = NULL,
+ *rd_InNoRoutes = NULL,
+ *rd_OutNoRoutes = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "errors"
+ , NULL
+ , "errors"
+ , NULL
+ , "IPv6 Errors"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ERRORS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_InDiscards = rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutDiscards = rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InHdrErrors = rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InAddrErrors = rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InUnknownProtos = rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InTooBigErrors = rrddim_add(st, "InTooBigErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InTruncatedPkts = rrddim_add(st, "InTruncatedPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InNoRoutes = rrddim_add(st, "InNoRoutes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutNoRoutes = rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InDiscards, Ip6InDiscards);
+ rrddim_set_by_pointer(st, rd_OutDiscards, Ip6OutDiscards);
+ rrddim_set_by_pointer(st, rd_InHdrErrors, Ip6InHdrErrors);
+ rrddim_set_by_pointer(st, rd_InAddrErrors, Ip6InAddrErrors);
+ rrddim_set_by_pointer(st, rd_InUnknownProtos, Ip6InUnknownProtos);
+ rrddim_set_by_pointer(st, rd_InTooBigErrors, Ip6InTooBigErrors);
+ rrddim_set_by_pointer(st, rd_InTruncatedPkts, Ip6InTruncatedPkts);
+ rrddim_set_by_pointer(st, rd_InNoRoutes, Ip6InNoRoutes);
+ rrddim_set_by_pointer(st, rd_OutNoRoutes, Ip6OutNoRoutes);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_udp_packets == CONFIG_BOOLEAN_YES || (do_ip6_udp_packets == CONFIG_BOOLEAN_AUTO &&
+ (Udp6InDatagrams ||
+ Udp6OutDatagrams ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udp_packets = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_received = NULL,
+ *rd_sent = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "udppackets"
+ , NULL
+ , "udp6"
+ , NULL
+ , "IPv6 UDP Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_UDP_PACKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_received = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_received, Udp6InDatagrams);
+ rrddim_set_by_pointer(st, rd_sent, Udp6OutDatagrams);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_udp_errors == CONFIG_BOOLEAN_YES || (do_ip6_udp_errors == CONFIG_BOOLEAN_AUTO &&
+ (Udp6InErrors ||
+ Udp6NoPorts ||
+ Udp6RcvbufErrors ||
+ Udp6SndbufErrors ||
+ Udp6InCsumErrors ||
+ Udp6IgnoredMulti ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_udp_errors = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_RcvbufErrors = NULL,
+ *rd_SndbufErrors = NULL,
+ *rd_InErrors = NULL,
+ *rd_NoPorts = NULL,
+ *rd_InCsumErrors = NULL,
+ *rd_IgnoredMulti = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "udperrors"
+ , NULL
+ , "udp6"
+ , NULL
+ , "IPv6 UDP Errors"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_UDP_ERRORS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_RcvbufErrors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_SndbufErrors = rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_NoPorts = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_IgnoredMulti = rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_RcvbufErrors, Udp6RcvbufErrors);
+ rrddim_set_by_pointer(st, rd_SndbufErrors, Udp6SndbufErrors);
+ rrddim_set_by_pointer(st, rd_InErrors, Udp6InErrors);
+ rrddim_set_by_pointer(st, rd_NoPorts, Udp6NoPorts);
+ rrddim_set_by_pointer(st, rd_InCsumErrors, Udp6InCsumErrors);
+ rrddim_set_by_pointer(st, rd_IgnoredMulti, Udp6IgnoredMulti);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_udplite_packets == CONFIG_BOOLEAN_YES || (do_ip6_udplite_packets == CONFIG_BOOLEAN_AUTO &&
+ (UdpLite6InDatagrams ||
+ UdpLite6OutDatagrams ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udplite_packets = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_received = NULL,
+ *rd_sent = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "udplitepackets"
+ , NULL
+ , "udplite6"
+ , NULL
+ , "IPv6 UDPlite Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_UDPLITE_PACKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_received = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_received, UdpLite6InDatagrams);
+ rrddim_set_by_pointer(st, rd_sent, UdpLite6OutDatagrams);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_udplite_errors == CONFIG_BOOLEAN_YES || (do_ip6_udplite_errors == CONFIG_BOOLEAN_AUTO &&
+ (UdpLite6InErrors ||
+ UdpLite6NoPorts ||
+ UdpLite6RcvbufErrors ||
+ UdpLite6SndbufErrors ||
+ Udp6InCsumErrors ||
+ UdpLite6InCsumErrors ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_udplite_errors = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_RcvbufErrors = NULL,
+ *rd_SndbufErrors = NULL,
+ *rd_InErrors = NULL,
+ *rd_NoPorts = NULL,
+ *rd_InCsumErrors = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "udpliteerrors"
+ , NULL
+ , "udplite6"
+ , NULL
+ , "IPv6 UDP Lite Errors"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_UDPLITE_ERRORS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_RcvbufErrors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_SndbufErrors = rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_NoPorts = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InErrors, UdpLite6InErrors);
+ rrddim_set_by_pointer(st, rd_NoPorts, UdpLite6NoPorts);
+ rrddim_set_by_pointer(st, rd_RcvbufErrors, UdpLite6RcvbufErrors);
+ rrddim_set_by_pointer(st, rd_SndbufErrors, UdpLite6SndbufErrors);
+ rrddim_set_by_pointer(st, rd_InCsumErrors, UdpLite6InCsumErrors);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_mcast == CONFIG_BOOLEAN_YES || (do_ip6_mcast == CONFIG_BOOLEAN_AUTO &&
+ (Ip6OutMcastOctets ||
+ Ip6InMcastOctets ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_mcast = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_Ip6InMcastOctets = NULL,
+ *rd_Ip6OutMcastOctets = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "mcast"
+ , NULL
+ , "multicast6"
+ , NULL
+ , "IPv6 Multicast Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_MCAST
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_Ip6InMcastOctets = rrddim_add(st, "InMcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_Ip6OutMcastOctets = rrddim_add(st, "OutMcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_Ip6InMcastOctets, Ip6InMcastOctets);
+ rrddim_set_by_pointer(st, rd_Ip6OutMcastOctets, Ip6OutMcastOctets);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_bcast == CONFIG_BOOLEAN_YES || (do_ip6_bcast == CONFIG_BOOLEAN_AUTO &&
+ (Ip6OutBcastOctets ||
+ Ip6InBcastOctets ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_bcast = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_Ip6InBcastOctets = NULL,
+ *rd_Ip6OutBcastOctets = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "bcast"
+ , NULL
+ , "broadcast6"
+ , NULL
+ , "IPv6 Broadcast Bandwidth"
+ , "kilobits/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_BCAST
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_Ip6InBcastOctets = rrddim_add(st, "InBcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_Ip6OutBcastOctets = rrddim_add(st, "OutBcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_Ip6InBcastOctets, Ip6InBcastOctets);
+ rrddim_set_by_pointer(st, rd_Ip6OutBcastOctets, Ip6OutBcastOctets);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_mcast_p == CONFIG_BOOLEAN_YES || (do_ip6_mcast_p == CONFIG_BOOLEAN_AUTO &&
+ (Ip6OutMcastPkts ||
+ Ip6InMcastPkts ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_mcast_p = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_Ip6InMcastPkts = NULL,
+ *rd_Ip6OutMcastPkts = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "mcastpkts"
+ , NULL
+ , "multicast6"
+ , NULL
+ , "IPv6 Multicast Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_MCAST_PACKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_Ip6InMcastPkts = rrddim_add(st, "InMcastPkts", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_Ip6OutMcastPkts = rrddim_add(st, "OutMcastPkts", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_Ip6InMcastPkts, Ip6InMcastPkts);
+ rrddim_set_by_pointer(st, rd_Ip6OutMcastPkts, Ip6OutMcastPkts);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp == CONFIG_BOOLEAN_YES || (do_ip6_icmp == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InMsgs ||
+ Icmp6OutMsgs ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_Icmp6InMsgs = NULL,
+ *rd_Icmp6OutMsgs = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "icmp"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 ICMP Messages"
+ , "messages/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_Icmp6InMsgs = rrddim_add(st, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_Icmp6OutMsgs = rrddim_add(st, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_Icmp6InMsgs, Icmp6InMsgs);
+ rrddim_set_by_pointer(st, rd_Icmp6OutMsgs, Icmp6OutMsgs);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp_redir == CONFIG_BOOLEAN_YES || (do_ip6_icmp_redir == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InRedirects ||
+ Icmp6OutRedirects ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp_redir = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_Icmp6InRedirects = NULL,
+ *rd_Icmp6OutRedirects = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "icmpredir"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 ICMP Redirects"
+ , "redirects/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP_REDIR
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_Icmp6InRedirects = rrddim_add(st, "InRedirects", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_Icmp6OutRedirects = rrddim_add(st, "OutRedirects", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_Icmp6InRedirects, Icmp6InRedirects);
+ rrddim_set_by_pointer(st, rd_Icmp6OutRedirects, Icmp6OutRedirects);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp_errors == CONFIG_BOOLEAN_YES || (do_ip6_icmp_errors == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InErrors ||
+ Icmp6OutErrors ||
+ Icmp6InCsumErrors ||
+ Icmp6InDestUnreachs ||
+ Icmp6InPktTooBigs ||
+ Icmp6InTimeExcds ||
+ Icmp6InParmProblems ||
+ Icmp6OutDestUnreachs ||
+ Icmp6OutPktTooBigs ||
+ Icmp6OutTimeExcds ||
+ Icmp6OutParmProblems ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp_errors = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InErrors = NULL,
+ *rd_OutErrors = NULL,
+ *rd_InCsumErrors = NULL,
+ *rd_InDestUnreachs = NULL,
+ *rd_InPktTooBigs = NULL,
+ *rd_InTimeExcds = NULL,
+ *rd_InParmProblems = NULL,
+ *rd_OutDestUnreachs = NULL,
+ *rd_OutPktTooBigs = NULL,
+ *rd_OutTimeExcds = NULL,
+ *rd_OutParmProblems = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "icmperrors"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 ICMP Errors"
+ , "errors/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP_ERRORS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutErrors = rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InDestUnreachs = rrddim_add(st, "InDestUnreachs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InPktTooBigs = rrddim_add(st, "InPktTooBigs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InTimeExcds = rrddim_add(st, "InTimeExcds", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InParmProblems = rrddim_add(st, "InParmProblems", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutDestUnreachs = rrddim_add(st, "OutDestUnreachs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutPktTooBigs = rrddim_add(st, "OutPktTooBigs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutTimeExcds = rrddim_add(st, "OutTimeExcds", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutParmProblems = rrddim_add(st, "OutParmProblems", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InErrors, Icmp6InErrors);
+ rrddim_set_by_pointer(st, rd_OutErrors, Icmp6OutErrors);
+ rrddim_set_by_pointer(st, rd_InCsumErrors, Icmp6InCsumErrors);
+ rrddim_set_by_pointer(st, rd_InDestUnreachs, Icmp6InDestUnreachs);
+ rrddim_set_by_pointer(st, rd_InPktTooBigs, Icmp6InPktTooBigs);
+ rrddim_set_by_pointer(st, rd_InTimeExcds, Icmp6InTimeExcds);
+ rrddim_set_by_pointer(st, rd_InParmProblems, Icmp6InParmProblems);
+ rrddim_set_by_pointer(st, rd_OutDestUnreachs, Icmp6OutDestUnreachs);
+ rrddim_set_by_pointer(st, rd_OutPktTooBigs, Icmp6OutPktTooBigs);
+ rrddim_set_by_pointer(st, rd_OutTimeExcds, Icmp6OutTimeExcds);
+ rrddim_set_by_pointer(st, rd_OutParmProblems, Icmp6OutParmProblems);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp_echos == CONFIG_BOOLEAN_YES || (do_ip6_icmp_echos == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InEchos ||
+ Icmp6OutEchos ||
+ Icmp6InEchoReplies ||
+ Icmp6OutEchoReplies ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp_echos = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InEchos = NULL,
+ *rd_OutEchos = NULL,
+ *rd_InEchoReplies = NULL,
+ *rd_OutEchoReplies = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "icmpechos"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 ICMP Echo"
+ , "messages/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP_ECHOS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InEchos = rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutEchos = rrddim_add(st, "OutEchos", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InEchoReplies = rrddim_add(st, "InEchoReplies", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutEchoReplies = rrddim_add(st, "OutEchoReplies", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InEchos, Icmp6InEchos);
+ rrddim_set_by_pointer(st, rd_OutEchos, Icmp6OutEchos);
+ rrddim_set_by_pointer(st, rd_InEchoReplies, Icmp6InEchoReplies);
+ rrddim_set_by_pointer(st, rd_OutEchoReplies, Icmp6OutEchoReplies);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp_groupmemb == CONFIG_BOOLEAN_YES || (do_ip6_icmp_groupmemb == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InGroupMembQueries ||
+ Icmp6OutGroupMembQueries ||
+ Icmp6InGroupMembResponses ||
+ Icmp6OutGroupMembResponses ||
+ Icmp6InGroupMembReductions ||
+ Icmp6OutGroupMembReductions ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp_groupmemb = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InQueries = NULL,
+ *rd_OutQueries = NULL,
+ *rd_InResponses = NULL,
+ *rd_OutResponses = NULL,
+ *rd_InReductions = NULL,
+ *rd_OutReductions = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "groupmemb"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 ICMP Group Membership"
+ , "messages/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP_GROUPMEMB
+ , update_every
+ , RRDSET_TYPE_LINE);
+
+ rd_InQueries = rrddim_add(st, "InQueries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutQueries = rrddim_add(st, "OutQueries", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InResponses = rrddim_add(st, "InResponses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutResponses = rrddim_add(st, "OutResponses", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InReductions = rrddim_add(st, "InReductions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutReductions = rrddim_add(st, "OutReductions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InQueries, Icmp6InGroupMembQueries);
+ rrddim_set_by_pointer(st, rd_OutQueries, Icmp6OutGroupMembQueries);
+ rrddim_set_by_pointer(st, rd_InResponses, Icmp6InGroupMembResponses);
+ rrddim_set_by_pointer(st, rd_OutResponses, Icmp6OutGroupMembResponses);
+ rrddim_set_by_pointer(st, rd_InReductions, Icmp6InGroupMembReductions);
+ rrddim_set_by_pointer(st, rd_OutReductions, Icmp6OutGroupMembReductions);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp_router == CONFIG_BOOLEAN_YES || (do_ip6_icmp_router == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InRouterSolicits ||
+ Icmp6OutRouterSolicits ||
+ Icmp6InRouterAdvertisements ||
+ Icmp6OutRouterAdvertisements ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp_router = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InSolicits = NULL,
+ *rd_OutSolicits = NULL,
+ *rd_InAdvertisements = NULL,
+ *rd_OutAdvertisements = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "icmprouter"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 Router Messages"
+ , "messages/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP_ROUTER
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InSolicits = rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutSolicits = rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InAdvertisements = rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutAdvertisements = rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InSolicits, Icmp6InRouterSolicits);
+ rrddim_set_by_pointer(st, rd_OutSolicits, Icmp6OutRouterSolicits);
+ rrddim_set_by_pointer(st, rd_InAdvertisements, Icmp6InRouterAdvertisements);
+ rrddim_set_by_pointer(st, rd_OutAdvertisements, Icmp6OutRouterAdvertisements);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp_neighbor == CONFIG_BOOLEAN_YES || (do_ip6_icmp_neighbor == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InNeighborSolicits ||
+ Icmp6OutNeighborSolicits ||
+ Icmp6InNeighborAdvertisements ||
+ Icmp6OutNeighborAdvertisements ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp_neighbor = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InSolicits = NULL,
+ *rd_OutSolicits = NULL,
+ *rd_InAdvertisements = NULL,
+ *rd_OutAdvertisements = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "icmpneighbor"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 Neighbor Messages"
+ , "messages/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP_NEIGHBOR
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InSolicits = rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutSolicits = rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InAdvertisements = rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutAdvertisements = rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InSolicits, Icmp6InNeighborSolicits);
+ rrddim_set_by_pointer(st, rd_OutSolicits, Icmp6OutNeighborSolicits);
+ rrddim_set_by_pointer(st, rd_InAdvertisements, Icmp6InNeighborAdvertisements);
+ rrddim_set_by_pointer(st, rd_OutAdvertisements, Icmp6OutNeighborAdvertisements);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp_mldv2 == CONFIG_BOOLEAN_YES || (do_ip6_icmp_mldv2 == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InMLDv2Reports ||
+ Icmp6OutMLDv2Reports ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp_mldv2 = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InMLDv2Reports = NULL,
+ *rd_OutMLDv2Reports = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "icmpmldv2"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 ICMP MLDv2 Reports"
+ , "reports/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP_LDV2
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InMLDv2Reports = rrddim_add(st, "InMLDv2Reports", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutMLDv2Reports = rrddim_add(st, "OutMLDv2Reports", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InMLDv2Reports, Icmp6InMLDv2Reports);
+ rrddim_set_by_pointer(st, rd_OutMLDv2Reports, Icmp6OutMLDv2Reports);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_icmp_types == CONFIG_BOOLEAN_YES || (do_ip6_icmp_types == CONFIG_BOOLEAN_AUTO &&
+ (Icmp6InType1 ||
+ Icmp6InType128 ||
+ Icmp6InType129 ||
+ Icmp6InType136 ||
+ Icmp6OutType1 ||
+ Icmp6OutType128 ||
+ Icmp6OutType129 ||
+ Icmp6OutType133 ||
+ Icmp6OutType135 ||
+ Icmp6OutType143 ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_icmp_types = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InType1 = NULL,
+ *rd_InType128 = NULL,
+ *rd_InType129 = NULL,
+ *rd_InType136 = NULL,
+ *rd_OutType1 = NULL,
+ *rd_OutType128 = NULL,
+ *rd_OutType129 = NULL,
+ *rd_OutType133 = NULL,
+ *rd_OutType135 = NULL,
+ *rd_OutType143 = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "icmptypes"
+ , NULL
+ , "icmp6"
+ , NULL
+ , "IPv6 ICMP Types"
+ , "messages/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ICMP_TYPES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InType1 = rrddim_add(st, "InType1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InType128 = rrddim_add(st, "InType128", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InType129 = rrddim_add(st, "InType129", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InType136 = rrddim_add(st, "InType136", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutType1 = rrddim_add(st, "OutType1", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutType128 = rrddim_add(st, "OutType128", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutType129 = rrddim_add(st, "OutType129", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutType133 = rrddim_add(st, "OutType133", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutType135 = rrddim_add(st, "OutType135", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutType143 = rrddim_add(st, "OutType143", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InType1, Icmp6InType1);
+ rrddim_set_by_pointer(st, rd_InType128, Icmp6InType128);
+ rrddim_set_by_pointer(st, rd_InType129, Icmp6InType129);
+ rrddim_set_by_pointer(st, rd_InType136, Icmp6InType136);
+ rrddim_set_by_pointer(st, rd_OutType1, Icmp6OutType1);
+ rrddim_set_by_pointer(st, rd_OutType128, Icmp6OutType128);
+ rrddim_set_by_pointer(st, rd_OutType129, Icmp6OutType129);
+ rrddim_set_by_pointer(st, rd_OutType133, Icmp6OutType133);
+ rrddim_set_by_pointer(st, rd_OutType135, Icmp6OutType135);
+ rrddim_set_by_pointer(st, rd_OutType143, Icmp6OutType143);
+ rrdset_done(st);
+ }
+
+ if(do_ip6_ect == CONFIG_BOOLEAN_YES || (do_ip6_ect == CONFIG_BOOLEAN_AUTO &&
+ (Ip6InNoECTPkts ||
+ Ip6InECT1Pkts ||
+ Ip6InECT0Pkts ||
+ Ip6InCEPkts ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ip6_ect = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_InNoECTPkts = NULL,
+ *rd_InECT1Pkts = NULL,
+ *rd_InECT0Pkts = NULL,
+ *rd_InCEPkts = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP6
+ , "ect"
+ , NULL
+ , "packets"
+ , NULL
+ , "IPv6 ECT Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NETSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV6_ECT
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_InNoECTPkts = rrddim_add(st, "InNoECTPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InECT1Pkts = rrddim_add(st, "InECT1Pkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InECT0Pkts = rrddim_add(st, "InECT0Pkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InCEPkts = rrddim_add(st, "InCEPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InNoECTPkts, Ip6InNoECTPkts);
+ rrddim_set_by_pointer(st, rd_InECT1Pkts, Ip6InECT1Pkts);
+ rrddim_set_by_pointer(st, rd_InECT0Pkts, Ip6InECT0Pkts);
+ rrddim_set_by_pointer(st, rd_InCEPkts, Ip6InCEPkts);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_rpc_nfs.c b/collectors/proc.plugin/proc_net_rpc_nfs.c
new file mode 100644
index 0000000..b1ff4e0
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_rpc_nfs.c
@@ -0,0 +1,439 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_NFS_NAME "/proc/net/rpc/nfs"
+#define CONFIG_SECTION_PLUGIN_PROC_NFS "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_NFS_NAME
+
+struct nfs_procs {
+ char name[30];
+ unsigned long long value;
+ int present;
+ RRDDIM *rd;
+};
+
+struct nfs_procs nfs_proc2_values[] = {
+ { "null" , 0ULL, 0, NULL}
+ , {"getattr" , 0ULL, 0, NULL}
+ , {"setattr" , 0ULL, 0, NULL}
+ , {"root" , 0ULL, 0, NULL}
+ , {"lookup" , 0ULL, 0, NULL}
+ , {"readlink", 0ULL, 0, NULL}
+ , {"read" , 0ULL, 0, NULL}
+ , {"wrcache" , 0ULL, 0, NULL}
+ , {"write" , 0ULL, 0, NULL}
+ , {"create" , 0ULL, 0, NULL}
+ , {"remove" , 0ULL, 0, NULL}
+ , {"rename" , 0ULL, 0, NULL}
+ , {"link" , 0ULL, 0, NULL}
+ , {"symlink" , 0ULL, 0, NULL}
+ , {"mkdir" , 0ULL, 0, NULL}
+ , {"rmdir" , 0ULL, 0, NULL}
+ , {"readdir" , 0ULL, 0, NULL}
+ , {"fsstat" , 0ULL, 0, NULL}
+ ,
+
+ /* termination */
+ { "" , 0ULL, 0, NULL}
+};
+
+struct nfs_procs nfs_proc3_values[] = {
+ { "null" , 0ULL, 0, NULL}
+ , {"getattr" , 0ULL, 0, NULL}
+ , {"setattr" , 0ULL, 0, NULL}
+ , {"lookup" , 0ULL, 0, NULL}
+ , {"access" , 0ULL, 0, NULL}
+ , {"readlink" , 0ULL, 0, NULL}
+ , {"read" , 0ULL, 0, NULL}
+ , {"write" , 0ULL, 0, NULL}
+ , {"create" , 0ULL, 0, NULL}
+ , {"mkdir" , 0ULL, 0, NULL}
+ , {"symlink" , 0ULL, 0, NULL}
+ , {"mknod" , 0ULL, 0, NULL}
+ , {"remove" , 0ULL, 0, NULL}
+ , {"rmdir" , 0ULL, 0, NULL}
+ , {"rename" , 0ULL, 0, NULL}
+ , {"link" , 0ULL, 0, NULL}
+ , {"readdir" , 0ULL, 0, NULL}
+ , {"readdirplus", 0ULL, 0, NULL}
+ , {"fsstat" , 0ULL, 0, NULL}
+ , {"fsinfo" , 0ULL, 0, NULL}
+ , {"pathconf" , 0ULL, 0, NULL}
+ , {"commit" , 0ULL, 0, NULL}
+ ,
+
+ /* termination */
+ { "" , 0ULL, 0, NULL}
+};
+
+struct nfs_procs nfs_proc4_values[] = {
+ { "null" , 0ULL, 0, NULL}
+ , {"read" , 0ULL, 0, NULL}
+ , {"write" , 0ULL, 0, NULL}
+ , {"commit" , 0ULL, 0, NULL}
+ , {"open" , 0ULL, 0, NULL}
+ , {"open_conf" , 0ULL, 0, NULL}
+ , {"open_noat" , 0ULL, 0, NULL}
+ , {"open_dgrd" , 0ULL, 0, NULL}
+ , {"close" , 0ULL, 0, NULL}
+ , {"setattr" , 0ULL, 0, NULL}
+ , {"fsinfo" , 0ULL, 0, NULL}
+ , {"renew" , 0ULL, 0, NULL}
+ , {"setclntid" , 0ULL, 0, NULL}
+ , {"confirm" , 0ULL, 0, NULL}
+ , {"lock" , 0ULL, 0, NULL}
+ , {"lockt" , 0ULL, 0, NULL}
+ , {"locku" , 0ULL, 0, NULL}
+ , {"access" , 0ULL, 0, NULL}
+ , {"getattr" , 0ULL, 0, NULL}
+ , {"lookup" , 0ULL, 0, NULL}
+ , {"lookup_root" , 0ULL, 0, NULL}
+ , {"remove" , 0ULL, 0, NULL}
+ , {"rename" , 0ULL, 0, NULL}
+ , {"link" , 0ULL, 0, NULL}
+ , {"symlink" , 0ULL, 0, NULL}
+ , {"create" , 0ULL, 0, NULL}
+ , {"pathconf" , 0ULL, 0, NULL}
+ , {"statfs" , 0ULL, 0, NULL}
+ , {"readlink" , 0ULL, 0, NULL}
+ , {"readdir" , 0ULL, 0, NULL}
+ , {"server_caps" , 0ULL, 0, NULL}
+ , {"delegreturn" , 0ULL, 0, NULL}
+ , {"getacl" , 0ULL, 0, NULL}
+ , {"setacl" , 0ULL, 0, NULL}
+ , {"fs_locations" , 0ULL, 0, NULL}
+ , {"rel_lkowner" , 0ULL, 0, NULL}
+ , {"secinfo" , 0ULL, 0, NULL}
+ , {"fsid_present" , 0ULL, 0, NULL}
+ ,
+
+ /* nfsv4.1 client ops */
+ { "exchange_id" , 0ULL, 0, NULL}
+ , {"create_session" , 0ULL, 0, NULL}
+ , {"destroy_session" , 0ULL, 0, NULL}
+ , {"sequence" , 0ULL, 0, NULL}
+ , {"get_lease_time" , 0ULL, 0, NULL}
+ , {"reclaim_comp" , 0ULL, 0, NULL}
+ , {"layoutget" , 0ULL, 0, NULL}
+ , {"getdevinfo" , 0ULL, 0, NULL}
+ , {"layoutcommit" , 0ULL, 0, NULL}
+ , {"layoutreturn" , 0ULL, 0, NULL}
+ , {"secinfo_no" , 0ULL, 0, NULL}
+ , {"test_stateid" , 0ULL, 0, NULL}
+ , {"free_stateid" , 0ULL, 0, NULL}
+ , {"getdevicelist" , 0ULL, 0, NULL}
+ , {"bind_conn_to_ses", 0ULL, 0, NULL}
+ , {"destroy_clientid", 0ULL, 0, NULL}
+ ,
+
+ /* nfsv4.2 client ops */
+ { "seek" , 0ULL, 0, NULL}
+ , {"allocate" , 0ULL, 0, NULL}
+ , {"deallocate" , 0ULL, 0, NULL}
+ , {"layoutstats" , 0ULL, 0, NULL}
+ , {"clone" , 0ULL, 0, NULL}
+ ,
+
+ /* termination */
+ { "" , 0ULL, 0, NULL}
+};
+
+int do_proc_net_rpc_nfs(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+ static int do_net = -1, do_rpc = -1, do_proc2 = -1, do_proc3 = -1, do_proc4 = -1;
+ static int proc2_warning = 0, proc3_warning = 0, proc4_warning = 0;
+
+ if(!ff) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/rpc/nfs");
+ ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_NFS, "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT);
+ }
+ if(!ff) return 1;
+
+ ff = procfile_readall(ff);
+ if(!ff) return 0; // we return 0, so that we will retry to open it next time
+
+ if(do_net == -1) do_net = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_NFS, "network", 1);
+ if(do_rpc == -1) do_rpc = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_NFS, "rpc", 1);
+ if(do_proc2 == -1) do_proc2 = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_NFS, "NFS v2 procedures", 1);
+ if(do_proc3 == -1) do_proc3 = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_NFS, "NFS v3 procedures", 1);
+ if(do_proc4 == -1) do_proc4 = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_NFS, "NFS v4 procedures", 1);
+
+ // if they are enabled, reset them to 1
+ // later we do them =2 to avoid doing strcmp() for all lines
+ if(do_net) do_net = 1;
+ if(do_rpc) do_rpc = 1;
+ if(do_proc2) do_proc2 = 1;
+ if(do_proc3) do_proc3 = 1;
+ if(do_proc4) do_proc4 = 1;
+
+ size_t lines = procfile_lines(ff), l;
+
+ char *type;
+ unsigned long long net_count = 0, net_udp_count = 0, net_tcp_count = 0, net_tcp_connections = 0;
+ unsigned long long rpc_calls = 0, rpc_retransmits = 0, rpc_auth_refresh = 0;
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(!words) continue;
+
+ type = procfile_lineword(ff, l, 0);
+
+ if(do_net == 1 && strcmp(type, "net") == 0) {
+ if(words < 5) {
+ error("%s line of /proc/net/rpc/nfs has %zu words, expected %d", type, words, 5);
+ continue;
+ }
+
+ net_count = str2ull(procfile_lineword(ff, l, 1));
+ net_udp_count = str2ull(procfile_lineword(ff, l, 2));
+ net_tcp_count = str2ull(procfile_lineword(ff, l, 3));
+ net_tcp_connections = str2ull(procfile_lineword(ff, l, 4));
+
+ unsigned long long sum = net_count + net_udp_count + net_tcp_count + net_tcp_connections;
+ if(sum == 0ULL) do_net = -1;
+ else do_net = 2;
+ }
+ else if(do_rpc == 1 && strcmp(type, "rpc") == 0) {
+ if(words < 4) {
+ error("%s line of /proc/net/rpc/nfs has %zu words, expected %d", type, words, 6);
+ continue;
+ }
+
+ rpc_calls = str2ull(procfile_lineword(ff, l, 1));
+ rpc_retransmits = str2ull(procfile_lineword(ff, l, 2));
+ rpc_auth_refresh = str2ull(procfile_lineword(ff, l, 3));
+
+ unsigned long long sum = rpc_calls + rpc_retransmits + rpc_auth_refresh;
+ if(sum == 0ULL) do_rpc = -1;
+ else do_rpc = 2;
+ }
+ else if(do_proc2 == 1 && strcmp(type, "proc2") == 0) {
+ // the first number is the count of numbers present
+ // so we start for word 2
+
+ unsigned long long sum = 0;
+ unsigned int i, j;
+ for(i = 0, j = 2; j < words && nfs_proc2_values[i].name[0] ; i++, j++) {
+ nfs_proc2_values[i].value = str2ull(procfile_lineword(ff, l, j));
+ nfs_proc2_values[i].present = 1;
+ sum += nfs_proc2_values[i].value;
+ }
+
+ if(sum == 0ULL) {
+ if(!proc2_warning) {
+ error("Disabling /proc/net/rpc/nfs v2 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it.");
+ proc2_warning = 1;
+ }
+ do_proc2 = 0;
+ }
+ else do_proc2 = 2;
+ }
+ else if(do_proc3 == 1 && strcmp(type, "proc3") == 0) {
+ // the first number is the count of numbers present
+ // so we start for word 2
+
+ unsigned long long sum = 0;
+ unsigned int i, j;
+ for(i = 0, j = 2; j < words && nfs_proc3_values[i].name[0] ; i++, j++) {
+ nfs_proc3_values[i].value = str2ull(procfile_lineword(ff, l, j));
+ nfs_proc3_values[i].present = 1;
+ sum += nfs_proc3_values[i].value;
+ }
+
+ if(sum == 0ULL) {
+ if(!proc3_warning) {
+ info("Disabling /proc/net/rpc/nfs v3 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it.");
+ proc3_warning = 1;
+ }
+ do_proc3 = 0;
+ }
+ else do_proc3 = 2;
+ }
+ else if(do_proc4 == 1 && strcmp(type, "proc4") == 0) {
+ // the first number is the count of numbers present
+ // so we start for word 2
+
+ unsigned long long sum = 0;
+ unsigned int i, j;
+ for(i = 0, j = 2; j < words && nfs_proc4_values[i].name[0] ; i++, j++) {
+ nfs_proc4_values[i].value = str2ull(procfile_lineword(ff, l, j));
+ nfs_proc4_values[i].present = 1;
+ sum += nfs_proc4_values[i].value;
+ }
+
+ if(sum == 0ULL) {
+ if(!proc4_warning) {
+ info("Disabling /proc/net/rpc/nfs v4 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it.");
+ proc4_warning = 1;
+ }
+ do_proc4 = 0;
+ }
+ else do_proc4 = 2;
+ }
+ }
+
+ if(do_net == 2) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_udp = NULL,
+ *rd_tcp = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfs"
+ , "net"
+ , NULL
+ , "network"
+ , NULL
+ , "NFS Client Network"
+ , "operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFS_NAME
+ , NETDATA_CHART_PRIO_NFS_NET
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_udp = rrddim_add(st, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_tcp = rrddim_add(st, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ // ignore net_count, net_tcp_connections
+ (void)net_count;
+ (void)net_tcp_connections;
+
+ rrddim_set_by_pointer(st, rd_udp, net_udp_count);
+ rrddim_set_by_pointer(st, rd_tcp, net_tcp_count);
+ rrdset_done(st);
+ }
+
+ if(do_rpc == 2) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_calls = NULL,
+ *rd_retransmits = NULL,
+ *rd_auth_refresh = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfs"
+ , "rpc"
+ , NULL
+ , "rpc"
+ , NULL
+ , "NFS Client Remote Procedure Calls Statistics"
+ , "calls/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFS_NAME
+ , NETDATA_CHART_PRIO_NFS_RPC
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_calls = rrddim_add(st, "calls", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_retransmits = rrddim_add(st, "retransmits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_auth_refresh = rrddim_add(st, "auth_refresh", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_calls, rpc_calls);
+ rrddim_set_by_pointer(st, rd_retransmits, rpc_retransmits);
+ rrddim_set_by_pointer(st, rd_auth_refresh, rpc_auth_refresh);
+ rrdset_done(st);
+ }
+
+ if(do_proc2 == 2) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfs"
+ , "proc2"
+ , NULL
+ , "nfsv2rpc"
+ , NULL
+ , "NFS v2 Client Remote Procedure Calls"
+ , "calls/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFS_NAME
+ , NETDATA_CHART_PRIO_NFS_PROC2
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ size_t i;
+ for(i = 0; nfs_proc2_values[i].present ; i++) {
+ if(unlikely(!nfs_proc2_values[i].rd))
+ nfs_proc2_values[i].rd = rrddim_add(st, nfs_proc2_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st, nfs_proc2_values[i].rd, nfs_proc2_values[i].value);
+ }
+
+ rrdset_done(st);
+ }
+
+ if(do_proc3 == 2) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfs"
+ , "proc3"
+ , NULL
+ , "nfsv3rpc"
+ , NULL
+ , "NFS v3 Client Remote Procedure Calls"
+ , "calls/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFS_NAME
+ , NETDATA_CHART_PRIO_NFS_PROC3
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ size_t i;
+ for(i = 0; nfs_proc3_values[i].present ; i++) {
+ if(unlikely(!nfs_proc3_values[i].rd))
+ nfs_proc3_values[i].rd = rrddim_add(st, nfs_proc3_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st, nfs_proc3_values[i].rd, nfs_proc3_values[i].value);
+ }
+
+ rrdset_done(st);
+ }
+
+ if(do_proc4 == 2) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfs"
+ , "proc4"
+ , NULL
+ , "nfsv4rpc"
+ , NULL
+ , "NFS v4 Client Remote Procedure Calls"
+ , "calls/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFS_NAME
+ , NETDATA_CHART_PRIO_NFS_PROC4
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ size_t i;
+ for(i = 0; nfs_proc4_values[i].present ; i++) {
+ if(unlikely(!nfs_proc4_values[i].rd))
+ nfs_proc4_values[i].rd = rrddim_add(st, nfs_proc4_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st, nfs_proc4_values[i].rd, nfs_proc4_values[i].value);
+ }
+
+ rrdset_done(st);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_rpc_nfsd.c b/collectors/proc.plugin/proc_net_rpc_nfsd.c
new file mode 100644
index 0000000..bd1da88
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_rpc_nfsd.c
@@ -0,0 +1,763 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_NFSD_NAME "/proc/net/rpc/nfsd"
+
+struct nfsd_procs {
+ char name[30];
+ unsigned long long value;
+ int present;
+ RRDDIM *rd;
+};
+
+struct nfsd_procs nfsd_proc2_values[] = {
+ { "null" , 0ULL, 0, NULL}
+ , {"getattr" , 0ULL, 0, NULL}
+ , {"setattr" , 0ULL, 0, NULL}
+ , {"root" , 0ULL, 0, NULL}
+ , {"lookup" , 0ULL, 0, NULL}
+ , {"readlink", 0ULL, 0, NULL}
+ , {"read" , 0ULL, 0, NULL}
+ , {"wrcache" , 0ULL, 0, NULL}
+ , {"write" , 0ULL, 0, NULL}
+ , {"create" , 0ULL, 0, NULL}
+ , {"remove" , 0ULL, 0, NULL}
+ , {"rename" , 0ULL, 0, NULL}
+ , {"link" , 0ULL, 0, NULL}
+ , {"symlink" , 0ULL, 0, NULL}
+ , {"mkdir" , 0ULL, 0, NULL}
+ , {"rmdir" , 0ULL, 0, NULL}
+ , {"readdir" , 0ULL, 0, NULL}
+ , {"fsstat" , 0ULL, 0, NULL}
+ ,
+
+ /* termination */
+ { "" , 0ULL, 0, NULL}
+};
+
+struct nfsd_procs nfsd_proc3_values[] = {
+ { "null" , 0ULL, 0, NULL}
+ , {"getattr" , 0ULL, 0, NULL}
+ , {"setattr" , 0ULL, 0, NULL}
+ , {"lookup" , 0ULL, 0, NULL}
+ , {"access" , 0ULL, 0, NULL}
+ , {"readlink" , 0ULL, 0, NULL}
+ , {"read" , 0ULL, 0, NULL}
+ , {"write" , 0ULL, 0, NULL}
+ , {"create" , 0ULL, 0, NULL}
+ , {"mkdir" , 0ULL, 0, NULL}
+ , {"symlink" , 0ULL, 0, NULL}
+ , {"mknod" , 0ULL, 0, NULL}
+ , {"remove" , 0ULL, 0, NULL}
+ , {"rmdir" , 0ULL, 0, NULL}
+ , {"rename" , 0ULL, 0, NULL}
+ , {"link" , 0ULL, 0, NULL}
+ , {"readdir" , 0ULL, 0, NULL}
+ , {"readdirplus", 0ULL, 0, NULL}
+ , {"fsstat" , 0ULL, 0, NULL}
+ , {"fsinfo" , 0ULL, 0, NULL}
+ , {"pathconf" , 0ULL, 0, NULL}
+ , {"commit" , 0ULL, 0, NULL}
+ ,
+
+ /* termination */
+ { "" , 0ULL, 0, NULL}
+};
+
+struct nfsd_procs nfsd_proc4_values[] = {
+ { "null" , 0ULL, 0, NULL}
+ , {"read" , 0ULL, 0, NULL}
+ , {"write" , 0ULL, 0, NULL}
+ , {"commit" , 0ULL, 0, NULL}
+ , {"open" , 0ULL, 0, NULL}
+ , {"open_conf" , 0ULL, 0, NULL}
+ , {"open_noat" , 0ULL, 0, NULL}
+ , {"open_dgrd" , 0ULL, 0, NULL}
+ , {"close" , 0ULL, 0, NULL}
+ , {"setattr" , 0ULL, 0, NULL}
+ , {"fsinfo" , 0ULL, 0, NULL}
+ , {"renew" , 0ULL, 0, NULL}
+ , {"setclntid" , 0ULL, 0, NULL}
+ , {"confirm" , 0ULL, 0, NULL}
+ , {"lock" , 0ULL, 0, NULL}
+ , {"lockt" , 0ULL, 0, NULL}
+ , {"locku" , 0ULL, 0, NULL}
+ , {"access" , 0ULL, 0, NULL}
+ , {"getattr" , 0ULL, 0, NULL}
+ , {"lookup" , 0ULL, 0, NULL}
+ , {"lookup_root" , 0ULL, 0, NULL}
+ , {"remove" , 0ULL, 0, NULL}
+ , {"rename" , 0ULL, 0, NULL}
+ , {"link" , 0ULL, 0, NULL}
+ , {"symlink" , 0ULL, 0, NULL}
+ , {"create" , 0ULL, 0, NULL}
+ , {"pathconf" , 0ULL, 0, NULL}
+ , {"statfs" , 0ULL, 0, NULL}
+ , {"readlink" , 0ULL, 0, NULL}
+ , {"readdir" , 0ULL, 0, NULL}
+ , {"server_caps" , 0ULL, 0, NULL}
+ , {"delegreturn" , 0ULL, 0, NULL}
+ , {"getacl" , 0ULL, 0, NULL}
+ , {"setacl" , 0ULL, 0, NULL}
+ , {"fs_locations" , 0ULL, 0, NULL}
+ , {"rel_lkowner" , 0ULL, 0, NULL}
+ , {"secinfo" , 0ULL, 0, NULL}
+ , {"fsid_present" , 0ULL, 0, NULL}
+ ,
+
+ /* nfsv4.1 client ops */
+ { "exchange_id" , 0ULL, 0, NULL}
+ , {"create_session" , 0ULL, 0, NULL}
+ , {"destroy_session" , 0ULL, 0, NULL}
+ , {"sequence" , 0ULL, 0, NULL}
+ , {"get_lease_time" , 0ULL, 0, NULL}
+ , {"reclaim_comp" , 0ULL, 0, NULL}
+ , {"layoutget" , 0ULL, 0, NULL}
+ , {"getdevinfo" , 0ULL, 0, NULL}
+ , {"layoutcommit" , 0ULL, 0, NULL}
+ , {"layoutreturn" , 0ULL, 0, NULL}
+ , {"secinfo_no" , 0ULL, 0, NULL}
+ , {"test_stateid" , 0ULL, 0, NULL}
+ , {"free_stateid" , 0ULL, 0, NULL}
+ , {"getdevicelist" , 0ULL, 0, NULL}
+ , {"bind_conn_to_ses", 0ULL, 0, NULL}
+ , {"destroy_clientid", 0ULL, 0, NULL}
+ ,
+
+ /* nfsv4.2 client ops */
+ { "seek" , 0ULL, 0, NULL}
+ , {"allocate" , 0ULL, 0, NULL}
+ , {"deallocate" , 0ULL, 0, NULL}
+ , {"layoutstats" , 0ULL, 0, NULL}
+ , {"clone" , 0ULL, 0, NULL}
+ ,
+
+ /* termination */
+ { "" , 0ULL, 0, NULL}
+};
+
+struct nfsd_procs nfsd4_ops_values[] = {
+ { "unused_op0" , 0ULL, 0, NULL}
+ , {"unused_op1" , 0ULL, 0, NULL}
+ , {"future_op2" , 0ULL, 0, NULL}
+ , {"access" , 0ULL, 0, NULL}
+ , {"close" , 0ULL, 0, NULL}
+ , {"commit" , 0ULL, 0, NULL}
+ , {"create" , 0ULL, 0, NULL}
+ , {"delegpurge" , 0ULL, 0, NULL}
+ , {"delegreturn" , 0ULL, 0, NULL}
+ , {"getattr" , 0ULL, 0, NULL}
+ , {"getfh" , 0ULL, 0, NULL}
+ , {"link" , 0ULL, 0, NULL}
+ , {"lock" , 0ULL, 0, NULL}
+ , {"lockt" , 0ULL, 0, NULL}
+ , {"locku" , 0ULL, 0, NULL}
+ , {"lookup" , 0ULL, 0, NULL}
+ , {"lookup_root" , 0ULL, 0, NULL}
+ , {"nverify" , 0ULL, 0, NULL}
+ , {"open" , 0ULL, 0, NULL}
+ , {"openattr" , 0ULL, 0, NULL}
+ , {"open_confirm" , 0ULL, 0, NULL}
+ , {"open_downgrade" , 0ULL, 0, NULL}
+ , {"putfh" , 0ULL, 0, NULL}
+ , {"putpubfh" , 0ULL, 0, NULL}
+ , {"putrootfh" , 0ULL, 0, NULL}
+ , {"read" , 0ULL, 0, NULL}
+ , {"readdir" , 0ULL, 0, NULL}
+ , {"readlink" , 0ULL, 0, NULL}
+ , {"remove" , 0ULL, 0, NULL}
+ , {"rename" , 0ULL, 0, NULL}
+ , {"renew" , 0ULL, 0, NULL}
+ , {"restorefh" , 0ULL, 0, NULL}
+ , {"savefh" , 0ULL, 0, NULL}
+ , {"secinfo" , 0ULL, 0, NULL}
+ , {"setattr" , 0ULL, 0, NULL}
+ , {"setclientid" , 0ULL, 0, NULL}
+ , {"setclientid_confirm" , 0ULL, 0, NULL}
+ , {"verify" , 0ULL, 0, NULL}
+ , {"write" , 0ULL, 0, NULL}
+ , {"release_lockowner" , 0ULL, 0, NULL}
+ ,
+
+ /* nfs41 */
+ { "backchannel_ctl" , 0ULL, 0, NULL}
+ , {"bind_conn_to_session", 0ULL, 0, NULL}
+ , {"exchange_id" , 0ULL, 0, NULL}
+ , {"create_session" , 0ULL, 0, NULL}
+ , {"destroy_session" , 0ULL, 0, NULL}
+ , {"free_stateid" , 0ULL, 0, NULL}
+ , {"get_dir_delegation" , 0ULL, 0, NULL}
+ , {"getdeviceinfo" , 0ULL, 0, NULL}
+ , {"getdevicelist" , 0ULL, 0, NULL}
+ , {"layoutcommit" , 0ULL, 0, NULL}
+ , {"layoutget" , 0ULL, 0, NULL}
+ , {"layoutreturn" , 0ULL, 0, NULL}
+ , {"secinfo_no_name" , 0ULL, 0, NULL}
+ , {"sequence" , 0ULL, 0, NULL}
+ , {"set_ssv" , 0ULL, 0, NULL}
+ , {"test_stateid" , 0ULL, 0, NULL}
+ , {"want_delegation" , 0ULL, 0, NULL}
+ , {"destroy_clientid" , 0ULL, 0, NULL}
+ , {"reclaim_complete" , 0ULL, 0, NULL}
+ ,
+
+ /* nfs42 */
+ { "allocate" , 0ULL, 0, NULL}
+ , {"copy" , 0ULL, 0, NULL}
+ , {"copy_notify" , 0ULL, 0, NULL}
+ , {"deallocate" , 0ULL, 0, NULL}
+ , {"ioadvise" , 0ULL, 0, NULL}
+ , {"layouterror" , 0ULL, 0, NULL}
+ , {"layoutstats" , 0ULL, 0, NULL}
+ , {"offload_cancel" , 0ULL, 0, NULL}
+ , {"offload_status" , 0ULL, 0, NULL}
+ , {"read_plus" , 0ULL, 0, NULL}
+ , {"seek" , 0ULL, 0, NULL}
+ , {"write_same" , 0ULL, 0, NULL}
+ ,
+
+ /* termination */
+ { "" , 0ULL, 0, NULL}
+};
+
+
+int do_proc_net_rpc_nfsd(int update_every, usec_t dt) {
+ (void)dt;
+ static procfile *ff = NULL;
+ static int do_rc = -1, do_fh = -1, do_io = -1, do_th = -1, do_net = -1, do_rpc = -1, do_proc2 = -1, do_proc3 = -1, do_proc4 = -1, do_proc4ops = -1;
+ static int proc2_warning = 0, proc3_warning = 0, proc4_warning = 0, proc4ops_warning = 0;
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/rpc/nfsd");
+ ff = procfile_open(config_get("plugin:proc:/proc/net/rpc/nfsd", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ if(unlikely(do_rc == -1)) {
+ do_rc = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "read cache", 1);
+ do_fh = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "file handles", 1);
+ do_io = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "I/O", 1);
+ do_th = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "threads", 1);
+ do_net = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "network", 1);
+ do_rpc = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "rpc", 1);
+ do_proc2 = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v2 procedures", 1);
+ do_proc3 = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v3 procedures", 1);
+ do_proc4 = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v4 procedures", 1);
+ do_proc4ops = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v4 operations", 1);
+ }
+
+ // if they are enabled, reset them to 1
+ // later we do them = 2 to avoid doing strcmp() for all lines
+ if(do_rc) do_rc = 1;
+ if(do_fh) do_fh = 1;
+ if(do_io) do_io = 1;
+ if(do_th) do_th = 1;
+ if(do_net) do_net = 1;
+ if(do_rpc) do_rpc = 1;
+ if(do_proc2) do_proc2 = 1;
+ if(do_proc3) do_proc3 = 1;
+ if(do_proc4) do_proc4 = 1;
+ if(do_proc4ops) do_proc4ops = 1;
+
+ size_t lines = procfile_lines(ff), l;
+
+ char *type;
+ unsigned long long rc_hits = 0, rc_misses = 0, rc_nocache = 0;
+ unsigned long long fh_stale = 0;
+ unsigned long long io_read = 0, io_write = 0;
+ unsigned long long th_threads = 0;
+ unsigned long long net_count = 0, net_udp_count = 0, net_tcp_count = 0, net_tcp_connections = 0;
+ unsigned long long rpc_calls = 0, rpc_bad_format = 0, rpc_bad_auth = 0, rpc_bad_client = 0;
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(!words)) continue;
+
+ type = procfile_lineword(ff, l, 0);
+
+ if(do_rc == 1 && strcmp(type, "rc") == 0) {
+ if(unlikely(words < 4)) {
+ error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 4);
+ continue;
+ }
+
+ rc_hits = str2ull(procfile_lineword(ff, l, 1));
+ rc_misses = str2ull(procfile_lineword(ff, l, 2));
+ rc_nocache = str2ull(procfile_lineword(ff, l, 3));
+
+ unsigned long long sum = rc_hits + rc_misses + rc_nocache;
+ if(sum == 0ULL) do_rc = -1;
+ else do_rc = 2;
+ }
+ else if(do_fh == 1 && strcmp(type, "fh") == 0) {
+ if(unlikely(words < 6)) {
+ error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 6);
+ continue;
+ }
+
+ fh_stale = str2ull(procfile_lineword(ff, l, 1));
+
+ // other file handler metrics were never used and are always zero
+
+ if(fh_stale == 0ULL) do_fh = -1;
+ else do_fh = 2;
+ }
+ else if(do_io == 1 && strcmp(type, "io") == 0) {
+ if(unlikely(words < 3)) {
+ error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 3);
+ continue;
+ }
+
+ io_read = str2ull(procfile_lineword(ff, l, 1));
+ io_write = str2ull(procfile_lineword(ff, l, 2));
+
+ unsigned long long sum = io_read + io_write;
+ if(sum == 0ULL) do_io = -1;
+ else do_io = 2;
+ }
+ else if(do_th == 1 && strcmp(type, "th") == 0) {
+ if(unlikely(words < 13)) {
+ error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 13);
+ continue;
+ }
+
+ th_threads = str2ull(procfile_lineword(ff, l, 1));
+
+ // thread histogram has been disabled since 2009 (kernel 2.6.30)
+ // https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=8bbfa9f3889b643fc7de82c0c761ef17097f8faf
+
+ do_th = 2;
+ }
+ else if(do_net == 1 && strcmp(type, "net") == 0) {
+ if(unlikely(words < 5)) {
+ error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 5);
+ continue;
+ }
+
+ net_count = str2ull(procfile_lineword(ff, l, 1));
+ net_udp_count = str2ull(procfile_lineword(ff, l, 2));
+ net_tcp_count = str2ull(procfile_lineword(ff, l, 3));
+ net_tcp_connections = str2ull(procfile_lineword(ff, l, 4));
+
+ unsigned long long sum = net_count + net_udp_count + net_tcp_count + net_tcp_connections;
+ if(sum == 0ULL) do_net = -1;
+ else do_net = 2;
+ }
+ else if(do_rpc == 1 && strcmp(type, "rpc") == 0) {
+ if(unlikely(words < 6)) {
+ error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 6);
+ continue;
+ }
+
+ rpc_calls = str2ull(procfile_lineword(ff, l, 1));
+ rpc_bad_format = str2ull(procfile_lineword(ff, l, 3));
+ rpc_bad_auth = str2ull(procfile_lineword(ff, l, 4));
+ rpc_bad_client = str2ull(procfile_lineword(ff, l, 5));
+
+ unsigned long long sum = rpc_calls + rpc_bad_format + rpc_bad_auth + rpc_bad_client;
+ if(sum == 0ULL) do_rpc = -1;
+ else do_rpc = 2;
+ }
+ else if(do_proc2 == 1 && strcmp(type, "proc2") == 0) {
+ // the first number is the count of numbers present
+ // so we start for word 2
+
+ unsigned long long sum = 0;
+ unsigned int i, j;
+ for(i = 0, j = 2; j < words && nfsd_proc2_values[i].name[0] ; i++, j++) {
+ nfsd_proc2_values[i].value = str2ull(procfile_lineword(ff, l, j));
+ nfsd_proc2_values[i].present = 1;
+ sum += nfsd_proc2_values[i].value;
+ }
+
+ if(sum == 0ULL) {
+ if(!proc2_warning) {
+ error("Disabling /proc/net/rpc/nfsd v2 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it.");
+ proc2_warning = 1;
+ }
+ do_proc2 = 0;
+ }
+ else do_proc2 = 2;
+ }
+ else if(do_proc3 == 1 && strcmp(type, "proc3") == 0) {
+ // the first number is the count of numbers present
+ // so we start for word 2
+
+ unsigned long long sum = 0;
+ unsigned int i, j;
+ for(i = 0, j = 2; j < words && nfsd_proc3_values[i].name[0] ; i++, j++) {
+ nfsd_proc3_values[i].value = str2ull(procfile_lineword(ff, l, j));
+ nfsd_proc3_values[i].present = 1;
+ sum += nfsd_proc3_values[i].value;
+ }
+
+ if(sum == 0ULL) {
+ if(!proc3_warning) {
+ info("Disabling /proc/net/rpc/nfsd v3 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it.");
+ proc3_warning = 1;
+ }
+ do_proc3 = 0;
+ }
+ else do_proc3 = 2;
+ }
+ else if(do_proc4 == 1 && strcmp(type, "proc4") == 0) {
+ // the first number is the count of numbers present
+ // so we start for word 2
+
+ unsigned long long sum = 0;
+ unsigned int i, j;
+ for(i = 0, j = 2; j < words && nfsd_proc4_values[i].name[0] ; i++, j++) {
+ nfsd_proc4_values[i].value = str2ull(procfile_lineword(ff, l, j));
+ nfsd_proc4_values[i].present = 1;
+ sum += nfsd_proc4_values[i].value;
+ }
+
+ if(sum == 0ULL) {
+ if(!proc4_warning) {
+ info("Disabling /proc/net/rpc/nfsd v4 procedure calls chart. It seems unused on this machine. It will be enabled automatically when found with data in it.");
+ proc4_warning = 1;
+ }
+ do_proc4 = 0;
+ }
+ else do_proc4 = 2;
+ }
+ else if(do_proc4ops == 1 && strcmp(type, "proc4ops") == 0) {
+ // the first number is the count of numbers present
+ // so we start for word 2
+
+ unsigned long long sum = 0;
+ unsigned int i, j;
+ for(i = 0, j = 2; j < words && nfsd4_ops_values[i].name[0] ; i++, j++) {
+ nfsd4_ops_values[i].value = str2ull(procfile_lineword(ff, l, j));
+ nfsd4_ops_values[i].present = 1;
+ sum += nfsd4_ops_values[i].value;
+ }
+
+ if(sum == 0ULL) {
+ if(!proc4ops_warning) {
+ info("Disabling /proc/net/rpc/nfsd v4 operations chart. It seems unused on this machine. It will be enabled automatically when found with data in it.");
+ proc4ops_warning = 1;
+ }
+ do_proc4ops = 0;
+ }
+ else do_proc4ops = 2;
+ }
+ }
+
+ if(do_rc == 2) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_hits = NULL,
+ *rd_misses = NULL,
+ *rd_nocache = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "readcache"
+ , NULL
+ , "cache"
+ , NULL
+ , "NFS Server Read Cache"
+ , "reads/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_READCACHE
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_hits = rrddim_add(st, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_misses = rrddim_add(st, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_nocache = rrddim_add(st, "nocache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_hits, rc_hits);
+ rrddim_set_by_pointer(st, rd_misses, rc_misses);
+ rrddim_set_by_pointer(st, rd_nocache, rc_nocache);
+ rrdset_done(st);
+ }
+
+ if(do_fh == 2) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_stale = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "filehandles"
+ , NULL
+ , "filehandles"
+ , NULL
+ , "NFS Server File Handles"
+ , "handles/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_FILEHANDLES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_stale = rrddim_add(st, "stale", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_stale, fh_stale);
+ rrdset_done(st);
+ }
+
+ if(do_io == 2) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_read = NULL,
+ *rd_write = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "io"
+ , NULL
+ , "io"
+ , NULL
+ , "NFS Server I/O"
+ , "kilobytes/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_IO
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_read = rrddim_add(st, "read", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
+ rd_write = rrddim_add(st, "write", NULL, -1, 1000, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_read, io_read);
+ rrddim_set_by_pointer(st, rd_write, io_write);
+ rrdset_done(st);
+ }
+
+ if(do_th == 2) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_threads = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "threads"
+ , NULL
+ , "threads"
+ , NULL
+ , "NFS Server Threads"
+ , "threads"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_THREADS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_threads = rrddim_add(st, "threads", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_threads, th_threads);
+ rrdset_done(st);
+ }
+
+ if(do_net == 2) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_udp = NULL,
+ *rd_tcp = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "net"
+ , NULL
+ , "network"
+ , NULL
+ , "NFS Server Network Statistics"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_NET
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_udp = rrddim_add(st, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_tcp = rrddim_add(st, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ // ignore net_count, net_tcp_connections
+ (void)net_count;
+ (void)net_tcp_connections;
+
+ rrddim_set_by_pointer(st, rd_udp, net_udp_count);
+ rrddim_set_by_pointer(st, rd_tcp, net_tcp_count);
+ rrdset_done(st);
+ }
+
+ if(do_rpc == 2) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_calls = NULL,
+ *rd_bad_format = NULL,
+ *rd_bad_auth = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "rpc"
+ , NULL
+ , "rpc"
+ , NULL
+ , "NFS Server Remote Procedure Calls Statistics"
+ , "calls/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_RPC
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_calls = rrddim_add(st, "calls", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_bad_format = rrddim_add(st, "bad_format", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_bad_auth = rrddim_add(st, "bad_auth", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ // ignore rpc_bad_client
+ (void)rpc_bad_client;
+
+ rrddim_set_by_pointer(st, rd_calls, rpc_calls);
+ rrddim_set_by_pointer(st, rd_bad_format, rpc_bad_format);
+ rrddim_set_by_pointer(st, rd_bad_auth, rpc_bad_auth);
+ rrdset_done(st);
+ }
+
+ if(do_proc2 == 2) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "proc2"
+ , NULL
+ , "nfsv2rpc"
+ , NULL
+ , "NFS v2 Server Remote Procedure Calls"
+ , "calls/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_PROC2
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ size_t i;
+ for(i = 0; nfsd_proc2_values[i].present ; i++) {
+ if(unlikely(!nfsd_proc2_values[i].rd))
+ nfsd_proc2_values[i].rd = rrddim_add(st, nfsd_proc2_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st, nfsd_proc2_values[i].rd, nfsd_proc2_values[i].value);
+ }
+
+ rrdset_done(st);
+ }
+
+ if(do_proc3 == 2) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "proc3"
+ , NULL
+ , "nfsv3rpc"
+ , NULL
+ , "NFS v3 Server Remote Procedure Calls"
+ , "calls/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_PROC3
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ size_t i;
+ for(i = 0; nfsd_proc3_values[i].present ; i++) {
+ if(unlikely(!nfsd_proc3_values[i].rd))
+ nfsd_proc3_values[i].rd = rrddim_add(st, nfsd_proc3_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st, nfsd_proc3_values[i].rd, nfsd_proc3_values[i].value);
+ }
+
+ rrdset_done(st);
+ }
+
+ if(do_proc4 == 2) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "proc4"
+ , NULL
+ , "nfsv4rpc"
+ , NULL
+ , "NFS v4 Server Remote Procedure Calls"
+ , "calls/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_PROC4
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ size_t i;
+ for(i = 0; nfsd_proc4_values[i].present ; i++) {
+ if(unlikely(!nfsd_proc4_values[i].rd))
+ nfsd_proc4_values[i].rd = rrddim_add(st, nfsd_proc4_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st, nfsd_proc4_values[i].rd, nfsd_proc4_values[i].value);
+ }
+
+ rrdset_done(st);
+ }
+
+ if(do_proc4ops == 2) {
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "nfsd"
+ , "proc4ops"
+ , NULL
+ , "nfsv4ops"
+ , NULL
+ , "NFS v4 Server Operations"
+ , "operations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NFSD_NAME
+ , NETDATA_CHART_PRIO_NFSD_PROC4OPS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ size_t i;
+ for(i = 0; nfsd4_ops_values[i].present ; i++) {
+ if(unlikely(!nfsd4_ops_values[i].rd))
+ nfsd4_ops_values[i].rd = rrddim_add(st, nfsd4_ops_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(st, nfsd4_ops_values[i].rd, nfsd4_ops_values[i].value);
+ }
+
+ rrdset_done(st);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_sctp_snmp.c b/collectors/proc.plugin/proc_net_sctp_snmp.c
new file mode 100644
index 0000000..292449a
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_sctp_snmp.c
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+#define PLUGIN_PROC_MODULE_NET_SCTP_SNMP_NAME "/proc/net/sctp/snmp"
+
+int do_proc_net_sctp_snmp(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+
+ static int
+ do_associations = -1,
+ do_transitions = -1,
+ do_packet_errors = -1,
+ do_packets = -1,
+ do_fragmentation = -1,
+ do_chunk_types = -1;
+
+ static ARL_BASE *arl_base = NULL;
+
+ static unsigned long long SctpCurrEstab = 0ULL;
+ static unsigned long long SctpActiveEstabs = 0ULL;
+ static unsigned long long SctpPassiveEstabs = 0ULL;
+ static unsigned long long SctpAborteds = 0ULL;
+ static unsigned long long SctpShutdowns = 0ULL;
+ static unsigned long long SctpOutOfBlues = 0ULL;
+ static unsigned long long SctpChecksumErrors = 0ULL;
+ static unsigned long long SctpOutCtrlChunks = 0ULL;
+ static unsigned long long SctpOutOrderChunks = 0ULL;
+ static unsigned long long SctpOutUnorderChunks = 0ULL;
+ static unsigned long long SctpInCtrlChunks = 0ULL;
+ static unsigned long long SctpInOrderChunks = 0ULL;
+ static unsigned long long SctpInUnorderChunks = 0ULL;
+ static unsigned long long SctpFragUsrMsgs = 0ULL;
+ static unsigned long long SctpReasmUsrMsgs = 0ULL;
+ static unsigned long long SctpOutSCTPPacks = 0ULL;
+ static unsigned long long SctpInSCTPPacks = 0ULL;
+ static unsigned long long SctpT1InitExpireds = 0ULL;
+ static unsigned long long SctpT1CookieExpireds = 0ULL;
+ static unsigned long long SctpT2ShutdownExpireds = 0ULL;
+ static unsigned long long SctpT3RtxExpireds = 0ULL;
+ static unsigned long long SctpT4RtoExpireds = 0ULL;
+ static unsigned long long SctpT5ShutdownGuardExpireds = 0ULL;
+ static unsigned long long SctpDelaySackExpireds = 0ULL;
+ static unsigned long long SctpAutocloseExpireds = 0ULL;
+ static unsigned long long SctpT3Retransmits = 0ULL;
+ static unsigned long long SctpPmtudRetransmits = 0ULL;
+ static unsigned long long SctpFastRetransmits = 0ULL;
+ static unsigned long long SctpInPktSoftirq = 0ULL;
+ static unsigned long long SctpInPktBacklog = 0ULL;
+ static unsigned long long SctpInPktDiscards = 0ULL;
+ static unsigned long long SctpInDataChunkDiscards = 0ULL;
+
+ if(unlikely(!arl_base)) {
+ do_associations = config_get_boolean_ondemand("plugin:proc:/proc/net/sctp/snmp", "established associations", CONFIG_BOOLEAN_AUTO);
+ do_transitions = config_get_boolean_ondemand("plugin:proc:/proc/net/sctp/snmp", "association transitions", CONFIG_BOOLEAN_AUTO);
+ do_fragmentation = config_get_boolean_ondemand("plugin:proc:/proc/net/sctp/snmp", "fragmentation", CONFIG_BOOLEAN_AUTO);
+ do_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/sctp/snmp", "packets", CONFIG_BOOLEAN_AUTO);
+ do_packet_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/sctp/snmp", "packet errors", CONFIG_BOOLEAN_AUTO);
+ do_chunk_types = config_get_boolean_ondemand("plugin:proc:/proc/net/sctp/snmp", "chunk types", CONFIG_BOOLEAN_AUTO);
+
+ arl_base = arl_create("sctp", NULL, 60);
+ arl_expect(arl_base, "SctpCurrEstab", &SctpCurrEstab);
+ arl_expect(arl_base, "SctpActiveEstabs", &SctpActiveEstabs);
+ arl_expect(arl_base, "SctpPassiveEstabs", &SctpPassiveEstabs);
+ arl_expect(arl_base, "SctpAborteds", &SctpAborteds);
+ arl_expect(arl_base, "SctpShutdowns", &SctpShutdowns);
+ arl_expect(arl_base, "SctpOutOfBlues", &SctpOutOfBlues);
+ arl_expect(arl_base, "SctpChecksumErrors", &SctpChecksumErrors);
+ arl_expect(arl_base, "SctpOutCtrlChunks", &SctpOutCtrlChunks);
+ arl_expect(arl_base, "SctpOutOrderChunks", &SctpOutOrderChunks);
+ arl_expect(arl_base, "SctpOutUnorderChunks", &SctpOutUnorderChunks);
+ arl_expect(arl_base, "SctpInCtrlChunks", &SctpInCtrlChunks);
+ arl_expect(arl_base, "SctpInOrderChunks", &SctpInOrderChunks);
+ arl_expect(arl_base, "SctpInUnorderChunks", &SctpInUnorderChunks);
+ arl_expect(arl_base, "SctpFragUsrMsgs", &SctpFragUsrMsgs);
+ arl_expect(arl_base, "SctpReasmUsrMsgs", &SctpReasmUsrMsgs);
+ arl_expect(arl_base, "SctpOutSCTPPacks", &SctpOutSCTPPacks);
+ arl_expect(arl_base, "SctpInSCTPPacks", &SctpInSCTPPacks);
+ arl_expect(arl_base, "SctpT1InitExpireds", &SctpT1InitExpireds);
+ arl_expect(arl_base, "SctpT1CookieExpireds", &SctpT1CookieExpireds);
+ arl_expect(arl_base, "SctpT2ShutdownExpireds", &SctpT2ShutdownExpireds);
+ arl_expect(arl_base, "SctpT3RtxExpireds", &SctpT3RtxExpireds);
+ arl_expect(arl_base, "SctpT4RtoExpireds", &SctpT4RtoExpireds);
+ arl_expect(arl_base, "SctpT5ShutdownGuardExpireds", &SctpT5ShutdownGuardExpireds);
+ arl_expect(arl_base, "SctpDelaySackExpireds", &SctpDelaySackExpireds);
+ arl_expect(arl_base, "SctpAutocloseExpireds", &SctpAutocloseExpireds);
+ arl_expect(arl_base, "SctpT3Retransmits", &SctpT3Retransmits);
+ arl_expect(arl_base, "SctpPmtudRetransmits", &SctpPmtudRetransmits);
+ arl_expect(arl_base, "SctpFastRetransmits", &SctpFastRetransmits);
+ arl_expect(arl_base, "SctpInPktSoftirq", &SctpInPktSoftirq);
+ arl_expect(arl_base, "SctpInPktBacklog", &SctpInPktBacklog);
+ arl_expect(arl_base, "SctpInPktDiscards", &SctpInPktDiscards);
+ arl_expect(arl_base, "SctpInDataChunkDiscards", &SctpInDataChunkDiscards);
+ }
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/sctp/snmp");
+ ff = procfile_open(config_get("plugin:proc:/proc/net/sctp/snmp", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff))
+ return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ arl_begin(arl_base);
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 2)) {
+ if(unlikely(words)) error("Cannot read /proc/net/sctp/snmp line %zu. Expected 2 params, read %zu.", l, words);
+ continue;
+ }
+
+ if(unlikely(arl_check(arl_base,
+ procfile_lineword(ff, l, 0),
+ procfile_lineword(ff, l, 1)))) break;
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_associations == CONFIG_BOOLEAN_YES || (do_associations == CONFIG_BOOLEAN_AUTO &&
+ (SctpCurrEstab || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_associations = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_established = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "sctp"
+ , "established"
+ , NULL
+ , "associations"
+ , NULL
+ , "SCTP current total number of established associations"
+ , "associations"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SCTP_SNMP_NAME
+ , NETDATA_CHART_PRIO_SCTP
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_established = rrddim_add(st, "SctpCurrEstab", "established", 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_established, SctpCurrEstab);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_transitions == CONFIG_BOOLEAN_YES || (do_transitions == CONFIG_BOOLEAN_AUTO &&
+ (SctpActiveEstabs ||
+ SctpPassiveEstabs ||
+ SctpAborteds ||
+ SctpShutdowns ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_transitions = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_active = NULL,
+ *rd_passive = NULL,
+ *rd_aborted = NULL,
+ *rd_shutdown = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "sctp"
+ , "transitions"
+ , NULL
+ , "transitions"
+ , NULL
+ , "SCTP Association Transitions"
+ , "transitions/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SCTP_SNMP_NAME
+ , NETDATA_CHART_PRIO_SCTP + 10
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_active = rrddim_add(st, "SctpActiveEstabs", "active", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_passive = rrddim_add(st, "SctpPassiveEstabs", "passive", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_aborted = rrddim_add(st, "SctpAborteds", "aborted", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_shutdown = rrddim_add(st, "SctpShutdowns", "shutdown", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_active, SctpActiveEstabs);
+ rrddim_set_by_pointer(st, rd_passive, SctpPassiveEstabs);
+ rrddim_set_by_pointer(st, rd_aborted, SctpAborteds);
+ rrddim_set_by_pointer(st, rd_shutdown, SctpShutdowns);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_packets == CONFIG_BOOLEAN_YES || (do_packets == CONFIG_BOOLEAN_AUTO &&
+ (SctpInSCTPPacks ||
+ SctpOutSCTPPacks ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_packets = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_received = NULL,
+ *rd_sent = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "sctp"
+ , "packets"
+ , NULL
+ , "packets"
+ , NULL
+ , "SCTP Packets"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SCTP_SNMP_NAME
+ , NETDATA_CHART_PRIO_SCTP + 20
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_received = rrddim_add(st, "SctpInSCTPPacks", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_sent = rrddim_add(st, "SctpOutSCTPPacks", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_received, SctpInSCTPPacks);
+ rrddim_set_by_pointer(st, rd_sent, SctpOutSCTPPacks);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_packet_errors == CONFIG_BOOLEAN_YES || (do_packet_errors == CONFIG_BOOLEAN_AUTO &&
+ (SctpOutOfBlues ||
+ SctpChecksumErrors ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_packet_errors = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_invalid = NULL,
+ *rd_csum = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "sctp"
+ , "packet_errors"
+ , NULL
+ , "packets"
+ , NULL
+ , "SCTP Packet Errors"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SCTP_SNMP_NAME
+ , NETDATA_CHART_PRIO_SCTP + 30
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_invalid = rrddim_add(st, "SctpOutOfBlues", "invalid", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_csum = rrddim_add(st, "SctpChecksumErrors", "checksum", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_invalid, SctpOutOfBlues);
+ rrddim_set_by_pointer(st, rd_csum, SctpChecksumErrors);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_fragmentation == CONFIG_BOOLEAN_YES || (do_fragmentation == CONFIG_BOOLEAN_AUTO &&
+ (SctpFragUsrMsgs ||
+ SctpReasmUsrMsgs ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_fragmentation = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_fragmented = NULL,
+ *rd_reassembled = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "sctp"
+ , "fragmentation"
+ , NULL
+ , "fragmentation"
+ , NULL
+ , "SCTP Fragmentation"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SCTP_SNMP_NAME
+ , NETDATA_CHART_PRIO_SCTP + 40
+ , update_every
+ , RRDSET_TYPE_LINE);
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_reassembled = rrddim_add(st, "SctpReasmUsrMsgs", "reassembled", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_fragmented = rrddim_add(st, "SctpFragUsrMsgs", "fragmented", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_reassembled, SctpReasmUsrMsgs);
+ rrddim_set_by_pointer(st, rd_fragmented, SctpFragUsrMsgs);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_chunk_types == CONFIG_BOOLEAN_YES || (do_chunk_types == CONFIG_BOOLEAN_AUTO &&
+ (SctpInCtrlChunks ||
+ SctpInOrderChunks ||
+ SctpInUnorderChunks ||
+ SctpOutCtrlChunks ||
+ SctpOutOrderChunks ||
+ SctpOutUnorderChunks ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_chunk_types = CONFIG_BOOLEAN_YES;
+ static RRDSET *st = NULL;
+ static RRDDIM
+ *rd_InCtrl = NULL,
+ *rd_InOrder = NULL,
+ *rd_InUnorder = NULL,
+ *rd_OutCtrl = NULL,
+ *rd_OutOrder = NULL,
+ *rd_OutUnorder = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "sctp"
+ , "chunks"
+ , NULL
+ , "chunks"
+ , NULL
+ , "SCTP Chunk Types"
+ , "chunks/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SCTP_SNMP_NAME
+ , NETDATA_CHART_PRIO_SCTP + 50
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_InCtrl = rrddim_add(st, "SctpInCtrlChunks", "InCtrl", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InOrder = rrddim_add(st, "SctpInOrderChunks", "InOrder", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_InUnorder = rrddim_add(st, "SctpInUnorderChunks", "InUnorder", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutCtrl = rrddim_add(st, "SctpOutCtrlChunks", "OutCtrl", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutOrder = rrddim_add(st, "SctpOutOrderChunks", "OutOrder", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_OutUnorder = rrddim_add(st, "SctpOutUnorderChunks", "OutUnorder", -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_InCtrl, SctpInCtrlChunks);
+ rrddim_set_by_pointer(st, rd_InOrder, SctpInOrderChunks);
+ rrddim_set_by_pointer(st, rd_InUnorder, SctpInUnorderChunks);
+ rrddim_set_by_pointer(st, rd_OutCtrl, SctpOutCtrlChunks);
+ rrddim_set_by_pointer(st, rd_OutOrder, SctpOutOrderChunks);
+ rrddim_set_by_pointer(st, rd_OutUnorder, SctpOutUnorderChunks);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
diff --git a/collectors/proc.plugin/proc_net_sockstat.c b/collectors/proc.plugin/proc_net_sockstat.c
new file mode 100644
index 0000000..e94b891
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_sockstat.c
@@ -0,0 +1,529 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME "/proc/net/sockstat"
+
+static struct proc_net_sockstat {
+ kernel_uint_t sockets_used;
+
+ kernel_uint_t tcp_inuse;
+ kernel_uint_t tcp_orphan;
+ kernel_uint_t tcp_tw;
+ kernel_uint_t tcp_alloc;
+ kernel_uint_t tcp_mem;
+
+ kernel_uint_t udp_inuse;
+ kernel_uint_t udp_mem;
+
+ kernel_uint_t udplite_inuse;
+
+ kernel_uint_t raw_inuse;
+
+ kernel_uint_t frag_inuse;
+ kernel_uint_t frag_memory;
+} sockstat_root = { 0 };
+
+
+static int read_tcp_mem(void) {
+ static char *filename = NULL;
+ static const RRDVAR_ACQUIRED *tcp_mem_low_threshold = NULL,
+ *tcp_mem_pressure_threshold = NULL,
+ *tcp_mem_high_threshold = NULL;
+
+ if(unlikely(!tcp_mem_low_threshold)) {
+ tcp_mem_low_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_low");
+ tcp_mem_pressure_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_pressure");
+ tcp_mem_high_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_high");
+ }
+
+ if(unlikely(!filename)) {
+ char buffer[FILENAME_MAX + 1];
+ snprintfz(buffer, FILENAME_MAX, "%s/proc/sys/net/ipv4/tcp_mem", netdata_configured_host_prefix);
+ filename = strdupz(buffer);
+ }
+
+ char buffer[200 + 1], *start, *end;
+ if(read_file(filename, buffer, 200) != 0) return 1;
+ buffer[200] = '\0';
+
+ unsigned long long low = 0, pressure = 0, high = 0;
+
+ start = buffer;
+ low = strtoull(start, &end, 10);
+
+ start = end;
+ pressure = strtoull(start, &end, 10);
+
+ start = end;
+ high = strtoull(start, &end, 10);
+
+ // fprintf(stderr, "TCP MEM low = %llu, pressure = %llu, high = %llu\n", low, pressure, high);
+
+ rrdvar_custom_host_variable_set(localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE) / 1024.0);
+ rrdvar_custom_host_variable_set(localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE) / 1024.0);
+ rrdvar_custom_host_variable_set(localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE) / 1024.0);
+
+ return 0;
+}
+
+static kernel_uint_t read_tcp_max_orphans(void) {
+ static char *filename = NULL;
+ static const RRDVAR_ACQUIRED *tcp_max_orphans_var = NULL;
+
+ if(unlikely(!filename)) {
+ char buffer[FILENAME_MAX + 1];
+ snprintfz(buffer, FILENAME_MAX, "%s/proc/sys/net/ipv4/tcp_max_orphans", netdata_configured_host_prefix);
+ filename = strdupz(buffer);
+ }
+
+ unsigned long long tcp_max_orphans = 0;
+ if(read_single_number_file(filename, &tcp_max_orphans) == 0) {
+
+ if(unlikely(!tcp_max_orphans_var))
+ tcp_max_orphans_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_orphans");
+
+ rrdvar_custom_host_variable_set(localhost, tcp_max_orphans_var, tcp_max_orphans);
+ return tcp_max_orphans;
+ }
+
+ return 0;
+}
+
+int do_proc_net_sockstat(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+
+ static uint32_t hash_sockets = 0,
+ hash_raw = 0,
+ hash_frag = 0,
+ hash_tcp = 0,
+ hash_udp = 0,
+ hash_udplite = 0;
+
+ static long long update_constants_every = 60, update_constants_count = 0;
+
+ static ARL_BASE *arl_sockets = NULL;
+ static ARL_BASE *arl_tcp = NULL;
+ static ARL_BASE *arl_udp = NULL;
+ static ARL_BASE *arl_udplite = NULL;
+ static ARL_BASE *arl_raw = NULL;
+ static ARL_BASE *arl_frag = NULL;
+
+ static int do_sockets = -1, do_tcp_sockets = -1, do_tcp_mem = -1, do_udp_sockets = -1, do_udp_mem = -1, do_udplite_sockets = -1, do_raw_sockets = -1, do_frag_sockets = -1, do_frag_mem = -1;
+
+ static char *keys[7] = { NULL };
+ static uint32_t hashes[7] = { 0 };
+ static ARL_BASE *bases[7] = { NULL };
+
+ if(unlikely(!arl_sockets)) {
+ do_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 sockets", CONFIG_BOOLEAN_AUTO);
+ do_tcp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 TCP sockets", CONFIG_BOOLEAN_AUTO);
+ do_tcp_mem = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 TCP memory", CONFIG_BOOLEAN_AUTO);
+ do_udp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 UDP sockets", CONFIG_BOOLEAN_AUTO);
+ do_udp_mem = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 UDP memory", CONFIG_BOOLEAN_AUTO);
+ do_udplite_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 UDPLITE sockets", CONFIG_BOOLEAN_AUTO);
+ do_raw_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 RAW sockets", CONFIG_BOOLEAN_AUTO);
+ do_frag_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 FRAG sockets", CONFIG_BOOLEAN_AUTO);
+ do_frag_mem = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 FRAG memory", CONFIG_BOOLEAN_AUTO);
+
+ update_constants_every = config_get_number("plugin:proc:/proc/net/sockstat", "update constants every", update_constants_every);
+ update_constants_count = update_constants_every;
+
+ arl_sockets = arl_create("sockstat/sockets", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_sockets, "used", &sockstat_root.sockets_used);
+
+ arl_tcp = arl_create("sockstat/TCP", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_tcp, "inuse", &sockstat_root.tcp_inuse);
+ arl_expect(arl_tcp, "orphan", &sockstat_root.tcp_orphan);
+ arl_expect(arl_tcp, "tw", &sockstat_root.tcp_tw);
+ arl_expect(arl_tcp, "alloc", &sockstat_root.tcp_alloc);
+ arl_expect(arl_tcp, "mem", &sockstat_root.tcp_mem);
+
+ arl_udp = arl_create("sockstat/UDP", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_udp, "inuse", &sockstat_root.udp_inuse);
+ arl_expect(arl_udp, "mem", &sockstat_root.udp_mem);
+
+ arl_udplite = arl_create("sockstat/UDPLITE", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_udplite, "inuse", &sockstat_root.udplite_inuse);
+
+ arl_raw = arl_create("sockstat/RAW", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_raw, "inuse", &sockstat_root.raw_inuse);
+
+ arl_frag = arl_create("sockstat/FRAG", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_frag, "inuse", &sockstat_root.frag_inuse);
+ arl_expect(arl_frag, "memory", &sockstat_root.frag_memory);
+
+ hash_sockets = simple_hash("sockets");
+ hash_tcp = simple_hash("TCP");
+ hash_udp = simple_hash("UDP");
+ hash_udplite = simple_hash("UDPLITE");
+ hash_raw = simple_hash("RAW");
+ hash_frag = simple_hash("FRAG");
+
+ keys[0] = "sockets"; hashes[0] = hash_sockets; bases[0] = arl_sockets;
+ keys[1] = "TCP"; hashes[1] = hash_tcp; bases[1] = arl_tcp;
+ keys[2] = "UDP"; hashes[2] = hash_udp; bases[2] = arl_udp;
+ keys[3] = "UDPLITE"; hashes[3] = hash_udplite; bases[3] = arl_udplite;
+ keys[4] = "RAW"; hashes[4] = hash_raw; bases[4] = arl_raw;
+ keys[5] = "FRAG"; hashes[5] = hash_frag; bases[5] = arl_frag;
+ keys[6] = NULL; // terminator
+ }
+
+ update_constants_count += update_every;
+ if(unlikely(update_constants_count > update_constants_every)) {
+ read_tcp_max_orphans();
+ read_tcp_mem();
+ update_constants_count = 0;
+ }
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/sockstat");
+ ff = procfile_open(config_get("plugin:proc:/proc/net/sockstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ char *key = procfile_lineword(ff, l, 0);
+ uint32_t hash = simple_hash(key);
+
+ int k;
+ for(k = 0; keys[k] ; k++) {
+ if(unlikely(hash == hashes[k] && strcmp(key, keys[k]) == 0)) {
+ // fprintf(stderr, "KEY: '%s', l=%zu, w=1, words=%zu\n", key, l, words);
+ ARL_BASE *arl = bases[k];
+ arl_begin(arl);
+ size_t w = 1;
+
+ while(w + 1 < words) {
+ char *name = procfile_lineword(ff, l, w); w++;
+ char *value = procfile_lineword(ff, l, w); w++;
+ // fprintf(stderr, " > NAME '%s', VALUE '%s', l=%zu, w=%zu, words=%zu\n", name, value, l, w, words);
+ if(unlikely(arl_check(arl, name, value) != 0))
+ break;
+ }
+
+ break;
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_sockets == CONFIG_BOOLEAN_YES || (do_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.sockets_used ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_used = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_sockets"
+ , NULL
+ , "sockets"
+ , NULL
+ , "IPv4 Sockets Used"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_SOCKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_used = rrddim_add(st, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_used, (collected_number)sockstat_root.sockets_used);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_tcp_sockets == CONFIG_BOOLEAN_YES || (do_tcp_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.tcp_inuse ||
+ sockstat_root.tcp_orphan ||
+ sockstat_root.tcp_tw ||
+ sockstat_root.tcp_alloc ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcp_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL,
+ *rd_orphan = NULL,
+ *rd_timewait = NULL,
+ *rd_alloc = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_tcp_sockets"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Sockets"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_TCP_SOCKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_alloc = rrddim_add(st, "alloc", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_orphan = rrddim_add(st, "orphan", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_timewait = rrddim_add(st, "timewait", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.tcp_inuse);
+ rrddim_set_by_pointer(st, rd_orphan, (collected_number)sockstat_root.tcp_orphan);
+ rrddim_set_by_pointer(st, rd_timewait, (collected_number)sockstat_root.tcp_tw);
+ rrddim_set_by_pointer(st, rd_alloc, (collected_number)sockstat_root.tcp_alloc);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_tcp_mem == CONFIG_BOOLEAN_YES || (do_tcp_mem == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.tcp_mem || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcp_mem = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_mem = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_tcp_mem"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Sockets Memory"
+ , "KiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_TCP_MEM
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_mem = rrddim_add(st, "mem", NULL, sysconf(_SC_PAGESIZE), 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_mem, (collected_number)sockstat_root.tcp_mem);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_udp_sockets == CONFIG_BOOLEAN_YES || (do_udp_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.udp_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udp_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_udp_sockets"
+ , NULL
+ , "udp"
+ , NULL
+ , "IPv4 UDP Sockets"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_UDP
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.udp_inuse);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_udp_mem == CONFIG_BOOLEAN_YES || (do_udp_mem == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.udp_mem ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udp_mem = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_mem = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_udp_mem"
+ , NULL
+ , "udp"
+ , NULL
+ , "IPv4 UDP Sockets Memory"
+ , "KiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_UDP_MEM
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_mem = rrddim_add(st, "mem", NULL, sysconf(_SC_PAGESIZE), 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_mem, (collected_number)sockstat_root.udp_mem);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_udplite_sockets == CONFIG_BOOLEAN_YES || (do_udplite_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.udplite_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udplite_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_udplite_sockets"
+ , NULL
+ , "udplite"
+ , NULL
+ , "IPv4 UDPLITE Sockets"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_UDPLITE
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.udplite_inuse);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_raw_sockets == CONFIG_BOOLEAN_YES || (do_raw_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.raw_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_raw_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_raw_sockets"
+ , NULL
+ , "raw"
+ , NULL
+ , "IPv4 RAW Sockets"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_RAW
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.raw_inuse);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_frag_sockets == CONFIG_BOOLEAN_YES || (do_frag_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.frag_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_frag_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_frag_sockets"
+ , NULL
+ , "fragments"
+ , NULL
+ , "IPv4 FRAG Sockets"
+ , "fragments"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_FRAGMENTS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.frag_inuse);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_frag_mem == CONFIG_BOOLEAN_YES || (do_frag_mem == CONFIG_BOOLEAN_AUTO &&
+ (sockstat_root.frag_memory ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_frag_mem = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_mem = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv4"
+ , "sockstat_frag_mem"
+ , NULL
+ , "fragments"
+ , NULL
+ , "IPv4 FRAG Sockets Memory"
+ , "KiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT_NAME
+ , NETDATA_CHART_PRIO_IPV4_FRAGMENTS_MEM
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_mem = rrddim_add(st, "mem", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_mem, (collected_number)sockstat_root.frag_memory);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
+
diff --git a/collectors/proc.plugin/proc_net_sockstat6.c b/collectors/proc.plugin/proc_net_sockstat6.c
new file mode 100644
index 0000000..065cf60
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_sockstat6.c
@@ -0,0 +1,278 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_NET_SOCKSTAT6_NAME "/proc/net/sockstat6"
+
+static struct proc_net_sockstat6 {
+ kernel_uint_t tcp6_inuse;
+ kernel_uint_t udp6_inuse;
+ kernel_uint_t udplite6_inuse;
+ kernel_uint_t raw6_inuse;
+ kernel_uint_t frag6_inuse;
+} sockstat6_root = { 0 };
+
+int do_proc_net_sockstat6(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+
+ static uint32_t hash_raw = 0,
+ hash_frag = 0,
+ hash_tcp = 0,
+ hash_udp = 0,
+ hash_udplite = 0;
+
+ static ARL_BASE *arl_tcp = NULL;
+ static ARL_BASE *arl_udp = NULL;
+ static ARL_BASE *arl_udplite = NULL;
+ static ARL_BASE *arl_raw = NULL;
+ static ARL_BASE *arl_frag = NULL;
+
+ static int do_tcp_sockets = -1, do_udp_sockets = -1, do_udplite_sockets = -1, do_raw_sockets = -1, do_frag_sockets = -1;
+
+ static char *keys[6] = { NULL };
+ static uint32_t hashes[6] = { 0 };
+ static ARL_BASE *bases[6] = { NULL };
+
+ if(unlikely(!arl_tcp)) {
+ do_tcp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 TCP sockets", CONFIG_BOOLEAN_AUTO);
+ do_udp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 UDP sockets", CONFIG_BOOLEAN_AUTO);
+ do_udplite_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 UDPLITE sockets", CONFIG_BOOLEAN_AUTO);
+ do_raw_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 RAW sockets", CONFIG_BOOLEAN_AUTO);
+ do_frag_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 FRAG sockets", CONFIG_BOOLEAN_AUTO);
+
+ arl_tcp = arl_create("sockstat6/TCP6", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_tcp, "inuse", &sockstat6_root.tcp6_inuse);
+
+ arl_udp = arl_create("sockstat6/UDP6", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_udp, "inuse", &sockstat6_root.udp6_inuse);
+
+ arl_udplite = arl_create("sockstat6/UDPLITE6", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_udplite, "inuse", &sockstat6_root.udplite6_inuse);
+
+ arl_raw = arl_create("sockstat6/RAW6", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_raw, "inuse", &sockstat6_root.raw6_inuse);
+
+ arl_frag = arl_create("sockstat6/FRAG6", arl_callback_str2kernel_uint_t, 60);
+ arl_expect(arl_frag, "inuse", &sockstat6_root.frag6_inuse);
+
+ hash_tcp = simple_hash("TCP6");
+ hash_udp = simple_hash("UDP6");
+ hash_udplite = simple_hash("UDPLITE6");
+ hash_raw = simple_hash("RAW6");
+ hash_frag = simple_hash("FRAG6");
+
+ keys[0] = "TCP6"; hashes[0] = hash_tcp; bases[0] = arl_tcp;
+ keys[1] = "UDP6"; hashes[1] = hash_udp; bases[1] = arl_udp;
+ keys[2] = "UDPLITE6"; hashes[2] = hash_udplite; bases[2] = arl_udplite;
+ keys[3] = "RAW6"; hashes[3] = hash_raw; bases[3] = arl_raw;
+ keys[4] = "FRAG6"; hashes[4] = hash_frag; bases[4] = arl_frag;
+ keys[5] = NULL; // terminator
+ }
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/sockstat6");
+ ff = procfile_open(config_get("plugin:proc:/proc/net/sockstat6", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ char *key = procfile_lineword(ff, l, 0);
+ uint32_t hash = simple_hash(key);
+
+ int k;
+ for(k = 0; keys[k] ; k++) {
+ if(unlikely(hash == hashes[k] && strcmp(key, keys[k]) == 0)) {
+ // fprintf(stderr, "KEY: '%s', l=%zu, w=1, words=%zu\n", key, l, words);
+ ARL_BASE *arl = bases[k];
+ arl_begin(arl);
+ size_t w = 1;
+
+ while(w + 1 < words) {
+ char *name = procfile_lineword(ff, l, w); w++;
+ char *value = procfile_lineword(ff, l, w); w++;
+ // fprintf(stderr, " > NAME '%s', VALUE '%s', l=%zu, w=%zu, words=%zu\n", name, value, l, w, words);
+ if(unlikely(arl_check(arl, name, value) != 0))
+ break;
+ }
+
+ break;
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_tcp_sockets == CONFIG_BOOLEAN_YES || (do_tcp_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat6_root.tcp6_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_tcp_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "sockstat6_tcp_sockets"
+ , NULL
+ , "tcp6"
+ , NULL
+ , "IPv6 TCP Sockets"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT6_NAME
+ , NETDATA_CHART_PRIO_IPV6_TCP
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.tcp6_inuse);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_udp_sockets == CONFIG_BOOLEAN_YES || (do_udp_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat6_root.udp6_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udp_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "sockstat6_udp_sockets"
+ , NULL
+ , "udp6"
+ , NULL
+ , "IPv6 UDP Sockets"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT6_NAME
+ , NETDATA_CHART_PRIO_IPV6_UDP
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.udp6_inuse);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_udplite_sockets == CONFIG_BOOLEAN_YES || (do_udplite_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat6_root.udplite6_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_udplite_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "sockstat6_udplite_sockets"
+ , NULL
+ , "udplite6"
+ , NULL
+ , "IPv6 UDPLITE Sockets"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT6_NAME
+ , NETDATA_CHART_PRIO_IPV6_UDPLITE
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.udplite6_inuse);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_raw_sockets == CONFIG_BOOLEAN_YES || (do_raw_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat6_root.raw6_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_raw_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "sockstat6_raw_sockets"
+ , NULL
+ , "raw6"
+ , NULL
+ , "IPv6 RAW Sockets"
+ , "sockets"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT6_NAME
+ , NETDATA_CHART_PRIO_IPV6_RAW
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.raw6_inuse);
+ rrdset_done(st);
+ }
+
+ // ------------------------------------------------------------------------
+
+ if(do_frag_sockets == CONFIG_BOOLEAN_YES || (do_frag_sockets == CONFIG_BOOLEAN_AUTO &&
+ (sockstat6_root.frag6_inuse ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_frag_sockets = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_inuse = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "ipv6"
+ , "sockstat6_frag_sockets"
+ , NULL
+ , "fragments6"
+ , NULL
+ , "IPv6 FRAG Sockets"
+ , "fragments"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOCKSTAT6_NAME
+ , NETDATA_CHART_PRIO_IPV6_FRAGMENTS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.frag6_inuse);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_softnet_stat.c b/collectors/proc.plugin/proc_net_softnet_stat.c
new file mode 100644
index 0000000..6523924
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_softnet_stat.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_NET_SOFTNET_NAME "/proc/net/softnet_stat"
+
+static inline char *softnet_column_name(size_t column) {
+ switch(column) {
+ // https://github.com/torvalds/linux/blob/a7fd20d1c476af4563e66865213474a2f9f473a4/net/core/net-procfs.c#L161-L166
+ case 0: return "processed";
+ case 1: return "dropped";
+ case 2: return "squeezed";
+ case 9: return "received_rps";
+ case 10: return "flow_limit_count";
+ default: return NULL;
+ }
+}
+
+int do_proc_net_softnet_stat(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+ static int do_per_core = -1;
+ static size_t allocated_lines = 0, allocated_columns = 0;
+ static uint32_t *data = NULL;
+
+ if(unlikely(do_per_core == -1)) do_per_core = config_get_boolean("plugin:proc:/proc/net/softnet_stat", "softnet_stat per core", 1);
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/softnet_stat");
+ ff = procfile_open(config_get("plugin:proc:/proc/net/softnet_stat", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+ size_t words = procfile_linewords(ff, 0), w;
+
+ if(unlikely(!lines || !words)) {
+ error("Cannot read /proc/net/softnet_stat, %zu lines and %zu columns reported.", lines, words);
+ return 1;
+ }
+
+ if(unlikely(lines > 200)) lines = 200;
+ if(unlikely(words > 50)) words = 50;
+
+ if(unlikely(!data || lines > allocated_lines || words > allocated_columns)) {
+ freez(data);
+ allocated_lines = lines;
+ allocated_columns = words;
+ data = mallocz((allocated_lines + 1) * allocated_columns * sizeof(uint32_t));
+ }
+
+ // initialize to zero
+ memset(data, 0, (allocated_lines + 1) * allocated_columns * sizeof(uint32_t));
+
+ // parse the values
+ for(l = 0; l < lines ;l++) {
+ words = procfile_linewords(ff, l);
+ if(unlikely(!words)) continue;
+
+ if(unlikely(words > allocated_columns))
+ words = allocated_columns;
+
+ for(w = 0; w < words ; w++) {
+ if(unlikely(softnet_column_name(w))) {
+ uint32_t t = (uint32_t)strtoul(procfile_lineword(ff, l, w), NULL, 16);
+ data[w] += t;
+ data[((l + 1) * allocated_columns) + w] = t;
+ }
+ }
+ }
+
+ if(unlikely(data[(lines * allocated_columns)] == 0))
+ lines--;
+
+ RRDSET *st;
+
+ // --------------------------------------------------------------------
+
+ st = rrdset_find_active_bytype_localhost("system", "softnet_stat");
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "softnet_stat"
+ , NULL
+ , "softnet_stat"
+ , "system.softnet_stat"
+ , "System softnet_stat"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOFTNET_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_SOFTNET_STAT
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ for(w = 0; w < allocated_columns ;w++)
+ if(unlikely(softnet_column_name(w)))
+ rrddim_add(st, softnet_column_name(w), NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ for(w = 0; w < allocated_columns ;w++)
+ if(unlikely(softnet_column_name(w)))
+ rrddim_set(st, softnet_column_name(w), data[w]);
+
+ rrdset_done(st);
+
+ if(do_per_core) {
+ for(l = 0; l < lines ;l++) {
+ char id[50+1];
+ snprintfz(id, 50, "cpu%zu_softnet_stat", l);
+
+ st = rrdset_find_active_bytype_localhost("cpu", id);
+ if(unlikely(!st)) {
+ char title[100+1];
+ snprintfz(title, 100, "CPU softnet_stat");
+
+ st = rrdset_create_localhost(
+ "cpu"
+ , id
+ , NULL
+ , "softnet_stat"
+ , "cpu.softnet_stat"
+ , title
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_NET_SOFTNET_NAME
+ , NETDATA_CHART_PRIO_SOFTNET_PER_CORE + l
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ for(w = 0; w < allocated_columns ;w++)
+ if(unlikely(softnet_column_name(w)))
+ rrddim_add(st, softnet_column_name(w), NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ for(w = 0; w < allocated_columns ;w++)
+ if(unlikely(softnet_column_name(w)))
+ rrddim_set(st, softnet_column_name(w), data[((l + 1) * allocated_columns) + w]);
+
+ rrdset_done(st);
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_stat_conntrack.c b/collectors/proc.plugin/proc_net_stat_conntrack.c
new file mode 100644
index 0000000..f9dbdf4
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_stat_conntrack.c
@@ -0,0 +1,345 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define RRD_TYPE_NET_STAT_NETFILTER "netfilter"
+#define RRD_TYPE_NET_STAT_CONNTRACK "conntrack"
+#define PLUGIN_PROC_MODULE_CONNTRACK_NAME "/proc/net/stat/nf_conntrack"
+
+int do_proc_net_stat_conntrack(int update_every, usec_t dt) {
+ static procfile *ff = NULL;
+ static int do_sockets = -1, do_new = -1, do_changes = -1, do_expect = -1, do_search = -1, do_errors = -1;
+ static usec_t get_max_every = 10 * USEC_PER_SEC, usec_since_last_max = 0;
+ static int read_full = 1;
+ static char *nf_conntrack_filename, *nf_conntrack_count_filename, *nf_conntrack_max_filename;
+ static const RRDVAR_ACQUIRED *rrdvar_max = NULL;
+
+ unsigned long long aentries = 0, asearched = 0, afound = 0, anew = 0, ainvalid = 0, aignore = 0, adelete = 0, adelete_list = 0,
+ ainsert = 0, ainsert_failed = 0, adrop = 0, aearly_drop = 0, aicmp_error = 0, aexpect_new = 0, aexpect_create = 0, aexpect_delete = 0, asearch_restart = 0;
+
+ if(unlikely(do_sockets == -1)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/stat/nf_conntrack");
+ nf_conntrack_filename = config_get("plugin:proc:/proc/net/stat/nf_conntrack", "filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/sys/net/netfilter/nf_conntrack_max");
+ nf_conntrack_max_filename = config_get("plugin:proc:/proc/sys/net/netfilter/nf_conntrack_max", "filename to monitor", filename);
+ usec_since_last_max = get_max_every = config_get_number("plugin:proc:/proc/sys/net/netfilter/nf_conntrack_max", "read every seconds", 10) * USEC_PER_SEC;
+
+ read_full = 1;
+ ff = procfile_open(nf_conntrack_filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if(!ff) read_full = 0;
+
+ do_new = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter new connections", read_full);
+ do_changes = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection changes", read_full);
+ do_expect = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection expectations", read_full);
+ do_search = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection searches", read_full);
+ do_errors = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter errors", read_full);
+
+ do_sockets = 1;
+ if(!read_full) {
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/sys/net/netfilter/nf_conntrack_count");
+ nf_conntrack_count_filename = config_get("plugin:proc:/proc/sys/net/netfilter/nf_conntrack_count", "filename to monitor", filename);
+
+ if(read_single_number_file(nf_conntrack_count_filename, &aentries))
+ do_sockets = 0;
+ }
+
+ do_sockets = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connections", do_sockets);
+
+ if(!do_sockets && !read_full)
+ return 1;
+
+ rrdvar_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "netfilter_conntrack_max");
+ }
+
+ if(likely(read_full)) {
+ if(unlikely(!ff)) {
+ ff = procfile_open(nf_conntrack_filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ for(l = 1; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 17)) {
+ if(unlikely(words)) error("Cannot read /proc/net/stat/nf_conntrack line. Expected 17 params, read %zu.", words);
+ continue;
+ }
+
+ unsigned long long tentries = 0, tsearched = 0, tfound = 0, tnew = 0, tinvalid = 0, tignore = 0, tdelete = 0, tdelete_list = 0, tinsert = 0, tinsert_failed = 0, tdrop = 0, tearly_drop = 0, ticmp_error = 0, texpect_new = 0, texpect_create = 0, texpect_delete = 0, tsearch_restart = 0;
+
+ tentries = strtoull(procfile_lineword(ff, l, 0), NULL, 16);
+ tsearched = strtoull(procfile_lineword(ff, l, 1), NULL, 16);
+ tfound = strtoull(procfile_lineword(ff, l, 2), NULL, 16);
+ tnew = strtoull(procfile_lineword(ff, l, 3), NULL, 16);
+ tinvalid = strtoull(procfile_lineword(ff, l, 4), NULL, 16);
+ tignore = strtoull(procfile_lineword(ff, l, 5), NULL, 16);
+ tdelete = strtoull(procfile_lineword(ff, l, 6), NULL, 16);
+ tdelete_list = strtoull(procfile_lineword(ff, l, 7), NULL, 16);
+ tinsert = strtoull(procfile_lineword(ff, l, 8), NULL, 16);
+ tinsert_failed = strtoull(procfile_lineword(ff, l, 9), NULL, 16);
+ tdrop = strtoull(procfile_lineword(ff, l, 10), NULL, 16);
+ tearly_drop = strtoull(procfile_lineword(ff, l, 11), NULL, 16);
+ ticmp_error = strtoull(procfile_lineword(ff, l, 12), NULL, 16);
+ texpect_new = strtoull(procfile_lineword(ff, l, 13), NULL, 16);
+ texpect_create = strtoull(procfile_lineword(ff, l, 14), NULL, 16);
+ texpect_delete = strtoull(procfile_lineword(ff, l, 15), NULL, 16);
+ tsearch_restart = strtoull(procfile_lineword(ff, l, 16), NULL, 16);
+
+ if(unlikely(!aentries)) aentries = tentries;
+
+ // sum all the cpus together
+ asearched += tsearched; // conntrack.search
+ afound += tfound; // conntrack.search
+ anew += tnew; // conntrack.new
+ ainvalid += tinvalid; // conntrack.new
+ aignore += tignore; // conntrack.new
+ adelete += tdelete; // conntrack.changes
+ adelete_list += tdelete_list; // conntrack.changes
+ ainsert += tinsert; // conntrack.changes
+ ainsert_failed += tinsert_failed; // conntrack.errors
+ adrop += tdrop; // conntrack.errors
+ aearly_drop += tearly_drop; // conntrack.errors
+ aicmp_error += ticmp_error; // conntrack.errors
+ aexpect_new += texpect_new; // conntrack.expect
+ aexpect_create += texpect_create; // conntrack.expect
+ aexpect_delete += texpect_delete; // conntrack.expect
+ asearch_restart += tsearch_restart; // conntrack.search
+ }
+ }
+ else {
+ if(unlikely(read_single_number_file(nf_conntrack_count_filename, &aentries)))
+ return 0; // we return 0, so that we will retry to open it next time
+ }
+
+ usec_since_last_max += dt;
+ if(unlikely(rrdvar_max && usec_since_last_max >= get_max_every)) {
+ usec_since_last_max = 0;
+
+ unsigned long long max;
+ if(likely(!read_single_number_file(nf_conntrack_max_filename, &max)))
+ rrdvar_custom_host_variable_set(localhost, rrdvar_max, max);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_sockets) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_connections = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_sockets"
+ , NULL
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NULL
+ , "Connection Tracker Connections"
+ , "active connections"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_CONNTRACK_NAME
+ , NETDATA_CHART_PRIO_NETFILTER_SOCKETS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_connections = rrddim_add(st, "connections", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd_connections, aentries);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_new) {
+ static RRDSET *st = NULL;
+ static RRDDIM
+ *rd_new = NULL,
+ *rd_ignore = NULL,
+ *rd_invalid = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_new"
+ , NULL
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NULL
+ , "Connection Tracker New Connections"
+ , "connections/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_CONNTRACK_NAME
+ , NETDATA_CHART_PRIO_NETFILTER_NEW
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_new = rrddim_add(st, "new", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_ignore = rrddim_add(st, "ignore", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_invalid = rrddim_add(st, "invalid", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_new, anew);
+ rrddim_set_by_pointer(st, rd_ignore, aignore);
+ rrddim_set_by_pointer(st, rd_invalid, ainvalid);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_changes) {
+ static RRDSET *st = NULL;
+ static RRDDIM
+ *rd_inserted = NULL,
+ *rd_deleted = NULL,
+ *rd_delete_list = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_changes"
+ , NULL
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NULL
+ , "Connection Tracker Changes"
+ , "changes/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_CONNTRACK_NAME
+ , NETDATA_CHART_PRIO_NETFILTER_CHANGES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_inserted = rrddim_add(st, "inserted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_deleted = rrddim_add(st, "deleted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_delete_list = rrddim_add(st, "delete_list", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_inserted, ainsert);
+ rrddim_set_by_pointer(st, rd_deleted, adelete);
+ rrddim_set_by_pointer(st, rd_delete_list, adelete_list);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_expect) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_created = NULL,
+ *rd_deleted = NULL,
+ *rd_new = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_expect"
+ , NULL
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NULL
+ , "Connection Tracker Expectations"
+ , "expectations/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_CONNTRACK_NAME
+ , NETDATA_CHART_PRIO_NETFILTER_EXPECT
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_created = rrddim_add(st, "created", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_deleted = rrddim_add(st, "deleted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_new = rrddim_add(st, "new", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_created, aexpect_create);
+ rrddim_set_by_pointer(st, rd_deleted, aexpect_delete);
+ rrddim_set_by_pointer(st, rd_new, aexpect_new);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_search) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_searched = NULL,
+ *rd_restarted = NULL,
+ *rd_found = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_search"
+ , NULL
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NULL
+ , "Connection Tracker Searches"
+ , "searches/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_CONNTRACK_NAME
+ , NETDATA_CHART_PRIO_NETFILTER_SEARCH
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_searched = rrddim_add(st, "searched", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_restarted = rrddim_add(st, "restarted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_found = rrddim_add(st, "found", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_searched, asearched);
+ rrddim_set_by_pointer(st, rd_restarted, asearch_restart);
+ rrddim_set_by_pointer(st, rd_found, afound);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_errors) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_icmp_error = NULL,
+ *rd_insert_failed = NULL,
+ *rd_drop = NULL,
+ *rd_early_drop = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_CONNTRACK "_errors"
+ , NULL
+ , RRD_TYPE_NET_STAT_CONNTRACK
+ , NULL
+ , "Connection Tracker Errors"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_CONNTRACK_NAME
+ , NETDATA_CHART_PRIO_NETFILTER_ERRORS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+
+ rd_icmp_error = rrddim_add(st, "icmp_error", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_insert_failed = rrddim_add(st, "insert_failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_drop = rrddim_add(st, "drop", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_early_drop = rrddim_add(st, "early_drop", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st, rd_icmp_error, aicmp_error);
+ rrddim_set_by_pointer(st, rd_insert_failed, ainsert_failed);
+ rrddim_set_by_pointer(st, rd_drop, adrop);
+ rrddim_set_by_pointer(st, rd_early_drop, aearly_drop);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_stat_synproxy.c b/collectors/proc.plugin/proc_net_stat_synproxy.c
new file mode 100644
index 0000000..0a74b35
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_stat_synproxy.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_SYNPROXY_NAME "/proc/net/stat/synproxy"
+
+#define RRD_TYPE_NET_STAT_NETFILTER "netfilter"
+#define RRD_TYPE_NET_STAT_SYNPROXY "synproxy"
+
+int do_proc_net_stat_synproxy(int update_every, usec_t dt) {
+ (void)dt;
+
+ static int do_cookies = -1, do_syns = -1, do_reopened = -1;
+ static procfile *ff = NULL;
+
+ if(unlikely(do_cookies == -1)) {
+ do_cookies = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY cookies", CONFIG_BOOLEAN_AUTO);
+ do_syns = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY SYN received", CONFIG_BOOLEAN_AUTO);
+ do_reopened = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY connections reopened", CONFIG_BOOLEAN_AUTO);
+ }
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/stat/synproxy");
+ ff = procfile_open(config_get("plugin:proc:/proc/net/stat/synproxy", "filename to monitor", filename), " \t,:|", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff))
+ return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ // make sure we have 3 lines
+ size_t lines = procfile_lines(ff), l;
+ if(unlikely(lines < 2)) {
+ error("/proc/net/stat/synproxy has %zu lines, expected no less than 2. Disabling it.", lines);
+ return 1;
+ }
+
+ unsigned long long syn_received = 0, cookie_invalid = 0, cookie_valid = 0, cookie_retrans = 0, conn_reopened = 0;
+
+ // synproxy gives its values per CPU
+ for(l = 1; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 6))
+ continue;
+
+ syn_received += strtoull(procfile_lineword(ff, l, 1), NULL, 16);
+ cookie_invalid += strtoull(procfile_lineword(ff, l, 2), NULL, 16);
+ cookie_valid += strtoull(procfile_lineword(ff, l, 3), NULL, 16);
+ cookie_retrans += strtoull(procfile_lineword(ff, l, 4), NULL, 16);
+ conn_reopened += strtoull(procfile_lineword(ff, l, 5), NULL, 16);
+ }
+
+ unsigned long long events = syn_received + cookie_invalid + cookie_valid + cookie_retrans + conn_reopened;
+
+ // --------------------------------------------------------------------
+
+ if(do_syns == CONFIG_BOOLEAN_YES || (do_syns == CONFIG_BOOLEAN_AUTO &&
+ (events || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_syns = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_SYNPROXY "_syn_received"
+ , NULL
+ , RRD_TYPE_NET_STAT_SYNPROXY
+ , NULL
+ , "SYNPROXY SYN Packets received"
+ , "packets/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_SYNPROXY_NAME
+ , NETDATA_CHART_PRIO_SYNPROXY_SYN_RECEIVED
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "received", syn_received);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_reopened == CONFIG_BOOLEAN_YES || (do_reopened == CONFIG_BOOLEAN_AUTO &&
+ (events || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_reopened = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_SYNPROXY "_conn_reopened"
+ , NULL
+ , RRD_TYPE_NET_STAT_SYNPROXY
+ , NULL
+ , "SYNPROXY Connections Reopened"
+ , "connections/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_SYNPROXY_NAME
+ , NETDATA_CHART_PRIO_SYNPROXY_CONN_OPEN
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "reopened", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "reopened", conn_reopened);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_cookies == CONFIG_BOOLEAN_YES || (do_cookies == CONFIG_BOOLEAN_AUTO &&
+ (events || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_cookies = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st = NULL;
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_STAT_NETFILTER
+ , RRD_TYPE_NET_STAT_SYNPROXY "_cookies"
+ , NULL
+ , RRD_TYPE_NET_STAT_SYNPROXY
+ , NULL
+ , "SYNPROXY TCP Cookies"
+ , "cookies/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_SYNPROXY_NAME
+ , NETDATA_CHART_PRIO_SYNPROXY_COOKIES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrddim_add(st, "valid", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "invalid", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "retransmits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set(st, "valid", cookie_valid);
+ rrddim_set(st, "invalid", cookie_invalid);
+ rrddim_set(st, "retransmits", cookie_retrans);
+ rrdset_done(st);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_net_wireless.c b/collectors/proc.plugin/proc_net_wireless.c
new file mode 100644
index 0000000..08ab2ea
--- /dev/null
+++ b/collectors/proc.plugin/proc_net_wireless.c
@@ -0,0 +1,432 @@
+#include <stdbool.h>
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_NETWIRELESS_NAME "/proc/net/wireless"
+
+#define CONFIG_SECTION_PLUGIN_PROC_NETWIRELESS "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_NETWIRELESS_NAME
+
+
+static struct netwireless {
+ char *name;
+ uint32_t hash;
+
+ //flags
+ bool configured;
+ struct timeval updated;
+
+ int do_status;
+ int do_quality;
+ int do_discarded_packets;
+ int do_missed_beacon;
+
+ // Data collected
+ // status
+ kernel_uint_t status;
+
+ // Quality
+ NETDATA_DOUBLE link;
+ NETDATA_DOUBLE level;
+ NETDATA_DOUBLE noise;
+
+ // Discarded packets
+ kernel_uint_t nwid;
+ kernel_uint_t crypt;
+ kernel_uint_t frag;
+ kernel_uint_t retry;
+ kernel_uint_t misc;
+
+ // missed beacon
+ kernel_uint_t missed_beacon;
+
+ const char *chart_id_net_status;
+ const char *chart_id_net_link;
+ const char *chart_id_net_level;
+ const char *chart_id_net_noise;
+ const char *chart_id_net_discarded_packets;
+ const char *chart_id_net_missed_beacon;
+
+ const char *chart_family;
+
+ // charts
+ // status
+ RRDSET *st_status;
+
+ // Quality
+ RRDSET *st_link;
+ RRDSET *st_level;
+ RRDSET *st_noise;
+
+ // Discarded Packets
+ RRDSET *st_discarded_packets;
+ // Missed beacon
+ RRDSET *st_missed_beacon;
+
+ // Dimensions
+ // status
+ RRDDIM *rd_status;
+
+ // Quality
+ RRDDIM *rd_link;
+ RRDDIM *rd_level;
+ RRDDIM *rd_noise;
+
+ // Discarded packets
+ RRDDIM *rd_nwid;
+ RRDDIM *rd_crypt;
+ RRDDIM *rd_frag;
+ RRDDIM *rd_retry;
+ RRDDIM *rd_misc;
+
+ // missed beacon
+ RRDDIM *rd_missed_beacon;
+
+ struct netwireless *next;
+} *netwireless_root = NULL;
+
+static void netwireless_free_st(struct netwireless *wireless_dev)
+{
+ if (wireless_dev->st_status) rrdset_is_obsolete(wireless_dev->st_status);
+ if (wireless_dev->st_link) rrdset_is_obsolete(wireless_dev->st_link);
+ if (wireless_dev->st_level) rrdset_is_obsolete(wireless_dev->st_level);
+ if (wireless_dev->st_noise) rrdset_is_obsolete(wireless_dev->st_noise);
+ if (wireless_dev->st_discarded_packets) rrdset_is_obsolete(wireless_dev->st_discarded_packets);
+ if (wireless_dev->st_missed_beacon) rrdset_is_obsolete(wireless_dev->st_missed_beacon);
+
+ wireless_dev->st_status = NULL;
+ wireless_dev->st_link = NULL;
+ wireless_dev->st_level = NULL;
+ wireless_dev->st_noise = NULL;
+ wireless_dev->st_discarded_packets = NULL;
+ wireless_dev->st_missed_beacon = NULL;
+}
+
+static void netwireless_free(struct netwireless *wireless_dev)
+{
+ wireless_dev->next = NULL;
+ freez((void *)wireless_dev->name);
+ netwireless_free_st(wireless_dev);
+ freez((void *)wireless_dev->chart_id_net_status);
+ freez((void *)wireless_dev->chart_id_net_link);
+ freez((void *)wireless_dev->chart_id_net_level);
+ freez((void *)wireless_dev->chart_id_net_noise);
+ freez((void *)wireless_dev->chart_id_net_discarded_packets);
+ freez((void *)wireless_dev->chart_id_net_missed_beacon);
+
+ freez((void *)wireless_dev);
+}
+
+static void netwireless_cleanup(struct timeval *timestamp)
+{
+ struct netwireless *previous = NULL;
+ struct netwireless *current;
+ // search it, from beginning to the end
+ for (current = netwireless_root; current;) {
+
+ if (timercmp(&current->updated, timestamp, <)) {
+ struct netwireless *to_free = current;
+ current = current->next;
+ netwireless_free(to_free);
+
+ if (previous) {
+ previous->next = current;
+ } else {
+ netwireless_root = current;
+ }
+ } else {
+ previous = current;
+ current = current->next;
+ }
+ }
+}
+
+// finds an existing interface or creates a new entry
+static struct netwireless *find_or_create_wireless(const char *name)
+{
+ struct netwireless *wireless;
+ uint32_t hash = simple_hash(name);
+
+ // search it, from beginning to the end
+ for (wireless = netwireless_root ; wireless ; wireless = wireless->next) {
+ if (unlikely(hash == wireless->hash && !strcmp(name, wireless->name))) {
+ return wireless;
+ }
+ }
+
+ // create a new one
+ wireless = callocz(1, sizeof(struct netwireless));
+ wireless->name = strdupz(name);
+ wireless->hash = hash;
+
+ // link it to the end
+ if (netwireless_root) {
+ struct netwireless *last_node;
+ for (last_node = netwireless_root; last_node->next ; last_node = last_node->next);
+
+ last_node->next = wireless;
+ } else
+ netwireless_root = wireless;
+
+ return wireless;
+}
+
+static void configure_device(int do_status, int do_quality, int do_discarded_packets, int do_missed,
+ struct netwireless *wireless_dev) {
+ wireless_dev->do_status = do_status;
+ wireless_dev->do_quality = do_quality;
+ wireless_dev->do_discarded_packets = do_discarded_packets;
+ wireless_dev->do_missed_beacon = do_missed;
+ wireless_dev->configured = true;
+
+ char buffer[RRD_ID_LENGTH_MAX + 1];
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s_status", wireless_dev->name);
+ wireless_dev->chart_id_net_status = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s_link_quality", wireless_dev->name);
+ wireless_dev->chart_id_net_link = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s_signal_level", wireless_dev->name);
+ wireless_dev->chart_id_net_level = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s_noise_level", wireless_dev->name);
+ wireless_dev->chart_id_net_noise = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s_discarded_packets", wireless_dev->name);
+ wireless_dev->chart_id_net_discarded_packets = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s_missed_beacon", wireless_dev->name);
+ wireless_dev->chart_id_net_missed_beacon = strdupz(buffer);
+}
+
+static void add_labels_to_wireless(struct netwireless *w, RRDSET *st) {
+ rrdlabels_add(st->rrdlabels, "device", w->name, RRDLABEL_SRC_AUTO);
+}
+
+int do_proc_net_wireless(int update_every, usec_t dt)
+{
+ UNUSED(dt);
+ static procfile *ff = NULL;
+ static int do_status, do_quality = -1, do_discarded_packets, do_beacon;
+ static char *proc_net_wireless_filename = NULL;
+
+ if (unlikely(do_quality == -1)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/wireless");
+
+ proc_net_wireless_filename = config_get(CONFIG_SECTION_PLUGIN_PROC_NETWIRELESS,"filename to monitor", filename);
+ do_status = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETWIRELESS, "status for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_quality = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETWIRELESS, "quality for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_discarded_packets = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETWIRELESS, "discarded packets for all interfaces", CONFIG_BOOLEAN_AUTO);
+ do_beacon = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETWIRELESS, "missed beacon for all interface", CONFIG_BOOLEAN_AUTO);
+ }
+
+ if (unlikely(!ff)) {
+ ff = procfile_open(proc_net_wireless_filename, " \t,|", PROCFILE_FLAG_DEFAULT);
+ if (unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if (unlikely(!ff)) return 1;
+
+ size_t lines = procfile_lines(ff);
+ struct timeval timestamp;
+ size_t l;
+ gettimeofday(&timestamp, NULL);
+ for (l = 2; l < lines; l++) {
+ if (unlikely(procfile_linewords(ff, l) < 11)) continue;
+
+ char *name = procfile_lineword(ff, l, 0);
+ size_t len = strlen(name);
+ if (name[len - 1] == ':') name[len - 1] = '\0';
+
+ struct netwireless *wireless_dev = find_or_create_wireless(name);
+
+ if (unlikely(!wireless_dev->configured)) {
+ configure_device(do_status, do_quality, do_discarded_packets, do_beacon, wireless_dev);
+ }
+
+ if (likely(do_status != CONFIG_BOOLEAN_NO)) {
+ wireless_dev->status = str2kernel_uint_t(procfile_lineword(ff, l, 1));
+
+ if (unlikely(!wireless_dev->st_status)) {
+ wireless_dev->st_status = rrdset_create_localhost(
+ "wireless",
+ wireless_dev->chart_id_net_status,
+ NULL,
+ wireless_dev->name,
+ "wireless.status",
+ "Internal status reported by interface.",
+ "status",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_NETWIRELESS_NAME,
+ NETDATA_CHART_PRIO_WIRELESS_IFACE,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_flag_set(wireless_dev->st_status, RRDSET_FLAG_DETAIL);
+
+ wireless_dev->rd_status = rrddim_add(wireless_dev->st_status, "status", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_wireless(wireless_dev, wireless_dev->st_status);
+ }
+
+ rrddim_set_by_pointer(wireless_dev->st_status, wireless_dev->rd_status,
+ (collected_number)wireless_dev->status);
+ rrdset_done(wireless_dev->st_status);
+ }
+
+ if (likely(do_quality != CONFIG_BOOLEAN_NO)) {
+ wireless_dev->link = str2ndd(procfile_lineword(ff, l, 2), NULL);
+ wireless_dev->level = str2ndd(procfile_lineword(ff, l, 3), NULL);
+ wireless_dev->noise = str2ndd(procfile_lineword(ff, l, 4), NULL);
+
+ if (unlikely(!wireless_dev->st_link)) {
+ wireless_dev->st_link = rrdset_create_localhost(
+ "wireless",
+ wireless_dev->chart_id_net_link,
+ NULL,
+ wireless_dev->name,
+ "wireless.link_quality",
+ "Overall quality of the link. This is an aggregate value, and depends on the driver and hardware.",
+ "value",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_NETWIRELESS_NAME,
+ NETDATA_CHART_PRIO_WIRELESS_IFACE + 1,
+ update_every,
+ RRDSET_TYPE_LINE);
+ rrdset_flag_set(wireless_dev->st_link, RRDSET_FLAG_DETAIL);
+
+ wireless_dev->rd_link = rrddim_add(wireless_dev->st_link, "link_quality", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_wireless(wireless_dev, wireless_dev->st_link);
+ }
+
+ if (unlikely(!wireless_dev->st_level)) {
+ wireless_dev->st_level = rrdset_create_localhost(
+ "wireless",
+ wireless_dev->chart_id_net_level,
+ NULL,
+ wireless_dev->name,
+ "wireless.signal_level",
+ "The signal level is the wireless signal power level received by the wireless client. The closer the value is to 0, the stronger the signal.",
+ "dBm",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_NETWIRELESS_NAME,
+ NETDATA_CHART_PRIO_WIRELESS_IFACE + 2,
+ update_every,
+ RRDSET_TYPE_LINE);
+ rrdset_flag_set(wireless_dev->st_level, RRDSET_FLAG_DETAIL);
+
+ wireless_dev->rd_level = rrddim_add(wireless_dev->st_level, "signal_level", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_wireless(wireless_dev, wireless_dev->st_level);
+ }
+
+ if (unlikely(!wireless_dev->st_noise)) {
+ wireless_dev->st_noise = rrdset_create_localhost(
+ "wireless",
+ wireless_dev->chart_id_net_noise,
+ NULL,
+ wireless_dev->name,
+ "wireless.noise_level",
+ "The noise level indicates the amount of background noise in your environment. The closer the value to 0, the greater the noise level.",
+ "dBm",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_NETWIRELESS_NAME,
+ NETDATA_CHART_PRIO_WIRELESS_IFACE + 3,
+ update_every,
+ RRDSET_TYPE_LINE);
+ rrdset_flag_set(wireless_dev->st_noise, RRDSET_FLAG_DETAIL);
+
+ wireless_dev->rd_noise = rrddim_add(wireless_dev->st_noise, "noise_level", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_wireless(wireless_dev, wireless_dev->st_noise);
+ }
+
+ rrddim_set_by_pointer(wireless_dev->st_link, wireless_dev->rd_link, (collected_number)wireless_dev->link);
+ rrdset_done(wireless_dev->st_link);
+
+ rrddim_set_by_pointer(wireless_dev->st_level, wireless_dev->rd_level, (collected_number)wireless_dev->level);
+ rrdset_done(wireless_dev->st_level);
+
+ rrddim_set_by_pointer(wireless_dev->st_noise, wireless_dev->rd_noise, (collected_number)wireless_dev->noise);
+ rrdset_done(wireless_dev->st_noise);
+ }
+
+ if (likely(do_discarded_packets)) {
+ wireless_dev->nwid = str2kernel_uint_t(procfile_lineword(ff, l, 5));
+ wireless_dev->crypt = str2kernel_uint_t(procfile_lineword(ff, l, 6));
+ wireless_dev->frag = str2kernel_uint_t(procfile_lineword(ff, l, 7));
+ wireless_dev->retry = str2kernel_uint_t(procfile_lineword(ff, l, 8));
+ wireless_dev->misc = str2kernel_uint_t(procfile_lineword(ff, l, 9));
+
+ if (unlikely(!wireless_dev->st_discarded_packets)) {
+ wireless_dev->st_discarded_packets = rrdset_create_localhost(
+ "wireless",
+ wireless_dev->chart_id_net_discarded_packets,
+ NULL,
+ wireless_dev->name,
+ "wireless.discarded_packets",
+ "Packet discarded in the wireless adapter due to \"wireless\" specific problems.",
+ "packets/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_NETWIRELESS_NAME,
+ NETDATA_CHART_PRIO_WIRELESS_IFACE + 4,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_flag_set(wireless_dev->st_discarded_packets, RRDSET_FLAG_DETAIL);
+
+ wireless_dev->rd_nwid = rrddim_add(wireless_dev->st_discarded_packets, "nwid", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ wireless_dev->rd_crypt = rrddim_add(wireless_dev->st_discarded_packets, "crypt", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ wireless_dev->rd_frag = rrddim_add(wireless_dev->st_discarded_packets, "frag", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ wireless_dev->rd_retry = rrddim_add(wireless_dev->st_discarded_packets, "retry", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ wireless_dev->rd_misc = rrddim_add(wireless_dev->st_discarded_packets, "misc", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_wireless(wireless_dev, wireless_dev->st_discarded_packets);
+ }
+
+ rrddim_set_by_pointer(wireless_dev->st_discarded_packets, wireless_dev->rd_nwid, (collected_number)wireless_dev->nwid);
+ rrddim_set_by_pointer(wireless_dev->st_discarded_packets, wireless_dev->rd_crypt, (collected_number)wireless_dev->crypt);
+ rrddim_set_by_pointer(wireless_dev->st_discarded_packets, wireless_dev->rd_frag, (collected_number)wireless_dev->frag);
+ rrddim_set_by_pointer(wireless_dev->st_discarded_packets, wireless_dev->rd_retry, (collected_number)wireless_dev->retry);
+ rrddim_set_by_pointer(wireless_dev->st_discarded_packets, wireless_dev->rd_misc, (collected_number)wireless_dev->misc);
+
+ rrdset_done(wireless_dev->st_discarded_packets);
+ }
+
+ if (likely(do_beacon)) {
+ wireless_dev->missed_beacon = str2kernel_uint_t(procfile_lineword(ff, l, 10));
+
+ if (unlikely(!wireless_dev->st_missed_beacon)) {
+ wireless_dev->st_missed_beacon = rrdset_create_localhost(
+ "wireless",
+ wireless_dev->chart_id_net_missed_beacon,
+ NULL,
+ wireless_dev->name,
+ "wireless.missed_beacons",
+ "Number of missed beacons.",
+ "frames/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_NETWIRELESS_NAME,
+ NETDATA_CHART_PRIO_WIRELESS_IFACE + 5,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_flag_set(wireless_dev->st_missed_beacon, RRDSET_FLAG_DETAIL);
+
+ wireless_dev->rd_missed_beacon = rrddim_add(wireless_dev->st_missed_beacon, "missed_beacons", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ add_labels_to_wireless(wireless_dev, wireless_dev->st_missed_beacon);
+ }
+
+ rrddim_set_by_pointer(wireless_dev->st_missed_beacon, wireless_dev->rd_missed_beacon, (collected_number)wireless_dev->missed_beacon);
+ rrdset_done(wireless_dev->st_missed_beacon);
+ }
+
+ wireless_dev->updated = timestamp;
+ }
+
+ netwireless_cleanup(&timestamp);
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_pagetypeinfo.c b/collectors/proc.plugin/proc_pagetypeinfo.c
new file mode 100644
index 0000000..dc006aa
--- /dev/null
+++ b/collectors/proc.plugin/proc_pagetypeinfo.c
@@ -0,0 +1,338 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+// For ULONG_MAX
+#include <limits.h>
+
+#define PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME "/proc/pagetypeinfo"
+#define CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME
+
+// Zone struct is pglist_data, in include/linux/mmzone.h
+// MAX_NR_ZONES is from __MAX_NR_ZONE, which is the last value of the enum.
+#define MAX_PAGETYPE_ORDER 11
+
+// Names are in mm/page_alloc.c :: migratetype_names. Max size = 10.
+#define MAX_ZONETYPE_NAME 16
+#define MAX_PAGETYPE_NAME 16
+
+// Defined in include/linux/mmzone.h as __MAX_NR_ZONE (last enum of zone_type)
+#define MAX_ZONETYPE 6
+// Defined in include/linux/mmzone.h as MIGRATE_TYPES (last enum of migratetype)
+#define MAX_PAGETYPE 7
+
+
+//
+// /proc/pagetypeinfo is declared in mm/vmstat.c :: init_mm_internals
+//
+
+// One line of /proc/pagetypeinfo
+struct pageline {
+ int node;
+ char *zone;
+ char *type;
+ int line;
+ uint64_t free_pages_size[MAX_PAGETYPE_ORDER];
+ RRDDIM *rd[MAX_PAGETYPE_ORDER];
+};
+
+// Sum of all orders
+struct systemorder {
+ uint64_t size;
+ RRDDIM *rd;
+};
+
+
+static inline uint64_t pageline_total_count(struct pageline *p) {
+ uint64_t sum = 0, o;
+ for (o=0; o<MAX_PAGETYPE_ORDER; o++)
+ sum += p->free_pages_size[o];
+ return sum;
+}
+
+// Check if a line of /proc/pagetypeinfo is valid to use
+// Free block lines starts by "Node" && 4th col is "type"
+#define pagetypeinfo_line_valid(ff, l) (strncmp(procfile_lineword(ff, l, 0), "Node", 4) == 0 && strncmp(procfile_lineword(ff, l, 4), "type", 4) == 0)
+
+// Dimension name from the order
+#define dim_name(s, o, pagesize) (snprintfz(s, 16,"%ldKB (%lu)", (1 << o) * pagesize / 1024, o))
+
+int do_proc_pagetypeinfo(int update_every, usec_t dt) {
+ (void)dt;
+
+ // Config
+ static int do_global, do_detail;
+ static SIMPLE_PATTERN *filter_types = NULL;
+
+ // Counters from parsing the file, that doesn't change after boot
+ static struct systemorder systemorders[MAX_PAGETYPE_ORDER] = {};
+ static struct pageline* pagelines = NULL;
+ static long pagesize = 0;
+ static size_t pageorders_cnt = 0, pagelines_cnt = 0, ff_lines = 0;
+
+ // Handle
+ static procfile *ff = NULL;
+ static char ff_path[FILENAME_MAX + 1];
+
+ // RRD Sets
+ static RRDSET *st_order = NULL;
+ static RRDSET **st_nodezonetype = NULL;
+
+ // Local temp variables
+ long unsigned int l, o, p;
+ struct pageline *pgl = NULL;
+
+ // --------------------------------------------------------------------
+ // Startup: Init arch and open /proc/pagetypeinfo
+ if (unlikely(!pagesize)) {
+ pagesize = sysconf(_SC_PAGESIZE);
+ }
+
+ if(unlikely(!ff)) {
+ snprintfz(ff_path, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME);
+ ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "filename to monitor", ff_path), " \t:", PROCFILE_FLAG_DEFAULT);
+
+ if(unlikely(!ff)) {
+ strncpyz(ff_path, PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME, FILENAME_MAX);
+ ff = procfile_open(PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME, " \t,", PROCFILE_FLAG_DEFAULT);
+ }
+ }
+ if(unlikely(!ff))
+ return 1;
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ // --------------------------------------------------------------------
+ // Init: find how many Nodes, Zones and Types
+ if(unlikely(pagelines_cnt == 0)) {
+ size_t nodenumlast = -1;
+ char *zonenamelast = NULL;
+
+ ff_lines = procfile_lines(ff);
+ if(unlikely(!ff_lines)) {
+ error("PLUGIN: PROC_PAGETYPEINFO: Cannot read %s, zero lines reported.", ff_path);
+ return 1;
+ }
+
+ // Configuration
+ do_global = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable system summary", CONFIG_BOOLEAN_YES);
+ do_detail = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable detail per-type", CONFIG_BOOLEAN_AUTO);
+ filter_types = simple_pattern_create(
+ config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "hide charts id matching", "")
+ , NULL
+ , SIMPLE_PATTERN_SUFFIX
+ );
+
+ pagelines_cnt = 0;
+
+ // Pass 1: how many lines would be valid
+ for (l = 4; l < ff_lines; l++) {
+ if (!pagetypeinfo_line_valid(ff, l))
+ continue;
+
+ pagelines_cnt++;
+ }
+ if (pagelines_cnt == 0) {
+ error("PLUGIN: PROC_PAGETYPEINFO: Unable to parse any valid line in %s", ff_path);
+ return 1;
+ }
+
+ // 4th line is the "Free pages count per migrate type at order". Just subtract these 8 words.
+ pageorders_cnt = procfile_linewords(ff, 3);
+ if (pageorders_cnt < 9) {
+ error("PLUGIN: PROC_PAGETYPEINFO: Unable to parse Line 4 of %s", ff_path);
+ return 1;
+ }
+
+ pageorders_cnt -= 9;
+
+ if (pageorders_cnt > MAX_PAGETYPE_ORDER) {
+ error("PLUGIN: PROC_PAGETYPEINFO: pageorder found (%lu) is higher than max %d",
+ (long unsigned int) pageorders_cnt, MAX_PAGETYPE_ORDER);
+ return 1;
+ }
+
+ // Init pagelines from scanned lines
+ if (!pagelines) {
+ pagelines = callocz(pagelines_cnt, sizeof(struct pageline));
+ if (!pagelines) {
+ error("PLUGIN: PROC_PAGETYPEINFO: Cannot allocate %lu pagelines of %lu B",
+ (long unsigned int) pagelines_cnt, (long unsigned int) sizeof(struct pageline));
+ return 1;
+ }
+ }
+
+ // Pass 2: Scan the file again, with details
+ p = 0;
+ for (l=4; l < ff_lines; l++) {
+
+ if (!pagetypeinfo_line_valid(ff, l))
+ continue;
+
+ size_t nodenum = strtoul(procfile_lineword(ff, l, 1), NULL, 10);
+ char *zonename = procfile_lineword(ff, l, 3);
+ char *typename = procfile_lineword(ff, l, 5);
+
+ // We changed node or zone
+ if (nodenum != nodenumlast || !zonenamelast || strncmp(zonename, zonenamelast, 6) != 0) {
+ zonenamelast = zonename;
+ }
+
+ // Populate the line
+ pgl = &pagelines[p];
+
+ pgl->line = l;
+ pgl->node = nodenum;
+ pgl->type = typename;
+ pgl->zone = zonename;
+ for (o = 0; o < pageorders_cnt; o++)
+ pgl->free_pages_size[o] = str2uint64_t(procfile_lineword(ff, l, o+6)) * 1 << o;
+
+ p++;
+ }
+
+ // Init the RRD graphs
+
+ // Per-Order: sum of all node, zone, type Grouped by order
+ if (do_global != CONFIG_BOOLEAN_NO) {
+ st_order = rrdset_create_localhost(
+ "mem"
+ , "pagetype_global"
+ , NULL
+ , "pagetype"
+ , NULL
+ , "System orders available"
+ , "B"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_PAGEFRAG
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ for (o = 0; o < pageorders_cnt; o++) {
+ char id[3+1];
+ snprintfz(id, 3, "%lu", o);
+
+ char name[20+1];
+ dim_name(name, o, pagesize);
+
+ systemorders[o].rd = rrddim_add(st_order, id, name, pagesize, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+ }
+
+
+ // Per-Numa Node & Zone & Type (full detail). Only if sum(line) > 0
+ st_nodezonetype = callocz(pagelines_cnt, sizeof(RRDSET *));
+ for (p = 0; p < pagelines_cnt; p++) {
+ pgl = &pagelines[p];
+
+ // Skip invalid, refused or empty pagelines if not explicitly requested
+ if (!pgl
+ || do_detail == CONFIG_BOOLEAN_NO
+ || (do_detail == CONFIG_BOOLEAN_AUTO && pageline_total_count(pgl) == 0 && netdata_zero_metrics_enabled != CONFIG_BOOLEAN_YES))
+ continue;
+
+ // "pagetype Node" + NUMA-NodeId + ZoneName + TypeName
+ char setid[13+1+2+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME+1];
+ snprintfz(setid, 13+1+2+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME, "pagetype_Node%d_%s_%s", pgl->node, pgl->zone, pgl->type);
+
+ // Skip explicitly refused charts
+ if (simple_pattern_matches(filter_types, setid))
+ continue;
+
+ // "Node" + NUMA-NodeID + ZoneName + TypeName
+ char setname[4+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME +1];
+ snprintfz(setname, MAX_ZONETYPE_NAME + MAX_PAGETYPE_NAME, "Node %d %s %s", pgl->node, pgl->zone, pgl->type);
+
+ st_nodezonetype[p] = rrdset_create_localhost(
+ "mem"
+ , setid
+ , NULL
+ , "pagetype"
+ , "mem.pagetype"
+ , setname
+ , "B"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME
+ , NETDATA_CHART_PRIO_MEM_PAGEFRAG + 1 + p
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ char node[50+1];
+ snprintfz(node, 50, "node%d", pgl->node);
+ rrdlabels_add(st_nodezonetype[p]->rrdlabels, "node_id", node, RRDLABEL_SRC_AUTO);
+ rrdlabels_add(st_nodezonetype[p]->rrdlabels, "node_zone", pgl->zone, RRDLABEL_SRC_AUTO);
+ rrdlabels_add(st_nodezonetype[p]->rrdlabels, "node_type", pgl->type, RRDLABEL_SRC_AUTO);
+
+ for (o = 0; o < pageorders_cnt; o++) {
+ char dimid[3+1];
+ snprintfz(dimid, 3, "%lu", o);
+ char dimname[20+1];
+ dim_name(dimname, o, pagesize);
+
+ pgl->rd[o] = rrddim_add(st_nodezonetype[p], dimid, dimname, pagesize, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------
+ // Update pagelines
+
+ // Process each line
+ p = 0;
+ for (l=4; l<ff_lines; l++) {
+
+ if (!pagetypeinfo_line_valid(ff, l))
+ continue;
+
+ size_t words = procfile_linewords(ff, l);
+
+ if (words != 7+pageorders_cnt) {
+ error("PLUGIN: PROC_PAGETYPEINFO: Unable to read line %lu, %lu words found instead of %lu",
+ l+1, (long unsigned int) words, (long unsigned int) 7+pageorders_cnt);
+ break;
+ }
+
+ for (o = 0; o < pageorders_cnt; o++) {
+ // Reset counter
+ if (p == 0)
+ systemorders[o].size = 0;
+
+ // Update orders of the current line
+ pagelines[p].free_pages_size[o] = str2uint64_t(procfile_lineword(ff, l, o+6)) * 1 << o;
+
+ // Update sum by order
+ systemorders[o].size += pagelines[p].free_pages_size[o];
+ }
+
+ p++;
+ }
+
+ // --------------------------------------------------------------------
+ // update RRD values
+
+ // Global system per order
+ if (st_order) {
+ for (o = 0; o < pageorders_cnt; o++)
+ rrddim_set_by_pointer(st_order, systemorders[o].rd, systemorders[o].size);
+ rrdset_done(st_order);
+ }
+
+ // Per Node-Zone-Type
+ if (do_detail) {
+ for (p = 0; p < pagelines_cnt; p++) {
+ // Skip empty graphs
+ if (!st_nodezonetype[p])
+ continue;
+
+ for (o = 0; o < pageorders_cnt; o++)
+ rrddim_set_by_pointer(st_nodezonetype[p], pagelines[p].rd[o], pagelines[p].free_pages_size[o]);
+ rrdset_done(st_nodezonetype[p]);
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_pressure.c b/collectors/proc.plugin/proc_pressure.c
new file mode 100644
index 0000000..6649aa6
--- /dev/null
+++ b/collectors/proc.plugin/proc_pressure.c
@@ -0,0 +1,209 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_PRESSURE_NAME "/proc/pressure"
+#define CONFIG_SECTION_PLUGIN_PROC_PRESSURE "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_PRESSURE_NAME
+
+// linux calculates this every 2 seconds, see kernel/sched/psi.c PSI_FREQ
+#define MIN_PRESSURE_UPDATE_EVERY 2
+
+static int pressure_update_every = 0;
+
+static struct pressure resources[PRESSURE_NUM_RESOURCES] = {
+ {
+ .some =
+ {.share_time = {.id = "cpu_some_pressure", .title = "CPU some pressure"},
+ .total_time = {.id = "cpu_some_pressure_stall_time", .title = "CPU some pressure stall time"}},
+ .full =
+ {.share_time = {.id = "cpu_full_pressure", .title = "CPU full pressure"},
+ .total_time = {.id = "cpu_full_pressure_stall_time", .title = "CPU full pressure stall time"}},
+ },
+ {
+ .some =
+ {.share_time = {.id = "memory_some_pressure", .title = "Memory some pressure"},
+ .total_time = {.id = "memory_some_pressure_stall_time", .title = "Memory some pressure stall time"}},
+ .full =
+ {.share_time = {.id = "memory_full_pressure", .title = "Memory full pressure"},
+ .total_time = {.id = "memory_full_pressure_stall_time", .title = "Memory full pressure stall time"}},
+ },
+ {
+ .some =
+ {.share_time = {.id = "io_some_pressure", .title = "I/O some pressure"},
+ .total_time = {.id = "io_some_pressure_stall_time", .title = "I/O some pressure stall time"}},
+ .full =
+ {.share_time = {.id = "io_full_pressure", .title = "I/O full pressure"},
+ .total_time = {.id = "io_full_pressure_stall_time", .title = "I/O full pressure stall time"}},
+ },
+};
+
+static struct resource_info {
+ procfile *pf;
+ const char *name; // metric file name
+ const char *family; // webui section name
+ int section_priority;
+} resource_info[PRESSURE_NUM_RESOURCES] = {
+ { .name = "cpu", .family = "cpu", .section_priority = NETDATA_CHART_PRIO_SYSTEM_CPU },
+ { .name = "memory", .family = "ram", .section_priority = NETDATA_CHART_PRIO_SYSTEM_RAM },
+ { .name = "io", .family = "disk", .section_priority = NETDATA_CHART_PRIO_SYSTEM_IO },
+};
+
+void update_pressure_charts(struct pressure_charts *pcs) {
+ if (pcs->share_time.st) {
+ rrddim_set_by_pointer(
+ pcs->share_time.st, pcs->share_time.rd10, (collected_number)(pcs->share_time.value10 * 100));
+ rrddim_set_by_pointer(
+ pcs->share_time.st, pcs->share_time.rd60, (collected_number)(pcs->share_time.value60 * 100));
+ rrddim_set_by_pointer(
+ pcs->share_time.st, pcs->share_time.rd300, (collected_number)(pcs->share_time.value300 * 100));
+ rrdset_done(pcs->share_time.st);
+ }
+ if (pcs->total_time.st) {
+ rrddim_set_by_pointer(
+ pcs->total_time.st, pcs->total_time.rdtotal, (collected_number)(pcs->total_time.value_total));
+ rrdset_done(pcs->total_time.st);
+ }
+}
+
+static void proc_pressure_do_resource(procfile *ff, int res_idx, int some) {
+ struct pressure_charts *pcs;
+ struct resource_info ri;
+ pcs = some ? &resources[res_idx].some : &resources[res_idx].full;
+ ri = resource_info[res_idx];
+
+ if (unlikely(!pcs->share_time.st)) {
+ pcs->share_time.st = rrdset_create_localhost(
+ "system",
+ pcs->share_time.id,
+ NULL,
+ ri.family,
+ NULL,
+ pcs->share_time.title,
+ "percentage",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_PRESSURE_NAME,
+ ri.section_priority + (some ? 40 : 50),
+ pressure_update_every,
+ RRDSET_TYPE_LINE);
+ pcs->share_time.rd10 =
+ rrddim_add(pcs->share_time.st, some ? "some 10" : "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd60 =
+ rrddim_add(pcs->share_time.st, some ? "some 60" : "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ pcs->share_time.rd300 =
+ rrddim_add(pcs->share_time.st, some ? "some 300" : "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ pcs->share_time.value10 = strtod(procfile_lineword(ff, some ? 0 : 1, 2), NULL);
+ pcs->share_time.value60 = strtod(procfile_lineword(ff, some ? 0 : 1, 4), NULL);
+ pcs->share_time.value300 = strtod(procfile_lineword(ff, some ? 0 : 1, 6), NULL);
+
+ if (unlikely(!pcs->total_time.st)) {
+ pcs->total_time.st = rrdset_create_localhost(
+ "system",
+ pcs->total_time.id,
+ NULL,
+ ri.family,
+ NULL,
+ pcs->total_time.title,
+ "ms",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_PRESSURE_NAME,
+ ri.section_priority + (some ? 45 : 55),
+ pressure_update_every,
+ RRDSET_TYPE_LINE);
+ pcs->total_time.rdtotal = rrddim_add(pcs->total_time.st, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ pcs->total_time.value_total = str2ull(procfile_lineword(ff, some ? 0 : 1, 8)) / 1000;
+}
+
+static void proc_pressure_do_resource_some(procfile *ff, int res_idx) {
+ proc_pressure_do_resource(ff, res_idx, 1);
+}
+
+static void proc_pressure_do_resource_full(procfile *ff, int res_idx) {
+ proc_pressure_do_resource(ff, res_idx, 0);
+}
+
+int do_proc_pressure(int update_every, usec_t dt) {
+ int fail_count = 0;
+ int i;
+
+ static usec_t next_pressure_dt = 0;
+ static char *base_path = NULL;
+
+ update_every = (update_every < MIN_PRESSURE_UPDATE_EVERY) ? MIN_PRESSURE_UPDATE_EVERY : update_every;
+ pressure_update_every = update_every;
+
+ if (next_pressure_dt <= dt) {
+ next_pressure_dt = update_every * USEC_PER_SEC;
+ } else {
+ next_pressure_dt -= dt;
+ return 0;
+ }
+
+ if (unlikely(!base_path)) {
+ base_path = config_get(CONFIG_SECTION_PLUGIN_PROC_PRESSURE, "base path of pressure metrics", "/proc/pressure");
+ }
+
+ for (i = 0; i < PRESSURE_NUM_RESOURCES; i++) {
+ procfile *ff = resource_info[i].pf;
+ int do_some = resources[i].some.enabled, do_full = resources[i].full.enabled;
+
+ if (unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ char config_key[CONFIG_MAX_NAME + 1];
+
+ snprintfz(filename
+ , FILENAME_MAX
+ , "%s%s/%s"
+ , netdata_configured_host_prefix
+ , base_path
+ , resource_info[i].name);
+
+ snprintfz(config_key, CONFIG_MAX_NAME, "enable %s some pressure", resource_info[i].name);
+ do_some = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_PRESSURE, config_key, CONFIG_BOOLEAN_YES);
+ resources[i].some.enabled = do_some;
+
+ snprintfz(config_key, CONFIG_MAX_NAME, "enable %s full pressure", resource_info[i].name);
+ do_full = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_PRESSURE, config_key, CONFIG_BOOLEAN_YES);
+ resources[i].full.enabled = do_full;
+
+ ff = procfile_open(filename, " =", PROCFILE_FLAG_DEFAULT);
+ if (unlikely(!ff)) {
+ error("Cannot read pressure information from %s.", filename);
+ fail_count++;
+ continue;
+ }
+ }
+
+ ff = procfile_readall(ff);
+ resource_info[i].pf = ff;
+ if (unlikely(!ff)) {
+ fail_count++;
+ continue;
+ }
+
+ size_t lines = procfile_lines(ff);
+ if (unlikely(lines < 1)) {
+ error("%s has no lines.", procfile_filename(ff));
+ fail_count++;
+ continue;
+ }
+
+ if (do_some) {
+ proc_pressure_do_resource_some(ff, i);
+ update_pressure_charts(&resources[i].some);
+ }
+ if (do_full && lines > 2) {
+ proc_pressure_do_resource_full(ff, i);
+ update_pressure_charts(&resources[i].full);
+ }
+ }
+
+ if (PRESSURE_NUM_RESOURCES == fail_count) {
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_pressure.h b/collectors/proc.plugin/proc_pressure.h
new file mode 100644
index 0000000..0cb2331
--- /dev/null
+++ b/collectors/proc.plugin/proc_pressure.h
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PROC_PRESSURE_H
+#define NETDATA_PROC_PRESSURE_H
+
+#define PRESSURE_NUM_RESOURCES 3
+
+struct pressure {
+ int updated;
+ char *filename;
+
+ struct pressure_charts {
+ int enabled;
+
+ struct pressure_share_time_chart {
+ const char *id;
+ const char *title;
+
+ double value10;
+ double value60;
+ double value300;
+
+ RRDSET *st;
+ RRDDIM *rd10;
+ RRDDIM *rd60;
+ RRDDIM *rd300;
+ } share_time;
+
+ struct pressure_total_time_chart {
+ const char *id;
+ const char *title;
+
+ unsigned long long value_total;
+
+ RRDSET *st;
+ RRDDIM *rdtotal;
+ } total_time;
+ } some, full;
+};
+
+void update_pressure_charts(struct pressure_charts *charts);
+
+#endif //NETDATA_PROC_PRESSURE_H
diff --git a/collectors/proc.plugin/proc_self_mountinfo.c b/collectors/proc.plugin/proc_self_mountinfo.c
new file mode 100644
index 0000000..9310f2f
--- /dev/null
+++ b/collectors/proc.plugin/proc_self_mountinfo.c
@@ -0,0 +1,458 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+// ----------------------------------------------------------------------------
+// taken from gnulib/mountlist.c
+
+#ifndef ME_REMOTE
+/* A file system is "remote" if its Fs_name contains a ':'
+ or if (it is of type (smbfs or cifs) and its Fs_name starts with '//')
+ or Fs_name is equal to "-hosts" (used by autofs to mount remote fs). */
+# define ME_REMOTE(Fs_name, Fs_type) \
+ (strchr (Fs_name, ':') != NULL \
+ || ((Fs_name)[0] == '/' \
+ && (Fs_name)[1] == '/' \
+ && (strcmp (Fs_type, "smbfs") == 0 \
+ || strcmp (Fs_type, "cifs") == 0)) \
+ || (strcmp("-hosts", Fs_name) == 0))
+#endif
+
+#define ME_DUMMY_0(Fs_name, Fs_type) \
+ (strcmp (Fs_type, "autofs") == 0 \
+ || strcmp (Fs_type, "proc") == 0 \
+ || strcmp (Fs_type, "subfs") == 0 \
+ /* for Linux 2.6/3.x */ \
+ || strcmp (Fs_type, "debugfs") == 0 \
+ || strcmp (Fs_type, "devpts") == 0 \
+ || strcmp (Fs_type, "fusectl") == 0 \
+ || strcmp (Fs_type, "mqueue") == 0 \
+ || strcmp (Fs_type, "rpc_pipefs") == 0 \
+ || strcmp (Fs_type, "sysfs") == 0 \
+ /* FreeBSD, Linux 2.4 */ \
+ || strcmp (Fs_type, "devfs") == 0 \
+ /* for NetBSD 3.0 */ \
+ || strcmp (Fs_type, "kernfs") == 0 \
+ /* for Irix 6.5 */ \
+ || strcmp (Fs_type, "ignore") == 0)
+
+/* Historically, we have marked as "dummy" any file system of type "none",
+ but now that programs like du need to know about bind-mounted directories,
+ we grant an exception to any with "bind" in its list of mount options.
+ I.e., those are *not* dummy entries. */
+# define ME_DUMMY(Fs_name, Fs_type) \
+ (ME_DUMMY_0 (Fs_name, Fs_type) || strcmp (Fs_type, "none") == 0)
+
+// ----------------------------------------------------------------------------
+
+// find the mount info with the given major:minor
+// in the supplied linked list of mountinfo structures
+struct mountinfo *mountinfo_find(struct mountinfo *root, unsigned long major, unsigned long minor, char *device) {
+ struct mountinfo *mi;
+
+ uint32_t hash = simple_hash(device);
+
+ for(mi = root; mi ; mi = mi->next)
+ if (unlikely(
+ mi->major == major &&
+ mi->minor == minor &&
+ mi->mount_source_name_hash == hash &&
+ !strcmp(mi->mount_source_name, device)))
+ return mi;
+
+ return NULL;
+}
+
+// find the mount info with the given filesystem and mount_source
+// in the supplied linked list of mountinfo structures
+struct mountinfo *mountinfo_find_by_filesystem_mount_source(struct mountinfo *root, const char *filesystem, const char *mount_source) {
+ struct mountinfo *mi;
+ uint32_t filesystem_hash = simple_hash(filesystem), mount_source_hash = simple_hash(mount_source);
+
+ for(mi = root; mi ; mi = mi->next)
+ if(unlikely(mi->filesystem
+ && mi->mount_source
+ && mi->filesystem_hash == filesystem_hash
+ && mi->mount_source_hash == mount_source_hash
+ && !strcmp(mi->filesystem, filesystem)
+ && !strcmp(mi->mount_source, mount_source)))
+ return mi;
+
+ return NULL;
+}
+
+struct mountinfo *mountinfo_find_by_filesystem_super_option(struct mountinfo *root, const char *filesystem, const char *super_options) {
+ struct mountinfo *mi;
+ uint32_t filesystem_hash = simple_hash(filesystem);
+
+ size_t solen = strlen(super_options);
+
+ for(mi = root; mi ; mi = mi->next)
+ if(unlikely(mi->filesystem
+ && mi->super_options
+ && mi->filesystem_hash == filesystem_hash
+ && !strcmp(mi->filesystem, filesystem))) {
+
+ // super_options is a comma separated list
+ char *s = mi->super_options, *e;
+ while(*s) {
+ e = s + 1;
+ while(*e && *e != ',') e++;
+
+ size_t len = e - s;
+ if(unlikely(len == solen && !strncmp(s, super_options, len)))
+ return mi;
+
+ if(*e == ',') s = ++e;
+ else s = e;
+ }
+ }
+
+ return NULL;
+}
+
+static void mountinfo_free(struct mountinfo *mi) {
+ freez(mi->root);
+ freez(mi->mount_point);
+ freez(mi->mount_options);
+ freez(mi->persistent_id);
+/*
+ if(mi->optional_fields_count) {
+ int i;
+ for(i = 0; i < mi->optional_fields_count ; i++)
+ free(*mi->optional_fields[i]);
+ }
+ free(mi->optional_fields);
+*/
+ freez(mi->filesystem);
+ freez(mi->mount_source);
+ freez(mi->mount_source_name);
+ freez(mi->super_options);
+ freez(mi);
+}
+
+// free a linked list of mountinfo structures
+void mountinfo_free_all(struct mountinfo *mi) {
+ while(mi) {
+ struct mountinfo *t = mi;
+ mi = mi->next;
+
+ mountinfo_free(t);
+ }
+}
+
+static char *strdupz_decoding_octal(const char *string) {
+ char *buffer = strdupz(string);
+
+ char *d = buffer;
+ const char *s = string;
+
+ while(*s) {
+ if(unlikely(*s == '\\')) {
+ s++;
+ if(likely(isdigit(*s) && isdigit(s[1]) && isdigit(s[2]))) {
+ char c = *s++ - '0';
+ c <<= 3;
+ c |= *s++ - '0';
+ c <<= 3;
+ c |= *s++ - '0';
+ *d++ = c;
+ }
+ else *d++ = '_';
+ }
+ else *d++ = *s++;
+ }
+ *d = '\0';
+
+ return buffer;
+}
+
+static inline int is_read_only(const char *s) {
+ if(!s) return 0;
+
+ size_t len = strlen(s);
+ if(len < 2) return 0;
+ if(len == 2) {
+ if(!strcmp(s, "ro")) return 1;
+ return 0;
+ }
+ if(!strncmp(s, "ro,", 3)) return 1;
+ if(!strncmp(&s[len - 3], ",ro", 3)) return 1;
+ if(strstr(s, ",ro,")) return 1;
+ return 0;
+}
+
+// for the full list of protected mount points look at
+// https://github.com/systemd/systemd/blob/1eb3ef78b4df28a9e9f464714208f2682f957e36/src/core/namespace.c#L142-L149
+// https://github.com/systemd/systemd/blob/1eb3ef78b4df28a9e9f464714208f2682f957e36/src/core/namespace.c#L180-L194
+static const char *systemd_protected_mount_points[] = {
+ "/home",
+ "/root",
+ "/usr",
+ "/boot",
+ "/efi",
+ "/etc",
+ "/run/user",
+ "/lib",
+ "/lib64",
+ "/bin",
+ "/sbin",
+ NULL
+};
+
+static inline int mount_point_is_protected(char *mount_point)
+{
+ for (size_t i = 0; systemd_protected_mount_points[i] != NULL; i++)
+ if (!strcmp(mount_point, systemd_protected_mount_points[i]))
+ return 1;
+
+ return 0;
+}
+
+// read the whole mountinfo into a linked list
+struct mountinfo *mountinfo_read(int do_statvfs) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/self/mountinfo", netdata_configured_host_prefix);
+ procfile *ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) {
+ snprintfz(filename, FILENAME_MAX, "%s/proc/1/mountinfo", netdata_configured_host_prefix);
+ ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return NULL;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return NULL;
+
+ struct mountinfo *root = NULL, *last = NULL, *mi = NULL;
+
+ // create a dictionary to track uniqueness
+ DICTIONARY *dict = dictionary_create(
+ DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_NAME_LINK_DONT_CLONE);
+
+ unsigned long l, lines = procfile_lines(ff);
+ for(l = 0; l < lines ;l++) {
+ if(unlikely(procfile_linewords(ff, l) < 5))
+ continue;
+
+ // make sure we don't add the same item twice
+ char *v = (char *)dictionary_set(dict, procfile_lineword(ff, l, 4), "N", 2);
+ if(v) {
+ if(*v == 'O') continue;
+ *v = 'O';
+ }
+
+ mi = mallocz(sizeof(struct mountinfo));
+
+ unsigned long w = 0;
+ mi->id = str2ul(procfile_lineword(ff, l, w)); w++;
+ mi->parentid = str2ul(procfile_lineword(ff, l, w)); w++;
+
+ char *major = procfile_lineword(ff, l, w), *minor; w++;
+ for(minor = major; *minor && *minor != ':' ;minor++) ;
+
+ if(unlikely(!*minor)) {
+ error("Cannot parse major:minor on '%s' at line %lu of '%s'", major, l + 1, filename);
+ freez(mi);
+ continue;
+ }
+
+ *minor = '\0';
+ minor++;
+
+ mi->flags = 0;
+
+ mi->major = str2ul(major);
+ mi->minor = str2ul(minor);
+
+ mi->root = strdupz(procfile_lineword(ff, l, w)); w++;
+ mi->root_hash = simple_hash(mi->root);
+
+ mi->mount_point = strdupz_decoding_octal(procfile_lineword(ff, l, w)); w++;
+ mi->mount_point_hash = simple_hash(mi->mount_point);
+
+ mi->persistent_id = strdupz(mi->mount_point);
+ netdata_fix_chart_id(mi->persistent_id);
+ mi->persistent_id_hash = simple_hash(mi->persistent_id);
+
+ mi->mount_options = strdupz(procfile_lineword(ff, l, w)); w++;
+
+ if(unlikely(is_read_only(mi->mount_options)))
+ mi->flags |= MOUNTINFO_READONLY;
+
+ if(unlikely(mount_point_is_protected(mi->mount_point)))
+ mi->flags |= MOUNTINFO_IS_IN_SYSD_PROTECTED_LIST;
+
+ // count the optional fields
+/*
+ unsigned long wo = w;
+*/
+ mi->optional_fields_count = 0;
+ char *s = procfile_lineword(ff, l, w);
+ while(*s && *s != '-') {
+ w++;
+ s = procfile_lineword(ff, l, w);
+ mi->optional_fields_count++;
+ }
+
+/*
+ if(unlikely(mi->optional_fields_count)) {
+ // we have some optional fields
+ // read them into a new array of pointers;
+
+ mi->optional_fields = mallocz(mi->optional_fields_count * sizeof(char *));
+
+ int i;
+ for(i = 0; i < mi->optional_fields_count ; i++) {
+ *mi->optional_fields[wo] = strdupz(procfile_lineword(ff, l, w));
+ wo++;
+ }
+ }
+ else
+ mi->optional_fields = NULL;
+*/
+
+ if(likely(*s == '-')) {
+ w++;
+
+ mi->filesystem = strdupz(procfile_lineword(ff, l, w)); w++;
+ mi->filesystem_hash = simple_hash(mi->filesystem);
+
+ mi->mount_source = strdupz_decoding_octal(procfile_lineword(ff, l, w)); w++;
+ mi->mount_source_hash = simple_hash(mi->mount_source);
+
+ mi->mount_source_name = strdupz(basename(mi->mount_source));
+ mi->mount_source_name_hash = simple_hash(mi->mount_source_name);
+
+ mi->super_options = strdupz(procfile_lineword(ff, l, w)); w++;
+
+ if(unlikely(is_read_only(mi->super_options)))
+ mi->flags |= MOUNTINFO_READONLY;
+
+ if(unlikely(ME_DUMMY(mi->mount_source, mi->filesystem)))
+ mi->flags |= MOUNTINFO_IS_DUMMY;
+
+ if(unlikely(ME_REMOTE(mi->mount_source, mi->filesystem)))
+ mi->flags |= MOUNTINFO_IS_REMOTE;
+
+ // mark as BIND the duplicates (i.e. same filesystem + same source)
+ if(do_statvfs) {
+ struct stat buf;
+ if(unlikely(stat(mi->mount_point, &buf) == -1)) {
+ mi->st_dev = 0;
+ mi->flags |= MOUNTINFO_NO_STAT;
+ }
+ else {
+ mi->st_dev = buf.st_dev;
+
+ struct mountinfo *mt;
+ for(mt = root; mt; mt = mt->next) {
+ if(unlikely(mt->st_dev == mi->st_dev && !(mt->flags & MOUNTINFO_IS_SAME_DEV))) {
+ if(strlen(mi->mount_point) < strlen(mt->mount_point))
+ mt->flags |= MOUNTINFO_IS_SAME_DEV;
+ else
+ mi->flags |= MOUNTINFO_IS_SAME_DEV;
+ }
+ }
+ }
+ }
+ else {
+ mi->st_dev = 0;
+ }
+ }
+ else {
+ mi->filesystem = NULL;
+ mi->filesystem_hash = 0;
+
+ mi->mount_source = NULL;
+ mi->mount_source_hash = 0;
+
+ mi->mount_source_name = NULL;
+ mi->mount_source_name_hash = 0;
+
+ mi->super_options = NULL;
+
+ mi->st_dev = 0;
+ }
+
+ // check if it has size
+ if(do_statvfs && !(mi->flags & MOUNTINFO_IS_DUMMY)) {
+ struct statvfs buff_statvfs;
+ if(unlikely(statvfs(mi->mount_point, &buff_statvfs) < 0)) {
+ mi->flags |= MOUNTINFO_NO_STAT;
+ }
+ else if(unlikely(!buff_statvfs.f_blocks /* || !buff_statvfs.f_files */)) {
+ mi->flags |= MOUNTINFO_NO_SIZE;
+ }
+ }
+
+ // link it
+ if(unlikely(!root))
+ root = mi;
+ else
+ last->next = mi;
+
+ last = mi;
+ mi->next = NULL;
+
+/*
+#ifdef NETDATA_INTERNAL_CHECKS
+ fprintf(stderr, "MOUNTINFO: %ld %ld %lu:%lu root '%s', persistent id '%s', mount point '%s', mount options '%s', filesystem '%s', mount source '%s', super options '%s'%s%s%s%s%s%s\n",
+ mi->id,
+ mi->parentid,
+ mi->major,
+ mi->minor,
+ mi->root,
+ mi->persistent_id,
+ (mi->mount_point)?mi->mount_point:"",
+ (mi->mount_options)?mi->mount_options:"",
+ (mi->filesystem)?mi->filesystem:"",
+ (mi->mount_source)?mi->mount_source:"",
+ (mi->super_options)?mi->super_options:"",
+ (mi->flags & MOUNTINFO_IS_DUMMY)?" DUMMY":"",
+ (mi->flags & MOUNTINFO_IS_BIND)?" BIND":"",
+ (mi->flags & MOUNTINFO_IS_REMOTE)?" REMOTE":"",
+ (mi->flags & MOUNTINFO_NO_STAT)?" NOSTAT":"",
+ (mi->flags & MOUNTINFO_NO_SIZE)?" NOSIZE":"",
+ (mi->flags & MOUNTINFO_IS_SAME_DEV)?" SAMEDEV":""
+ );
+#endif
+*/
+ }
+
+/* find if the mount options have "bind" in them
+ {
+ FILE *fp = setmntent(MOUNTED, "r");
+ if (fp != NULL) {
+ struct mntent mntbuf;
+ struct mntent *mnt;
+ char buf[4096 + 1];
+
+ while ((mnt = getmntent_r(fp, &mntbuf, buf, 4096))) {
+ char *bind = hasmntopt(mnt, "bind");
+ if(unlikely(bind)) {
+ struct mountinfo *mi;
+ for(mi = root; mi ; mi = mi->next) {
+ if(unlikely(strcmp(mnt->mnt_dir, mi->mount_point) == 0)) {
+ fprintf(stderr, "Mount point '%s' is BIND\n", mi->mount_point);
+ mi->flags |= MOUNTINFO_IS_BIND;
+ break;
+ }
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(!mi)) {
+ error("Mount point '%s' not found in /proc/self/mountinfo", mnt->mnt_dir);
+ }
+#endif
+ }
+ }
+ endmntent(fp);
+ }
+ }
+*/
+
+ dictionary_destroy(dict);
+ procfile_close(ff);
+ return root;
+}
diff --git a/collectors/proc.plugin/proc_self_mountinfo.h b/collectors/proc.plugin/proc_self_mountinfo.h
new file mode 100644
index 0000000..4bd24d2
--- /dev/null
+++ b/collectors/proc.plugin/proc_self_mountinfo.h
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PROC_SELF_MOUNTINFO_H
+#define NETDATA_PROC_SELF_MOUNTINFO_H 1
+
+#define MOUNTINFO_IS_DUMMY 0x00000001
+#define MOUNTINFO_IS_REMOTE 0x00000002
+#define MOUNTINFO_IS_BIND 0x00000004
+#define MOUNTINFO_IS_SAME_DEV 0x00000008
+#define MOUNTINFO_NO_STAT 0x00000010
+#define MOUNTINFO_NO_SIZE 0x00000020
+#define MOUNTINFO_READONLY 0x00000040
+#define MOUNTINFO_IS_IN_SYSD_PROTECTED_LIST 0x00000080
+
+struct mountinfo {
+ long id; // mount ID: unique identifier of the mount (may be reused after umount(2)).
+ long parentid; // parent ID: ID of parent mount (or of self for the top of the mount tree).
+ unsigned long major; // major:minor: value of st_dev for files on filesystem (see stat(2)).
+ unsigned long minor;
+
+ char *persistent_id; // a calculated persistent id for the mount point
+ uint32_t persistent_id_hash;
+
+ char *root; // root: root of the mount within the filesystem.
+ uint32_t root_hash;
+
+ char *mount_point; // mount point: mount point relative to the process's root.
+ uint32_t mount_point_hash;
+
+ char *mount_options; // mount options: per-mount options.
+
+ int optional_fields_count;
+/*
+ char ***optional_fields; // optional fields: zero or more fields of the form "tag[:value]".
+*/
+ char *filesystem; // filesystem type: name of filesystem in the form "type[.subtype]".
+ uint32_t filesystem_hash;
+
+ char *mount_source; // mount source: filesystem-specific information or "none".
+ uint32_t mount_source_hash;
+
+ char *mount_source_name;
+ uint32_t mount_source_name_hash;
+
+ char *super_options; // super options: per-superblock options.
+
+ uint32_t flags;
+
+ dev_t st_dev; // id of device as given by stat()
+
+ struct mountinfo *next;
+};
+
+struct mountinfo *mountinfo_find(struct mountinfo *root, unsigned long major, unsigned long minor, char *device);
+struct mountinfo *mountinfo_find_by_filesystem_mount_source(struct mountinfo *root, const char *filesystem, const char *mount_source);
+struct mountinfo *mountinfo_find_by_filesystem_super_option(struct mountinfo *root, const char *filesystem, const char *super_options);
+
+void mountinfo_free_all(struct mountinfo *mi);
+struct mountinfo *mountinfo_read(int do_statvfs);
+
+#endif /* NETDATA_PROC_SELF_MOUNTINFO_H */
diff --git a/collectors/proc.plugin/proc_softirqs.c b/collectors/proc.plugin/proc_softirqs.c
new file mode 100644
index 0000000..4c4df76
--- /dev/null
+++ b/collectors/proc.plugin/proc_softirqs.c
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_SOFTIRQS_NAME "/proc/softirqs"
+
+#define MAX_INTERRUPT_NAME 50
+
+struct cpu_interrupt {
+ unsigned long long value;
+ RRDDIM *rd;
+};
+
+struct interrupt {
+ int used;
+ char *id;
+ char name[MAX_INTERRUPT_NAME + 1];
+ RRDDIM *rd;
+ unsigned long long total;
+ struct cpu_interrupt cpu[];
+};
+
+// since each interrupt is variable in size
+// we use this to calculate its record size
+#define recordsize(cpus) (sizeof(struct interrupt) + ((cpus) * sizeof(struct cpu_interrupt)))
+
+// given a base, get a pointer to each record
+#define irrindex(base, line, cpus) ((struct interrupt *)&((char *)(base))[(line) * recordsize(cpus)])
+
+static inline struct interrupt *get_interrupts_array(size_t lines, int cpus) {
+ static struct interrupt *irrs = NULL;
+ static size_t allocated = 0;
+
+ if(unlikely(lines != allocated)) {
+ uint32_t l;
+ int c;
+
+ irrs = (struct interrupt *)reallocz(irrs, lines * recordsize(cpus));
+
+ // reset all interrupt RRDDIM pointers as any line could have shifted
+ for(l = 0; l < lines ;l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+ irr->rd = NULL;
+ irr->name[0] = '\0';
+ for(c = 0; c < cpus ;c++)
+ irr->cpu[c].rd = NULL;
+ }
+
+ allocated = lines;
+ }
+
+ return irrs;
+}
+
+int do_proc_softirqs(int update_every, usec_t dt) {
+ (void)dt;
+ static procfile *ff = NULL;
+ static int cpus = -1, do_per_core = CONFIG_BOOLEAN_INVALID;
+ struct interrupt *irrs = NULL;
+
+ if(unlikely(do_per_core == CONFIG_BOOLEAN_INVALID))
+ do_per_core = config_get_boolean_ondemand("plugin:proc:/proc/softirqs", "interrupts per core", CONFIG_BOOLEAN_AUTO);
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/softirqs");
+ ff = procfile_open(config_get("plugin:proc:/proc/softirqs", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+ size_t words = procfile_linewords(ff, 0);
+
+ if(unlikely(!lines)) {
+ error("Cannot read /proc/softirqs, zero lines reported.");
+ return 1;
+ }
+
+ // find how many CPUs are there
+ if(unlikely(cpus == -1)) {
+ uint32_t w;
+ cpus = 0;
+ for(w = 0; w < words ; w++) {
+ if(likely(strncmp(procfile_lineword(ff, 0, w), "CPU", 3) == 0))
+ cpus++;
+ }
+ }
+
+ if(unlikely(!cpus)) {
+ error("PLUGIN: PROC_SOFTIRQS: Cannot find the number of CPUs in /proc/softirqs");
+ return 1;
+ }
+
+ // allocate the size we need;
+ irrs = get_interrupts_array(lines, cpus);
+ irrs[0].used = 0;
+
+ // loop through all lines
+ for(l = 1; l < lines ;l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+ irr->used = 0;
+ irr->total = 0;
+
+ words = procfile_linewords(ff, l);
+ if(unlikely(!words)) continue;
+
+ irr->id = procfile_lineword(ff, l, 0);
+ if(unlikely(!irr->id || !irr->id[0])) continue;
+
+ int c;
+ for(c = 0; c < cpus ;c++) {
+ if(likely((c + 1) < (int)words))
+ irr->cpu[c].value = str2ull(procfile_lineword(ff, l, (uint32_t)(c + 1)));
+ else
+ irr->cpu[c].value = 0;
+
+ irr->total += irr->cpu[c].value;
+ }
+
+ strncpyz(irr->name, irr->id, MAX_INTERRUPT_NAME);
+
+ irr->used = 1;
+ }
+
+ // --------------------------------------------------------------------
+
+ static RRDSET *st_system_softirqs = NULL;
+ if(unlikely(!st_system_softirqs)) {
+ st_system_softirqs = rrdset_create_localhost(
+ "system"
+ , "softirqs"
+ , NULL
+ , "softirqs"
+ , NULL
+ , "System softirqs"
+ , "softirqs/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_SOFTIRQS_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_SOFTIRQS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+ }
+
+ for(l = 0; l < lines ;l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+
+ if(irr->used && irr->total) {
+ // some interrupt may have changed without changing the total number of lines
+ // if the same number of interrupts have been added and removed between two
+ // calls of this function.
+ if(unlikely(!irr->rd || strncmp(irr->name, rrddim_name(irr->rd), MAX_INTERRUPT_NAME) != 0)) {
+ irr->rd = rrddim_add(st_system_softirqs, irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_reset_name(st_system_softirqs, irr->rd, irr->name);
+
+ // also reset per cpu RRDDIMs to avoid repeating strncmp() in the per core loop
+ if(likely(do_per_core != CONFIG_BOOLEAN_NO)) {
+ int c;
+ for(c = 0; c < cpus; c++) irr->cpu[c].rd = NULL;
+ }
+ }
+
+ rrddim_set_by_pointer(st_system_softirqs, irr->rd, irr->total);
+ }
+ }
+
+ rrdset_done(st_system_softirqs);
+
+ // --------------------------------------------------------------------
+
+ if(do_per_core != CONFIG_BOOLEAN_NO) {
+ static RRDSET **core_st = NULL;
+ static int old_cpus = 0;
+
+ if(old_cpus < cpus) {
+ core_st = reallocz(core_st, sizeof(RRDSET *) * cpus);
+ memset(&core_st[old_cpus], 0, sizeof(RRDSET *) * (cpus - old_cpus));
+ old_cpus = cpus;
+ }
+
+ int c;
+
+ for(c = 0; c < cpus ; c++) {
+ if(unlikely(!core_st[c])) {
+ // find if everything is just zero
+ unsigned long long core_sum = 0;
+
+ for (l = 0; l < lines; l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+ if (unlikely(!irr->used)) continue;
+ core_sum += irr->cpu[c].value;
+ }
+
+ if (unlikely(core_sum == 0)) continue; // try next core
+
+ char id[50 + 1];
+ snprintfz(id, 50, "cpu%d_softirqs", c);
+
+ char title[100 + 1];
+ snprintfz(title, 100, "CPU softirqs");
+
+ core_st[c] = rrdset_create_localhost(
+ "cpu"
+ , id
+ , NULL
+ , "softirqs"
+ , "cpu.softirqs"
+ , title
+ , "softirqs/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_SOFTIRQS_NAME
+ , NETDATA_CHART_PRIO_SOFTIRQS_PER_CORE + c
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ char core[50+1];
+ snprintfz(core, 50, "cpu%d", c);
+ rrdlabels_add(core_st[c]->rrdlabels, "cpu", core, RRDLABEL_SRC_AUTO);
+ }
+
+ for(l = 0; l < lines ;l++) {
+ struct interrupt *irr = irrindex(irrs, l, cpus);
+
+ if(irr->used && (do_per_core == CONFIG_BOOLEAN_YES || irr->cpu[c].value)) {
+ if(unlikely(!irr->cpu[c].rd)) {
+ irr->cpu[c].rd = rrddim_add(core_st[c], irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_reset_name(core_st[c], irr->cpu[c].rd, irr->name);
+ }
+
+ rrddim_set_by_pointer(core_st[c], irr->cpu[c].rd, irr->cpu[c].value);
+ }
+ }
+
+ rrdset_done(core_st[c]);
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_spl_kstat_zfs.c b/collectors/proc.plugin/proc_spl_kstat_zfs.c
new file mode 100644
index 0000000..8938d64
--- /dev/null
+++ b/collectors/proc.plugin/proc_spl_kstat_zfs.c
@@ -0,0 +1,423 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+#include "zfs_common.h"
+
+#define ZFS_PROC_ARCSTATS "/proc/spl/kstat/zfs/arcstats"
+#define ZFS_PROC_POOLS "/proc/spl/kstat/zfs"
+
+#define STATE_SIZE 9
+#define MAX_CHART_ID 256
+
+extern struct arcstats arcstats;
+
+unsigned long long zfs_arcstats_shrinkable_cache_size_bytes = 0;
+
+int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt) {
+ (void)dt;
+
+ static int show_zero_charts = 0, do_zfs_stats = 0;
+ static procfile *ff = NULL;
+ static char *dirname = NULL;
+ static ARL_BASE *arl_base = NULL;
+
+ arcstats.l2exist = -1;
+
+ if(unlikely(!arl_base)) {
+ arl_base = arl_create("arcstats", NULL, 60);
+
+ arl_expect(arl_base, "hits", &arcstats.hits);
+ arl_expect(arl_base, "misses", &arcstats.misses);
+ arl_expect(arl_base, "demand_data_hits", &arcstats.demand_data_hits);
+ arl_expect(arl_base, "demand_data_misses", &arcstats.demand_data_misses);
+ arl_expect(arl_base, "demand_metadata_hits", &arcstats.demand_metadata_hits);
+ arl_expect(arl_base, "demand_metadata_misses", &arcstats.demand_metadata_misses);
+ arl_expect(arl_base, "prefetch_data_hits", &arcstats.prefetch_data_hits);
+ arl_expect(arl_base, "prefetch_data_misses", &arcstats.prefetch_data_misses);
+ arl_expect(arl_base, "prefetch_metadata_hits", &arcstats.prefetch_metadata_hits);
+ arl_expect(arl_base, "prefetch_metadata_misses", &arcstats.prefetch_metadata_misses);
+ arl_expect(arl_base, "mru_hits", &arcstats.mru_hits);
+ arl_expect(arl_base, "mru_ghost_hits", &arcstats.mru_ghost_hits);
+ arl_expect(arl_base, "mfu_hits", &arcstats.mfu_hits);
+ arl_expect(arl_base, "mfu_ghost_hits", &arcstats.mfu_ghost_hits);
+ arl_expect(arl_base, "deleted", &arcstats.deleted);
+ arl_expect(arl_base, "mutex_miss", &arcstats.mutex_miss);
+ arl_expect(arl_base, "evict_skip", &arcstats.evict_skip);
+ arl_expect(arl_base, "evict_not_enough", &arcstats.evict_not_enough);
+ arl_expect(arl_base, "evict_l2_cached", &arcstats.evict_l2_cached);
+ arl_expect(arl_base, "evict_l2_eligible", &arcstats.evict_l2_eligible);
+ arl_expect(arl_base, "evict_l2_ineligible", &arcstats.evict_l2_ineligible);
+ arl_expect(arl_base, "evict_l2_skip", &arcstats.evict_l2_skip);
+ arl_expect(arl_base, "hash_elements", &arcstats.hash_elements);
+ arl_expect(arl_base, "hash_elements_max", &arcstats.hash_elements_max);
+ arl_expect(arl_base, "hash_collisions", &arcstats.hash_collisions);
+ arl_expect(arl_base, "hash_chains", &arcstats.hash_chains);
+ arl_expect(arl_base, "hash_chain_max", &arcstats.hash_chain_max);
+ arl_expect(arl_base, "p", &arcstats.p);
+ arl_expect(arl_base, "c", &arcstats.c);
+ arl_expect(arl_base, "c_min", &arcstats.c_min);
+ arl_expect(arl_base, "c_max", &arcstats.c_max);
+ arl_expect(arl_base, "size", &arcstats.size);
+ arl_expect(arl_base, "hdr_size", &arcstats.hdr_size);
+ arl_expect(arl_base, "data_size", &arcstats.data_size);
+ arl_expect(arl_base, "metadata_size", &arcstats.metadata_size);
+ arl_expect(arl_base, "other_size", &arcstats.other_size);
+ arl_expect(arl_base, "anon_size", &arcstats.anon_size);
+ arl_expect(arl_base, "anon_evictable_data", &arcstats.anon_evictable_data);
+ arl_expect(arl_base, "anon_evictable_metadata", &arcstats.anon_evictable_metadata);
+ arl_expect(arl_base, "mru_size", &arcstats.mru_size);
+ arl_expect(arl_base, "mru_evictable_data", &arcstats.mru_evictable_data);
+ arl_expect(arl_base, "mru_evictable_metadata", &arcstats.mru_evictable_metadata);
+ arl_expect(arl_base, "mru_ghost_size", &arcstats.mru_ghost_size);
+ arl_expect(arl_base, "mru_ghost_evictable_data", &arcstats.mru_ghost_evictable_data);
+ arl_expect(arl_base, "mru_ghost_evictable_metadata", &arcstats.mru_ghost_evictable_metadata);
+ arl_expect(arl_base, "mfu_size", &arcstats.mfu_size);
+ arl_expect(arl_base, "mfu_evictable_data", &arcstats.mfu_evictable_data);
+ arl_expect(arl_base, "mfu_evictable_metadata", &arcstats.mfu_evictable_metadata);
+ arl_expect(arl_base, "mfu_ghost_size", &arcstats.mfu_ghost_size);
+ arl_expect(arl_base, "mfu_ghost_evictable_data", &arcstats.mfu_ghost_evictable_data);
+ arl_expect(arl_base, "mfu_ghost_evictable_metadata", &arcstats.mfu_ghost_evictable_metadata);
+ arl_expect(arl_base, "l2_hits", &arcstats.l2_hits);
+ arl_expect(arl_base, "l2_misses", &arcstats.l2_misses);
+ arl_expect(arl_base, "l2_feeds", &arcstats.l2_feeds);
+ arl_expect(arl_base, "l2_rw_clash", &arcstats.l2_rw_clash);
+ arl_expect(arl_base, "l2_read_bytes", &arcstats.l2_read_bytes);
+ arl_expect(arl_base, "l2_write_bytes", &arcstats.l2_write_bytes);
+ arl_expect(arl_base, "l2_writes_sent", &arcstats.l2_writes_sent);
+ arl_expect(arl_base, "l2_writes_done", &arcstats.l2_writes_done);
+ arl_expect(arl_base, "l2_writes_error", &arcstats.l2_writes_error);
+ arl_expect(arl_base, "l2_writes_lock_retry", &arcstats.l2_writes_lock_retry);
+ arl_expect(arl_base, "l2_evict_lock_retry", &arcstats.l2_evict_lock_retry);
+ arl_expect(arl_base, "l2_evict_reading", &arcstats.l2_evict_reading);
+ arl_expect(arl_base, "l2_evict_l1cached", &arcstats.l2_evict_l1cached);
+ arl_expect(arl_base, "l2_free_on_write", &arcstats.l2_free_on_write);
+ arl_expect(arl_base, "l2_cdata_free_on_write", &arcstats.l2_cdata_free_on_write);
+ arl_expect(arl_base, "l2_abort_lowmem", &arcstats.l2_abort_lowmem);
+ arl_expect(arl_base, "l2_cksum_bad", &arcstats.l2_cksum_bad);
+ arl_expect(arl_base, "l2_io_error", &arcstats.l2_io_error);
+ arl_expect(arl_base, "l2_size", &arcstats.l2_size);
+ arl_expect(arl_base, "l2_asize", &arcstats.l2_asize);
+ arl_expect(arl_base, "l2_hdr_size", &arcstats.l2_hdr_size);
+ arl_expect(arl_base, "l2_compress_successes", &arcstats.l2_compress_successes);
+ arl_expect(arl_base, "l2_compress_zeros", &arcstats.l2_compress_zeros);
+ arl_expect(arl_base, "l2_compress_failures", &arcstats.l2_compress_failures);
+ arl_expect(arl_base, "memory_throttle_count", &arcstats.memory_throttle_count);
+ arl_expect(arl_base, "duplicate_buffers", &arcstats.duplicate_buffers);
+ arl_expect(arl_base, "duplicate_buffers_size", &arcstats.duplicate_buffers_size);
+ arl_expect(arl_base, "duplicate_reads", &arcstats.duplicate_reads);
+ arl_expect(arl_base, "memory_direct_count", &arcstats.memory_direct_count);
+ arl_expect(arl_base, "memory_indirect_count", &arcstats.memory_indirect_count);
+ arl_expect(arl_base, "arc_no_grow", &arcstats.arc_no_grow);
+ arl_expect(arl_base, "arc_tempreserve", &arcstats.arc_tempreserve);
+ arl_expect(arl_base, "arc_loaned_bytes", &arcstats.arc_loaned_bytes);
+ arl_expect(arl_base, "arc_prune", &arcstats.arc_prune);
+ arl_expect(arl_base, "arc_meta_used", &arcstats.arc_meta_used);
+ arl_expect(arl_base, "arc_meta_limit", &arcstats.arc_meta_limit);
+ arl_expect(arl_base, "arc_meta_max", &arcstats.arc_meta_max);
+ arl_expect(arl_base, "arc_meta_min", &arcstats.arc_meta_min);
+ arl_expect(arl_base, "arc_need_free", &arcstats.arc_need_free);
+ arl_expect(arl_base, "arc_sys_free", &arcstats.arc_sys_free);
+ }
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, ZFS_PROC_ARCSTATS);
+ ff = procfile_open(config_get("plugin:proc:" ZFS_PROC_ARCSTATS, "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff))
+ return 1;
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/spl/kstat/zfs");
+ dirname = config_get("plugin:proc:" ZFS_PROC_ARCSTATS, "directory to monitor", filename);
+
+ show_zero_charts = config_get_boolean_ondemand("plugin:proc:" ZFS_PROC_ARCSTATS, "show zero charts", CONFIG_BOOLEAN_NO);
+ if(show_zero_charts == CONFIG_BOOLEAN_AUTO && netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)
+ show_zero_charts = CONFIG_BOOLEAN_YES;
+ if(unlikely(show_zero_charts == CONFIG_BOOLEAN_YES))
+ do_zfs_stats = 1;
+ }
+
+ // check if any pools exist
+ if(likely(!do_zfs_stats)) {
+ DIR *dir = opendir(dirname);
+ if(unlikely(!dir)) {
+ error("Cannot read directory '%s'", dirname);
+ return 1;
+ }
+
+ struct dirent *de = NULL;
+ while(likely(de = readdir(dir))) {
+ if(likely(de->d_type == DT_DIR
+ && (
+ (de->d_name[0] == '.' && de->d_name[1] == '\0')
+ || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ )))
+ continue;
+
+ if(unlikely(de->d_type == DT_LNK || de->d_type == DT_DIR)) {
+ do_zfs_stats = 1;
+ break;
+ }
+ }
+
+ closedir(dir);
+ }
+
+ // do not show ZFS filesystem metrics if there haven't been any pools in the system yet
+ if(unlikely(!do_zfs_stats))
+ return 0;
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ arl_begin(arl_base);
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 3)) {
+ if(unlikely(words)) error("Cannot read " ZFS_PROC_ARCSTATS " line %zu. Expected 3 params, read %zu.", l, words);
+ continue;
+ }
+
+ const char *key = procfile_lineword(ff, l, 0);
+ const char *value = procfile_lineword(ff, l, 2);
+
+ if(unlikely(arcstats.l2exist == -1)) {
+ if(key[0] == 'l' && key[1] == '2' && key[2] == '_')
+ arcstats.l2exist = 1;
+ }
+
+ if(unlikely(arl_check(arl_base, key, value))) break;
+ }
+
+ if (arcstats.size > arcstats.c_min) {
+ zfs_arcstats_shrinkable_cache_size_bytes = arcstats.size - arcstats.c_min;
+ } else {
+ zfs_arcstats_shrinkable_cache_size_bytes = 0;
+ }
+
+ if(unlikely(arcstats.l2exist == -1))
+ arcstats.l2exist = 0;
+
+ generate_charts_arcstats(PLUGIN_PROC_NAME, ZFS_PROC_ARCSTATS, show_zero_charts, update_every);
+ generate_charts_arc_summary(PLUGIN_PROC_NAME, ZFS_PROC_ARCSTATS, show_zero_charts, update_every);
+
+ return 0;
+}
+
+struct zfs_pool {
+ RRDSET *st;
+
+ RRDDIM *rd_online;
+ RRDDIM *rd_degraded;
+ RRDDIM *rd_faulted;
+ RRDDIM *rd_offline;
+ RRDDIM *rd_removed;
+ RRDDIM *rd_unavail;
+
+ int updated;
+ int disabled;
+
+ int online;
+ int degraded;
+ int faulted;
+ int offline;
+ int removed;
+ int unavail;
+};
+
+struct deleted_zfs_pool {
+ char *name;
+ struct deleted_zfs_pool *next;
+} *deleted_zfs_pools = NULL;
+
+DICTIONARY *zfs_pools = NULL;
+
+void disable_zfs_pool_state(struct zfs_pool *pool)
+{
+ if (pool->st)
+ rrdset_is_obsolete(pool->st);
+
+ pool->st = NULL;
+
+ pool->rd_online = NULL;
+ pool->rd_degraded = NULL;
+ pool->rd_faulted = NULL;
+ pool->rd_offline = NULL;
+ pool->rd_removed = NULL;
+ pool->rd_unavail = NULL;
+
+ pool->disabled = 1;
+}
+
+int update_zfs_pool_state_chart(const DICTIONARY_ITEM *item, void *pool_p, void *update_every_p) {
+ const char *name = dictionary_acquired_item_name(item);
+ struct zfs_pool *pool = (struct zfs_pool *)pool_p;
+ int update_every = *(int *)update_every_p;
+
+ if (pool->updated) {
+ pool->updated = 0;
+
+ if (!pool->disabled) {
+ if (unlikely(!pool->st)) {
+ char chart_id[MAX_CHART_ID + 1];
+ snprintf(chart_id, MAX_CHART_ID, "state_%s", name);
+
+ pool->st = rrdset_create_localhost(
+ "zfspool",
+ chart_id,
+ NULL,
+ name,
+ "zfspool.state",
+ "ZFS pool state",
+ "boolean",
+ PLUGIN_PROC_NAME,
+ ZFS_PROC_POOLS,
+ NETDATA_CHART_PRIO_ZFS_POOL_STATE,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ pool->rd_online = rrddim_add(pool->st, "online", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ pool->rd_degraded = rrddim_add(pool->st, "degraded", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ pool->rd_faulted = rrddim_add(pool->st, "faulted", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ pool->rd_offline = rrddim_add(pool->st, "offline", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ pool->rd_removed = rrddim_add(pool->st, "removed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ pool->rd_unavail = rrddim_add(pool->st, "unavail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(pool->st, pool->rd_online, pool->online);
+ rrddim_set_by_pointer(pool->st, pool->rd_degraded, pool->degraded);
+ rrddim_set_by_pointer(pool->st, pool->rd_faulted, pool->faulted);
+ rrddim_set_by_pointer(pool->st, pool->rd_offline, pool->offline);
+ rrddim_set_by_pointer(pool->st, pool->rd_removed, pool->removed);
+ rrddim_set_by_pointer(pool->st, pool->rd_unavail, pool->unavail);
+ rrdset_done(pool->st);
+ }
+ } else {
+ disable_zfs_pool_state(pool);
+ struct deleted_zfs_pool *new = callocz(1, sizeof(struct deleted_zfs_pool));
+ new->name = strdupz(name);
+ new->next = deleted_zfs_pools;
+ deleted_zfs_pools = new;
+ }
+
+ return 0;
+}
+
+int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt)
+{
+ (void)dt;
+
+ static int do_zfs_pool_state = -1;
+ static char *dirname = NULL;
+
+ int pool_found = 0, state_file_found = 0;
+
+ if (unlikely(do_zfs_pool_state == -1)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/spl/kstat/zfs");
+ dirname = config_get("plugin:proc:" ZFS_PROC_POOLS, "directory to monitor", filename);
+
+ zfs_pools = dictionary_create(DICT_OPTION_SINGLE_THREADED);
+
+ do_zfs_pool_state = 1;
+ }
+
+ if (likely(do_zfs_pool_state)) {
+ DIR *dir = opendir(dirname);
+ if (unlikely(!dir)) {
+ error("Cannot read directory '%s'", dirname);
+ return 1;
+ }
+
+ struct dirent *de = NULL;
+ while (likely(de = readdir(dir))) {
+ if (likely(
+ de->d_type == DT_DIR && ((de->d_name[0] == '.' && de->d_name[1] == '\0') ||
+ (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0'))))
+ continue;
+
+ if (unlikely(de->d_type == DT_LNK || de->d_type == DT_DIR)) {
+ pool_found = 1;
+
+ struct zfs_pool *pool = dictionary_get(zfs_pools, de->d_name);
+
+ if (unlikely(!pool)) {
+ struct zfs_pool new_zfs_pool = {};
+ pool = dictionary_set(zfs_pools, de->d_name, &new_zfs_pool, sizeof(struct zfs_pool));
+ };
+
+ pool->updated = 1;
+
+ if (pool->disabled) {
+ state_file_found = 1;
+ continue;
+ }
+
+ pool->online = 0;
+ pool->degraded = 0;
+ pool->faulted = 0;
+ pool->offline = 0;
+ pool->removed = 0;
+ pool->unavail = 0;
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(
+ filename, FILENAME_MAX, "%s%s/%s/state", netdata_configured_host_prefix, dirname, de->d_name);
+
+ char state[STATE_SIZE + 1];
+ int ret = read_file(filename, state, STATE_SIZE);
+
+ if (!ret) {
+ state_file_found = 1;
+
+ // ZFS pool states are described at https://openzfs.github.io/openzfs-docs/man/8/zpoolconcepts.8.html?#Device_Failure_and_Recovery
+ if (!strcmp(state, "ONLINE\n")) {
+ pool->online = 1;
+ } else if (!strcmp(state, "DEGRADED\n")) {
+ pool->degraded = 1;
+ } else if (!strcmp(state, "FAULTED\n")) {
+ pool->faulted = 1;
+ } else if (!strcmp(state, "OFFLINE\n")) {
+ pool->offline = 1;
+ } else if (!strcmp(state, "REMOVED\n")) {
+ pool->removed = 1;
+ } else if (!strcmp(state, "UNAVAIL\n")) {
+ pool->unavail = 1;
+ } else {
+ disable_zfs_pool_state(pool);
+
+ char *c = strchr(state, '\n');
+ if (c)
+ *c = '\0';
+ error("ZFS POOLS: Undefined state %s for zpool %s, disabling the chart", state, de->d_name);
+ }
+ }
+ }
+ }
+
+ closedir(dir);
+ }
+
+ if (do_zfs_pool_state && pool_found && !state_file_found) {
+ info("ZFS POOLS: State files not found. Disabling the module.");
+ do_zfs_pool_state = 0;
+ }
+
+ if (do_zfs_pool_state)
+ dictionary_walkthrough_read(zfs_pools, update_zfs_pool_state_chart, &update_every);
+
+ while (deleted_zfs_pools) {
+ struct deleted_zfs_pool *current_pool = deleted_zfs_pools;
+ dictionary_del(zfs_pools, current_pool->name);
+
+ deleted_zfs_pools = deleted_zfs_pools->next;
+
+ freez(current_pool->name);
+ freez(current_pool);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_stat.c b/collectors/proc.plugin/proc_stat.c
new file mode 100644
index 0000000..33fe932
--- /dev/null
+++ b/collectors/proc.plugin/proc_stat.c
@@ -0,0 +1,1064 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_STAT_NAME "/proc/stat"
+
+struct per_core_single_number_file {
+ unsigned char found:1;
+ const char *filename;
+ int fd;
+ collected_number value;
+ RRDDIM *rd;
+};
+
+struct last_ticks {
+ collected_number frequency;
+ collected_number ticks;
+};
+
+// This is an extension of struct per_core_single_number_file at CPU_FREQ_INDEX.
+// Either scaling_cur_freq or time_in_state file is used at one time.
+struct per_core_time_in_state_file {
+ const char *filename;
+ procfile *ff;
+ size_t last_ticks_len;
+ struct last_ticks *last_ticks;
+};
+
+#define CORE_THROTTLE_COUNT_INDEX 0
+#define PACKAGE_THROTTLE_COUNT_INDEX 1
+#define CPU_FREQ_INDEX 2
+#define PER_CORE_FILES 3
+
+struct cpu_chart {
+ const char *id;
+
+ RRDSET *st;
+ RRDDIM *rd_user;
+ RRDDIM *rd_nice;
+ RRDDIM *rd_system;
+ RRDDIM *rd_idle;
+ RRDDIM *rd_iowait;
+ RRDDIM *rd_irq;
+ RRDDIM *rd_softirq;
+ RRDDIM *rd_steal;
+ RRDDIM *rd_guest;
+ RRDDIM *rd_guest_nice;
+
+ struct per_core_single_number_file files[PER_CORE_FILES];
+
+ struct per_core_time_in_state_file time_in_state_files;
+};
+
+static int keep_per_core_fds_open = CONFIG_BOOLEAN_YES;
+static int keep_cpuidle_fds_open = CONFIG_BOOLEAN_YES;
+
+static int read_per_core_files(struct cpu_chart *all_cpu_charts, size_t len, size_t index) {
+ char buf[50 + 1];
+ size_t x, files_read = 0, files_nonzero = 0;
+
+ for(x = 0; x < len ; x++) {
+ struct per_core_single_number_file *f = &all_cpu_charts[x].files[index];
+
+ f->found = 0;
+
+ if(unlikely(!f->filename))
+ continue;
+
+ if(unlikely(f->fd == -1)) {
+ f->fd = open(f->filename, O_RDONLY);
+ if (unlikely(f->fd == -1)) {
+ error("Cannot open file '%s'", f->filename);
+ continue;
+ }
+ }
+
+ ssize_t ret = read(f->fd, buf, 50);
+ if(unlikely(ret < 0)) {
+ // cannot read that file
+
+ error("Cannot read file '%s'", f->filename);
+ close(f->fd);
+ f->fd = -1;
+ continue;
+ }
+ else {
+ // successful read
+
+ // terminate the buffer
+ buf[ret] = '\0';
+
+ if(unlikely(keep_per_core_fds_open != CONFIG_BOOLEAN_YES)) {
+ close(f->fd);
+ f->fd = -1;
+ }
+ else if(lseek(f->fd, 0, SEEK_SET) == -1) {
+ error("Cannot seek in file '%s'", f->filename);
+ close(f->fd);
+ f->fd = -1;
+ }
+ }
+
+ files_read++;
+ f->found = 1;
+
+ f->value = str2ll(buf, NULL);
+ if(likely(f->value != 0))
+ files_nonzero++;
+ }
+
+ if(files_read == 0)
+ return -1;
+
+ if(files_nonzero == 0)
+ return 0;
+
+ return (int)files_nonzero;
+}
+
+static int read_per_core_time_in_state_files(struct cpu_chart *all_cpu_charts, size_t len, size_t index) {
+ size_t x, files_read = 0, files_nonzero = 0;
+
+ for(x = 0; x < len ; x++) {
+ struct per_core_single_number_file *f = &all_cpu_charts[x].files[index];
+ struct per_core_time_in_state_file *tsf = &all_cpu_charts[x].time_in_state_files;
+
+ f->found = 0;
+
+ if(unlikely(!tsf->filename))
+ continue;
+
+ if(unlikely(!tsf->ff)) {
+ tsf->ff = procfile_open(tsf->filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!tsf->ff))
+ {
+ error("Cannot open file '%s'", tsf->filename);
+ continue;
+ }
+ }
+
+ tsf->ff = procfile_readall(tsf->ff);
+ if(unlikely(!tsf->ff)) {
+ error("Cannot read file '%s'", tsf->filename);
+ procfile_close(tsf->ff);
+ tsf->ff = NULL;
+ continue;
+ }
+ else {
+ // successful read
+
+ size_t lines = procfile_lines(tsf->ff), l;
+ size_t words;
+ unsigned long long total_ticks_since_last = 0, avg_freq = 0;
+
+ // Check if there is at least one frequency in time_in_state
+ if (procfile_word(tsf->ff, 0)[0] == '\0') {
+ if(unlikely(keep_per_core_fds_open != CONFIG_BOOLEAN_YES)) {
+ procfile_close(tsf->ff);
+ tsf->ff = NULL;
+ }
+ // TODO: Is there a better way to avoid spikes than calculating the average over
+ // the whole period under schedutil governor?
+ // freez(tsf->last_ticks);
+ // tsf->last_ticks = NULL;
+ // tsf->last_ticks_len = 0;
+ continue;
+ }
+
+ if (unlikely(tsf->last_ticks_len < lines || tsf->last_ticks == NULL)) {
+ tsf->last_ticks = reallocz(tsf->last_ticks, sizeof(struct last_ticks) * lines);
+ memset(tsf->last_ticks, 0, sizeof(struct last_ticks) * lines);
+ tsf->last_ticks_len = lines;
+ }
+
+ f->value = 0;
+
+ for(l = 0; l < lines - 1 ;l++) {
+ unsigned long long frequency = 0, ticks = 0, ticks_since_last = 0;
+
+ words = procfile_linewords(tsf->ff, l);
+ if(unlikely(words < 2)) {
+ error("Cannot read time_in_state line. Expected 2 params, read %zu.", words);
+ continue;
+ }
+ frequency = str2ull(procfile_lineword(tsf->ff, l, 0));
+ ticks = str2ull(procfile_lineword(tsf->ff, l, 1));
+
+ // It is assumed that frequencies are static and sorted
+ ticks_since_last = ticks - tsf->last_ticks[l].ticks;
+ tsf->last_ticks[l].frequency = frequency;
+ tsf->last_ticks[l].ticks = ticks;
+
+ total_ticks_since_last += ticks_since_last;
+ avg_freq += frequency * ticks_since_last;
+
+ }
+
+ if (likely(total_ticks_since_last)) {
+ avg_freq /= total_ticks_since_last;
+ f->value = avg_freq;
+ }
+
+ if(unlikely(keep_per_core_fds_open != CONFIG_BOOLEAN_YES)) {
+ procfile_close(tsf->ff);
+ tsf->ff = NULL;
+ }
+ }
+
+ files_read++;
+
+ f->found = 1;
+
+ if(likely(f->value != 0))
+ files_nonzero++;
+ }
+
+ if(unlikely(files_read == 0))
+ return -1;
+
+ if(unlikely(files_nonzero == 0))
+ return 0;
+
+ return (int)files_nonzero;
+}
+
+static void chart_per_core_files(struct cpu_chart *all_cpu_charts, size_t len, size_t index, RRDSET *st, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm) {
+ size_t x;
+ for(x = 0; x < len ; x++) {
+ struct per_core_single_number_file *f = &all_cpu_charts[x].files[index];
+
+ if(unlikely(!f->found))
+ continue;
+
+ if(unlikely(!f->rd))
+ f->rd = rrddim_add(st, all_cpu_charts[x].id, NULL, multiplier, divisor, algorithm);
+
+ rrddim_set_by_pointer(st, f->rd, f->value);
+ }
+}
+
+struct cpuidle_state {
+ char *name;
+
+ char *time_filename;
+ int time_fd;
+
+ collected_number value;
+
+ RRDDIM *rd;
+};
+
+struct per_core_cpuidle_chart {
+ RRDSET *st;
+
+ RRDDIM *active_time_rd;
+ collected_number active_time;
+ collected_number last_active_time;
+
+ struct cpuidle_state *cpuidle_state;
+ size_t cpuidle_state_len;
+ int rescan_cpu_states;
+};
+
+static void* wake_cpu_thread(void* core) {
+ pthread_t thread;
+ cpu_set_t cpu_set;
+ static size_t cpu_wakeups = 0;
+ static int errors = 0;
+
+ CPU_ZERO(&cpu_set);
+ CPU_SET(*(int*)core, &cpu_set);
+
+ thread = pthread_self();
+ if(unlikely(pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpu_set))) {
+ if(unlikely(errors < 8)) {
+ error("Cannot set CPU affinity for core %d", *(int*)core);
+ errors++;
+ }
+ else if(unlikely(errors < 9)) {
+ error("CPU affinity errors are disabled");
+ errors++;
+ }
+ }
+
+ // Make the CPU core do something to force it to update its idle counters
+ cpu_wakeups++;
+
+ return 0;
+}
+
+static int read_schedstat(char *schedstat_filename, struct per_core_cpuidle_chart **cpuidle_charts_address, size_t *schedstat_cores_found) {
+ static size_t cpuidle_charts_len = 0;
+ static procfile *ff = NULL;
+ struct per_core_cpuidle_chart *cpuidle_charts = *cpuidle_charts_address;
+ size_t cores_found = 0;
+
+ if(unlikely(!ff)) {
+ ff = procfile_open(schedstat_filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 1;
+
+ size_t lines = procfile_lines(ff), l;
+ size_t words;
+
+ for(l = 0; l < lines ;l++) {
+ char *row_key = procfile_lineword(ff, l, 0);
+
+ // faster strncmp(row_key, "cpu", 3) == 0
+ if(likely(row_key[0] == 'c' && row_key[1] == 'p' && row_key[2] == 'u')) {
+ words = procfile_linewords(ff, l);
+ if(unlikely(words < 10)) {
+ error("Cannot read /proc/schedstat cpu line. Expected 9 params, read %zu.", words);
+ return 1;
+ }
+ cores_found++;
+
+ size_t core = str2ul(&row_key[3]);
+ if(unlikely(core >= cores_found)) {
+ error("Core %zu found but no more than %zu cores were expected.", core, cores_found);
+ return 1;
+ }
+
+ if(unlikely(cpuidle_charts_len < cores_found)) {
+ cpuidle_charts = reallocz(cpuidle_charts, sizeof(struct per_core_cpuidle_chart) * cores_found);
+ *cpuidle_charts_address = cpuidle_charts;
+ memset(cpuidle_charts + cpuidle_charts_len, 0, sizeof(struct per_core_cpuidle_chart) * (cores_found - cpuidle_charts_len));
+ cpuidle_charts_len = cores_found;
+ }
+
+ cpuidle_charts[core].active_time = str2ull(procfile_lineword(ff, l, 7)) / 1000;
+ }
+ }
+
+ *schedstat_cores_found = cores_found;
+ return 0;
+}
+
+static int read_one_state(char *buf, const char *filename, int *fd) {
+ ssize_t ret = read(*fd, buf, 50);
+
+ if(unlikely(ret <= 0)) {
+ // cannot read that file
+ error("Cannot read file '%s'", filename);
+ close(*fd);
+ *fd = -1;
+ return 0;
+ }
+ else {
+ // successful read
+
+ // terminate the buffer
+ buf[ret - 1] = '\0';
+
+ if(unlikely(keep_cpuidle_fds_open != CONFIG_BOOLEAN_YES)) {
+ close(*fd);
+ *fd = -1;
+ }
+ else if(lseek(*fd, 0, SEEK_SET) == -1) {
+ error("Cannot seek in file '%s'", filename);
+ close(*fd);
+ *fd = -1;
+ }
+ }
+
+ return 1;
+}
+
+static int read_cpuidle_states(char *cpuidle_name_filename , char *cpuidle_time_filename, struct per_core_cpuidle_chart *cpuidle_charts, size_t core) {
+ char filename[FILENAME_MAX + 1];
+ static char next_state_filename[FILENAME_MAX + 1];
+ struct stat stbuf;
+ struct per_core_cpuidle_chart *cc = &cpuidle_charts[core];
+ size_t state;
+
+ if(unlikely(!cc->cpuidle_state_len || cc->rescan_cpu_states)) {
+ int state_file_found = 1; // check at least one state
+
+ if(cc->cpuidle_state_len) {
+ for(state = 0; state < cc->cpuidle_state_len; state++) {
+ freez(cc->cpuidle_state[state].name);
+
+ freez(cc->cpuidle_state[state].time_filename);
+ close(cc->cpuidle_state[state].time_fd);
+ cc->cpuidle_state[state].time_fd = -1;
+ }
+
+ freez(cc->cpuidle_state);
+ cc->cpuidle_state = NULL;
+ cc->cpuidle_state_len = 0;
+
+ cc->active_time_rd = NULL;
+ cc->st = NULL;
+ }
+
+ while(likely(state_file_found)) {
+ snprintfz(filename, FILENAME_MAX, cpuidle_name_filename, core, cc->cpuidle_state_len);
+ if (stat(filename, &stbuf) == 0)
+ cc->cpuidle_state_len++;
+ else
+ state_file_found = 0;
+ }
+ snprintfz(next_state_filename, FILENAME_MAX, cpuidle_name_filename, core, cc->cpuidle_state_len);
+
+ if(likely(cc->cpuidle_state_len))
+ cc->cpuidle_state = callocz(cc->cpuidle_state_len, sizeof(struct cpuidle_state));
+
+ for(state = 0; state < cc->cpuidle_state_len; state++) {
+ char name_buf[50 + 1];
+ snprintfz(filename, FILENAME_MAX, cpuidle_name_filename, core, state);
+
+ int fd = open(filename, O_RDONLY, 0666);
+ if(unlikely(fd == -1)) {
+ error("Cannot open file '%s'", filename);
+ cc->rescan_cpu_states = 1;
+ return 1;
+ }
+
+ ssize_t r = read(fd, name_buf, 50);
+ if(unlikely(r < 1)) {
+ error("Cannot read file '%s'", filename);
+ close(fd);
+ cc->rescan_cpu_states = 1;
+ return 1;
+ }
+
+ name_buf[r - 1] = '\0'; // erase extra character
+ cc->cpuidle_state[state].name = strdupz(trim(name_buf));
+ close(fd);
+
+ snprintfz(filename, FILENAME_MAX, cpuidle_time_filename, core, state);
+ cc->cpuidle_state[state].time_filename = strdupz(filename);
+ cc->cpuidle_state[state].time_fd = -1;
+ }
+
+ cc->rescan_cpu_states = 0;
+ }
+
+ for(state = 0; state < cc->cpuidle_state_len; state++) {
+
+ struct cpuidle_state *cs = &cc->cpuidle_state[state];
+
+ if(unlikely(cs->time_fd == -1)) {
+ cs->time_fd = open(cs->time_filename, O_RDONLY);
+ if (unlikely(cs->time_fd == -1)) {
+ error("Cannot open file '%s'", cs->time_filename);
+ cc->rescan_cpu_states = 1;
+ return 1;
+ }
+ }
+
+ char time_buf[50 + 1];
+ if(likely(read_one_state(time_buf, cs->time_filename, &cs->time_fd))) {
+ cs->value = str2ll(time_buf, NULL);
+ }
+ else {
+ cc->rescan_cpu_states = 1;
+ return 1;
+ }
+ }
+
+ // check if the number of states was increased
+ if(unlikely(stat(next_state_filename, &stbuf) == 0)) {
+ cc->rescan_cpu_states = 1;
+ return 1;
+ }
+
+ return 0;
+}
+
+int do_proc_stat(int update_every, usec_t dt) {
+ (void)dt;
+
+ static struct cpu_chart *all_cpu_charts = NULL;
+ static size_t all_cpu_charts_size = 0;
+ static procfile *ff = NULL;
+ static int do_cpu = -1, do_cpu_cores = -1, do_interrupts = -1, do_context = -1, do_forks = -1, do_processes = -1,
+ do_core_throttle_count = -1, do_package_throttle_count = -1, do_cpu_freq = -1, do_cpuidle = -1;
+ static uint32_t hash_intr, hash_ctxt, hash_processes, hash_procs_running, hash_procs_blocked;
+ static char *core_throttle_count_filename = NULL, *package_throttle_count_filename = NULL, *scaling_cur_freq_filename = NULL,
+ *time_in_state_filename = NULL, *schedstat_filename = NULL, *cpuidle_name_filename = NULL, *cpuidle_time_filename = NULL;
+ static const RRDVAR_ACQUIRED *cpus_var = NULL;
+ static int accurate_freq_avail = 0, accurate_freq_is_used = 0;
+ size_t cores_found = (size_t)processors;
+
+ if(unlikely(do_cpu == -1)) {
+ do_cpu = config_get_boolean("plugin:proc:/proc/stat", "cpu utilization", CONFIG_BOOLEAN_YES);
+ do_cpu_cores = config_get_boolean("plugin:proc:/proc/stat", "per cpu core utilization", CONFIG_BOOLEAN_YES);
+ do_interrupts = config_get_boolean("plugin:proc:/proc/stat", "cpu interrupts", CONFIG_BOOLEAN_YES);
+ do_context = config_get_boolean("plugin:proc:/proc/stat", "context switches", CONFIG_BOOLEAN_YES);
+ do_forks = config_get_boolean("plugin:proc:/proc/stat", "processes started", CONFIG_BOOLEAN_YES);
+ do_processes = config_get_boolean("plugin:proc:/proc/stat", "processes running", CONFIG_BOOLEAN_YES);
+
+ // give sane defaults based on the number of processors
+ if(unlikely(processors > 50)) {
+ // the system has too many processors
+ keep_per_core_fds_open = CONFIG_BOOLEAN_NO;
+ do_core_throttle_count = CONFIG_BOOLEAN_NO;
+ do_package_throttle_count = CONFIG_BOOLEAN_NO;
+ do_cpu_freq = CONFIG_BOOLEAN_NO;
+ do_cpuidle = CONFIG_BOOLEAN_NO;
+ }
+ else {
+ // the system has a reasonable number of processors
+ keep_per_core_fds_open = CONFIG_BOOLEAN_YES;
+ do_core_throttle_count = CONFIG_BOOLEAN_AUTO;
+ do_package_throttle_count = CONFIG_BOOLEAN_NO;
+ do_cpu_freq = CONFIG_BOOLEAN_YES;
+ do_cpuidle = CONFIG_BOOLEAN_YES;
+ }
+ if(unlikely(processors > 24)) {
+ // the system has too many processors
+ keep_cpuidle_fds_open = CONFIG_BOOLEAN_NO;
+ }
+ else {
+ // the system has a reasonable number of processors
+ keep_cpuidle_fds_open = CONFIG_BOOLEAN_YES;
+ }
+
+ keep_per_core_fds_open = config_get_boolean("plugin:proc:/proc/stat", "keep per core files open", keep_per_core_fds_open);
+ keep_cpuidle_fds_open = config_get_boolean("plugin:proc:/proc/stat", "keep cpuidle files open", keep_cpuidle_fds_open);
+ do_core_throttle_count = config_get_boolean_ondemand("plugin:proc:/proc/stat", "core_throttle_count", do_core_throttle_count);
+ do_package_throttle_count = config_get_boolean_ondemand("plugin:proc:/proc/stat", "package_throttle_count", do_package_throttle_count);
+ do_cpu_freq = config_get_boolean_ondemand("plugin:proc:/proc/stat", "cpu frequency", do_cpu_freq);
+ do_cpuidle = config_get_boolean_ondemand("plugin:proc:/proc/stat", "cpu idle states", do_cpuidle);
+
+ hash_intr = simple_hash("intr");
+ hash_ctxt = simple_hash("ctxt");
+ hash_processes = simple_hash("processes");
+ hash_procs_running = simple_hash("procs_running");
+ hash_procs_blocked = simple_hash("procs_blocked");
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/cpu/%s/thermal_throttle/core_throttle_count");
+ core_throttle_count_filename = config_get("plugin:proc:/proc/stat", "core_throttle_count filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/cpu/%s/thermal_throttle/package_throttle_count");
+ package_throttle_count_filename = config_get("plugin:proc:/proc/stat", "package_throttle_count filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/cpu/%s/cpufreq/scaling_cur_freq");
+ scaling_cur_freq_filename = config_get("plugin:proc:/proc/stat", "scaling_cur_freq filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/cpu/%s/cpufreq/stats/time_in_state");
+ time_in_state_filename = config_get("plugin:proc:/proc/stat", "time_in_state filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/schedstat");
+ schedstat_filename = config_get("plugin:proc:/proc/stat", "schedstat filename to monitor", filename);
+
+ if(do_cpuidle != CONFIG_BOOLEAN_NO) {
+ struct stat stbuf;
+
+ if (stat(schedstat_filename, &stbuf))
+ do_cpuidle = CONFIG_BOOLEAN_NO;
+ }
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/cpu/cpu%zu/cpuidle/state%zu/name");
+ cpuidle_name_filename = config_get("plugin:proc:/proc/stat", "cpuidle name filename to monitor", filename);
+
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/cpu/cpu%zu/cpuidle/state%zu/time");
+ cpuidle_time_filename = config_get("plugin:proc:/proc/stat", "cpuidle time filename to monitor", filename);
+ }
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/stat");
+ ff = procfile_open(config_get("plugin:proc:/proc/stat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+ size_t words;
+
+ unsigned long long processes = 0, running = 0 , blocked = 0;
+
+ for(l = 0; l < lines ;l++) {
+ char *row_key = procfile_lineword(ff, l, 0);
+ uint32_t hash = simple_hash(row_key);
+
+ // faster strncmp(row_key, "cpu", 3) == 0
+ if(likely(row_key[0] == 'c' && row_key[1] == 'p' && row_key[2] == 'u')) {
+ words = procfile_linewords(ff, l);
+ if(unlikely(words < 9)) {
+ error("Cannot read /proc/stat cpu line. Expected 9 params, read %zu.", words);
+ continue;
+ }
+
+ size_t core = (row_key[3] == '\0') ? 0 : str2ul(&row_key[3]) + 1;
+ if(likely(core > 0)) cores_found = core;
+
+ if(likely((core == 0 && do_cpu) || (core > 0 && do_cpu_cores))) {
+ char *id;
+ unsigned long long user = 0, nice = 0, system = 0, idle = 0, iowait = 0, irq = 0, softirq = 0, steal = 0, guest = 0, guest_nice = 0;
+
+ id = row_key;
+ user = str2ull(procfile_lineword(ff, l, 1));
+ nice = str2ull(procfile_lineword(ff, l, 2));
+ system = str2ull(procfile_lineword(ff, l, 3));
+ idle = str2ull(procfile_lineword(ff, l, 4));
+ iowait = str2ull(procfile_lineword(ff, l, 5));
+ irq = str2ull(procfile_lineword(ff, l, 6));
+ softirq = str2ull(procfile_lineword(ff, l, 7));
+ steal = str2ull(procfile_lineword(ff, l, 8));
+
+ guest = str2ull(procfile_lineword(ff, l, 9));
+ user -= guest;
+
+ guest_nice = str2ull(procfile_lineword(ff, l, 10));
+ nice -= guest_nice;
+
+ char *title, *type, *context, *family;
+ long priority;
+
+ if(unlikely(core >= all_cpu_charts_size)) {
+ size_t old_cpu_charts_size = all_cpu_charts_size;
+ all_cpu_charts_size = core + 1;
+ all_cpu_charts = reallocz(all_cpu_charts, sizeof(struct cpu_chart) * all_cpu_charts_size);
+ memset(&all_cpu_charts[old_cpu_charts_size], 0, sizeof(struct cpu_chart) * (all_cpu_charts_size - old_cpu_charts_size));
+ }
+ struct cpu_chart *cpu_chart = &all_cpu_charts[core];
+
+ if(unlikely(!cpu_chart->st)) {
+ cpu_chart->id = strdupz(id);
+
+ if(unlikely(core == 0)) {
+ title = "Total CPU utilization";
+ type = "system";
+ context = "system.cpu";
+ family = id;
+ priority = NETDATA_CHART_PRIO_SYSTEM_CPU;
+ }
+ else {
+ title = "Core utilization";
+ type = "cpu";
+ context = "cpu.cpu";
+ family = "utilization";
+ priority = NETDATA_CHART_PRIO_CPU_PER_CORE;
+
+ char filename[FILENAME_MAX + 1];
+ struct stat stbuf;
+
+ if(do_core_throttle_count != CONFIG_BOOLEAN_NO) {
+ snprintfz(filename, FILENAME_MAX, core_throttle_count_filename, id);
+ if (stat(filename, &stbuf) == 0) {
+ cpu_chart->files[CORE_THROTTLE_COUNT_INDEX].filename = strdupz(filename);
+ cpu_chart->files[CORE_THROTTLE_COUNT_INDEX].fd = -1;
+ do_core_throttle_count = CONFIG_BOOLEAN_YES;
+ }
+ }
+
+ if(do_package_throttle_count != CONFIG_BOOLEAN_NO) {
+ snprintfz(filename, FILENAME_MAX, package_throttle_count_filename, id);
+ if (stat(filename, &stbuf) == 0) {
+ cpu_chart->files[PACKAGE_THROTTLE_COUNT_INDEX].filename = strdupz(filename);
+ cpu_chart->files[PACKAGE_THROTTLE_COUNT_INDEX].fd = -1;
+ do_package_throttle_count = CONFIG_BOOLEAN_YES;
+ }
+ }
+
+ if(do_cpu_freq != CONFIG_BOOLEAN_NO) {
+
+ snprintfz(filename, FILENAME_MAX, scaling_cur_freq_filename, id);
+
+ if (stat(filename, &stbuf) == 0) {
+ cpu_chart->files[CPU_FREQ_INDEX].filename = strdupz(filename);
+ cpu_chart->files[CPU_FREQ_INDEX].fd = -1;
+ do_cpu_freq = CONFIG_BOOLEAN_YES;
+ }
+
+ snprintfz(filename, FILENAME_MAX, time_in_state_filename, id);
+
+ if (stat(filename, &stbuf) == 0) {
+ cpu_chart->time_in_state_files.filename = strdupz(filename);
+ cpu_chart->time_in_state_files.ff = NULL;
+ do_cpu_freq = CONFIG_BOOLEAN_YES;
+ accurate_freq_avail = 1;
+ }
+ }
+ }
+
+ cpu_chart->st = rrdset_create_localhost(
+ type
+ , id
+ , NULL
+ , family
+ , context
+ , title
+ , "percentage"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , priority + core
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ long multiplier = 1;
+ long divisor = 1; // sysconf(_SC_CLK_TCK);
+
+ cpu_chart->rd_guest_nice = rrddim_add(cpu_chart->st, "guest_nice", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_guest = rrddim_add(cpu_chart->st, "guest", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_steal = rrddim_add(cpu_chart->st, "steal", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_softirq = rrddim_add(cpu_chart->st, "softirq", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_irq = rrddim_add(cpu_chart->st, "irq", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_user = rrddim_add(cpu_chart->st, "user", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_system = rrddim_add(cpu_chart->st, "system", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_nice = rrddim_add(cpu_chart->st, "nice", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_iowait = rrddim_add(cpu_chart->st, "iowait", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ cpu_chart->rd_idle = rrddim_add(cpu_chart->st, "idle", NULL, multiplier, divisor, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rrddim_hide(cpu_chart->st, "idle");
+
+ if(unlikely(core == 0 && cpus_var == NULL))
+ cpus_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "active_processors");
+ }
+
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_user, user);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_nice, nice);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_system, system);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_idle, idle);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_iowait, iowait);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_irq, irq);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_softirq, softirq);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_steal, steal);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_guest, guest);
+ rrddim_set_by_pointer(cpu_chart->st, cpu_chart->rd_guest_nice, guest_nice);
+ rrdset_done(cpu_chart->st);
+ }
+ }
+ else if(unlikely(hash == hash_intr && strcmp(row_key, "intr") == 0)) {
+ if(likely(do_interrupts)) {
+ static RRDSET *st_intr = NULL;
+ static RRDDIM *rd_interrupts = NULL;
+ unsigned long long value = str2ull(procfile_lineword(ff, l, 1));
+
+ if(unlikely(!st_intr)) {
+ st_intr = rrdset_create_localhost(
+ "system"
+ , "intr"
+ , NULL
+ , "interrupts"
+ , NULL
+ , "CPU Interrupts"
+ , "interrupts/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_INTR
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_intr, RRDSET_FLAG_DETAIL);
+
+ rd_interrupts = rrddim_add(st_intr, "interrupts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_intr, rd_interrupts, value);
+ rrdset_done(st_intr);
+ }
+ }
+ else if(unlikely(hash == hash_ctxt && strcmp(row_key, "ctxt") == 0)) {
+ if(likely(do_context)) {
+ static RRDSET *st_ctxt = NULL;
+ static RRDDIM *rd_switches = NULL;
+ unsigned long long value = str2ull(procfile_lineword(ff, l, 1));
+
+ if(unlikely(!st_ctxt)) {
+ st_ctxt = rrdset_create_localhost(
+ "system"
+ , "ctxt"
+ , NULL
+ , "processes"
+ , NULL
+ , "CPU Context Switches"
+ , "context switches/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_CTXT
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_switches = rrddim_add(st_ctxt, "switches", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ctxt, rd_switches, value);
+ rrdset_done(st_ctxt);
+ }
+ }
+ else if(unlikely(hash == hash_processes && !processes && strcmp(row_key, "processes") == 0)) {
+ processes = str2ull(procfile_lineword(ff, l, 1));
+ }
+ else if(unlikely(hash == hash_procs_running && !running && strcmp(row_key, "procs_running") == 0)) {
+ running = str2ull(procfile_lineword(ff, l, 1));
+ }
+ else if(unlikely(hash == hash_procs_blocked && !blocked && strcmp(row_key, "procs_blocked") == 0)) {
+ blocked = str2ull(procfile_lineword(ff, l, 1));
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_forks)) {
+ static RRDSET *st_forks = NULL;
+ static RRDDIM *rd_started = NULL;
+
+ if(unlikely(!st_forks)) {
+ st_forks = rrdset_create_localhost(
+ "system"
+ , "forks"
+ , NULL
+ , "processes"
+ , NULL
+ , "Started Processes"
+ , "processes/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_FORKS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ rrdset_flag_set(st_forks, RRDSET_FLAG_DETAIL);
+
+ rd_started = rrddim_add(st_forks, "started", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_forks, rd_started, processes);
+ rrdset_done(st_forks);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_processes)) {
+ static RRDSET *st_processes = NULL;
+ static RRDDIM *rd_running = NULL;
+ static RRDDIM *rd_blocked = NULL;
+
+ if(unlikely(!st_processes)) {
+ st_processes = rrdset_create_localhost(
+ "system"
+ , "processes"
+ , NULL
+ , "processes"
+ , NULL
+ , "System Processes"
+ , "processes"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_PROCESSES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_running = rrddim_add(st_processes, "running", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_blocked = rrddim_add(st_processes, "blocked", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_processes, rd_running, running);
+ rrddim_set_by_pointer(st_processes, rd_blocked, blocked);
+ rrdset_done(st_processes);
+ }
+
+ if(likely(all_cpu_charts_size > 1)) {
+ if(likely(do_core_throttle_count != CONFIG_BOOLEAN_NO)) {
+ int r = read_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, CORE_THROTTLE_COUNT_INDEX);
+ if(likely(r != -1 && (do_core_throttle_count == CONFIG_BOOLEAN_YES || r > 0))) {
+ do_core_throttle_count = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_core_throttle_count = NULL;
+
+ if (unlikely(!st_core_throttle_count)) {
+ st_core_throttle_count = rrdset_create_localhost(
+ "cpu"
+ , "core_throttling"
+ , NULL
+ , "throttling"
+ , "cpu.core_throttling"
+ , "Core Thermal Throttling Events"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , NETDATA_CHART_PRIO_CORE_THROTTLING
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ }
+
+ chart_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, CORE_THROTTLE_COUNT_INDEX, st_core_throttle_count, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrdset_done(st_core_throttle_count);
+ }
+ }
+
+ if(likely(do_package_throttle_count != CONFIG_BOOLEAN_NO)) {
+ int r = read_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, PACKAGE_THROTTLE_COUNT_INDEX);
+ if(likely(r != -1 && (do_package_throttle_count == CONFIG_BOOLEAN_YES || r > 0))) {
+ do_package_throttle_count = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_package_throttle_count = NULL;
+
+ if(unlikely(!st_package_throttle_count)) {
+ st_package_throttle_count = rrdset_create_localhost(
+ "cpu"
+ , "package_throttling"
+ , NULL
+ , "throttling"
+ , "cpu.package_throttling"
+ , "Package Thermal Throttling Events"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , NETDATA_CHART_PRIO_PACKAGE_THROTTLING
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ }
+
+ chart_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, PACKAGE_THROTTLE_COUNT_INDEX, st_package_throttle_count, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrdset_done(st_package_throttle_count);
+ }
+ }
+
+ if(likely(do_cpu_freq != CONFIG_BOOLEAN_NO)) {
+ char filename[FILENAME_MAX + 1];
+ int r = 0;
+
+ if (accurate_freq_avail) {
+ r = read_per_core_time_in_state_files(&all_cpu_charts[1], all_cpu_charts_size - 1, CPU_FREQ_INDEX);
+ if(r > 0 && !accurate_freq_is_used) {
+ accurate_freq_is_used = 1;
+ snprintfz(filename, FILENAME_MAX, time_in_state_filename, "cpu*");
+ info("cpufreq is using %s", filename);
+ }
+ }
+ if (r < 1) {
+ r = read_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, CPU_FREQ_INDEX);
+ if(accurate_freq_is_used) {
+ accurate_freq_is_used = 0;
+ snprintfz(filename, FILENAME_MAX, scaling_cur_freq_filename, "cpu*");
+ info("cpufreq fell back to %s", filename);
+ }
+ }
+
+ if(likely(r != -1 && (do_cpu_freq == CONFIG_BOOLEAN_YES || r > 0))) {
+ do_cpu_freq = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_scaling_cur_freq = NULL;
+
+ if(unlikely(!st_scaling_cur_freq)) {
+ st_scaling_cur_freq = rrdset_create_localhost(
+ "cpu"
+ , "cpufreq"
+ , NULL
+ , "cpufreq"
+ , "cpufreq.cpufreq"
+ , "Current CPU Frequency"
+ , "MHz"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , NETDATA_CHART_PRIO_CPUFREQ_SCALING_CUR_FREQ
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ }
+
+ chart_per_core_files(&all_cpu_charts[1], all_cpu_charts_size - 1, CPU_FREQ_INDEX, st_scaling_cur_freq, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ rrdset_done(st_scaling_cur_freq);
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ static struct per_core_cpuidle_chart *cpuidle_charts = NULL;
+ size_t schedstat_cores_found = 0;
+
+ if(likely(do_cpuidle != CONFIG_BOOLEAN_NO && !read_schedstat(schedstat_filename, &cpuidle_charts, &schedstat_cores_found))) {
+ int cpu_states_updated = 0;
+ size_t core, state;
+
+
+ // proc.plugin runs on Linux systems only. Multi-platform compatibility is not needed here,
+ // so bare pthread functions are used to avoid unneeded overheads.
+ for(core = 0; core < schedstat_cores_found; core++) {
+ if(unlikely(!(cpuidle_charts[core].active_time - cpuidle_charts[core].last_active_time))) {
+ pthread_t thread;
+ cpu_set_t global_cpu_set;
+
+ if (likely(!pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &global_cpu_set))) {
+ if (unlikely(!CPU_ISSET(core, &global_cpu_set))) {
+ continue;
+ }
+ }
+ else
+ error("Cannot read current process affinity");
+
+ // These threads are very ephemeral and don't need to have a specific name
+ if(unlikely(pthread_create(&thread, NULL, wake_cpu_thread, (void *)&core)))
+ error("Cannot create wake_cpu_thread");
+ else if(unlikely(pthread_join(thread, NULL)))
+ error("Cannot join wake_cpu_thread");
+ cpu_states_updated = 1;
+ }
+ }
+
+ if(unlikely(!cpu_states_updated || !read_schedstat(schedstat_filename, &cpuidle_charts, &schedstat_cores_found))) {
+ for(core = 0; core < schedstat_cores_found; core++) {
+ cpuidle_charts[core].last_active_time = cpuidle_charts[core].active_time;
+
+ int r = read_cpuidle_states(cpuidle_name_filename, cpuidle_time_filename, cpuidle_charts, core);
+ if(likely(r != -1 && (do_cpuidle == CONFIG_BOOLEAN_YES || r > 0))) {
+ do_cpuidle = CONFIG_BOOLEAN_YES;
+
+ char cpuidle_chart_id[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(cpuidle_chart_id, RRD_ID_LENGTH_MAX, "cpu%zu_cpuidle", core);
+
+ if(unlikely(!cpuidle_charts[core].st)) {
+ cpuidle_charts[core].st = rrdset_create_localhost(
+ "cpu"
+ , cpuidle_chart_id
+ , NULL
+ , "cpuidle"
+ , "cpuidle.cpu_cstate_residency_time"
+ , "C-state residency time"
+ , "percentage"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_STAT_NAME
+ , NETDATA_CHART_PRIO_CPUIDLE + core
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ char corebuf[50+1];
+ snprintfz(corebuf, 50, "cpu%zu", core);
+ rrdlabels_add(cpuidle_charts[core].st->rrdlabels, "cpu", corebuf, RRDLABEL_SRC_AUTO);
+
+ char cpuidle_dim_id[RRD_ID_LENGTH_MAX + 1];
+ cpuidle_charts[core].active_time_rd = rrddim_add(cpuidle_charts[core].st, "active", "C0 (active)", 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ for(state = 0; state < cpuidle_charts[core].cpuidle_state_len; state++) {
+ strncpyz(cpuidle_dim_id, cpuidle_charts[core].cpuidle_state[state].name, RRD_ID_LENGTH_MAX);
+ for(int i = 0; cpuidle_dim_id[i]; i++)
+ cpuidle_dim_id[i] = tolower(cpuidle_dim_id[i]);
+ cpuidle_charts[core].cpuidle_state[state].rd = rrddim_add(cpuidle_charts[core].st, cpuidle_dim_id,
+ cpuidle_charts[core].cpuidle_state[state].name,
+ 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ }
+
+ rrddim_set_by_pointer(cpuidle_charts[core].st, cpuidle_charts[core].active_time_rd, cpuidle_charts[core].active_time);
+ for(state = 0; state < cpuidle_charts[core].cpuidle_state_len; state++) {
+ rrddim_set_by_pointer(cpuidle_charts[core].st, cpuidle_charts[core].cpuidle_state[state].rd, cpuidle_charts[core].cpuidle_state[state].value);
+ }
+ rrdset_done(cpuidle_charts[core].st);
+ }
+ }
+ }
+ }
+
+ if(cpus_var)
+ rrdvar_custom_host_variable_set(localhost, cpus_var, cores_found);
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_sys_kernel_random_entropy_avail.c b/collectors/proc.plugin/proc_sys_kernel_random_entropy_avail.c
new file mode 100644
index 0000000..a04d430
--- /dev/null
+++ b/collectors/proc.plugin/proc_sys_kernel_random_entropy_avail.c
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+int do_proc_sys_kernel_random_entropy_avail(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/sys/kernel/random/entropy_avail");
+ ff = procfile_open(config_get("plugin:proc:/proc/sys/kernel/random/entropy_avail", "filename to monitor", filename), "", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ unsigned long long entropy = str2ull(procfile_lineword(ff, 0, 0));
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if(unlikely(!st)) {
+ st = rrdset_create_localhost(
+ "system"
+ , "entropy"
+ , NULL
+ , "entropy"
+ , NULL
+ , "Available Entropy"
+ , "entropy"
+ , PLUGIN_PROC_NAME
+ , "/proc/sys/kernel/random/entropy_avail"
+ , NETDATA_CHART_PRIO_SYSTEM_ENTROPY
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd = rrddim_add(st, "entropy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd, entropy);
+ rrdset_done(st);
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_uptime.c b/collectors/proc.plugin/proc_uptime.c
new file mode 100644
index 0000000..ddab726
--- /dev/null
+++ b/collectors/proc.plugin/proc_uptime.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+int do_proc_uptime(int update_every, usec_t dt) {
+ (void)dt;
+
+ static char *uptime_filename = NULL;
+ if(!uptime_filename) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/uptime");
+
+ uptime_filename = config_get("plugin:proc:/proc/uptime", "filename to monitor", filename);
+ }
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd = NULL;
+
+ if(unlikely(!st)) {
+
+ st = rrdset_create_localhost(
+ "system"
+ , "uptime"
+ , NULL
+ , "uptime"
+ , NULL
+ , "System Uptime"
+ , "seconds"
+ , PLUGIN_PROC_NAME
+ , "/proc/uptime"
+ , NETDATA_CHART_PRIO_SYSTEM_UPTIME
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd = rrddim_add(st, "uptime", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st, rd, uptime_msec(uptime_filename));
+ rrdset_done(st);
+ return 0;
+}
diff --git a/collectors/proc.plugin/proc_vmstat.c b/collectors/proc.plugin/proc_vmstat.c
new file mode 100644
index 0000000..b8defc4
--- /dev/null
+++ b/collectors/proc.plugin/proc_vmstat.c
@@ -0,0 +1,311 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_VMSTAT_NAME "/proc/vmstat"
+
+#define OOM_KILL_STRING "oom_kill"
+
+int do_proc_vmstat(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+ static int do_swapio = -1, do_io = -1, do_pgfaults = -1, do_oom_kill = -1, do_numa = -1;
+ static int has_numa = -1;
+
+ static ARL_BASE *arl_base = NULL;
+ static unsigned long long numa_foreign = 0ULL;
+ static unsigned long long numa_hint_faults = 0ULL;
+ static unsigned long long numa_hint_faults_local = 0ULL;
+ static unsigned long long numa_huge_pte_updates = 0ULL;
+ static unsigned long long numa_interleave = 0ULL;
+ static unsigned long long numa_local = 0ULL;
+ static unsigned long long numa_other = 0ULL;
+ static unsigned long long numa_pages_migrated = 0ULL;
+ static unsigned long long numa_pte_updates = 0ULL;
+ static unsigned long long pgfault = 0ULL;
+ static unsigned long long pgmajfault = 0ULL;
+ static unsigned long long pgpgin = 0ULL;
+ static unsigned long long pgpgout = 0ULL;
+ static unsigned long long pswpin = 0ULL;
+ static unsigned long long pswpout = 0ULL;
+ static unsigned long long oom_kill = 0ULL;
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/vmstat");
+ ff = procfile_open(config_get("plugin:proc:/proc/vmstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ if(unlikely(!arl_base)) {
+ do_swapio = config_get_boolean_ondemand("plugin:proc:/proc/vmstat", "swap i/o", CONFIG_BOOLEAN_AUTO);
+ do_io = config_get_boolean("plugin:proc:/proc/vmstat", "disk i/o", CONFIG_BOOLEAN_YES);
+ do_pgfaults = config_get_boolean("plugin:proc:/proc/vmstat", "memory page faults", CONFIG_BOOLEAN_YES);
+ do_oom_kill = config_get_boolean("plugin:proc:/proc/vmstat", "out of memory kills", CONFIG_BOOLEAN_AUTO);
+ do_numa = config_get_boolean_ondemand("plugin:proc:/proc/vmstat", "system-wide numa metric summary", CONFIG_BOOLEAN_AUTO);
+
+
+ arl_base = arl_create("vmstat", NULL, 60);
+ arl_expect(arl_base, "pgfault", &pgfault);
+ arl_expect(arl_base, "pgmajfault", &pgmajfault);
+ arl_expect(arl_base, "pgpgin", &pgpgin);
+ arl_expect(arl_base, "pgpgout", &pgpgout);
+ arl_expect(arl_base, "pswpin", &pswpin);
+ arl_expect(arl_base, "pswpout", &pswpout);
+
+ int has_oom_kill = 0;
+
+ for (l = 0; l < lines; l++) {
+ if (!strcmp(procfile_lineword(ff, l, 0), OOM_KILL_STRING)) {
+ has_oom_kill = 1;
+ break;
+ }
+ }
+
+ if (has_oom_kill)
+ arl_expect(arl_base, OOM_KILL_STRING, &oom_kill);
+ else
+ do_oom_kill = CONFIG_BOOLEAN_NO;
+
+ if(do_numa == CONFIG_BOOLEAN_YES || (do_numa == CONFIG_BOOLEAN_AUTO &&
+ (get_numa_node_count() >= 2 ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ arl_expect(arl_base, "numa_foreign", &numa_foreign);
+ arl_expect(arl_base, "numa_hint_faults_local", &numa_hint_faults_local);
+ arl_expect(arl_base, "numa_hint_faults", &numa_hint_faults);
+ arl_expect(arl_base, "numa_huge_pte_updates", &numa_huge_pte_updates);
+ arl_expect(arl_base, "numa_interleave", &numa_interleave);
+ arl_expect(arl_base, "numa_local", &numa_local);
+ arl_expect(arl_base, "numa_other", &numa_other);
+ arl_expect(arl_base, "numa_pages_migrated", &numa_pages_migrated);
+ arl_expect(arl_base, "numa_pte_updates", &numa_pte_updates);
+ }
+ else {
+ // Do not expect numa metrics when they are not needed.
+ // By not adding them, the ARL will stop processing the file
+ // when all the expected metrics are collected.
+ // Also ARL will not parse their values.
+ has_numa = 0;
+ do_numa = CONFIG_BOOLEAN_NO;
+ }
+ }
+
+ arl_begin(arl_base);
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 2)) {
+ if(unlikely(words)) error("Cannot read /proc/vmstat line %zu. Expected 2 params, read %zu.", l, words);
+ continue;
+ }
+
+ if(unlikely(arl_check(arl_base,
+ procfile_lineword(ff, l, 0),
+ procfile_lineword(ff, l, 1)))) break;
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_swapio == CONFIG_BOOLEAN_YES || (do_swapio == CONFIG_BOOLEAN_AUTO &&
+ (pswpin || pswpout ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_swapio = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_swapio = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_swapio)) {
+ st_swapio = rrdset_create_localhost(
+ "system"
+ , "swapio"
+ , NULL
+ , "swap"
+ , NULL
+ , "Swap I/O"
+ , "KiB/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_VMSTAT_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_SWAPIO
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st_swapio, "in", NULL, sysconf(_SC_PAGESIZE), 1024, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_swapio, "out", NULL, -sysconf(_SC_PAGESIZE), 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_swapio, rd_in, pswpin);
+ rrddim_set_by_pointer(st_swapio, rd_out, pswpout);
+ rrdset_done(st_swapio);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_io) {
+ static RRDSET *st_io = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if(unlikely(!st_io)) {
+ st_io = rrdset_create_localhost(
+ "system"
+ , "pgpgio"
+ , NULL
+ , "disk"
+ , NULL
+ , "Memory Paged from/to disk"
+ , "KiB/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_VMSTAT_NAME
+ , NETDATA_CHART_PRIO_SYSTEM_PGPGIO
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st_io, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st_io, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_io, rd_in, pgpgin);
+ rrddim_set_by_pointer(st_io, rd_out, pgpgout);
+ rrdset_done(st_io);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_pgfaults) {
+ static RRDSET *st_pgfaults = NULL;
+ static RRDDIM *rd_minor = NULL, *rd_major = NULL;
+
+ if(unlikely(!st_pgfaults)) {
+ st_pgfaults = rrdset_create_localhost(
+ "mem"
+ , "pgfaults"
+ , NULL
+ , "system"
+ , NULL
+ , "Memory Page Faults"
+ , "faults/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_VMSTAT_NAME
+ , NETDATA_CHART_PRIO_MEM_SYSTEM_PGFAULTS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_pgfaults, RRDSET_FLAG_DETAIL);
+
+ rd_minor = rrddim_add(st_pgfaults, "minor", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_major = rrddim_add(st_pgfaults, "major", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_pgfaults, rd_minor, pgfault);
+ rrddim_set_by_pointer(st_pgfaults, rd_major, pgmajfault);
+ rrdset_done(st_pgfaults);
+ }
+
+ // --------------------------------------------------------------------
+
+ if (do_oom_kill == CONFIG_BOOLEAN_YES ||
+ (do_oom_kill == CONFIG_BOOLEAN_AUTO && (oom_kill || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ static RRDSET *st_oom_kill = NULL;
+ static RRDDIM *rd_oom_kill = NULL;
+
+ do_oom_kill = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!st_oom_kill)) {
+ st_oom_kill = rrdset_create_localhost(
+ "mem"
+ , "oom_kill"
+ , NULL
+ , "system"
+ , NULL
+ , "Out of Memory Kills"
+ , "kills/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_VMSTAT_NAME
+ , NETDATA_CHART_PRIO_MEM_SYSTEM_OOM_KILL
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_oom_kill, RRDSET_FLAG_DETAIL);
+
+ rd_oom_kill = rrddim_add(st_oom_kill, "kills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_oom_kill, rd_oom_kill, oom_kill);
+ rrdset_done(st_oom_kill);
+ }
+
+ // --------------------------------------------------------------------
+
+ // Ondemand criteria for NUMA. Since this won't change at run time, we
+ // check it only once. We check whether the node count is >= 2 because
+ // single-node systems have uninteresting statistics (since all accesses
+ // are local).
+ if(unlikely(has_numa == -1))
+
+ has_numa = (numa_local || numa_foreign || numa_interleave || numa_other || numa_pte_updates ||
+ numa_huge_pte_updates || numa_hint_faults || numa_hint_faults_local || numa_pages_migrated) ? 1 : 0;
+
+ if(do_numa == CONFIG_BOOLEAN_YES || (do_numa == CONFIG_BOOLEAN_AUTO && has_numa)) {
+ do_numa = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_numa = NULL;
+ static RRDDIM *rd_local = NULL, *rd_foreign = NULL, *rd_interleave = NULL, *rd_other = NULL, *rd_pte_updates = NULL, *rd_huge_pte_updates = NULL, *rd_hint_faults = NULL, *rd_hint_faults_local = NULL, *rd_pages_migrated = NULL;
+
+ if(unlikely(!st_numa)) {
+ st_numa = rrdset_create_localhost(
+ "mem"
+ , "numa"
+ , NULL
+ , "numa"
+ , NULL
+ , "NUMA events"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_VMSTAT_NAME
+ , NETDATA_CHART_PRIO_MEM_NUMA
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(st_numa, RRDSET_FLAG_DETAIL);
+
+ // These depend on CONFIG_NUMA in the kernel.
+ rd_local = rrddim_add(st_numa, "local", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_foreign = rrddim_add(st_numa, "foreign", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_interleave = rrddim_add(st_numa, "interleave", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_other = rrddim_add(st_numa, "other", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ // The following stats depend on CONFIG_NUMA_BALANCING in the
+ // kernel.
+ rd_pte_updates = rrddim_add(st_numa, "pte_updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_huge_pte_updates = rrddim_add(st_numa, "huge_pte_updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_hint_faults = rrddim_add(st_numa, "hint_faults", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_hint_faults_local = rrddim_add(st_numa, "hint_faults_local", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_pages_migrated = rrddim_add(st_numa, "pages_migrated", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_numa, rd_local, numa_local);
+ rrddim_set_by_pointer(st_numa, rd_foreign, numa_foreign);
+ rrddim_set_by_pointer(st_numa, rd_interleave, numa_interleave);
+ rrddim_set_by_pointer(st_numa, rd_other, numa_other);
+
+ rrddim_set_by_pointer(st_numa, rd_pte_updates, numa_pte_updates);
+ rrddim_set_by_pointer(st_numa, rd_huge_pte_updates, numa_huge_pte_updates);
+ rrddim_set_by_pointer(st_numa, rd_hint_faults, numa_hint_faults);
+ rrddim_set_by_pointer(st_numa, rd_hint_faults_local, numa_hint_faults_local);
+ rrddim_set_by_pointer(st_numa, rd_pages_migrated, numa_pages_migrated);
+
+ rrdset_done(st_numa);
+ }
+
+ return 0;
+}
+
diff --git a/collectors/proc.plugin/sys_block_zram.c b/collectors/proc.plugin/sys_block_zram.c
new file mode 100644
index 0000000..6bae542
--- /dev/null
+++ b/collectors/proc.plugin/sys_block_zram.c
@@ -0,0 +1,279 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_ZRAM_NAME "/sys/block/zram"
+#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); (st) = NULL; } } while(st)
+
+typedef struct mm_stat {
+ unsigned long long orig_data_size;
+ unsigned long long compr_data_size;
+ unsigned long long mem_used_total;
+ unsigned long long mem_limit;
+ unsigned long long mem_used_max;
+ unsigned long long same_pages;
+ unsigned long long pages_compacted;
+} MM_STAT;
+
+typedef struct zram_device {
+ procfile *file;
+
+ RRDSET *st_usage;
+ RRDDIM *rd_compr_data_size;
+ RRDDIM *rd_metadata_size;
+
+ RRDSET *st_savings;
+ RRDDIM *rd_original_size;
+ RRDDIM *rd_savings_size;
+
+ RRDSET *st_comp_ratio;
+ RRDDIM *rd_comp_ratio;
+
+ RRDSET *st_alloc_efficiency;
+ RRDDIM *rd_alloc_efficiency;
+} ZRAM_DEVICE;
+
+static int try_get_zram_major_number(procfile *file) {
+ size_t i;
+ unsigned int lines = procfile_lines(file);
+ int id = -1;
+ char *name = NULL;
+ for (i = 0; i < lines; i++)
+ {
+ if (procfile_linewords(file, i) < 2)
+ continue;
+ name = procfile_lineword(file, i, 1);
+ if (strcmp(name, "zram") == 0)
+ {
+ id = str2i(procfile_lineword(file, i, 0));
+ if (id == 0)
+ return -1;
+ return id;
+ }
+ }
+ return -1;
+}
+
+static inline void init_rrd(const char *name, ZRAM_DEVICE *d, int update_every) {
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
+
+ snprintfz(chart_name, RRD_ID_LENGTH_MAX, "zram_usage.%s", name);
+ d->st_usage = rrdset_create_localhost(
+ "mem"
+ , chart_name
+ , chart_name
+ , name
+ , "mem.zram_usage"
+ , "ZRAM Memory Usage"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_ZRAM_NAME
+ , NETDATA_CHART_PRIO_MEM_ZRAM
+ , update_every
+ , RRDSET_TYPE_AREA);
+ d->rd_compr_data_size = rrddim_add(d->st_usage, "compressed", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_metadata_size = rrddim_add(d->st_usage, "metadata", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrdlabels_add(d->st_usage->rrdlabels, "device", name, RRDLABEL_SRC_AUTO);
+
+ snprintfz(chart_name, RRD_ID_LENGTH_MAX, "zram_savings.%s", name);
+ d->st_savings = rrdset_create_localhost(
+ "mem"
+ , chart_name
+ , chart_name
+ , name
+ , "mem.zram_savings"
+ , "ZRAM Memory Savings"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_ZRAM_NAME
+ , NETDATA_CHART_PRIO_MEM_ZRAM_SAVINGS
+ , update_every
+ , RRDSET_TYPE_AREA);
+ d->rd_savings_size = rrddim_add(d->st_savings, "savings", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ d->rd_original_size = rrddim_add(d->st_savings, "original", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rrdlabels_add(d->st_savings->rrdlabels, "device", name, RRDLABEL_SRC_AUTO);
+
+ snprintfz(chart_name, RRD_ID_LENGTH_MAX, "zram_ratio.%s", name);
+ d->st_comp_ratio = rrdset_create_localhost(
+ "mem"
+ , chart_name
+ , chart_name
+ , name
+ , "mem.zram_ratio"
+ , "ZRAM Compression Ratio (original to compressed)"
+ , "ratio"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_ZRAM_NAME
+ , NETDATA_CHART_PRIO_MEM_ZRAM_RATIO
+ , update_every
+ , RRDSET_TYPE_LINE);
+ d->rd_comp_ratio = rrddim_add(d->st_comp_ratio, "ratio", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+ rrdlabels_add(d->st_comp_ratio->rrdlabels, "device", name, RRDLABEL_SRC_AUTO);
+
+ snprintfz(chart_name, RRD_ID_LENGTH_MAX, "zram_efficiency.%s", name);
+ d->st_alloc_efficiency = rrdset_create_localhost(
+ "mem"
+ , chart_name
+ , chart_name
+ , name
+ , "mem.zram_efficiency"
+ , "ZRAM Efficiency"
+ , "percentage"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_ZRAM_NAME
+ , NETDATA_CHART_PRIO_MEM_ZRAM_EFFICIENCY
+ , update_every
+ , RRDSET_TYPE_LINE);
+ d->rd_alloc_efficiency = rrddim_add(d->st_alloc_efficiency, "percent", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
+ rrdlabels_add(d->st_alloc_efficiency->rrdlabels, "device", name, RRDLABEL_SRC_AUTO);
+}
+
+static int init_devices(DICTIONARY *devices, unsigned int zram_id, int update_every) {
+ int count = 0;
+ DIR *dir = opendir("/dev");
+ struct dirent *de;
+ struct stat st;
+ char filename[FILENAME_MAX + 1];
+ procfile *ff = NULL;
+ ZRAM_DEVICE device;
+
+ if (unlikely(!dir))
+ return 0;
+ while ((de = readdir(dir)))
+ {
+ snprintfz(filename, FILENAME_MAX, "/dev/%s", de->d_name);
+ if (unlikely(stat(filename, &st) != 0))
+ {
+ error("ZRAM : Unable to stat %s: %s", filename, strerror(errno));
+ continue;
+ }
+ if (major(st.st_rdev) == zram_id)
+ {
+ info("ZRAM : Found device %s", filename);
+ snprintfz(filename, FILENAME_MAX, "/sys/block/%s/mm_stat", de->d_name);
+ ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ if (ff == NULL)
+ {
+ error("ZRAM : Failed to open %s: %s", filename, strerror(errno));
+ continue;
+ }
+ device.file = ff;
+ init_rrd(de->d_name, &device, update_every);
+ dictionary_set(devices, de->d_name, &device, sizeof(ZRAM_DEVICE));
+ count++;
+ }
+ }
+ closedir(dir);
+ return count;
+}
+
+static void free_device(DICTIONARY *dict, const char *name)
+{
+ ZRAM_DEVICE *d = (ZRAM_DEVICE*)dictionary_get(dict, name);
+ info("ZRAM : Disabling monitoring of device %s", name);
+ rrdset_obsolete_and_pointer_null(d->st_usage);
+ rrdset_obsolete_and_pointer_null(d->st_savings);
+ rrdset_obsolete_and_pointer_null(d->st_alloc_efficiency);
+ rrdset_obsolete_and_pointer_null(d->st_comp_ratio);
+ dictionary_del(dict, name);
+}
+
+static inline int read_mm_stat(procfile *ff, MM_STAT *stats) {
+ ff = procfile_readall(ff);
+ if (!ff)
+ return -1;
+ if (procfile_lines(ff) < 1) {
+ procfile_close(ff);
+ return -1;
+ }
+ if (procfile_linewords(ff, 0) < 7) {
+ procfile_close(ff);
+ return -1;
+ }
+
+ stats->orig_data_size = str2ull(procfile_word(ff, 0));
+ stats->compr_data_size = str2ull(procfile_word(ff, 1));
+ stats->mem_used_total = str2ull(procfile_word(ff, 2));
+ stats->mem_limit = str2ull(procfile_word(ff, 3));
+ stats->mem_used_max = str2ull(procfile_word(ff, 4));
+ stats->same_pages = str2ull(procfile_word(ff, 5));
+ stats->pages_compacted = str2ull(procfile_word(ff, 6));
+ return 0;
+}
+
+static int collect_zram_metrics(const DICTIONARY_ITEM *item, void *entry, void *data) {
+ const char *name = dictionary_acquired_item_name(item);
+ ZRAM_DEVICE *dev = entry;
+ DICTIONARY *dict = data;
+
+ MM_STAT mm;
+ int value;
+
+ if (unlikely(read_mm_stat(dev->file, &mm) < 0)) {
+ free_device(dict, name);
+ return -1;
+ }
+
+ // zram_usage
+ rrddim_set_by_pointer(dev->st_usage, dev->rd_compr_data_size, mm.compr_data_size);
+ rrddim_set_by_pointer(dev->st_usage, dev->rd_metadata_size, mm.mem_used_total - mm.compr_data_size);
+ rrdset_done(dev->st_usage);
+
+ // zram_savings
+ rrddim_set_by_pointer(dev->st_savings, dev->rd_savings_size, mm.compr_data_size - mm.orig_data_size);
+ rrddim_set_by_pointer(dev->st_savings, dev->rd_original_size, mm.orig_data_size);
+ rrdset_done(dev->st_savings);
+
+ // zram_ratio
+ value = mm.compr_data_size == 0 ? 1 : mm.orig_data_size * 100 / mm.compr_data_size;
+ rrddim_set_by_pointer(dev->st_comp_ratio, dev->rd_comp_ratio, value);
+ rrdset_done(dev->st_comp_ratio);
+
+ // zram_efficiency
+ value = mm.mem_used_total == 0 ? 100 : (mm.compr_data_size * 1000000 / mm.mem_used_total);
+ rrddim_set_by_pointer(dev->st_alloc_efficiency, dev->rd_alloc_efficiency, value);
+ rrdset_done(dev->st_alloc_efficiency);
+
+ return 0;
+}
+
+int do_sys_block_zram(int update_every, usec_t dt) {
+ static procfile *ff = NULL;
+ static DICTIONARY *devices = NULL;
+ static int initialized = 0;
+ static int device_count = 0;
+ int zram_id = -1;
+
+ (void)dt;
+
+ if (unlikely(!initialized))
+ {
+ initialized = 1;
+ ff = procfile_open("/proc/devices", " \t:", PROCFILE_FLAG_DEFAULT);
+ if (ff == NULL)
+ {
+ error("Cannot read /proc/devices");
+ return 1;
+ }
+ ff = procfile_readall(ff);
+ if (!ff)
+ return 1;
+ zram_id = try_get_zram_major_number(ff);
+ if (zram_id == -1)
+ {
+ if (ff != NULL)
+ procfile_close(ff);
+ return 1;
+ }
+ procfile_close(ff);
+
+ devices = dictionary_create(DICT_OPTION_SINGLE_THREADED);
+ device_count = init_devices(devices, (unsigned int)zram_id, update_every);
+ }
+
+ if (unlikely(device_count < 1))
+ return 1;
+
+ dictionary_walkthrough_write(devices, collect_zram_metrics, devices);
+ return 0;
+} \ No newline at end of file
diff --git a/collectors/proc.plugin/sys_class_infiniband.c b/collectors/proc.plugin/sys_class_infiniband.c
new file mode 100644
index 0000000..fca0cb8
--- /dev/null
+++ b/collectors/proc.plugin/sys_class_infiniband.c
@@ -0,0 +1,703 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Heavily inspired from proc_net_dev.c
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_INFINIBAND_NAME "/sys/class/infiniband"
+#define CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND \
+ "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_INFINIBAND_NAME
+
+// ib_device::name[IB_DEVICE_NAME_MAX(64)] + "-" + ib_device::phys_port_cnt[u8 = 3 chars]
+#define IBNAME_MAX 68
+
+// ----------------------------------------------------------------------------
+// infiniband & omnipath standard counters
+
+// I use macro as there's no single file acting as summary, but a lot of different files, so can't use helpers like
+// procfile(). Also, omnipath generates other counters, that are not provided by infiniband
+#define FOREACH_COUNTER(GEN, ...) \
+ FOREACH_COUNTER_BYTES(GEN, __VA_ARGS__) \
+ FOREACH_COUNTER_PACKETS(GEN, __VA_ARGS__) \
+ FOREACH_COUNTER_ERRORS(GEN, __VA_ARGS__)
+
+#define FOREACH_COUNTER_BYTES(GEN, ...) \
+ GEN(port_rcv_data, bytes, "Received", 1, __VA_ARGS__) \
+ GEN(port_xmit_data, bytes, "Sent", -1, __VA_ARGS__)
+
+#define FOREACH_COUNTER_PACKETS(GEN, ...) \
+ GEN(port_rcv_packets, packets, "Received", 1, __VA_ARGS__) \
+ GEN(port_xmit_packets, packets, "Sent", -1, __VA_ARGS__) \
+ GEN(multicast_rcv_packets, packets, "Mcast rcvd", 1, __VA_ARGS__) \
+ GEN(multicast_xmit_packets, packets, "Mcast sent", -1, __VA_ARGS__) \
+ GEN(unicast_rcv_packets, packets, "Ucast rcvd", 1, __VA_ARGS__) \
+ GEN(unicast_xmit_packets, packets, "Ucast sent", -1, __VA_ARGS__)
+
+#define FOREACH_COUNTER_ERRORS(GEN, ...) \
+ GEN(port_rcv_errors, errors, "Pkts malformated", 1, __VA_ARGS__) \
+ GEN(port_rcv_constraint_errors, errors, "Pkts rcvd discarded ", 1, __VA_ARGS__) \
+ GEN(port_xmit_discards, errors, "Pkts sent discarded", 1, __VA_ARGS__) \
+ GEN(port_xmit_wait, errors, "Tick Wait to send", 1, __VA_ARGS__) \
+ GEN(VL15_dropped, errors, "Pkts missed resource", 1, __VA_ARGS__) \
+ GEN(excessive_buffer_overrun_errors, errors, "Buffer overrun", 1, __VA_ARGS__) \
+ GEN(link_downed, errors, "Link Downed", 1, __VA_ARGS__) \
+ GEN(link_error_recovery, errors, "Link recovered", 1, __VA_ARGS__) \
+ GEN(local_link_integrity_errors, errors, "Link integrity err", 1, __VA_ARGS__) \
+ GEN(symbol_error, errors, "Link minor errors", 1, __VA_ARGS__) \
+ GEN(port_rcv_remote_physical_errors, errors, "Pkts rcvd with EBP", 1, __VA_ARGS__) \
+ GEN(port_rcv_switch_relay_errors, errors, "Pkts rcvd discarded by switch", 1, __VA_ARGS__) \
+ GEN(port_xmit_constraint_errors, errors, "Pkts sent discarded by switch", 1, __VA_ARGS__)
+
+//
+// Hardware Counters
+//
+
+// IMPORTANT: These are vendor-specific fields.
+// If you want to add a new vendor, search this for for 'VENDORS:' keyword and
+// add your definition as 'VENDOR-<key>:' where <key> if the string part that
+// is shown in /sys/class/infiniband/<key>X_Y
+// EG: for Mellanox, shown as mlx0_1, it's 'mlx'
+// for Intel, shown as hfi1_1, it's 'hfi'
+
+// VENDORS: List of implemented hardware vendors
+#define FOREACH_HWCOUNTER_NAME(GEN, ...) GEN(mlx, __VA_ARGS__)
+
+// VENDOR-MLX: HW Counters for Mellanox ConnectX Devices
+#define FOREACH_HWCOUNTER_MLX(GEN, ...) \
+ FOREACH_HWCOUNTER_MLX_PACKETS(GEN, __VA_ARGS__) \
+ FOREACH_HWCOUNTER_MLX_ERRORS(GEN, __VA_ARGS__)
+
+#define FOREACH_HWCOUNTER_MLX_PACKETS(GEN, ...) \
+ GEN(np_cnp_sent, hwpackets, "RoCEv2 Congestion sent", 1, __VA_ARGS__) \
+ GEN(np_ecn_marked_roce_packets, hwpackets, "RoCEv2 Congestion rcvd", -1, __VA_ARGS__) \
+ GEN(rp_cnp_handled, hwpackets, "IB Congestion handled", 1, __VA_ARGS__) \
+ GEN(rx_atomic_requests, hwpackets, "ATOMIC req. rcvd", 1, __VA_ARGS__) \
+ GEN(rx_dct_connect, hwpackets, "Connection req. rcvd", 1, __VA_ARGS__) \
+ GEN(rx_read_requests, hwpackets, "Read req. rcvd", 1, __VA_ARGS__) \
+ GEN(rx_write_requests, hwpackets, "Write req. rcvd", 1, __VA_ARGS__) \
+ GEN(roce_adp_retrans, hwpackets, "RoCE retrans adaptive", 1, __VA_ARGS__) \
+ GEN(roce_adp_retrans_to, hwpackets, "RoCE retrans timeout", 1, __VA_ARGS__) \
+ GEN(roce_slow_restart, hwpackets, "RoCE slow restart", 1, __VA_ARGS__) \
+ GEN(roce_slow_restart_cnps, hwpackets, "RoCE slow restart congestion", 1, __VA_ARGS__) \
+ GEN(roce_slow_restart_trans, hwpackets, "RoCE slow restart count", 1, __VA_ARGS__)
+
+#define FOREACH_HWCOUNTER_MLX_ERRORS(GEN, ...) \
+ GEN(duplicate_request, hwerrors, "Duplicated packets", -1, __VA_ARGS__) \
+ GEN(implied_nak_seq_err, hwerrors, "Pkt Seq Num gap", 1, __VA_ARGS__) \
+ GEN(local_ack_timeout_err, hwerrors, "Ack timer expired", 1, __VA_ARGS__) \
+ GEN(out_of_buffer, hwerrors, "Drop missing buffer", 1, __VA_ARGS__) \
+ GEN(out_of_sequence, hwerrors, "Drop out of sequence", 1, __VA_ARGS__) \
+ GEN(packet_seq_err, hwerrors, "NAK sequence rcvd", 1, __VA_ARGS__) \
+ GEN(req_cqe_error, hwerrors, "CQE err Req", 1, __VA_ARGS__) \
+ GEN(resp_cqe_error, hwerrors, "CQE err Resp", 1, __VA_ARGS__) \
+ GEN(req_cqe_flush_error, hwerrors, "CQE Flushed err Req", 1, __VA_ARGS__) \
+ GEN(resp_cqe_flush_error, hwerrors, "CQE Flushed err Resp", 1, __VA_ARGS__) \
+ GEN(req_remote_access_errors, hwerrors, "Remote access err Req", 1, __VA_ARGS__) \
+ GEN(resp_remote_access_errors, hwerrors, "Remote access err Resp", 1, __VA_ARGS__) \
+ GEN(req_remote_invalid_request, hwerrors, "Remote invalid req", 1, __VA_ARGS__) \
+ GEN(resp_local_length_error, hwerrors, "Local length err Resp", 1, __VA_ARGS__) \
+ GEN(rnr_nak_retry_err, hwerrors, "RNR NAK Packets", 1, __VA_ARGS__) \
+ GEN(rp_cnp_ignored, hwerrors, "CNP Pkts ignored", 1, __VA_ARGS__) \
+ GEN(rx_icrc_encapsulated, hwerrors, "RoCE ICRC Errors", 1, __VA_ARGS__)
+
+// Common definitions used more than once
+#define GEN_RRD_DIM_ADD(NAME, GRP, DESC, DIR, PORT) \
+ GEN_RRD_DIM_ADD_CUSTOM(NAME, GRP, DESC, DIR, PORT, 1, 1, RRD_ALGORITHM_INCREMENTAL)
+
+#define GEN_RRD_DIM_ADD_CUSTOM(NAME, GRP, DESC, DIR, PORT, MULT, DIV, ALGO) \
+ PORT->rd_##NAME = rrddim_add(PORT->st_##GRP, DESC, NULL, DIR * MULT, DIV, ALGO);
+
+#define GEN_RRD_DIM_ADD_HW(NAME, GRP, DESC, DIR, PORT, HW) \
+ HW->rd_##NAME = rrddim_add(PORT->st_##GRP, DESC, NULL, DIR, 1, RRD_ALGORITHM_INCREMENTAL);
+
+#define GEN_RRD_DIM_SETP(NAME, GRP, DESC, DIR, PORT) \
+ rrddim_set_by_pointer(PORT->st_##GRP, PORT->rd_##NAME, (collected_number)PORT->NAME);
+
+#define GEN_RRD_DIM_SETP_HW(NAME, GRP, DESC, DIR, PORT, HW) \
+ rrddim_set_by_pointer(PORT->st_##GRP, HW->rd_##NAME, (collected_number)HW->NAME);
+
+// https://community.mellanox.com/s/article/understanding-mlx5-linux-counters-and-status-parameters
+// https://community.mellanox.com/s/article/infiniband-port-counters
+static struct ibport {
+ char *name;
+ char *counters_path;
+ char *hwcounters_path;
+ int len;
+
+ // flags
+ int configured;
+ int enabled;
+ int discovered;
+
+ int do_bytes;
+ int do_packets;
+ int do_errors;
+ int do_hwpackets;
+ int do_hwerrors;
+
+ const char *chart_type_bytes;
+ const char *chart_type_packets;
+ const char *chart_type_errors;
+ const char *chart_type_hwpackets;
+ const char *chart_type_hwerrors;
+
+ const char *chart_id_bytes;
+ const char *chart_id_packets;
+ const char *chart_id_errors;
+ const char *chart_id_hwpackets;
+ const char *chart_id_hwerrors;
+
+ const char *chart_family;
+
+ unsigned long priority;
+
+ // Port details using drivers/infiniband/core/sysfs.c :: rate_show()
+ RRDDIM *rd_speed;
+ uint64_t speed;
+ uint64_t width;
+
+// Stats from /$device/ports/$portid/counters
+// as drivers/infiniband/hw/qib/qib_verbs.h
+// All uint64 except vl15_dropped, local_link_integrity_errors, excessive_buffer_overrun_errors uint32
+// Will generate 2 elements for each counter:
+// - uint64_t to store the value
+// - char* to store the filename path
+// - RRDDIM* to store the RRD Dimension
+#define GEN_DEF_COUNTER(NAME, ...) \
+ uint64_t NAME; \
+ char *file_##NAME; \
+ RRDDIM *rd_##NAME;
+ FOREACH_COUNTER(GEN_DEF_COUNTER)
+
+// Vendor specific hwcounters from /$device/ports/$portid/hw_counters
+// We will generate one struct pointer per vendor to avoid future casting
+#define GEN_DEF_HWCOUNTER_PTR(VENDOR, ...) struct ibporthw_##VENDOR *hwcounters_##VENDOR;
+ FOREACH_HWCOUNTER_NAME(GEN_DEF_HWCOUNTER_PTR)
+
+ // Function pointer to the "infiniband_hwcounters_parse_<vendor>" function
+ void (*hwcounters_parse)(struct ibport *);
+ void (*hwcounters_dorrd)(struct ibport *);
+
+ // charts and dim
+ RRDSET *st_bytes;
+ RRDSET *st_packets;
+ RRDSET *st_errors;
+ RRDSET *st_hwpackets;
+ RRDSET *st_hwerrors;
+
+ const RRDSETVAR_ACQUIRED *stv_speed;
+
+ usec_t speed_last_collected_usec;
+
+ struct ibport *next;
+} *ibport_root = NULL, *ibport_last_used = NULL;
+
+// VENDORS: reading / calculation functions
+#define GEN_DEF_HWCOUNTER(NAME, ...) \
+ uint64_t NAME; \
+ char *file_##NAME; \
+ RRDDIM *rd_##NAME;
+
+#define GEN_DO_HWCOUNTER_READ(NAME, GRP, DESC, DIR, PORT, HW, ...) \
+ if (HW->file_##NAME) { \
+ if (read_single_number_file(HW->file_##NAME, (unsigned long long *)&HW->NAME)) { \
+ error("cannot read iface '%s' hwcounter '" #HW "'", PORT->name); \
+ HW->file_##NAME = NULL; \
+ } \
+ }
+
+// VENDOR-MLX: Mellanox
+struct ibporthw_mlx {
+ FOREACH_HWCOUNTER_MLX(GEN_DEF_HWCOUNTER)
+};
+void infiniband_hwcounters_parse_mlx(struct ibport *port)
+{
+ if (port->do_hwerrors != CONFIG_BOOLEAN_NO)
+ FOREACH_HWCOUNTER_MLX_ERRORS(GEN_DO_HWCOUNTER_READ, port, port->hwcounters_mlx)
+ if (port->do_hwpackets != CONFIG_BOOLEAN_NO)
+ FOREACH_HWCOUNTER_MLX_PACKETS(GEN_DO_HWCOUNTER_READ, port, port->hwcounters_mlx)
+}
+void infiniband_hwcounters_dorrd_mlx(struct ibport *port)
+{
+ if (port->do_hwerrors != CONFIG_BOOLEAN_NO) {
+ FOREACH_HWCOUNTER_MLX_ERRORS(GEN_RRD_DIM_SETP_HW, port, port->hwcounters_mlx)
+ rrdset_done(port->st_hwerrors);
+ }
+ if (port->do_hwpackets != CONFIG_BOOLEAN_NO) {
+ FOREACH_HWCOUNTER_MLX_PACKETS(GEN_RRD_DIM_SETP_HW, port, port->hwcounters_mlx)
+ rrdset_done(port->st_hwpackets);
+ }
+}
+
+// ----------------------------------------------------------------------------
+
+static struct ibport *get_ibport(const char *dev, const char *port)
+{
+ struct ibport *p;
+
+ char name[IBNAME_MAX + 1];
+ snprintfz(name, IBNAME_MAX, "%s-%s", dev, port);
+
+ // search it, resuming from the last position in sequence
+ for (p = ibport_last_used; p; p = p->next) {
+ if (unlikely(!strcmp(name, p->name))) {
+ ibport_last_used = p->next;
+ return p;
+ }
+ }
+
+ // new round, from the beginning to the last position used this time
+ for (p = ibport_root; p != ibport_last_used; p = p->next) {
+ if (unlikely(!strcmp(name, p->name))) {
+ ibport_last_used = p->next;
+ return p;
+ }
+ }
+
+ // create a new one
+ p = callocz(1, sizeof(struct ibport));
+ p->name = strdupz(name);
+ p->len = strlen(p->name);
+
+ p->chart_type_bytes = strdupz("infiniband_cnt_bytes");
+ p->chart_type_packets = strdupz("infiniband_cnt_packets");
+ p->chart_type_errors = strdupz("infiniband_cnt_errors");
+ p->chart_type_hwpackets = strdupz("infiniband_hwc_packets");
+ p->chart_type_hwerrors = strdupz("infiniband_hwc_errors");
+
+ char buffer[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cntbytes_%s", p->name);
+ p->chart_id_bytes = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cntpackets_%s", p->name);
+ p->chart_id_packets = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cnterrors_%s", p->name);
+ p->chart_id_errors = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_hwcntpackets_%s", p->name);
+ p->chart_id_hwpackets = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_hwcnterrors_%s", p->name);
+ p->chart_id_hwerrors = strdupz(buffer);
+
+ p->chart_family = strdupz(p->name);
+ p->priority = NETDATA_CHART_PRIO_INFINIBAND;
+
+ // Link current ibport to last one in the list
+ if (ibport_root) {
+ struct ibport *t;
+ for (t = ibport_root; t->next; t = t->next)
+ ;
+ t->next = p;
+ } else
+ ibport_root = p;
+
+ return p;
+}
+
+int do_sys_class_infiniband(int update_every, usec_t dt)
+{
+ (void)dt;
+ static SIMPLE_PATTERN *disabled_list = NULL;
+ static int initialized = 0;
+ static int enable_new_ports = -1, enable_only_active = CONFIG_BOOLEAN_YES;
+ static int do_bytes = -1, do_packets = -1, do_errors = -1, do_hwpackets = -1, do_hwerrors = -1;
+ static char *sys_class_infiniband_dirname = NULL;
+
+ static long long int dt_to_refresh_ports = 0, last_refresh_ports_usec = 0;
+
+ if (unlikely(enable_new_ports == -1)) {
+ char dirname[FILENAME_MAX + 1];
+
+ snprintfz(dirname, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/infiniband");
+ sys_class_infiniband_dirname =
+ config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "dirname to monitor", dirname);
+
+ do_bytes = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "bandwidth counters", CONFIG_BOOLEAN_YES);
+ do_packets = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "packets counters", CONFIG_BOOLEAN_YES);
+ do_errors = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "errors counters", CONFIG_BOOLEAN_YES);
+ do_hwpackets = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "hardware packets counters", CONFIG_BOOLEAN_AUTO);
+ do_hwerrors = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "hardware errors counters", CONFIG_BOOLEAN_AUTO);
+
+ enable_only_active = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "monitor only active ports", CONFIG_BOOLEAN_AUTO);
+ disabled_list = simple_pattern_create(
+ config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "disable by default interfaces matching", ""), NULL,
+ SIMPLE_PATTERN_EXACT);
+
+ dt_to_refresh_ports =
+ config_get_number(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "refresh ports state every seconds", 30) *
+ USEC_PER_SEC;
+ if (dt_to_refresh_ports < 0)
+ dt_to_refresh_ports = 0;
+ }
+
+ // init listing of /sys/class/infiniband/ (or rediscovery)
+ if (unlikely(!initialized) || unlikely(last_refresh_ports_usec >= dt_to_refresh_ports)) {
+ // If folder does not exists, return 1 to disable
+ DIR *devices_dir = opendir(sys_class_infiniband_dirname);
+ if (unlikely(!devices_dir))
+ return 1;
+
+ // Work on all device available
+ struct dirent *dev_dent;
+ while ((dev_dent = readdir(devices_dir))) {
+ // Skip special folders
+ if (!strcmp(dev_dent->d_name, "..") || !strcmp(dev_dent->d_name, "."))
+ continue;
+
+ // /sys/class/infiniband/<dev>/ports
+ char ports_dirname[FILENAME_MAX + 1];
+ snprintfz(ports_dirname, FILENAME_MAX, "%s/%s/%s", sys_class_infiniband_dirname, dev_dent->d_name, "ports");
+
+ DIR *ports_dir = opendir(ports_dirname);
+ if (unlikely(!ports_dir))
+ continue;
+
+ struct dirent *port_dent;
+ while ((port_dent = readdir(ports_dir))) {
+ // Skip special folders
+ if (!strcmp(port_dent->d_name, "..") || !strcmp(port_dent->d_name, "."))
+ continue;
+
+ char buffer[FILENAME_MAX + 1];
+
+ // Check if counters are available (mandatory)
+ // /sys/class/infiniband/<device>/ports/<port>/counters
+ char counters_dirname[FILENAME_MAX + 1];
+ snprintfz(counters_dirname, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "counters");
+ DIR *counters_dir = opendir(counters_dirname);
+ // Standard counters are mandatory
+ if (!counters_dir)
+ continue;
+ closedir(counters_dir);
+
+ // Hardware Counters are optional, used later
+ char hwcounters_dirname[FILENAME_MAX + 1];
+ snprintfz(
+ hwcounters_dirname, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "hw_counters");
+
+ // Get new or existing ibport
+ struct ibport *p = get_ibport(dev_dent->d_name, port_dent->d_name);
+ if (!p)
+ continue;
+
+ // Prepare configuration
+ if (!p->configured) {
+ p->configured = 1;
+
+ // Enable by default, will be filtered out later
+ p->enabled = 1;
+
+ p->counters_path = strdupz(counters_dirname);
+ p->hwcounters_path = strdupz(hwcounters_dirname);
+
+ snprintfz(buffer, FILENAME_MAX, "plugin:proc:/sys/class/infiniband:%s", p->name);
+
+ // Standard counters
+ p->do_bytes = config_get_boolean_ondemand(buffer, "bytes", do_bytes);
+ p->do_packets = config_get_boolean_ondemand(buffer, "packets", do_packets);
+ p->do_errors = config_get_boolean_ondemand(buffer, "errors", do_errors);
+
+// Gen filename allocation and concatenation
+#define GEN_DO_COUNTER_NAME(NAME, GRP, DESC, DIR, PORT, ...) \
+ PORT->file_##NAME = callocz(1, strlen(PORT->counters_path) + sizeof(#NAME) + 3); \
+ strcat(PORT->file_##NAME, PORT->counters_path); \
+ strcat(PORT->file_##NAME, "/" #NAME);
+ FOREACH_COUNTER(GEN_DO_COUNTER_NAME, p)
+
+ // Check HW Counters vendor dependent
+ DIR *hwcounters_dir = opendir(hwcounters_dirname);
+ if (hwcounters_dir) {
+ // By default set standard
+ p->do_hwpackets = config_get_boolean_ondemand(buffer, "hwpackets", do_hwpackets);
+ p->do_hwerrors = config_get_boolean_ondemand(buffer, "hwerrors", do_hwerrors);
+
+// VENDORS: Set your own
+
+// Allocate the chars to the filenames
+#define GEN_DO_HWCOUNTER_NAME(NAME, GRP, DESC, DIR, PORT, HW, ...) \
+ HW->file_##NAME = callocz(1, strlen(PORT->hwcounters_path) + sizeof(#NAME) + 3); \
+ strcat(HW->file_##NAME, PORT->hwcounters_path); \
+ strcat(HW->file_##NAME, "/" #NAME);
+
+ // VENDOR-MLX: Mellanox
+ if (strncmp(dev_dent->d_name, "mlx", 3) == 0) {
+ // Allocate the vendor specific struct
+ p->hwcounters_mlx = callocz(1, sizeof(struct ibporthw_mlx));
+
+ FOREACH_HWCOUNTER_MLX(GEN_DO_HWCOUNTER_NAME, p, p->hwcounters_mlx)
+
+ // Set the function pointer for hwcounter parsing
+ p->hwcounters_parse = &infiniband_hwcounters_parse_mlx;
+ p->hwcounters_dorrd = &infiniband_hwcounters_dorrd_mlx;
+ }
+
+ // VENDOR: Unknown
+ else {
+ p->do_hwpackets = CONFIG_BOOLEAN_NO;
+ p->do_hwerrors = CONFIG_BOOLEAN_NO;
+ }
+ closedir(hwcounters_dir);
+ }
+ }
+
+ // Check port state to keep activation
+ if (enable_only_active) {
+ snprintfz(buffer, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "state");
+ unsigned long long active;
+ // File is "1: DOWN" or "4: ACTIVE", but str2ull will stop on first non-decimal char
+ read_single_number_file(buffer, &active);
+
+ // Want "IB_PORT_ACTIVE" == "4", as defined by drivers/infiniband/core/sysfs.c::state_show()
+ if (active == 4)
+ p->enabled = 1;
+ else
+ p->enabled = 0;
+ }
+
+ if (p->enabled)
+ p->enabled = !simple_pattern_matches(disabled_list, p->name);
+
+ // Check / Update the link speed & width frm "rate" file
+ // Sample output: "100 Gb/sec (4X EDR)"
+ snprintfz(buffer, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "rate");
+ char buffer_rate[65];
+ if (read_file(buffer, buffer_rate, 64)) {
+ error("Unable to read '%s'", buffer);
+ p->width = 1;
+ } else {
+ char *buffer_width = strstr(buffer_rate, "(");
+ buffer_width++;
+ // str2ull will stop on first non-decimal value
+ p->speed = str2ull(buffer_rate);
+ p->width = str2ull(buffer_width);
+ }
+
+ if (!p->discovered)
+ info(
+ "Infiniband card %s port %s at speed %" PRIu64 " width %" PRIu64 "",
+ dev_dent->d_name,
+ port_dent->d_name,
+ p->speed,
+ p->width);
+
+ p->discovered = 1;
+ }
+ closedir(ports_dir);
+ }
+ closedir(devices_dir);
+
+ initialized = 1;
+ last_refresh_ports_usec = 0;
+ }
+ last_refresh_ports_usec += dt;
+
+ // Update all ports values
+ struct ibport *port;
+ for (port = ibport_root; port; port = port->next) {
+ if (!port->enabled)
+ continue;
+ //
+ // Read values from system to struct
+ //
+
+// counter from file and place it in ibport struct
+#define GEN_DO_COUNTER_READ(NAME, GRP, DESC, DIR, PORT, ...) \
+ if (PORT->file_##NAME) { \
+ if (read_single_number_file(PORT->file_##NAME, (unsigned long long *)&PORT->NAME)) { \
+ error("cannot read iface '%s' counter '" #NAME "'", PORT->name); \
+ PORT->file_##NAME = NULL; \
+ } \
+ }
+
+ // Update charts
+ if (port->do_bytes != CONFIG_BOOLEAN_NO) {
+ // Read values from sysfs
+ FOREACH_COUNTER_BYTES(GEN_DO_COUNTER_READ, port)
+
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_bytes)) {
+ port->st_bytes = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_bytes,
+ NULL,
+ port->chart_family,
+ "ib.bytes",
+ "Bandwidth usage",
+ "kilobits/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 1,
+ update_every,
+ RRDSET_TYPE_AREA);
+ // Create Dimensions
+ rrdset_flag_set(port->st_bytes, RRDSET_FLAG_DETAIL);
+ // On this chart, we want to have a KB/s so the dashboard will autoscale it
+ // The reported values are also per-lane, so we must multiply it by the width
+ // x4 lanes multiplier as per Documentation/ABI/stable/sysfs-class-infiniband
+ FOREACH_COUNTER_BYTES(GEN_RRD_DIM_ADD_CUSTOM, port, 4 * 8 * port->width, 1024, RRD_ALGORITHM_INCREMENTAL)
+
+ port->stv_speed = rrdsetvar_custom_chart_variable_add_and_acquire(port->st_bytes, "link_speed");
+ }
+
+ // Link read values to dimensions
+ FOREACH_COUNTER_BYTES(GEN_RRD_DIM_SETP, port)
+
+ // For link speed set only variable
+ rrdsetvar_custom_chart_variable_set(port->st_bytes, port->stv_speed, port->speed);
+
+ rrdset_done(port->st_bytes);
+ }
+
+ if (port->do_packets != CONFIG_BOOLEAN_NO) {
+ // Read values from sysfs
+ FOREACH_COUNTER_PACKETS(GEN_DO_COUNTER_READ, port)
+
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_packets)) {
+ port->st_packets = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_packets,
+ NULL,
+ port->chart_family,
+ "ib.packets",
+ "Packets Statistics",
+ "packets/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 2,
+ update_every,
+ RRDSET_TYPE_AREA);
+ // Create Dimensions
+ rrdset_flag_set(port->st_packets, RRDSET_FLAG_DETAIL);
+ FOREACH_COUNTER_PACKETS(GEN_RRD_DIM_ADD, port)
+ }
+
+ // Link read values to dimensions
+ FOREACH_COUNTER_PACKETS(GEN_RRD_DIM_SETP, port)
+ rrdset_done(port->st_packets);
+ }
+
+ if (port->do_errors != CONFIG_BOOLEAN_NO) {
+ // Read values from sysfs
+ FOREACH_COUNTER_ERRORS(GEN_DO_COUNTER_READ, port)
+
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_errors)) {
+ port->st_errors = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_errors,
+ NULL,
+ port->chart_family,
+ "ib.errors",
+ "Error Counters",
+ "errors/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 3,
+ update_every,
+ RRDSET_TYPE_LINE);
+ // Create Dimensions
+ rrdset_flag_set(port->st_errors, RRDSET_FLAG_DETAIL);
+ FOREACH_COUNTER_ERRORS(GEN_RRD_DIM_ADD, port)
+ }
+
+ // Link read values to dimensions
+ FOREACH_COUNTER_ERRORS(GEN_RRD_DIM_SETP, port)
+ rrdset_done(port->st_errors);
+ }
+
+ //
+ // HW Counters
+ //
+
+ // Call the function for parsing and setting hwcounters
+ if (port->hwcounters_parse && port->hwcounters_dorrd) {
+ // Read all values (done by vendor-specific function)
+ (*port->hwcounters_parse)(port);
+
+ if (port->do_hwerrors != CONFIG_BOOLEAN_NO) {
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_hwerrors)) {
+ port->st_hwerrors = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_hwerrors,
+ NULL,
+ port->chart_family,
+ "ib.hwerrors",
+ "Hardware Errors",
+ "errors/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 4,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_flag_set(port->st_hwerrors, RRDSET_FLAG_DETAIL);
+
+ // VENDORS: Set your selection
+
+ // VENDOR: Mellanox
+ if (strncmp(port->name, "mlx", 3) == 0) {
+ FOREACH_HWCOUNTER_MLX_ERRORS(GEN_RRD_DIM_ADD_HW, port, port->hwcounters_mlx)
+ }
+
+ // Unknown vendor, should not happen
+ else {
+ error(
+ "Unmanaged vendor for '%s', do_hwerrors should have been set to no. Please report this bug",
+ port->name);
+ port->do_hwerrors = CONFIG_BOOLEAN_NO;
+ }
+ }
+ }
+
+ if (port->do_hwpackets != CONFIG_BOOLEAN_NO) {
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_hwpackets)) {
+ port->st_hwpackets = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_hwpackets,
+ NULL,
+ port->chart_family,
+ "ib.hwpackets",
+ "Hardware Packets Statistics",
+ "packets/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 5,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_flag_set(port->st_hwpackets, RRDSET_FLAG_DETAIL);
+
+ // VENDORS: Set your selection
+
+ // VENDOR: Mellanox
+ if (strncmp(port->name, "mlx", 3) == 0) {
+ FOREACH_HWCOUNTER_MLX_PACKETS(GEN_RRD_DIM_ADD_HW, port, port->hwcounters_mlx)
+ }
+
+ // Unknown vendor, should not happen
+ else {
+ error(
+ "Unmanaged vendor for '%s', do_hwpackets should have been set to no. Please report this bug",
+ port->name);
+ port->do_hwpackets = CONFIG_BOOLEAN_NO;
+ }
+ }
+ }
+
+ // Update values to rrd (done by vendor-specific function)
+ (*port->hwcounters_dorrd)(port);
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/sys_class_power_supply.c b/collectors/proc.plugin/sys_class_power_supply.c
new file mode 100644
index 0000000..dde4215
--- /dev/null
+++ b/collectors/proc.plugin/sys_class_power_supply.c
@@ -0,0 +1,414 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_POWER_SUPPLY_NAME "/sys/class/power_supply"
+
+const char *ps_property_names[] = { "charge", "energy", "voltage"};
+const char *ps_property_titles[] = {"Battery charge", "Battery energy", "Power supply voltage"};
+const char *ps_property_units[] = { "Ah", "Wh", "V"};
+
+const char *ps_property_dim_names[] = {"empty_design", "empty", "now", "full", "full_design",
+ "empty_design", "empty", "now", "full", "full_design",
+ "min_design", "min", "now", "max", "max_design"};
+
+struct ps_property_dim {
+ char *name;
+ char *filename;
+ int fd;
+
+ RRDDIM *rd;
+ unsigned long long value;
+ int always_zero;
+
+ struct ps_property_dim *next;
+};
+
+struct ps_property {
+ char *name;
+ char *title;
+ char *units;
+
+ RRDSET *st;
+
+ struct ps_property_dim *property_dim_root;
+
+ struct ps_property *next;
+};
+
+struct capacity {
+ char *filename;
+ int fd;
+
+ RRDSET *st;
+ RRDDIM *rd;
+ unsigned long long value;
+};
+
+struct power_supply {
+ char *name;
+ uint32_t hash;
+ int found;
+
+ struct capacity *capacity;
+
+ struct ps_property *property_root;
+
+ struct power_supply *next;
+};
+
+static struct power_supply *power_supply_root = NULL;
+static int files_num = 0;
+
+void power_supply_free(struct power_supply *ps) {
+ if(likely(ps)) {
+
+ // free capacity structure
+ if(likely(ps->capacity)) {
+ if(likely(ps->capacity->st)) rrdset_is_obsolete(ps->capacity->st);
+ freez(ps->capacity->filename);
+ if(likely(ps->capacity->fd != -1)) close(ps->capacity->fd);
+ files_num--;
+ freez(ps->capacity);
+ }
+ freez(ps->name);
+
+ struct ps_property *pr = ps->property_root;
+ while(likely(pr)) {
+
+ // free dimensions
+ struct ps_property_dim *pd = pr->property_dim_root;
+ while(likely(pd)) {
+ freez(pd->name);
+ freez(pd->filename);
+ if(likely(pd->fd != -1)) close(pd->fd);
+ files_num--;
+ struct ps_property_dim *d = pd;
+ pd = pd->next;
+ freez(d);
+ }
+
+ // free properties
+ if(likely(pr->st)) rrdset_is_obsolete(pr->st);
+ freez(pr->name);
+ freez(pr->title);
+ freez(pr->units);
+ struct ps_property *p = pr;
+ pr = pr->next;
+ freez(p);
+ }
+
+ // remove power supply from linked list
+ if(likely(ps == power_supply_root)) {
+ power_supply_root = ps->next;
+ }
+ else {
+ struct power_supply *last;
+ for(last = power_supply_root; last && last->next != ps; last = last->next);
+ if(likely(last)) last->next = ps->next;
+ }
+
+ freez(ps);
+ }
+}
+
+static void add_labels_to_power_supply(struct power_supply *ps, RRDSET *st) {
+ rrdlabels_add(st->rrdlabels, "device", ps->name, RRDLABEL_SRC_AUTO);
+}
+
+int do_sys_class_power_supply(int update_every, usec_t dt) {
+ (void)dt;
+ static int do_capacity = -1, do_property[3] = {-1};
+ static int keep_fds_open = CONFIG_BOOLEAN_NO, keep_fds_open_config = -1;
+ static char *dirname = NULL;
+
+ if(unlikely(do_capacity == -1)) {
+ do_capacity = config_get_boolean("plugin:proc:/sys/class/power_supply", "battery capacity", CONFIG_BOOLEAN_YES);
+ do_property[0] = config_get_boolean("plugin:proc:/sys/class/power_supply", "battery charge", CONFIG_BOOLEAN_NO);
+ do_property[1] = config_get_boolean("plugin:proc:/sys/class/power_supply", "battery energy", CONFIG_BOOLEAN_NO);
+ do_property[2] = config_get_boolean("plugin:proc:/sys/class/power_supply", "power supply voltage", CONFIG_BOOLEAN_NO);
+
+ keep_fds_open_config = config_get_boolean_ondemand("plugin:proc:/sys/class/power_supply", "keep files open", CONFIG_BOOLEAN_AUTO);
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/power_supply");
+ dirname = config_get("plugin:proc:/sys/class/power_supply", "directory to monitor", filename);
+ }
+
+ DIR *dir = opendir(dirname);
+ if(unlikely(!dir)) {
+ error("Cannot read directory '%s'", dirname);
+ return 1;
+ }
+
+ struct dirent *de = NULL;
+ while(likely(de = readdir(dir))) {
+ if(likely(de->d_type == DT_DIR
+ && (
+ (de->d_name[0] == '.' && de->d_name[1] == '\0')
+ || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ )))
+ continue;
+
+ if(likely(de->d_type == DT_LNK || de->d_type == DT_DIR)) {
+ uint32_t hash = simple_hash(de->d_name);
+
+ struct power_supply *ps;
+ for(ps = power_supply_root; ps; ps = ps->next) {
+ if(unlikely(ps->hash == hash && !strcmp(ps->name, de->d_name))) {
+ ps->found = 1;
+ break;
+ }
+ }
+
+ // allocate memory for power supply and initialize it
+ if(unlikely(!ps)) {
+ ps = callocz(sizeof(struct power_supply), 1);
+ ps->name = strdupz(de->d_name);
+ ps->hash = simple_hash(de->d_name);
+ ps->found = 1;
+ ps->next = power_supply_root;
+ power_supply_root = ps;
+
+ struct stat stbuf;
+ if(likely(do_capacity != CONFIG_BOOLEAN_NO)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/%s/%s", dirname, de->d_name, "capacity");
+ if (stat(filename, &stbuf) == 0) {
+ ps->capacity = callocz(sizeof(struct capacity), 1);
+ ps->capacity->filename = strdupz(filename);
+ ps->capacity->fd = -1;
+ files_num++;
+ }
+ }
+
+ // allocate memory and initialize structures for every property and file found
+ size_t pr_idx, pd_idx;
+ size_t prev_idx = 3; // there is no property with this index
+
+ for(pr_idx = 0; pr_idx < 3; pr_idx++) {
+ if(unlikely(do_property[pr_idx] != CONFIG_BOOLEAN_NO)) {
+ struct ps_property *pr = NULL;
+ int min_value_found = 0, max_value_found = 0;
+
+ for(pd_idx = pr_idx * 5; pd_idx < pr_idx * 5 + 5; pd_idx++) {
+
+ // check if file exists
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/%s/%s_%s", dirname, de->d_name,
+ ps_property_names[pr_idx], ps_property_dim_names[pd_idx]);
+ if (stat(filename, &stbuf) == 0) {
+
+ if(unlikely(pd_idx == pr_idx * 5 + 1))
+ min_value_found = 1;
+ if(unlikely(pd_idx == pr_idx * 5 + 3))
+ max_value_found = 1;
+
+ // add chart
+ if(unlikely(prev_idx != pr_idx)) {
+ pr = callocz(sizeof(struct ps_property), 1);
+ pr->name = strdupz(ps_property_names[pr_idx]);
+ pr->title = strdupz(ps_property_titles[pr_idx]);
+ pr->units = strdupz(ps_property_units[pr_idx]);
+ prev_idx = pr_idx;
+ pr->next = ps->property_root;
+ ps->property_root = pr;
+ }
+
+ // add dimension
+ struct ps_property_dim *pd;
+ pd= callocz(sizeof(struct ps_property_dim), 1);
+ pd->name = strdupz(ps_property_dim_names[pd_idx]);
+ pd->filename = strdupz(filename);
+ pd->fd = -1;
+ files_num++;
+ pd->next = pr->property_dim_root;
+ pr->property_dim_root = pd;
+ }
+ }
+
+ // create a zero empty/min dimension
+ if(unlikely(max_value_found && !min_value_found)) {
+ struct ps_property_dim *pd;
+ pd= callocz(sizeof(struct ps_property_dim), 1);
+ pd->name = strdupz(ps_property_dim_names[pr_idx * 5 + 1]);
+ pd->always_zero = 1;
+ pd->next = pr->property_dim_root;
+ pr->property_dim_root = pd;
+ }
+ }
+ }
+ }
+
+ // read capacity file
+ if(likely(ps->capacity)) {
+ char buffer[30 + 1];
+
+ if(unlikely(ps->capacity->fd == -1)) {
+ ps->capacity->fd = open(ps->capacity->filename, O_RDONLY, 0666);
+ if(unlikely(ps->capacity->fd == -1)) {
+ error("Cannot open file '%s'", ps->capacity->filename);
+ power_supply_free(ps);
+ ps = NULL;
+ }
+ }
+
+ if (ps)
+ {
+ ssize_t r = read(ps->capacity->fd, buffer, 30);
+ if(unlikely(r < 1)) {
+ error("Cannot read file '%s'", ps->capacity->filename);
+ power_supply_free(ps);
+ ps = NULL;
+ }
+ else {
+ buffer[r] = '\0';
+ ps->capacity->value = str2ull(buffer);
+
+ if(unlikely(!keep_fds_open)) {
+ close(ps->capacity->fd);
+ ps->capacity->fd = -1;
+ }
+ else if(unlikely(lseek(ps->capacity->fd, 0, SEEK_SET) == -1)) {
+ error("Cannot seek in file '%s'", ps->capacity->filename);
+ close(ps->capacity->fd);
+ ps->capacity->fd = -1;
+ }
+ }
+ }
+ }
+
+ // read property files
+ int read_error = 0;
+ struct ps_property *pr;
+ if (ps)
+ {
+ for(pr = ps->property_root; pr && !read_error; pr = pr->next) {
+ struct ps_property_dim *pd;
+ for(pd = pr->property_dim_root; pd; pd = pd->next) {
+ if(likely(!pd->always_zero)) {
+ char buffer[30 + 1];
+
+ if(unlikely(pd->fd == -1)) {
+ pd->fd = open(pd->filename, O_RDONLY, 0666);
+ if(unlikely(pd->fd == -1)) {
+ error("Cannot open file '%s'", pd->filename);
+ read_error = 1;
+ power_supply_free(ps);
+ break;
+ }
+ }
+
+ ssize_t r = read(pd->fd, buffer, 30);
+ if(unlikely(r < 1)) {
+ error("Cannot read file '%s'", pd->filename);
+ read_error = 1;
+ power_supply_free(ps);
+ break;
+ }
+ buffer[r] = '\0';
+ pd->value = str2ull(buffer);
+
+ if(unlikely(!keep_fds_open)) {
+ close(pd->fd);
+ pd->fd = -1;
+ }
+ else if(unlikely(lseek(pd->fd, 0, SEEK_SET) == -1)) {
+ error("Cannot seek in file '%s'", pd->filename);
+ close(pd->fd);
+ pd->fd = -1;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ closedir(dir);
+
+ keep_fds_open = keep_fds_open_config;
+ if(likely(keep_fds_open_config == CONFIG_BOOLEAN_AUTO)) {
+ if(unlikely(files_num > 32))
+ keep_fds_open = CONFIG_BOOLEAN_NO;
+ else
+ keep_fds_open = CONFIG_BOOLEAN_YES;
+ }
+
+ // --------------------------------------------------------------------
+
+ struct power_supply *ps = power_supply_root;
+ while(unlikely(ps)) {
+ if(unlikely(!ps->found)) {
+ struct power_supply *f = ps;
+ ps = ps->next;
+ power_supply_free(f);
+ continue;
+ }
+
+ if(likely(ps->capacity)) {
+ if(unlikely(!ps->capacity->st)) {
+ ps->capacity->st = rrdset_create_localhost(
+ "powersupply_capacity"
+ , ps->name
+ , NULL
+ , ps->name
+ , "powersupply.capacity"
+ , "Battery capacity"
+ , "percentage"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_POWER_SUPPLY_NAME
+ , NETDATA_CHART_PRIO_POWER_SUPPLY_CAPACITY
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ add_labels_to_power_supply(ps, ps->capacity->st);
+ }
+
+ if(unlikely(!ps->capacity->rd)) ps->capacity->rd = rrddim_add(ps->capacity->st, "capacity", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_set_by_pointer(ps->capacity->st, ps->capacity->rd, ps->capacity->value);
+
+ rrdset_done(ps->capacity->st);
+ }
+
+ struct ps_property *pr;
+ for(pr = ps->property_root; pr; pr = pr->next) {
+ if(unlikely(!pr->st)) {
+ char id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(id, RRD_ID_LENGTH_MAX, "powersupply_%s", pr->name);
+ snprintfz(context, RRD_ID_LENGTH_MAX, "powersupply.%s", pr->name);
+
+ pr->st = rrdset_create_localhost(
+ id
+ , ps->name
+ , NULL
+ , ps->name
+ , context
+ , pr->title
+ , pr->units
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_POWER_SUPPLY_NAME
+ , NETDATA_CHART_PRIO_POWER_SUPPLY_CAPACITY
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ add_labels_to_power_supply(ps, pr->st);
+ }
+
+ struct ps_property_dim *pd;
+ for(pd = pr->property_dim_root; pd; pd = pd->next) {
+ if(unlikely(!pd->rd)) pd->rd = rrddim_add(pr->st, pd->name, NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_set_by_pointer(pr->st, pd->rd, pd->value);
+ }
+
+ rrdset_done(pr->st);
+ }
+
+ ps->found = 0;
+ ps = ps->next;
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/sys_devices_system_edac_mc.c b/collectors/proc.plugin/sys_devices_system_edac_mc.c
new file mode 100644
index 0000000..13d2097
--- /dev/null
+++ b/collectors/proc.plugin/sys_devices_system_edac_mc.c
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+struct mc {
+ char *name;
+ char ce_updated;
+ char ue_updated;
+
+ char *ce_count_filename;
+ char *ue_count_filename;
+
+ procfile *ce_ff;
+ procfile *ue_ff;
+
+ collected_number ce_count;
+ collected_number ue_count;
+
+ RRDDIM *ce_rd;
+ RRDDIM *ue_rd;
+
+ struct mc *next;
+};
+static struct mc *mc_root = NULL;
+
+static void find_all_mc() {
+ char name[FILENAME_MAX + 1];
+ snprintfz(name, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/edac/mc");
+ char *dirname = config_get("plugin:proc:/sys/devices/system/edac/mc", "directory to monitor", name);
+
+ DIR *dir = opendir(dirname);
+ if(unlikely(!dir)) {
+ error("Cannot read ECC memory errors directory '%s'", dirname);
+ return;
+ }
+
+ struct dirent *de = NULL;
+ while((de = readdir(dir))) {
+ if(de->d_type == DT_DIR && de->d_name[0] == 'm' && de->d_name[1] == 'c' && isdigit(de->d_name[2])) {
+ struct mc *m = callocz(1, sizeof(struct mc));
+ m->name = strdupz(de->d_name);
+
+ struct stat st;
+
+ snprintfz(name, FILENAME_MAX, "%s/%s/ce_count", dirname, de->d_name);
+ if(stat(name, &st) != -1)
+ m->ce_count_filename = strdupz(name);
+
+ snprintfz(name, FILENAME_MAX, "%s/%s/ue_count", dirname, de->d_name);
+ if(stat(name, &st) != -1)
+ m->ue_count_filename = strdupz(name);
+
+ if(!m->ce_count_filename && !m->ue_count_filename) {
+ freez(m->name);
+ freez(m);
+ }
+ else {
+ m->next = mc_root;
+ mc_root = m;
+ }
+ }
+ }
+
+ closedir(dir);
+}
+
+int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt) {
+ (void)dt;
+
+ if(unlikely(mc_root == NULL)) {
+ find_all_mc();
+ if(unlikely(mc_root == NULL))
+ return 1;
+ }
+
+ static int do_ce = -1, do_ue = -1;
+ NETDATA_DOUBLE ce_sum = 0, ue_sum = 0;
+ struct mc *m;
+
+ if(unlikely(do_ce == -1)) {
+ do_ce = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/edac/mc", "enable ECC memory correctable errors", CONFIG_BOOLEAN_YES);
+ do_ue = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/edac/mc", "enable ECC memory uncorrectable errors", CONFIG_BOOLEAN_YES);
+ }
+
+ if(do_ce != CONFIG_BOOLEAN_NO) {
+ for(m = mc_root; m; m = m->next) {
+ if(m->ce_count_filename) {
+ m->ce_updated = 0;
+
+ if(unlikely(!m->ce_ff)) {
+ m->ce_ff = procfile_open(m->ce_count_filename, " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!m->ce_ff))
+ continue;
+ }
+
+ m->ce_ff = procfile_readall(m->ce_ff);
+ if(unlikely(!m->ce_ff || procfile_lines(m->ce_ff) < 1 || procfile_linewords(m->ce_ff, 0) < 1))
+ continue;
+
+ m->ce_count = str2ull(procfile_lineword(m->ce_ff, 0, 0));
+ ce_sum += m->ce_count;
+ m->ce_updated = 1;
+ }
+ }
+ }
+
+ if(do_ue != CONFIG_BOOLEAN_NO) {
+ for(m = mc_root; m; m = m->next) {
+ if(m->ue_count_filename) {
+ m->ue_updated = 0;
+
+ if(unlikely(!m->ue_ff)) {
+ m->ue_ff = procfile_open(m->ue_count_filename, " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!m->ue_ff))
+ continue;
+ }
+
+ m->ue_ff = procfile_readall(m->ue_ff);
+ if(unlikely(!m->ue_ff || procfile_lines(m->ue_ff) < 1 || procfile_linewords(m->ue_ff, 0) < 1))
+ continue;
+
+ m->ue_count = str2ull(procfile_lineword(m->ue_ff, 0, 0));
+ ue_sum += m->ue_count;
+ m->ue_updated = 1;
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_ce == CONFIG_BOOLEAN_YES || (do_ce == CONFIG_BOOLEAN_AUTO &&
+ (ce_sum > 0 || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ce = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *ce_st = NULL;
+
+ if(unlikely(!ce_st)) {
+ ce_st = rrdset_create_localhost(
+ "mem"
+ , "ecc_ce"
+ , NULL
+ , "ecc"
+ , NULL
+ , "ECC Memory Correctable Errors"
+ , "errors"
+ , PLUGIN_PROC_NAME
+ , "/sys/devices/system/edac/mc"
+ , NETDATA_CHART_PRIO_MEM_HW_ECC_CE
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ }
+
+ for(m = mc_root; m; m = m->next) {
+ if (m->ce_count_filename && m->ce_updated) {
+ if(unlikely(!m->ce_rd))
+ m->ce_rd = rrddim_add(ce_st, m->name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(ce_st, m->ce_rd, m->ce_count);
+ }
+ }
+
+ rrdset_done(ce_st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(do_ue == CONFIG_BOOLEAN_YES || (do_ue == CONFIG_BOOLEAN_AUTO &&
+ (ue_sum > 0 || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_ue = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *ue_st = NULL;
+
+ if(unlikely(!ue_st)) {
+ ue_st = rrdset_create_localhost(
+ "mem"
+ , "ecc_ue"
+ , NULL
+ , "ecc"
+ , NULL
+ , "ECC Memory Uncorrectable Errors"
+ , "errors"
+ , PLUGIN_PROC_NAME
+ , "/sys/devices/system/edac/mc"
+ , NETDATA_CHART_PRIO_MEM_HW_ECC_UE
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+ }
+
+ for(m = mc_root; m; m = m->next) {
+ if (m->ue_count_filename && m->ue_updated) {
+ if(unlikely(!m->ue_rd))
+ m->ue_rd = rrddim_add(ue_st, m->name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ rrddim_set_by_pointer(ue_st, m->ue_rd, m->ue_count);
+ }
+ }
+
+ rrdset_done(ue_st);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/sys_devices_system_node.c b/collectors/proc.plugin/sys_devices_system_node.c
new file mode 100644
index 0000000..90aafd5
--- /dev/null
+++ b/collectors/proc.plugin/sys_devices_system_node.c
@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+struct node {
+ char *name;
+ char *numastat_filename;
+ procfile *numastat_ff;
+ RRDSET *numastat_st;
+ struct node *next;
+};
+static struct node *numa_root = NULL;
+
+static int find_all_nodes() {
+ int numa_node_count = 0;
+ char name[FILENAME_MAX + 1];
+ snprintfz(name, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/system/node");
+ char *dirname = config_get("plugin:proc:/sys/devices/system/node", "directory to monitor", name);
+
+ DIR *dir = opendir(dirname);
+ if(!dir) {
+ error("Cannot read NUMA node directory '%s'", dirname);
+ return 0;
+ }
+
+ struct dirent *de = NULL;
+ while((de = readdir(dir))) {
+ if(de->d_type != DT_DIR)
+ continue;
+
+ if(strncmp(de->d_name, "node", 4) != 0)
+ continue;
+
+ if(!isdigit(de->d_name[4]))
+ continue;
+
+ numa_node_count++;
+
+ struct node *m = callocz(1, sizeof(struct node));
+ m->name = strdupz(de->d_name);
+
+ struct stat st;
+
+ snprintfz(name, FILENAME_MAX, "%s/%s/numastat", dirname, de->d_name);
+ if(stat(name, &st) == -1) {
+ freez(m->name);
+ freez(m);
+ continue;
+ }
+
+ m->numastat_filename = strdupz(name);
+
+ m->next = numa_root;
+ numa_root = m;
+ }
+
+ closedir(dir);
+
+ return numa_node_count;
+}
+
+int do_proc_sys_devices_system_node(int update_every, usec_t dt) {
+ (void)dt;
+
+ static uint32_t hash_local_node = 0, hash_numa_foreign = 0, hash_interleave_hit = 0, hash_other_node = 0, hash_numa_hit = 0, hash_numa_miss = 0;
+ static int do_numastat = -1, numa_node_count = 0;
+ struct node *m;
+
+ if(unlikely(numa_root == NULL)) {
+ numa_node_count = find_all_nodes();
+ if(unlikely(numa_root == NULL))
+ return 1;
+ }
+
+ if(unlikely(do_numastat == -1)) {
+ do_numastat = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/node", "enable per-node numa metrics", CONFIG_BOOLEAN_AUTO);
+
+ hash_local_node = simple_hash("local_node");
+ hash_numa_foreign = simple_hash("numa_foreign");
+ hash_interleave_hit = simple_hash("interleave_hit");
+ hash_other_node = simple_hash("other_node");
+ hash_numa_hit = simple_hash("numa_hit");
+ hash_numa_miss = simple_hash("numa_miss");
+ }
+
+ if(do_numastat == CONFIG_BOOLEAN_YES || (do_numastat == CONFIG_BOOLEAN_AUTO &&
+ (numa_node_count >= 2 || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ for(m = numa_root; m; m = m->next) {
+ if(m->numastat_filename) {
+
+ if(unlikely(!m->numastat_ff)) {
+ m->numastat_ff = procfile_open(m->numastat_filename, " ", PROCFILE_FLAG_DEFAULT);
+
+ if(unlikely(!m->numastat_ff))
+ continue;
+ }
+
+ m->numastat_ff = procfile_readall(m->numastat_ff);
+ if(unlikely(!m->numastat_ff || procfile_lines(m->numastat_ff) < 1 || procfile_linewords(m->numastat_ff, 0) < 1))
+ continue;
+
+ if(unlikely(!m->numastat_st)) {
+ m->numastat_st = rrdset_create_localhost(
+ "mem"
+ , m->name
+ , NULL
+ , "numa"
+ , NULL
+ , "NUMA events"
+ , "events/s"
+ , PLUGIN_PROC_NAME
+ , "/sys/devices/system/node"
+ , NETDATA_CHART_PRIO_MEM_NUMA_NODES
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rrdlabels_add(m->numastat_st->rrdlabels, "numa_node", m->name, RRDLABEL_SRC_AUTO);
+
+ rrdset_flag_set(m->numastat_st, RRDSET_FLAG_DETAIL);
+
+ rrddim_add(m->numastat_st, "numa_hit", "hit", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(m->numastat_st, "numa_miss", "miss", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(m->numastat_st, "local_node", "local", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(m->numastat_st, "numa_foreign", "foreign", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(m->numastat_st, "interleave_hit", "interleave", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(m->numastat_st, "other_node", "other", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ }
+
+ size_t lines = procfile_lines(m->numastat_ff), l;
+ for(l = 0; l < lines; l++) {
+ size_t words = procfile_linewords(m->numastat_ff, l);
+
+ if(unlikely(words < 2)) {
+ if(unlikely(words))
+ error("Cannot read %s numastat line %zu. Expected 2 params, read %zu.", m->name, l, words);
+ continue;
+ }
+
+ char *name = procfile_lineword(m->numastat_ff, l, 0);
+ char *value = procfile_lineword(m->numastat_ff, l, 1);
+
+ if (unlikely(!name || !*name || !value || !*value))
+ continue;
+
+ uint32_t hash = simple_hash(name);
+ if(likely(
+ (hash == hash_numa_hit && !strcmp(name, "numa_hit"))
+ || (hash == hash_numa_miss && !strcmp(name, "numa_miss"))
+ || (hash == hash_local_node && !strcmp(name, "local_node"))
+ || (hash == hash_numa_foreign && !strcmp(name, "numa_foreign"))
+ || (hash == hash_interleave_hit && !strcmp(name, "interleave_hit"))
+ || (hash == hash_other_node && !strcmp(name, "other_node"))
+ ))
+ rrddim_set(m->numastat_st, name, (collected_number)str2kernel_uint_t(value));
+ }
+
+ rrdset_done(m->numastat_st);
+ }
+ }
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/sys_fs_btrfs.c b/collectors/proc.plugin/sys_fs_btrfs.c
new file mode 100644
index 0000000..3b9841f
--- /dev/null
+++ b/collectors/proc.plugin/sys_fs_btrfs.c
@@ -0,0 +1,743 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_BTRFS_NAME "/sys/fs/btrfs"
+
+typedef struct btrfs_disk {
+ char *name;
+ uint32_t hash;
+ int exists;
+
+ char *size_filename;
+ char *hw_sector_size_filename;
+ unsigned long long size;
+ unsigned long long hw_sector_size;
+
+ struct btrfs_disk *next;
+} BTRFS_DISK;
+
+typedef struct btrfs_node {
+ int exists;
+ int logged_error;
+
+ char *id;
+ uint32_t hash;
+
+ char *label;
+
+ // unsigned long long int sectorsize;
+ // unsigned long long int nodesize;
+ // unsigned long long int quota_override;
+
+ #define declare_btrfs_allocation_section_field(SECTION, FIELD) \
+ char *allocation_ ## SECTION ## _ ## FIELD ## _filename; \
+ unsigned long long int allocation_ ## SECTION ## _ ## FIELD;
+
+ #define declare_btrfs_allocation_field(FIELD) \
+ char *allocation_ ## FIELD ## _filename; \
+ unsigned long long int allocation_ ## FIELD;
+
+ RRDSET *st_allocation_disks;
+ RRDDIM *rd_allocation_disks_unallocated;
+ RRDDIM *rd_allocation_disks_data_used;
+ RRDDIM *rd_allocation_disks_data_free;
+ RRDDIM *rd_allocation_disks_metadata_used;
+ RRDDIM *rd_allocation_disks_metadata_free;
+ RRDDIM *rd_allocation_disks_system_used;
+ RRDDIM *rd_allocation_disks_system_free;
+ unsigned long long all_disks_total;
+
+ RRDSET *st_allocation_data;
+ RRDDIM *rd_allocation_data_free;
+ RRDDIM *rd_allocation_data_used;
+ declare_btrfs_allocation_section_field(data, total_bytes)
+ declare_btrfs_allocation_section_field(data, bytes_used)
+ declare_btrfs_allocation_section_field(data, disk_total)
+ declare_btrfs_allocation_section_field(data, disk_used)
+
+ RRDSET *st_allocation_metadata;
+ RRDDIM *rd_allocation_metadata_free;
+ RRDDIM *rd_allocation_metadata_used;
+ RRDDIM *rd_allocation_metadata_reserved;
+ declare_btrfs_allocation_section_field(metadata, total_bytes)
+ declare_btrfs_allocation_section_field(metadata, bytes_used)
+ declare_btrfs_allocation_section_field(metadata, disk_total)
+ declare_btrfs_allocation_section_field(metadata, disk_used)
+ //declare_btrfs_allocation_field(global_rsv_reserved)
+ declare_btrfs_allocation_field(global_rsv_size)
+
+ RRDSET *st_allocation_system;
+ RRDDIM *rd_allocation_system_free;
+ RRDDIM *rd_allocation_system_used;
+ declare_btrfs_allocation_section_field(system, total_bytes)
+ declare_btrfs_allocation_section_field(system, bytes_used)
+ declare_btrfs_allocation_section_field(system, disk_total)
+ declare_btrfs_allocation_section_field(system, disk_used)
+
+ BTRFS_DISK *disks;
+
+ struct btrfs_node *next;
+} BTRFS_NODE;
+
+static BTRFS_NODE *nodes = NULL;
+
+static inline void btrfs_free_disk(BTRFS_DISK *d) {
+ freez(d->name);
+ freez(d->size_filename);
+ freez(d->hw_sector_size_filename);
+ freez(d);
+}
+
+static inline void btrfs_free_node(BTRFS_NODE *node) {
+ // info("BTRFS: destroying '%s'", node->id);
+
+ if(node->st_allocation_disks)
+ rrdset_is_obsolete(node->st_allocation_disks);
+
+ if(node->st_allocation_data)
+ rrdset_is_obsolete(node->st_allocation_data);
+
+ if(node->st_allocation_metadata)
+ rrdset_is_obsolete(node->st_allocation_metadata);
+
+ if(node->st_allocation_system)
+ rrdset_is_obsolete(node->st_allocation_system);
+
+ freez(node->allocation_data_bytes_used_filename);
+ freez(node->allocation_data_total_bytes_filename);
+
+ freez(node->allocation_metadata_bytes_used_filename);
+ freez(node->allocation_metadata_total_bytes_filename);
+
+ freez(node->allocation_system_bytes_used_filename);
+ freez(node->allocation_system_total_bytes_filename);
+
+ while(node->disks) {
+ BTRFS_DISK *d = node->disks;
+ node->disks = node->disks->next;
+ btrfs_free_disk(d);
+ }
+
+ freez(node->label);
+ freez(node->id);
+ freez(node);
+}
+
+static inline int find_btrfs_disks(BTRFS_NODE *node, const char *path) {
+ char filename[FILENAME_MAX + 1];
+
+ node->all_disks_total = 0;
+
+ BTRFS_DISK *d;
+ for(d = node->disks ; d ; d = d->next)
+ d->exists = 0;
+
+ DIR *dir = opendir(path);
+ if (!dir) {
+ if(!node->logged_error) {
+ error("BTRFS: Cannot open directory '%s'.", path);
+ node->logged_error = 1;
+ }
+ return 1;
+ }
+ node->logged_error = 0;
+
+ struct dirent *de = NULL;
+ while ((de = readdir(dir))) {
+ if (de->d_type != DT_LNK
+ || !strcmp(de->d_name, ".")
+ || !strcmp(de->d_name, "..")
+ ) {
+ // info("BTRFS: ignoring '%s'", de->d_name);
+ continue;
+ }
+
+ uint32_t hash = simple_hash(de->d_name);
+
+ // --------------------------------------------------------------------
+ // search for it
+
+ for(d = node->disks ; d ; d = d->next) {
+ if(hash == d->hash && !strcmp(de->d_name, d->name))
+ break;
+ }
+
+ // --------------------------------------------------------------------
+ // did we find it?
+
+ if(!d) {
+ d = callocz(sizeof(BTRFS_DISK), 1);
+
+ d->name = strdupz(de->d_name);
+ d->hash = simple_hash(d->name);
+
+ snprintfz(filename, FILENAME_MAX, "%s/%s/size", path, de->d_name);
+ d->size_filename = strdupz(filename);
+
+ // for bcache
+ snprintfz(filename, FILENAME_MAX, "%s/%s/bcache/../queue/hw_sector_size", path, de->d_name);
+ struct stat sb;
+ if(stat(filename, &sb) == -1) {
+ // for disks
+ snprintfz(filename, FILENAME_MAX, "%s/%s/queue/hw_sector_size", path, de->d_name);
+ if(stat(filename, &sb) == -1)
+ // for partitions
+ snprintfz(filename, FILENAME_MAX, "%s/%s/../queue/hw_sector_size", path, de->d_name);
+ }
+
+ d->hw_sector_size_filename = strdupz(filename);
+
+ // link it
+ d->next = node->disks;
+ node->disks = d;
+ }
+
+ d->exists = 1;
+
+
+ // --------------------------------------------------------------------
+ // update the values
+
+ if(read_single_number_file(d->size_filename, &d->size) != 0) {
+ error("BTRFS: failed to read '%s'", d->size_filename);
+ d->exists = 0;
+ continue;
+ }
+
+ if(read_single_number_file(d->hw_sector_size_filename, &d->hw_sector_size) != 0) {
+ error("BTRFS: failed to read '%s'", d->hw_sector_size_filename);
+ d->exists = 0;
+ continue;
+ }
+
+ node->all_disks_total += d->size * d->hw_sector_size;
+ }
+ closedir(dir);
+
+ // ------------------------------------------------------------------------
+ // cleanup
+
+ BTRFS_DISK *last = NULL;
+ d = node->disks;
+
+ while(d) {
+ if(unlikely(!d->exists)) {
+ if(unlikely(node->disks == d)) {
+ node->disks = d->next;
+ btrfs_free_disk(d);
+ d = node->disks;
+ last = NULL;
+ }
+ else {
+ last->next = d->next;
+ btrfs_free_disk(d);
+ d = last->next;
+ }
+
+ continue;
+ }
+
+ last = d;
+ d = d->next;
+ }
+
+ return 0;
+}
+
+
+static inline int find_all_btrfs_pools(const char *path) {
+ static int logged_error = 0;
+ char filename[FILENAME_MAX + 1];
+
+ BTRFS_NODE *node;
+ for(node = nodes ; node ; node = node->next)
+ node->exists = 0;
+
+ DIR *dir = opendir(path);
+ if (!dir) {
+ if(!logged_error) {
+ error("BTRFS: Cannot open directory '%s'.", path);
+ logged_error = 1;
+ }
+ return 1;
+ }
+ logged_error = 0;
+
+ struct dirent *de = NULL;
+ while ((de = readdir(dir))) {
+ if(de->d_type != DT_DIR
+ || !strcmp(de->d_name, ".")
+ || !strcmp(de->d_name, "..")
+ || !strcmp(de->d_name, "features")
+ ) {
+ // info("BTRFS: ignoring '%s'", de->d_name);
+ continue;
+ }
+
+ uint32_t hash = simple_hash(de->d_name);
+
+ // search for it
+ for(node = nodes ; node ; node = node->next) {
+ if(hash == node->hash && !strcmp(de->d_name, node->id))
+ break;
+ }
+
+ // did we find it?
+ if(node) {
+ // info("BTRFS: already exists '%s'", de->d_name);
+ node->exists = 1;
+
+ // update the disk sizes
+ snprintfz(filename, FILENAME_MAX, "%s/%s/devices", path, de->d_name);
+ find_btrfs_disks(node, filename);
+
+ continue;
+ }
+
+ // info("BTRFS: adding '%s'", de->d_name);
+
+ // not found, create it
+ node = callocz(sizeof(BTRFS_NODE), 1);
+
+ node->id = strdupz(de->d_name);
+ node->hash = simple_hash(node->id);
+ node->exists = 1;
+
+ {
+ char label[FILENAME_MAX + 1] = "";
+
+ snprintfz(filename, FILENAME_MAX, "%s/%s/label", path, de->d_name);
+ if(read_file(filename, label, FILENAME_MAX) != 0) {
+ error("BTRFS: failed to read '%s'", filename);
+ btrfs_free_node(node);
+ continue;
+ }
+
+ char *s = label;
+ if (s[0])
+ s = trim(label);
+
+ if(s && s[0])
+ node->label = strdupz(s);
+ else
+ node->label = strdupz(node->id);
+ }
+
+ //snprintfz(filename, FILENAME_MAX, "%s/%s/sectorsize", path, de->d_name);
+ //if(read_single_number_file(filename, &node->sectorsize) != 0) {
+ // error("BTRFS: failed to read '%s'", filename);
+ // btrfs_free_node(node);
+ // continue;
+ //}
+
+ //snprintfz(filename, FILENAME_MAX, "%s/%s/nodesize", path, de->d_name);
+ //if(read_single_number_file(filename, &node->nodesize) != 0) {
+ // error("BTRFS: failed to read '%s'", filename);
+ // btrfs_free_node(node);
+ // continue;
+ //}
+
+ //snprintfz(filename, FILENAME_MAX, "%s/%s/quota_override", path, de->d_name);
+ //if(read_single_number_file(filename, &node->quota_override) != 0) {
+ // error("BTRFS: failed to read '%s'", filename);
+ // btrfs_free_node(node);
+ // continue;
+ //}
+
+ // --------------------------------------------------------------------
+ // macros to simplify our life
+
+ #define init_btrfs_allocation_field(FIELD) {\
+ snprintfz(filename, FILENAME_MAX, "%s/%s/allocation/" #FIELD, path, de->d_name); \
+ if(read_single_number_file(filename, &node->allocation_ ## FIELD) != 0) {\
+ error("BTRFS: failed to read '%s'", filename);\
+ btrfs_free_node(node);\
+ continue;\
+ }\
+ if(!node->allocation_ ## FIELD ## _filename)\
+ node->allocation_ ## FIELD ## _filename = strdupz(filename);\
+ }
+
+ #define init_btrfs_allocation_section_field(SECTION, FIELD) {\
+ snprintfz(filename, FILENAME_MAX, "%s/%s/allocation/" #SECTION "/" #FIELD, path, de->d_name); \
+ if(read_single_number_file(filename, &node->allocation_ ## SECTION ## _ ## FIELD) != 0) {\
+ error("BTRFS: failed to read '%s'", filename);\
+ btrfs_free_node(node);\
+ continue;\
+ }\
+ if(!node->allocation_ ## SECTION ## _ ## FIELD ## _filename)\
+ node->allocation_ ## SECTION ## _ ## FIELD ## _filename = strdupz(filename);\
+ }
+
+ // --------------------------------------------------------------------
+ // allocation/data
+
+ init_btrfs_allocation_section_field(data, total_bytes);
+ init_btrfs_allocation_section_field(data, bytes_used);
+ init_btrfs_allocation_section_field(data, disk_total);
+ init_btrfs_allocation_section_field(data, disk_used);
+
+
+ // --------------------------------------------------------------------
+ // allocation/metadata
+
+ init_btrfs_allocation_section_field(metadata, total_bytes);
+ init_btrfs_allocation_section_field(metadata, bytes_used);
+ init_btrfs_allocation_section_field(metadata, disk_total);
+ init_btrfs_allocation_section_field(metadata, disk_used);
+
+ init_btrfs_allocation_field(global_rsv_size);
+ // init_btrfs_allocation_field(global_rsv_reserved);
+
+
+ // --------------------------------------------------------------------
+ // allocation/system
+
+ init_btrfs_allocation_section_field(system, total_bytes);
+ init_btrfs_allocation_section_field(system, bytes_used);
+ init_btrfs_allocation_section_field(system, disk_total);
+ init_btrfs_allocation_section_field(system, disk_used);
+
+
+ // --------------------------------------------------------------------
+ // find all disks related to this node
+ // and collect their sizes
+
+ snprintfz(filename, FILENAME_MAX, "%s/%s/devices", path, de->d_name);
+ find_btrfs_disks(node, filename);
+
+
+ // --------------------------------------------------------------------
+ // link it
+
+ // info("BTRFS: linking '%s'", node->id);
+ node->next = nodes;
+ nodes = node;
+ }
+ closedir(dir);
+
+
+ // ------------------------------------------------------------------------
+ // cleanup
+
+ BTRFS_NODE *last = NULL;
+ node = nodes;
+
+ while(node) {
+ if(unlikely(!node->exists)) {
+ if(unlikely(nodes == node)) {
+ nodes = node->next;
+ btrfs_free_node(node);
+ node = nodes;
+ last = NULL;
+ }
+ else {
+ last->next = node->next;
+ btrfs_free_node(node);
+ node = last->next;
+ }
+
+ continue;
+ }
+
+ last = node;
+ node = node->next;
+ }
+
+ return 0;
+}
+
+static void add_labels_to_btrfs(BTRFS_NODE *n, RRDSET *st) {
+ rrdlabels_add(st->rrdlabels, "device", n->id, RRDLABEL_SRC_AUTO);
+ rrdlabels_add(st->rrdlabels, "device_label", n->label, RRDLABEL_SRC_AUTO);
+}
+
+int do_sys_fs_btrfs(int update_every, usec_t dt) {
+ static int initialized = 0
+ , do_allocation_disks = CONFIG_BOOLEAN_AUTO
+ , do_allocation_system = CONFIG_BOOLEAN_AUTO
+ , do_allocation_data = CONFIG_BOOLEAN_AUTO
+ , do_allocation_metadata = CONFIG_BOOLEAN_AUTO;
+
+ static usec_t refresh_delta = 0, refresh_every = 60 * USEC_PER_SEC;
+ static char *btrfs_path = NULL;
+
+ (void)dt;
+
+ if(unlikely(!initialized)) {
+ initialized = 1;
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/fs/btrfs");
+ btrfs_path = config_get("plugin:proc:/sys/fs/btrfs", "path to monitor", filename);
+
+ refresh_every = config_get_number("plugin:proc:/sys/fs/btrfs", "check for btrfs changes every", refresh_every / USEC_PER_SEC) * USEC_PER_SEC;
+ refresh_delta = refresh_every;
+
+ do_allocation_disks = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "physical disks allocation", do_allocation_disks);
+ do_allocation_data = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "data allocation", do_allocation_data);
+ do_allocation_metadata = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "metadata allocation", do_allocation_metadata);
+ do_allocation_system = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "system allocation", do_allocation_system);
+ }
+
+ refresh_delta += dt;
+ if(refresh_delta >= refresh_every) {
+ refresh_delta = 0;
+ find_all_btrfs_pools(btrfs_path);
+ }
+
+ BTRFS_NODE *node;
+ for(node = nodes; node ; node = node->next) {
+ // --------------------------------------------------------------------
+ // allocation/system
+
+ #define collect_btrfs_allocation_field(FIELD) \
+ read_single_number_file(node->allocation_ ## FIELD ## _filename, &node->allocation_ ## FIELD)
+
+ #define collect_btrfs_allocation_section_field(SECTION, FIELD) \
+ read_single_number_file(node->allocation_ ## SECTION ## _ ## FIELD ## _filename, &node->allocation_ ## SECTION ## _ ## FIELD)
+
+ if(do_allocation_disks != CONFIG_BOOLEAN_NO) {
+ if( collect_btrfs_allocation_section_field(data, disk_total) != 0
+ || collect_btrfs_allocation_section_field(data, disk_used) != 0
+ || collect_btrfs_allocation_section_field(metadata, disk_total) != 0
+ || collect_btrfs_allocation_section_field(metadata, disk_used) != 0
+ || collect_btrfs_allocation_section_field(system, disk_total) != 0
+ || collect_btrfs_allocation_section_field(system, disk_used) != 0) {
+ error("BTRFS: failed to collect physical disks allocation for '%s'", node->id);
+ // make it refresh btrfs at the next iteration
+ refresh_delta = refresh_every;
+ continue;
+ }
+ }
+
+ if(do_allocation_data != CONFIG_BOOLEAN_NO) {
+ if (collect_btrfs_allocation_section_field(data, total_bytes) != 0
+ || collect_btrfs_allocation_section_field(data, bytes_used) != 0) {
+ error("BTRFS: failed to collect allocation/data for '%s'", node->id);
+ // make it refresh btrfs at the next iteration
+ refresh_delta = refresh_every;
+ continue;
+ }
+ }
+
+ if(do_allocation_metadata != CONFIG_BOOLEAN_NO) {
+ if (collect_btrfs_allocation_section_field(metadata, total_bytes) != 0
+ || collect_btrfs_allocation_section_field(metadata, bytes_used) != 0
+ || collect_btrfs_allocation_field(global_rsv_size) != 0
+ ) {
+ error("BTRFS: failed to collect allocation/metadata for '%s'", node->id);
+ // make it refresh btrfs at the next iteration
+ refresh_delta = refresh_every;
+ continue;
+ }
+ }
+
+ if(do_allocation_system != CONFIG_BOOLEAN_NO) {
+ if (collect_btrfs_allocation_section_field(system, total_bytes) != 0
+ || collect_btrfs_allocation_section_field(system, bytes_used) != 0) {
+ error("BTRFS: failed to collect allocation/system for '%s'", node->id);
+ // make it refresh btrfs at the next iteration
+ refresh_delta = refresh_every;
+ continue;
+ }
+ }
+
+ // --------------------------------------------------------------------
+ // allocation/disks
+
+ if(do_allocation_disks == CONFIG_BOOLEAN_YES || (do_allocation_disks == CONFIG_BOOLEAN_AUTO &&
+ ((node->all_disks_total && node->allocation_data_disk_total) ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_allocation_disks = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!node->st_allocation_disks)) {
+ char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
+
+ snprintf(id, RRD_ID_LENGTH_MAX, "disk_%s", node->id);
+ snprintf(name, RRD_ID_LENGTH_MAX, "disk_%s", node->label);
+ snprintf(title, 200, "BTRFS Physical Disk Allocation");
+
+ netdata_fix_chart_id(id);
+ netdata_fix_chart_name(name);
+
+ node->st_allocation_disks = rrdset_create_localhost(
+ "btrfs"
+ , id
+ , name
+ , node->label
+ , "btrfs.disk"
+ , title
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_BTRFS_NAME
+ , NETDATA_CHART_PRIO_BTRFS_DISK
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ node->rd_allocation_disks_unallocated = rrddim_add(node->st_allocation_disks, "unallocated", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_disks_data_free = rrddim_add(node->st_allocation_disks, "data_free", "data free", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_disks_data_used = rrddim_add(node->st_allocation_disks, "data_used", "data used", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_disks_metadata_free = rrddim_add(node->st_allocation_disks, "meta_free", "meta free", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_disks_metadata_used = rrddim_add(node->st_allocation_disks, "meta_used", "meta used", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_disks_system_free = rrddim_add(node->st_allocation_disks, "sys_free", "sys free", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_disks_system_used = rrddim_add(node->st_allocation_disks, "sys_used", "sys used", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_btrfs(node, node->st_allocation_disks);
+ }
+
+ // unsigned long long disk_used = node->allocation_data_disk_used + node->allocation_metadata_disk_used + node->allocation_system_disk_used;
+ unsigned long long disk_total = node->allocation_data_disk_total + node->allocation_metadata_disk_total + node->allocation_system_disk_total;
+ unsigned long long disk_unallocated = node->all_disks_total - disk_total;
+
+ rrddim_set_by_pointer(node->st_allocation_disks, node->rd_allocation_disks_unallocated, disk_unallocated);
+ rrddim_set_by_pointer(node->st_allocation_disks, node->rd_allocation_disks_data_used, node->allocation_data_disk_used);
+ rrddim_set_by_pointer(node->st_allocation_disks, node->rd_allocation_disks_data_free, node->allocation_data_disk_total - node->allocation_data_disk_used);
+ rrddim_set_by_pointer(node->st_allocation_disks, node->rd_allocation_disks_metadata_used, node->allocation_metadata_disk_used);
+ rrddim_set_by_pointer(node->st_allocation_disks, node->rd_allocation_disks_metadata_free, node->allocation_metadata_disk_total - node->allocation_metadata_disk_used);
+ rrddim_set_by_pointer(node->st_allocation_disks, node->rd_allocation_disks_system_used, node->allocation_system_disk_used);
+ rrddim_set_by_pointer(node->st_allocation_disks, node->rd_allocation_disks_system_free, node->allocation_system_disk_total - node->allocation_system_disk_used);
+ rrdset_done(node->st_allocation_disks);
+ }
+
+
+ // --------------------------------------------------------------------
+ // allocation/data
+
+ if(do_allocation_data == CONFIG_BOOLEAN_YES || (do_allocation_data == CONFIG_BOOLEAN_AUTO &&
+ (node->allocation_data_total_bytes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_allocation_data = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!node->st_allocation_data)) {
+ char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
+
+ snprintf(id, RRD_ID_LENGTH_MAX, "data_%s", node->id);
+ snprintf(name, RRD_ID_LENGTH_MAX, "data_%s", node->label);
+ snprintf(title, 200, "BTRFS Data Allocation");
+
+ netdata_fix_chart_id(id);
+ netdata_fix_chart_name(name);
+
+ node->st_allocation_data = rrdset_create_localhost(
+ "btrfs"
+ , id
+ , name
+ , node->label
+ , "btrfs.data"
+ , title
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_BTRFS_NAME
+ , NETDATA_CHART_PRIO_BTRFS_DATA
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ node->rd_allocation_data_free = rrddim_add(node->st_allocation_data, "free", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_data_used = rrddim_add(node->st_allocation_data, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_btrfs(node, node->st_allocation_data);
+ }
+
+ rrddim_set_by_pointer(node->st_allocation_data, node->rd_allocation_data_free, node->allocation_data_total_bytes - node->allocation_data_bytes_used);
+ rrddim_set_by_pointer(node->st_allocation_data, node->rd_allocation_data_used, node->allocation_data_bytes_used);
+ rrdset_done(node->st_allocation_data);
+ }
+
+ // --------------------------------------------------------------------
+ // allocation/metadata
+
+ if(do_allocation_metadata == CONFIG_BOOLEAN_YES || (do_allocation_metadata == CONFIG_BOOLEAN_AUTO &&
+ (node->allocation_metadata_total_bytes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_allocation_metadata = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!node->st_allocation_metadata)) {
+ char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
+
+ snprintf(id, RRD_ID_LENGTH_MAX, "metadata_%s", node->id);
+ snprintf(name, RRD_ID_LENGTH_MAX, "metadata_%s", node->label);
+ snprintf(title, 200, "BTRFS Metadata Allocation");
+
+ netdata_fix_chart_id(id);
+ netdata_fix_chart_name(name);
+
+ node->st_allocation_metadata = rrdset_create_localhost(
+ "btrfs"
+ , id
+ , name
+ , node->label
+ , "btrfs.metadata"
+ , title
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_BTRFS_NAME
+ , NETDATA_CHART_PRIO_BTRFS_METADATA
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ node->rd_allocation_metadata_free = rrddim_add(node->st_allocation_metadata, "free", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_metadata_used = rrddim_add(node->st_allocation_metadata, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_metadata_reserved = rrddim_add(node->st_allocation_metadata, "reserved", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_btrfs(node, node->st_allocation_metadata);
+ }
+
+ rrddim_set_by_pointer(node->st_allocation_metadata, node->rd_allocation_metadata_free, node->allocation_metadata_total_bytes - node->allocation_metadata_bytes_used - node->allocation_global_rsv_size);
+ rrddim_set_by_pointer(node->st_allocation_metadata, node->rd_allocation_metadata_used, node->allocation_metadata_bytes_used);
+ rrddim_set_by_pointer(node->st_allocation_metadata, node->rd_allocation_metadata_reserved, node->allocation_global_rsv_size);
+ rrdset_done(node->st_allocation_metadata);
+ }
+
+ // --------------------------------------------------------------------
+ // allocation/system
+
+ if(do_allocation_system == CONFIG_BOOLEAN_YES || (do_allocation_system == CONFIG_BOOLEAN_AUTO &&
+ (node->allocation_system_total_bytes ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ do_allocation_system = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!node->st_allocation_system)) {
+ char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
+
+ snprintf(id, RRD_ID_LENGTH_MAX, "system_%s", node->id);
+ snprintf(name, RRD_ID_LENGTH_MAX, "system_%s", node->label);
+ snprintf(title, 200, "BTRFS System Allocation");
+
+ netdata_fix_chart_id(id);
+ netdata_fix_chart_name(name);
+
+ node->st_allocation_system = rrdset_create_localhost(
+ "btrfs"
+ , id
+ , name
+ , node->label
+ , "btrfs.system"
+ , title
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_BTRFS_NAME
+ , NETDATA_CHART_PRIO_BTRFS_SYSTEM
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ node->rd_allocation_system_free = rrddim_add(node->st_allocation_system, "free", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ node->rd_allocation_system_used = rrddim_add(node->st_allocation_system, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+
+ add_labels_to_btrfs(node, node->st_allocation_system);
+ }
+
+ rrddim_set_by_pointer(node->st_allocation_system, node->rd_allocation_system_free, node->allocation_system_total_bytes - node->allocation_system_bytes_used);
+ rrddim_set_by_pointer(node->st_allocation_system, node->rd_allocation_system_used, node->allocation_system_bytes_used);
+ rrdset_done(node->st_allocation_system);
+ }
+ }
+
+ return 0;
+}
+
diff --git a/collectors/proc.plugin/sys_kernel_mm_ksm.c b/collectors/proc.plugin/sys_kernel_mm_ksm.c
new file mode 100644
index 0000000..e586d55
--- /dev/null
+++ b/collectors/proc.plugin/sys_kernel_mm_ksm.c
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_KSM_NAME "/sys/kernel/mm/ksm"
+
+typedef struct ksm_name_value {
+ char filename[FILENAME_MAX + 1];
+ unsigned long long value;
+} KSM_NAME_VALUE;
+
+#define PAGES_SHARED 0
+#define PAGES_SHARING 1
+#define PAGES_UNSHARED 2
+#define PAGES_VOLATILE 3
+#define PAGES_TO_SCAN 4
+
+KSM_NAME_VALUE values[] = {
+ [PAGES_SHARED] = { "/sys/kernel/mm/ksm/pages_shared", 0ULL },
+ [PAGES_SHARING] = { "/sys/kernel/mm/ksm/pages_sharing", 0ULL },
+ [PAGES_UNSHARED] = { "/sys/kernel/mm/ksm/pages_unshared", 0ULL },
+ [PAGES_VOLATILE] = { "/sys/kernel/mm/ksm/pages_volatile", 0ULL },
+ // [PAGES_TO_SCAN] = { "/sys/kernel/mm/ksm/pages_to_scan", 0ULL },
+};
+
+int do_sys_kernel_mm_ksm(int update_every, usec_t dt) {
+ (void)dt;
+ static procfile *ff_pages_shared = NULL, *ff_pages_sharing = NULL, *ff_pages_unshared = NULL, *ff_pages_volatile = NULL/*, *ff_pages_to_scan = NULL*/;
+ static unsigned long page_size = 0;
+
+ if(unlikely(page_size == 0))
+ page_size = (unsigned long)sysconf(_SC_PAGESIZE);
+
+ if(unlikely(!ff_pages_shared)) {
+ snprintfz(values[PAGES_SHARED].filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/kernel/mm/ksm/pages_shared");
+ snprintfz(values[PAGES_SHARED].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_shared", values[PAGES_SHARED].filename));
+ ff_pages_shared = procfile_open(values[PAGES_SHARED].filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ }
+
+ if(unlikely(!ff_pages_sharing)) {
+ snprintfz(values[PAGES_SHARING].filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/kernel/mm/ksm/pages_sharing");
+ snprintfz(values[PAGES_SHARING].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_sharing", values[PAGES_SHARING].filename));
+ ff_pages_sharing = procfile_open(values[PAGES_SHARING].filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ }
+
+ if(unlikely(!ff_pages_unshared)) {
+ snprintfz(values[PAGES_UNSHARED].filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/kernel/mm/ksm/pages_unshared");
+ snprintfz(values[PAGES_UNSHARED].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_unshared", values[PAGES_UNSHARED].filename));
+ ff_pages_unshared = procfile_open(values[PAGES_UNSHARED].filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ }
+
+ if(unlikely(!ff_pages_volatile)) {
+ snprintfz(values[PAGES_VOLATILE].filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/kernel/mm/ksm/pages_volatile");
+ snprintfz(values[PAGES_VOLATILE].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_volatile", values[PAGES_VOLATILE].filename));
+ ff_pages_volatile = procfile_open(values[PAGES_VOLATILE].filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ }
+
+ //if(unlikely(!ff_pages_to_scan)) {
+ // snprintfz(values[PAGES_TO_SCAN].filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/kernel/mm/ksm/pages_to_scan");
+ // snprintfz(values[PAGES_TO_SCAN].filename, FILENAME_MAX, "%s", config_get("plugin:proc:/sys/kernel/mm/ksm", "/sys/kernel/mm/ksm/pages_to_scan", values[PAGES_TO_SCAN].filename));
+ // ff_pages_to_scan = procfile_open(values[PAGES_TO_SCAN].filename, " \t:", PROCFILE_FLAG_DEFAULT);
+ //}
+
+ if(unlikely(!ff_pages_shared || !ff_pages_sharing || !ff_pages_unshared || !ff_pages_volatile /*|| !ff_pages_to_scan */))
+ return 1;
+
+ unsigned long long pages_shared = 0, pages_sharing = 0, pages_unshared = 0, pages_volatile = 0, /*pages_to_scan = 0,*/ offered = 0, saved = 0;
+
+ ff_pages_shared = procfile_readall(ff_pages_shared);
+ if(unlikely(!ff_pages_shared)) return 0; // we return 0, so that we will retry to open it next time
+ pages_shared = str2ull(procfile_lineword(ff_pages_shared, 0, 0));
+
+ ff_pages_sharing = procfile_readall(ff_pages_sharing);
+ if(unlikely(!ff_pages_sharing)) return 0; // we return 0, so that we will retry to open it next time
+ pages_sharing = str2ull(procfile_lineword(ff_pages_sharing, 0, 0));
+
+ ff_pages_unshared = procfile_readall(ff_pages_unshared);
+ if(unlikely(!ff_pages_unshared)) return 0; // we return 0, so that we will retry to open it next time
+ pages_unshared = str2ull(procfile_lineword(ff_pages_unshared, 0, 0));
+
+ ff_pages_volatile = procfile_readall(ff_pages_volatile);
+ if(unlikely(!ff_pages_volatile)) return 0; // we return 0, so that we will retry to open it next time
+ pages_volatile = str2ull(procfile_lineword(ff_pages_volatile, 0, 0));
+
+ //ff_pages_to_scan = procfile_readall(ff_pages_to_scan);
+ //if(unlikely(!ff_pages_to_scan)) return 0; // we return 0, so that we will retry to open it next time
+ //pages_to_scan = str2ull(procfile_lineword(ff_pages_to_scan, 0, 0));
+
+ offered = pages_sharing + pages_shared + pages_unshared + pages_volatile;
+ saved = pages_sharing;
+
+ if(unlikely(!offered /*|| !pages_to_scan*/ && netdata_zero_metrics_enabled == CONFIG_BOOLEAN_NO)) return 0;
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_mem_ksm = NULL;
+ static RRDDIM *rd_shared = NULL, *rd_unshared = NULL, *rd_sharing = NULL, *rd_volatile = NULL/*, *rd_to_scan = NULL*/;
+
+ if (unlikely(!st_mem_ksm)) {
+ st_mem_ksm = rrdset_create_localhost(
+ "mem"
+ , "ksm"
+ , NULL
+ , "ksm"
+ , NULL
+ , "Kernel Same Page Merging"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_KSM_NAME
+ , NETDATA_CHART_PRIO_MEM_KSM
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_shared = rrddim_add(st_mem_ksm, "shared", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_unshared = rrddim_add(st_mem_ksm, "unshared", NULL, -1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_sharing = rrddim_add(st_mem_ksm, "sharing", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_volatile = rrddim_add(st_mem_ksm, "volatile", NULL, -1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ //rd_to_scan = rrddim_add(st_mem_ksm, "to_scan", "to scan", -1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_ksm, rd_shared, pages_shared * page_size);
+ rrddim_set_by_pointer(st_mem_ksm, rd_unshared, pages_unshared * page_size);
+ rrddim_set_by_pointer(st_mem_ksm, rd_sharing, pages_sharing * page_size);
+ rrddim_set_by_pointer(st_mem_ksm, rd_volatile, pages_volatile * page_size);
+ //rrddim_set_by_pointer(st_mem_ksm, rd_to_scan, pages_to_scan * page_size);
+
+ rrdset_done(st_mem_ksm);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_mem_ksm_savings = NULL;
+ static RRDDIM *rd_savings = NULL, *rd_offered = NULL;
+
+ if (unlikely(!st_mem_ksm_savings)) {
+ st_mem_ksm_savings = rrdset_create_localhost(
+ "mem"
+ , "ksm_savings"
+ , NULL
+ , "ksm"
+ , NULL
+ , "Kernel Same Page Merging Savings"
+ , "MiB"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_KSM_NAME
+ , NETDATA_CHART_PRIO_MEM_KSM_SAVINGS
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_savings = rrddim_add(st_mem_ksm_savings, "savings", NULL, -1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_offered = rrddim_add(st_mem_ksm_savings, "offered", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_ksm_savings, rd_savings, saved * page_size);
+ rrddim_set_by_pointer(st_mem_ksm_savings, rd_offered, offered * page_size);
+
+ rrdset_done(st_mem_ksm_savings);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_mem_ksm_ratios = NULL;
+ static RRDDIM *rd_savings = NULL;
+
+ if (unlikely(!st_mem_ksm_ratios)) {
+ st_mem_ksm_ratios = rrdset_create_localhost(
+ "mem"
+ , "ksm_ratios"
+ , NULL
+ , "ksm"
+ , NULL
+ , "Kernel Same Page Merging Effectiveness"
+ , "percentage"
+ , PLUGIN_PROC_NAME
+ , PLUGIN_PROC_MODULE_KSM_NAME
+ , NETDATA_CHART_PRIO_MEM_KSM_RATIOS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_savings = rrddim_add(st_mem_ksm_ratios, "savings", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_mem_ksm_ratios, rd_savings, offered ? (saved * 1000000) / offered : 0);
+ rrdset_done(st_mem_ksm_ratios);
+ }
+
+ return 0;
+}
diff --git a/collectors/proc.plugin/zfs_common.c b/collectors/proc.plugin/zfs_common.c
new file mode 100644
index 0000000..cca0ae0
--- /dev/null
+++ b/collectors/proc.plugin/zfs_common.c
@@ -0,0 +1,960 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "zfs_common.h"
+
+struct arcstats arcstats = { 0 };
+
+void generate_charts_arcstats(const char *plugin, const char *module, int show_zero_charts, int update_every) {
+ static int do_arc_size = -1, do_l2_size = -1, do_reads = -1, do_l2bytes = -1, do_ahits = -1, do_dhits = -1, \
+ do_phits = -1, do_mhits = -1, do_l2hits = -1, do_list_hits = -1;
+
+ if(unlikely(do_arc_size == -1))
+ do_arc_size = do_l2_size = do_reads = do_l2bytes = do_ahits = do_dhits = do_phits = do_mhits \
+ = do_l2hits = do_list_hits = show_zero_charts;
+
+ // ARC reads
+ unsigned long long aread = arcstats.hits + arcstats.misses;
+
+ // Demand reads
+ unsigned long long dhit = arcstats.demand_data_hits + arcstats.demand_metadata_hits;
+ unsigned long long dmiss = arcstats.demand_data_misses + arcstats.demand_metadata_misses;
+ unsigned long long dread = dhit + dmiss;
+
+ // Prefetch reads
+ unsigned long long phit = arcstats.prefetch_data_hits + arcstats.prefetch_metadata_hits;
+ unsigned long long pmiss = arcstats.prefetch_data_misses + arcstats.prefetch_metadata_misses;
+ unsigned long long pread = phit + pmiss;
+
+ // Metadata reads
+ unsigned long long mhit = arcstats.prefetch_metadata_hits + arcstats.demand_metadata_hits;
+ unsigned long long mmiss = arcstats.prefetch_metadata_misses + arcstats.demand_metadata_misses;
+ unsigned long long mread = mhit + mmiss;
+
+ // l2 reads
+ unsigned long long l2hit = arcstats.l2_hits;
+ unsigned long long l2miss = arcstats.l2_misses;
+ unsigned long long l2read = l2hit + l2miss;
+
+ // --------------------------------------------------------------------
+
+ if(do_arc_size == CONFIG_BOOLEAN_YES || arcstats.size || arcstats.c || arcstats.c_min || arcstats.c_max) {
+ do_arc_size = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_arc_size = NULL;
+ static RRDDIM *rd_arc_size = NULL;
+ static RRDDIM *rd_arc_target_size = NULL;
+ static RRDDIM *rd_arc_target_min_size = NULL;
+ static RRDDIM *rd_arc_target_max_size = NULL;
+
+ if (unlikely(!st_arc_size)) {
+ st_arc_size = rrdset_create_localhost(
+ "zfs"
+ , "arc_size"
+ , NULL
+ , ZFS_FAMILY_SIZE
+ , NULL
+ , "ZFS ARC Size"
+ , "MiB"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_ARC_SIZE
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_arc_size = rrddim_add(st_arc_size, "size", "arcsz", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_arc_target_size = rrddim_add(st_arc_size, "target", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_arc_target_min_size = rrddim_add(st_arc_size, "min", "min (hard limit)", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_arc_target_max_size = rrddim_add(st_arc_size, "max", "max (high water)", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_arc_size, rd_arc_size, arcstats.size);
+ rrddim_set_by_pointer(st_arc_size, rd_arc_target_size, arcstats.c);
+ rrddim_set_by_pointer(st_arc_size, rd_arc_target_min_size, arcstats.c_min);
+ rrddim_set_by_pointer(st_arc_size, rd_arc_target_max_size, arcstats.c_max);
+ rrdset_done(st_arc_size);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(arcstats.l2exist) && (do_l2_size == CONFIG_BOOLEAN_YES || arcstats.l2_size || arcstats.l2_asize)) {
+ do_l2_size = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_l2_size = NULL;
+ static RRDDIM *rd_l2_size = NULL;
+ static RRDDIM *rd_l2_asize = NULL;
+
+ if (unlikely(!st_l2_size)) {
+ st_l2_size = rrdset_create_localhost(
+ "zfs"
+ , "l2_size"
+ , NULL
+ , ZFS_FAMILY_SIZE
+ , NULL
+ , "ZFS L2 ARC Size"
+ , "MiB"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_L2_SIZE
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_l2_asize = rrddim_add(st_l2_size, "actual", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_l2_size = rrddim_add(st_l2_size, "size", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_l2_size, rd_l2_size, arcstats.l2_size);
+ rrddim_set_by_pointer(st_l2_size, rd_l2_asize, arcstats.l2_asize);
+ rrdset_done(st_l2_size);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_reads == CONFIG_BOOLEAN_YES || aread || dread || pread || mread || l2read)) {
+ do_reads = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_reads = NULL;
+ static RRDDIM *rd_aread = NULL;
+ static RRDDIM *rd_dread = NULL;
+ static RRDDIM *rd_pread = NULL;
+ static RRDDIM *rd_mread = NULL;
+ static RRDDIM *rd_l2read = NULL;
+
+ if (unlikely(!st_reads)) {
+ st_reads = rrdset_create_localhost(
+ "zfs"
+ , "reads"
+ , NULL
+ , ZFS_FAMILY_ACCESSES
+ , NULL
+ , "ZFS Reads"
+ , "reads/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_READS
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_aread = rrddim_add(st_reads, "areads", "arc", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_dread = rrddim_add(st_reads, "dreads", "demand", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_pread = rrddim_add(st_reads, "preads", "prefetch", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mread = rrddim_add(st_reads, "mreads", "metadata", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(arcstats.l2exist)
+ rd_l2read = rrddim_add(st_reads, "l2reads", "l2", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_reads, rd_aread, aread);
+ rrddim_set_by_pointer(st_reads, rd_dread, dread);
+ rrddim_set_by_pointer(st_reads, rd_pread, pread);
+ rrddim_set_by_pointer(st_reads, rd_mread, mread);
+
+ if(arcstats.l2exist)
+ rrddim_set_by_pointer(st_reads, rd_l2read, l2read);
+
+ rrdset_done(st_reads);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(arcstats.l2exist && (do_l2bytes == CONFIG_BOOLEAN_YES || arcstats.l2_read_bytes || arcstats.l2_write_bytes))) {
+ do_l2bytes = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_l2bytes = NULL;
+ static RRDDIM *rd_l2_read_bytes = NULL;
+ static RRDDIM *rd_l2_write_bytes = NULL;
+
+ if (unlikely(!st_l2bytes)) {
+ st_l2bytes = rrdset_create_localhost(
+ "zfs"
+ , "bytes"
+ , NULL
+ , ZFS_FAMILY_ACCESSES
+ , NULL
+ , "ZFS ARC L2 Read/Write Rate"
+ , "KiB/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_IO
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_l2_read_bytes = rrddim_add(st_l2bytes, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ rd_l2_write_bytes = rrddim_add(st_l2bytes, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_l2bytes, rd_l2_read_bytes, arcstats.l2_read_bytes);
+ rrddim_set_by_pointer(st_l2bytes, rd_l2_write_bytes, arcstats.l2_write_bytes);
+ rrdset_done(st_l2bytes);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_ahits == CONFIG_BOOLEAN_YES || arcstats.hits || arcstats.misses)) {
+ do_ahits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_ahits = NULL;
+ static RRDDIM *rd_ahits = NULL;
+ static RRDDIM *rd_amisses = NULL;
+
+ if (unlikely(!st_ahits)) {
+ st_ahits = rrdset_create_localhost(
+ "zfs"
+ , "hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS ARC Hits"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_HITS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_ahits = rrddim_add(st_ahits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_amisses = rrddim_add(st_ahits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_ahits, rd_ahits, arcstats.hits);
+ rrddim_set_by_pointer(st_ahits, rd_amisses, arcstats.misses);
+ rrdset_done(st_ahits);
+
+ static RRDSET *st_ahits_rate = NULL;
+ static RRDDIM *rd_ahits_rate = NULL;
+ static RRDDIM *rd_amisses_rate = NULL;
+
+ if (unlikely(!st_ahits_rate)) {
+ st_ahits_rate = rrdset_create_localhost(
+ "zfs"
+ , "hits_rate"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS ARC Hits Rate"
+ , "events/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_HITS + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_ahits_rate = rrddim_add(st_ahits_rate, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_amisses_rate = rrddim_add(st_ahits_rate, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_ahits_rate, rd_ahits_rate, arcstats.hits);
+ rrddim_set_by_pointer(st_ahits_rate, rd_amisses_rate, arcstats.misses);
+ rrdset_done(st_ahits_rate);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_dhits == CONFIG_BOOLEAN_YES || dhit || dmiss)) {
+ do_dhits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_dhits = NULL;
+ static RRDDIM *rd_dhits = NULL;
+ static RRDDIM *rd_dmisses = NULL;
+
+ if (unlikely(!st_dhits)) {
+ st_dhits = rrdset_create_localhost(
+ "zfs"
+ , "dhits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Demand Hits"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_DHITS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_dhits = rrddim_add(st_dhits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_dmisses = rrddim_add(st_dhits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_dhits, rd_dhits, dhit);
+ rrddim_set_by_pointer(st_dhits, rd_dmisses, dmiss);
+ rrdset_done(st_dhits);
+
+ static RRDSET *st_dhits_rate = NULL;
+ static RRDDIM *rd_dhits_rate = NULL;
+ static RRDDIM *rd_dmisses_rate = NULL;
+
+ if (unlikely(!st_dhits_rate)) {
+ st_dhits_rate = rrdset_create_localhost(
+ "zfs"
+ , "dhits_rate"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Demand Hits Rate"
+ , "events/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_DHITS + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_dhits_rate = rrddim_add(st_dhits_rate, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_dmisses_rate = rrddim_add(st_dhits_rate, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_dhits_rate, rd_dhits_rate, dhit);
+ rrddim_set_by_pointer(st_dhits_rate, rd_dmisses_rate, dmiss);
+ rrdset_done(st_dhits_rate);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_phits == CONFIG_BOOLEAN_YES || phit || pmiss)) {
+ do_phits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_phits = NULL;
+ static RRDDIM *rd_phits = NULL;
+ static RRDDIM *rd_pmisses = NULL;
+
+ if (unlikely(!st_phits)) {
+ st_phits = rrdset_create_localhost(
+ "zfs"
+ , "phits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Prefetch Hits"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_PHITS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_phits = rrddim_add(st_phits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_pmisses = rrddim_add(st_phits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_phits, rd_phits, phit);
+ rrddim_set_by_pointer(st_phits, rd_pmisses, pmiss);
+ rrdset_done(st_phits);
+
+ static RRDSET *st_phits_rate = NULL;
+ static RRDDIM *rd_phits_rate = NULL;
+ static RRDDIM *rd_pmisses_rate = NULL;
+
+ if (unlikely(!st_phits_rate)) {
+ st_phits_rate = rrdset_create_localhost(
+ "zfs"
+ , "phits_rate"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Prefetch Hits Rate"
+ , "events/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_PHITS + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_phits_rate = rrddim_add(st_phits_rate, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_pmisses_rate = rrddim_add(st_phits_rate, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_phits_rate, rd_phits_rate, phit);
+ rrddim_set_by_pointer(st_phits_rate, rd_pmisses_rate, pmiss);
+ rrdset_done(st_phits_rate);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_mhits == CONFIG_BOOLEAN_YES || mhit || mmiss)) {
+ do_mhits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_mhits = NULL;
+ static RRDDIM *rd_mhits = NULL;
+ static RRDDIM *rd_mmisses = NULL;
+
+ if (unlikely(!st_mhits)) {
+ st_mhits = rrdset_create_localhost(
+ "zfs"
+ , "mhits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Metadata Hits"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_MHITS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_mhits = rrddim_add(st_mhits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_mmisses = rrddim_add(st_mhits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_mhits, rd_mhits, mhit);
+ rrddim_set_by_pointer(st_mhits, rd_mmisses, mmiss);
+ rrdset_done(st_mhits);
+
+ static RRDSET *st_mhits_rate = NULL;
+ static RRDDIM *rd_mhits_rate = NULL;
+ static RRDDIM *rd_mmisses_rate = NULL;
+
+ if (unlikely(!st_mhits_rate)) {
+ st_mhits_rate = rrdset_create_localhost(
+ "zfs"
+ , "mhits_rate"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Metadata Hits Rate"
+ , "events/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_MHITS + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_mhits_rate = rrddim_add(st_mhits_rate, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mmisses_rate = rrddim_add(st_mhits_rate, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_mhits_rate, rd_mhits_rate, mhit);
+ rrddim_set_by_pointer(st_mhits_rate, rd_mmisses_rate, mmiss);
+ rrdset_done(st_mhits_rate);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(arcstats.l2exist && (do_l2hits == CONFIG_BOOLEAN_YES || l2hit || l2miss))) {
+ do_l2hits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_l2hits = NULL;
+ static RRDDIM *rd_l2hits = NULL;
+ static RRDDIM *rd_l2misses = NULL;
+
+ if (unlikely(!st_l2hits)) {
+ st_l2hits = rrdset_create_localhost(
+ "zfs"
+ , "l2hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS L2 Hits"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_L2HITS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_l2hits = rrddim_add(st_l2hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_l2misses = rrddim_add(st_l2hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_l2hits, rd_l2hits, l2hit);
+ rrddim_set_by_pointer(st_l2hits, rd_l2misses, l2miss);
+ rrdset_done(st_l2hits);
+
+ static RRDSET *st_l2hits_rate = NULL;
+ static RRDDIM *rd_l2hits_rate = NULL;
+ static RRDDIM *rd_l2misses_rate = NULL;
+
+ if (unlikely(!st_l2hits_rate)) {
+ st_l2hits_rate = rrdset_create_localhost(
+ "zfs"
+ , "l2hits_rate"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS L2 Hits Rate"
+ , "events/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_L2HITS + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_l2hits_rate = rrddim_add(st_l2hits_rate, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_l2misses_rate = rrddim_add(st_l2hits_rate, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_l2hits_rate, rd_l2hits_rate, l2hit);
+ rrddim_set_by_pointer(st_l2hits_rate, rd_l2misses_rate, l2miss);
+ rrdset_done(st_l2hits_rate);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_list_hits == CONFIG_BOOLEAN_YES || arcstats.mfu_hits \
+ || arcstats.mru_hits \
+ || arcstats.mfu_ghost_hits \
+ || arcstats.mru_ghost_hits)) {
+ do_list_hits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_list_hits = NULL;
+ static RRDDIM *rd_mfu = NULL;
+ static RRDDIM *rd_mru = NULL;
+ static RRDDIM *rd_mfug = NULL;
+ static RRDDIM *rd_mrug = NULL;
+
+ if (unlikely(!st_list_hits)) {
+ st_list_hits = rrdset_create_localhost(
+ "zfs"
+ , "list_hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS List Hits"
+ , "hits/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_LIST_HITS
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_mfu = rrddim_add(st_list_hits, "mfu", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mfug = rrddim_add(st_list_hits, "mfug", "mfu ghost", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mru = rrddim_add(st_list_hits, "mru", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mrug = rrddim_add(st_list_hits, "mrug", "mru ghost", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_list_hits, rd_mfu, arcstats.mfu_hits);
+ rrddim_set_by_pointer(st_list_hits, rd_mru, arcstats.mru_hits);
+ rrddim_set_by_pointer(st_list_hits, rd_mfug, arcstats.mfu_ghost_hits);
+ rrddim_set_by_pointer(st_list_hits, rd_mrug, arcstats.mru_ghost_hits);
+ rrdset_done(st_list_hits);
+ }
+}
+
+void generate_charts_arc_summary(const char *plugin, const char *module, int show_zero_charts, int update_every) {
+ static int do_arc_size_breakdown = -1, do_memory = -1, do_important_ops = -1, do_actual_hits = -1, \
+ do_demand_data_hits = -1, do_prefetch_data_hits = -1, do_hash_elements = -1, do_hash_chains = -1;
+
+ if(unlikely(do_arc_size_breakdown == -1))
+ do_arc_size_breakdown = do_memory = do_important_ops = do_actual_hits = do_demand_data_hits \
+ = do_prefetch_data_hits = do_hash_elements = do_hash_chains = show_zero_charts;
+
+ unsigned long long arc_accesses_total = arcstats.hits + arcstats.misses;
+ unsigned long long real_hits = arcstats.mfu_hits + arcstats.mru_hits;
+ unsigned long long real_misses = arc_accesses_total - real_hits;
+
+ //unsigned long long anon_hits = arcstats.hits - (arcstats.mfu_hits + arcstats.mru_hits + arcstats.mfu_ghost_hits + arcstats.mru_ghost_hits);
+
+ unsigned long long arc_size = arcstats.size;
+ unsigned long long mru_size = arcstats.p;
+ //unsigned long long target_min_size = arcstats.c_min;
+ //unsigned long long target_max_size = arcstats.c_max;
+ unsigned long long target_size = arcstats.c;
+ //unsigned long long target_size_ratio = (target_max_size / target_min_size);
+
+ unsigned long long mfu_size;
+ if(arc_size > target_size)
+ mfu_size = arc_size - mru_size;
+ else
+ mfu_size = target_size - mru_size;
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_arc_size_breakdown == CONFIG_BOOLEAN_YES || mru_size || mfu_size)) {
+ do_arc_size_breakdown = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_arc_size_breakdown = NULL;
+ static RRDDIM *rd_most_recent = NULL;
+ static RRDDIM *rd_most_frequent = NULL;
+
+ if (unlikely(!st_arc_size_breakdown)) {
+ st_arc_size_breakdown = rrdset_create_localhost(
+ "zfs"
+ , "arc_size_breakdown"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS ARC Size Breakdown"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_ARC_SIZE_BREAKDOWN
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_most_recent = rrddim_add(st_arc_size_breakdown, "recent", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL);
+ rd_most_frequent = rrddim_add(st_arc_size_breakdown, "frequent", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_arc_size_breakdown, rd_most_recent, mru_size);
+ rrddim_set_by_pointer(st_arc_size_breakdown, rd_most_frequent, mfu_size);
+ rrdset_done(st_arc_size_breakdown);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_memory == CONFIG_BOOLEAN_YES || arcstats.memory_direct_count \
+ || arcstats.memory_throttle_count \
+ || arcstats.memory_indirect_count)) {
+ do_memory = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_memory = NULL;
+#ifndef __FreeBSD__
+ static RRDDIM *rd_direct = NULL;
+#endif
+ static RRDDIM *rd_throttled = NULL;
+#ifndef __FreeBSD__
+ static RRDDIM *rd_indirect = NULL;
+#endif
+
+ if (unlikely(!st_memory)) {
+ st_memory = rrdset_create_localhost(
+ "zfs"
+ , "memory_ops"
+ , NULL
+ , ZFS_FAMILY_OPERATIONS
+ , NULL
+ , "ZFS Memory Operations"
+ , "operations/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_MEMORY_OPS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+#ifndef __FreeBSD__
+ rd_direct = rrddim_add(st_memory, "direct", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#endif
+ rd_throttled = rrddim_add(st_memory, "throttled", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#ifndef __FreeBSD__
+ rd_indirect = rrddim_add(st_memory, "indirect", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#endif
+ }
+
+#ifndef __FreeBSD__
+ rrddim_set_by_pointer(st_memory, rd_direct, arcstats.memory_direct_count);
+#endif
+ rrddim_set_by_pointer(st_memory, rd_throttled, arcstats.memory_throttle_count);
+#ifndef __FreeBSD__
+ rrddim_set_by_pointer(st_memory, rd_indirect, arcstats.memory_indirect_count);
+#endif
+ rrdset_done(st_memory);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_important_ops == CONFIG_BOOLEAN_YES || arcstats.deleted \
+ || arcstats.evict_skip \
+ || arcstats.mutex_miss \
+ || arcstats.hash_collisions)) {
+ do_important_ops = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_important_ops = NULL;
+ static RRDDIM *rd_deleted = NULL;
+ static RRDDIM *rd_mutex_misses = NULL;
+ static RRDDIM *rd_evict_skips = NULL;
+ static RRDDIM *rd_hash_collisions = NULL;
+
+ if (unlikely(!st_important_ops)) {
+ st_important_ops = rrdset_create_localhost(
+ "zfs"
+ , "important_ops"
+ , NULL
+ , ZFS_FAMILY_OPERATIONS
+ , NULL
+ , "ZFS Important Operations"
+ , "operations/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_IMPORTANT_OPS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_evict_skips = rrddim_add(st_important_ops, "eskip", "evict skip", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_deleted = rrddim_add(st_important_ops, "deleted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mutex_misses = rrddim_add(st_important_ops, "mtxmis", "mutex miss", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_hash_collisions = rrddim_add(st_important_ops, "hash_collisions", "hash collisions", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_important_ops, rd_deleted, arcstats.deleted);
+ rrddim_set_by_pointer(st_important_ops, rd_evict_skips, arcstats.evict_skip);
+ rrddim_set_by_pointer(st_important_ops, rd_mutex_misses, arcstats.mutex_miss);
+ rrddim_set_by_pointer(st_important_ops, rd_hash_collisions, arcstats.hash_collisions);
+ rrdset_done(st_important_ops);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_actual_hits == CONFIG_BOOLEAN_YES || real_hits || real_misses)) {
+ do_actual_hits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_actual_hits = NULL;
+ static RRDDIM *rd_actual_hits = NULL;
+ static RRDDIM *rd_actual_misses = NULL;
+
+ if (unlikely(!st_actual_hits)) {
+ st_actual_hits = rrdset_create_localhost(
+ "zfs"
+ , "actual_hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Actual Cache Hits"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_ACTUAL_HITS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_actual_hits = rrddim_add(st_actual_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_actual_misses = rrddim_add(st_actual_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_actual_hits, rd_actual_hits, real_hits);
+ rrddim_set_by_pointer(st_actual_hits, rd_actual_misses, real_misses);
+ rrdset_done(st_actual_hits);
+
+ static RRDSET *st_actual_hits_rate = NULL;
+ static RRDDIM *rd_actual_hits_rate = NULL;
+ static RRDDIM *rd_actual_misses_rate = NULL;
+
+ if (unlikely(!st_actual_hits_rate)) {
+ st_actual_hits_rate = rrdset_create_localhost(
+ "zfs"
+ , "actual_hits_rate"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Actual Cache Hits Rate"
+ , "events/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_ACTUAL_HITS + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_actual_hits_rate = rrddim_add(st_actual_hits_rate, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_actual_misses_rate = rrddim_add(st_actual_hits_rate, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_actual_hits_rate, rd_actual_hits_rate, real_hits);
+ rrddim_set_by_pointer(st_actual_hits_rate, rd_actual_misses_rate, real_misses);
+ rrdset_done(st_actual_hits_rate);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_demand_data_hits == CONFIG_BOOLEAN_YES || arcstats.demand_data_hits || arcstats.demand_data_misses)) {
+ do_demand_data_hits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_demand_data_hits = NULL;
+ static RRDDIM *rd_demand_data_hits = NULL;
+ static RRDDIM *rd_demand_data_misses = NULL;
+
+ if (unlikely(!st_demand_data_hits)) {
+ st_demand_data_hits = rrdset_create_localhost(
+ "zfs"
+ , "demand_data_hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Data Demand Efficiency"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_DEMAND_DATA_HITS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_demand_data_hits = rrddim_add(st_demand_data_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_demand_data_misses = rrddim_add(st_demand_data_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_demand_data_hits, rd_demand_data_hits, arcstats.demand_data_hits);
+ rrddim_set_by_pointer(st_demand_data_hits, rd_demand_data_misses, arcstats.demand_data_misses);
+ rrdset_done(st_demand_data_hits);
+
+ static RRDSET *st_demand_data_hits_rate = NULL;
+ static RRDDIM *rd_demand_data_hits_rate = NULL;
+ static RRDDIM *rd_demand_data_misses_rate = NULL;
+
+ if (unlikely(!st_demand_data_hits_rate)) {
+ st_demand_data_hits_rate = rrdset_create_localhost(
+ "zfs"
+ , "demand_data_hits_rate"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Data Demand Efficiency Rate"
+ , "events/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_DEMAND_DATA_HITS + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_demand_data_hits_rate = rrddim_add(st_demand_data_hits_rate, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_demand_data_misses_rate = rrddim_add(st_demand_data_hits_rate, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_demand_data_hits_rate, rd_demand_data_hits_rate, arcstats.demand_data_hits);
+ rrddim_set_by_pointer(st_demand_data_hits_rate, rd_demand_data_misses_rate, arcstats.demand_data_misses);
+ rrdset_done(st_demand_data_hits_rate);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_prefetch_data_hits == CONFIG_BOOLEAN_YES || arcstats.prefetch_data_hits \
+ || arcstats.prefetch_data_misses)) {
+ do_prefetch_data_hits = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_prefetch_data_hits = NULL;
+ static RRDDIM *rd_prefetch_data_hits = NULL;
+ static RRDDIM *rd_prefetch_data_misses = NULL;
+
+ if (unlikely(!st_prefetch_data_hits)) {
+ st_prefetch_data_hits = rrdset_create_localhost(
+ "zfs"
+ , "prefetch_data_hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Data Prefetch Efficiency"
+ , "percentage"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_PREFETCH_DATA_HITS
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_prefetch_data_hits = rrddim_add(st_prefetch_data_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_prefetch_data_misses = rrddim_add(st_prefetch_data_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+
+ rrddim_set_by_pointer(st_prefetch_data_hits, rd_prefetch_data_hits, arcstats.prefetch_data_hits);
+ rrddim_set_by_pointer(st_prefetch_data_hits, rd_prefetch_data_misses, arcstats.prefetch_data_misses);
+ rrdset_done(st_prefetch_data_hits);
+
+ static RRDSET *st_prefetch_data_hits_rate = NULL;
+ static RRDDIM *rd_prefetch_data_hits_rate = NULL;
+ static RRDDIM *rd_prefetch_data_misses_rate = NULL;
+
+ if (unlikely(!st_prefetch_data_hits_rate)) {
+ st_prefetch_data_hits_rate = rrdset_create_localhost(
+ "zfs"
+ , "prefetch_data_hits_rate"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Data Prefetch Efficiency Rate"
+ , "events/s"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_PREFETCH_DATA_HITS + 1
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_prefetch_data_hits_rate = rrddim_add(st_prefetch_data_hits_rate, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_prefetch_data_misses_rate = rrddim_add(st_prefetch_data_hits_rate, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(st_prefetch_data_hits_rate, rd_prefetch_data_hits_rate, arcstats.prefetch_data_hits);
+ rrddim_set_by_pointer(st_prefetch_data_hits_rate, rd_prefetch_data_misses_rate, arcstats.prefetch_data_misses);
+ rrdset_done(st_prefetch_data_hits_rate);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_hash_elements == CONFIG_BOOLEAN_YES || arcstats.hash_elements || arcstats.hash_elements_max)) {
+ do_hash_elements = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_hash_elements = NULL;
+ static RRDDIM *rd_hash_elements_current = NULL;
+ static RRDDIM *rd_hash_elements_max = NULL;
+
+ if (unlikely(!st_hash_elements)) {
+ st_hash_elements = rrdset_create_localhost(
+ "zfs"
+ , "hash_elements"
+ , NULL
+ , ZFS_FAMILY_HASH
+ , NULL
+ , "ZFS ARC Hash Elements"
+ , "elements"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_HASH_ELEMENTS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_hash_elements_current = rrddim_add(st_hash_elements, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_hash_elements_max = rrddim_add(st_hash_elements, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_hash_elements, rd_hash_elements_current, arcstats.hash_elements);
+ rrddim_set_by_pointer(st_hash_elements, rd_hash_elements_max, arcstats.hash_elements_max);
+ rrdset_done(st_hash_elements);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(do_hash_chains == CONFIG_BOOLEAN_YES || arcstats.hash_chains || arcstats.hash_chain_max)) {
+ do_hash_chains = CONFIG_BOOLEAN_YES;
+
+ static RRDSET *st_hash_chains = NULL;
+ static RRDDIM *rd_hash_chains_current = NULL;
+ static RRDDIM *rd_hash_chains_max = NULL;
+
+ if (unlikely(!st_hash_chains)) {
+ st_hash_chains = rrdset_create_localhost(
+ "zfs"
+ , "hash_chains"
+ , NULL
+ , ZFS_FAMILY_HASH
+ , NULL
+ , "ZFS ARC Hash Chains"
+ , "chains"
+ , plugin
+ , module
+ , NETDATA_CHART_PRIO_ZFS_HASH_CHAINS
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_hash_chains_current = rrddim_add(st_hash_chains, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_hash_chains_max = rrddim_add(st_hash_chains, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_hash_chains, rd_hash_chains_current, arcstats.hash_chains);
+ rrddim_set_by_pointer(st_hash_chains, rd_hash_chains_max, arcstats.hash_chain_max);
+ rrdset_done(st_hash_chains);
+ }
+
+ // --------------------------------------------------------------------
+
+}
diff --git a/collectors/proc.plugin/zfs_common.h b/collectors/proc.plugin/zfs_common.h
new file mode 100644
index 0000000..9d61de2
--- /dev/null
+++ b/collectors/proc.plugin/zfs_common.h
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_ZFS_COMMON_H
+#define NETDATA_ZFS_COMMON_H 1
+
+#include "daemon/common.h"
+
+#define ZFS_FAMILY_SIZE "size"
+#define ZFS_FAMILY_EFFICIENCY "efficiency"
+#define ZFS_FAMILY_ACCESSES "accesses"
+#define ZFS_FAMILY_OPERATIONS "operations"
+#define ZFS_FAMILY_HASH "hashes"
+
+struct arcstats {
+ // values
+ unsigned long long hits;
+ unsigned long long misses;
+ unsigned long long demand_data_hits;
+ unsigned long long demand_data_misses;
+ unsigned long long demand_metadata_hits;
+ unsigned long long demand_metadata_misses;
+ unsigned long long prefetch_data_hits;
+ unsigned long long prefetch_data_misses;
+ unsigned long long prefetch_metadata_hits;
+ unsigned long long prefetch_metadata_misses;
+ unsigned long long mru_hits;
+ unsigned long long mru_ghost_hits;
+ unsigned long long mfu_hits;
+ unsigned long long mfu_ghost_hits;
+ unsigned long long deleted;
+ unsigned long long mutex_miss;
+ unsigned long long evict_skip;
+ unsigned long long evict_not_enough;
+ unsigned long long evict_l2_cached;
+ unsigned long long evict_l2_eligible;
+ unsigned long long evict_l2_ineligible;
+ unsigned long long evict_l2_skip;
+ unsigned long long hash_elements;
+ unsigned long long hash_elements_max;
+ unsigned long long hash_collisions;
+ unsigned long long hash_chains;
+ unsigned long long hash_chain_max;
+ unsigned long long p;
+ unsigned long long c;
+ unsigned long long c_min;
+ unsigned long long c_max;
+ unsigned long long size;
+ unsigned long long hdr_size;
+ unsigned long long data_size;
+ unsigned long long metadata_size;
+ unsigned long long other_size;
+ unsigned long long anon_size;
+ unsigned long long anon_evictable_data;
+ unsigned long long anon_evictable_metadata;
+ unsigned long long mru_size;
+ unsigned long long mru_evictable_data;
+ unsigned long long mru_evictable_metadata;
+ unsigned long long mru_ghost_size;
+ unsigned long long mru_ghost_evictable_data;
+ unsigned long long mru_ghost_evictable_metadata;
+ unsigned long long mfu_size;
+ unsigned long long mfu_evictable_data;
+ unsigned long long mfu_evictable_metadata;
+ unsigned long long mfu_ghost_size;
+ unsigned long long mfu_ghost_evictable_data;
+ unsigned long long mfu_ghost_evictable_metadata;
+ unsigned long long l2_hits;
+ unsigned long long l2_misses;
+ unsigned long long l2_feeds;
+ unsigned long long l2_rw_clash;
+ unsigned long long l2_read_bytes;
+ unsigned long long l2_write_bytes;
+ unsigned long long l2_writes_sent;
+ unsigned long long l2_writes_done;
+ unsigned long long l2_writes_error;
+ unsigned long long l2_writes_lock_retry;
+ unsigned long long l2_evict_lock_retry;
+ unsigned long long l2_evict_reading;
+ unsigned long long l2_evict_l1cached;
+ unsigned long long l2_free_on_write;
+ unsigned long long l2_cdata_free_on_write;
+ unsigned long long l2_abort_lowmem;
+ unsigned long long l2_cksum_bad;
+ unsigned long long l2_io_error;
+ unsigned long long l2_size;
+ unsigned long long l2_asize;
+ unsigned long long l2_hdr_size;
+ unsigned long long l2_compress_successes;
+ unsigned long long l2_compress_zeros;
+ unsigned long long l2_compress_failures;
+ unsigned long long memory_throttle_count;
+ unsigned long long duplicate_buffers;
+ unsigned long long duplicate_buffers_size;
+ unsigned long long duplicate_reads;
+ unsigned long long memory_direct_count;
+ unsigned long long memory_indirect_count;
+ unsigned long long arc_no_grow;
+ unsigned long long arc_tempreserve;
+ unsigned long long arc_loaned_bytes;
+ unsigned long long arc_prune;
+ unsigned long long arc_meta_used;
+ unsigned long long arc_meta_limit;
+ unsigned long long arc_meta_max;
+ unsigned long long arc_meta_min;
+ unsigned long long arc_need_free;
+ unsigned long long arc_sys_free;
+
+ // flags
+ int l2exist;
+};
+
+void generate_charts_arcstats(const char *plugin, const char *module, int show_zero_charts, int update_every);
+void generate_charts_arc_summary(const char *plugin, const char *module, int show_zero_charts, int update_every);
+
+#endif //NETDATA_ZFS_COMMON_H
diff --git a/collectors/python.d.plugin/Makefile.am b/collectors/python.d.plugin/Makefile.am
new file mode 100644
index 0000000..1bbbf8c
--- /dev/null
+++ b/collectors/python.d.plugin/Makefile.am
@@ -0,0 +1,236 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+CLEANFILES = \
+ python.d.plugin \
+ $(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+dist_libconfig_DATA = \
+ python.d.conf \
+ $(NULL)
+
+dist_plugins_SCRIPTS = \
+ python.d.plugin \
+ $(NULL)
+
+dist_noinst_DATA = \
+ python.d.plugin.in \
+ README.md \
+ $(NULL)
+
+dist_python_SCRIPTS = \
+ $(NULL)
+
+dist_python_DATA = \
+ $(NULL)
+
+userpythonconfigdir=$(configdir)/python.d
+dist_userpythonconfig_DATA = \
+ $(NULL)
+
+# Explicitly install directories to avoid permission issues due to umask
+install-exec-local:
+ $(INSTALL) -d $(DESTDIR)$(userpythonconfigdir)
+
+pythonconfigdir=$(libconfigdir)/python.d
+dist_pythonconfig_DATA = \
+ $(NULL)
+
+include adaptec_raid/Makefile.inc
+include alarms/Makefile.inc
+include am2320/Makefile.inc
+include anomalies/Makefile.inc
+include beanstalk/Makefile.inc
+include bind_rndc/Makefile.inc
+include boinc/Makefile.inc
+include ceph/Makefile.inc
+include changefinder/Makefile.inc
+include dockerd/Makefile.inc
+include dovecot/Makefile.inc
+include example/Makefile.inc
+include exim/Makefile.inc
+include fail2ban/Makefile.inc
+include gearman/Makefile.inc
+include go_expvar/Makefile.inc
+include haproxy/Makefile.inc
+include hddtemp/Makefile.inc
+include hpssa/Makefile.inc
+include icecast/Makefile.inc
+include ipfs/Makefile.inc
+include litespeed/Makefile.inc
+include logind/Makefile.inc
+include megacli/Makefile.inc
+include memcached/Makefile.inc
+include mongodb/Makefile.inc
+include monit/Makefile.inc
+include nvidia_smi/Makefile.inc
+include nsd/Makefile.inc
+include ntpd/Makefile.inc
+include openldap/Makefile.inc
+include oracledb/Makefile.inc
+include pandas/Makefile.inc
+include postfix/Makefile.inc
+include proxysql/Makefile.inc
+include puppet/Makefile.inc
+include rabbitmq/Makefile.inc
+include rethinkdbs/Makefile.inc
+include retroshare/Makefile.inc
+include riakkv/Makefile.inc
+include samba/Makefile.inc
+include sensors/Makefile.inc
+include smartd_log/Makefile.inc
+include spigotmc/Makefile.inc
+include springboot/Makefile.inc
+include squid/Makefile.inc
+include tomcat/Makefile.inc
+include tor/Makefile.inc
+include traefik/Makefile.inc
+include uwsgi/Makefile.inc
+include varnish/Makefile.inc
+include w1sensor/Makefile.inc
+include zscores/Makefile.inc
+
+pythonmodulesdir=$(pythondir)/python_modules
+dist_pythonmodules_DATA = \
+ python_modules/__init__.py \
+ $(NULL)
+
+basesdir=$(pythonmodulesdir)/bases
+dist_bases_DATA = \
+ python_modules/bases/__init__.py \
+ python_modules/bases/charts.py \
+ python_modules/bases/collection.py \
+ python_modules/bases/loaders.py \
+ python_modules/bases/loggers.py \
+ $(NULL)
+
+bases_framework_servicesdir=$(basesdir)/FrameworkServices
+dist_bases_framework_services_DATA = \
+ python_modules/bases/FrameworkServices/__init__.py \
+ python_modules/bases/FrameworkServices/ExecutableService.py \
+ python_modules/bases/FrameworkServices/LogService.py \
+ python_modules/bases/FrameworkServices/MySQLService.py \
+ python_modules/bases/FrameworkServices/SimpleService.py \
+ python_modules/bases/FrameworkServices/SocketService.py \
+ python_modules/bases/FrameworkServices/UrlService.py \
+ $(NULL)
+
+third_partydir=$(pythonmodulesdir)/third_party
+dist_third_party_DATA = \
+ python_modules/third_party/__init__.py \
+ python_modules/third_party/ordereddict.py \
+ python_modules/third_party/lm_sensors.py \
+ python_modules/third_party/mcrcon.py \
+ python_modules/third_party/boinc_client.py \
+ python_modules/third_party/monotonic.py \
+ python_modules/third_party/filelock.py \
+ $(NULL)
+
+pythonyaml2dir=$(pythonmodulesdir)/pyyaml2
+dist_pythonyaml2_DATA = \
+ python_modules/pyyaml2/__init__.py \
+ python_modules/pyyaml2/composer.py \
+ python_modules/pyyaml2/constructor.py \
+ python_modules/pyyaml2/cyaml.py \
+ python_modules/pyyaml2/dumper.py \
+ python_modules/pyyaml2/emitter.py \
+ python_modules/pyyaml2/error.py \
+ python_modules/pyyaml2/events.py \
+ python_modules/pyyaml2/loader.py \
+ python_modules/pyyaml2/nodes.py \
+ python_modules/pyyaml2/parser.py \
+ python_modules/pyyaml2/reader.py \
+ python_modules/pyyaml2/representer.py \
+ python_modules/pyyaml2/resolver.py \
+ python_modules/pyyaml2/scanner.py \
+ python_modules/pyyaml2/serializer.py \
+ python_modules/pyyaml2/tokens.py \
+ $(NULL)
+
+pythonyaml3dir=$(pythonmodulesdir)/pyyaml3
+dist_pythonyaml3_DATA = \
+ python_modules/pyyaml3/__init__.py \
+ python_modules/pyyaml3/composer.py \
+ python_modules/pyyaml3/constructor.py \
+ python_modules/pyyaml3/cyaml.py \
+ python_modules/pyyaml3/dumper.py \
+ python_modules/pyyaml3/emitter.py \
+ python_modules/pyyaml3/error.py \
+ python_modules/pyyaml3/events.py \
+ python_modules/pyyaml3/loader.py \
+ python_modules/pyyaml3/nodes.py \
+ python_modules/pyyaml3/parser.py \
+ python_modules/pyyaml3/reader.py \
+ python_modules/pyyaml3/representer.py \
+ python_modules/pyyaml3/resolver.py \
+ python_modules/pyyaml3/scanner.py \
+ python_modules/pyyaml3/serializer.py \
+ python_modules/pyyaml3/tokens.py \
+ $(NULL)
+
+python_urllib3dir=$(pythonmodulesdir)/urllib3
+dist_python_urllib3_DATA = \
+ python_modules/urllib3/__init__.py \
+ python_modules/urllib3/_collections.py \
+ python_modules/urllib3/connection.py \
+ python_modules/urllib3/connectionpool.py \
+ python_modules/urllib3/exceptions.py \
+ python_modules/urllib3/fields.py \
+ python_modules/urllib3/filepost.py \
+ python_modules/urllib3/response.py \
+ python_modules/urllib3/poolmanager.py \
+ python_modules/urllib3/request.py \
+ $(NULL)
+
+python_urllib3_utildir=$(python_urllib3dir)/util
+dist_python_urllib3_util_DATA = \
+ python_modules/urllib3/util/__init__.py \
+ python_modules/urllib3/util/connection.py \
+ python_modules/urllib3/util/request.py \
+ python_modules/urllib3/util/response.py \
+ python_modules/urllib3/util/retry.py \
+ python_modules/urllib3/util/selectors.py \
+ python_modules/urllib3/util/ssl_.py \
+ python_modules/urllib3/util/timeout.py \
+ python_modules/urllib3/util/url.py \
+ python_modules/urllib3/util/wait.py \
+ $(NULL)
+
+python_urllib3_packagesdir=$(python_urllib3dir)/packages
+dist_python_urllib3_packages_DATA = \
+ python_modules/urllib3/packages/__init__.py \
+ python_modules/urllib3/packages/ordered_dict.py \
+ python_modules/urllib3/packages/six.py \
+ $(NULL)
+
+python_urllib3_backportsdir=$(python_urllib3_packagesdir)/backports
+dist_python_urllib3_backports_DATA = \
+ python_modules/urllib3/packages/backports/__init__.py \
+ python_modules/urllib3/packages/backports/makefile.py \
+ $(NULL)
+
+python_urllib3_ssl_match_hostnamedir=$(python_urllib3_packagesdir)/ssl_match_hostname
+dist_python_urllib3_ssl_match_hostname_DATA = \
+ python_modules/urllib3/packages/ssl_match_hostname/__init__.py \
+ python_modules/urllib3/packages/ssl_match_hostname/_implementation.py \
+ $(NULL)
+
+python_urllib3_contribdir=$(python_urllib3dir)/contrib
+dist_python_urllib3_contrib_DATA = \
+ python_modules/urllib3/contrib/__init__.py \
+ python_modules/urllib3/contrib/appengine.py \
+ python_modules/urllib3/contrib/ntlmpool.py \
+ python_modules/urllib3/contrib/pyopenssl.py \
+ python_modules/urllib3/contrib/securetransport.py \
+ python_modules/urllib3/contrib/socks.py \
+ $(NULL)
+
+python_urllib3_securetransportdir=$(python_urllib3_contribdir)/_securetransport
+dist_python_urllib3_securetransport_DATA = \
+ python_modules/urllib3/contrib/_securetransport/__init__.py \
+ python_modules/urllib3/contrib/_securetransport/bindings.py \
+ python_modules/urllib3/contrib/_securetransport/low_level.py \
+ $(NULL)
diff --git a/collectors/python.d.plugin/README.md b/collectors/python.d.plugin/README.md
new file mode 100644
index 0000000..2f5ebfc
--- /dev/null
+++ b/collectors/python.d.plugin/README.md
@@ -0,0 +1,270 @@
+<!--
+title: "python.d.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/README.md
+-->
+
+# python.d.plugin
+
+`python.d.plugin` is a Netdata external plugin. It is an **orchestrator** for data collection modules written in `python`.
+
+1. It runs as an independent process `ps fax` shows it
+2. It is started and stopped automatically by Netdata
+3. It communicates with Netdata via a unidirectional pipe (sending data to the `netdata` daemon)
+4. Supports any number of data collection **modules**
+5. Allows each **module** to have one or more data collection **jobs**
+6. Each **job** is collecting one or more metrics from a single data source
+
+## Disclaimer
+
+All third party libraries should be installed system-wide or in `python_modules` directory.
+Module configurations are written in YAML and **pyYAML is required**.
+
+Every configuration file must have one of two formats:
+
+- Configuration for only one job:
+
+```yaml
+update_every : 2 # update frequency
+priority : 20000 # where it is shown on dashboard
+
+other_var1 : bla # variables passed to module
+other_var2 : alb
+```
+
+- Configuration for many jobs (ex. mysql):
+
+```yaml
+# module defaults:
+update_every : 2
+priority : 20000
+
+local: # job name
+ update_every : 5 # job update frequency
+ other_var1 : some_val # module specific variable
+
+other_job:
+ priority : 5 # job position on dashboard
+ other_var2 : val # module specific variable
+```
+
+`update_every` and `priority` are always optional.
+
+## How to debug a python module
+
+```
+# become user netdata
+sudo su -s /bin/bash netdata
+```
+
+Depending on where Netdata was installed, execute one of the following commands to trace the execution of a python module:
+
+```
+# execute the plugin in debug mode, for a specific module
+/opt/netdata/usr/libexec/netdata/plugins.d/python.d.plugin <module> debug trace
+/usr/libexec/netdata/plugins.d/python.d.plugin <module> debug trace
+```
+
+Where `[module]` is the directory name under <https://github.com/netdata/netdata/tree/master/collectors/python.d.plugin>
+
+**Note**: If you would like execute a collector in debug mode while it is still running by Netdata, you can pass the `nolock` CLI option to the above commands.
+
+## How to write a new module
+
+Writing new python module is simple. You just need to remember to include 5 major things:
+
+- **ORDER** global list
+- **CHART** global dictionary
+- **Service** class
+- **\_get_data** method
+
+If you plan to submit the module in a PR, make sure and go through the [PR checklist for new modules](#pull-request-checklist-for-python-plugins) beforehand to make sure you have updated all the files you need to.
+
+For a quick start, you can look at the [example
+plugin](https://raw.githubusercontent.com/netdata/netdata/master/collectors/python.d.plugin/example/example.chart.py).
+
+**Note**: If you are working 'locally' on a new collector and would like to run it in an already installed and running
+Netdata (as opposed to having to install Netdata from source again with your new changes) to can copy over the relevant
+file to where Netdata expects it and then either `sudo systemctl restart netdata` to have it be picked up and used by
+Netdata or you can just run the updated collector in debug mode by following a process like below (this assumes you have
+[installed Netdata from a GitHub fork](https://learn.netdata.cloud/docs/agent/packaging/installer/methods/manual) you
+have made to do your development on).
+
+```bash
+# clone your fork (done once at the start but shown here for clarity)
+#git clone --branch my-example-collector https://github.com/mygithubusername/netdata.git --depth=100 --recursive
+# go into your netdata source folder
+cd netdata
+# git pull your latest changes (assuming you built from a fork you are using to develop on)
+git pull
+# instead of running the installer we can just copy over the updated collector files
+#sudo ./netdata-installer.sh --dont-wait
+# copy over the file you have updated locally (pretending we are working on the 'example' collector)
+sudo cp collectors/python.d.plugin/example/example.chart.py /usr/libexec/netdata/python.d/
+# become user netdata
+sudo su -s /bin/bash netdata
+# run your updated collector in debug mode to see if it works without having to reinstall netdata
+/usr/libexec/netdata/plugins.d/python.d.plugin example debug trace nolock
+```
+
+### Global variables `ORDER` and `CHART`
+
+`ORDER` list should contain the order of chart ids. Example:
+
+```py
+ORDER = ['first_chart', 'second_chart', 'third_chart']
+```
+
+`CHART` dictionary is a little bit trickier. It should contain the chart definition in following format:
+
+```py
+CHART = {
+ id: {
+ 'options': [name, title, units, family, context, charttype],
+ 'lines': [
+ [unique_dimension_name, name, algorithm, multiplier, divisor]
+ ]}
+```
+
+All names are better explained in the [External Plugins](/collectors/plugins.d/README.md) section.
+Parameters like `priority` and `update_every` are handled by `python.d.plugin`.
+
+### `Service` class
+
+Every module needs to implement its own `Service` class. This class should inherit from one of the framework classes:
+
+- `SimpleService`
+- `UrlService`
+- `SocketService`
+- `LogService`
+- `ExecutableService`
+
+Also it needs to invoke the parent class constructor in a specific way as well as assign global variables to class variables.
+
+Simple example:
+
+```py
+from base import UrlService
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+```
+
+### `_get_data` collector/parser
+
+This method should grab raw data from `_get_raw_data`, parse it, and return a dictionary where keys are unique dimension names or `None` if no data is collected.
+
+Example:
+
+```py
+def _get_data(self):
+ try:
+ raw = self._get_raw_data().split(" ")
+ return {'active': int(raw[2])}
+ except (ValueError, AttributeError):
+ return None
+```
+
+# More about framework classes
+
+Every framework class has some user-configurable variables which are specific to this particular class. Those variables should have default values initialized in the child class constructor.
+
+If module needs some additional user-configurable variable, it can be accessed from the `self.configuration` list and assigned in constructor or custom `check` method. Example:
+
+```py
+def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ try:
+ self.baseurl = str(self.configuration['baseurl'])
+ except (KeyError, TypeError):
+ self.baseurl = "http://localhost:5001"
+```
+
+Classes implement `_get_raw_data` which should be used to grab raw data. This method usually returns a list of strings.
+
+### `SimpleService`
+
+_This is last resort class, if a new module cannot be written by using other framework class this one can be used._
+
+_Example: `ceph`, `sensors`_
+
+It is the lowest-level class which implements most of module logic, like:
+
+- threading
+- handling run times
+- chart formatting
+- logging
+- chart creation and updating
+
+### `LogService`
+
+_Examples: `apache_cache`, `nginx_log`_
+
+_Variable from config file_: `log_path`.
+
+Object created from this class reads new lines from file specified in `log_path` variable. It will check if file exists and is readable. Also `_get_raw_data` returns list of strings where each string is one line from file specified in `log_path`.
+
+### `ExecutableService`
+
+_Examples: `exim`, `postfix`_
+
+_Variable from config file_: `command`.
+
+This allows to execute a shell command in a secure way. It will check for invalid characters in `command` variable and won't proceed if there is one of:
+
+- '&'
+- '|'
+- ';'
+- '>'
+- '\<'
+
+For additional security it uses python `subprocess.Popen` (without `shell=True` option) to execute command. Command can be specified with absolute or relative name. When using relative name, it will try to find `command` in `PATH` environment variable as well as in `/sbin` and `/usr/sbin`.
+
+`_get_raw_data` returns list of decoded lines returned by `command`.
+
+### UrlService
+
+_Examples: `apache`, `nginx`, `tomcat`_
+
+_Multiple Endpoints (urls) Examples: [`rabbitmq`](/collectors/python.d.plugin/rabbitmq/README.md) (simpler).
+
+
+_Variables from config file_: `url`, `user`, `pass`.
+
+If data is grabbed by accessing service via HTTP protocol, this class can be used. It can handle HTTP Basic Auth when specified with `user` and `pass` credentials.
+
+Please note that the config file can use different variables according to the specification of each module.
+
+`_get_raw_data` returns list of utf-8 decoded strings (lines).
+
+### SocketService
+
+_Examples: `dovecot`, `redis`_
+
+_Variables from config file_: `unix_socket`, `host`, `port`, `request`.
+
+Object will try execute `request` using either `unix_socket` or TCP/IP socket with combination of `host` and `port`. This can access unix sockets with SOCK_STREAM or SOCK_DGRAM protocols and TCP/IP sockets in version 4 and 6 with SOCK_STREAM setting.
+
+Sockets are accessed in non-blocking mode with 15 second timeout.
+
+After every execution of `_get_raw_data` socket is closed, to prevent this module needs to set `_keep_alive` variable to `True` and implement custom `_check_raw_data` method.
+
+`_check_raw_data` should take raw data and return `True` if all data is received otherwise it should return `False`. Also it should do it in fast and efficient way.
+
+## Pull Request Checklist for Python Plugins
+
+This is a generic checklist for submitting a new Python plugin for Netdata. It is by no means comprehensive.
+
+At minimum, to be buildable and testable, the PR needs to include:
+
+- The module itself, following proper naming conventions: `collectors/python.d.plugin/<module_dir>/<module_name>.chart.py`
+- A README.md file for the plugin under `collectors/python.d.plugin/<module_dir>`.
+- The configuration file for the module: `collectors/python.d.plugin/<module_dir>/<module_name>.conf`. Python config files are in YAML format, and should include comments describing what options are present. The instructions are also needed in the configuration section of the README.md
+- A basic configuration for the plugin in the appropriate global config file: `collectors/python.d.plugin/python.d.conf`, which is also in YAML format. Either add a line that reads `# <module_name>: yes` if the module is to be enabled by default, or one that reads `<module_name>: no` if it is to be disabled by default.
+- A makefile for the plugin at `collectors/python.d.plugin/<module_dir>/Makefile.inc`. Check an existing plugin for what this should look like.
+- A line in `collectors/python.d.plugin/Makefile.am` including the above-mentioned makefile. Place it with the other plugin includes (please keep the includes sorted alphabetically).
+- Optionally, chart information in `web/gui/dashboard_info.js`. This generally involves specifying a name and icon for the section, and may include descriptions for the section or individual charts.
+- Optionally, some default alarm configurations for your collector in `health/health.d/<module_name>.conf` and a line adding `<module_name>.conf` in `health/Makefile.am`.
+
+
diff --git a/collectors/python.d.plugin/adaptec_raid/Makefile.inc b/collectors/python.d.plugin/adaptec_raid/Makefile.inc
new file mode 100644
index 0000000..716cdb2
--- /dev/null
+++ b/collectors/python.d.plugin/adaptec_raid/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += adaptec_raid/adaptec_raid.chart.py
+dist_pythonconfig_DATA += adaptec_raid/adaptec_raid.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += adaptec_raid/README.md adaptec_raid/Makefile.inc
+
diff --git a/collectors/python.d.plugin/adaptec_raid/README.md b/collectors/python.d.plugin/adaptec_raid/README.md
new file mode 100644
index 0000000..da5d13b
--- /dev/null
+++ b/collectors/python.d.plugin/adaptec_raid/README.md
@@ -0,0 +1,80 @@
+<!--
+title: "Adaptec RAID controller monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/adaptec_raid/README.md
+sidebar_label: "Adaptec RAID"
+-->
+
+# Adaptec RAID controller monitoring with Netdata
+
+Collects logical and physical devices metrics using `arcconf` command-line utility.
+
+Executed commands:
+
+- `sudo -n arcconf GETCONFIG 1 LD`
+- `sudo -n arcconf GETCONFIG 1 PD`
+
+## Requirements
+
+The module uses `arcconf`, which can only be executed by `root`. It uses
+`sudo` and assumes that it is configured such that the `netdata` user can execute `arcconf` as root without a password.
+
+- Add to your `/etc/sudoers` file:
+
+`which arcconf` shows the full path to the binary.
+
+```bash
+netdata ALL=(root) NOPASSWD: /path/to/arcconf
+```
+
+- Reset Netdata's systemd
+ unit [CapabilityBoundingSet](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#Capabilities) (Linux
+ distributions with systemd)
+
+The default CapabilityBoundingSet doesn't allow using `sudo`, and is quite strict in general. Resetting is not optimal, but a next-best solution given the inability to execute `arcconf` using `sudo`.
+
+
+As the `root` user, do the following:
+
+```cmd
+mkdir /etc/systemd/system/netdata.service.d
+echo -e '[Service]\nCapabilityBoundingSet=~' | tee /etc/systemd/system/netdata.service.d/unset-capability-bounding-set.conf
+systemctl daemon-reload
+systemctl restart netdata.service
+```
+
+## Charts
+
+- Logical Device Status
+- Physical Device State
+- Physical Device S.M.A.R.T warnings
+- Physical Device Temperature
+
+## Enable the collector
+
+The `adaptec_raid` collector is disabled by default. To enable it, use `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`, to edit the `python.d.conf`
+file.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d.conf
+```
+
+Change the value of the `adaptec_raid` setting to `yes`. Save the file and restart the Netdata Agent with `sudo
+systemctl restart netdata`, or the [appropriate method](/docs/configure/start-stop-restart.md) for your system.
+
+## Configuration
+
+Edit the `python.d/adaptec_raid.conf` configuration file using `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/adaptec_raid.conf
+```
+
+![image](https://user-images.githubusercontent.com/22274335/47278133-6d306680-d601-11e8-87c2-cc9c0f42d686.png)
+
+---
+
+
diff --git a/collectors/python.d.plugin/adaptec_raid/adaptec_raid.chart.py b/collectors/python.d.plugin/adaptec_raid/adaptec_raid.chart.py
new file mode 100644
index 0000000..bb59d88
--- /dev/null
+++ b/collectors/python.d.plugin/adaptec_raid/adaptec_raid.chart.py
@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+# Description: adaptec_raid netdata python.d module
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+
+import re
+from copy import deepcopy
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+from bases.collection import find_binary
+
+disabled_by_default = True
+
+update_every = 5
+
+ORDER = [
+ 'ld_status',
+ 'pd_state',
+ 'pd_smart_warnings',
+ 'pd_temperature',
+]
+
+CHARTS = {
+ 'ld_status': {
+ 'options': [None, 'Status of logical devices (1: Failed or Degraded)', 'bool', 'logical devices',
+ 'adaptec_raid.ld_status', 'line'],
+ 'lines': []
+ },
+ 'pd_state': {
+ 'options': [None, 'State of physical devices (1: not Online)', 'bool', 'physical devices',
+ 'adaptec_raid.pd_state', 'line'],
+ 'lines': []
+ },
+ 'pd_smart_warnings': {
+ 'options': [None, 'S.M.A.R.T warnings', 'count', 'physical devices',
+ 'adaptec_raid.smart_warnings', 'line'],
+ 'lines': []
+ },
+ 'pd_temperature': {
+ 'options': [None, 'Temperature', 'celsius', 'physical devices', 'adaptec_raid.temperature', 'line'],
+ 'lines': []
+ },
+}
+
+SUDO = 'sudo'
+ARCCONF = 'arcconf'
+
+BAD_LD_STATUS = (
+ 'Degraded',
+ 'Failed',
+)
+
+GOOD_PD_STATUS = (
+ 'Online',
+)
+
+RE_LD = re.compile(
+ r'Logical [dD]evice number\s+([0-9]+).*?'
+ r'Status of [lL]ogical [dD]evice\s+: ([a-zA-Z]+)'
+)
+
+
+def find_lds(d):
+ d = ' '.join(v.strip() for v in d)
+ return [LD(*v) for v in RE_LD.findall(d)]
+
+
+def find_pds(d):
+ pds = list()
+ pd = PD()
+
+ for row in d:
+ row = row.strip()
+ if row.startswith('Device #'):
+ pd = PD()
+ pd.id = row.split('#')[-1]
+ elif not pd.id:
+ continue
+
+ if row.startswith('State'):
+ v = row.split()[-1]
+ pd.state = v
+ elif row.startswith('S.M.A.R.T. warnings'):
+ v = row.split()[-1]
+ pd.smart_warnings = v
+ elif row.startswith('Temperature'):
+ v = row.split(':')[-1].split()[0]
+ pd.temperature = v
+ elif row.startswith('NCQ status'):
+ if pd.id and pd.state and pd.smart_warnings:
+ pds.append(pd)
+ pd = PD()
+
+ return pds
+
+
+class LD:
+ def __init__(self, ld_id, status):
+ self.id = ld_id
+ self.status = status
+
+ def data(self):
+ return {
+ 'ld_{0}_status'.format(self.id): int(self.status in BAD_LD_STATUS)
+ }
+
+
+class PD:
+ def __init__(self):
+ self.id = None
+ self.state = None
+ self.smart_warnings = None
+ self.temperature = None
+
+ def data(self):
+ data = {
+ 'pd_{0}_state'.format(self.id): int(self.state not in GOOD_PD_STATUS),
+ 'pd_{0}_smart_warnings'.format(self.id): self.smart_warnings,
+ }
+ if self.temperature and self.temperature.isdigit():
+ data['pd_{0}_temperature'.format(self.id)] = self.temperature
+
+ return data
+
+
+class Arcconf:
+ def __init__(self, arcconf):
+ self.arcconf = arcconf
+
+ def ld_info(self):
+ return [self.arcconf, 'GETCONFIG', '1', 'LD']
+
+ def pd_info(self):
+ return [self.arcconf, 'GETCONFIG', '1', 'PD']
+
+
+# TODO: hardcoded sudo...
+class SudoArcconf:
+ def __init__(self, arcconf, sudo):
+ self.arcconf = Arcconf(arcconf)
+ self.sudo = sudo
+
+ def ld_info(self):
+ return [self.sudo, '-n'] + self.arcconf.ld_info()
+
+ def pd_info(self):
+ return [self.sudo, '-n'] + self.arcconf.pd_info()
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ ExecutableService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = deepcopy(CHARTS)
+ self.use_sudo = self.configuration.get('use_sudo', True)
+ self.arcconf = None
+
+ def execute(self, command, stderr=False):
+ return self._get_raw_data(command=command, stderr=stderr)
+
+ def check(self):
+ arcconf = find_binary(ARCCONF)
+ if not arcconf:
+ self.error('can\'t locate "{0}" binary'.format(ARCCONF))
+ return False
+
+ sudo = find_binary(SUDO)
+ if self.use_sudo:
+ if not sudo:
+ self.error('can\'t locate "{0}" binary'.format(SUDO))
+ return False
+ err = self.execute([sudo, '-n', '-v'], True)
+ if err:
+ self.error(' '.join(err))
+ return False
+
+ if self.use_sudo:
+ self.arcconf = SudoArcconf(arcconf, sudo)
+ else:
+ self.arcconf = Arcconf(arcconf)
+
+ lds = self.get_lds()
+ if not lds:
+ return False
+
+ self.debug('discovered logical devices ids: {0}'.format([ld.id for ld in lds]))
+
+ pds = self.get_pds()
+ if not pds:
+ return False
+
+ self.debug('discovered physical devices ids: {0}'.format([pd.id for pd in pds]))
+
+ self.update_charts(lds, pds)
+ return True
+
+ def get_data(self):
+ data = dict()
+
+ for ld in self.get_lds():
+ data.update(ld.data())
+
+ for pd in self.get_pds():
+ data.update(pd.data())
+
+ return data
+
+ def get_lds(self):
+ raw_lds = self.execute(self.arcconf.ld_info())
+ if not raw_lds:
+ return None
+
+ lds = find_lds(raw_lds)
+ if not lds:
+ self.error('failed to parse "{0}" output'.format(' '.join(self.arcconf.ld_info())))
+ self.debug('output: {0}'.format(raw_lds))
+ return None
+ return lds
+
+ def get_pds(self):
+ raw_pds = self.execute(self.arcconf.pd_info())
+ if not raw_pds:
+ return None
+
+ pds = find_pds(raw_pds)
+ if not pds:
+ self.error('failed to parse "{0}" output'.format(' '.join(self.arcconf.pd_info())))
+ self.debug('output: {0}'.format(raw_pds))
+ return None
+ return pds
+
+ def update_charts(self, lds, pds):
+ charts = self.definitions
+ for ld in lds:
+ dim = ['ld_{0}_status'.format(ld.id), 'ld {0}'.format(ld.id)]
+ charts['ld_status']['lines'].append(dim)
+
+ for pd in pds:
+ dim = ['pd_{0}_state'.format(pd.id), 'pd {0}'.format(pd.id)]
+ charts['pd_state']['lines'].append(dim)
+
+ dim = ['pd_{0}_smart_warnings'.format(pd.id), 'pd {0}'.format(pd.id)]
+ charts['pd_smart_warnings']['lines'].append(dim)
+
+ dim = ['pd_{0}_temperature'.format(pd.id), 'pd {0}'.format(pd.id)]
+ charts['pd_temperature']['lines'].append(dim)
diff --git a/collectors/python.d.plugin/adaptec_raid/adaptec_raid.conf b/collectors/python.d.plugin/adaptec_raid/adaptec_raid.conf
new file mode 100644
index 0000000..fa462ec
--- /dev/null
+++ b/collectors/python.d.plugin/adaptec_raid/adaptec_raid.conf
@@ -0,0 +1,53 @@
+# netdata python.d.plugin configuration for adaptec raid
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+# ----------------------------------------------------------------------
diff --git a/collectors/python.d.plugin/alarms/Makefile.inc b/collectors/python.d.plugin/alarms/Makefile.inc
new file mode 100644
index 0000000..c2de117
--- /dev/null
+++ b/collectors/python.d.plugin/alarms/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += alarms/alarms.chart.py
+dist_pythonconfig_DATA += alarms/alarms.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += alarms/README.md alarms/Makefile.inc
+
diff --git a/collectors/python.d.plugin/alarms/README.md b/collectors/python.d.plugin/alarms/README.md
new file mode 100644
index 0000000..8dc666f
--- /dev/null
+++ b/collectors/python.d.plugin/alarms/README.md
@@ -0,0 +1,66 @@
+<!--
+title: "Alarms"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/alarms/README.md
+-->
+
+# Alarms - graphing Netdata alarm states over time
+
+This collector creates an 'Alarms' menu with one line plot showing alarm states over time. Alarm states are mapped to integer values according to the below default mapping. Any alarm status types not in this mapping will be ignored (Note: This mapping can be changed by editing the `status_map` in the `alarms.conf` file). If you would like to learn more about the different alarm statuses check out the docs [here](https://learn.netdata.cloud/docs/agent/health/reference#alarm-statuses).
+
+```
+{
+ 'CLEAR': 0,
+ 'WARNING': 1,
+ 'CRITICAL': 2
+}
+```
+
+## Charts
+
+Below is an example of the chart produced when running `stress-ng --all 2` for a few minutes. You can see the various warning and critical alarms raised.
+
+![alarms collector](https://user-images.githubusercontent.com/1153921/101641493-0b086a80-39ef-11eb-9f55-0713e5dfb19f.png)
+
+## Configuration
+
+Enable the collector and [restart Netdata](/docs/configure/start-stop-restart.md).
+
+```bash
+cd /etc/netdata/
+sudo ./edit-config python.d.conf
+# Set `alarms: no` to `alarms: yes`
+sudo systemctl restart netdata
+```
+
+If needed, edit the `python.d/alarms.conf` configuration file using `edit-config` from the your agent's [config
+directory](/docs/configure/nodes.md), which is usually at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/alarms.conf
+```
+
+The `alarms` specific part of the `alarms.conf` file should look like this:
+
+```yaml
+# what url to pull data from
+local:
+ url: 'http://127.0.0.1:19999/api/v1/alarms?all'
+ # define how to map alarm status to numbers for the chart
+ status_map:
+ CLEAR: 0
+ WARNING: 1
+ CRITICAL: 2
+ # set to true to include a chart with calculated alarm values over time
+ collect_alarm_values: false
+ # define the type of chart for plotting status over time e.g. 'line' or 'stacked'
+ alarm_status_chart_type: 'line'
+ # a "," separated list of words you want to filter alarm names for. For example 'cpu,load' would filter for only
+ # alarms with "cpu" or "load" in alarm name. Default includes all.
+ alarm_contains_words: ''
+ # a "," separated list of words you want to exclude based on alarm name. For example 'cpu,load' would exclude
+ # all alarms with "cpu" or "load" in alarm name. Default excludes None.
+ alarm_excludes_words: ''
+```
+
+It will default to pulling all alarms at each time step from the Netdata rest api at `http://127.0.0.1:19999/api/v1/alarms?all`
diff --git a/collectors/python.d.plugin/alarms/alarms.chart.py b/collectors/python.d.plugin/alarms/alarms.chart.py
new file mode 100644
index 0000000..d194273
--- /dev/null
+++ b/collectors/python.d.plugin/alarms/alarms.chart.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# Description: alarms netdata python.d module
+# Author: andrewm4894
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from json import loads
+
+from bases.FrameworkServices.UrlService import UrlService
+
+update_every = 10
+disabled_by_default = True
+
+
+def charts_template(sm, alarm_status_chart_type='line'):
+ order = [
+ 'alarms',
+ 'values'
+ ]
+
+ mappings = ', '.join(['{0}={1}'.format(k, v) for k, v in sm.items()])
+ charts = {
+ 'alarms': {
+ 'options': [None, 'Alarms ({0})'.format(mappings), 'status', 'status', 'alarms.status', alarm_status_chart_type],
+ 'lines': [],
+ 'variables': [
+ ['alarms_num'],
+ ]
+ },
+ 'values': {
+ 'options': [None, 'Alarm Values', 'value', 'value', 'alarms.value', 'line'],
+ 'lines': [],
+ }
+ }
+ return order, charts
+
+
+DEFAULT_STATUS_MAP = {'CLEAR': 0, 'WARNING': 1, 'CRITICAL': 2}
+DEFAULT_URL = 'http://127.0.0.1:19999/api/v1/alarms?all'
+DEFAULT_COLLECT_ALARM_VALUES = False
+DEFAULT_ALARM_STATUS_CHART_TYPE = 'line'
+DEFAULT_ALARM_CONTAINS_WORDS = ''
+DEFAULT_ALARM_EXCLUDES_WORDS = ''
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.sm = self.configuration.get('status_map', DEFAULT_STATUS_MAP)
+ self.alarm_status_chart_type = self.configuration.get('alarm_status_chart_type', DEFAULT_ALARM_STATUS_CHART_TYPE)
+ self.order, self.definitions = charts_template(self.sm, self.alarm_status_chart_type)
+ self.url = self.configuration.get('url', DEFAULT_URL)
+ self.collect_alarm_values = bool(self.configuration.get('collect_alarm_values', DEFAULT_COLLECT_ALARM_VALUES))
+ self.collected_dims = {'alarms': set(), 'values': set()}
+ self.alarm_contains_words = self.configuration.get('alarm_contains_words', DEFAULT_ALARM_CONTAINS_WORDS)
+ self.alarm_contains_words_list = [alarm_contains_word.lstrip(' ').rstrip(' ') for alarm_contains_word in self.alarm_contains_words.split(',')]
+ self.alarm_excludes_words = self.configuration.get('alarm_excludes_words', DEFAULT_ALARM_EXCLUDES_WORDS)
+ self.alarm_excludes_words_list = [alarm_excludes_word.lstrip(' ').rstrip(' ') for alarm_excludes_word in self.alarm_excludes_words.split(',')]
+
+ def _get_data(self):
+ raw_data = self._get_raw_data()
+ if raw_data is None:
+ return None
+
+ raw_data = loads(raw_data)
+ alarms = raw_data.get('alarms', {})
+ if self.alarm_contains_words != '':
+ alarms = {alarm_name: alarms[alarm_name] for alarm_name in alarms for alarm_contains_word in
+ self.alarm_contains_words_list if alarm_contains_word in alarm_name}
+ if self.alarm_excludes_words != '':
+ alarms = {alarm_name: alarms[alarm_name] for alarm_name in alarms for alarm_excludes_word in
+ self.alarm_excludes_words_list if alarm_excludes_word not in alarm_name}
+
+ data = {a: self.sm[alarms[a]['status']] for a in alarms if alarms[a]['status'] in self.sm}
+ self.update_charts('alarms', data)
+ data['alarms_num'] = len(data)
+
+ if self.collect_alarm_values:
+ data_values = {'{}_value'.format(a): alarms[a]['value'] * 100 for a in alarms if 'value' in alarms[a] and alarms[a]['value'] is not None}
+ self.update_charts('values', data_values, divisor=100)
+ data.update(data_values)
+
+ return data
+
+ def update_charts(self, chart, data, algorithm='absolute', multiplier=1, divisor=1):
+ if not self.charts:
+ return
+
+ for dim in data:
+ if dim not in self.collected_dims[chart]:
+ self.collected_dims[chart].add(dim)
+ self.charts[chart].add_dimension([dim, dim, algorithm, multiplier, divisor])
+
+ for dim in list(self.collected_dims[chart]):
+ if dim not in data:
+ self.collected_dims[chart].remove(dim)
+ self.charts[chart].del_dimension(dim, hide=False)
diff --git a/collectors/python.d.plugin/alarms/alarms.conf b/collectors/python.d.plugin/alarms/alarms.conf
new file mode 100644
index 0000000..06d76c3
--- /dev/null
+++ b/collectors/python.d.plugin/alarms/alarms.conf
@@ -0,0 +1,60 @@
+# netdata python.d.plugin configuration for example
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 10
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+
+# what url to pull data from
+local:
+ url: 'http://127.0.0.1:19999/api/v1/alarms?all'
+ # define how to map alarm status to numbers for the chart
+ status_map:
+ CLEAR: 0
+ WARNING: 1
+ CRITICAL: 2
+ # set to true to include a chart with calculated alarm values over time
+ collect_alarm_values: false
+ # define the type of chart for plotting status over time e.g. 'line' or 'stacked'
+ alarm_status_chart_type: 'line'
+ # a "," separated list of words you want to filter alarm names for. For example 'cpu,load' would filter for only
+ # alarms with "cpu" or "load" in alarm name. Default includes all.
+ alarm_contains_words: ''
+ # a "," separated list of words you want to exclude based on alarm name. For example 'cpu,load' would exclude
+ # all alarms with "cpu" or "load" in alarm name. Default excludes None.
+ alarm_excludes_words: ''
diff --git a/collectors/python.d.plugin/am2320/Makefile.inc b/collectors/python.d.plugin/am2320/Makefile.inc
new file mode 100644
index 0000000..48e5a88
--- /dev/null
+++ b/collectors/python.d.plugin/am2320/Makefile.inc
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# install these files
+dist_python_DATA += am2320/am2320.chart.py
+dist_pythonconfig_DATA += am2320/am2320.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += am2320/README.md am2320/Makefile.inc
diff --git a/collectors/python.d.plugin/am2320/README.md b/collectors/python.d.plugin/am2320/README.md
new file mode 100644
index 0000000..3503d7c
--- /dev/null
+++ b/collectors/python.d.plugin/am2320/README.md
@@ -0,0 +1,53 @@
+<!--
+title: "AM2320 sensor monitoring with netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/am2320/README.md
+sidebar_label: "AM2320"
+-->
+
+# AM2320 sensor monitoring with netdata
+
+Displays a graph of the temperature and humidity from a AM2320 sensor.
+
+## Requirements
+ - Adafruit Circuit Python AM2320 library
+ - Adafruit AM2320 I2C sensor
+ - Python 3 (Adafruit libraries are not Python 2.x compatible)
+
+
+It produces the following charts:
+1. **Temperature**
+2. **Humidity**
+
+## Configuration
+
+Edit the `python.d/am2320.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/am2320.conf
+```
+
+Raspberry Pi Instructions:
+
+Hardware install:
+Connect the am2320 to the Raspberry Pi I2C pins
+
+Raspberry Pi 3B/4 Pins:
+
+- Board 3.3V (pin 1) to sensor VIN (pin 1)
+- Board SDA (pin 3) to sensor SDA (pin 2)
+- Board GND (pin 6) to sensor GND (pin 3)
+- Board SCL (pin 5) to sensor SCL (pin 4)
+
+You may also need to add two I2C pullup resistors if your board does not already have them. The Raspberry Pi does have internal pullup resistors but it doesn't hurt to add them anyway. You can use 2.2K - 10K but we will just use 10K. The resistors go from VDD to SCL and SDA each.
+
+Software install:
+- `sudo pip3 install adafruit-circuitpython-am2320`
+- edit `/etc/netdata/netdata.conf`
+- find `[plugin:python.d]`
+- add `command options = -ppython3`
+- save the file.
+- restart the netdata service.
+- check the dashboard.
+
diff --git a/collectors/python.d.plugin/am2320/am2320.chart.py b/collectors/python.d.plugin/am2320/am2320.chart.py
new file mode 100644
index 0000000..8e66544
--- /dev/null
+++ b/collectors/python.d.plugin/am2320/am2320.chart.py
@@ -0,0 +1,68 @@
+# _*_ coding: utf-8 _*_
+# Description: AM2320 netdata module
+# Author: tommybuck
+# SPDX-License-Identifier: GPL-3.0-or-Later
+
+try:
+ import board
+ import busio
+ import adafruit_am2320
+
+ HAS_AM2320 = True
+except ImportError:
+ HAS_AM2320 = False
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+ORDER = [
+ 'temperature',
+ 'humidity',
+]
+
+CHARTS = {
+ 'temperature': {
+ 'options': [None, 'Temperature', 'celsius', 'temperature', 'am2320.temperature', 'line'],
+ 'lines': [
+ ['temperature']
+ ]
+ },
+ 'humidity': {
+ 'options': [None, 'Relative Humidity', 'percentage', 'humidity', 'am2320.humidity', 'line'],
+ 'lines': [
+ ['humidity']
+ ]
+ }
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.am = None
+
+ def check(self):
+ if not HAS_AM2320:
+ self.error("Could not find the adafruit-circuitpython-am2320 package.")
+ return False
+
+ try:
+ i2c = busio.I2C(board.SCL, board.SDA)
+ self.am = adafruit_am2320.AM2320(i2c)
+ except ValueError as error:
+ self.error("error on creating I2C shared bus : {0}".format(error))
+ return False
+
+ return True
+
+ def get_data(self):
+ try:
+ return {
+ 'temperature': self.am.temperature,
+ 'humidity': self.am.relative_humidity,
+ }
+
+ except (OSError, RuntimeError) as error:
+ self.error(error)
+ return None
diff --git a/collectors/python.d.plugin/am2320/am2320.conf b/collectors/python.d.plugin/am2320/am2320.conf
new file mode 100644
index 0000000..c6b9885
--- /dev/null
+++ b/collectors/python.d.plugin/am2320/am2320.conf
@@ -0,0 +1,68 @@
+# netdata python.d.plugin configuration for am2320 temperature/humidity sensor
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, example also supports the following:
+#
+# - none
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
diff --git a/collectors/python.d.plugin/anomalies/Makefile.inc b/collectors/python.d.plugin/anomalies/Makefile.inc
new file mode 100644
index 0000000..94937b3
--- /dev/null
+++ b/collectors/python.d.plugin/anomalies/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += anomalies/anomalies.chart.py
+dist_pythonconfig_DATA += anomalies/anomalies.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += anomalies/README.md anomalies/Makefile.inc
+
diff --git a/collectors/python.d.plugin/anomalies/README.md b/collectors/python.d.plugin/anomalies/README.md
new file mode 100644
index 0000000..aaf39ab
--- /dev/null
+++ b/collectors/python.d.plugin/anomalies/README.md
@@ -0,0 +1,245 @@
+<!--
+title: "Anomaly detection with Netdata"
+description: "Use ML-driven anomaly detection to narrow your focus to only affected metrics and services/processes on your node to shorten root cause analysis."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/anomalies/README.md
+sidebar_url: Anomalies
+-->
+
+# Anomaly detection with Netdata
+
+**Note**: Check out the [Netdata Anomaly Advisor](https://learn.netdata.cloud/docs/cloud/insights/anomaly-advisor) for a more native anomaly detection experience within Netdata.
+
+This collector uses the Python [PyOD](https://pyod.readthedocs.io/en/latest/index.html) library to perform unsupervised [anomaly detection](https://en.wikipedia.org/wiki/Anomaly_detection) on your Netdata charts and/or dimensions.
+
+Instead of this collector just _collecting_ data, it also does some computation on the data it collects to return an anomaly probability and anomaly flag for each chart or custom model you define. This computation consists of a **train** function that runs every `train_n_secs` to train the ML models to learn what 'normal' typically looks like on your node. At each iteration there is also a **predict** function that uses the latest trained models and most recent metrics to produce an anomaly probability and anomaly flag for each chart or custom model you define.
+
+> As this is a somewhat unique collector and involves often subjective concepts like anomalies and anomaly probabilities, we would love to hear any feedback on it from the community. Please let us know on the [community forum](https://community.netdata.cloud/t/anomalies-collector-feedback-megathread/767) or drop us a note at [analytics-ml-team@netdata.cloud](mailto:analytics-ml-team@netdata.cloud) for any and all feedback, both positive and negative. This sort of feedback is priceless to help us make complex features more useful.
+
+## Charts
+
+Two charts are produced:
+
+- **Anomaly Probability** (`anomalies.probability`): This chart shows the probability that the latest observed data is anomalous based on the trained model for that chart (using the [`predict_proba()`](https://pyod.readthedocs.io/en/latest/api_cc.html#pyod.models.base.BaseDetector.predict_proba) method of the trained PyOD model).
+- **Anomaly** (`anomalies.anomaly`): This chart shows `1` or `0` predictions of if the latest observed data is considered anomalous or not based on the trained model (using the [`predict()`](https://pyod.readthedocs.io/en/latest/api_cc.html#pyod.models.base.BaseDetector.predict) method of the trained PyOD model).
+
+Below is an example of the charts produced by this collector and how they might look when things are 'normal' on the node. The anomaly probabilities tend to bounce randomly around a typically low probability range, one or two might randomly jump or drift outside of this range every now and then and show up as anomalies on the anomaly chart.
+
+![netdata-anomalies-collector-normal](https://user-images.githubusercontent.com/2178292/100663699-99755000-334e-11eb-922f-0c41a0176484.jpg)
+
+If we then go onto the system and run a command like `stress-ng --all 2` to create some [stress](https://wiki.ubuntu.com/Kernel/Reference/stress-ng), we see some charts begin to have anomaly probabilities that jump outside the typical range. When the anomaly probabilities change enough, we will start seeing anomalies being flagged on the `anomalies.anomaly` chart. The idea is that these charts are the most anomalous right now so could be a good place to start your troubleshooting.
+
+![netdata-anomalies-collector-abnormal](https://user-images.githubusercontent.com/2178292/100663710-9bd7aa00-334e-11eb-9d14-76fda73bc309.jpg)
+
+Then, as the issue passes, the anomaly probabilities should settle back down into their 'normal' range again.
+
+![netdata-anomalies-collector-normal-again](https://user-images.githubusercontent.com/2178292/100666681-481a9000-3351-11eb-9979-64728ee2dfb6.jpg)
+
+## Requirements
+
+- This collector will only work with Python 3 and requires the packages below be installed.
+- Typically you will not need to do this, but, if needed, to ensure Python 3 is used you can add the below line to the `[plugin:python.d]` section of `netdata.conf`
+
+```conf
+[plugin:python.d]
+ # update every = 1
+ command options = -ppython3
+```
+
+Install the required python libraries.
+
+```bash
+# become netdata user
+sudo su -s /bin/bash netdata
+# install required packages for the netdata user
+pip3 install --user netdata-pandas==0.0.38 numba==0.50.1 scikit-learn==0.23.2 pyod==0.8.3
+```
+
+## Configuration
+
+Install the Python requirements above, enable the collector and restart Netdata.
+
+```bash
+cd /etc/netdata/
+sudo ./edit-config python.d.conf
+# Set `anomalies: no` to `anomalies: yes`
+sudo systemctl restart netdata
+```
+
+The configuration for the anomalies collector defines how it will behave on your system and might take some experimentation with over time to set it optimally for your node. Out of the box, the config comes with some [sane defaults](https://www.netdata.cloud/blog/redefining-monitoring-netdata/) to get you started that try to balance the flexibility and power of the ML models with the goal of being as cheap as possible in term of cost on the node resources.
+
+_**Note**: If you are unsure about any of the below configuration options then it's best to just ignore all this and leave the `anomalies.conf` file alone to begin with. Then you can return to it later if you would like to tune things a bit more once the collector is running for a while and you have a feeling for its performance on your node._
+
+Edit the `python.d/anomalies.conf` configuration file using `edit-config` from the your agent's [config
+directory](/docs/configure/nodes.md), which is usually at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/anomalies.conf
+```
+
+The default configuration should look something like this. Here you can see each parameter (with sane defaults) and some information about each one and what it does.
+
+```conf
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+
+# Pull data from local Netdata node.
+anomalies:
+ name: 'Anomalies'
+
+ # Host to pull data from.
+ host: '127.0.0.1:19999'
+
+ # Username and Password for Netdata if using basic auth.
+ # username: '???'
+ # password: '???'
+
+ # Use http or https to pull data
+ protocol: 'http'
+
+ # SSL verify parameter for requests.get() calls
+ tls_verify: true
+
+ # What charts to pull data for - A regex like 'system\..*|' or 'system\..*|apps.cpu|apps.mem' etc.
+ charts_regex: 'system\..*'
+
+ # Charts to exclude, useful if you would like to exclude some specific charts.
+ # Note: should be a ',' separated string like 'chart.name,chart.name'.
+ charts_to_exclude: 'system.uptime,system.entropy'
+
+ # What model to use - can be one of 'pca', 'hbos', 'iforest', 'cblof', 'loda', 'copod' or 'feature_bagging'.
+ # More details here: https://pyod.readthedocs.io/en/latest/pyod.models.html.
+ model: 'pca'
+
+ # Max number of observations to train on, to help cap compute cost of training model if you set a very large train_n_secs.
+ train_max_n: 100000
+
+ # How often to re-train the model (assuming update_every=1 then train_every_n=1800 represents (re)training every 30 minutes).
+ # Note: If you want to turn off re-training set train_every_n=0 and after initial training the models will not be retrained.
+ train_every_n: 1800
+
+ # The length of the window of data to train on (14400 = last 4 hours).
+ train_n_secs: 14400
+
+ # How many prediction steps after a train event to just use previous prediction value for.
+ # Used to reduce possibility of the training step itself appearing as an anomaly on the charts.
+ train_no_prediction_n: 10
+
+ # If you would like to train the model for the first time on a specific window then you can define it using the below two variables.
+ # Start of training data for initial model.
+ # initial_train_data_after: 1604578857
+
+ # End of training data for initial model.
+ # initial_train_data_before: 1604593257
+
+ # If you would like to ignore recent data in training then you can offset it by offset_n_secs.
+ offset_n_secs: 0
+
+ # How many lagged values of each dimension to include in the 'feature vector' each model is trained on.
+ lags_n: 5
+
+ # How much smoothing to apply to each dimension in the 'feature vector' each model is trained on.
+ smooth_n: 3
+
+ # How many differences to take in preprocessing your data.
+ # More info on differencing here: https://en.wikipedia.org/wiki/Autoregressive_integrated_moving_average#Differencing
+ # diffs_n=0 would mean training models on the raw values of each dimension.
+ # diffs_n=1 means everything is done in terms of differences.
+ diffs_n: 1
+
+ # What is the typical proportion of anomalies in your data on average?
+ # This parameter can control the sensitivity of your models to anomalies.
+ # Some discussion here: https://github.com/yzhao062/pyod/issues/144
+ contamination: 0.001
+
+ # Set to true to include an "average_prob" dimension on anomalies probability chart which is
+ # just the average of all anomaly probabilities at each time step
+ include_average_prob: true
+
+ # Define any custom models you would like to create anomaly probabilities for, some examples below to show how.
+ # For example below example creates two custom models, one to run anomaly detection user and system cpu for our demo servers
+ # and one on the cpu and mem apps metrics for the python.d.plugin.
+ # custom_models:
+ # - name: 'demos_cpu'
+ # dimensions: 'london.my-netdata.io::system.cpu|user,london.my-netdata.io::system.cpu|system,newyork.my-netdata.io::system.cpu|user,newyork.my-netdata.io::system.cpu|system'
+ # - name: 'apps_python_d_plugin'
+ # dimensions: 'apps.cpu|python.d.plugin,apps.mem|python.d.plugin'
+
+ # Set to true to normalize, using min-max standardization, features used for the custom models.
+ # Useful if your custom models contain dimensions on very different scales an model you use does
+ # not internally do its own normalization. Usually best to leave as false.
+ # custom_models_normalize: false
+```
+
+## Custom models
+
+In the `anomalies.conf` file you can also define some "custom models" which you can use to group one or more metrics into a single model much like is done by default for the charts you specify. This is useful if you have a handful of metrics that exist in different charts but perhaps are related to the same underlying thing you would like to perform anomaly detection on, for example a specific app or user.
+
+To define a custom model you would include configuration like below in `anomalies.conf`. By default there should already be some commented out examples in there.
+
+`name` is a name you give your custom model, this is what will appear alongside any other specified charts in the `anomalies.probability` and `anomalies.anomaly` charts. `dimensions` is a string of metrics you want to include in your custom model. By default the [netdata-pandas](https://github.com/netdata/netdata-pandas) library used to pull the data from Netdata uses a "chart.a|dim.1" type of naming convention in the pandas columns it returns, hence the `dimensions` string should look like "chart.name|dimension.name,chart.name|dimension.name". The examples below hopefully make this clear.
+
+```yaml
+custom_models:
+ # a model for anomaly detection on the netdata user in terms of cpu, mem, threads, processes and sockets.
+ - name: 'user_netdata'
+ dimensions: 'users.cpu|netdata,users.mem|netdata,users.threads|netdata,users.processes|netdata,users.sockets|netdata'
+ # a model for anomaly detection on the netdata python.d.plugin app in terms of cpu, mem, threads, processes and sockets.
+ - name: 'apps_python_d_plugin'
+ dimensions: 'apps.cpu|python.d.plugin,apps.mem|python.d.plugin,apps.threads|python.d.plugin,apps.processes|python.d.plugin,apps.sockets|python.d.plugin'
+
+custom_models_normalize: false
+```
+
+## Troubleshooting
+
+To see any relevant log messages you can use a command like below.
+
+```bash
+`grep 'anomalies' /var/log/netdata/error.log`
+```
+
+If you would like to log in as `netdata` user and run the collector in debug mode to see more detail.
+
+```bash
+# become netdata user
+sudo su -s /bin/bash netdata
+# run collector in debug using `nolock` option if netdata is already running the collector itself.
+/usr/libexec/netdata/plugins.d/python.d.plugin anomalies debug trace nolock
+```
+
+## Deepdive tutorial
+
+If you would like to go deeper on what exactly the anomalies collector is doing under the hood then check out this [deepdive tutorial](https://github.com/netdata/community/blob/main/netdata-agent-api/netdata-pandas/anomalies_collector_deepdive.ipynb) in our community repo where you can play around with some data from our demo servers (or your own if its accessible to you) and work through the calculations step by step.
+
+(Note: as its a Jupyter Notebook it might render a little prettier on [nbviewer](https://nbviewer.jupyter.org/github/netdata/community/blob/main/netdata-agent-api/netdata-pandas/anomalies_collector_deepdive.ipynb))
+
+## Notes
+
+- Python 3 is required as the [`netdata-pandas`](https://github.com/netdata/netdata-pandas) package uses Python async libraries ([asks](https://pypi.org/project/asks/) and [trio](https://pypi.org/project/trio/)) to make asynchronous calls to the [Netdata REST API](https://learn.netdata.cloud/docs/agent/web/api) to get the required data for each chart.
+- Python 3 is also required for the underlying ML libraries of [numba](https://pypi.org/project/numba/), [scikit-learn](https://pypi.org/project/scikit-learn/), and [PyOD](https://pypi.org/project/pyod/).
+- It may take a few hours or so (depending on your choice of `train_secs_n`) for the collector to 'settle' into it's typical behaviour in terms of the trained models and probabilities you will see in the normal running of your node.
+- As this collector does most of the work in Python itself, with [PyOD](https://pyod.readthedocs.io/en/latest/) leveraging [numba](https://numba.pydata.org/) under the hood, you may want to try it out first on a test or development system to get a sense of its performance characteristics on a node similar to where you would like to use it.
+- `lags_n`, `smooth_n`, and `diffs_n` together define the preprocessing done to the raw data before models are trained and before each prediction. This essentially creates a [feature vector](https://en.wikipedia.org/wiki/Feature_(machine_learning)#:~:text=In%20pattern%20recognition%20and%20machine,features%20that%20represent%20some%20object.&text=Feature%20vectors%20are%20often%20combined,score%20for%20making%20a%20prediction.) for each chart model (or each custom model). The default settings for these parameters aim to create a rolling matrix of recent smoothed [differenced](https://en.wikipedia.org/wiki/Autoregressive_integrated_moving_average#Differencing) values for each chart. The aim of the model then is to score how unusual this 'matrix' of features is for each chart based on what it has learned as 'normal' from the training data. So as opposed to just looking at the single most recent value of a dimension and considering how strange it is, this approach looks at a recent smoothed window of all dimensions for a chart (or dimensions in a custom model) and asks how unusual the data as a whole looks. This should be more flexible in capturing a wider range of [anomaly types](https://andrewm4894.com/2020/10/19/different-types-of-time-series-anomalies/) and be somewhat more robust to temporary 'spikes' in the data that tend to always be happening somewhere in your metrics but often are not the most important type of anomaly (this is all covered in a lot more detail in the [deepdive tutorial](https://nbviewer.jupyter.org/github/netdata/community/blob/main/netdata-agent-api/netdata-pandas/anomalies_collector_deepdive.ipynb)).
+- You can see how long model training is taking by looking in the logs for the collector `grep 'anomalies' /var/log/netdata/error.log | grep 'training'` and you should see lines like `2020-12-01 22:02:14: python.d INFO: anomalies[local] : training complete in 2.81 seconds (runs_counter=2700, model=pca, train_n_secs=14400, models=26, n_fit_success=26, n_fit_fails=0, after=1606845731, before=1606860131).`.
+ - This also gives counts of the number of models, if any, that failed to fit and so had to default back to the DefaultModel (which is currently [HBOS](https://pyod.readthedocs.io/en/latest/_modules/pyod/models/hbos.html)).
+ - `after` and `before` here refer to the start and end of the training data used to train the models.
+- On a development n1-standard-2 (2 vCPUs, 7.5 GB memory) vm running Ubuntu 18.04 LTS and not doing any work some of the typical performance characteristics we saw from running this collector (with defaults) were:
+ - A runtime (`netdata.runtime_anomalies`) of ~80ms when doing scoring and ~3 seconds when training or retraining the models.
+ - Typically ~3%-3.5% additional cpu usage from scoring, jumping to ~60% for a couple of seconds during model training.
+ - About ~150mb of ram (`apps.mem`) being continually used by the `python.d.plugin`.
+- If you activate this collector on a fresh node, it might take a little while to build up enough data to calculate a realistic and useful model.
+- Some models like `iforest` can be comparatively expensive (on same n1-standard-2 system above ~2s runtime during predict, ~40s training time, ~50% cpu on both train and predict) so if you would like to use it you might be advised to set a relatively high `update_every` maybe 10, 15 or 30 in `anomalies.conf`.
+- Setting a higher `train_every_n` and `update_every` is an easy way to devote less resources on the node to anomaly detection. Specifying less charts and a lower `train_n_secs` will also help reduce resources at the expense of covering less charts and maybe a more noisy model if you set `train_n_secs` to be too small for how your node tends to behave.
+- If you would like to enable this on a Rasberry Pi, then check out [this guide](https://learn.netdata.cloud/guides/monitor/raspberry-pi-anomaly-detection) which will guide you through first installing LLVM.
+
+## Useful links and further reading
+
+- [PyOD documentation](https://pyod.readthedocs.io/en/latest/), [PyOD Github](https://github.com/yzhao062/pyod).
+- [Anomaly Detection](https://en.wikipedia.org/wiki/Anomaly_detection) wikipedia page.
+- [Anomaly Detection YouTube playlist](https://www.youtube.com/playlist?list=PL6Zhl9mK2r0KxA6rB87oi4kWzoqGd5vp0) maintained by [andrewm4894](https://github.com/andrewm4894/) from Netdata.
+- [awesome-TS-anomaly-detection](https://github.com/rob-med/awesome-TS-anomaly-detection) Github list of useful tools, libraries and resources.
+- [Mendeley public group](https://www.mendeley.com/community/interesting-anomaly-detection-papers/) with some interesting anomaly detection papers we have been reading.
+- Good [blog post](https://www.anodot.com/blog/what-is-anomaly-detection/) from Anodot on time series anomaly detection. Anodot also have some great whitepapers in this space too that some may find useful.
+- Novelty and outlier detection in the [scikit-learn documentation](https://scikit-learn.org/stable/modules/outlier_detection.html).
+
diff --git a/collectors/python.d.plugin/anomalies/anomalies.chart.py b/collectors/python.d.plugin/anomalies/anomalies.chart.py
new file mode 100644
index 0000000..8ca3df6
--- /dev/null
+++ b/collectors/python.d.plugin/anomalies/anomalies.chart.py
@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+# Description: anomalies netdata python.d module
+# Author: andrewm4894
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import sys
+import time
+from datetime import datetime
+import re
+import warnings
+
+import requests
+import numpy as np
+import pandas as pd
+from netdata_pandas.data import get_data, get_allmetrics_async
+from pyod.models.hbos import HBOS
+from pyod.models.pca import PCA
+from pyod.models.loda import LODA
+from pyod.models.iforest import IForest
+from pyod.models.cblof import CBLOF
+from pyod.models.feature_bagging import FeatureBagging
+from pyod.models.copod import COPOD
+from sklearn.preprocessing import MinMaxScaler
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+# ignore some sklearn/numpy warnings that are ok
+warnings.filterwarnings('ignore', r'All-NaN slice encountered')
+warnings.filterwarnings('ignore', r'invalid value encountered in true_divide')
+warnings.filterwarnings('ignore', r'divide by zero encountered in true_divide')
+warnings.filterwarnings('ignore', r'invalid value encountered in subtract')
+
+disabled_by_default = True
+
+ORDER = ['probability', 'anomaly']
+
+CHARTS = {
+ 'probability': {
+ 'options': ['probability', 'Anomaly Probability', 'probability', 'anomalies', 'anomalies.probability', 'line'],
+ 'lines': []
+ },
+ 'anomaly': {
+ 'options': ['anomaly', 'Anomaly', 'count', 'anomalies', 'anomalies.anomaly', 'stacked'],
+ 'lines': []
+ },
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.basic_init()
+ self.charts_init()
+ self.custom_models_init()
+ self.data_init()
+ self.model_params_init()
+ self.models_init()
+ self.collected_dims = {'probability': set(), 'anomaly': set()}
+
+ def check(self):
+ python_version = float('{}.{}'.format(sys.version_info[0], sys.version_info[1]))
+ if python_version < 3.6:
+ self.error("anomalies collector only works with Python>=3.6")
+ if len(self.host_charts_dict[self.host]) > 0:
+ _ = get_allmetrics_async(host_charts_dict=self.host_charts_dict, protocol=self.protocol, user=self.username, pwd=self.password)
+ return True
+
+ def basic_init(self):
+ """Perform some basic initialization.
+ """
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.protocol = self.configuration.get('protocol', 'http')
+ self.host = self.configuration.get('host', '127.0.0.1:19999')
+ self.username = self.configuration.get('username', None)
+ self.password = self.configuration.get('password', None)
+ self.tls_verify = self.configuration.get('tls_verify', True)
+ self.fitted_at = {}
+ self.df_allmetrics = pd.DataFrame()
+ self.last_train_at = 0
+ self.include_average_prob = bool(self.configuration.get('include_average_prob', True))
+ self.reinitialize_at_every_step = bool(self.configuration.get('reinitialize_at_every_step', False))
+
+ def charts_init(self):
+ """Do some initialisation of charts in scope related variables.
+ """
+ self.charts_regex = re.compile(self.configuration.get('charts_regex','None'))
+ self.charts_available = [c for c in list(requests.get(f'{self.protocol}://{self.host}/api/v1/charts', verify=self.tls_verify).json().get('charts', {}).keys())]
+ self.charts_in_scope = list(filter(self.charts_regex.match, self.charts_available))
+ self.charts_to_exclude = self.configuration.get('charts_to_exclude', '').split(',')
+ if len(self.charts_to_exclude) > 0:
+ self.charts_in_scope = [c for c in self.charts_in_scope if c not in self.charts_to_exclude]
+
+ def custom_models_init(self):
+ """Perform initialization steps related to custom models.
+ """
+ self.custom_models = self.configuration.get('custom_models', None)
+ self.custom_models_normalize = bool(self.configuration.get('custom_models_normalize', False))
+ if self.custom_models:
+ self.custom_models_names = [model['name'] for model in self.custom_models]
+ self.custom_models_dims = [i for s in [model['dimensions'].split(',') for model in self.custom_models] for i in s]
+ self.custom_models_dims = [dim if '::' in dim else f'{self.host}::{dim}' for dim in self.custom_models_dims]
+ self.custom_models_charts = list(set([dim.split('|')[0].split('::')[1] for dim in self.custom_models_dims]))
+ self.custom_models_hosts = list(set([dim.split('::')[0] for dim in self.custom_models_dims]))
+ self.custom_models_host_charts_dict = {}
+ for host in self.custom_models_hosts:
+ self.custom_models_host_charts_dict[host] = list(set([dim.split('::')[1].split('|')[0] for dim in self.custom_models_dims if dim.startswith(host)]))
+ self.custom_models_dims_renamed = [f"{model['name']}|{dim}" for model in self.custom_models for dim in model['dimensions'].split(',')]
+ self.models_in_scope = list(set([f'{self.host}::{c}' for c in self.charts_in_scope] + self.custom_models_names))
+ self.charts_in_scope = list(set(self.charts_in_scope + self.custom_models_charts))
+ self.host_charts_dict = {self.host: self.charts_in_scope}
+ for host in self.custom_models_host_charts_dict:
+ if host not in self.host_charts_dict:
+ self.host_charts_dict[host] = self.custom_models_host_charts_dict[host]
+ else:
+ for chart in self.custom_models_host_charts_dict[host]:
+ if chart not in self.host_charts_dict[host]:
+ self.host_charts_dict[host].extend(chart)
+ else:
+ self.models_in_scope = [f'{self.host}::{c}' for c in self.charts_in_scope]
+ self.host_charts_dict = {self.host: self.charts_in_scope}
+ self.model_display_names = {model: model.split('::')[1] if '::' in model else model for model in self.models_in_scope}
+ #self.info(f'self.host_charts_dict (len={len(self.host_charts_dict[self.host])}): {self.host_charts_dict}')
+
+ def data_init(self):
+ """Initialize some empty data objects.
+ """
+ self.data_probability_latest = {f'{m}_prob': 0 for m in self.charts_in_scope}
+ self.data_anomaly_latest = {f'{m}_anomaly': 0 for m in self.charts_in_scope}
+ self.data_latest = {**self.data_probability_latest, **self.data_anomaly_latest}
+
+ def model_params_init(self):
+ """Model parameters initialisation.
+ """
+ self.train_max_n = self.configuration.get('train_max_n', 100000)
+ self.train_n_secs = self.configuration.get('train_n_secs', 14400)
+ self.offset_n_secs = self.configuration.get('offset_n_secs', 0)
+ self.train_every_n = self.configuration.get('train_every_n', 1800)
+ self.train_no_prediction_n = self.configuration.get('train_no_prediction_n', 10)
+ self.initial_train_data_after = self.configuration.get('initial_train_data_after', 0)
+ self.initial_train_data_before = self.configuration.get('initial_train_data_before', 0)
+ self.contamination = self.configuration.get('contamination', 0.001)
+ self.lags_n = {model: self.configuration.get('lags_n', 5) for model in self.models_in_scope}
+ self.smooth_n = {model: self.configuration.get('smooth_n', 5) for model in self.models_in_scope}
+ self.diffs_n = {model: self.configuration.get('diffs_n', 5) for model in self.models_in_scope}
+
+ def models_init(self):
+ """Models initialisation.
+ """
+ self.model = self.configuration.get('model', 'pca')
+ if self.model == 'pca':
+ self.models = {model: PCA(contamination=self.contamination) for model in self.models_in_scope}
+ elif self.model == 'loda':
+ self.models = {model: LODA(contamination=self.contamination) for model in self.models_in_scope}
+ elif self.model == 'iforest':
+ self.models = {model: IForest(n_estimators=50, bootstrap=True, behaviour='new', contamination=self.contamination) for model in self.models_in_scope}
+ elif self.model == 'cblof':
+ self.models = {model: CBLOF(n_clusters=3, contamination=self.contamination) for model in self.models_in_scope}
+ elif self.model == 'feature_bagging':
+ self.models = {model: FeatureBagging(base_estimator=PCA(contamination=self.contamination), contamination=self.contamination) for model in self.models_in_scope}
+ elif self.model == 'copod':
+ self.models = {model: COPOD(contamination=self.contamination) for model in self.models_in_scope}
+ elif self.model == 'hbos':
+ self.models = {model: HBOS(contamination=self.contamination) for model in self.models_in_scope}
+ else:
+ self.models = {model: HBOS(contamination=self.contamination) for model in self.models_in_scope}
+ self.custom_model_scalers = {model: MinMaxScaler() for model in self.models_in_scope}
+
+ def model_init(self, model):
+ """Model initialisation of a single model.
+ """
+ if self.model == 'pca':
+ self.models[model] = PCA(contamination=self.contamination)
+ elif self.model == 'loda':
+ self.models[model] = LODA(contamination=self.contamination)
+ elif self.model == 'iforest':
+ self.models[model] = IForest(n_estimators=50, bootstrap=True, behaviour='new', contamination=self.contamination)
+ elif self.model == 'cblof':
+ self.models[model] = CBLOF(n_clusters=3, contamination=self.contamination)
+ elif self.model == 'feature_bagging':
+ self.models[model] = FeatureBagging(base_estimator=PCA(contamination=self.contamination), contamination=self.contamination)
+ elif self.model == 'copod':
+ self.models[model] = COPOD(contamination=self.contamination)
+ elif self.model == 'hbos':
+ self.models[model] = HBOS(contamination=self.contamination)
+ else:
+ self.models[model] = HBOS(contamination=self.contamination)
+ self.custom_model_scalers[model] = MinMaxScaler()
+
+ def reinitialize(self):
+ """Reinitialize charts, models and data to a beginning state.
+ """
+ self.charts_init()
+ self.custom_models_init()
+ self.data_init()
+ self.model_params_init()
+ self.models_init()
+
+ def save_data_latest(self, data, data_probability, data_anomaly):
+ """Save the most recent data objects to be used if needed in the future.
+ """
+ self.data_latest = data
+ self.data_probability_latest = data_probability
+ self.data_anomaly_latest = data_anomaly
+
+ def validate_charts(self, chart, data, algorithm='absolute', multiplier=1, divisor=1):
+ """If dimension not in chart then add it.
+ """
+ for dim in data:
+ if dim not in self.collected_dims[chart]:
+ self.collected_dims[chart].add(dim)
+ self.charts[chart].add_dimension([dim, dim, algorithm, multiplier, divisor])
+
+ for dim in list(self.collected_dims[chart]):
+ if dim not in data:
+ self.collected_dims[chart].remove(dim)
+ self.charts[chart].del_dimension(dim, hide=False)
+
+ def add_custom_models_dims(self, df):
+ """Given a df, select columns used by custom models, add custom model name as prefix, and append to df.
+
+ :param df <pd.DataFrame>: dataframe to append new renamed columns to.
+ :return: <pd.DataFrame> dataframe with additional columns added relating to the specified custom models.
+ """
+ df_custom = df[self.custom_models_dims].copy()
+ df_custom.columns = self.custom_models_dims_renamed
+ df = df.join(df_custom)
+
+ return df
+
+ def make_features(self, arr, train=False, model=None):
+ """Take in numpy array and preprocess accordingly by taking diffs, smoothing and adding lags.
+
+ :param arr <np.ndarray>: numpy array we want to make features from.
+ :param train <bool>: True if making features for training, in which case need to fit_transform scaler and maybe sample train_max_n.
+ :param model <str>: model to make features for.
+ :return: <np.ndarray> transformed numpy array.
+ """
+
+ def lag(arr, n):
+ res = np.empty_like(arr)
+ res[:n] = np.nan
+ res[n:] = arr[:-n]
+
+ return res
+
+ arr = np.nan_to_num(arr)
+
+ diffs_n = self.diffs_n[model]
+ smooth_n = self.smooth_n[model]
+ lags_n = self.lags_n[model]
+
+ if self.custom_models_normalize and model in self.custom_models_names:
+ if train:
+ arr = self.custom_model_scalers[model].fit_transform(arr)
+ else:
+ arr = self.custom_model_scalers[model].transform(arr)
+
+ if diffs_n > 0:
+ arr = np.diff(arr, diffs_n, axis=0)
+ arr = arr[~np.isnan(arr).any(axis=1)]
+
+ if smooth_n > 1:
+ arr = np.cumsum(arr, axis=0, dtype=float)
+ arr[smooth_n:] = arr[smooth_n:] - arr[:-smooth_n]
+ arr = arr[smooth_n - 1:] / smooth_n
+ arr = arr[~np.isnan(arr).any(axis=1)]
+
+ if lags_n > 0:
+ arr_orig = np.copy(arr)
+ for lag_n in range(1, lags_n + 1):
+ arr = np.concatenate((arr, lag(arr_orig, lag_n)), axis=1)
+ arr = arr[~np.isnan(arr).any(axis=1)]
+
+ if train:
+ if len(arr) > self.train_max_n:
+ arr = arr[np.random.randint(arr.shape[0], size=self.train_max_n), :]
+
+ arr = np.nan_to_num(arr)
+
+ return arr
+
+ def train(self, models_to_train=None, train_data_after=0, train_data_before=0):
+ """Pull required training data and train a model for each specified model.
+
+ :param models_to_train <list>: list of models to train on.
+ :param train_data_after <int>: integer timestamp for start of train data.
+ :param train_data_before <int>: integer timestamp for end of train data.
+ """
+ now = datetime.now().timestamp()
+ if train_data_after > 0 and train_data_before > 0:
+ before = train_data_before
+ after = train_data_after
+ else:
+ before = int(now) - self.offset_n_secs
+ after = before - self.train_n_secs
+
+ # get training data
+ df_train = get_data(
+ host_charts_dict=self.host_charts_dict, host_prefix=True, host_sep='::', after=after, before=before,
+ sort_cols=True, numeric_only=True, protocol=self.protocol, float_size='float32', user=self.username, pwd=self.password,
+ verify=self.tls_verify
+ ).ffill()
+ if self.custom_models:
+ df_train = self.add_custom_models_dims(df_train)
+
+ # train model
+ self.try_fit(df_train, models_to_train=models_to_train)
+ self.info(f'training complete in {round(time.time() - now, 2)} seconds (runs_counter={self.runs_counter}, model={self.model}, train_n_secs={self.train_n_secs}, models={len(self.fitted_at)}, n_fit_success={self.n_fit_success}, n_fit_fails={self.n_fit_fail}, after={after}, before={before}).')
+ self.last_train_at = self.runs_counter
+
+ def try_fit(self, df_train, models_to_train=None):
+ """Try fit each model and try to fallback to a default model if fit fails for any reason.
+
+ :param df_train <pd.DataFrame>: data to train on.
+ :param models_to_train <list>: list of models to train.
+ """
+ if models_to_train is None:
+ models_to_train = list(self.models.keys())
+ self.n_fit_fail, self.n_fit_success = 0, 0
+ for model in models_to_train:
+ if model not in self.models:
+ self.model_init(model)
+ X_train = self.make_features(
+ df_train[df_train.columns[df_train.columns.str.startswith(f'{model}|')]].values,
+ train=True, model=model)
+ try:
+ self.models[model].fit(X_train)
+ self.n_fit_success += 1
+ except Exception as e:
+ self.n_fit_fail += 1
+ self.info(e)
+ self.info(f'training failed for {model} at run_counter {self.runs_counter}, defaulting to hbos model.')
+ self.models[model] = HBOS(contamination=self.contamination)
+ self.models[model].fit(X_train)
+ self.fitted_at[model] = self.runs_counter
+
+ def predict(self):
+ """Get latest data, make it into a feature vector, and get predictions for each available model.
+
+ :return: (<dict>,<dict>) tuple of dictionaries, one for probability scores and the other for anomaly predictions.
+ """
+ # get recent data to predict on
+ df_allmetrics = get_allmetrics_async(
+ host_charts_dict=self.host_charts_dict, host_prefix=True, host_sep='::', wide=True, sort_cols=True,
+ protocol=self.protocol, numeric_only=True, float_size='float32', user=self.username, pwd=self.password
+ )
+ if self.custom_models:
+ df_allmetrics = self.add_custom_models_dims(df_allmetrics)
+ self.df_allmetrics = self.df_allmetrics.append(df_allmetrics).ffill().tail((max(self.lags_n.values()) + max(self.smooth_n.values()) + max(self.diffs_n.values())) * 2)
+
+ # get predictions
+ data_probability, data_anomaly = self.try_predict()
+
+ return data_probability, data_anomaly
+
+ def try_predict(self):
+ """Try make prediction and fall back to last known prediction if fails.
+
+ :return: (<dict>,<dict>) tuple of dictionaries, one for probability scores and the other for anomaly predictions.
+ """
+ data_probability, data_anomaly = {}, {}
+ for model in self.fitted_at.keys():
+ model_display_name = self.model_display_names[model]
+ try:
+ X_model = np.nan_to_num(
+ self.make_features(
+ self.df_allmetrics[self.df_allmetrics.columns[self.df_allmetrics.columns.str.startswith(f'{model}|')]].values,
+ model=model
+ )[-1,:].reshape(1, -1)
+ )
+ data_probability[model_display_name + '_prob'] = np.nan_to_num(self.models[model].predict_proba(X_model)[-1][1]) * 10000
+ data_anomaly[model_display_name + '_anomaly'] = self.models[model].predict(X_model)[-1]
+ except Exception as _:
+ #self.info(e)
+ if model_display_name + '_prob' in self.data_latest:
+ #self.info(f'prediction failed for {model} at run_counter {self.runs_counter}, using last prediction instead.')
+ data_probability[model_display_name + '_prob'] = self.data_latest[model_display_name + '_prob']
+ data_anomaly[model_display_name + '_anomaly'] = self.data_latest[model_display_name + '_anomaly']
+ else:
+ #self.info(f'prediction failed for {model} at run_counter {self.runs_counter}, skipping as no previous prediction.')
+ continue
+
+ return data_probability, data_anomaly
+
+ def get_data(self):
+
+ # initialize to what's available right now
+ if self.reinitialize_at_every_step or len(self.host_charts_dict[self.host]) == 0:
+ self.charts_init()
+ self.custom_models_init()
+ self.model_params_init()
+
+ # if not all models have been trained then train those we need to
+ if len(self.fitted_at) < len(self.models_in_scope):
+ self.train(
+ models_to_train=[m for m in self.models_in_scope if m not in self.fitted_at],
+ train_data_after=self.initial_train_data_after,
+ train_data_before=self.initial_train_data_before
+ )
+ # retrain all models as per schedule from config
+ elif self.train_every_n > 0 and self.runs_counter % self.train_every_n == 0:
+ self.reinitialize()
+ self.train()
+
+ # roll forward previous predictions around a training step to avoid the possibility of having the training itself trigger an anomaly
+ if (self.runs_counter - self.last_train_at) <= self.train_no_prediction_n:
+ data_probability = self.data_probability_latest
+ data_anomaly = self.data_anomaly_latest
+ else:
+ data_probability, data_anomaly = self.predict()
+ if self.include_average_prob:
+ average_prob = np.mean(list(data_probability.values()))
+ data_probability['average_prob'] = 0 if np.isnan(average_prob) else average_prob
+
+ data = {**data_probability, **data_anomaly}
+
+ self.validate_charts('probability', data_probability, divisor=100)
+ self.validate_charts('anomaly', data_anomaly)
+
+ self.save_data_latest(data, data_probability, data_anomaly)
+
+ #self.info(f'len(data)={len(data)}')
+ #self.info(f'data')
+
+ return data
diff --git a/collectors/python.d.plugin/anomalies/anomalies.conf b/collectors/python.d.plugin/anomalies/anomalies.conf
new file mode 100644
index 0000000..ef86770
--- /dev/null
+++ b/collectors/python.d.plugin/anomalies/anomalies.conf
@@ -0,0 +1,184 @@
+# netdata python.d.plugin configuration for anomalies
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 2
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+
+# Pull data from local Netdata node.
+anomalies:
+ name: 'Anomalies'
+
+ # Host to pull data from.
+ host: '127.0.0.1:19999'
+
+ # Username and Password for Netdata if using basic auth.
+ # username: '???'
+ # password: '???'
+
+ # Use http or https to pull data
+ protocol: 'http'
+
+ # SSL verify parameter for requests.get() calls
+ tls_verify: true
+
+ # What charts to pull data for - A regex like 'system\..*|' or 'system\..*|apps.cpu|apps.mem' etc.
+ charts_regex: 'system\..*'
+
+ # Charts to exclude, useful if you would like to exclude some specific charts.
+ # Note: should be a ',' separated string like 'chart.name,chart.name'.
+ charts_to_exclude: 'system.uptime,system.entropy'
+
+ # What model to use - can be one of 'pca', 'hbos', 'iforest', 'cblof', 'loda', 'copod' or 'feature_bagging'.
+ # More details here: https://pyod.readthedocs.io/en/latest/pyod.models.html.
+ model: 'pca'
+
+ # Max number of observations to train on, to help cap compute cost of training model if you set a very large train_n_secs.
+ train_max_n: 100000
+
+ # How often to re-train the model (assuming update_every=1 then train_every_n=1800 represents (re)training every 30 minutes).
+ # Note: If you want to turn off re-training set train_every_n=0 and after initial training the models will not be retrained.
+ train_every_n: 1800
+
+ # The length of the window of data to train on (14400 = last 4 hours).
+ train_n_secs: 14400
+
+ # How many prediction steps after a train event to just use previous prediction value for.
+ # Used to reduce possibility of the training step itself appearing as an anomaly on the charts.
+ train_no_prediction_n: 10
+
+ # If you would like to train the model for the first time on a specific window then you can define it using the below two variables.
+ # Start of training data for initial model.
+ # initial_train_data_after: 1604578857
+
+ # End of training data for initial model.
+ # initial_train_data_before: 1604593257
+
+ # If you would like to ignore recent data in training then you can offset it by offset_n_secs.
+ offset_n_secs: 0
+
+ # How many lagged values of each dimension to include in the 'feature vector' each model is trained on.
+ lags_n: 5
+
+ # How much smoothing to apply to each dimension in the 'feature vector' each model is trained on.
+ smooth_n: 3
+
+ # How many differences to take in preprocessing your data.
+ # More info on differencing here: https://en.wikipedia.org/wiki/Autoregressive_integrated_moving_average#Differencing
+ # diffs_n=0 would mean training models on the raw values of each dimension.
+ # diffs_n=1 means everything is done in terms of differences.
+ diffs_n: 1
+
+ # What is the typical proportion of anomalies in your data on average?
+ # This parameter can control the sensitivity of your models to anomalies.
+ # Some discussion here: https://github.com/yzhao062/pyod/issues/144
+ contamination: 0.001
+
+ # Set to true to include an "average_prob" dimension on anomalies probability chart which is
+ # just the average of all anomaly probabilities at each time step
+ include_average_prob: true
+
+ # Define any custom models you would like to create anomaly probabilities for, some examples below to show how.
+ # For example below example creates two custom models, one to run anomaly detection user and system cpu for our demo servers
+ # and one on the cpu and mem apps metrics for the python.d.plugin.
+ # custom_models:
+ # - name: 'demos_cpu'
+ # dimensions: 'london.my-netdata.io::system.cpu|user,london.my-netdata.io::system.cpu|system,newyork.my-netdata.io::system.cpu|user,newyork.my-netdata.io::system.cpu|system'
+ # - name: 'apps_python_d_plugin'
+ # dimensions: 'apps.cpu|python.d.plugin,apps.mem|python.d.plugin'
+
+ # Set to true to normalize, using min-max standardization, features used for the custom models.
+ # Useful if your custom models contain dimensions on very different scales an model you use does
+ # not internally do its own normalization. Usually best to leave as false.
+ # custom_models_normalize: false
+
+# Standalone Custom models example as an additional collector job.
+# custom:
+# name: 'custom'
+# host: '127.0.0.1:19999'
+# protocol: 'http'
+# charts_regex: 'None'
+# charts_to_exclude: 'None'
+# model: 'pca'
+# train_max_n: 100000
+# train_every_n: 1800
+# train_n_secs: 14400
+# offset_n_secs: 0
+# lags_n: 5
+# smooth_n: 3
+# diffs_n: 1
+# contamination: 0.001
+# custom_models:
+# - name: 'user_netdata'
+# dimensions: 'users.cpu|netdata,users.mem|netdata,users.threads|netdata,users.processes|netdata,users.sockets|netdata'
+# - name: 'apps_python_d_plugin'
+# dimensions: 'apps.cpu|python.d.plugin,apps.mem|python.d.plugin,apps.threads|python.d.plugin,apps.processes|python.d.plugin,apps.sockets|python.d.plugin'
+
+# Pull data from some demo nodes for cross node custom models.
+# demos:
+# name: 'demos'
+# host: '127.0.0.1:19999'
+# protocol: 'http'
+# charts_regex: 'None'
+# charts_to_exclude: 'None'
+# model: 'pca'
+# train_max_n: 100000
+# train_every_n: 1800
+# train_n_secs: 14400
+# offset_n_secs: 0
+# lags_n: 5
+# smooth_n: 3
+# diffs_n: 1
+# contamination: 0.001
+# custom_models:
+# - name: 'system.cpu'
+# dimensions: 'london.my-netdata.io::system.cpu|user,london.my-netdata.io::system.cpu|system,newyork.my-netdata.io::system.cpu|user,newyork.my-netdata.io::system.cpu|system'
+# - name: 'system.ip'
+# dimensions: 'london.my-netdata.io::system.ip|received,london.my-netdata.io::system.ip|sent,newyork.my-netdata.io::system.ip|received,newyork.my-netdata.io::system.ip|sent'
+# - name: 'system.net'
+# dimensions: 'london.my-netdata.io::system.net|received,london.my-netdata.io::system.net|sent,newyork.my-netdata.io::system.net|received,newyork.my-netdata.io::system.net|sent'
+# - name: 'system.io'
+# dimensions: 'london.my-netdata.io::system.io|in,london.my-netdata.io::system.io|out,newyork.my-netdata.io::system.io|in,newyork.my-netdata.io::system.io|out'
+
+# Example additional job if you want to also pull data from a child streaming to your
+# local parent or even a remote node so long as the Netdata REST API is accessible.
+# mychildnode1:
+# name: 'mychildnode1'
+# host: '127.0.0.1:19999/host/mychildnode1'
+# protocol: 'http'
+# charts_regex: 'system\..*'
+# charts_to_exclude: 'None'
+# model: 'pca'
+# train_max_n: 100000
+# train_every_n: 1800
+# train_n_secs: 14400
+# offset_n_secs: 0
+# lags_n: 5
+# smooth_n: 3
+# diffs_n: 1
+# contamination: 0.001
diff --git a/collectors/python.d.plugin/beanstalk/Makefile.inc b/collectors/python.d.plugin/beanstalk/Makefile.inc
new file mode 100644
index 0000000..4bbb708
--- /dev/null
+++ b/collectors/python.d.plugin/beanstalk/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += beanstalk/beanstalk.chart.py
+dist_pythonconfig_DATA += beanstalk/beanstalk.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += beanstalk/README.md beanstalk/Makefile.inc
+
diff --git a/collectors/python.d.plugin/beanstalk/README.md b/collectors/python.d.plugin/beanstalk/README.md
new file mode 100644
index 0000000..3b63259
--- /dev/null
+++ b/collectors/python.d.plugin/beanstalk/README.md
@@ -0,0 +1,133 @@
+<!--
+title: "Beanstalk monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/beanstalk/README.md
+sidebar_label: "Beanstalk"
+-->
+
+# Beanstalk monitoring with Netdata
+
+Provides server and tube-level statistics.
+
+## Requirements
+
+- `python-beanstalkc`
+
+**Server statistics:**
+
+1. **Cpu usage** in cpu time
+
+ - user
+ - system
+
+2. **Jobs rate** in jobs/s
+
+ - total
+ - timeouts
+
+3. **Connections rate** in connections/s
+
+ - connections
+
+4. **Commands rate** in commands/s
+
+ - put
+ - peek
+ - peek-ready
+ - peek-delayed
+ - peek-buried
+ - reserve
+ - use
+ - watch
+ - ignore
+ - delete
+ - release
+ - bury
+ - kick
+ - stats
+ - stats-job
+ - stats-tube
+ - list-tubes
+ - list-tube-used
+ - list-tubes-watched
+ - pause-tube
+
+5. **Current tubes** in tubes
+
+ - tubes
+
+6. **Current jobs** in jobs
+
+ - urgent
+ - ready
+ - reserved
+ - delayed
+ - buried
+
+7. **Current connections** in connections
+
+ - written
+ - producers
+ - workers
+ - waiting
+
+8. **Binlog** in records/s
+
+ - written
+ - migrated
+
+9. **Uptime** in seconds
+
+ - uptime
+
+**Per tube statistics:**
+
+1. **Jobs rate** in jobs/s
+
+ - jobs
+
+2. **Jobs** in jobs
+
+ - using
+ - ready
+ - reserved
+ - delayed
+ - buried
+
+3. **Connections** in connections
+
+ - using
+ - waiting
+ - watching
+
+4. **Commands** in commands/s
+
+ - deletes
+ - pauses
+
+5. **Pause** in seconds
+
+ - since
+ - left
+
+## Configuration
+
+Edit the `python.d/beanstalk.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/beanstalk.conf
+```
+
+Sample:
+
+```yaml
+host : '127.0.0.1'
+port : 11300
+```
+
+If no configuration is given, module will attempt to connect to beanstalkd on `127.0.0.1:11300` address
+
+---
+
+
diff --git a/collectors/python.d.plugin/beanstalk/beanstalk.chart.py b/collectors/python.d.plugin/beanstalk/beanstalk.chart.py
new file mode 100644
index 0000000..396543e
--- /dev/null
+++ b/collectors/python.d.plugin/beanstalk/beanstalk.chart.py
@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+# Description: beanstalk netdata python.d module
+# Author: ilyam8
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+try:
+ import beanstalkc
+
+ BEANSTALKC = True
+except ImportError:
+ BEANSTALKC = False
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from bases.loaders import load_yaml
+
+ORDER = [
+ 'cpu_usage',
+ 'jobs_rate',
+ 'connections_rate',
+ 'commands_rate',
+ 'current_tubes',
+ 'current_jobs',
+ 'current_connections',
+ 'binlog',
+ 'uptime',
+]
+
+CHARTS = {
+ 'cpu_usage': {
+ 'options': [None, 'Cpu Usage', 'cpu time', 'server statistics', 'beanstalk.cpu_usage', 'area'],
+ 'lines': [
+ ['rusage-utime', 'user', 'incremental'],
+ ['rusage-stime', 'system', 'incremental']
+ ]
+ },
+ 'jobs_rate': {
+ 'options': [None, 'Jobs Rate', 'jobs/s', 'server statistics', 'beanstalk.jobs_rate', 'line'],
+ 'lines': [
+ ['total-jobs', 'total', 'incremental'],
+ ['job-timeouts', 'timeouts', 'incremental']
+ ]
+ },
+ 'connections_rate': {
+ 'options': [None, 'Connections Rate', 'connections/s', 'server statistics', 'beanstalk.connections_rate',
+ 'area'],
+ 'lines': [
+ ['total-connections', 'connections', 'incremental']
+ ]
+ },
+ 'commands_rate': {
+ 'options': [None, 'Commands Rate', 'commands/s', 'server statistics', 'beanstalk.commands_rate', 'stacked'],
+ 'lines': [
+ ['cmd-put', 'put', 'incremental'],
+ ['cmd-peek', 'peek', 'incremental'],
+ ['cmd-peek-ready', 'peek-ready', 'incremental'],
+ ['cmd-peek-delayed', 'peek-delayed', 'incremental'],
+ ['cmd-peek-buried', 'peek-buried', 'incremental'],
+ ['cmd-reserve', 'reserve', 'incremental'],
+ ['cmd-use', 'use', 'incremental'],
+ ['cmd-watch', 'watch', 'incremental'],
+ ['cmd-ignore', 'ignore', 'incremental'],
+ ['cmd-delete', 'delete', 'incremental'],
+ ['cmd-release', 'release', 'incremental'],
+ ['cmd-bury', 'bury', 'incremental'],
+ ['cmd-kick', 'kick', 'incremental'],
+ ['cmd-stats', 'stats', 'incremental'],
+ ['cmd-stats-job', 'stats-job', 'incremental'],
+ ['cmd-stats-tube', 'stats-tube', 'incremental'],
+ ['cmd-list-tubes', 'list-tubes', 'incremental'],
+ ['cmd-list-tube-used', 'list-tube-used', 'incremental'],
+ ['cmd-list-tubes-watched', 'list-tubes-watched', 'incremental'],
+ ['cmd-pause-tube', 'pause-tube', 'incremental']
+ ]
+ },
+ 'current_tubes': {
+ 'options': [None, 'Current Tubes', 'tubes', 'server statistics', 'beanstalk.current_tubes', 'area'],
+ 'lines': [
+ ['current-tubes', 'tubes']
+ ]
+ },
+ 'current_jobs': {
+ 'options': [None, 'Current Jobs', 'jobs', 'server statistics', 'beanstalk.current_jobs', 'stacked'],
+ 'lines': [
+ ['current-jobs-urgent', 'urgent'],
+ ['current-jobs-ready', 'ready'],
+ ['current-jobs-reserved', 'reserved'],
+ ['current-jobs-delayed', 'delayed'],
+ ['current-jobs-buried', 'buried']
+ ]
+ },
+ 'current_connections': {
+ 'options': [None, 'Current Connections', 'connections', 'server statistics',
+ 'beanstalk.current_connections', 'line'],
+ 'lines': [
+ ['current-connections', 'written'],
+ ['current-producers', 'producers'],
+ ['current-workers', 'workers'],
+ ['current-waiting', 'waiting']
+ ]
+ },
+ 'binlog': {
+ 'options': [None, 'Binlog', 'records/s', 'server statistics', 'beanstalk.binlog', 'line'],
+ 'lines': [
+ ['binlog-records-written', 'written', 'incremental'],
+ ['binlog-records-migrated', 'migrated', 'incremental']
+ ]
+ },
+ 'uptime': {
+ 'options': [None, 'Uptime', 'seconds', 'server statistics', 'beanstalk.uptime', 'line'],
+ 'lines': [
+ ['uptime'],
+ ]
+ }
+}
+
+
+def tube_chart_template(name):
+ order = [
+ '{0}_jobs_rate'.format(name),
+ '{0}_jobs'.format(name),
+ '{0}_connections'.format(name),
+ '{0}_commands'.format(name),
+ '{0}_pause'.format(name)
+ ]
+ family = 'tube {0}'.format(name)
+
+ charts = {
+ order[0]: {
+ 'options': [None, 'Job Rate', 'jobs/s', family, 'beanstalk.jobs_rate', 'area'],
+ 'lines': [
+ ['_'.join([name, 'total-jobs']), 'jobs', 'incremental']
+ ]
+ },
+ order[1]: {
+ 'options': [None, 'Jobs', 'jobs', family, 'beanstalk.jobs', 'stacked'],
+ 'lines': [
+ ['_'.join([name, 'current-jobs-urgent']), 'urgent'],
+ ['_'.join([name, 'current-jobs-ready']), 'ready'],
+ ['_'.join([name, 'current-jobs-reserved']), 'reserved'],
+ ['_'.join([name, 'current-jobs-delayed']), 'delayed'],
+ ['_'.join([name, 'current-jobs-buried']), 'buried']
+ ]
+ },
+ order[2]: {
+ 'options': [None, 'Connections', 'connections', family, 'beanstalk.connections', 'stacked'],
+ 'lines': [
+ ['_'.join([name, 'current-using']), 'using'],
+ ['_'.join([name, 'current-waiting']), 'waiting'],
+ ['_'.join([name, 'current-watching']), 'watching']
+ ]
+ },
+ order[3]: {
+ 'options': [None, 'Commands', 'commands/s', family, 'beanstalk.commands', 'stacked'],
+ 'lines': [
+ ['_'.join([name, 'cmd-delete']), 'deletes', 'incremental'],
+ ['_'.join([name, 'cmd-pause-tube']), 'pauses', 'incremental']
+ ]
+ },
+ order[4]: {
+ 'options': [None, 'Pause', 'seconds', family, 'beanstalk.pause', 'stacked'],
+ 'lines': [
+ ['_'.join([name, 'pause']), 'since'],
+ ['_'.join([name, 'pause-time-left']), 'left']
+ ]
+ }
+ }
+
+ return order, charts
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.configuration = configuration
+ self.order = list(ORDER)
+ self.definitions = dict(CHARTS)
+ self.conn = None
+ self.alive = True
+
+ def check(self):
+ if not BEANSTALKC:
+ self.error("'beanstalkc' module is needed to use beanstalk.chart.py")
+ return False
+
+ self.conn = self.connect()
+
+ return True if self.conn else False
+
+ def get_data(self):
+ """
+ :return: dict
+ """
+ if not self.is_alive():
+ return None
+
+ active_charts = self.charts.active_charts()
+ data = dict()
+
+ try:
+ data.update(self.conn.stats())
+
+ for tube in self.conn.tubes():
+ stats = self.conn.stats_tube(tube)
+
+ if tube + '_jobs_rate' not in active_charts:
+ self.create_new_tube_charts(tube)
+
+ for stat in stats:
+ data['_'.join([tube, stat])] = stats[stat]
+
+ except beanstalkc.SocketError:
+ self.alive = False
+ return None
+
+ return data or None
+
+ def create_new_tube_charts(self, tube):
+ order, charts = tube_chart_template(tube)
+
+ for chart_name in order:
+ params = [chart_name] + charts[chart_name]['options']
+ dimensions = charts[chart_name]['lines']
+
+ new_chart = self.charts.add_chart(params)
+ for dimension in dimensions:
+ new_chart.add_dimension(dimension)
+
+ def connect(self):
+ host = self.configuration.get('host', '127.0.0.1')
+ port = self.configuration.get('port', 11300)
+ timeout = self.configuration.get('timeout', 1)
+ try:
+ return beanstalkc.Connection(host=host,
+ port=port,
+ connect_timeout=timeout,
+ parse_yaml=load_yaml)
+ except beanstalkc.SocketError as error:
+ self.error('Connection to {0}:{1} failed: {2}'.format(host, port, error))
+ return None
+
+ def reconnect(self):
+ try:
+ self.conn.reconnect()
+ self.alive = True
+ return True
+ except beanstalkc.SocketError:
+ return False
+
+ def is_alive(self):
+ if not self.alive:
+ return self.reconnect()
+ return True
diff --git a/collectors/python.d.plugin/beanstalk/beanstalk.conf b/collectors/python.d.plugin/beanstalk/beanstalk.conf
new file mode 100644
index 0000000..6d9773a
--- /dev/null
+++ b/collectors/python.d.plugin/beanstalk/beanstalk.conf
@@ -0,0 +1,78 @@
+# netdata python.d.plugin configuration for beanstalk
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# chart_cleanup sets the default chart cleanup interval in iterations.
+# A chart is marked as obsolete if it has not been updated
+# 'chart_cleanup' iterations in a row.
+# When a plugin sends the obsolete flag, the charts are not deleted
+# from netdata immediately.
+# They will be hidden immediately (not offered to dashboard viewer,
+# streamed upstream and archived to external databases) and deleted one hour
+# later (configurable from netdata.conf).
+# chart_cleanup: 10
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+# chart_cleanup: 10 # the JOB's chart cleanup interval in iterations
+#
+# Additionally to the above, beanstalk also supports the following:
+#
+# host: 'host' # Server ip address or hostname. Default: 127.0.0.1
+# port: port # Beanstalkd port. Default:
+#
+# ----------------------------------------------------------------------
diff --git a/collectors/python.d.plugin/bind_rndc/Makefile.inc b/collectors/python.d.plugin/bind_rndc/Makefile.inc
new file mode 100644
index 0000000..72f3914
--- /dev/null
+++ b/collectors/python.d.plugin/bind_rndc/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += bind_rndc/bind_rndc.chart.py
+dist_pythonconfig_DATA += bind_rndc/bind_rndc.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += bind_rndc/README.md bind_rndc/Makefile.inc
+
diff --git a/collectors/python.d.plugin/bind_rndc/README.md b/collectors/python.d.plugin/bind_rndc/README.md
new file mode 100644
index 0000000..2d747f8
--- /dev/null
+++ b/collectors/python.d.plugin/bind_rndc/README.md
@@ -0,0 +1,79 @@
+<!--
+title: "ISC Bind monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/bind_rndc/README.md
+sidebar_label: "ISC Bind"
+-->
+
+# ISC Bind monitoring with Netdata
+
+Collects Name server summary performance statistics using `rndc` tool.
+
+## Requirements
+
+- Version of bind must be 9.6 +
+- Netdata must have permissions to run `rndc stats`
+
+It produces:
+
+1. **Name server statistics**
+
+ - requests
+ - responses
+ - success
+ - auth_answer
+ - nonauth_answer
+ - nxrrset
+ - failure
+ - nxdomain
+ - recursion
+ - duplicate
+ - rejections
+
+2. **Incoming queries**
+
+ - RESERVED0
+ - A
+ - NS
+ - CNAME
+ - SOA
+ - PTR
+ - MX
+ - TXT
+ - X25
+ - AAAA
+ - SRV
+ - NAPTR
+ - A6
+ - DS
+ - RSIG
+ - DNSKEY
+ - SPF
+ - ANY
+ - DLV
+
+3. **Outgoing queries**
+
+- Same as Incoming queries
+
+## Configuration
+
+Edit the `python.d/bind_rndc.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/bind_rndc.conf
+```
+
+Sample:
+
+```yaml
+local:
+ named_stats_path : '/var/log/bind/named.stats'
+```
+
+If no configuration is given, module will attempt to read named.stats file at `/var/log/bind/named.stats`
+
+---
+
+
diff --git a/collectors/python.d.plugin/bind_rndc/bind_rndc.chart.py b/collectors/python.d.plugin/bind_rndc/bind_rndc.chart.py
new file mode 100644
index 0000000..9d6c9fe
--- /dev/null
+++ b/collectors/python.d.plugin/bind_rndc/bind_rndc.chart.py
@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+# Description: bind rndc netdata python.d module
+# Author: ilyam8
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+from collections import defaultdict
+from subprocess import Popen
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from bases.collection import find_binary
+
+update_every = 30
+
+ORDER = [
+ 'name_server_statistics',
+ 'incoming_queries',
+ 'outgoing_queries',
+ 'named_stats_size',
+]
+
+CHARTS = {
+ 'name_server_statistics': {
+ 'options': [None, 'Name Server Statistics', 'stats', 'name server statistics',
+ 'bind_rndc.name_server_statistics', 'line'],
+ 'lines': [
+ ['nms_requests', 'requests', 'incremental'],
+ ['nms_rejected_queries', 'rejected_queries', 'incremental'],
+ ['nms_success', 'success', 'incremental'],
+ ['nms_failure', 'failure', 'incremental'],
+ ['nms_responses', 'responses', 'incremental'],
+ ['nms_duplicate', 'duplicate', 'incremental'],
+ ['nms_recursion', 'recursion', 'incremental'],
+ ['nms_nxrrset', 'nxrrset', 'incremental'],
+ ['nms_nxdomain', 'nxdomain', 'incremental'],
+ ['nms_non_auth_answer', 'non_auth_answer', 'incremental'],
+ ['nms_auth_answer', 'auth_answer', 'incremental'],
+ ['nms_dropped_queries', 'dropped_queries', 'incremental'],
+ ]},
+ 'incoming_queries': {
+ 'options': [None, 'Incoming Queries', 'queries', 'incoming queries', 'bind_rndc.incoming_queries', 'line'],
+ 'lines': [
+ ]},
+ 'outgoing_queries': {
+ 'options': [None, 'Outgoing Queries', 'queries', 'outgoing queries', 'bind_rndc.outgoing_queries', 'line'],
+ 'lines': [
+ ]},
+ 'named_stats_size': {
+ 'options': [None, 'Named Stats File Size', 'MiB', 'file size', 'bind_rndc.stats_size', 'line'],
+ 'lines': [
+ ['stats_size', None, 'absolute', 1, 1 << 20]
+ ]
+ }
+}
+
+NMS = {
+ 'nms_requests': [
+ 'IPv4 requests received',
+ 'IPv6 requests received',
+ 'TCP requests received',
+ 'requests with EDNS(0) receive'
+ ],
+ 'nms_responses': [
+ 'responses sent',
+ 'truncated responses sent',
+ 'responses with EDNS(0) sent',
+ 'requests with unsupported EDNS version received'
+ ],
+ 'nms_failure': [
+ 'other query failures',
+ 'queries resulted in SERVFAIL'
+ ],
+ 'nms_auth_answer': ['queries resulted in authoritative answer'],
+ 'nms_non_auth_answer': ['queries resulted in non authoritative answer'],
+ 'nms_nxrrset': ['queries resulted in nxrrset'],
+ 'nms_success': ['queries resulted in successful answer'],
+ 'nms_nxdomain': ['queries resulted in NXDOMAIN'],
+ 'nms_recursion': ['queries caused recursion'],
+ 'nms_duplicate': ['duplicate queries received'],
+ 'nms_rejected_queries': [
+ 'auth queries rejected',
+ 'recursive queries rejected'
+ ],
+ 'nms_dropped_queries': ['queries dropped']
+}
+
+STATS = ['Name Server Statistics', 'Incoming Queries', 'Outgoing Queries']
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.named_stats_path = self.configuration.get('named_stats_path', '/var/log/bind/named.stats')
+ self.rndc = find_binary('rndc')
+ self.data = dict(
+ nms_requests=0,
+ nms_responses=0,
+ nms_failure=0,
+ nms_auth=0,
+ nms_non_auth=0,
+ nms_nxrrset=0,
+ nms_success=0,
+ nms_nxdomain=0,
+ nms_recursion=0,
+ nms_duplicate=0,
+ nms_rejected_queries=0,
+ nms_dropped_queries=0,
+ )
+
+ def check(self):
+ if not self.rndc:
+ self.error('Can\'t locate "rndc" binary or binary is not executable by netdata')
+ return False
+
+ if not (os.path.isfile(self.named_stats_path) and os.access(self.named_stats_path, os.R_OK)):
+ self.error('Cannot access file %s' % self.named_stats_path)
+ return False
+
+ run_rndc = Popen([self.rndc, 'stats'], shell=False)
+ run_rndc.wait()
+
+ if not run_rndc.returncode:
+ return True
+ self.error('Not enough permissions to run "%s stats"' % self.rndc)
+ return False
+
+ def _get_raw_data(self):
+ """
+ Run 'rndc stats' and read last dump from named.stats
+ :return: dict
+ """
+ result = dict()
+ try:
+ current_size = os.path.getsize(self.named_stats_path)
+ run_rndc = Popen([self.rndc, 'stats'], shell=False)
+ run_rndc.wait()
+
+ if run_rndc.returncode:
+ return None
+ with open(self.named_stats_path) as named_stats:
+ named_stats.seek(current_size)
+ result['stats'] = named_stats.readlines()
+ result['size'] = current_size
+ return result
+ except (OSError, IOError):
+ return None
+
+ def _get_data(self):
+ """
+ Parse data from _get_raw_data()
+ :return: dict
+ """
+
+ raw_data = self._get_raw_data()
+
+ if raw_data is None:
+ return None
+ parsed = dict()
+ for stat in STATS:
+ parsed[stat] = parse_stats(field=stat,
+ named_stats=raw_data['stats'])
+
+ self.data.update(nms_mapper(data=parsed['Name Server Statistics']))
+
+ for elem in zip(['Incoming Queries', 'Outgoing Queries'], ['incoming_queries', 'outgoing_queries']):
+ parsed_key, chart_name = elem[0], elem[1]
+ for dimension_id, value in queries_mapper(data=parsed[parsed_key],
+ add=chart_name[:9]).items():
+
+ if dimension_id not in self.data:
+ dimension = dimension_id.replace(chart_name[:9], '')
+ if dimension_id not in self.charts[chart_name]:
+ self.charts[chart_name].add_dimension([dimension_id, dimension, 'incremental'])
+
+ self.data[dimension_id] = value
+
+ self.data['stats_size'] = raw_data['size']
+ return self.data
+
+
+def parse_stats(field, named_stats):
+ """
+ :param field: str:
+ :param named_stats: list:
+ :return: dict
+
+ Example:
+ filed: 'Incoming Queries'
+ names_stats (list of lines):
+ ++ Incoming Requests ++
+ 1405660 QUERY
+ 3 NOTIFY
+ ++ Incoming Queries ++
+ 1214961 A
+ 75 NS
+ 2 CNAME
+ 2897 SOA
+ 35544 PTR
+ 14 MX
+ 5822 TXT
+ 145974 AAAA
+ 371 SRV
+ ++ Outgoing Queries ++
+ ...
+
+ result:
+ {'A', 1214961, 'NS': 75, 'CNAME': 2, 'SOA': 2897, ...}
+ """
+ data = dict()
+ ns = iter(named_stats)
+ for line in ns:
+ if field not in line:
+ continue
+ while True:
+ try:
+ line = next(ns)
+ except StopIteration:
+ break
+ if '++' not in line:
+ if '[' in line:
+ continue
+ v, k = line.strip().split(' ', 1)
+ if k not in data:
+ data[k] = 0
+ data[k] += int(v)
+ continue
+ break
+ break
+ return data
+
+
+def nms_mapper(data):
+ """
+ :param data: dict
+ :return: dict(defaultdict)
+ """
+ result = defaultdict(int)
+ for k, v in NMS.items():
+ for elem in v:
+ result[k] += data.get(elem, 0)
+ return result
+
+
+def queries_mapper(data, add):
+ """
+ :param data: dict
+ :param add: str
+ :return: dict
+ """
+ return dict([(add + k, v) for k, v in data.items()])
diff --git a/collectors/python.d.plugin/bind_rndc/bind_rndc.conf b/collectors/python.d.plugin/bind_rndc/bind_rndc.conf
new file mode 100644
index 0000000..3b7e9a2
--- /dev/null
+++ b/collectors/python.d.plugin/bind_rndc/bind_rndc.conf
@@ -0,0 +1,110 @@
+# netdata python.d.plugin configuration for bind_rndc
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, bind_rndc also supports the following:
+#
+# named_stats_path: 'path to named.stats' # Default: '/var/log/bind/named.stats'
+#------------------------------------------------------------------------------------------------------------------
+# IMPORTANT Information
+#
+# BIND APPEND logs at EVERY RUN. Its NOT RECOMMENDED to set update_every below 30 sec.
+# STRONGLY RECOMMENDED to create a bind-rndc conf file for logrotate
+#
+# To set up your BIND to dump stats do the following:
+#
+# 1. add to 'named.conf.options' options {}:
+# statistics-file "/var/log/bind/named.stats";
+#
+# 2. Create bind/ directory in /var/log
+# cd /var/log/ && mkdir bind
+#
+# 3. Change owner of directory to 'bind' user
+# chown bind bind/
+#
+# 4. RELOAD (NOT restart) BIND
+# systemctl reload bind9.service
+#
+# 5. Run as a root 'rndc stats' to dump (BIND will create named.stats in new directory)
+#
+#
+# To ALLOW NETDATA TO RUN 'rndc stats' change '/etc/bind/rndc.key' group to netdata
+# chown :netdata rndc.key
+#
+# The last BUT NOT least is to create bind-rndc.conf in logrotate.d/
+# The working one
+# /var/log/bind/named.stats {
+#
+# daily
+# rotate 4
+# compress
+# delaycompress
+# create 0644 bind bind
+# missingok
+# postrotate
+# rndc reload > /dev/null
+# endscript
+# }
+#
+# To test your logrotate conf file run as root:
+#
+# logrotate /etc/logrotate.d/bind-rndc -d (debug dry-run mode)
+#
+# ----------------------------------------------------------------------
diff --git a/collectors/python.d.plugin/boinc/Makefile.inc b/collectors/python.d.plugin/boinc/Makefile.inc
new file mode 100644
index 0000000..319e19c
--- /dev/null
+++ b/collectors/python.d.plugin/boinc/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += boinc/boinc.chart.py
+dist_pythonconfig_DATA += boinc/boinc.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += boinc/README.md boinc/Makefile.inc
+
diff --git a/collectors/python.d.plugin/boinc/README.md b/collectors/python.d.plugin/boinc/README.md
new file mode 100644
index 0000000..4da2d52
--- /dev/null
+++ b/collectors/python.d.plugin/boinc/README.md
@@ -0,0 +1,41 @@
+<!--
+title: "BOINC monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/boinc/README.md
+sidebar_label: "BOINC"
+-->
+
+# BOINC monitoring with Netdata
+
+Monitors task counts for the Berkeley Open Infrastructure Networking Computing (BOINC) distributed computing client using the same RPC interface that the BOINC monitoring GUI does.
+
+It provides charts tracking the total number of tasks and active tasks, as well as ones tracking each of the possible states for tasks.
+
+## Configuration
+
+Edit the `python.d/boinc.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/boinc.conf
+```
+
+BOINC requires use of a password to access it's RPC interface. You can
+find this password in the `gui_rpc_auth.cfg` file in your BOINC directory.
+
+By default, the module will try to auto-detect the password by looking
+in `/var/lib/boinc` for this file (this is the location most Linux
+distributions use for a system-wide BOINC installation), so things may
+just work without needing configuration for the local system.
+
+You can monitor remote systems as well:
+
+```yaml
+remote:
+ hostname: some-host
+ password: some-password
+```
+
+---
+
+
diff --git a/collectors/python.d.plugin/boinc/boinc.chart.py b/collectors/python.d.plugin/boinc/boinc.chart.py
new file mode 100644
index 0000000..a31eda1
--- /dev/null
+++ b/collectors/python.d.plugin/boinc/boinc.chart.py
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+# Description: BOINC netdata python.d module
+# Author: Austin S. Hemmelgarn (Ferroin)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import socket
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from third_party import boinc_client
+
+ORDER = [
+ 'tasks',
+ 'states',
+ 'sched_states',
+ 'process_states',
+]
+
+CHARTS = {
+ 'tasks': {
+ 'options': [None, 'Overall Tasks', 'tasks', 'boinc', 'boinc.tasks', 'line'],
+ 'lines': [
+ ['total', 'Total', 'absolute', 1, 1],
+ ['active', 'Active', 'absolute', 1, 1]
+ ]
+ },
+ 'states': {
+ 'options': [None, 'Tasks per State', 'tasks', 'boinc', 'boinc.states', 'line'],
+ 'lines': [
+ ['new', 'New', 'absolute', 1, 1],
+ ['downloading', 'Downloading', 'absolute', 1, 1],
+ ['downloaded', 'Ready to Run', 'absolute', 1, 1],
+ ['comperror', 'Compute Errors', 'absolute', 1, 1],
+ ['uploading', 'Uploading', 'absolute', 1, 1],
+ ['uploaded', 'Uploaded', 'absolute', 1, 1],
+ ['aborted', 'Aborted', 'absolute', 1, 1],
+ ['upload_failed', 'Failed Uploads', 'absolute', 1, 1]
+ ]
+ },
+ 'sched_states': {
+ 'options': [None, 'Tasks per Scheduler State', 'tasks', 'boinc', 'boinc.sched', 'line'],
+ 'lines': [
+ ['uninit_sched', 'Uninitialized', 'absolute', 1, 1],
+ ['preempted', 'Preempted', 'absolute', 1, 1],
+ ['scheduled', 'Scheduled', 'absolute', 1, 1]
+ ]
+ },
+ 'process_states': {
+ 'options': [None, 'Tasks per Process State', 'tasks', 'boinc', 'boinc.process', 'line'],
+ 'lines': [
+ ['uninit_proc', 'Uninitialized', 'absolute', 1, 1],
+ ['executing', 'Executing', 'absolute', 1, 1],
+ ['suspended', 'Suspended', 'absolute', 1, 1],
+ ['aborting', 'Aborted', 'absolute', 1, 1],
+ ['quit', 'Quit', 'absolute', 1, 1],
+ ['copy_pending', 'Copy Pending', 'absolute', 1, 1]
+ ]
+ }
+}
+
+# A simple template used for pre-loading the return dictionary to make
+# the _get_data() method simpler.
+_DATA_TEMPLATE = {
+ 'total': 0,
+ 'active': 0,
+ 'new': 0,
+ 'downloading': 0,
+ 'downloaded': 0,
+ 'comperror': 0,
+ 'uploading': 0,
+ 'uploaded': 0,
+ 'aborted': 0,
+ 'upload_failed': 0,
+ 'uninit_sched': 0,
+ 'preempted': 0,
+ 'scheduled': 0,
+ 'uninit_proc': 0,
+ 'executing': 0,
+ 'suspended': 0,
+ 'aborting': 0,
+ 'quit': 0,
+ 'copy_pending': 0
+}
+
+# Map task states to dimensions
+_TASK_MAP = {
+ boinc_client.ResultState.NEW: 'new',
+ boinc_client.ResultState.FILES_DOWNLOADING: 'downloading',
+ boinc_client.ResultState.FILES_DOWNLOADED: 'downloaded',
+ boinc_client.ResultState.COMPUTE_ERROR: 'comperror',
+ boinc_client.ResultState.FILES_UPLOADING: 'uploading',
+ boinc_client.ResultState.FILES_UPLOADED: 'uploaded',
+ boinc_client.ResultState.ABORTED: 'aborted',
+ boinc_client.ResultState.UPLOAD_FAILED: 'upload_failed'
+}
+
+# Map scheduler states to dimensions
+_SCHED_MAP = {
+ boinc_client.CpuSched.UNINITIALIZED: 'uninit_sched',
+ boinc_client.CpuSched.PREEMPTED: 'preempted',
+ boinc_client.CpuSched.SCHEDULED: 'scheduled',
+}
+
+# Maps process states to dimensions
+_PROC_MAP = {
+ boinc_client.Process.UNINITIALIZED: 'uninit_proc',
+ boinc_client.Process.EXECUTING: 'executing',
+ boinc_client.Process.SUSPENDED: 'suspended',
+ boinc_client.Process.ABORT_PENDING: 'aborted',
+ boinc_client.Process.QUIT_PENDING: 'quit',
+ boinc_client.Process.COPY_PENDING: 'copy_pending'
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.host = self.configuration.get('host', 'localhost')
+ self.port = self.configuration.get('port', 0)
+ self.password = self.configuration.get('password', '')
+ self.client = boinc_client.BoincClient(host=self.host, port=self.port, passwd=self.password)
+ self.alive = False
+
+ def check(self):
+ return self.connect()
+
+ def connect(self):
+ self.client.connect()
+ self.alive = self.client.connected and self.client.authorized
+ return self.alive
+
+ def reconnect(self):
+ # The client class itself actually disconnects existing
+ # connections when it is told to connect, so we don't need to
+ # explicitly disconnect when we're just trying to reconnect.
+ return self.connect()
+
+ def is_alive(self):
+ if not self.alive:
+ return self.reconnect()
+ return True
+
+ def _get_data(self):
+ if not self.is_alive():
+ return None
+
+ data = dict(_DATA_TEMPLATE)
+
+ try:
+ results = self.client.get_tasks()
+ except socket.error:
+ self.error('Connection is dead')
+ self.alive = False
+ return None
+
+ for task in results:
+ data['total'] += 1
+ data[_TASK_MAP[task.state]] += 1
+ try:
+ if task.active_task:
+ data['active'] += 1
+ data[_SCHED_MAP[task.scheduler_state]] += 1
+ data[_PROC_MAP[task.active_task_state]] += 1
+ except AttributeError:
+ pass
+
+ return data or None
diff --git a/collectors/python.d.plugin/boinc/boinc.conf b/collectors/python.d.plugin/boinc/boinc.conf
new file mode 100644
index 0000000..16edf55
--- /dev/null
+++ b/collectors/python.d.plugin/boinc/boinc.conf
@@ -0,0 +1,66 @@
+# netdata python.d.plugin configuration for boinc
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, boinc also supports the following:
+#
+# hostname: localhost # The host running the BOINC client
+# port: 31416 # The remote GUI RPC port for BOINC
+# password: '' # The remote GUI RPC password
diff --git a/collectors/python.d.plugin/ceph/Makefile.inc b/collectors/python.d.plugin/ceph/Makefile.inc
new file mode 100644
index 0000000..15b039e
--- /dev/null
+++ b/collectors/python.d.plugin/ceph/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += ceph/ceph.chart.py
+dist_pythonconfig_DATA += ceph/ceph.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += ceph/README.md ceph/Makefile.inc
+
diff --git a/collectors/python.d.plugin/ceph/README.md b/collectors/python.d.plugin/ceph/README.md
new file mode 100644
index 0000000..b75ba6d
--- /dev/null
+++ b/collectors/python.d.plugin/ceph/README.md
@@ -0,0 +1,48 @@
+<!--
+title: "CEPH monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/ceph/README.md
+sidebar_label: "CEPH"
+-->
+
+# CEPH monitoring with Netdata
+
+Monitors the ceph cluster usage and consumption data of a server, and produces:
+
+- Cluster statistics (usage, available, latency, objects, read/write rate)
+- OSD usage
+- OSD latency
+- Pool usage
+- Pool read/write operations
+- Pool read/write rate
+- number of objects per pool
+
+## Requirements
+
+- `rados` python module
+- Granting read permissions to ceph group from keyring file
+
+```shell
+# chmod 640 /etc/ceph/ceph.client.admin.keyring
+```
+
+## Configuration
+
+Edit the `python.d/ceph.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/ceph.conf
+```
+
+Sample:
+
+```yaml
+local:
+ config_file: '/etc/ceph/ceph.conf'
+ keyring_file: '/etc/ceph/ceph.client.admin.keyring'
+```
+
+---
+
+
diff --git a/collectors/python.d.plugin/ceph/ceph.chart.py b/collectors/python.d.plugin/ceph/ceph.chart.py
new file mode 100644
index 0000000..494eef4
--- /dev/null
+++ b/collectors/python.d.plugin/ceph/ceph.chart.py
@@ -0,0 +1,374 @@
+# -*- coding: utf-8 -*-
+# Description: ceph netdata python.d module
+# Author: Luis Eduardo (lets00)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+try:
+ import rados
+
+ CEPH = True
+except ImportError:
+ CEPH = False
+
+import json
+import os
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+# default module values (can be overridden per job in `config`)
+update_every = 10
+
+ORDER = [
+ 'general_usage',
+ 'general_objects',
+ 'general_bytes',
+ 'general_operations',
+ 'general_latency',
+ 'pool_usage',
+ 'pool_objects',
+ 'pool_read_bytes',
+ 'pool_write_bytes',
+ 'pool_read_operations',
+ 'pool_write_operations',
+ 'osd_usage',
+ 'osd_size',
+ 'osd_apply_latency',
+ 'osd_commit_latency'
+]
+
+CHARTS = {
+ 'general_usage': {
+ 'options': [None, 'Ceph General Space', 'KiB', 'general', 'ceph.general_usage', 'stacked'],
+ 'lines': [
+ ['general_available', 'avail', 'absolute'],
+ ['general_usage', 'used', 'absolute']
+ ]
+ },
+ 'general_objects': {
+ 'options': [None, 'Ceph General Objects', 'objects', 'general', 'ceph.general_objects', 'area'],
+ 'lines': [
+ ['general_objects', 'cluster', 'absolute']
+ ]
+ },
+ 'general_bytes': {
+ 'options': [None, 'Ceph General Read/Write Data/s', 'KiB/s', 'general', 'ceph.general_bytes',
+ 'area'],
+ 'lines': [
+ ['general_read_bytes', 'read', 'absolute', 1, 1024],
+ ['general_write_bytes', 'write', 'absolute', -1, 1024]
+ ]
+ },
+ 'general_operations': {
+ 'options': [None, 'Ceph General Read/Write Operations/s', 'operations', 'general', 'ceph.general_operations',
+ 'area'],
+ 'lines': [
+ ['general_read_operations', 'read', 'absolute', 1],
+ ['general_write_operations', 'write', 'absolute', -1]
+ ]
+ },
+ 'general_latency': {
+ 'options': [None, 'Ceph General Apply/Commit latency', 'milliseconds', 'general', 'ceph.general_latency',
+ 'area'],
+ 'lines': [
+ ['general_apply_latency', 'apply', 'absolute'],
+ ['general_commit_latency', 'commit', 'absolute']
+ ]
+ },
+ 'pool_usage': {
+ 'options': [None, 'Ceph Pools', 'KiB', 'pool', 'ceph.pool_usage', 'line'],
+ 'lines': []
+ },
+ 'pool_objects': {
+ 'options': [None, 'Ceph Pools', 'objects', 'pool', 'ceph.pool_objects', 'line'],
+ 'lines': []
+ },
+ 'pool_read_bytes': {
+ 'options': [None, 'Ceph Read Pool Data/s', 'KiB/s', 'pool', 'ceph.pool_read_bytes', 'area'],
+ 'lines': []
+ },
+ 'pool_write_bytes': {
+ 'options': [None, 'Ceph Write Pool Data/s', 'KiB/s', 'pool', 'ceph.pool_write_bytes', 'area'],
+ 'lines': []
+ },
+ 'pool_read_operations': {
+ 'options': [None, 'Ceph Read Pool Operations/s', 'operations', 'pool', 'ceph.pool_read_operations', 'area'],
+ 'lines': []
+ },
+ 'pool_write_operations': {
+ 'options': [None, 'Ceph Write Pool Operations/s', 'operations', 'pool', 'ceph.pool_write_operations', 'area'],
+ 'lines': []
+ },
+ 'osd_usage': {
+ 'options': [None, 'Ceph OSDs', 'KiB', 'osd', 'ceph.osd_usage', 'line'],
+ 'lines': []
+ },
+ 'osd_size': {
+ 'options': [None, 'Ceph OSDs size', 'KiB', 'osd', 'ceph.osd_size', 'line'],
+ 'lines': []
+ },
+ 'osd_apply_latency': {
+ 'options': [None, 'Ceph OSDs apply latency', 'milliseconds', 'osd', 'ceph.apply_latency', 'line'],
+ 'lines': []
+ },
+ 'osd_commit_latency': {
+ 'options': [None, 'Ceph OSDs commit latency', 'milliseconds', 'osd', 'ceph.commit_latency', 'line'],
+ 'lines': []
+ }
+
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.config_file = self.configuration.get('config_file')
+ self.keyring_file = self.configuration.get('keyring_file')
+ self.rados_id = self.configuration.get('rados_id', 'admin')
+
+ def check(self):
+ """
+ Checks module
+ :return:
+ """
+ if not CEPH:
+ self.error('rados module is needed to use ceph.chart.py')
+ return False
+ if not (self.config_file and self.keyring_file):
+ self.error('config_file and/or keyring_file is not defined')
+ return False
+
+ # Verify files and permissions
+ if not (os.access(self.config_file, os.F_OK)):
+ self.error('{0} does not exist'.format(self.config_file))
+ return False
+ if not (os.access(self.keyring_file, os.F_OK)):
+ self.error('{0} does not exist'.format(self.keyring_file))
+ return False
+ if not (os.access(self.config_file, os.R_OK)):
+ self.error('Ceph plugin does not read {0}, define read permission.'.format(self.config_file))
+ return False
+ if not (os.access(self.keyring_file, os.R_OK)):
+ self.error('Ceph plugin does not read {0}, define read permission.'.format(self.keyring_file))
+ return False
+ try:
+ self.cluster = rados.Rados(conffile=self.config_file,
+ conf=dict(keyring=self.keyring_file),
+ rados_id=self.rados_id)
+ self.cluster.connect()
+ except rados.Error as error:
+ self.error(error)
+ return False
+ self.create_definitions()
+ return True
+
+ def create_definitions(self):
+ """
+ Create dynamically charts options
+ :return: None
+ """
+ # Pool lines
+ for pool in sorted(self._get_df()['pools'], key=lambda x: sorted(x.keys())):
+ self.definitions['pool_usage']['lines'].append([pool['name'],
+ pool['name'],
+ 'absolute'])
+ self.definitions['pool_objects']['lines'].append(["obj_{0}".format(pool['name']),
+ pool['name'],
+ 'absolute'])
+ self.definitions['pool_read_bytes']['lines'].append(['read_{0}'.format(pool['name']),
+ pool['name'],
+ 'absolute', 1, 1024])
+ self.definitions['pool_write_bytes']['lines'].append(['write_{0}'.format(pool['name']),
+ pool['name'],
+ 'absolute', 1, 1024])
+ self.definitions['pool_read_operations']['lines'].append(['read_operations_{0}'.format(pool['name']),
+ pool['name'],
+ 'absolute'])
+ self.definitions['pool_write_operations']['lines'].append(['write_operations_{0}'.format(pool['name']),
+ pool['name'],
+ 'absolute'])
+
+ # OSD lines
+ for osd in sorted(self._get_osd_df()['nodes'], key=lambda x: sorted(x.keys())):
+ self.definitions['osd_usage']['lines'].append([osd['name'],
+ osd['name'],
+ 'absolute'])
+ self.definitions['osd_size']['lines'].append(['size_{0}'.format(osd['name']),
+ osd['name'],
+ 'absolute'])
+ self.definitions['osd_apply_latency']['lines'].append(['apply_latency_{0}'.format(osd['name']),
+ osd['name'],
+ 'absolute'])
+ self.definitions['osd_commit_latency']['lines'].append(['commit_latency_{0}'.format(osd['name']),
+ osd['name'],
+ 'absolute'])
+
+ def get_data(self):
+ """
+ Catch all ceph data
+ :return: dict
+ """
+ try:
+ data = {}
+ df = self._get_df()
+ osd_df = self._get_osd_df()
+ osd_perf = self._get_osd_perf()
+ osd_perf_infos = get_osd_perf_infos(osd_perf)
+ pool_stats = self._get_osd_pool_stats()
+
+ data.update(self._get_general(osd_perf_infos, pool_stats))
+ for pool in df['pools']:
+ data.update(self._get_pool_usage(pool))
+ data.update(self._get_pool_objects(pool))
+ for pool_io in pool_stats:
+ data.update(self._get_pool_rw(pool_io))
+ for osd in osd_df['nodes']:
+ data.update(self._get_osd_usage(osd))
+ data.update(self._get_osd_size(osd))
+ for osd_apply_commit in osd_perf_infos:
+ data.update(self._get_osd_latency(osd_apply_commit))
+ return data
+ except (ValueError, AttributeError) as error:
+ self.error(error)
+ return None
+
+ def _get_general(self, osd_perf_infos, pool_stats):
+ """
+ Get ceph's general usage
+ :return: dict
+ """
+ status = self.cluster.get_cluster_stats()
+ read_bytes_sec = 0
+ write_bytes_sec = 0
+ read_op_per_sec = 0
+ write_op_per_sec = 0
+ apply_latency = 0
+ commit_latency = 0
+
+ for pool_rw_io_b in pool_stats:
+ read_bytes_sec += pool_rw_io_b['client_io_rate'].get('read_bytes_sec', 0)
+ write_bytes_sec += pool_rw_io_b['client_io_rate'].get('write_bytes_sec', 0)
+ read_op_per_sec += pool_rw_io_b['client_io_rate'].get('read_op_per_sec', 0)
+ write_op_per_sec += pool_rw_io_b['client_io_rate'].get('write_op_per_sec', 0)
+ for perf in osd_perf_infos:
+ apply_latency += perf['perf_stats']['apply_latency_ms']
+ commit_latency += perf['perf_stats']['commit_latency_ms']
+
+ return {
+ 'general_usage': int(status['kb_used']),
+ 'general_available': int(status['kb_avail']),
+ 'general_objects': int(status['num_objects']),
+ 'general_read_bytes': read_bytes_sec,
+ 'general_write_bytes': write_bytes_sec,
+ 'general_read_operations': read_op_per_sec,
+ 'general_write_operations': write_op_per_sec,
+ 'general_apply_latency': apply_latency,
+ 'general_commit_latency': commit_latency
+ }
+
+ @staticmethod
+ def _get_pool_usage(pool):
+ """
+ Process raw data into pool usage dict information
+ :return: A pool dict with pool name's key and usage bytes' value
+ """
+ return {pool['name']: pool['stats']['kb_used']}
+
+ @staticmethod
+ def _get_pool_objects(pool):
+ """
+ Process raw data into pool usage dict information
+ :return: A pool dict with pool name's key and object numbers
+ """
+ return {'obj_{0}'.format(pool['name']): pool['stats']['objects']}
+
+ @staticmethod
+ def _get_pool_rw(pool):
+ """
+ Get read/write kb and operations in a pool
+ :return: A pool dict with both read/write bytes and operations.
+ """
+ return {
+ 'read_{0}'.format(pool['pool_name']): int(pool['client_io_rate'].get('read_bytes_sec', 0)),
+ 'write_{0}'.format(pool['pool_name']): int(pool['client_io_rate'].get('write_bytes_sec', 0)),
+ 'read_operations_{0}'.format(pool['pool_name']): int(pool['client_io_rate'].get('read_op_per_sec', 0)),
+ 'write_operations_{0}'.format(pool['pool_name']): int(pool['client_io_rate'].get('write_op_per_sec', 0))
+ }
+
+ @staticmethod
+ def _get_osd_usage(osd):
+ """
+ Process raw data into osd dict information to get osd usage
+ :return: A osd dict with osd name's key and usage bytes' value
+ """
+ return {osd['name']: float(osd['kb_used'])}
+
+ @staticmethod
+ def _get_osd_size(osd):
+ """
+ Process raw data into osd dict information to get osd size (kb)
+ :return: A osd dict with osd name's key and size bytes' value
+ """
+ return {'size_{0}'.format(osd['name']): float(osd['kb'])}
+
+ @staticmethod
+ def _get_osd_latency(osd):
+ """
+ Get ceph osd apply and commit latency
+ :return: A osd dict with osd name's key with both apply and commit latency values
+ """
+ return {
+ 'apply_latency_osd.{0}'.format(osd['id']): osd['perf_stats']['apply_latency_ms'],
+ 'commit_latency_osd.{0}'.format(osd['id']): osd['perf_stats']['commit_latency_ms']
+ }
+
+ def _get_df(self):
+ """
+ Get ceph df output
+ :return: ceph df --format json
+ """
+ return json.loads(self.cluster.mon_command(json.dumps({
+ 'prefix': 'df',
+ 'format': 'json'
+ }), '')[1].decode('utf-8'))
+
+ def _get_osd_df(self):
+ """
+ Get ceph osd df output
+ :return: ceph osd df --format json
+ """
+ return json.loads(self.cluster.mon_command(json.dumps({
+ 'prefix': 'osd df',
+ 'format': 'json'
+ }), '')[1].decode('utf-8').replace('-nan', '"-nan"'))
+
+ def _get_osd_perf(self):
+ """
+ Get ceph osd performance
+ :return: ceph osd perf --format json
+ """
+ return json.loads(self.cluster.mon_command(json.dumps({
+ 'prefix': 'osd perf',
+ 'format': 'json'
+ }), '')[1].decode('utf-8'))
+
+ def _get_osd_pool_stats(self):
+ """
+ Get ceph osd pool status.
+ This command is used to get information about both
+ read/write operation and bytes per second on each pool
+ :return: ceph osd pool stats --format json
+ """
+ return json.loads(self.cluster.mon_command(json.dumps({
+ 'prefix': 'osd pool stats',
+ 'format': 'json'
+ }), '')[1].decode('utf-8'))
+
+
+def get_osd_perf_infos(osd_perf):
+ # https://github.com/netdata/netdata/issues/8247
+ # module uses 'osd_perf_infos' data, its been moved under 'osdstats` since Ceph v14.2
+ if 'osd_perf_infos' in osd_perf:
+ return osd_perf['osd_perf_infos']
+ return osd_perf['osdstats']['osd_perf_infos']
diff --git a/collectors/python.d.plugin/ceph/ceph.conf b/collectors/python.d.plugin/ceph/ceph.conf
new file mode 100644
index 0000000..81788e8
--- /dev/null
+++ b/collectors/python.d.plugin/ceph/ceph.conf
@@ -0,0 +1,75 @@
+# netdata python.d.plugin configuration for ceph stats
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 10
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 10 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, ceph plugin also supports the following:
+#
+# config_file: 'config_file' # Ceph config file.
+# keyring_file: 'keyring_file' # Ceph keyring file. netdata user must be added into ceph group
+# # and keyring file must be read group permission.
+# rados_id: 'rados username' # ID used to connect to ceph cluster. Allows
+# # creating a read only key for pulling data v.s. admin
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+#
+config_file: '/etc/ceph/ceph.conf'
+keyring_file: '/etc/ceph/ceph.client.admin.keyring'
+rados_id: 'admin'
diff --git a/collectors/python.d.plugin/changefinder/Makefile.inc b/collectors/python.d.plugin/changefinder/Makefile.inc
new file mode 100644
index 0000000..01a9240
--- /dev/null
+++ b/collectors/python.d.plugin/changefinder/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += changefinder/changefinder.chart.py
+dist_pythonconfig_DATA += changefinder/changefinder.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += changefinder/README.md changefinder/Makefile.inc
+
diff --git a/collectors/python.d.plugin/changefinder/README.md b/collectors/python.d.plugin/changefinder/README.md
new file mode 100644
index 0000000..7ec3a25
--- /dev/null
+++ b/collectors/python.d.plugin/changefinder/README.md
@@ -0,0 +1,217 @@
+<!--
+title: "Online change point detection with Netdata"
+description: "Use ML-driven change point detection to narrow your focus and shorten root cause analysis."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/changefinder/README.md
+-->
+
+# Online changepoint detection with Netdata
+
+This collector uses the Python [changefinder](https://github.com/shunsukeaihara/changefinder) library to
+perform [online](https://en.wikipedia.org/wiki/Online_machine_learning) [changepoint detection](https://en.wikipedia.org/wiki/Change_detection)
+on your Netdata charts and/or dimensions.
+
+Instead of this collector just _collecting_ data, it also does some computation on the data it collects to return a
+changepoint score for each chart or dimension you configure it to work on. This is
+an [online](https://en.wikipedia.org/wiki/Online_machine_learning) machine learning algorithm so there is no batch step
+to train the model, instead it evolves over time as more data arrives. That makes this particular algorithm quite cheap
+to compute at each step of data collection (see the notes section below for more details) and it should scale fairly
+well to work on lots of charts or hosts (if running on a parent node for example).
+
+> As this is a somewhat unique collector and involves often subjective concepts like changepoints and anomalies, we would love to hear any feedback on it from the community. Please let us know on the [community forum](https://community.netdata.cloud/t/changefinder-collector-feedback/972) or drop us a note at [analytics-ml-team@netdata.cloud](mailto:analytics-ml-team@netdata.cloud) for any and all feedback, both positive and negative. This sort of feedback is priceless to help us make complex features more useful.
+
+## Charts
+
+Two charts are available:
+
+### ChangeFinder Scores (`changefinder.scores`)
+
+This chart shows the percentile of the score that is output from the ChangeFinder library (it is turned off by default
+but available with `show_scores: true`).
+
+A high observed score is more likely to be a valid changepoint worth exploring, even more so when multiple charts or
+dimensions have high changepoint scores at the same time or very close together.
+
+### ChangeFinder Flags (`changefinder.flags`)
+
+This chart shows `1` or `0` if the latest score has a percentile value that exceeds the `cf_threshold` threshold. By
+default, any scores that are in the 99th or above percentile will raise a flag on this chart.
+
+The raw changefinder score itself can be a little noisy and so limiting ourselves to just periods where it surpasses
+the 99th percentile can help manage the "[signal to noise ratio](https://en.wikipedia.org/wiki/Signal-to-noise_ratio)"
+better.
+
+The `cf_threshold` parameter might be one you want to play around with to tune things specifically for the workloads on
+your node and the specific charts you want to monitor. For example, maybe the 95th percentile might work better for you
+than the 99th percentile.
+
+Below is an example of the chart produced by this collector. The first 3/4 of the period looks normal in that we see a
+few individual changes being picked up somewhat randomly over time. But then at around 14:59 towards the end of the
+chart we see two periods with 'spikes' of multiple changes for a small period of time. This is the sort of pattern that
+might be a sign something on the system that has changed sufficiently enough to merit some investigation.
+
+![changepoint-collector](https://user-images.githubusercontent.com/2178292/108773528-665de980-7556-11eb-895d-798669bcd695.png)
+
+## Requirements
+
+- This collector will only work with Python 3 and requires the packages below be installed.
+
+```bash
+# become netdata user
+sudo su -s /bin/bash netdata
+# install required packages for the netdata user
+pip3 install --user numpy==1.19.5 changefinder==0.03 scipy==1.5.4
+```
+
+**Note**: if you need to tell Netdata to use Python 3 then you can pass the below command in the python plugin section
+of your `netdata.conf` file.
+
+```yaml
+[ plugin:python.d ]
+ # update every = 1
+ command options = -ppython3
+```
+
+## Configuration
+
+Install the Python requirements above, enable the collector and restart Netdata.
+
+```bash
+cd /etc/netdata/
+sudo ./edit-config python.d.conf
+# Set `changefinder: no` to `changefinder: yes`
+sudo systemctl restart netdata
+```
+
+The configuration for the changefinder collector defines how it will behave on your system and might take some
+experimentation with over time to set it optimally for your node. Out of the box, the config comes with
+some [sane defaults](https://www.netdata.cloud/blog/redefining-monitoring-netdata/) to get you started that try to
+balance the flexibility and power of the ML models with the goal of being as cheap as possible in term of cost on the
+node resources.
+
+_**Note**: If you are unsure about any of the below configuration options then it's best to just ignore all this and
+leave the `changefinder.conf` file alone to begin with. Then you can return to it later if you would like to tune things
+a bit more once the collector is running for a while and you have a feeling for its performance on your node._
+
+Edit the `python.d/changefinder.conf` configuration file using `edit-config` from the your
+agent's [config directory](/docs/configure/nodes.md), which is usually at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/changefinder.conf
+```
+
+The default configuration should look something like this. Here you can see each parameter (with sane defaults) and some
+information about each one and what it does.
+
+```yaml
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+
+# Pull data from local Netdata node.
+local:
+
+ # A friendly name for this job.
+ name: 'local'
+
+ # What host to pull data from.
+ host: '127.0.0.1:19999'
+
+ # What charts to pull data for - A regex like 'system\..*|' or 'system\..*|apps.cpu|apps.mem' etc.
+ charts_regex: 'system\..*'
+
+ # Charts to exclude, useful if you would like to exclude some specific charts.
+ # Note: should be a ',' separated string like 'chart.name,chart.name'.
+ charts_to_exclude: ''
+
+ # Get ChangeFinder scores 'per_dim' or 'per_chart'.
+ mode: 'per_chart'
+
+ # Default parameters that can be passed to the changefinder library.
+ cf_r: 0.5
+ cf_order: 1
+ cf_smooth: 15
+
+ # The percentile above which scores will be flagged.
+ cf_threshold: 99
+
+ # The number of recent scores to use when calculating the percentile of the changefinder score.
+ n_score_samples: 14400
+
+ # Set to true if you also want to chart the percentile scores in addition to the flags.
+ # Mainly useful for debugging or if you want to dive deeper on how the scores are evolving over time.
+ show_scores: false
+```
+
+## Troubleshooting
+
+To see any relevant log messages you can use a command like below.
+
+```bash
+grep 'changefinder' /var/log/netdata/error.log
+```
+
+If you would like to log in as `netdata` user and run the collector in debug mode to see more detail.
+
+```bash
+# become netdata user
+sudo su -s /bin/bash netdata
+# run collector in debug using `nolock` option if netdata is already running the collector itself.
+/usr/libexec/netdata/plugins.d/python.d.plugin changefinder debug trace nolock
+```
+
+## Notes
+
+- It may take an hour or two (depending on your choice of `n_score_samples`) for the collector to 'settle' into it's
+ typical behaviour in terms of the trained models and scores you will see in the normal running of your node. Mainly
+ this is because it can take a while to build up a proper distribution of previous scores in over to convert the raw
+ score returned by the ChangeFinder algorithm into a percentile based on the most recent `n_score_samples` that have
+ already been produced. So when you first turn the collector on, it will have a lot of flags in the beginning and then
+ should 'settle down' once it has built up enough history. This is a typical characteristic of online machine learning
+ approaches which need some initial window of time before they can be useful.
+- As this collector does most of the work in Python itself, you may want to try it out first on a test or development
+ system to get a sense of its performance characteristics on a node similar to where you would like to use it.
+- On a development n1-standard-2 (2 vCPUs, 7.5 GB memory) vm running Ubuntu 18.04 LTS and not doing any work some of the
+ typical performance characteristics we saw from running this collector (with defaults) were:
+ - A runtime (`netdata.runtime_changefinder`) of ~30ms.
+ - Typically ~1% additional cpu usage.
+ - About ~85mb of ram (`apps.mem`) being continually used by the `python.d.plugin` under default configuration.
+
+## Useful links and further reading
+
+- [PyPi changefinder](https://pypi.org/project/changefinder/) reference page.
+- [GitHub repo](https://github.com/shunsukeaihara/changefinder) for the changefinder library.
+- Relevant academic papers:
+ - Yamanishi K, Takeuchi J. A unifying framework for detecting outliers and change points from nonstationary time
+ series data. 8th ACM SIGKDD international conference on Knowledge discovery and data mining - KDD02. 2002:
+ 676. ([pdf](https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.12.3469&rep=rep1&type=pdf))
+ - Kawahara Y, Sugiyama M. Sequential Change-Point Detection Based on Direct Density-Ratio Estimation. SIAM
+ International Conference on Data Mining. 2009:
+ 389–400. ([pdf](https://onlinelibrary.wiley.com/doi/epdf/10.1002/sam.10124))
+ - Liu S, Yamada M, Collier N, Sugiyama M. Change-point detection in time-series data by relative density-ratio
+ estimation. Neural Networks. Jul.2013 43:72–83. [PubMed: 23500502] ([pdf](https://arxiv.org/pdf/1203.0453.pdf))
+ - T. Iwata, K. Nakamura, Y. Tokusashi, and H. Matsutani, “Accelerating Online Change-Point Detection Algorithm using
+ 10 GbE FPGA NIC,” Proc. International European Conference on Parallel and Distributed Computing (Euro-Par’18)
+ Workshops, vol.11339, pp.506–517, Aug.
+ 2018 ([pdf](https://www.arc.ics.keio.ac.jp/~matutani/papers/iwata_heteropar2018.pdf))
+- The [ruptures](https://github.com/deepcharles/ruptures) python package is also a good place to learn more about
+ changepoint detection (mostly offline as opposed to online but deals with similar concepts).
+- A nice [blog post](https://techrando.com/2019/08/14/a-brief-introduction-to-change-point-detection-using-python/)
+ showing some of the other options and libraries for changepoint detection in Python.
+- [Bayesian changepoint detection](https://github.com/hildensia/bayesian_changepoint_detection) library - we may explore
+ implementing a collector for this or integrating this approach into this collector at a future date if there is
+ interest and it proves computationaly feasible.
+- You might also find the
+ Netdata [anomalies collector](https://github.com/netdata/netdata/tree/master/collectors/python.d.plugin/anomalies)
+ interesting.
+- [Anomaly Detection](https://en.wikipedia.org/wiki/Anomaly_detection) wikipedia page.
+- [Anomaly Detection YouTube playlist](https://www.youtube.com/playlist?list=PL6Zhl9mK2r0KxA6rB87oi4kWzoqGd5vp0)
+ maintained by [andrewm4894](https://github.com/andrewm4894/) from Netdata.
+- [awesome-TS-anomaly-detection](https://github.com/rob-med/awesome-TS-anomaly-detection) Github list of useful tools,
+ libraries and resources.
+- [Mendeley public group](https://www.mendeley.com/community/interesting-anomaly-detection-papers/) with some
+ interesting anomaly detection papers we have been reading.
+- Good [blog post](https://www.anodot.com/blog/what-is-anomaly-detection/) from Anodot on time series anomaly detection.
+ Anodot also have some great whitepapers in this space too that some may find useful.
+- Novelty and outlier detection in
+ the [scikit-learn documentation](https://scikit-learn.org/stable/modules/outlier_detection.html).
+
diff --git a/collectors/python.d.plugin/changefinder/changefinder.chart.py b/collectors/python.d.plugin/changefinder/changefinder.chart.py
new file mode 100644
index 0000000..c18e560
--- /dev/null
+++ b/collectors/python.d.plugin/changefinder/changefinder.chart.py
@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*-
+# Description: changefinder netdata python.d module
+# Author: andrewm4894
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from json import loads
+import re
+
+from bases.FrameworkServices.UrlService import UrlService
+
+import numpy as np
+import changefinder
+from scipy.stats import percentileofscore
+
+update_every = 5
+disabled_by_default = True
+
+ORDER = [
+ 'scores',
+ 'flags'
+]
+
+CHARTS = {
+ 'scores': {
+ 'options': [None, 'ChangeFinder', 'score', 'Scores', 'scores', 'line'],
+ 'lines': []
+ },
+ 'flags': {
+ 'options': [None, 'ChangeFinder', 'flag', 'Flags', 'flags', 'stacked'],
+ 'lines': []
+ }
+}
+
+DEFAULT_PROTOCOL = 'http'
+DEFAULT_HOST = '127.0.0.1:19999'
+DEFAULT_CHARTS_REGEX = 'system.*'
+DEFAULT_MODE = 'per_chart'
+DEFAULT_CF_R = 0.5
+DEFAULT_CF_ORDER = 1
+DEFAULT_CF_SMOOTH = 15
+DEFAULT_CF_DIFF = False
+DEFAULT_CF_THRESHOLD = 99
+DEFAULT_N_SCORE_SAMPLES = 14400
+DEFAULT_SHOW_SCORES = False
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.protocol = self.configuration.get('protocol', DEFAULT_PROTOCOL)
+ self.host = self.configuration.get('host', DEFAULT_HOST)
+ self.url = '{}://{}/api/v1/allmetrics?format=json'.format(self.protocol, self.host)
+ self.charts_regex = re.compile(self.configuration.get('charts_regex', DEFAULT_CHARTS_REGEX))
+ self.charts_to_exclude = self.configuration.get('charts_to_exclude', '').split(',')
+ self.mode = self.configuration.get('mode', DEFAULT_MODE)
+ self.n_score_samples = int(self.configuration.get('n_score_samples', DEFAULT_N_SCORE_SAMPLES))
+ self.show_scores = int(self.configuration.get('show_scores', DEFAULT_SHOW_SCORES))
+ self.cf_r = float(self.configuration.get('cf_r', DEFAULT_CF_R))
+ self.cf_order = int(self.configuration.get('cf_order', DEFAULT_CF_ORDER))
+ self.cf_smooth = int(self.configuration.get('cf_smooth', DEFAULT_CF_SMOOTH))
+ self.cf_diff = bool(self.configuration.get('cf_diff', DEFAULT_CF_DIFF))
+ self.cf_threshold = float(self.configuration.get('cf_threshold', DEFAULT_CF_THRESHOLD))
+ self.collected_dims = {'scores': set(), 'flags': set()}
+ self.models = {}
+ self.x_latest = {}
+ self.scores_latest = {}
+ self.scores_samples = {}
+
+ def get_score(self, x, model):
+ """Update the score for the model based on most recent data, flag if it's percentile passes self.cf_threshold.
+ """
+
+ # get score
+ if model not in self.models:
+ # initialise empty model if needed
+ self.models[model] = changefinder.ChangeFinder(r=self.cf_r, order=self.cf_order, smooth=self.cf_smooth)
+ # if the update for this step fails then just fallback to last known score
+ try:
+ score = self.models[model].update(x)
+ self.scores_latest[model] = score
+ except Exception as _:
+ score = self.scores_latest.get(model, 0)
+ score = 0 if np.isnan(score) else score
+
+ # update sample scores used to calculate percentiles
+ if model in self.scores_samples:
+ self.scores_samples[model].append(score)
+ else:
+ self.scores_samples[model] = [score]
+ self.scores_samples[model] = self.scores_samples[model][-self.n_score_samples:]
+
+ # convert score to percentile
+ score = percentileofscore(self.scores_samples[model], score)
+
+ # flag based on score percentile
+ flag = 1 if score >= self.cf_threshold else 0
+
+ return score, flag
+
+ def validate_charts(self, chart, data, algorithm='absolute', multiplier=1, divisor=1):
+ """If dimension not in chart then add it.
+ """
+ if not self.charts:
+ return
+
+ for dim in data:
+ if dim not in self.collected_dims[chart]:
+ self.collected_dims[chart].add(dim)
+ self.charts[chart].add_dimension([dim, dim, algorithm, multiplier, divisor])
+
+ for dim in list(self.collected_dims[chart]):
+ if dim not in data:
+ self.collected_dims[chart].remove(dim)
+ self.charts[chart].del_dimension(dim, hide=False)
+
+ def diff(self, x, model):
+ """Take difference of data.
+ """
+ x_diff = x - self.x_latest.get(model, 0)
+ self.x_latest[model] = x
+ x = x_diff
+ return x
+
+ def _get_data(self):
+
+ # pull data from self.url
+ raw_data = self._get_raw_data()
+ if raw_data is None:
+ return None
+
+ raw_data = loads(raw_data)
+
+ # filter to just the data for the charts specified
+ charts_in_scope = list(filter(self.charts_regex.match, raw_data.keys()))
+ charts_in_scope = [c for c in charts_in_scope if c not in self.charts_to_exclude]
+
+ data_score = {}
+ data_flag = {}
+
+ # process each chart
+ for chart in charts_in_scope:
+
+ if self.mode == 'per_chart':
+
+ # average dims on chart and run changefinder on that average
+ x = [raw_data[chart]['dimensions'][dim]['value'] for dim in raw_data[chart]['dimensions']]
+ x = [x for x in x if x is not None]
+
+ if len(x) > 0:
+
+ x = sum(x) / len(x)
+ x = self.diff(x, chart) if self.cf_diff else x
+
+ score, flag = self.get_score(x, chart)
+ if self.show_scores:
+ data_score['{}_score'.format(chart)] = score * 100
+ data_flag[chart] = flag
+
+ else:
+
+ # run changefinder on each individual dim
+ for dim in raw_data[chart]['dimensions']:
+
+ chart_dim = '{}|{}'.format(chart, dim)
+
+ x = raw_data[chart]['dimensions'][dim]['value']
+ x = x if x else 0
+ x = self.diff(x, chart_dim) if self.cf_diff else x
+
+ score, flag = self.get_score(x, chart_dim)
+ if self.show_scores:
+ data_score['{}_score'.format(chart_dim)] = score * 100
+ data_flag[chart_dim] = flag
+
+ self.validate_charts('flags', data_flag)
+
+ if self.show_scores & len(data_score) > 0:
+ data_score['average_score'] = sum(data_score.values()) / len(data_score)
+ self.validate_charts('scores', data_score, divisor=100)
+
+ data = {**data_score, **data_flag}
+
+ return data
diff --git a/collectors/python.d.plugin/changefinder/changefinder.conf b/collectors/python.d.plugin/changefinder/changefinder.conf
new file mode 100644
index 0000000..56a681f
--- /dev/null
+++ b/collectors/python.d.plugin/changefinder/changefinder.conf
@@ -0,0 +1,74 @@
+# netdata python.d.plugin configuration for example
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 5
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+
+local:
+
+ # A friendly name for this job.
+ name: 'local'
+
+ # What host to pull data from.
+ host: '127.0.0.1:19999'
+
+ # What charts to pull data for - A regex like 'system\..*|' or 'system\..*|apps.cpu|apps.mem' etc.
+ charts_regex: 'system\..*'
+
+ # Charts to exclude, useful if you would like to exclude some specific charts.
+ # Note: should be a ',' separated string like 'chart.name,chart.name'.
+ charts_to_exclude: ''
+
+ # Get ChangeFinder scores 'per_dim' or 'per_chart'.
+ mode: 'per_chart'
+
+ # Default parameters that can be passed to the changefinder library.
+ cf_r: 0.5
+ cf_order: 1
+ cf_smooth: 15
+
+ # The percentile above which scores will be flagged.
+ cf_threshold: 99
+
+ # The number of recent scores to use when calculating the percentile of the changefinder score.
+ n_score_samples: 14400
+
+ # Set to true if you also want to chart the percentile scores in addition to the flags.
+ # Mainly useful for debugging or if you want to dive deeper on how the scores are evolving over time.
+ show_scores: false
diff --git a/collectors/python.d.plugin/dockerd/Makefile.inc b/collectors/python.d.plugin/dockerd/Makefile.inc
new file mode 100644
index 0000000..b100bc6
--- /dev/null
+++ b/collectors/python.d.plugin/dockerd/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += dockerd/dockerd.chart.py
+dist_pythonconfig_DATA += dockerd/dockerd.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += dockerd/README.md dockerd/Makefile.inc
+
diff --git a/collectors/python.d.plugin/dockerd/README.md b/collectors/python.d.plugin/dockerd/README.md
new file mode 100644
index 0000000..6470a7c
--- /dev/null
+++ b/collectors/python.d.plugin/dockerd/README.md
@@ -0,0 +1,46 @@
+<!--
+title: "Docker Engine monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/dockerd/README.md
+sidebar_label: "Docker Engine"
+-->
+
+# Docker Engine monitoring with Netdata
+
+Collects docker container health metrics.
+
+**Requirement:**
+
+- `docker` package, required version 3.2.0+
+
+Following charts are drawn:
+
+1. **running containers**
+
+ - count
+
+2. **healthy containers**
+
+ - count
+
+3. **unhealthy containers**
+
+ - count
+
+## Configuration
+
+Edit the `python.d/dockerd.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/dockerd.conf
+```
+
+```yaml
+ update_every : 1
+ priority : 60000
+```
+
+---
+
+
diff --git a/collectors/python.d.plugin/dockerd/dockerd.chart.py b/collectors/python.d.plugin/dockerd/dockerd.chart.py
new file mode 100644
index 0000000..bd9640b
--- /dev/null
+++ b/collectors/python.d.plugin/dockerd/dockerd.chart.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+# Description: docker netdata python.d module
+# Author: Kévin Darcel (@tuxity)
+
+try:
+ import docker
+
+ HAS_DOCKER = True
+except ImportError:
+ HAS_DOCKER = False
+
+from distutils.version import StrictVersion
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = [
+ 'running_containers',
+ 'healthy_containers',
+ 'unhealthy_containers'
+]
+
+CHARTS = {
+ 'running_containers': {
+ 'options': [None, 'Number of running containers', 'containers', 'running containers',
+ 'docker.running_containers', 'line'],
+ 'lines': [
+ ['running_containers', 'running']
+ ]
+ },
+ 'healthy_containers': {
+ 'options': [None, 'Number of healthy containers', 'containers', 'healthy containers',
+ 'docker.healthy_containers', 'line'],
+ 'lines': [
+ ['healthy_containers', 'healthy']
+ ]
+ },
+ 'unhealthy_containers': {
+ 'options': [None, 'Number of unhealthy containers', 'containers', 'unhealthy containers',
+ 'docker.unhealthy_containers', 'line'],
+ 'lines': [
+ ['unhealthy_containers', 'unhealthy']
+ ]
+ }
+}
+
+MIN_REQUIRED_VERSION = '3.2.0'
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.client = None
+
+ def check(self):
+ if not HAS_DOCKER:
+ self.error("'docker' package is needed to use dockerd module")
+ return False
+
+ if StrictVersion(docker.__version__) < StrictVersion(MIN_REQUIRED_VERSION):
+ self.error("installed 'docker' package version {0}, minimum required version {1}, please upgrade".format(
+ docker.__version__,
+ MIN_REQUIRED_VERSION,
+ ))
+ return False
+
+ self.client = docker.DockerClient(base_url=self.configuration.get('url', 'unix://var/run/docker.sock'))
+
+ try:
+ self.client.ping()
+ except docker.errors.APIError as error:
+ self.error(error)
+ return False
+
+ return True
+
+ def get_data(self):
+ data = dict()
+
+ data['running_containers'] = len(self.client.containers.list(sparse=True))
+ data['healthy_containers'] = len(self.client.containers.list(filters={'health': 'healthy'}, sparse=True))
+ data['unhealthy_containers'] = len(self.client.containers.list(filters={'health': 'unhealthy'}, sparse=True))
+
+ return data or None
diff --git a/collectors/python.d.plugin/dockerd/dockerd.conf b/collectors/python.d.plugin/dockerd/dockerd.conf
new file mode 100644
index 0000000..96c8ee0
--- /dev/null
+++ b/collectors/python.d.plugin/dockerd/dockerd.conf
@@ -0,0 +1,77 @@
+# netdata python.d.plugin configuration for dockerd health data API
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, dockerd plugin also supports the following:
+#
+# url: '<scheme>://<host>:<port>/<health_page_api>'
+# # http://localhost:8080/health
+#
+# if the URL is password protected, the following are supported:
+#
+# user: 'username'
+# pass: 'password'
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+#
+local:
+ url: 'unix://var/run/docker.sock'
diff --git a/collectors/python.d.plugin/dovecot/Makefile.inc b/collectors/python.d.plugin/dovecot/Makefile.inc
new file mode 100644
index 0000000..fd7d13b
--- /dev/null
+++ b/collectors/python.d.plugin/dovecot/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += dovecot/dovecot.chart.py
+dist_pythonconfig_DATA += dovecot/dovecot.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += dovecot/README.md dovecot/Makefile.inc
+
diff --git a/collectors/python.d.plugin/dovecot/README.md b/collectors/python.d.plugin/dovecot/README.md
new file mode 100644
index 0000000..e6bbf0d
--- /dev/null
+++ b/collectors/python.d.plugin/dovecot/README.md
@@ -0,0 +1,105 @@
+<!--
+title: "Dovecot monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/dovecot/README.md
+sidebar_label: "Dovecot"
+-->
+
+# Dovecot monitoring with Netdata
+
+Provides statistics information from Dovecot server.
+
+Statistics are taken from dovecot socket by executing `EXPORT global` command.
+More information about dovecot stats can be found on [project wiki page.](http://wiki2.dovecot.org/Statistics)
+
+Module isn't compatible with new statistic api (v2.3), but you are still able to use the module with Dovecot v2.3
+by following [upgrading steps.](https://wiki2.dovecot.org/Upgrading/2.3).
+
+**Requirement:**
+Dovecot UNIX socket with R/W permissions for user `netdata` or Dovecot with configured TCP/IP socket.
+
+Module gives information with following charts:
+
+1. **sessions**
+
+ - active sessions
+
+2. **logins**
+
+ - logins
+
+3. **commands** - number of IMAP commands
+
+ - commands
+
+4. **Faults**
+
+ - minor
+ - major
+
+5. **Context Switches**
+
+ - voluntary
+ - involuntary
+
+6. **disk** in bytes/s
+
+ - read
+ - write
+
+7. **bytes** in bytes/s
+
+ - read
+ - write
+
+8. **number of syscalls** in syscalls/s
+
+ - read
+ - write
+
+9. **lookups** - number of lookups per second
+
+ - path
+ - attr
+
+10. **hits** - number of cache hits
+
+ - hits
+
+11. **attempts** - authorization attempts
+
+ - success
+ - failure
+
+12. **cache** - cached authorization hits
+
+ - hit
+ - miss
+
+## Configuration
+
+Edit the `python.d/dovecot.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/dovecot.conf
+```
+
+Sample:
+
+```yaml
+localtcpip:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 24242
+
+localsocket:
+ name : 'local'
+ socket : '/var/run/dovecot/stats'
+```
+
+If no configuration is given, module will attempt to connect to dovecot using unix socket localized in `/var/run/dovecot/stats`
+
+---
+
+
diff --git a/collectors/python.d.plugin/dovecot/dovecot.chart.py b/collectors/python.d.plugin/dovecot/dovecot.chart.py
new file mode 100644
index 0000000..dfaef28
--- /dev/null
+++ b/collectors/python.d.plugin/dovecot/dovecot.chart.py
@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# Description: dovecot netdata python.d module
+# Author: Pawel Krupa (paulfantom)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from bases.FrameworkServices.SocketService import SocketService
+
+UNIX_SOCKET = '/var/run/dovecot/stats'
+
+ORDER = [
+ 'sessions',
+ 'logins',
+ 'commands',
+ 'faults',
+ 'context_switches',
+ 'io',
+ 'net',
+ 'syscalls',
+ 'lookup',
+ 'cache',
+ 'auth',
+ 'auth_cache'
+]
+
+CHARTS = {
+ 'sessions': {
+ 'options': [None, 'Dovecot Active Sessions', 'number', 'sessions', 'dovecot.sessions', 'line'],
+ 'lines': [
+ ['num_connected_sessions', 'active sessions', 'absolute']
+ ]
+ },
+ 'logins': {
+ 'options': [None, 'Dovecot Logins', 'number', 'logins', 'dovecot.logins', 'line'],
+ 'lines': [
+ ['num_logins', 'logins', 'absolute']
+ ]
+ },
+ 'commands': {
+ 'options': [None, 'Dovecot Commands', 'commands', 'commands', 'dovecot.commands', 'line'],
+ 'lines': [
+ ['num_cmds', 'commands', 'absolute']
+ ]
+ },
+ 'faults': {
+ 'options': [None, 'Dovecot Page Faults', 'faults', 'page faults', 'dovecot.faults', 'line'],
+ 'lines': [
+ ['min_faults', 'minor', 'absolute'],
+ ['maj_faults', 'major', 'absolute']
+ ]
+ },
+ 'context_switches': {
+ 'options': [None, 'Dovecot Context Switches', 'switches', 'context switches', 'dovecot.context_switches',
+ 'line'],
+ 'lines': [
+ ['vol_cs', 'voluntary', 'absolute'],
+ ['invol_cs', 'involuntary', 'absolute']
+ ]
+ },
+ 'io': {
+ 'options': [None, 'Dovecot Disk I/O', 'KiB/s', 'disk', 'dovecot.io', 'area'],
+ 'lines': [
+ ['disk_input', 'read', 'incremental', 1, 1024],
+ ['disk_output', 'write', 'incremental', -1, 1024]
+ ]
+ },
+ 'net': {
+ 'options': [None, 'Dovecot Network Bandwidth', 'kilobits/s', 'network', 'dovecot.net', 'area'],
+ 'lines': [
+ ['read_bytes', 'read', 'incremental', 8, 1000],
+ ['write_bytes', 'write', 'incremental', -8, 1000]
+ ]
+ },
+ 'syscalls': {
+ 'options': [None, 'Dovecot Number of SysCalls', 'syscalls/s', 'system', 'dovecot.syscalls', 'line'],
+ 'lines': [
+ ['read_count', 'read', 'incremental'],
+ ['write_count', 'write', 'incremental']
+ ]
+ },
+ 'lookup': {
+ 'options': [None, 'Dovecot Lookups', 'number/s', 'lookups', 'dovecot.lookup', 'stacked'],
+ 'lines': [
+ ['mail_lookup_path', 'path', 'incremental'],
+ ['mail_lookup_attr', 'attr', 'incremental']
+ ]
+ },
+ 'cache': {
+ 'options': [None, 'Dovecot Cache Hits', 'hits/s', 'cache', 'dovecot.cache', 'line'],
+ 'lines': [
+ ['mail_cache_hits', 'hits', 'incremental']
+ ]
+ },
+ 'auth': {
+ 'options': [None, 'Dovecot Authentications', 'attempts', 'logins', 'dovecot.auth', 'stacked'],
+ 'lines': [
+ ['auth_successes', 'ok', 'absolute'],
+ ['auth_failures', 'failed', 'absolute']
+ ]
+ },
+ 'auth_cache': {
+ 'options': [None, 'Dovecot Authentication Cache', 'number', 'cache', 'dovecot.auth_cache', 'stacked'],
+ 'lines': [
+ ['auth_cache_hits', 'hit', 'absolute'],
+ ['auth_cache_misses', 'miss', 'absolute']
+ ]
+ }
+}
+
+
+class Service(SocketService):
+ def __init__(self, configuration=None, name=None):
+ SocketService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.host = None # localhost
+ self.port = None # 24242
+ self.unix_socket = UNIX_SOCKET
+ self.request = 'EXPORT\tglobal\r\n'
+
+ def _get_data(self):
+ """
+ Format data received from socket
+ :return: dict
+ """
+ try:
+ raw = self._get_raw_data()
+ except (ValueError, AttributeError):
+ return None
+
+ if raw is None:
+ self.debug('dovecot returned no data')
+ return None
+
+ data = raw.split('\n')[:2]
+ desc = data[0].split('\t')
+ vals = data[1].split('\t')
+ ret = dict()
+ for i, _ in enumerate(desc):
+ try:
+ ret[str(desc[i])] = int(vals[i])
+ except ValueError:
+ continue
+ return ret or None
diff --git a/collectors/python.d.plugin/dovecot/dovecot.conf b/collectors/python.d.plugin/dovecot/dovecot.conf
new file mode 100644
index 0000000..451dbc9
--- /dev/null
+++ b/collectors/python.d.plugin/dovecot/dovecot.conf
@@ -0,0 +1,98 @@
+# netdata python.d.plugin configuration for dovecot
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, dovecot also supports the following:
+#
+# socket: 'path/to/dovecot/stats'
+#
+# or
+# host: 'IP or HOSTNAME' # the host to connect to
+# port: PORT # the port to connect to
+#
+#
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name : 'local'
+ host : 'localhost'
+ port : 24242
+
+localipv4:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 24242
+
+localipv6:
+ name : 'local'
+ host : '::1'
+ port : 24242
+
+localsocket:
+ name : 'local'
+ socket : '/var/run/dovecot/stats'
+
+localsocket_old:
+ name : 'local'
+ socket : '/var/run/dovecot/old-stats'
+
diff --git a/collectors/python.d.plugin/example/Makefile.inc b/collectors/python.d.plugin/example/Makefile.inc
new file mode 100644
index 0000000..1b027d5
--- /dev/null
+++ b/collectors/python.d.plugin/example/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += example/example.chart.py
+dist_pythonconfig_DATA += example/example.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += example/README.md example/Makefile.inc
+
diff --git a/collectors/python.d.plugin/example/README.md b/collectors/python.d.plugin/example/README.md
new file mode 100644
index 0000000..0b80aa9
--- /dev/null
+++ b/collectors/python.d.plugin/example/README.md
@@ -0,0 +1,14 @@
+<!--
+title: "Example"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/example/README.md
+-->
+
+# Example
+
+You can add custom data collectors using Python.
+
+Netdata provides an [example python data collection module](https://github.com/netdata/netdata/tree/master/collectors/python.d.plugin/example).
+
+If you want to write your own collector, read our [writing a new Python module](/collectors/python.d.plugin/README.md#how-to-write-a-new-module) tutorial.
+
+
diff --git a/collectors/python.d.plugin/example/example.chart.py b/collectors/python.d.plugin/example/example.chart.py
new file mode 100644
index 0000000..d6c0b66
--- /dev/null
+++ b/collectors/python.d.plugin/example/example.chart.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Description: example netdata python.d module
+# Author: Put your name here (your github login)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from random import SystemRandom
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+priority = 90000
+
+ORDER = [
+ 'random',
+]
+
+CHARTS = {
+ 'random': {
+ 'options': [None, 'A random number', 'random number', 'random', 'random', 'line'],
+ 'lines': [
+ ['random1']
+ ]
+ }
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.random = SystemRandom()
+ self.num_lines = self.configuration.get('num_lines', 4)
+ self.lower = self.configuration.get('lower', 0)
+ self.upper = self.configuration.get('upper', 100)
+
+ @staticmethod
+ def check():
+ return True
+
+ def get_data(self):
+ data = dict()
+
+ for i in range(0, self.num_lines):
+ dimension_id = ''.join(['random', str(i)])
+
+ if dimension_id not in self.charts['random']:
+ self.charts['random'].add_dimension([dimension_id])
+
+ data[dimension_id] = self.random.randint(self.lower, self.upper)
+
+ return data
diff --git a/collectors/python.d.plugin/example/example.conf b/collectors/python.d.plugin/example/example.conf
new file mode 100644
index 0000000..31261b8
--- /dev/null
+++ b/collectors/python.d.plugin/example/example.conf
@@ -0,0 +1,87 @@
+# netdata python.d.plugin configuration for example
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear on the dashboard
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, example also supports the following:
+#
+# num_lines: 4 # the number of lines to create
+# lower: 0 # the lower bound of numbers to randomly sample from
+# upper: 100 # the upper bound of numbers to randomly sample from
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+
+four_lines:
+ name: "Four Lines" # the JOB's name as it will appear on the dashboard
+ update_every: 1 # the JOB's data collection frequency
+ priority: 60000 # the JOB's order on the dashboard
+ penalty: yes # the JOB's penalty
+ autodetection_retry: 0 # the JOB's re-check interval in seconds
+ num_lines: 4 # the number of lines to create
+ lower: 0 # the lower bound of numbers to randomly sample from
+ upper: 100 # the upper bound of numbers to randomly sample from
+
+# if you wanted to make another job to run in addition to the one above then
+# you would just uncomment the job configuration below.
+# two_lines:
+# name: "Two Lines" # the JOB's name as it will appear on the dashboard
+# num_lines: 2 # the number of lines to create
+# lower: 50 # the lower bound of numbers to randomly sample from
+# upper: 75 # the upper bound of numbers to randomly sample from
diff --git a/collectors/python.d.plugin/exim/Makefile.inc b/collectors/python.d.plugin/exim/Makefile.inc
new file mode 100644
index 0000000..36ffa56
--- /dev/null
+++ b/collectors/python.d.plugin/exim/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += exim/exim.chart.py
+dist_pythonconfig_DATA += exim/exim.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += exim/README.md exim/Makefile.inc
+
diff --git a/collectors/python.d.plugin/exim/README.md b/collectors/python.d.plugin/exim/README.md
new file mode 100644
index 0000000..92b2d7a
--- /dev/null
+++ b/collectors/python.d.plugin/exim/README.md
@@ -0,0 +1,41 @@
+<!--
+title: "Exim monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/exim/README.md
+sidebar_label: "Exim"
+-->
+
+# Exim monitoring with Netdata
+
+Simple module executing `exim -bpc` to grab exim queue.
+This command can take a lot of time to finish its execution thus it is not recommended to run it every second.
+
+## Requirements
+
+The module uses the `exim` binary, which can only be executed as root by default. We need to allow other users to `exim` binary. We solve that adding `queue_list_requires_admin` statement in exim configuration and set to `false`, because it is `true` by default. On many Linux distributions, the default location of `exim` configuration is in `/etc/exim.conf`.
+
+1. Edit the `exim` configuration with your preferred editor and add:
+`queue_list_requires_admin = false`
+2. Restart `exim` and Netdata
+
+*WHM (CPanel) server*
+
+On a WHM server, you can reconfigure `exim` over the WHM interface with the following steps.
+
+1. Login to WHM
+2. Navigate to Service Configuration --> Exim Configuration Manager --> tab Advanced Editor
+3. Scroll down to the button **Add additional configuration setting** and click on it.
+4. In the new dropdown which will appear above we need to find and choose:
+`queue_list_requires_admin` and set to `false`
+5. Scroll to the end and click the **Save** button.
+
+It produces only one chart:
+
+1. **Exim Queue Emails**
+
+ - emails
+
+Configuration is not needed.
+
+---
+
+
diff --git a/collectors/python.d.plugin/exim/exim.chart.py b/collectors/python.d.plugin/exim/exim.chart.py
new file mode 100644
index 0000000..7238a1b
--- /dev/null
+++ b/collectors/python.d.plugin/exim/exim.chart.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Description: exim netdata python.d module
+# Author: Pawel Krupa (paulfantom)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+
+EXIM_COMMAND = 'exim -bpc'
+
+ORDER = [
+ 'qemails',
+]
+
+CHARTS = {
+ 'qemails': {
+ 'options': [None, 'Exim Queue Emails', 'emails', 'queue', 'exim.qemails', 'line'],
+ 'lines': [
+ ['emails', None, 'absolute']
+ ]
+ }
+}
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ ExecutableService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.command = EXIM_COMMAND
+
+ def _get_data(self):
+ """
+ Format data received from shell command
+ :return: dict
+ """
+ try:
+ return {'emails': int(self._get_raw_data()[0])}
+ except (ValueError, AttributeError):
+ return None
diff --git a/collectors/python.d.plugin/exim/exim.conf b/collectors/python.d.plugin/exim/exim.conf
new file mode 100644
index 0000000..3b7e659
--- /dev/null
+++ b/collectors/python.d.plugin/exim/exim.conf
@@ -0,0 +1,91 @@
+# netdata python.d.plugin configuration for exim
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# exim is slow, so once every 10 seconds
+update_every: 10
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, exim also supports the following:
+#
+# command: 'exim -bpc' # the command to run
+#
+
+# ----------------------------------------------------------------------
+# REQUIRED exim CONFIGURATION
+#
+# netdata will query exim as user netdata.
+# By default exim will refuse to respond.
+#
+# To allow querying exim as non-admin user, please set the following
+# to your exim configuration:
+#
+# queue_list_requires_admin = false
+#
+# Your exim configuration should be in
+#
+# /etc/exim/exim4.conf
+# or
+# /etc/exim4/conf.d/main/000_local_options
+#
+# Please consult your distribution information to find the exact file.
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+
+local:
+ command: 'exim -bpc'
diff --git a/collectors/python.d.plugin/fail2ban/Makefile.inc b/collectors/python.d.plugin/fail2ban/Makefile.inc
new file mode 100644
index 0000000..31e117e
--- /dev/null
+++ b/collectors/python.d.plugin/fail2ban/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += fail2ban/fail2ban.chart.py
+dist_pythonconfig_DATA += fail2ban/fail2ban.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += fail2ban/README.md fail2ban/Makefile.inc
+
diff --git a/collectors/python.d.plugin/fail2ban/README.md b/collectors/python.d.plugin/fail2ban/README.md
new file mode 100644
index 0000000..be09e18
--- /dev/null
+++ b/collectors/python.d.plugin/fail2ban/README.md
@@ -0,0 +1,82 @@
+<!--
+title: "Fail2ban monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/fail2ban/README.md
+sidebar_label: "Fail2ban"
+-->
+
+# Fail2ban monitoring with Netdata
+
+Monitors the fail2ban log file to show all bans for all active jails.
+
+## Requirements
+
+The `fail2ban.log` file must be readable by the user `netdata`:
+
+- change the file ownership and access permissions.
+- update `/etc/logrotate.d/fail2ban` to persists the changes after rotating the log file.
+
+<details>
+ <summary>Click to expand the instruction.</summary>
+
+To change the file ownership and access permissions, execute the following:
+
+```shell
+sudo chown root:netdata /var/log/fail2ban.log
+sudo chmod 640 /var/log/fail2ban.log
+```
+
+To persist the changes after rotating the log file, add `create 640 root netdata` to the `/etc/logrotate.d/fail2ban`:
+
+```shell
+/var/log/fail2ban.log {
+
+ weekly
+ rotate 4
+ compress
+
+ delaycompress
+ missingok
+ postrotate
+ fail2ban-client flushlogs 1>/dev/null
+ endscript
+
+ # If fail2ban runs as non-root it still needs to have write access
+ # to logfiles.
+ # create 640 fail2ban adm
+ create 640 root netdata
+}
+```
+
+</details>
+
+## Charts
+
+- Failed attempts in attempts/s
+- Bans in bans/s
+- Banned IP addresses (since the last restart of netdata) in ips
+
+## Configuration
+
+Edit the `python.d/fail2ban.conf` configuration file using `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/fail2ban.conf
+```
+
+Sample:
+
+```yaml
+local:
+ log_path: '/var/log/fail2ban.log'
+ conf_path: '/etc/fail2ban/jail.local'
+ exclude: 'dropbear apache'
+```
+
+If no configuration is given, module will attempt to read log file at `/var/log/fail2ban.log` and conf file
+at `/etc/fail2ban/jail.local`. If conf file is not found default jail is `ssh`.
+
+---
+
+
diff --git a/collectors/python.d.plugin/fail2ban/fail2ban.chart.py b/collectors/python.d.plugin/fail2ban/fail2ban.chart.py
new file mode 100644
index 0000000..76f6d92
--- /dev/null
+++ b/collectors/python.d.plugin/fail2ban/fail2ban.chart.py
@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+# Description: fail2ban log netdata python.d module
+# Author: ilyam8
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+import re
+from collections import defaultdict
+from glob import glob
+
+from bases.FrameworkServices.LogService import LogService
+
+ORDER = [
+ 'jails_failed_attempts',
+ 'jails_bans',
+ 'jails_banned_ips',
+]
+
+
+def charts(jails):
+ """
+ Chart definitions creating
+ """
+
+ ch = {
+ ORDER[0]: {
+ 'options': [None, 'Failed attempts', 'attempts/s', 'failed attempts', 'fail2ban.failed_attempts', 'line'],
+ 'lines': []
+ },
+ ORDER[1]: {
+ 'options': [None, 'Bans', 'bans/s', 'bans', 'fail2ban.bans', 'line'],
+ 'lines': []
+ },
+ ORDER[2]: {
+ 'options': [None, 'Banned IP addresses (since the last restart of netdata)', 'ips', 'banned ips',
+ 'fail2ban.banned_ips', 'line'],
+ 'lines': []
+ },
+ }
+ for jail in jails:
+ dim = ['{0}_failed_attempts'.format(jail), jail, 'incremental']
+ ch[ORDER[0]]['lines'].append(dim)
+
+ dim = [jail, jail, 'incremental']
+ ch[ORDER[1]]['lines'].append(dim)
+
+ dim = ['{0}_in_jail'.format(jail), jail, 'absolute']
+ ch[ORDER[2]]['lines'].append(dim)
+
+ return ch
+
+
+RE_JAILS = re.compile(r'\[([a-zA-Z0-9_-]+)\][^\[\]]+?enabled\s+= +(true|yes|false|no)')
+
+ACTION_BAN = 'Ban'
+ACTION_UNBAN = 'Unban'
+ACTION_RESTORE_BAN = 'Restore Ban'
+ACTION_FOUND = 'Found'
+
+# Example:
+# 2018-09-12 11:45:58,727 fail2ban.actions[25029]: WARNING [ssh] Found 203.0.113.1
+# 2018-09-12 11:45:58,727 fail2ban.actions[25029]: WARNING [ssh] Ban 203.0.113.1
+# 2018-09-12 11:45:58,727 fail2ban.actions[25029]: WARNING [ssh] Restore Ban 203.0.113.1
+# 2018-09-12 11:45:53,715 fail2ban.actions[25029]: WARNING [ssh] Unban 203.0.113.1
+RE_DATA = re.compile(
+ r'\[(?P<jail>[A-Za-z-_0-9]+)\] (?P<action>{0}|{1}|{2}|{3}) (?P<ip>[a-f0-9.:]+)'.format(
+ ACTION_BAN, ACTION_UNBAN, ACTION_RESTORE_BAN, ACTION_FOUND
+ )
+)
+
+DEFAULT_JAILS = [
+ 'ssh',
+]
+
+
+class Service(LogService):
+ def __init__(self, configuration=None, name=None):
+ LogService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = dict()
+ self.log_path = self.configuration.get('log_path', '/var/log/fail2ban.log')
+ self.conf_path = self.configuration.get('conf_path', '/etc/fail2ban/jail.local')
+ self.conf_dir = self.configuration.get('conf_dir', '/etc/fail2ban/jail.d/')
+ self.exclude = self.configuration.get('exclude', str())
+ self.monitoring_jails = list()
+ self.banned_ips = defaultdict(set)
+ self.data = dict()
+
+ def check(self):
+ """
+ :return: bool
+ """
+ if not self.conf_path.endswith(('.conf', '.local')):
+ self.error('{0} is a wrong conf path name, must be *.conf or *.local'.format(self.conf_path))
+ return False
+
+ if not os.access(self.log_path, os.R_OK):
+ self.error('{0} is not readable'.format(self.log_path))
+ return False
+
+ if os.path.getsize(self.log_path) == 0:
+ self.error('{0} is empty'.format(self.log_path))
+ return False
+
+ self.monitoring_jails = self.jails_auto_detection()
+ for jail in self.monitoring_jails:
+ self.data['{0}_failed_attempts'.format(jail)] = 0
+ self.data[jail] = 0
+ self.data['{0}_in_jail'.format(jail)] = 0
+
+ self.definitions = charts(self.monitoring_jails)
+ self.info('monitoring jails: {0}'.format(self.monitoring_jails))
+
+ return True
+
+ def get_data(self):
+ """
+ :return: dict
+ """
+ raw = self._get_raw_data()
+
+ if not raw:
+ return None if raw is None else self.data
+
+ for row in raw:
+ match = RE_DATA.search(row)
+
+ if not match:
+ continue
+
+ match = match.groupdict()
+
+ if match['jail'] not in self.monitoring_jails:
+ continue
+
+ jail, action, ip = match['jail'], match['action'], match['ip']
+
+ if action == ACTION_FOUND:
+ self.data['{0}_failed_attempts'.format(jail)] += 1
+ elif action in (ACTION_BAN, ACTION_RESTORE_BAN):
+ self.data[jail] += 1
+ if ip not in self.banned_ips[jail]:
+ self.banned_ips[jail].add(ip)
+ self.data['{0}_in_jail'.format(jail)] += 1
+ elif action == ACTION_UNBAN:
+ if ip in self.banned_ips[jail]:
+ self.banned_ips[jail].remove(ip)
+ self.data['{0}_in_jail'.format(jail)] -= 1
+
+ return self.data
+
+ def get_files_from_dir(self, dir_path, suffix):
+ """
+ :return: list
+ """
+ if not os.path.isdir(dir_path):
+ self.error('{0} is not a directory'.format(dir_path))
+ return list()
+
+ return glob('{0}/*.{1}'.format(self.conf_dir, suffix))
+
+ def get_jails_from_file(self, file_path):
+ """
+ :return: list
+ """
+ if not os.access(file_path, os.R_OK):
+ self.error('{0} is not readable or not exist'.format(file_path))
+ return list()
+
+ with open(file_path, 'rt') as f:
+ lines = f.readlines()
+ raw = ' '.join(line for line in lines if line.startswith(('[', 'enabled')))
+
+ match = RE_JAILS.findall(raw)
+ # Result: [('ssh', 'true'), ('dropbear', 'true'), ('pam-generic', 'true'), ...]
+
+ if not match:
+ self.debug('{0} parse failed'.format(file_path))
+ return list()
+
+ return match
+
+ def jails_auto_detection(self):
+ """
+ :return: list
+
+ Parses jail configuration files. Returns list of enabled jails.
+ According man jail.conf parse order must be
+ * jail.conf
+ * jail.d/*.conf (in alphabetical order)
+ * jail.local
+ * jail.d/*.local (in alphabetical order)
+ """
+ jails_files, all_jails, active_jails = list(), list(), list()
+
+ jails_files.append('{0}.conf'.format(self.conf_path.rsplit('.')[0]))
+ jails_files.extend(self.get_files_from_dir(self.conf_dir, 'conf'))
+ jails_files.append('{0}.local'.format(self.conf_path.rsplit('.')[0]))
+ jails_files.extend(self.get_files_from_dir(self.conf_dir, 'local'))
+
+ self.debug('config files to parse: {0}'.format(jails_files))
+
+ for f in jails_files:
+ all_jails.extend(self.get_jails_from_file(f))
+
+ exclude = self.exclude.split()
+
+ for name, status in all_jails:
+ if name in exclude:
+ continue
+
+ if status in ('true', 'yes') and name not in active_jails:
+ active_jails.append(name)
+ elif status in ('false', 'no') and name in active_jails:
+ active_jails.remove(name)
+
+ return active_jails or DEFAULT_JAILS
diff --git a/collectors/python.d.plugin/fail2ban/fail2ban.conf b/collectors/python.d.plugin/fail2ban/fail2ban.conf
new file mode 100644
index 0000000..a36436b
--- /dev/null
+++ b/collectors/python.d.plugin/fail2ban/fail2ban.conf
@@ -0,0 +1,68 @@
+# netdata python.d.plugin configuration for fail2ban
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, fail2ban also supports the following:
+#
+# log_path: 'path to fail2ban.log' # Default: '/var/log/fail2ban.log'
+# conf_path: 'path to jail.local/jail.conf' # Default: '/etc/fail2ban/jail.local'
+# conf_dir: 'path to jail.d/' # Default: '/etc/fail2ban/jail.d/'
+# exclude: 'jails you want to exclude from autodetection' # Default: none
+#------------------------------------------------------------------------------------------------------------------
diff --git a/collectors/python.d.plugin/gearman/Makefile.inc b/collectors/python.d.plugin/gearman/Makefile.inc
new file mode 100644
index 0000000..275adf1
--- /dev/null
+++ b/collectors/python.d.plugin/gearman/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += gearman/gearman.chart.py
+dist_pythonconfig_DATA += gearman/gearman.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += gearman/README.md gearman/Makefile.inc
+
diff --git a/collectors/python.d.plugin/gearman/README.md b/collectors/python.d.plugin/gearman/README.md
new file mode 100644
index 0000000..34ea584
--- /dev/null
+++ b/collectors/python.d.plugin/gearman/README.md
@@ -0,0 +1,50 @@
+<!--
+title: "Gearman monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/gearman/README.md
+sidebar_label: "Gearman"
+-->
+
+# Gearman monitoring with Netdata
+
+Monitors Gearman worker statistics. A chart is shown for each job as well as one showing a summary of all workers.
+
+Note: Charts may show as a line graph rather than an area
+graph if you load Netdata with no jobs running. To change
+this go to "Settings" > "Which dimensions to show?" and
+select "All".
+
+Plugin can obtain data from tcp socket **OR** unix socket.
+
+**Requirement:**
+Socket MUST be readable by netdata user.
+
+It produces:
+
+ * Workers queued
+ * Workers idle
+ * Workers running
+
+## Configuration
+
+Edit the `python.d/gearman.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/gearman.conf
+```
+
+```yaml
+localhost:
+ name : 'local'
+ host : 'localhost'
+ port : 4730
+
+ # TLS information can be provided as well
+ tls : no
+ cert : /path/to/cert
+ key : /path/to/key
+```
+
+When no configuration file is found, module tries to connect to TCP/IP socket: `localhost:4730`.
+
diff --git a/collectors/python.d.plugin/gearman/gearman.chart.py b/collectors/python.d.plugin/gearman/gearman.chart.py
new file mode 100644
index 0000000..5e280a4
--- /dev/null
+++ b/collectors/python.d.plugin/gearman/gearman.chart.py
@@ -0,0 +1,243 @@
+# Description: dovecot netdata python.d module
+# Author: Kyle Agronick (agronick)
+# SPDX-License-Identifier: GPL-3.0+
+
+# Gearman Netdata Plugin
+
+from copy import deepcopy
+
+from bases.FrameworkServices.SocketService import SocketService
+
+CHARTS = {
+ 'total_workers': {
+ 'options': [None, 'Total Jobs', 'Jobs', 'Total Jobs', 'gearman.total_jobs', 'line'],
+ 'lines': [
+ ['total_pending', 'Pending', 'absolute'],
+ ['total_running', 'Running', 'absolute'],
+ ]
+ },
+}
+
+
+def job_chart_template(job_name):
+ return {
+ 'options': [None, job_name, 'Jobs', 'Activity by Job', 'gearman.single_job', 'stacked'],
+ 'lines': [
+ ['{0}_pending'.format(job_name), 'Pending', 'absolute'],
+ ['{0}_idle'.format(job_name), 'Idle', 'absolute'],
+ ['{0}_running'.format(job_name), 'Running', 'absolute'],
+ ]
+ }
+
+
+def build_result_dict(job):
+ """
+ Get the status for each job
+ :return: dict
+ """
+
+ total, running, available = job['metrics']
+
+ idle = available - running
+ pending = total - running
+
+ return {
+ '{0}_pending'.format(job['job_name']): pending,
+ '{0}_idle'.format(job['job_name']): idle,
+ '{0}_running'.format(job['job_name']): running,
+ }
+
+
+def parse_worker_data(job):
+ job_name = job[0]
+ job_metrics = job[1:]
+
+ return {
+ 'job_name': job_name,
+ 'metrics': job_metrics,
+ }
+
+
+class GearmanReadException(BaseException):
+ pass
+
+
+class Service(SocketService):
+ def __init__(self, configuration=None, name=None):
+ super(Service, self).__init__(configuration=configuration, name=name)
+ self.request = "status\n"
+ self._keep_alive = True
+
+ self.host = self.configuration.get('host', 'localhost')
+ self.port = self.configuration.get('port', 4730)
+
+ self.tls = self.configuration.get('tls', False)
+ self.cert = self.configuration.get('cert', None)
+ self.key = self.configuration.get('key', None)
+
+ self.active_jobs = set()
+ self.definitions = deepcopy(CHARTS)
+ self.order = ['total_workers']
+
+ def _get_data(self):
+ """
+ Format data received from socket
+ :return: dict
+ """
+
+ try:
+ active_jobs = self.get_active_jobs()
+ except GearmanReadException:
+ return None
+
+ found_jobs, job_data = self.process_jobs(active_jobs)
+ self.remove_stale_jobs(found_jobs)
+ return job_data
+
+ def get_active_jobs(self):
+ active_jobs = []
+
+ for job in self.get_worker_data():
+ parsed_job = parse_worker_data(job)
+
+ # Gearman does not clean up old jobs
+ # We only care about jobs that have
+ # some relevant data
+ if not any(parsed_job['metrics']):
+ continue
+
+ active_jobs.append(parsed_job)
+
+ return active_jobs
+
+ def get_worker_data(self):
+ """
+ Split the data returned from Gearman
+ into a list of lists
+
+ This returns the same output that you
+ would get from a gearadmin --status
+ command.
+
+ Example output returned from
+ _get_raw_data():
+ prefix generic_worker4 78 78 500
+ generic_worker2 78 78 500
+ generic_worker3 0 0 760
+ generic_worker1 0 0 500
+
+ :return: list
+ """
+
+ try:
+ raw = self._get_raw_data()
+ except (ValueError, AttributeError):
+ raise GearmanReadException()
+
+ if raw is None:
+ self.debug("Gearman returned no data")
+ raise GearmanReadException()
+
+ workers = list()
+
+ for line in raw.splitlines()[:-1]:
+ parts = line.split()
+ if not parts:
+ continue
+
+ name = '_'.join(parts[:-3])
+ try:
+ values = [int(w) for w in parts[-3:]]
+ except ValueError:
+ continue
+
+ w = [name]
+ w.extend(values)
+ workers.append(w)
+
+ return workers
+
+ def process_jobs(self, active_jobs):
+
+ output = {
+ 'total_pending': 0,
+ 'total_idle': 0,
+ 'total_running': 0,
+ }
+ found_jobs = set()
+
+ for parsed_job in active_jobs:
+
+ job_name = self.add_job(parsed_job)
+ found_jobs.add(job_name)
+ job_data = build_result_dict(parsed_job)
+
+ for sum_value in ('pending', 'running', 'idle'):
+ output['total_{0}'.format(sum_value)] += job_data['{0}_{1}'.format(job_name, sum_value)]
+
+ output.update(job_data)
+
+ return found_jobs, output
+
+ def remove_stale_jobs(self, active_job_list):
+ """
+ Removes jobs that have no workers, pending jobs,
+ or running jobs
+ :param active_job_list: The latest list of active jobs
+ :type active_job_list: iterable
+ :return: None
+ """
+
+ for to_remove in self.active_jobs - active_job_list:
+ self.remove_job(to_remove)
+
+ def add_job(self, parsed_job):
+ """
+ Adds a job to the list of active jobs
+ :param parsed_job: A parsed job dict
+ :type parsed_job: dict
+ :return: None
+ """
+
+ def add_chart(job_name):
+ """
+ Adds a new job chart
+ :param job_name: The name of the job to add
+ :type job_name: string
+ :return: None
+ """
+
+ job_key = 'job_{0}'.format(job_name)
+ template = job_chart_template(job_name)
+ new_chart = self.charts.add_chart([job_key] + template['options'])
+ for dimension in template['lines']:
+ new_chart.add_dimension(dimension)
+
+ if parsed_job['job_name'] not in self.active_jobs:
+ add_chart(parsed_job['job_name'])
+ self.active_jobs.add(parsed_job['job_name'])
+
+ return parsed_job['job_name']
+
+ def remove_job(self, job_name):
+ """
+ Removes a job to the list of active jobs
+ :param job_name: The name of the job to remove
+ :type job_name: string
+ :return: None
+ """
+
+ def remove_chart(job_name):
+ """
+ Removes a job chart
+ :param job_name: The name of the job to remove
+ :type job_name: string
+ :return: None
+ """
+
+ job_key = 'job_{0}'.format(job_name)
+ self.charts[job_key].obsolete()
+ del self.charts[job_key]
+
+ remove_chart(job_name)
+ self.active_jobs.remove(job_name)
diff --git a/collectors/python.d.plugin/gearman/gearman.conf b/collectors/python.d.plugin/gearman/gearman.conf
new file mode 100644
index 0000000..c41fd9f
--- /dev/null
+++ b/collectors/python.d.plugin/gearman/gearman.conf
@@ -0,0 +1,72 @@
+# netdata python.d.plugin configuration for gearman
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, gearman also supports the following:
+#
+# hostname: localhost # The host running the Gearman server
+# port: 4730 # Port of the Gearman server
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOB
+
+localhost:
+ name : 'local'
+ host : 'localhost'
+ port : 4730 \ No newline at end of file
diff --git a/collectors/python.d.plugin/go_expvar/Makefile.inc b/collectors/python.d.plugin/go_expvar/Makefile.inc
new file mode 100644
index 0000000..74f50d7
--- /dev/null
+++ b/collectors/python.d.plugin/go_expvar/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += go_expvar/go_expvar.chart.py
+dist_pythonconfig_DATA += go_expvar/go_expvar.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += go_expvar/README.md go_expvar/Makefile.inc
+
diff --git a/collectors/python.d.plugin/go_expvar/README.md b/collectors/python.d.plugin/go_expvar/README.md
new file mode 100644
index 0000000..feb150d
--- /dev/null
+++ b/collectors/python.d.plugin/go_expvar/README.md
@@ -0,0 +1,319 @@
+<!--
+title: "Go applications monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/go_expvar/README.md
+sidebar_label: "Go applications"
+-->
+
+# Go applications monitoring with Netdata
+
+Monitors Go application that exposes its metrics with the use of `expvar` package from the Go standard library. The package produces charts for Go runtime memory statistics and optionally any number of custom charts.
+
+The `go_expvar` module produces the following charts:
+
+1. **Heap allocations** in kB
+
+ - alloc: size of objects allocated on the heap
+ - inuse: size of allocated heap spans
+
+2. **Stack allocations** in kB
+
+ - inuse: size of allocated stack spans
+
+3. **MSpan allocations** in kB
+
+ - inuse: size of allocated mspan structures
+
+4. **MCache allocations** in kB
+
+ - inuse: size of allocated mcache structures
+
+5. **Virtual memory** in kB
+
+ - sys: size of reserved virtual address space
+
+6. **Live objects**
+
+ - live: number of live objects in memory
+
+7. **GC pauses average** in ns
+
+ - avg: average duration of all GC stop-the-world pauses
+
+## Monitoring Go applications
+
+Netdata can be used to monitor running Go applications that expose their metrics with
+the use of the [expvar package](https://golang.org/pkg/expvar/) included in Go standard library.
+
+The `expvar` package exposes these metrics over HTTP and is very easy to use.
+Consider this minimal sample below:
+
+```go
+package main
+
+import (
+ _ "expvar"
+ "net/http"
+)
+
+func main() {
+ http.ListenAndServe("127.0.0.1:8080", nil)
+}
+```
+
+When imported this way, the `expvar` package registers a HTTP handler at `/debug/vars` that
+exposes Go runtime's memory statistics in JSON format. You can inspect the output by opening
+the URL in your browser (or by using `wget` or `curl`).
+
+Sample output:
+
+```json
+{
+"cmdline": ["./expvar-demo-binary"],
+"memstats": {"Alloc":630856,"TotalAlloc":630856,"Sys":3346432,"Lookups":27, <omitted for brevity>}
+}
+```
+
+You can of course expose and monitor your own variables as well.
+Here is a sample Go application that exposes a few custom variables:
+
+```go
+package main
+
+import (
+ "expvar"
+ "net/http"
+ "runtime"
+ "time"
+)
+
+func main() {
+
+ tick := time.NewTicker(1 * time.Second)
+ num_go := expvar.NewInt("runtime.goroutines")
+ counters := expvar.NewMap("counters")
+ counters.Set("cnt1", new(expvar.Int))
+ counters.Set("cnt2", new(expvar.Float))
+
+ go http.ListenAndServe(":8080", nil)
+
+ for {
+ select {
+ case <- tick.C:
+ num_go.Set(int64(runtime.NumGoroutine()))
+ counters.Add("cnt1", 1)
+ counters.AddFloat("cnt2", 1.452)
+ }
+ }
+}
+```
+
+Apart from the runtime memory stats, this application publishes two counters and the
+number of currently running Goroutines and updates these stats every second.
+
+In the next section, we will cover how to monitor and chart these exposed stats with
+the use of `netdata`s `go_expvar` module.
+
+### Using Netdata go_expvar module
+
+The `go_expvar` module is disabled by default. To enable it, edit `python.d.conf` (to edit it on your system run
+`/etc/netdata/edit-config python.d.conf`), and change the `go_expvar` variable to `yes`:
+
+```
+# Enable / Disable python.d.plugin modules
+#default_run: yes
+#
+# If "default_run" = "yes" the default for all modules is enabled (yes).
+# Setting any of these to "no" will disable it.
+#
+# If "default_run" = "no" the default for all modules is disabled (no).
+# Setting any of these to "yes" will enable it.
+...
+go_expvar: yes
+...
+```
+
+Next, we need to edit the module configuration file (found at `/etc/netdata/python.d/go_expvar.conf` by default) (to
+edit it on your system run `/etc/netdata/edit-config python.d/go_expvar.conf`). The module configuration consists of
+jobs, where each job can be used to monitor a separate Go application. Let's see a sample job configuration:
+
+```
+# /etc/netdata/python.d/go_expvar.conf
+
+app1:
+ name : 'app1'
+ url : 'http://127.0.0.1:8080/debug/vars'
+ collect_memstats: true
+ extra_charts: {}
+```
+
+Let's go over each of the defined options:
+
+```
+name: 'app1'
+```
+
+This is the job name that will appear at the Netdata dashboard.
+If not defined, the job_name (top level key) will be used.
+
+```
+url: 'http://127.0.0.1:8080/debug/vars'
+```
+
+This is the URL of the expvar endpoint. As the expvar handler can be installed
+in a custom path, the whole URL has to be specified. This value is mandatory.
+
+```
+collect_memstats: true
+```
+
+Whether to enable collecting stats about Go runtime's memory. You can find more
+information about the exposed values at the [runtime package docs](https://golang.org/pkg/runtime/#MemStats).
+
+```
+extra_charts: {}
+```
+
+Enables the user to specify custom expvars to monitor and chart.
+Will be explained in more detail below.
+
+**Note: if `collect_memstats` is disabled and no `extra_charts` are defined, the plugin will
+disable itself, as there will be no data to collect!**
+
+Apart from these options, each job supports options inherited from Netdata's `python.d.plugin`
+and its base `UrlService` class. These are:
+
+```
+update_every: 1 # the job's data collection frequency
+priority: 60000 # the job's order on the dashboard
+user: admin # use when the expvar endpoint is protected by HTTP Basic Auth
+password: sekret # use when the expvar endpoint is protected by HTTP Basic Auth
+```
+
+### Monitoring custom vars with go_expvar
+
+Now, memory stats might be useful, but what if you want Netdata to monitor some custom values
+that your Go application exposes? The `go_expvar` module can do that as well with the use of
+the `extra_charts` configuration variable.
+
+The `extra_charts` variable is a YaML list of Netdata chart definitions.
+Each chart definition has the following keys:
+
+```
+id: Netdata chart ID
+options: a key-value mapping of chart options
+lines: a list of line definitions
+```
+
+**Note: please do not use dots in the chart or line ID field.
+See [this issue](https://github.com/netdata/netdata/pull/1902#issuecomment-284494195) for explanation.**
+
+Please see these two links to the official Netdata documentation for more information about the values:
+
+- [External plugins - charts](/collectors/plugins.d/README.md#chart)
+- [Chart variables](/collectors/python.d.plugin/README.md#global-variables-order-and-chart)
+
+**Line definitions**
+
+Each chart can define multiple lines (dimensions).
+A line definition is a key-value mapping of line options.
+Each line can have the following options:
+
+```
+# mandatory
+expvar_key: the name of the expvar as present in the JSON output of /debug/vars endpoint
+expvar_type: value type; supported are "float" or "int"
+id: the id of this line/dimension in Netdata
+
+# optional - Netdata defaults are used if these options are not defined
+name: ''
+algorithm: absolute
+multiplier: 1
+divisor: 100 if expvar_type == float, 1 if expvar_type == int
+hidden: False
+```
+
+Please see the following link for more information about the options and their default values:
+[External plugins - dimensions](/collectors/plugins.d/README.md#dimension)
+
+Apart from top-level expvars, this plugin can also parse expvars stored in a multi-level map;
+All dicts in the resulting JSON document are then flattened to one level.
+Expvar names are joined together with '.' when flattening.
+
+Example:
+
+```
+{
+ "counters": {"cnt1": 1042, "cnt2": 1512.9839999999983},
+ "runtime.goroutines": 5
+}
+```
+
+In the above case, the exported variables will be available under `runtime.goroutines`,
+`counters.cnt1` and `counters.cnt2` expvar_keys. If the flattening results in a key collision,
+the first defined key wins and all subsequent keys with the same name are ignored.
+
+## Enable the collector
+
+The `go_expvar` collector is disabled by default. To enable it, use `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`, to edit the `python.d.conf` file.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d.conf
+```
+
+Change the value of the `go_expvar` setting to `yes`. Save the file and restart the Netdata Agent with `sudo systemctl
+restart netdata`, or the appropriate method for your system, to finish enabling the `go_expvar` collector.
+
+## Configuration
+
+Edit the `python.d/go_expvar.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/go_expvar.conf
+```
+
+The configuration below matches the second Go application described above.
+Netdata will monitor and chart memory stats for the application, as well as a custom chart of
+running goroutines and two dummy counters.
+
+```
+app1:
+ name : 'app1'
+ url : 'http://127.0.0.1:8080/debug/vars'
+ collect_memstats: true
+ extra_charts:
+ - id: "runtime_goroutines"
+ options:
+ name: num_goroutines
+ title: "runtime: number of goroutines"
+ units: goroutines
+ family: runtime
+ context: expvar.runtime.goroutines
+ chart_type: line
+ lines:
+ - {expvar_key: 'runtime.goroutines', expvar_type: int, id: runtime_goroutines}
+ - id: "foo_counters"
+ options:
+ name: counters
+ title: "some random counters"
+ units: awesomeness
+ family: counters
+ context: expvar.foo.counters
+ chart_type: line
+ lines:
+ - {expvar_key: 'counters.cnt1', expvar_type: int, id: counters_cnt1}
+ - {expvar_key: 'counters.cnt2', expvar_type: float, id: counters_cnt2}
+```
+
+**Netdata charts example**
+
+The images below show how do the final charts in Netdata look.
+
+![Memory stats charts](https://cloud.githubusercontent.com/assets/15180106/26762052/62b4af58-493b-11e7-9e69-146705acfc2c.png)
+
+![Custom charts](https://cloud.githubusercontent.com/assets/15180106/26762051/62ae915e-493b-11e7-8518-bd25a3886650.png)
+
+
diff --git a/collectors/python.d.plugin/go_expvar/go_expvar.chart.py b/collectors/python.d.plugin/go_expvar/go_expvar.chart.py
new file mode 100644
index 0000000..dca0108
--- /dev/null
+++ b/collectors/python.d.plugin/go_expvar/go_expvar.chart.py
@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+# Description: go_expvar netdata python.d module
+# Author: Jan Kral (kralewitz)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import division
+
+import json
+from collections import namedtuple
+
+from bases.FrameworkServices.UrlService import UrlService
+
+MEMSTATS_ORDER = [
+ 'memstats_heap',
+ 'memstats_stack',
+ 'memstats_mspan',
+ 'memstats_mcache',
+ 'memstats_sys',
+ 'memstats_live_objects',
+ 'memstats_gc_pauses',
+]
+
+MEMSTATS_CHARTS = {
+ 'memstats_heap': {
+ 'options': ['heap', 'memory: size of heap memory structures', 'KiB', 'memstats',
+ 'expvar.memstats.heap', 'line'],
+ 'lines': [
+ ['memstats_heap_alloc', 'alloc', 'absolute', 1, 1024],
+ ['memstats_heap_inuse', 'inuse', 'absolute', 1, 1024]
+ ]
+ },
+ 'memstats_stack': {
+ 'options': ['stack', 'memory: size of stack memory structures', 'KiB', 'memstats',
+ 'expvar.memstats.stack', 'line'],
+ 'lines': [
+ ['memstats_stack_inuse', 'inuse', 'absolute', 1, 1024]
+ ]
+ },
+ 'memstats_mspan': {
+ 'options': ['mspan', 'memory: size of mspan memory structures', 'KiB', 'memstats',
+ 'expvar.memstats.mspan', 'line'],
+ 'lines': [
+ ['memstats_mspan_inuse', 'inuse', 'absolute', 1, 1024]
+ ]
+ },
+ 'memstats_mcache': {
+ 'options': ['mcache', 'memory: size of mcache memory structures', 'KiB', 'memstats',
+ 'expvar.memstats.mcache', 'line'],
+ 'lines': [
+ ['memstats_mcache_inuse', 'inuse', 'absolute', 1, 1024]
+ ]
+ },
+ 'memstats_live_objects': {
+ 'options': ['live_objects', 'memory: number of live objects', 'objects', 'memstats',
+ 'expvar.memstats.live_objects', 'line'],
+ 'lines': [
+ ['memstats_live_objects', 'live']
+ ]
+ },
+ 'memstats_sys': {
+ 'options': ['sys', 'memory: size of reserved virtual address space', 'KiB', 'memstats',
+ 'expvar.memstats.sys', 'line'],
+ 'lines': [
+ ['memstats_sys', 'sys', 'absolute', 1, 1024]
+ ]
+ },
+ 'memstats_gc_pauses': {
+ 'options': ['gc_pauses', 'memory: average duration of GC pauses', 'ns', 'memstats',
+ 'expvar.memstats.gc_pauses', 'line'],
+ 'lines': [
+ ['memstats_gc_pauses', 'avg']
+ ]
+ }
+}
+
+EXPVAR = namedtuple(
+ "EXPVAR",
+ [
+ "key",
+ "type",
+ "id",
+ ]
+)
+
+
+def flatten(d, top='', sep='.'):
+ items = []
+ for key, val in d.items():
+ nkey = top + sep + key if top else key
+ if isinstance(val, dict):
+ items.extend(flatten(val, nkey, sep=sep).items())
+ else:
+ items.append((nkey, val))
+ return dict(items)
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ # if memstats collection is enabled, add the charts and their order
+ if self.configuration.get('collect_memstats'):
+ self.definitions = dict(MEMSTATS_CHARTS)
+ self.order = list(MEMSTATS_ORDER)
+ else:
+ self.definitions = dict()
+ self.order = list()
+
+ # if extra charts are defined, parse their config
+ extra_charts = self.configuration.get('extra_charts')
+ if extra_charts:
+ self._parse_extra_charts_config(extra_charts)
+
+ def check(self):
+ """
+ Check if the module can collect data:
+ 1) At least one JOB configuration has to be specified
+ 2) The JOB configuration needs to define the URL and either collect_memstats must be enabled or at least one
+ extra_chart must be defined.
+
+ The configuration and URL check is provided by the UrlService class.
+ """
+
+ if not (self.configuration.get('extra_charts') or self.configuration.get('collect_memstats')):
+ self.error('Memstats collection is disabled and no extra_charts are defined, disabling module.')
+ return False
+
+ return UrlService.check(self)
+
+ def _parse_extra_charts_config(self, extra_charts_config):
+
+ # a place to store the expvar keys and their types
+ self.expvars = list()
+
+ for chart in extra_charts_config:
+
+ chart_dict = dict()
+ chart_id = chart.get('id')
+ chart_lines = chart.get('lines')
+ chart_opts = chart.get('options', dict())
+
+ if not all([chart_id, chart_lines]):
+ self.info('Chart {0} has no ID or no lines defined, skipping'.format(chart))
+ continue
+
+ chart_dict['options'] = [
+ chart_opts.get('name', ''),
+ chart_opts.get('title', ''),
+ chart_opts.get('units', ''),
+ chart_opts.get('family', ''),
+ chart_opts.get('context', ''),
+ chart_opts.get('chart_type', 'line')
+ ]
+ chart_dict['lines'] = list()
+
+ # add the lines to the chart
+ for line in chart_lines:
+
+ ev_key = line.get('expvar_key')
+ ev_type = line.get('expvar_type')
+ line_id = line.get('id')
+
+ if not all([ev_key, ev_type, line_id]):
+ self.info('Line missing expvar_key, expvar_type, or line_id, skipping: {0}'.format(line))
+ continue
+
+ if ev_type not in ['int', 'float']:
+ self.info('Unsupported expvar_type "{0}". Must be "int" or "float"'.format(ev_type))
+ continue
+
+ # self.expvars[ev_key] = (ev_type, line_id)
+ self.expvars.append(EXPVAR(ev_key, ev_type, line_id))
+
+ chart_dict['lines'].append(
+ [
+ line.get('id', ''),
+ line.get('name', ''),
+ line.get('algorithm', ''),
+ line.get('multiplier', 1),
+ line.get('divisor', 100 if ev_type == 'float' else 1),
+ line.get('hidden', False)
+ ]
+ )
+
+ self.order.append(chart_id)
+ self.definitions[chart_id] = chart_dict
+
+ def _get_data(self):
+ """
+ Format data received from http request
+ :return: dict
+ """
+
+ raw_data = self._get_raw_data()
+ if not raw_data:
+ return None
+
+ data = json.loads(raw_data)
+
+ expvars = dict()
+ if self.configuration.get('collect_memstats'):
+ expvars.update(self._parse_memstats(data))
+
+ if self.configuration.get('extra_charts'):
+ # the memstats part of the data has been already parsed, so we remove it before flattening and checking
+ # the rest of the data, thus avoiding needless iterating over the multiply nested memstats dict.
+ del (data['memstats'])
+ flattened = flatten(data)
+
+ for ev in self.expvars:
+ v = flattened.get(ev.key)
+
+ if v is None:
+ continue
+
+ try:
+ if ev.type == 'int':
+ expvars[ev.id] = int(v)
+ elif ev.type == 'float':
+ expvars[ev.id] = float(v) * 100
+ except ValueError:
+ self.info('Failed to parse value for key {0} as {1}, ignoring key.'.format(ev.key, ev.type))
+ return None
+
+ return expvars
+
+ @staticmethod
+ def _parse_memstats(data):
+
+ memstats = data['memstats']
+
+ # calculate the number of live objects in memory
+ live_objs = int(memstats['Mallocs']) - int(memstats['Frees'])
+
+ # calculate GC pause times average
+ # the Go runtime keeps the last 256 GC pause durations in a circular buffer,
+ # so we need to filter out the 0 values before the buffer is filled
+ gc_pauses = memstats['PauseNs']
+ try:
+ gc_pause_avg = sum(gc_pauses) / len([x for x in gc_pauses if x > 0])
+ # no GC cycles have occurred yet
+ except ZeroDivisionError:
+ gc_pause_avg = 0
+
+ return {
+ 'memstats_heap_alloc': memstats['HeapAlloc'],
+ 'memstats_heap_inuse': memstats['HeapInuse'],
+ 'memstats_stack_inuse': memstats['StackInuse'],
+ 'memstats_mspan_inuse': memstats['MSpanInuse'],
+ 'memstats_mcache_inuse': memstats['MCacheInuse'],
+ 'memstats_sys': memstats['Sys'],
+ 'memstats_live_objects': live_objs,
+ 'memstats_gc_pauses': gc_pause_avg,
+ }
diff --git a/collectors/python.d.plugin/go_expvar/go_expvar.conf b/collectors/python.d.plugin/go_expvar/go_expvar.conf
new file mode 100644
index 0000000..4b821cd
--- /dev/null
+++ b/collectors/python.d.plugin/go_expvar/go_expvar.conf
@@ -0,0 +1,108 @@
+# netdata python.d.plugin configuration for go_expvar
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, this plugin also supports the following:
+#
+# url: 'http://127.0.0.1/debug/vars' # the URL of the expvar endpoint
+#
+# As the plugin cannot possibly know the port your application listens on, there is no default value. Please include
+# the whole path of the endpoint, as the expvar handler can be installed in a non-standard location.
+#
+# if the URL is password protected, the following are supported:
+#
+# user: 'username'
+# pass: 'password'
+#
+# collect_memstats: true # enables charts for Go runtime's memory statistics
+# extra_charts: {} # defines extra data/charts to monitor, please see the example below
+#
+# If collect_memstats is disabled and no extra charts are defined, this module will disable itself, as it has no data to
+# collect.
+#
+# Please visit the module wiki page for more information on how to use the extra_charts variable:
+#
+# https://github.com/netdata/netdata/tree/master/collectors/python.d.plugin/go_expvar
+#
+# Configuration example
+# ---------------------
+
+#app1:
+# name : 'app1'
+# url : 'http://127.0.0.1:8080/debug/vars'
+# collect_memstats: true
+# extra_charts:
+# - id: "runtime_goroutines"
+# options:
+# name: num_goroutines
+# title: "runtime: number of goroutines"
+# units: goroutines
+# family: runtime
+# context: expvar.runtime.goroutines
+# chart_type: line
+# lines:
+# - {expvar_key: 'runtime.goroutines', expvar_type: int, id: runtime_goroutines}
+# - id: "foo_counters"
+# options:
+# name: counters
+# title: "some random counters"
+# units: awesomeness
+# family: counters
+# context: expvar.foo.counters
+# chart_type: line
+# lines:
+# - {expvar_key: 'counters.cnt1', expvar_type: int, id: counters_cnt1}
+# - {expvar_key: 'counters.cnt2', expvar_type: float, id: counters_cnt2}
+
diff --git a/collectors/python.d.plugin/haproxy/Makefile.inc b/collectors/python.d.plugin/haproxy/Makefile.inc
new file mode 100644
index 0000000..ad24dea
--- /dev/null
+++ b/collectors/python.d.plugin/haproxy/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += haproxy/haproxy.chart.py
+dist_pythonconfig_DATA += haproxy/haproxy.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += haproxy/README.md haproxy/Makefile.inc
+
diff --git a/collectors/python.d.plugin/haproxy/README.md b/collectors/python.d.plugin/haproxy/README.md
new file mode 100644
index 0000000..f16e725
--- /dev/null
+++ b/collectors/python.d.plugin/haproxy/README.md
@@ -0,0 +1,67 @@
+<!--
+title: "HAProxy monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/haproxy/README.md
+sidebar_label: "HAProxy"
+-->
+
+# HAProxy monitoring with Netdata
+
+Monitors frontend and backend metrics such as bytes in, bytes out, sessions current, sessions in queue current.
+And health metrics such as backend servers status (server check should be used).
+
+Plugin can obtain data from URL or Unix socket.
+
+Requirement:
+
+- Socket must be readable and writable by the `netdata` user.
+- URL must have `stats uri <path>` present in the haproxy config, otherwise you will get HTTP 503 in the haproxy logs.
+
+It produces:
+
+1. **Frontend** family charts
+
+ - Kilobytes in/s
+ - Kilobytes out/s
+ - Sessions current
+ - Sessions in queue current
+
+2. **Backend** family charts
+
+ - Kilobytes in/s
+ - Kilobytes out/s
+ - Sessions current
+ - Sessions in queue current
+
+3. **Health** chart
+
+ - number of failed servers for every backend (in DOWN state)
+
+## Configuration
+
+Edit the `python.d/haproxy.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/haproxy.conf
+```
+
+Sample:
+
+```yaml
+via_url:
+ user: 'username' # ONLY IF stats auth is used
+ pass: 'password' # # ONLY IF stats auth is used
+ url: 'http://ip.address:port/url;csv;norefresh'
+```
+
+OR
+
+```yaml
+via_socket:
+ socket: 'path/to/haproxy/sock'
+```
+
+If no configuration is given, module will fail to run.
+
+---
diff --git a/collectors/python.d.plugin/haproxy/haproxy.chart.py b/collectors/python.d.plugin/haproxy/haproxy.chart.py
new file mode 100644
index 0000000..6f94c9a
--- /dev/null
+++ b/collectors/python.d.plugin/haproxy/haproxy.chart.py
@@ -0,0 +1,360 @@
+# -*- coding: utf-8 -*-
+# Description: haproxy netdata python.d module
+# Author: ilyam8, ktarasz
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from collections import defaultdict
+from re import compile as re_compile
+
+try:
+ from urlparse import urlparse
+except ImportError:
+ from urllib.parse import urlparse
+
+from bases.FrameworkServices.SocketService import SocketService
+from bases.FrameworkServices.UrlService import UrlService
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = [
+ 'fbin',
+ 'fbout',
+ 'fscur',
+ 'fqcur',
+ 'fhrsp_1xx',
+ 'fhrsp_2xx',
+ 'fhrsp_3xx',
+ 'fhrsp_4xx',
+ 'fhrsp_5xx',
+ 'fhrsp_other',
+ 'fhrsp_total',
+ 'bbin',
+ 'bbout',
+ 'bscur',
+ 'bqcur',
+ 'bhrsp_1xx',
+ 'bhrsp_2xx',
+ 'bhrsp_3xx',
+ 'bhrsp_4xx',
+ 'bhrsp_5xx',
+ 'bhrsp_other',
+ 'bhrsp_total',
+ 'bqtime',
+ 'bttime',
+ 'brtime',
+ 'bctime',
+ 'health_sup',
+ 'health_sdown',
+ 'health_bdown',
+ 'health_idle'
+]
+
+CHARTS = {
+ 'fbin': {
+ 'options': [None, 'Kilobytes In', 'KiB/s', 'frontend', 'haproxy_f.bin', 'line'],
+ 'lines': []
+ },
+ 'fbout': {
+ 'options': [None, 'Kilobytes Out', 'KiB/s', 'frontend', 'haproxy_f.bout', 'line'],
+ 'lines': []
+ },
+ 'fscur': {
+ 'options': [None, 'Sessions Active', 'sessions', 'frontend', 'haproxy_f.scur', 'line'],
+ 'lines': []
+ },
+ 'fqcur': {
+ 'options': [None, 'Session In Queue', 'sessions', 'frontend', 'haproxy_f.qcur', 'line'],
+ 'lines': []
+ },
+ 'fhrsp_1xx': {
+ 'options': [None, 'HTTP responses with 1xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_1xx', 'line'],
+ 'lines': []
+ },
+ 'fhrsp_2xx': {
+ 'options': [None, 'HTTP responses with 2xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_2xx', 'line'],
+ 'lines': []
+ },
+ 'fhrsp_3xx': {
+ 'options': [None, 'HTTP responses with 3xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_3xx', 'line'],
+ 'lines': []
+ },
+ 'fhrsp_4xx': {
+ 'options': [None, 'HTTP responses with 4xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_4xx', 'line'],
+ 'lines': []
+ },
+ 'fhrsp_5xx': {
+ 'options': [None, 'HTTP responses with 5xx code', 'responses/s', 'frontend', 'haproxy_f.hrsp_5xx', 'line'],
+ 'lines': []
+ },
+ 'fhrsp_other': {
+ 'options': [None, 'HTTP responses with other codes (protocol error)', 'responses/s', 'frontend',
+ 'haproxy_f.hrsp_other', 'line'],
+ 'lines': []
+ },
+ 'fhrsp_total': {
+ 'options': [None, 'HTTP responses', 'responses', 'frontend', 'haproxy_f.hrsp_total', 'line'],
+ 'lines': []
+ },
+ 'bbin': {
+ 'options': [None, 'Kilobytes In', 'KiB/s', 'backend', 'haproxy_b.bin', 'line'],
+ 'lines': []
+ },
+ 'bbout': {
+ 'options': [None, 'Kilobytes Out', 'KiB/s', 'backend', 'haproxy_b.bout', 'line'],
+ 'lines': []
+ },
+ 'bscur': {
+ 'options': [None, 'Sessions Active', 'sessions', 'backend', 'haproxy_b.scur', 'line'],
+ 'lines': []
+ },
+ 'bqcur': {
+ 'options': [None, 'Sessions In Queue', 'sessions', 'backend', 'haproxy_b.qcur', 'line'],
+ 'lines': []
+ },
+ 'bhrsp_1xx': {
+ 'options': [None, 'HTTP responses with 1xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_1xx', 'line'],
+ 'lines': []
+ },
+ 'bhrsp_2xx': {
+ 'options': [None, 'HTTP responses with 2xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_2xx', 'line'],
+ 'lines': []
+ },
+ 'bhrsp_3xx': {
+ 'options': [None, 'HTTP responses with 3xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_3xx', 'line'],
+ 'lines': []
+ },
+ 'bhrsp_4xx': {
+ 'options': [None, 'HTTP responses with 4xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_4xx', 'line'],
+ 'lines': []
+ },
+ 'bhrsp_5xx': {
+ 'options': [None, 'HTTP responses with 5xx code', 'responses/s', 'backend', 'haproxy_b.hrsp_5xx', 'line'],
+ 'lines': []
+ },
+ 'bhrsp_other': {
+ 'options': [None, 'HTTP responses with other codes (protocol error)', 'responses/s', 'backend',
+ 'haproxy_b.hrsp_other', 'line'],
+ 'lines': []
+ },
+ 'bhrsp_total': {
+ 'options': [None, 'HTTP responses (total)', 'responses/s', 'backend', 'haproxy_b.hrsp_total', 'line'],
+ 'lines': []
+ },
+ 'bqtime': {
+ 'options': [None, 'The average queue time over the 1024 last requests', 'milliseconds', 'backend',
+ 'haproxy_b.qtime', 'line'],
+ 'lines': []
+ },
+ 'bctime': {
+ 'options': [None, 'The average connect time over the 1024 last requests', 'milliseconds', 'backend',
+ 'haproxy_b.ctime', 'line'],
+ 'lines': []
+ },
+ 'brtime': {
+ 'options': [None, 'The average response time over the 1024 last requests', 'milliseconds', 'backend',
+ 'haproxy_b.rtime', 'line'],
+ 'lines': []
+ },
+ 'bttime': {
+ 'options': [None, 'The average total session time over the 1024 last requests', 'milliseconds', 'backend',
+ 'haproxy_b.ttime', 'line'],
+ 'lines': []
+ },
+ 'health_sdown': {
+ 'options': [None, 'Backend Servers In DOWN State', 'failed servers', 'health', 'haproxy_hs.down', 'line'],
+ 'lines': []
+ },
+ 'health_sup': {
+ 'options': [None, 'Backend Servers In UP State', 'health servers', 'health', 'haproxy_hs.up', 'line'],
+ 'lines': []
+ },
+ 'health_bdown': {
+ 'options': [None, 'Is Backend Failed?', 'boolean', 'health', 'haproxy_hb.down', 'line'],
+ 'lines': []
+ },
+ 'health_idle': {
+ 'options': [None, 'The Ratio Of Polling Time Vs Total Time', 'percentage', 'health', 'haproxy.idle', 'line'],
+ 'lines': [
+ ['idle', None, 'absolute']
+ ]
+ }
+}
+
+METRICS = {
+ 'bin': {'algorithm': 'incremental', 'divisor': 1024},
+ 'bout': {'algorithm': 'incremental', 'divisor': 1024},
+ 'scur': {'algorithm': 'absolute', 'divisor': 1},
+ 'qcur': {'algorithm': 'absolute', 'divisor': 1},
+ 'hrsp_1xx': {'algorithm': 'incremental', 'divisor': 1},
+ 'hrsp_2xx': {'algorithm': 'incremental', 'divisor': 1},
+ 'hrsp_3xx': {'algorithm': 'incremental', 'divisor': 1},
+ 'hrsp_4xx': {'algorithm': 'incremental', 'divisor': 1},
+ 'hrsp_5xx': {'algorithm': 'incremental', 'divisor': 1},
+ 'hrsp_other': {'algorithm': 'incremental', 'divisor': 1}
+}
+
+BACKEND_METRICS = {
+ 'qtime': {'algorithm': 'absolute', 'divisor': 1},
+ 'ctime': {'algorithm': 'absolute', 'divisor': 1},
+ 'rtime': {'algorithm': 'absolute', 'divisor': 1},
+ 'ttime': {'algorithm': 'absolute', 'divisor': 1}
+}
+
+REGEX = dict(url=re_compile(r'idle = (?P<idle>[0-9]+)'),
+ socket=re_compile(r'Idle_pct: (?P<idle>[0-9]+)'))
+
+
+# TODO: the code is unreadable
+class Service(UrlService, SocketService):
+ def __init__(self, configuration=None, name=None):
+ if 'socket' in configuration:
+ SocketService.__init__(self, configuration=configuration, name=name)
+ self.poll = SocketService
+ self.options_ = dict(regex=REGEX['socket'],
+ stat='show stat\n'.encode(),
+ info='show info\n'.encode())
+ else:
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.poll = UrlService
+ self.options_ = dict(regex=REGEX['url'],
+ stat=self.url,
+ info=url_remove_params(self.url))
+ self.order = ORDER
+ self.definitions = CHARTS
+
+ def check(self):
+ if self.poll.check(self):
+ self.create_charts()
+ self.info('We are using %s.' % self.poll.__name__)
+ return True
+ return False
+
+ def _get_data(self):
+ to_netdata = dict()
+ self.request, self.url = self.options_['stat'], self.options_['stat']
+ stat_data = self._get_stat_data()
+ self.request, self.url = self.options_['info'], self.options_['info']
+ info_data = self._get_info_data(regex=self.options_['regex'])
+
+ to_netdata.update(stat_data)
+ to_netdata.update(info_data)
+ return to_netdata or None
+
+ def _get_stat_data(self):
+ """
+ :return: dict
+ """
+ raw_data = self.poll._get_raw_data(self)
+
+ if not raw_data:
+ return dict()
+
+ raw_data = raw_data.splitlines()
+ self.data = parse_data_([dict(zip(raw_data[0].split(','), raw_data[_].split(',')))
+ for _ in range(1, len(raw_data))])
+ if not self.data:
+ return dict()
+
+ stat_data = dict()
+
+ for frontend in self.data['frontend']:
+ for metric in METRICS:
+ idx = frontend['# pxname'].replace('.', '_')
+ stat_data['_'.join(['frontend', metric, idx])] = frontend.get(metric) or 0
+
+ for backend in self.data['backend']:
+ name, idx = backend['# pxname'], backend['# pxname'].replace('.', '_')
+ stat_data['hsup_' + idx] = len([server for server in self.data['servers']
+ if server_status(server, name, 'UP')])
+ stat_data['hsdown_' + idx] = len([server for server in self.data['servers']
+ if server_status(server, name, 'DOWN')])
+ stat_data['hbdown_' + idx] = 1 if backend.get('status') == 'DOWN' else 0
+ for metric in BACKEND_METRICS:
+ stat_data['_'.join(['backend', metric, idx])] = backend.get(metric) or 0
+ hrsp_total = 0
+ for metric in METRICS:
+ stat_data['_'.join(['backend', metric, idx])] = backend.get(metric) or 0
+ if metric.startswith('hrsp_'):
+ hrsp_total += int(backend.get(metric) or 0)
+ stat_data['_'.join(['backend', 'hrsp_total', idx])] = hrsp_total
+ return stat_data
+
+ def _get_info_data(self, regex):
+ """
+ :return: dict
+ """
+ raw_data = self.poll._get_raw_data(self)
+ if not raw_data:
+ return dict()
+
+ match = regex.search(raw_data)
+ return match.groupdict() if match else dict()
+
+ @staticmethod
+ def _check_raw_data(data):
+ """
+ Check if all data has been gathered from socket
+ :param data: str
+ :return: boolean
+ """
+ return not bool(data)
+
+ def create_charts(self):
+ for front in self.data['frontend']:
+ name, idx = front['# pxname'], front['# pxname'].replace('.', '_')
+ for metric in METRICS:
+ self.definitions['f' + metric]['lines'].append(['_'.join(['frontend', metric, idx]),
+ name, METRICS[metric]['algorithm'], 1,
+ METRICS[metric]['divisor']])
+ self.definitions['fhrsp_total']['lines'].append(['_'.join(['frontend', 'hrsp_total', idx]),
+ name, 'incremental', 1, 1])
+ for back in self.data['backend']:
+ name, idx = back['# pxname'], back['# pxname'].replace('.', '_')
+ for metric in METRICS:
+ self.definitions['b' + metric]['lines'].append(['_'.join(['backend', metric, idx]),
+ name, METRICS[metric]['algorithm'], 1,
+ METRICS[metric]['divisor']])
+ self.definitions['bhrsp_total']['lines'].append(['_'.join(['backend', 'hrsp_total', idx]),
+ name, 'incremental', 1, 1])
+ for metric in BACKEND_METRICS:
+ self.definitions['b' + metric]['lines'].append(['_'.join(['backend', metric, idx]),
+ name, BACKEND_METRICS[metric]['algorithm'], 1,
+ BACKEND_METRICS[metric]['divisor']])
+ self.definitions['health_sup']['lines'].append(['hsup_' + idx, name, 'absolute'])
+ self.definitions['health_sdown']['lines'].append(['hsdown_' + idx, name, 'absolute'])
+ self.definitions['health_bdown']['lines'].append(['hbdown_' + idx, name, 'absolute'])
+
+
+def parse_data_(data):
+ def is_backend(backend):
+ return backend.get('svname') == 'BACKEND' and backend.get('# pxname') != 'stats'
+
+ def is_frontend(frontend):
+ return frontend.get('svname') == 'FRONTEND' and frontend.get('# pxname') != 'stats'
+
+ def is_server(server):
+ return not server.get('svname', '').startswith(('FRONTEND', 'BACKEND'))
+
+ if not data:
+ return None
+
+ result = defaultdict(list)
+ for elem in data:
+ if is_backend(elem):
+ result['backend'].append(elem)
+ continue
+ elif is_frontend(elem):
+ result['frontend'].append(elem)
+ continue
+ elif is_server(elem):
+ result['servers'].append(elem)
+
+ return result or None
+
+
+def server_status(server, backend_name, status='DOWN'):
+ return server.get('# pxname') == backend_name and server.get('status') == status
+
+
+def url_remove_params(url):
+ parsed = urlparse(url or str())
+ return '{scheme}://{netloc}{path}'.format(scheme=parsed.scheme, netloc=parsed.netloc, path=parsed.path)
diff --git a/collectors/python.d.plugin/haproxy/haproxy.conf b/collectors/python.d.plugin/haproxy/haproxy.conf
new file mode 100644
index 0000000..10a0df3
--- /dev/null
+++ b/collectors/python.d.plugin/haproxy/haproxy.conf
@@ -0,0 +1,83 @@
+# netdata python.d.plugin configuration for haproxy
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, haproxy also supports the following:
+#
+# IMPORTANT: socket MUST BE readable AND writable by netdata user
+#
+# socket: 'path/to/haproxy/sock'
+#
+# OR
+# url: 'http://<ip.address>:<port>/<url>;csv;norefresh'
+# [user: USERNAME] only if stats auth is used
+# [pass: PASSWORD] only if stats auth is used
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+#via_url:
+# user : 'admin'
+# pass : 'password'
+# url : 'http://127.0.0.1:7000/haproxy_stats;csv;norefresh'
+
+#via_socket:
+# socket: '/var/run/haproxy/admin.sock'
diff --git a/collectors/python.d.plugin/hddtemp/Makefile.inc b/collectors/python.d.plugin/hddtemp/Makefile.inc
new file mode 100644
index 0000000..22852b6
--- /dev/null
+++ b/collectors/python.d.plugin/hddtemp/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += hddtemp/hddtemp.chart.py
+dist_pythonconfig_DATA += hddtemp/hddtemp.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += hddtemp/README.md hddtemp/Makefile.inc
+
diff --git a/collectors/python.d.plugin/hddtemp/README.md b/collectors/python.d.plugin/hddtemp/README.md
new file mode 100644
index 0000000..d8aba62
--- /dev/null
+++ b/collectors/python.d.plugin/hddtemp/README.md
@@ -0,0 +1,38 @@
+<!--
+title: "Hard drive temperature monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/hddtemp/README.md
+sidebar_label: "Hard drive temperature"
+-->
+
+# Hard drive temperature monitoring with Netdata
+
+Monitors disk temperatures from one or more `hddtemp` daemons.
+
+**Requirement:**
+Running `hddtemp` in daemonized mode with access on tcp port
+
+It produces one chart **Temperature** with dynamic number of dimensions (one per disk)
+
+## Configuration
+
+Edit the `python.d/hddtemp.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/hddtemp.conf
+```
+
+Sample:
+
+```yaml
+update_every: 3
+host: "127.0.0.1"
+port: 7634
+```
+
+If no configuration is given, module will attempt to connect to hddtemp daemon on `127.0.0.1:7634` address
+
+---
+
+
diff --git a/collectors/python.d.plugin/hddtemp/hddtemp.chart.py b/collectors/python.d.plugin/hddtemp/hddtemp.chart.py
new file mode 100644
index 0000000..6427aa1
--- /dev/null
+++ b/collectors/python.d.plugin/hddtemp/hddtemp.chart.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+# Description: hddtemp netdata python.d module
+# Author: Pawel Krupa (paulfantom)
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+
+import re
+from copy import deepcopy
+
+from bases.FrameworkServices.SocketService import SocketService
+
+ORDER = [
+ 'temperatures',
+]
+
+CHARTS = {
+ 'temperatures': {
+ 'options': ['disks_temp', 'Disks Temperatures', 'Celsius', 'temperatures', 'hddtemp.temperatures', 'line'],
+ 'lines': [
+ # lines are created dynamically in `check()` method
+ ]}}
+
+RE = re.compile(r'\/dev\/([^|]+)\|([^|]+)\|([0-9]+|SLP|UNK)\|')
+
+
+class Disk:
+ def __init__(self, id_, name, temp):
+ self.id = id_.split('/')[-1]
+ self.name = name.replace(' ', '_')
+ self.temp = temp if temp.isdigit() else None
+
+ def __repr__(self):
+ return self.id
+
+
+class Service(SocketService):
+ def __init__(self, configuration=None, name=None):
+ SocketService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = deepcopy(CHARTS)
+ self.do_only = self.configuration.get('devices')
+ self._keep_alive = False
+ self.request = ""
+ self.host = "127.0.0.1"
+ self.port = 7634
+
+ def get_disks(self):
+ r = self._get_raw_data()
+
+ if not r:
+ return None
+
+ m = RE.findall(r)
+
+ if not m:
+ self.error("received data doesn't have needed records")
+ return None
+
+ rv = [Disk(*d) for d in m]
+ self.debug('available disks: {0}'.format(rv))
+
+ if self.do_only:
+ return [v for v in rv if v.id in self.do_only]
+ return rv
+
+ def get_data(self):
+ """
+ Get data from TCP/IP socket
+ :return: dict
+ """
+
+ disks = self.get_disks()
+
+ if not disks:
+ return None
+
+ return dict((d.id, d.temp) for d in disks)
+
+ def check(self):
+ """
+ Parse configuration, check if hddtemp is available, and dynamically create chart lines data
+ :return: boolean
+ """
+ self._parse_config()
+ disks = self.get_disks()
+
+ if not disks:
+ return False
+
+ for d in disks:
+ dim = [d.id]
+ self.definitions['temperatures']['lines'].append(dim)
+
+ return True
+
+ @staticmethod
+ def _check_raw_data(data):
+ return not bool(data)
diff --git a/collectors/python.d.plugin/hddtemp/hddtemp.conf b/collectors/python.d.plugin/hddtemp/hddtemp.conf
new file mode 100644
index 0000000..b2d7aef
--- /dev/null
+++ b/collectors/python.d.plugin/hddtemp/hddtemp.conf
@@ -0,0 +1,95 @@
+# netdata python.d.plugin configuration for hddtemp
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, hddtemp also supports the following:
+#
+# host: 'IP or HOSTNAME' # the host to connect to
+# port: PORT # the port to connect to
+#
+
+# By default this module will try to autodetect disks
+# (autodetection works only for disk which names start with "sd").
+# However this can be overridden by setting variable `disks` to
+# array of desired disks. Example for two disks:
+#
+# devices:
+# - sda
+# - sdb
+#
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name: 'local'
+ host: 'localhost'
+ port: 7634
+
+localipv4:
+ name: 'local'
+ host: '127.0.0.1'
+ port: 7634
+
+localipv6:
+ name: 'local'
+ host: '::1'
+ port: 7634
diff --git a/collectors/python.d.plugin/hpssa/Makefile.inc b/collectors/python.d.plugin/hpssa/Makefile.inc
new file mode 100644
index 0000000..1c04aa4
--- /dev/null
+++ b/collectors/python.d.plugin/hpssa/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += hpssa/hpssa.chart.py
+dist_pythonconfig_DATA += hpssa/hpssa.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += hpssa/README.md hpssa/Makefile.inc
+
diff --git a/collectors/python.d.plugin/hpssa/README.md b/collectors/python.d.plugin/hpssa/README.md
new file mode 100644
index 0000000..c1d2182
--- /dev/null
+++ b/collectors/python.d.plugin/hpssa/README.md
@@ -0,0 +1,83 @@
+<!--
+title: "HP Smart Storage Arrays monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/hpssa/README.md
+sidebar_label: "HP Smart Storage Arrays"
+-->
+
+# HP Smart Storage Arrays monitoring with Netdata
+
+Monitors controller, cache module, logical and physical drive state and temperature using `ssacli` tool.
+
+Executed commands:
+
+- `sudo -n ssacli ctrl all show config detail`
+
+## Requirements:
+
+This module uses `ssacli`, which can only be executed by root. It uses
+`sudo` and assumes that it is configured such that the `netdata` user can execute `ssacli` as root without a password.
+
+- Add to your `/etc/sudoers` file:
+
+`which ssacli` shows the full path to the binary.
+
+```bash
+netdata ALL=(root) NOPASSWD: /path/to/ssacli
+```
+
+- Reset Netdata's systemd
+ unit [CapabilityBoundingSet](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#Capabilities) (Linux
+ distributions with systemd)
+
+The default CapabilityBoundingSet doesn't allow using `sudo`, and is quite strict in general. Resetting is not optimal, but a next-best solution given the inability to execute `ssacli` using `sudo`.
+
+As the `root` user, do the following:
+
+```cmd
+mkdir /etc/systemd/system/netdata.service.d
+echo -e '[Service]\nCapabilityBoundingSet=~' | tee /etc/systemd/system/netdata.service.d/unset-capability-bounding-set.conf
+systemctl daemon-reload
+systemctl restart netdata.service
+```
+
+## Charts
+
+- Controller status
+- Controller temperature
+- Logical drive status
+- Physical drive status
+- Physical drive temperature
+
+## Enable the collector
+
+The `hpssa` collector is disabled by default. To enable it, use `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`, to edit the `python.d.conf`
+file.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d.conf
+```
+
+Change the value of the `hpssa` setting to `yes`. Save the file and restart the Netdata Agent with `sudo systemctl
+restart netdata`, or the [appropriate method](/docs/configure/start-stop-restart.md) for your system.
+
+## Configuration
+
+Edit the `python.d/hpssa.conf` configuration file using `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/hpssa.conf
+```
+
+If `ssacli` cannot be found in the `PATH`, configure it in `hpssa.conf`.
+
+```yaml
+ssacli_path: /usr/sbin/ssacli
+```
+
+Save the file and restart the Netdata Agent with `sudo systemctl restart netdata`, or the [appropriate
+method](/docs/configure/start-stop-restart.md) for your system.
+
diff --git a/collectors/python.d.plugin/hpssa/hpssa.chart.py b/collectors/python.d.plugin/hpssa/hpssa.chart.py
new file mode 100644
index 0000000..4da73dc
--- /dev/null
+++ b/collectors/python.d.plugin/hpssa/hpssa.chart.py
@@ -0,0 +1,396 @@
+# -*- coding: utf-8 -*-
+# Description: hpssa netdata python.d module
+# Author: Peter Gnodde (gnoddep)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+import re
+from copy import deepcopy
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+from bases.collection import find_binary
+
+disabled_by_default = True
+update_every = 5
+
+ORDER = [
+ 'ctrl_status',
+ 'ctrl_temperature',
+ 'ld_status',
+ 'pd_status',
+ 'pd_temperature',
+]
+
+CHARTS = {
+ 'ctrl_status': {
+ 'options': [
+ None,
+ 'Status 1 is OK, Status 0 is not OK',
+ 'Status',
+ 'Controller',
+ 'hpssa.ctrl_status',
+ 'line'
+ ],
+ 'lines': []
+ },
+ 'ctrl_temperature': {
+ 'options': [
+ None,
+ 'Temperature',
+ 'Celsius',
+ 'Controller',
+ 'hpssa.ctrl_temperature',
+ 'line'
+ ],
+ 'lines': []
+ },
+ 'ld_status': {
+ 'options': [
+ None,
+ 'Status 1 is OK, Status 0 is not OK',
+ 'Status',
+ 'Logical drives',
+ 'hpssa.ld_status',
+ 'line'
+ ],
+ 'lines': []
+ },
+ 'pd_status': {
+ 'options': [
+ None,
+ 'Status 1 is OK, Status 0 is not OK',
+ 'Status',
+ 'Physical drives',
+ 'hpssa.pd_status',
+ 'line'
+ ],
+ 'lines': []
+ },
+ 'pd_temperature': {
+ 'options': [
+ None,
+ 'Temperature',
+ 'Celsius',
+ 'Physical drives',
+ 'hpssa.pd_temperature',
+ 'line'
+ ],
+ 'lines': []
+ }
+}
+
+adapter_regex = re.compile(r'^(?P<adapter_type>.+) in Slot (?P<slot>\d+)')
+ignored_sections_regex = re.compile(
+ r'''
+ ^
+ Physical[ ]Drives
+ | None[ ]attached
+ | (?:Expander|Enclosure|SEP|Port[ ]Name:)[ ].+
+ | .+[ ]at[ ]Port[ ]\S+,[ ]Box[ ]\d+,[ ].+
+ | Mirror[ ]Group[ ]\d+:
+ $
+ ''',
+ re.X
+)
+mirror_group_regex = re.compile(r'^Mirror Group \d+:$')
+disk_partition_regex = re.compile(r'^Disk Partition Information$')
+array_regex = re.compile(r'^Array: (?P<id>[A-Z]+)$')
+drive_regex = re.compile(
+ r'''
+ ^
+ Logical[ ]Drive:[ ](?P<logical_drive_id>\d+)
+ | physicaldrive[ ](?P<fqn>[^:]+:\d+:\d+)
+ $
+ ''',
+ re.X
+)
+key_value_regex = re.compile(r'^(?P<key>[^:]+): ?(?P<value>.*)$')
+ld_status_regex = re.compile(r'^Status: (?P<status>[^,]+)(?:, (?P<percentage>[0-9.]+)% complete)?$')
+error_match = re.compile(r'Error:')
+
+
+class HPSSAException(Exception):
+ pass
+
+
+class HPSSA(object):
+ def __init__(self, lines):
+ self.lines = [line.strip() for line in lines if line.strip()]
+ self.current_line = 0
+ self.adapters = []
+ self.parse()
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ if self.current_line == len(self.lines):
+ raise StopIteration
+
+ line = self.lines[self.current_line]
+ self.current_line += 1
+
+ return line
+
+ def next(self):
+ """
+ This is for Python 2.7 compatibility
+ """
+ return self.__next__()
+
+ def rewind(self):
+ self.current_line = max(self.current_line - 1, 0)
+
+ @staticmethod
+ def match_any(line, *regexes):
+ return any([regex.match(line) for regex in regexes])
+
+ def parse(self):
+ for line in self:
+ match = adapter_regex.match(line)
+ if match:
+ self.adapters.append(self.parse_adapter(**match.groupdict()))
+
+ def parse_adapter(self, slot, adapter_type):
+ adapter = {
+ 'slot': int(slot),
+ 'type': adapter_type,
+
+ 'controller': {
+ 'status': None,
+ 'temperature': None,
+ },
+ 'cache': {
+ 'present': False,
+ 'status': None,
+ 'temperature': None,
+ },
+ 'battery': {
+ 'status': None,
+ 'count': 0,
+ },
+
+ 'logical_drives': [],
+ 'physical_drives': [],
+ }
+
+ for line in self:
+ if error_match.match(line):
+ raise HPSSAException('Error: {}'.format(line))
+ elif adapter_regex.match(line):
+ self.rewind()
+ break
+ elif array_regex.match(line):
+ self.parse_array(adapter)
+ elif line == 'Unassigned' or line == 'HBA Drives':
+ self.parse_physical_drives(adapter)
+ elif ignored_sections_regex.match(line):
+ self.parse_ignored_section()
+ else:
+ match = key_value_regex.match(line)
+ if match:
+ key, value = match.group('key', 'value')
+ if key == 'Controller Status':
+ adapter['controller']['status'] = value == 'OK'
+ elif key == 'Controller Temperature (C)':
+ adapter['controller']['temperature'] = int(value)
+ elif key == 'Cache Board Present':
+ adapter['cache']['present'] = value == 'True'
+ elif key == 'Cache Status':
+ adapter['cache']['status'] = value == 'OK'
+ elif key == 'Cache Module Temperature (C)':
+ adapter['cache']['temperature'] = int(value)
+ elif key == 'Battery/Capacitor Count':
+ adapter['battery']['count'] = int(value)
+ elif key == 'Battery/Capacitor Status':
+ adapter['battery']['status'] = value == 'OK'
+ else:
+ raise HPSSAException('Cannot parse line: {}'.format(line))
+
+ return adapter
+
+ def parse_array(self, adapter):
+ for line in self:
+ if HPSSA.match_any(line, adapter_regex, array_regex, ignored_sections_regex):
+ self.rewind()
+ break
+
+ match = drive_regex.match(line)
+ if match:
+ data = match.groupdict()
+ if data['logical_drive_id']:
+ self.parse_logical_drive(adapter, int(data['logical_drive_id']))
+ else:
+ self.parse_physical_drive(adapter, data['fqn'])
+ elif not key_value_regex.match(line):
+ self.rewind()
+ break
+
+ def parse_physical_drives(self, adapter):
+ for line in self:
+ match = drive_regex.match(line)
+ if match:
+ self.parse_physical_drive(adapter, match.group('fqn'))
+ else:
+ self.rewind()
+ break
+
+ def parse_logical_drive(self, adapter, logical_drive_id):
+ ld = {
+ 'id': logical_drive_id,
+ 'status': None,
+ 'status_complete': None,
+ }
+
+ for line in self:
+ if HPSSA.match_any(line, mirror_group_regex, disk_partition_regex):
+ self.parse_ignored_section()
+ continue
+
+ match = ld_status_regex.match(line)
+ if match:
+ ld['status'] = match.group('status') == 'OK'
+
+ if match.group('percentage'):
+ ld['status_complete'] = float(match.group('percentage')) / 100
+ elif HPSSA.match_any(line, adapter_regex, array_regex, drive_regex, ignored_sections_regex) \
+ or not key_value_regex.match(line):
+ self.rewind()
+ break
+
+ adapter['logical_drives'].append(ld)
+
+ def parse_physical_drive(self, adapter, fqn):
+ pd = {
+ 'fqn': fqn,
+ 'status': None,
+ 'temperature': None,
+ }
+
+ for line in self:
+ if HPSSA.match_any(line, adapter_regex, array_regex, drive_regex, ignored_sections_regex):
+ self.rewind()
+ break
+
+ match = key_value_regex.match(line)
+ if match:
+ key, value = match.group('key', 'value')
+ if key == 'Status':
+ pd['status'] = value == 'OK'
+ elif key == 'Current Temperature (C)':
+ pd['temperature'] = int(value)
+ else:
+ self.rewind()
+ break
+
+ adapter['physical_drives'].append(pd)
+
+ def parse_ignored_section(self):
+ for line in self:
+ if HPSSA.match_any(line, adapter_regex, array_regex, drive_regex, ignored_sections_regex) \
+ or not key_value_regex.match(line):
+ self.rewind()
+ break
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ super(Service, self).__init__(configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = deepcopy(CHARTS)
+ self.ssacli_path = self.configuration.get('ssacli_path', 'ssacli')
+ self.use_sudo = self.configuration.get('use_sudo', True)
+ self.cmd = []
+
+ def get_adapters(self):
+ try:
+ adapters = HPSSA(self._get_raw_data(command=self.cmd)).adapters
+ if not adapters:
+ # If no adapters are returned, run the command again but capture stderr
+ err = self._get_raw_data(command=self.cmd, stderr=True)
+ if err:
+ raise HPSSAException('Error executing cmd {}: {}'.format(' '.join(self.cmd), '\n'.join(err)))
+ return adapters
+ except HPSSAException as ex:
+ self.error(ex)
+ return []
+
+ def check(self):
+ if not os.path.isfile(self.ssacli_path):
+ ssacli_path = find_binary(self.ssacli_path)
+ if ssacli_path:
+ self.ssacli_path = ssacli_path
+ else:
+ self.error('Cannot locate "{}" binary'.format(self.ssacli_path))
+ return False
+
+ if self.use_sudo:
+ sudo = find_binary('sudo')
+ if not sudo:
+ self.error('Cannot locate "{}" binary'.format('sudo'))
+ return False
+
+ allowed = self._get_raw_data(command=[sudo, '-n', '-l', self.ssacli_path])
+ if not allowed or allowed[0].strip() != os.path.realpath(self.ssacli_path):
+ self.error('Not allowed to run sudo for command {}'.format(self.ssacli_path))
+ return False
+
+ self.cmd = [sudo, '-n']
+
+ self.cmd.extend([self.ssacli_path, 'ctrl', 'all', 'show', 'config', 'detail'])
+ self.info('Command: {}'.format(self.cmd))
+
+ adapters = self.get_adapters()
+
+ self.info('Discovered adapters: {}'.format([adapter['type'] for adapter in adapters]))
+ if not adapters:
+ self.error('No adapters discovered')
+ return False
+
+ return True
+
+ def get_data(self):
+ netdata = {}
+
+ for adapter in self.get_adapters():
+ status_key = '{}_status'.format(adapter['slot'])
+ temperature_key = '{}_temperature'.format(adapter['slot'])
+ ld_key = 'ld_{}_'.format(adapter['slot'])
+
+ data = {
+ 'ctrl_status': {
+ 'ctrl_' + status_key: adapter['controller']['status'],
+ 'cache_' + status_key: adapter['cache']['present'] and adapter['cache']['status'],
+ 'battery_' + status_key:
+ adapter['battery']['status'] if adapter['battery']['count'] > 0 else None
+ },
+
+ 'ctrl_temperature': {
+ 'ctrl_' + temperature_key: adapter['controller']['temperature'],
+ 'cache_' + temperature_key: adapter['cache']['temperature'],
+ },
+
+ 'ld_status': {
+ ld_key + '{}_status'.format(ld['id']): ld['status'] for ld in adapter['logical_drives']
+ },
+
+ 'pd_status': {},
+ 'pd_temperature': {},
+ }
+
+ for pd in adapter['physical_drives']:
+ pd_key = 'pd_{}_{}'.format(adapter['slot'], pd['fqn'])
+ data['pd_status'][pd_key + '_status'] = pd['status']
+ data['pd_temperature'][pd_key + '_temperature'] = pd['temperature']
+
+ for chart, dimension_data in data.items():
+ for dimension_id, value in dimension_data.items():
+ if value is None:
+ continue
+
+ if dimension_id not in self.charts[chart]:
+ self.charts[chart].add_dimension([dimension_id])
+
+ netdata[dimension_id] = value
+
+ return netdata
diff --git a/collectors/python.d.plugin/hpssa/hpssa.conf b/collectors/python.d.plugin/hpssa/hpssa.conf
new file mode 100644
index 0000000..cc50c98
--- /dev/null
+++ b/collectors/python.d.plugin/hpssa/hpssa.conf
@@ -0,0 +1,61 @@
+# netdata python.d.plugin configuration for hpssa
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 5
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 5 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, hpssa also supports the following:
+#
+# ssacli_path: /usr/sbin/ssacli # The path to the ssacli executable
+# use_sudo: True # Whether to use sudo or not
+# ----------------------------------------------------------------------
+
+# ssacli_path: /usr/sbin/ssacli
+# use_sudo: True
diff --git a/collectors/python.d.plugin/icecast/Makefile.inc b/collectors/python.d.plugin/icecast/Makefile.inc
new file mode 100644
index 0000000..cb7c6fa
--- /dev/null
+++ b/collectors/python.d.plugin/icecast/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += icecast/icecast.chart.py
+dist_pythonconfig_DATA += icecast/icecast.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += icecast/README.md icecast/Makefile.inc
+
diff --git a/collectors/python.d.plugin/icecast/README.md b/collectors/python.d.plugin/icecast/README.md
new file mode 100644
index 0000000..c122f76
--- /dev/null
+++ b/collectors/python.d.plugin/icecast/README.md
@@ -0,0 +1,44 @@
+<!--
+title: "Icecast monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/icecast/README.md
+sidebar_label: "Icecast"
+-->
+
+# Icecast monitoring with Netdata
+
+Monitors the number of listeners for active sources.
+
+## Requirements
+
+- icecast version >= 2.4.0
+
+It produces the following charts:
+
+1. **Listeners** in listeners
+
+- source number
+
+## Configuration
+
+Edit the `python.d/icecast.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/icecast.conf
+```
+
+Needs only `url` to server's `/status-json.xsl`
+
+Here is an example for remote server:
+
+```yaml
+remote:
+ url : 'http://1.2.3.4:8443/status-json.xsl'
+```
+
+Without configuration, module attempts to connect to `http://localhost:8443/status-json.xsl`
+
+---
+
+
diff --git a/collectors/python.d.plugin/icecast/icecast.chart.py b/collectors/python.d.plugin/icecast/icecast.chart.py
new file mode 100644
index 0000000..a967d17
--- /dev/null
+++ b/collectors/python.d.plugin/icecast/icecast.chart.py
@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# Description: icecast netdata python.d module
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import json
+
+from bases.FrameworkServices.UrlService import UrlService
+
+ORDER = [
+ 'listeners',
+]
+
+CHARTS = {
+ 'listeners': {
+ 'options': [None, 'Number Of Listeners', 'listeners', 'listeners', 'icecast.listeners', 'line'],
+ 'lines': [
+ ]
+ }
+}
+
+
+class Source:
+ def __init__(self, idx, data):
+ self.name = 'source_{0}'.format(idx)
+ self.is_active = data.get('stream_start') and data.get('server_name')
+ self.listeners = data['listeners']
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.url = self.configuration.get('url')
+ self._manager = self._build_manager()
+
+ def check(self):
+ """
+ Add active sources to the "listeners" chart
+ :return: bool
+ """
+ sources = self.get_sources()
+ if not sources:
+ return None
+
+ active_sources = 0
+ for idx, raw_source in enumerate(sources):
+ if Source(idx, raw_source).is_active:
+ active_sources += 1
+ dim_id = 'source_{0}'.format(idx)
+ dim = 'source {0}'.format(idx)
+ self.definitions['listeners']['lines'].append([dim_id, dim])
+
+ return bool(active_sources)
+
+ def _get_data(self):
+ """
+ Get number of listeners for every source
+ :return: dict
+ """
+ sources = self.get_sources()
+ if not sources:
+ return None
+
+ data = dict()
+
+ for idx, raw_source in enumerate(sources):
+ source = Source(idx, raw_source)
+ data[source.name] = source.listeners
+
+ return data
+
+ def get_sources(self):
+ """
+ Format data received from http request and return list of sources
+ :return: list
+ """
+
+ raw_data = self._get_raw_data()
+ if not raw_data:
+ return None
+
+ try:
+ data = json.loads(raw_data)
+ except ValueError as error:
+ self.error('JSON decode error:', error)
+ return None
+
+ sources = data['icestats'].get('source')
+ if not sources:
+ return None
+
+ return sources if isinstance(sources, list) else [sources]
diff --git a/collectors/python.d.plugin/icecast/icecast.conf b/collectors/python.d.plugin/icecast/icecast.conf
new file mode 100644
index 0000000..a33074a
--- /dev/null
+++ b/collectors/python.d.plugin/icecast/icecast.conf
@@ -0,0 +1,81 @@
+# netdata python.d.plugin configuration for icecast
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, icecast also supports the following:
+#
+# url: 'URL' # the URL to fetch icecast's stats
+#
+# if the URL is password protected, the following are supported:
+#
+# user: 'username'
+# pass: 'password'
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name : 'local'
+ url : 'http://localhost:8443/status-json.xsl'
+
+localipv4:
+ name : 'local'
+ url : 'http://127.0.0.1:8443/status-json.xsl' \ No newline at end of file
diff --git a/collectors/python.d.plugin/ipfs/Makefile.inc b/collectors/python.d.plugin/ipfs/Makefile.inc
new file mode 100644
index 0000000..68458cb
--- /dev/null
+++ b/collectors/python.d.plugin/ipfs/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += ipfs/ipfs.chart.py
+dist_pythonconfig_DATA += ipfs/ipfs.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += ipfs/README.md ipfs/Makefile.inc
+
diff --git a/collectors/python.d.plugin/ipfs/README.md b/collectors/python.d.plugin/ipfs/README.md
new file mode 100644
index 0000000..3a7c436
--- /dev/null
+++ b/collectors/python.d.plugin/ipfs/README.md
@@ -0,0 +1,51 @@
+<!--
+title: "IPFS monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/ipfs/README.md
+sidebar_label: "IPFS"
+-->
+
+# IPFS monitoring with Netdata
+
+Collects [`IPFS`](https://ipfs.io) basic information like file system bandwidth, peers and repo metrics.
+
+## Charts
+
+It produces the following charts:
+
+- Bandwidth in `kilobits/s`
+- Peers in `peers`
+- Repo Size in `GiB`
+- Repo Objects in `objects`
+
+## Configuration
+
+Edit the `python.d/ipfs.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/ipfs.conf
+```
+
+---
+
+Calls to the following endpoints are disabled due to `IPFS` bugs:
+
+- `/api/v0/stats/repo` (https://github.com/ipfs/go-ipfs/issues/3874)
+- `/api/v0/pin/ls` (https://github.com/ipfs/go-ipfs/issues/7528)
+
+Can be enabled in the collector configuration file.
+
+The configuration needs only `url` to `IPFS` server, here is an example for 2 `IPFS` instances:
+
+```yaml
+localhost:
+ url: 'http://localhost:5001'
+
+remote:
+ url: 'http://203.0.113.10::5001'
+```
+
+---
+
+
diff --git a/collectors/python.d.plugin/ipfs/ipfs.chart.py b/collectors/python.d.plugin/ipfs/ipfs.chart.py
new file mode 100644
index 0000000..abfc9c4
--- /dev/null
+++ b/collectors/python.d.plugin/ipfs/ipfs.chart.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+# Description: IPFS netdata python.d module
+# Authors: davidak
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import json
+
+from bases.FrameworkServices.UrlService import UrlService
+
+ORDER = [
+ 'bandwidth',
+ 'peers',
+ 'repo_size',
+ 'repo_objects',
+]
+
+CHARTS = {
+ 'bandwidth': {
+ 'options': [None, 'IPFS Bandwidth', 'kilobits/s', 'Bandwidth', 'ipfs.bandwidth', 'line'],
+ 'lines': [
+ ['in', None, 'absolute', 8, 1000],
+ ['out', None, 'absolute', -8, 1000]
+ ]
+ },
+ 'peers': {
+ 'options': [None, 'IPFS Peers', 'peers', 'Peers', 'ipfs.peers', 'line'],
+ 'lines': [
+ ['peers', None, 'absolute']
+ ]
+ },
+ 'repo_size': {
+ 'options': [None, 'IPFS Repo Size', 'GiB', 'Size', 'ipfs.repo_size', 'area'],
+ 'lines': [
+ ['avail', None, 'absolute', 1, 1 << 30],
+ ['size', None, 'absolute', 1, 1 << 30],
+ ]
+ },
+ 'repo_objects': {
+ 'options': [None, 'IPFS Repo Objects', 'objects', 'Objects', 'ipfs.repo_objects', 'line'],
+ 'lines': [
+ ['objects', None, 'absolute', 1, 1],
+ ['pinned', None, 'absolute', 1, 1],
+ ['recursive_pins', None, 'absolute', 1, 1]
+ ]
+ }
+}
+
+SI_zeroes = {
+ 'k': 3,
+ 'm': 6,
+ 'g': 9,
+ 't': 12,
+ 'p': 15,
+ 'e': 18,
+ 'z': 21,
+ 'y': 24
+}
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.baseurl = self.configuration.get('url', 'http://localhost:5001')
+ self.method = "POST"
+ self.do_pinapi = self.configuration.get('pinapi')
+ self.do_repoapi = self.configuration.get('repoapi')
+ self.__storage_max = None
+
+ def _get_json(self, sub_url):
+ """
+ :return: json decoding of the specified url
+ """
+ self.url = self.baseurl + sub_url
+ try:
+ return json.loads(self._get_raw_data())
+ except (TypeError, ValueError):
+ return dict()
+
+ @staticmethod
+ def _recursive_pins(keys):
+ return sum(1 for k in keys if keys[k]['Type'] == b'recursive')
+
+ @staticmethod
+ def _dehumanize(store_max):
+ # convert from '10Gb' to 10000000000
+ if not isinstance(store_max, int):
+ store_max = store_max.lower()
+ if store_max.endswith('b'):
+ val, units = store_max[:-2], store_max[-2]
+ if units in SI_zeroes:
+ val += '0' * SI_zeroes[units]
+ store_max = val
+ try:
+ store_max = int(store_max)
+ except (TypeError, ValueError):
+ store_max = None
+ return store_max
+
+ def _storagemax(self, store_cfg):
+ if self.__storage_max is None:
+ self.__storage_max = self._dehumanize(store_cfg)
+ return self.__storage_max
+
+ def _get_data(self):
+ """
+ Get data from API
+ :return: dict
+ """
+ # suburl : List of (result-key, original-key, transform-func)
+ cfg = {
+ '/api/v0/stats/bw':
+ [
+ ('in', 'RateIn', int),
+ ('out', 'RateOut', int),
+ ],
+ '/api/v0/swarm/peers':
+ [
+ ('peers', 'Peers', len),
+ ],
+ }
+ if self.do_repoapi:
+ cfg.update({
+ '/api/v0/stats/repo':
+ [
+ ('size', 'RepoSize', int),
+ ('objects', 'NumObjects', int),
+ ('avail', 'StorageMax', self._storagemax),
+ ],
+ })
+
+ if self.do_pinapi:
+ cfg.update({
+ '/api/v0/pin/ls':
+ [
+ ('pinned', 'Keys', len),
+ ('recursive_pins', 'Keys', self._recursive_pins),
+ ]
+ })
+ r = dict()
+ for suburl in cfg:
+ in_json = self._get_json(suburl)
+ for new_key, orig_key, xmute in cfg[suburl]:
+ try:
+ r[new_key] = xmute(in_json[orig_key])
+ except Exception as error:
+ self.debug(error)
+ return r or None
diff --git a/collectors/python.d.plugin/ipfs/ipfs.conf b/collectors/python.d.plugin/ipfs/ipfs.conf
new file mode 100644
index 0000000..8b167b3
--- /dev/null
+++ b/collectors/python.d.plugin/ipfs/ipfs.conf
@@ -0,0 +1,82 @@
+# netdata python.d.plugin configuration for ipfs
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, ipfs also supports the following:
+#
+# url: 'URL' # URL to the IPFS API
+# repoapi: no # Collect repo metrics
+# # Currently defaults to disabled due to IPFS Bug
+# # https://github.com/ipfs/go-ipfs/issues/7528
+# # resulting in very high CPU Usage
+# pinapi: no # Set status of IPFS pinned object polling
+# # Currently defaults to disabled due to IPFS Bug
+# # https://github.com/ipfs/go-ipfs/issues/3874
+# # resulting in very high CPU Usage
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name: 'local'
+ url: 'http://localhost:5001'
+ repoapi: no
+ pinapi: no
diff --git a/collectors/python.d.plugin/litespeed/Makefile.inc b/collectors/python.d.plugin/litespeed/Makefile.inc
new file mode 100644
index 0000000..5dd6450
--- /dev/null
+++ b/collectors/python.d.plugin/litespeed/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += litespeed/litespeed.chart.py
+dist_pythonconfig_DATA += litespeed/litespeed.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += litespeed/README.md litespeed/Makefile.inc
+
diff --git a/collectors/python.d.plugin/litespeed/README.md b/collectors/python.d.plugin/litespeed/README.md
new file mode 100644
index 0000000..b58b23d
--- /dev/null
+++ b/collectors/python.d.plugin/litespeed/README.md
@@ -0,0 +1,72 @@
+<!--
+title: "LiteSpeed monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/litespeed/README.md
+sidebar_label: "LiteSpeed"
+-->
+
+# LiteSpeed monitoring with Netdata
+
+Collects web server performance metrics for network, connection, requests, and cache.
+
+It produces:
+
+1. **Network Throughput HTTP** in kilobits/s
+
+ - in
+ - out
+
+2. **Network Throughput HTTPS** in kilobits/s
+
+ - in
+ - out
+
+3. **Connections HTTP** in connections
+
+ - free
+ - used
+
+4. **Connections HTTPS** in connections
+
+ - free
+ - used
+
+5. **Requests** in requests/s
+
+ - requests
+
+6. **Requests In Processing** in requests
+
+ - processing
+
+7. **Public Cache Hits** in hits/s
+
+ - hits
+
+8. **Private Cache Hits** in hits/s
+
+ - hits
+
+9. **Static Hits** in hits/s
+
+ - hits
+
+## Configuration
+
+Edit the `python.d/litespeed.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/litespeed.conf
+```
+
+```yaml
+local:
+ path : 'PATH'
+```
+
+If no configuration is given, module will use "/tmp/lshttpd/".
+
+---
+
+
diff --git a/collectors/python.d.plugin/litespeed/litespeed.chart.py b/collectors/python.d.plugin/litespeed/litespeed.chart.py
new file mode 100644
index 0000000..7ef8189
--- /dev/null
+++ b/collectors/python.d.plugin/litespeed/litespeed.chart.py
@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+# Description: litespeed netdata python.d module
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import glob
+import os
+import re
+from collections import namedtuple
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+update_every = 10
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = [
+ 'net_throughput_http', # net throughput
+ 'net_throughput_https', # net throughput
+ 'connections_http', # connections
+ 'connections_https', # connections
+ 'requests', # requests
+ 'requests_processing', # requests
+ 'pub_cache_hits', # cache
+ 'private_cache_hits', # cache
+ 'static_hits', # static
+]
+
+CHARTS = {
+ 'net_throughput_http': {
+ 'options': [None, 'Network Throughput HTTP', 'kilobits/s', 'net throughput',
+ 'litespeed.net_throughput', 'area'],
+ 'lines': [
+ ['bps_in', 'in', 'absolute'],
+ ['bps_out', 'out', 'absolute', -1]
+ ]
+ },
+ 'net_throughput_https': {
+ 'options': [None, 'Network Throughput HTTPS', 'kilobits/s', 'net throughput',
+ 'litespeed.net_throughput', 'area'],
+ 'lines': [
+ ['ssl_bps_in', 'in', 'absolute'],
+ ['ssl_bps_out', 'out', 'absolute', -1]
+ ]
+ },
+ 'connections_http': {
+ 'options': [None, 'Connections HTTP', 'conns', 'connections', 'litespeed.connections', 'stacked'],
+ 'lines': [
+ ['conn_free', 'free', 'absolute'],
+ ['conn_used', 'used', 'absolute']
+ ]
+ },
+ 'connections_https': {
+ 'options': [None, 'Connections HTTPS', 'conns', 'connections', 'litespeed.connections', 'stacked'],
+ 'lines': [
+ ['ssl_conn_free', 'free', 'absolute'],
+ ['ssl_conn_used', 'used', 'absolute']
+ ]
+ },
+ 'requests': {
+ 'options': [None, 'Requests', 'requests/s', 'requests', 'litespeed.requests', 'line'],
+ 'lines': [
+ ['requests', None, 'absolute', 1, 100]
+ ]
+ },
+ 'requests_processing': {
+ 'options': [None, 'Requests In Processing', 'requests', 'requests', 'litespeed.requests_processing', 'line'],
+ 'lines': [
+ ['requests_processing', 'processing', 'absolute']
+ ]
+ },
+ 'pub_cache_hits': {
+ 'options': [None, 'Public Cache Hits', 'hits/s', 'cache', 'litespeed.cache', 'line'],
+ 'lines': [
+ ['pub_cache_hits', 'hits', 'absolute', 1, 100]
+ ]
+ },
+ 'private_cache_hits': {
+ 'options': [None, 'Private Cache Hits', 'hits/s', 'cache', 'litespeed.cache', 'line'],
+ 'lines': [
+ ['private_cache_hits', 'hits', 'absolute', 1, 100]
+ ]
+ },
+ 'static_hits': {
+ 'options': [None, 'Static Hits', 'hits/s', 'static', 'litespeed.static', 'line'],
+ 'lines': [
+ ['static_hits', 'hits', 'absolute', 1, 100]
+ ]
+ }
+}
+
+t = namedtuple('T', ['key', 'id', 'mul'])
+
+T = [
+ t('BPS_IN', 'bps_in', 8),
+ t('BPS_OUT', 'bps_out', 8),
+ t('SSL_BPS_IN', 'ssl_bps_in', 8),
+ t('SSL_BPS_OUT', 'ssl_bps_out', 8),
+ t('REQ_PER_SEC', 'requests', 100),
+ t('REQ_PROCESSING', 'requests_processing', 1),
+ t('PUB_CACHE_HITS_PER_SEC', 'pub_cache_hits', 100),
+ t('PRIVATE_CACHE_HITS_PER_SEC', 'private_cache_hits', 100),
+ t('STATIC_HITS_PER_SEC', 'static_hits', 100),
+ t('PLAINCONN', 'conn_used', 1),
+ t('AVAILCONN', 'conn_free', 1),
+ t('SSLCONN', 'ssl_conn_used', 1),
+ t('AVAILSSL', 'ssl_conn_free', 1),
+]
+
+RE = re.compile(r'([A-Z_]+): ([0-9.]+)')
+
+ZERO_DATA = {
+ 'bps_in': 0,
+ 'bps_out': 0,
+ 'ssl_bps_in': 0,
+ 'ssl_bps_out': 0,
+ 'requests': 0,
+ 'requests_processing': 0,
+ 'pub_cache_hits': 0,
+ 'private_cache_hits': 0,
+ 'static_hits': 0,
+ 'conn_used': 0,
+ 'conn_free': 0,
+ 'ssl_conn_used': 0,
+ 'ssl_conn_free': 0,
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.path = self.configuration.get('path', '/tmp/lshttpd/')
+ self.files = list()
+
+ def check(self):
+ if not self.path:
+ self.error('"path" not specified')
+ return False
+
+ fs = glob.glob(os.path.join(self.path, '.rtreport*'))
+
+ if not fs:
+ self.error('"{0}" has no "rtreport" files or dir is not readable'.format(self.path))
+ return None
+
+ self.debug('stats files:', fs)
+
+ for f in fs:
+ if not is_readable_file(f):
+ self.error('{0} is not readable'.format(f))
+ continue
+ self.files.append(f)
+
+ return bool(self.files)
+
+ def get_data(self):
+ """
+ Format data received from http request
+ :return: dict
+ """
+ data = dict(ZERO_DATA)
+
+ for f in self.files:
+ try:
+ with open(f) as b:
+ lines = b.readlines()
+ except (OSError, IOError) as err:
+ self.error(err)
+ return None
+ else:
+ parse_file(data, lines)
+
+ return data
+
+
+def parse_file(data, lines):
+ for line in lines:
+ if not line.startswith(('BPS_IN:', 'MAXCONN:', 'PLAINCONN:', 'REQ_RATE []:')):
+ continue
+ m = dict(RE.findall(line))
+ for v in T:
+ if v.key in m:
+ data[v.id] += float(m[v.key]) * v.mul
+
+
+def is_readable_file(v):
+ return os.path.isfile(v) and os.access(v, os.R_OK)
diff --git a/collectors/python.d.plugin/litespeed/litespeed.conf b/collectors/python.d.plugin/litespeed/litespeed.conf
new file mode 100644
index 0000000..a326e18
--- /dev/null
+++ b/collectors/python.d.plugin/litespeed/litespeed.conf
@@ -0,0 +1,72 @@
+# netdata python.d.plugin configuration for litespeed
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, lightspeed also supports the following:
+#
+# path: 'PATH' # path to lightspeed stats files directory
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name : 'local'
+ path : '/tmp/lshttpd/'
diff --git a/collectors/python.d.plugin/logind/Makefile.inc b/collectors/python.d.plugin/logind/Makefile.inc
new file mode 100644
index 0000000..adadab1
--- /dev/null
+++ b/collectors/python.d.plugin/logind/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += logind/logind.chart.py
+dist_pythonconfig_DATA += logind/logind.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += logind/README.md logind/Makefile.inc
+
diff --git a/collectors/python.d.plugin/logind/README.md b/collectors/python.d.plugin/logind/README.md
new file mode 100644
index 0000000..442d388
--- /dev/null
+++ b/collectors/python.d.plugin/logind/README.md
@@ -0,0 +1,86 @@
+<!--
+title: "systemd-logind monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/logind/README.md
+sidebar_label: "systemd-logind"
+-->
+
+# Systemd-Logind monitoring with Netdata
+
+Monitors active sessions, users, and seats tracked by `systemd-logind` or `elogind`.
+
+It provides the following charts:
+
+1. **Sessions** Tracks the total number of sessions.
+
+ - Graphical: Local graphical sessions (running X11, or Wayland, or something else).
+ - Console: Local console sessions.
+ - Remote: Remote sessions.
+
+2. **Users** Tracks total number of unique user logins of each type.
+
+ - Graphical
+ - Console
+ - Remote
+
+3. **Seats** Total number of seats in use.
+
+ - Seats
+
+## Enable the collector
+
+The `logind` collector is disabled by default. To enable it, use `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`, to edit the `python.d.conf` file.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d.conf
+```
+
+Change the value of the `logind` setting to `yes`. Save the file and restart the Netdata Agent with `sudo systemctl
+restart netdata`, or the appropriate method for your system, to finish enabling the `logind` collector.
+
+## Configuration
+
+This module needs no configuration. Just make sure the `netdata` user
+can run the `loginctl` command and get a session list without having to
+specify a path.
+
+This will work with any command that can output data in the _exact_
+same format as `loginctl list-sessions --no-legend`. If you have some
+other command you want to use that outputs data in this format, you can
+specify it using the `command` key like so:
+
+```yaml
+command: '/path/to/other/command'
+```
+
+Edit the `python.d/logind.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/logind.conf
+```
+
+## Notes
+
+- This module's ability to track logins is dependent on what PAM services
+ are configured to register sessions with logind. In particular, for
+ most systems, it will only track TTY logins, local desktop logins,
+ and logins through remote shell connections.
+
+- The users chart counts _usernames_ not UID's. This is potentially
+ important in configurations where multiple users have the same UID.
+
+- The users chart counts any given user name up to once for _each_ type
+ of login. So if the same user has a graphical and a console login on a
+ system, they will show up once in the graphical count, and once in the
+ console count.
+
+- Because the data collection process is rather expensive, this plugin
+ is currently disabled by default, and needs to be explicitly enabled in
+ `/etc/netdata/python.d.conf` before it will run.
+
+---
+
+
diff --git a/collectors/python.d.plugin/logind/logind.chart.py b/collectors/python.d.plugin/logind/logind.chart.py
new file mode 100644
index 0000000..7086686
--- /dev/null
+++ b/collectors/python.d.plugin/logind/logind.chart.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+# Description: logind netdata python.d module
+# Author: Austin S. Hemmelgarn (Ferroin)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+
+priority = 59999
+disabled_by_default = True
+
+LOGINCTL_COMMAND = 'loginctl list-sessions --no-legend'
+
+ORDER = [
+ 'sessions',
+ 'users',
+ 'seats',
+]
+
+CHARTS = {
+ 'sessions': {
+ 'options': [None, 'Logind Sessions', 'sessions', 'sessions', 'logind.sessions', 'stacked'],
+ 'lines': [
+ ['sessions_graphical', 'Graphical', 'absolute', 1, 1],
+ ['sessions_console', 'Console', 'absolute', 1, 1],
+ ['sessions_remote', 'Remote', 'absolute', 1, 1]
+ ]
+ },
+ 'users': {
+ 'options': [None, 'Logind Users', 'users', 'users', 'logind.users', 'stacked'],
+ 'lines': [
+ ['users_graphical', 'Graphical', 'absolute', 1, 1],
+ ['users_console', 'Console', 'absolute', 1, 1],
+ ['users_remote', 'Remote', 'absolute', 1, 1]
+ ]
+ },
+ 'seats': {
+ 'options': [None, 'Logind Seats', 'seats', 'seats', 'logind.seats', 'line'],
+ 'lines': [
+ ['seats', 'Active Seats', 'absolute', 1, 1]
+ ]
+ }
+}
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ ExecutableService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.command = LOGINCTL_COMMAND
+
+ def _get_data(self):
+ ret = {
+ 'sessions_graphical': 0,
+ 'sessions_console': 0,
+ 'sessions_remote': 0,
+ }
+ users = {
+ 'graphical': list(),
+ 'console': list(),
+ 'remote': list()
+ }
+ seats = list()
+ data = self._get_raw_data()
+
+ for item in data:
+ fields = item.split()
+ if len(fields) == 3:
+ users['remote'].append(fields[2])
+ ret['sessions_remote'] += 1
+ elif len(fields) == 4:
+ users['graphical'].append(fields[2])
+ ret['sessions_graphical'] += 1
+ seats.append(fields[3])
+ elif len(fields) == 5:
+ users['console'].append(fields[2])
+ ret['sessions_console'] += 1
+ seats.append(fields[3])
+
+ ret['users_graphical'] = len(set(users['graphical']))
+ ret['users_console'] = len(set(users['console']))
+ ret['users_remote'] = len(set(users['remote']))
+ ret['seats'] = len(set(seats))
+
+ return ret
diff --git a/collectors/python.d.plugin/logind/logind.conf b/collectors/python.d.plugin/logind/logind.conf
new file mode 100644
index 0000000..01a859d
--- /dev/null
+++ b/collectors/python.d.plugin/logind/logind.conf
@@ -0,0 +1,60 @@
+# netdata python.d.plugin configuration for logind
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
diff --git a/collectors/python.d.plugin/megacli/Makefile.inc b/collectors/python.d.plugin/megacli/Makefile.inc
new file mode 100644
index 0000000..83680d7
--- /dev/null
+++ b/collectors/python.d.plugin/megacli/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += megacli/megacli.chart.py
+dist_pythonconfig_DATA += megacli/megacli.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += megacli/README.md megacli/Makefile.inc
+
diff --git a/collectors/python.d.plugin/megacli/README.md b/collectors/python.d.plugin/megacli/README.md
new file mode 100644
index 0000000..3c99c3d
--- /dev/null
+++ b/collectors/python.d.plugin/megacli/README.md
@@ -0,0 +1,86 @@
+<!--
+title: "MegaRAID controller monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/megacli/README.md
+sidebar_label: "MegaRAID controllers"
+-->
+
+# MegaRAID controller monitoring with Netdata
+
+Collects adapter, physical drives and battery stats using `megacli` command-line tool.
+
+Executed commands:
+
+- `sudo -n megacli -LDPDInfo -aAll`
+- `sudo -n megacli -AdpBbuCmd -a0`
+
+## Requirements
+
+The module uses `megacli`, which can only be executed by `root`. It uses
+`sudo` and assumes that it is configured such that the `netdata` user can execute `megacli` as root without a password.
+
+- Add to your `/etc/sudoers` file:
+
+`which megacli` shows the full path to the binary.
+
+```bash
+netdata ALL=(root) NOPASSWD: /path/to/megacli
+```
+
+- Reset Netdata's systemd
+ unit [CapabilityBoundingSet](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#Capabilities) (Linux
+ distributions with systemd)
+
+The default CapabilityBoundingSet doesn't allow using `sudo`, and is quite strict in general. Resetting is not optimal, but a next-best solution given the inability to execute `megacli` using `sudo`.
+
+
+As the `root` user, do the following:
+
+```cmd
+mkdir /etc/systemd/system/netdata.service.d
+echo -e '[Service]\nCapabilityBoundingSet=~' | tee /etc/systemd/system/netdata.service.d/unset-capability-bounding-set.conf
+systemctl daemon-reload
+systemctl restart netdata.service
+```
+
+## Charts
+
+- Adapter State
+- Physical Drives Media Errors
+- Physical Drives Predictive Failures
+- Battery Relative State of Charge
+- Battery Cycle Count
+
+## Enable the collector
+
+The `megacli` collector is disabled by default. To enable it, use `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`, to edit the `python.d.conf`
+file.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d.conf
+```
+
+Change the value of the `megacli` setting to `yes`. Save the file and restart the Netdata Agent
+with `sudo systemctl restart netdata`, or the appropriate method for your system.
+
+## Configuration
+
+Edit the `python.d/megacli.conf` configuration file using `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/megacli.conf
+```
+
+Battery stats disabled by default. To enable them, modify `megacli.conf`.
+
+```yaml
+do_battery: yes
+```
+
+Save the file and restart the Netdata Agent with `sudo systemctl restart netdata`, or the [appropriate
+method](/docs/configure/start-stop-restart.md) for your system.
+
+
diff --git a/collectors/python.d.plugin/megacli/megacli.chart.py b/collectors/python.d.plugin/megacli/megacli.chart.py
new file mode 100644
index 0000000..ef35ff6
--- /dev/null
+++ b/collectors/python.d.plugin/megacli/megacli.chart.py
@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+# Description: megacli netdata python.d module
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+
+import re
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+from bases.collection import find_binary
+
+disabled_by_default = True
+
+update_every = 5
+
+
+def adapter_charts(ads):
+ order = [
+ 'adapter_degraded',
+ ]
+
+ def dims(ad):
+ return [['adapter_{0}_degraded'.format(a.id), 'adapter {0}'.format(a.id)] for a in ad]
+
+ charts = {
+ 'adapter_degraded': {
+ 'options': [None, 'Adapter State', 'is degraded', 'adapter', 'megacli.adapter_degraded', 'line'],
+ 'lines': dims(ads)
+ },
+ }
+
+ return order, charts
+
+
+def pd_charts(pds):
+ order = [
+ 'pd_media_error',
+ 'pd_predictive_failure',
+ ]
+
+ def dims(k, pd):
+ return [['slot_{0}_{1}'.format(p.id, k), 'slot {0}'.format(p.id), 'incremental'] for p in pd]
+
+ charts = {
+ 'pd_media_error': {
+ 'options': [None, 'Physical Drives Media Errors', 'errors/s', 'pd', 'megacli.pd_media_error', 'line'],
+ 'lines': dims('media_error', pds)
+ },
+ 'pd_predictive_failure': {
+ 'options': [None, 'Physical Drives Predictive Failures', 'failures/s', 'pd',
+ 'megacli.pd_predictive_failure', 'line'],
+ 'lines': dims('predictive_failure', pds)
+ }
+ }
+
+ return order, charts
+
+
+def battery_charts(bats):
+ order = list()
+ charts = dict()
+
+ for b in bats:
+ order.append('bbu_{0}_relative_charge'.format(b.id))
+ charts.update(
+ {
+ 'bbu_{0}_relative_charge'.format(b.id): {
+ 'options': [None, 'Relative State of Charge', 'percentage', 'battery',
+ 'megacli.bbu_relative_charge', 'line'],
+ 'lines': [
+ ['bbu_{0}_relative_charge'.format(b.id), 'adapter {0}'.format(b.id)],
+ ]
+ }
+ }
+ )
+
+ for b in bats:
+ order.append('bbu_{0}_cycle_count'.format(b.id))
+ charts.update(
+ {
+ 'bbu_{0}_cycle_count'.format(b.id): {
+ 'options': [None, 'Cycle Count', 'cycle count', 'battery', 'megacli.bbu_cycle_count', 'line'],
+ 'lines': [
+ ['bbu_{0}_cycle_count'.format(b.id), 'adapter {0}'.format(b.id)],
+ ]
+ }
+ }
+ )
+
+ return order, charts
+
+
+RE_ADAPTER = re.compile(
+ r'Adapter #([0-9]+) State(?:\s+)?: ([a-zA-Z]+)'
+)
+
+RE_VD = re.compile(
+ r'Slot Number: ([0-9]+) Media Error Count: ([0-9]+) Predictive Failure Count: ([0-9]+)'
+)
+
+RE_BATTERY = re.compile(
+ r'BBU Capacity Info for Adapter: ([0-9]+) Relative State of Charge: ([0-9]+) % Cycle Count: ([0-9]+)'
+)
+
+
+def find_adapters(d):
+ keys = ('Adapter #', 'State')
+ d = ' '.join(v.strip() for v in d if v.startswith(keys))
+ return [Adapter(*v) for v in RE_ADAPTER.findall(d)]
+
+
+def find_pds(d):
+ keys = ('Slot Number', 'Media Error Count', 'Predictive Failure Count')
+ d = ' '.join(v.strip() for v in d if v.startswith(keys))
+ return [PD(*v) for v in RE_VD.findall(d)]
+
+
+def find_batteries(d):
+ keys = ('BBU Capacity Info for Adapter', 'Relative State of Charge', 'Cycle Count')
+ d = ' '.join(v.strip() for v in d if v.strip().startswith(keys))
+ return [Battery(*v) for v in RE_BATTERY.findall(d)]
+
+
+class Adapter:
+ def __init__(self, n, state):
+ self.id = n
+ self.state = int(state == 'Degraded')
+
+ def data(self):
+ return {
+ 'adapter_{0}_degraded'.format(self.id): self.state,
+ }
+
+
+class PD:
+ def __init__(self, n, media_err, predict_fail):
+ self.id = n
+ self.media_err = media_err
+ self.predict_fail = predict_fail
+
+ def data(self):
+ return {
+ 'slot_{0}_media_error'.format(self.id): self.media_err,
+ 'slot_{0}_predictive_failure'.format(self.id): self.predict_fail,
+ }
+
+
+class Battery:
+ def __init__(self, adapt_id, rel_charge, cycle_count):
+ self.id = adapt_id
+ self.rel_charge = rel_charge
+ self.cycle_count = cycle_count
+
+ def data(self):
+ return {
+ 'bbu_{0}_relative_charge'.format(self.id): self.rel_charge,
+ 'bbu_{0}_cycle_count'.format(self.id): self.cycle_count,
+ }
+
+
+# TODO: hardcoded sudo...
+class Megacli:
+ def __init__(self):
+ self.s = find_binary('sudo')
+ self.m = find_binary('megacli') or find_binary('MegaCli') # Binary on FreeBSD is MegaCli
+ self.sudo_check = [self.s, '-n', '-l']
+ self.disk_info = [self.s, '-n', self.m, '-LDPDInfo', '-aAll', '-NoLog']
+ self.battery_info = [self.s, '-n', self.m, '-AdpBbuCmd', '-a0', '-NoLog']
+
+ def __bool__(self):
+ return bool(self.s and self.m)
+
+ def __nonzero__(self):
+ return self.__bool__()
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ ExecutableService.__init__(self, configuration=configuration, name=name)
+ self.order = list()
+ self.definitions = dict()
+ self.do_battery = self.configuration.get('do_battery')
+ self.megacli = Megacli()
+
+ def check_sudo(self):
+ err = self._get_raw_data(command=self.megacli.sudo_check, stderr=True)
+ if err:
+ self.error(''.join(err))
+ return False
+ return True
+
+ def check_disk_info(self):
+ d = self._get_raw_data(command=self.megacli.disk_info)
+ if not d:
+ return False
+
+ ads = find_adapters(d)
+ pds = find_pds(d)
+
+ if not (ads and pds):
+ self.error('failed to parse "{0}" output'.format(' '.join(self.megacli.disk_info)))
+ return False
+
+ o, c = adapter_charts(ads)
+ self.order.extend(o)
+ self.definitions.update(c)
+
+ o, c = pd_charts(pds)
+ self.order.extend(o)
+ self.definitions.update(c)
+
+ return True
+
+ def check_battery(self):
+ d = self._get_raw_data(command=self.megacli.battery_info)
+ if not d:
+ return False
+
+ bats = find_batteries(d)
+
+ if not bats:
+ self.error('failed to parse "{0}" output'.format(' '.join(self.megacli.battery_info)))
+ return False
+
+ o, c = battery_charts(bats)
+ self.order.extend(o)
+ self.definitions.update(c)
+ return True
+
+ def check(self):
+ if not self.megacli:
+ self.error('can\'t locate "sudo" or "megacli" binary')
+ return None
+
+ if not (self.check_sudo() and self.check_disk_info()):
+ return False
+
+ if self.do_battery:
+ self.do_battery = self.check_battery()
+
+ return True
+
+ def get_data(self):
+ data = dict()
+
+ data.update(self.get_adapter_pd_data())
+
+ if self.do_battery:
+ data.update(self.get_battery_data())
+
+ return data or None
+
+ def get_adapter_pd_data(self):
+ raw = self._get_raw_data(command=self.megacli.disk_info)
+ data = dict()
+
+ if not raw:
+ return data
+
+ for a in find_adapters(raw):
+ data.update(a.data())
+
+ for p in find_pds(raw):
+ data.update(p.data())
+
+ return data
+
+ def get_battery_data(self):
+ raw = self._get_raw_data(command=self.megacli.battery_info)
+ data = dict()
+
+ if not raw:
+ return data
+
+ for b in find_batteries(raw):
+ data.update(b.data())
+
+ return data
diff --git a/collectors/python.d.plugin/megacli/megacli.conf b/collectors/python.d.plugin/megacli/megacli.conf
new file mode 100644
index 0000000..1af4292
--- /dev/null
+++ b/collectors/python.d.plugin/megacli/megacli.conf
@@ -0,0 +1,60 @@
+# netdata python.d.plugin configuration for megacli
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, megacli also supports the following:
+#
+# do_battery: yes/no # default is no. Battery stats (adds additional call to megacli `megacli -AdpBbuCmd -a0`).
+#
+# ----------------------------------------------------------------------
+# uncomment the line below to collect battery statistics
+# do_battery: yes
diff --git a/collectors/python.d.plugin/memcached/Makefile.inc b/collectors/python.d.plugin/memcached/Makefile.inc
new file mode 100644
index 0000000..e603571
--- /dev/null
+++ b/collectors/python.d.plugin/memcached/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += memcached/memcached.chart.py
+dist_pythonconfig_DATA += memcached/memcached.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += memcached/README.md memcached/Makefile.inc
+
diff --git a/collectors/python.d.plugin/memcached/README.md b/collectors/python.d.plugin/memcached/README.md
new file mode 100644
index 0000000..19139ee
--- /dev/null
+++ b/collectors/python.d.plugin/memcached/README.md
@@ -0,0 +1,99 @@
+<!--
+title: "Memcached monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/memcached/README.md
+sidebar_label: "Memcached"
+-->
+
+# Memcached monitoring with Netdata
+
+Collects memory-caching system performance metrics. It reads server response to stats command ([stats interface](https://github.com/memcached/memcached/wiki/Commands#stats)).
+
+
+1. **Network** in kilobytes/s
+
+ - read
+ - written
+
+2. **Connections** per second
+
+ - current
+ - rejected
+ - total
+
+3. **Items** in cluster
+
+ - current
+ - total
+
+4. **Evicted and Reclaimed** items
+
+ - evicted
+ - reclaimed
+
+5. **GET** requests/s
+
+ - hits
+ - misses
+
+6. **GET rate** rate in requests/s
+
+ - rate
+
+7. **SET rate** rate in requests/s
+
+ - rate
+
+8. **DELETE** requests/s
+
+ - hits
+ - misses
+
+9. **CAS** requests/s
+
+ - hits
+ - misses
+ - bad value
+
+10. **Increment** requests/s
+
+ - hits
+ - misses
+
+11. **Decrement** requests/s
+
+ - hits
+ - misses
+
+12. **Touch** requests/s
+
+ - hits
+ - misses
+
+13. **Touch rate** rate in requests/s
+
+ - rate
+
+## Configuration
+
+Edit the `python.d/memcached.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/memcached.conf
+```
+
+Sample:
+
+```yaml
+localtcpip:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 24242
+```
+
+If no configuration is given, module will attempt to connect to memcached instance on `127.0.0.1:11211` address.
+
+---
+
+
diff --git a/collectors/python.d.plugin/memcached/memcached.chart.py b/collectors/python.d.plugin/memcached/memcached.chart.py
new file mode 100644
index 0000000..bb656a2
--- /dev/null
+++ b/collectors/python.d.plugin/memcached/memcached.chart.py
@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+# Description: memcached netdata python.d module
+# Author: Pawel Krupa (paulfantom)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from bases.FrameworkServices.SocketService import SocketService
+
+ORDER = [
+ 'cache',
+ 'net',
+ 'connections',
+ 'items',
+ 'evicted_reclaimed',
+ 'get',
+ 'get_rate',
+ 'set_rate',
+ 'cas',
+ 'delete',
+ 'increment',
+ 'decrement',
+ 'touch',
+ 'touch_rate',
+]
+
+CHARTS = {
+ 'cache': {
+ 'options': [None, 'Cache Size', 'MiB', 'cache', 'memcached.cache', 'stacked'],
+ 'lines': [
+ ['avail', 'available', 'absolute', 1, 1 << 20],
+ ['used', 'used', 'absolute', 1, 1 << 20]
+ ]
+ },
+ 'net': {
+ 'options': [None, 'Network', 'kilobits/s', 'network', 'memcached.net', 'area'],
+ 'lines': [
+ ['bytes_read', 'in', 'incremental', 8, 1000],
+ ['bytes_written', 'out', 'incremental', -8, 1000],
+ ]
+ },
+ 'connections': {
+ 'options': [None, 'Connections', 'connections/s', 'connections', 'memcached.connections', 'line'],
+ 'lines': [
+ ['curr_connections', 'current', 'incremental'],
+ ['rejected_connections', 'rejected', 'incremental'],
+ ['total_connections', 'total', 'incremental']
+ ]
+ },
+ 'items': {
+ 'options': [None, 'Items', 'items', 'items', 'memcached.items', 'line'],
+ 'lines': [
+ ['curr_items', 'current', 'absolute'],
+ ['total_items', 'total', 'absolute']
+ ]
+ },
+ 'evicted_reclaimed': {
+ 'options': [None, 'Items', 'items', 'items', 'memcached.evicted_reclaimed', 'line'],
+ 'lines': [
+ ['reclaimed', 'reclaimed', 'absolute'],
+ ['evictions', 'evicted', 'absolute']
+ ]
+ },
+ 'get': {
+ 'options': [None, 'Requests', 'requests', 'get ops', 'memcached.get', 'stacked'],
+ 'lines': [
+ ['get_hits', 'hits', 'percent-of-absolute-row'],
+ ['get_misses', 'misses', 'percent-of-absolute-row']
+ ]
+ },
+ 'get_rate': {
+ 'options': [None, 'Rate', 'requests/s', 'get ops', 'memcached.get_rate', 'line'],
+ 'lines': [
+ ['cmd_get', 'rate', 'incremental']
+ ]
+ },
+ 'set_rate': {
+ 'options': [None, 'Rate', 'requests/s', 'set ops', 'memcached.set_rate', 'line'],
+ 'lines': [
+ ['cmd_set', 'rate', 'incremental']
+ ]
+ },
+ 'delete': {
+ 'options': [None, 'Requests', 'requests', 'delete ops', 'memcached.delete', 'stacked'],
+ 'lines': [
+ ['delete_hits', 'hits', 'percent-of-absolute-row'],
+ ['delete_misses', 'misses', 'percent-of-absolute-row'],
+ ]
+ },
+ 'cas': {
+ 'options': [None, 'Requests', 'requests', 'check and set ops', 'memcached.cas', 'stacked'],
+ 'lines': [
+ ['cas_hits', 'hits', 'percent-of-absolute-row'],
+ ['cas_misses', 'misses', 'percent-of-absolute-row'],
+ ['cas_badval', 'bad value', 'percent-of-absolute-row']
+ ]
+ },
+ 'increment': {
+ 'options': [None, 'Requests', 'requests', 'increment ops', 'memcached.increment', 'stacked'],
+ 'lines': [
+ ['incr_hits', 'hits', 'percent-of-absolute-row'],
+ ['incr_misses', 'misses', 'percent-of-absolute-row']
+ ]
+ },
+ 'decrement': {
+ 'options': [None, 'Requests', 'requests', 'decrement ops', 'memcached.decrement', 'stacked'],
+ 'lines': [
+ ['decr_hits', 'hits', 'percent-of-absolute-row'],
+ ['decr_misses', 'misses', 'percent-of-absolute-row']
+ ]
+ },
+ 'touch': {
+ 'options': [None, 'Requests', 'requests', 'touch ops', 'memcached.touch', 'stacked'],
+ 'lines': [
+ ['touch_hits', 'hits', 'percent-of-absolute-row'],
+ ['touch_misses', 'misses', 'percent-of-absolute-row']
+ ]
+ },
+ 'touch_rate': {
+ 'options': [None, 'Rate', 'requests/s', 'touch ops', 'memcached.touch_rate', 'line'],
+ 'lines': [
+ ['cmd_touch', 'rate', 'incremental']
+ ]
+ }
+}
+
+
+class Service(SocketService):
+ def __init__(self, configuration=None, name=None):
+ SocketService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.request = 'stats\r\n'
+ self.host = 'localhost'
+ self.port = 11211
+ self._keep_alive = True
+ self.unix_socket = None
+
+ def _get_data(self):
+ """
+ Get data from socket
+ :return: dict
+ """
+ response = self._get_raw_data()
+ if response is None:
+ # error has already been logged
+ return None
+
+ if response.startswith('ERROR'):
+ self.error('received ERROR')
+ return None
+
+ try:
+ parsed = response.split('\n')
+ except AttributeError:
+ self.error('response is invalid/empty')
+ return None
+
+ # split the response
+ data = {}
+ for line in parsed:
+ if line.startswith('STAT'):
+ try:
+ t = line[5:].split(' ')
+ data[t[0]] = t[1]
+ except (IndexError, ValueError):
+ self.debug('invalid line received: ' + str(line))
+
+ if not data:
+ self.error("received data doesn't have any records")
+ return None
+
+ # custom calculations
+ try:
+ data['avail'] = int(data['limit_maxbytes']) - int(data['bytes'])
+ data['used'] = int(data['bytes'])
+ except (KeyError, ValueError, TypeError):
+ pass
+
+ return data
+
+ def _check_raw_data(self, data):
+ if data.endswith('END\r\n'):
+ self.debug('received full response from memcached')
+ return True
+
+ self.debug('waiting more data from memcached')
+ return False
+
+ def check(self):
+ """
+ Parse configuration, check if memcached is available
+ :return: boolean
+ """
+ self._parse_config()
+ data = self._get_data()
+ if data is None:
+ return False
+ return True
diff --git a/collectors/python.d.plugin/memcached/memcached.conf b/collectors/python.d.plugin/memcached/memcached.conf
new file mode 100644
index 0000000..3286b46
--- /dev/null
+++ b/collectors/python.d.plugin/memcached/memcached.conf
@@ -0,0 +1,90 @@
+# netdata python.d.plugin configuration for memcached
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, memcached also supports the following:
+#
+# socket: 'path/to/memcached.sock'
+#
+# or
+# host: 'IP or HOSTNAME' # the host to connect to
+# port: PORT # the port to connect to
+#
+#
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name : 'local'
+ host : 'localhost'
+ port : 11211
+
+localipv4:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 11211
+
+localipv6:
+ name : 'local'
+ host : '::1'
+ port : 11211
+
diff --git a/collectors/python.d.plugin/mongodb/Makefile.inc b/collectors/python.d.plugin/mongodb/Makefile.inc
new file mode 100644
index 0000000..784945a
--- /dev/null
+++ b/collectors/python.d.plugin/mongodb/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += mongodb/mongodb.chart.py
+dist_pythonconfig_DATA += mongodb/mongodb.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += mongodb/README.md mongodb/Makefile.inc
+
diff --git a/collectors/python.d.plugin/mongodb/README.md b/collectors/python.d.plugin/mongodb/README.md
new file mode 100644
index 0000000..b6dd9c5
--- /dev/null
+++ b/collectors/python.d.plugin/mongodb/README.md
@@ -0,0 +1,210 @@
+<!--
+title: "MongoDB monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/mongodb/README.md
+sidebar_label: "MongoDB"
+-->
+
+# MongoDB monitoring with Netdata
+
+Monitors performance and health metrics of MongoDB.
+
+## Requirements
+
+- `python-pymongo` package v2.4+.
+
+You need to install it manually.
+
+Number of charts depends on mongodb version, storage engine and other features (replication):
+
+1. **Read requests**:
+
+ - query
+ - getmore (operation the cursor executes to get additional data from query)
+
+2. **Write requests**:
+
+ - insert
+ - delete
+ - update
+
+3. **Active clients**:
+
+ - readers (number of clients with read operations in progress or queued)
+ - writers (number of clients with write operations in progress or queued)
+
+4. **Journal transactions**:
+
+ - commits (count of transactions that have been written to the journal)
+
+5. **Data written to the journal**:
+
+ - volume (volume of data)
+
+6. **Background flush** (MMAPv1):
+
+ - average ms (average time taken by flushes to execute)
+ - last ms (time taken by the last flush)
+
+7. **Read tickets** (WiredTiger):
+
+ - in use (number of read tickets in use)
+ - available (number of available read tickets remaining)
+
+8. **Write tickets** (WiredTiger):
+
+ - in use (number of write tickets in use)
+ - available (number of available write tickets remaining)
+
+9. **Cursors**:
+
+- opened (number of cursors currently opened by MongoDB for clients)
+- timedOut (number of cursors that have timed)
+- noTimeout (number of open cursors with timeout disabled)
+
+10. **Connections**:
+
+ - connected (number of clients currently connected to the database server)
+ - unused (number of unused connections available for new clients)
+
+11. **Memory usage metrics**:
+
+ - virtual
+ - resident (amount of memory used by the database process)
+ - mapped
+ - non mapped
+
+12. **Page faults**:
+
+ - page faults (number of times MongoDB had to request from disk)
+
+13. **Cache metrics** (WiredTiger):
+
+ - percentage of bytes currently in the cache (amount of space taken by cached data)
+ - percentage of tracked dirty bytes in the cache (amount of space taken by dirty data)
+
+14. **Pages evicted from cache** (WiredTiger):
+
+ - modified
+ - unmodified
+
+15. **Queued requests**:
+
+ - readers (number of read request currently queued)
+ - writers (number of write request currently queued)
+
+16. **Errors**:
+
+ - msg (number of message assertions raised)
+ - warning (number of warning assertions raised)
+ - regular (number of regular assertions raised)
+ - user (number of assertions corresponding to errors generated by users)
+
+17. **Storage metrics** (one chart for every database)
+
+ - dataSize (size of all documents + padding in the database)
+ - indexSize (size of all indexes in the database)
+ - storageSize (size of all extents in the database)
+
+18. **Documents in the database** (one chart for all databases)
+
+- documents (number of objects in the database among all the collections)
+
+19. **tcmalloc metrics**
+
+ - central cache free
+ - current total thread cache
+ - pageheap free
+ - pageheap unmapped
+ - thread cache free
+ - transfer cache free
+ - heap size
+
+20. **Commands total/failed rate**
+
+ - count
+ - createIndex
+ - delete
+ - eval
+ - findAndModify
+ - insert
+
+21. **Locks metrics** (acquireCount metrics - number of times the lock was acquired in the specified mode)
+
+ - Global lock
+ - Database lock
+ - Collection lock
+ - Metadata lock
+ - oplog lock
+
+22. **Replica set members state**
+
+ - state
+
+23. **Oplog window**
+
+ - window (interval of time between the oldest and the latest entries in the oplog)
+
+24. **Replication lag**
+
+ - member (time when last entry from the oplog was applied for every member)
+
+25. **Replication set member heartbeat latency**
+
+ - member (time when last heartbeat was received from replica set member)
+
+## Prerequisite
+
+Create a read-only user for Netdata in the admin database.
+
+1. Authenticate as the admin user.
+
+```
+use admin
+db.auth("admin", "<MONGODB_ADMIN_PASSWORD>")
+```
+
+2. Create a user.
+
+```
+# MongoDB 2.x.
+db.addUser("netdata", "<UNIQUE_PASSWORD>", true)
+
+# MongoDB 3.x or higher.
+db.createUser({
+ "user":"netdata",
+ "pwd": "<UNIQUE_PASSWORD>",
+ "roles" : [
+ {role: 'read', db: 'admin' },
+ {role: 'clusterMonitor', db: 'admin'},
+ {role: 'read', db: 'local' }
+ ]
+})
+```
+
+## Configuration
+
+Edit the `python.d/mongodb.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/mongodb.conf
+```
+
+Sample:
+
+```yaml
+local:
+ name : 'local'
+ authdb: 'admin'
+ host : '127.0.0.1'
+ port : 27017
+ user : 'netdata'
+ pass : 'netdata'
+```
+
+If no configuration is given, module will attempt to connect to mongodb daemon on `127.0.0.1:27017` address
+
+---
+
+
diff --git a/collectors/python.d.plugin/mongodb/mongodb.chart.py b/collectors/python.d.plugin/mongodb/mongodb.chart.py
new file mode 100644
index 0000000..5e8fec8
--- /dev/null
+++ b/collectors/python.d.plugin/mongodb/mongodb.chart.py
@@ -0,0 +1,786 @@
+# -*- coding: utf-8 -*-
+# Description: mongodb netdata python.d module
+# Author: ilyam8
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import ssl
+
+from copy import deepcopy
+from datetime import datetime
+from sys import exc_info
+
+try:
+ from pymongo import MongoClient, ASCENDING, DESCENDING, version_tuple
+ from pymongo.errors import PyMongoError
+
+ PYMONGO = True
+except ImportError:
+ PYMONGO = False
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+REPL_SET_STATES = [
+ ('1', 'primary'),
+ ('8', 'down'),
+ ('2', 'secondary'),
+ ('3', 'recovering'),
+ ('5', 'startup2'),
+ ('4', 'fatal'),
+ ('7', 'arbiter'),
+ ('6', 'unknown'),
+ ('9', 'rollback'),
+ ('10', 'removed'),
+ ('0', 'startup')
+]
+
+
+def multiply_by_100(value):
+ return value * 100
+
+
+DEFAULT_METRICS = [
+ ('opcounters.delete', None, None),
+ ('opcounters.update', None, None),
+ ('opcounters.insert', None, None),
+ ('opcounters.query', None, None),
+ ('opcounters.getmore', None, None),
+ ('globalLock.activeClients.readers', 'activeClients_readers', None),
+ ('globalLock.activeClients.writers', 'activeClients_writers', None),
+ ('connections.available', 'connections_available', None),
+ ('connections.current', 'connections_current', None),
+ ('mem.mapped', None, None),
+ ('mem.resident', None, None),
+ ('mem.virtual', None, None),
+ ('globalLock.currentQueue.readers', 'currentQueue_readers', None),
+ ('globalLock.currentQueue.writers', 'currentQueue_writers', None),
+ ('asserts.msg', None, None),
+ ('asserts.regular', None, None),
+ ('asserts.user', None, None),
+ ('asserts.warning', None, None),
+ ('extra_info.page_faults', None, None),
+ ('metrics.record.moves', None, None),
+ ('backgroundFlushing.average_ms', None, multiply_by_100),
+ ('backgroundFlushing.last_ms', None, multiply_by_100),
+ ('backgroundFlushing.flushes', None, multiply_by_100),
+ ('metrics.cursor.timedOut', None, None),
+ ('metrics.cursor.open.total', 'cursor_total', None),
+ ('metrics.cursor.open.noTimeout', None, None),
+ ('cursors.timedOut', None, None),
+ ('cursors.totalOpen', 'cursor_total', None)
+]
+
+DUR = [
+ ('dur.commits', None, None),
+ ('dur.journaledMB', None, multiply_by_100)
+]
+
+WIREDTIGER = [
+ ('wiredTiger.concurrentTransactions.read.available', 'wiredTigerRead_available', None),
+ ('wiredTiger.concurrentTransactions.read.out', 'wiredTigerRead_out', None),
+ ('wiredTiger.concurrentTransactions.write.available', 'wiredTigerWrite_available', None),
+ ('wiredTiger.concurrentTransactions.write.out', 'wiredTigerWrite_out', None),
+ ('wiredTiger.cache.bytes currently in the cache', None, None),
+ ('wiredTiger.cache.tracked dirty bytes in the cache', None, None),
+ ('wiredTiger.cache.maximum bytes configured', None, None),
+ ('wiredTiger.cache.unmodified pages evicted', 'unmodified', None),
+ ('wiredTiger.cache.modified pages evicted', 'modified', None)
+]
+
+TCMALLOC = [
+ ('tcmalloc.generic.current_allocated_bytes', None, None),
+ ('tcmalloc.generic.heap_size', None, None),
+ ('tcmalloc.tcmalloc.central_cache_free_bytes', None, None),
+ ('tcmalloc.tcmalloc.current_total_thread_cache_bytes', None, None),
+ ('tcmalloc.tcmalloc.pageheap_free_bytes', None, None),
+ ('tcmalloc.tcmalloc.pageheap_unmapped_bytes', None, None),
+ ('tcmalloc.tcmalloc.thread_cache_free_bytes', None, None),
+ ('tcmalloc.tcmalloc.transfer_cache_free_bytes', None, None)
+]
+
+COMMANDS = [
+ ('metrics.commands.count.total', 'count_total', None),
+ ('metrics.commands.createIndexes.total', 'createIndexes_total', None),
+ ('metrics.commands.delete.total', 'delete_total', None),
+ ('metrics.commands.eval.total', 'eval_total', None),
+ ('metrics.commands.findAndModify.total', 'findAndModify_total', None),
+ ('metrics.commands.insert.total', 'insert_total', None),
+ ('metrics.commands.delete.total', 'delete_total', None),
+ ('metrics.commands.count.failed', 'count_failed', None),
+ ('metrics.commands.createIndexes.failed', 'createIndexes_failed', None),
+ ('metrics.commands.delete.failed', 'delete_failed', None),
+ ('metrics.commands.eval.failed', 'eval_failed', None),
+ ('metrics.commands.findAndModify.failed', 'findAndModify_failed', None),
+ ('metrics.commands.insert.failed', 'insert_failed', None),
+ ('metrics.commands.delete.failed', 'delete_failed', None)
+]
+
+LOCKS = [
+ ('locks.Collection.acquireCount.R', 'Collection_R', None),
+ ('locks.Collection.acquireCount.r', 'Collection_r', None),
+ ('locks.Collection.acquireCount.W', 'Collection_W', None),
+ ('locks.Collection.acquireCount.w', 'Collection_w', None),
+ ('locks.Database.acquireCount.R', 'Database_R', None),
+ ('locks.Database.acquireCount.r', 'Database_r', None),
+ ('locks.Database.acquireCount.W', 'Database_W', None),
+ ('locks.Database.acquireCount.w', 'Database_w', None),
+ ('locks.Global.acquireCount.R', 'Global_R', None),
+ ('locks.Global.acquireCount.r', 'Global_r', None),
+ ('locks.Global.acquireCount.W', 'Global_W', None),
+ ('locks.Global.acquireCount.w', 'Global_w', None),
+ ('locks.Metadata.acquireCount.R', 'Metadata_R', None),
+ ('locks.Metadata.acquireCount.w', 'Metadata_w', None),
+ ('locks.oplog.acquireCount.r', 'oplog_r', None),
+ ('locks.oplog.acquireCount.w', 'oplog_w', None)
+]
+
+DBSTATS = [
+ 'dataSize',
+ 'indexSize',
+ 'storageSize',
+ 'objects'
+]
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = [
+ 'read_operations',
+ 'write_operations',
+ 'active_clients',
+ 'journaling_transactions',
+ 'journaling_volume',
+ 'background_flush_average',
+ 'background_flush_last',
+ 'background_flush_rate',
+ 'wiredtiger_read',
+ 'wiredtiger_write',
+ 'cursors',
+ 'connections',
+ 'memory',
+ 'page_faults',
+ 'queued_requests',
+ 'record_moves',
+ 'wiredtiger_cache',
+ 'wiredtiger_pages_evicted',
+ 'asserts',
+ 'locks_collection',
+ 'locks_database',
+ 'locks_global',
+ 'locks_metadata',
+ 'locks_oplog',
+ 'dbstats_objects',
+ 'tcmalloc_generic',
+ 'tcmalloc_metrics',
+ 'command_total_rate',
+ 'command_failed_rate'
+]
+
+CHARTS = {
+ 'read_operations': {
+ 'options': [None, 'Received read requests', 'requests/s', 'throughput metrics',
+ 'mongodb.read_operations', 'line'],
+ 'lines': [
+ ['query', None, 'incremental'],
+ ['getmore', None, 'incremental']
+ ]
+ },
+ 'write_operations': {
+ 'options': [None, 'Received write requests', 'requests/s', 'throughput metrics',
+ 'mongodb.write_operations', 'line'],
+ 'lines': [
+ ['insert', None, 'incremental'],
+ ['update', None, 'incremental'],
+ ['delete', None, 'incremental']
+ ]
+ },
+ 'active_clients': {
+ 'options': [None, 'Clients with read or write operations in progress or queued', 'clients',
+ 'throughput metrics', 'mongodb.active_clients', 'line'],
+ 'lines': [
+ ['activeClients_readers', 'readers', 'absolute'],
+ ['activeClients_writers', 'writers', 'absolute']
+ ]
+ },
+ 'journaling_transactions': {
+ 'options': [None, 'Transactions that have been written to the journal', 'commits',
+ 'database performance', 'mongodb.journaling_transactions', 'line'],
+ 'lines': [
+ ['commits', None, 'absolute']
+ ]
+ },
+ 'journaling_volume': {
+ 'options': [None, 'Volume of data written to the journal', 'MiB', 'database performance',
+ 'mongodb.journaling_volume', 'line'],
+ 'lines': [
+ ['journaledMB', 'volume', 'absolute', 1, 100]
+ ]
+ },
+ 'background_flush_average': {
+ 'options': [None, 'Average time taken by flushes to execute', 'milliseconds', 'database performance',
+ 'mongodb.background_flush_average', 'line'],
+ 'lines': [
+ ['average_ms', 'time', 'absolute', 1, 100]
+ ]
+ },
+ 'background_flush_last': {
+ 'options': [None, 'Time taken by the last flush operation to execute', 'milliseconds', 'database performance',
+ 'mongodb.background_flush_last', 'line'],
+ 'lines': [
+ ['last_ms', 'time', 'absolute', 1, 100]
+ ]
+ },
+ 'background_flush_rate': {
+ 'options': [None, 'Flushes rate', 'flushes', 'database performance', 'mongodb.background_flush_rate', 'line'],
+ 'lines': [
+ ['flushes', 'flushes', 'incremental', 1, 1]
+ ]
+ },
+ 'wiredtiger_read': {
+ 'options': [None, 'Read tickets in use and remaining', 'tickets', 'database performance',
+ 'mongodb.wiredtiger_read', 'stacked'],
+ 'lines': [
+ ['wiredTigerRead_available', 'available', 'absolute', 1, 1],
+ ['wiredTigerRead_out', 'inuse', 'absolute', 1, 1]
+ ]
+ },
+ 'wiredtiger_write': {
+ 'options': [None, 'Write tickets in use and remaining', 'tickets', 'database performance',
+ 'mongodb.wiredtiger_write', 'stacked'],
+ 'lines': [
+ ['wiredTigerWrite_available', 'available', 'absolute', 1, 1],
+ ['wiredTigerWrite_out', 'inuse', 'absolute', 1, 1]
+ ]
+ },
+ 'cursors': {
+ 'options': [None, 'Currently opened cursors, cursors with timeout disabled and timed out cursors',
+ 'cursors', 'database performance', 'mongodb.cursors', 'stacked'],
+ 'lines': [
+ ['cursor_total', 'opened', 'absolute', 1, 1],
+ ['noTimeout', None, 'absolute', 1, 1],
+ ['timedOut', None, 'incremental', 1, 1]
+ ]
+ },
+ 'connections': {
+ 'options': [None, 'Currently connected clients and unused connections', 'connections',
+ 'resource utilization', 'mongodb.connections', 'stacked'],
+ 'lines': [
+ ['connections_available', 'unused', 'absolute', 1, 1],
+ ['connections_current', 'connected', 'absolute', 1, 1]
+ ]
+ },
+ 'memory': {
+ 'options': [None, 'Memory metrics', 'MiB', 'resource utilization', 'mongodb.memory', 'stacked'],
+ 'lines': [
+ ['virtual', None, 'absolute', 1, 1],
+ ['resident', None, 'absolute', 1, 1],
+ ['nonmapped', None, 'absolute', 1, 1],
+ ['mapped', None, 'absolute', 1, 1]
+ ]
+ },
+ 'page_faults': {
+ 'options': [None, 'Number of times MongoDB had to fetch data from disk', 'request/s',
+ 'resource utilization', 'mongodb.page_faults', 'line'],
+ 'lines': [
+ ['page_faults', None, 'incremental', 1, 1]
+ ]
+ },
+ 'queued_requests': {
+ 'options': [None, 'Currently queued read and write requests', 'requests', 'resource saturation',
+ 'mongodb.queued_requests', 'line'],
+ 'lines': [
+ ['currentQueue_readers', 'readers', 'absolute', 1, 1],
+ ['currentQueue_writers', 'writers', 'absolute', 1, 1]
+ ]
+ },
+ 'record_moves': {
+ 'options': [None, 'Number of times documents had to be moved on-disk', 'number',
+ 'resource saturation', 'mongodb.record_moves', 'line'],
+ 'lines': [
+ ['moves', None, 'incremental', 1, 1]
+ ]
+ },
+ 'asserts': {
+ 'options': [
+ None,
+ 'Number of message, warning, regular, corresponding to errors generated by users assertions raised',
+ 'number', 'errors (asserts)', 'mongodb.asserts', 'line'],
+ 'lines': [
+ ['msg', None, 'incremental', 1, 1],
+ ['warning', None, 'incremental', 1, 1],
+ ['regular', None, 'incremental', 1, 1],
+ ['user', None, 'incremental', 1, 1]
+ ]
+ },
+ 'wiredtiger_cache': {
+ 'options': [None, 'The percentage of the wiredTiger cache that is in use and cache with dirty bytes',
+ 'percentage', 'resource utilization', 'mongodb.wiredtiger_cache', 'stacked'],
+ 'lines': [
+ ['wiredTiger_percent_clean', 'inuse', 'absolute', 1, 1000],
+ ['wiredTiger_percent_dirty', 'dirty', 'absolute', 1, 1000]
+ ]
+ },
+ 'wiredtiger_pages_evicted': {
+ 'options': [None, 'Pages evicted from the cache',
+ 'pages', 'resource utilization', 'mongodb.wiredtiger_pages_evicted', 'stacked'],
+ 'lines': [
+ ['unmodified', None, 'absolute', 1, 1],
+ ['modified', None, 'absolute', 1, 1]
+ ]
+ },
+ 'dbstats_objects': {
+ 'options': [None, 'Number of documents in the database among all the collections', 'documents',
+ 'storage size metrics', 'mongodb.dbstats_objects', 'stacked'],
+ 'lines': []
+ },
+ 'tcmalloc_generic': {
+ 'options': [None, 'Tcmalloc generic metrics', 'MiB', 'tcmalloc', 'mongodb.tcmalloc_generic', 'stacked'],
+ 'lines': [
+ ['current_allocated_bytes', 'allocated', 'absolute', 1, 1 << 20],
+ ['heap_size', 'heap_size', 'absolute', 1, 1 << 20]
+ ]
+ },
+ 'tcmalloc_metrics': {
+ 'options': [None, 'Tcmalloc metrics', 'KiB', 'tcmalloc', 'mongodb.tcmalloc_metrics', 'stacked'],
+ 'lines': [
+ ['central_cache_free_bytes', 'central_cache_free', 'absolute', 1, 1024],
+ ['current_total_thread_cache_bytes', 'current_total_thread_cache', 'absolute', 1, 1024],
+ ['pageheap_free_bytes', 'pageheap_free', 'absolute', 1, 1024],
+ ['pageheap_unmapped_bytes', 'pageheap_unmapped', 'absolute', 1, 1024],
+ ['thread_cache_free_bytes', 'thread_cache_free', 'absolute', 1, 1024],
+ ['transfer_cache_free_bytes', 'transfer_cache_free', 'absolute', 1, 1024]
+ ]
+ },
+ 'command_total_rate': {
+ 'options': [None, 'Commands total rate', 'commands/s', 'commands', 'mongodb.command_total_rate', 'stacked'],
+ 'lines': [
+ ['count_total', 'count', 'incremental', 1, 1],
+ ['createIndexes_total', 'createIndexes', 'incremental', 1, 1],
+ ['delete_total', 'delete', 'incremental', 1, 1],
+ ['eval_total', 'eval', 'incremental', 1, 1],
+ ['findAndModify_total', 'findAndModify', 'incremental', 1, 1],
+ ['insert_total', 'insert', 'incremental', 1, 1],
+ ['update_total', 'update', 'incremental', 1, 1]
+ ]
+ },
+ 'command_failed_rate': {
+ 'options': [None, 'Commands failed rate', 'commands/s', 'commands', 'mongodb.command_failed_rate', 'stacked'],
+ 'lines': [
+ ['count_failed', 'count', 'incremental', 1, 1],
+ ['createIndexes_failed', 'createIndexes', 'incremental', 1, 1],
+ ['delete_failed', 'delete', 'incremental', 1, 1],
+ ['eval_failed', 'eval', 'incremental', 1, 1],
+ ['findAndModify_failed', 'findAndModify', 'incremental', 1, 1],
+ ['insert_failed', 'insert', 'incremental', 1, 1],
+ ['update_failed', 'update', 'incremental', 1, 1]
+ ]
+ },
+ 'locks_collection': {
+ 'options': [None, 'Collection lock. Number of times the lock was acquired in the specified mode',
+ 'locks', 'locks metrics', 'mongodb.locks_collection', 'stacked'],
+ 'lines': [
+ ['Collection_R', 'shared', 'incremental'],
+ ['Collection_W', 'exclusive', 'incremental'],
+ ['Collection_r', 'intent_shared', 'incremental'],
+ ['Collection_w', 'intent_exclusive', 'incremental']
+ ]
+ },
+ 'locks_database': {
+ 'options': [None, 'Database lock. Number of times the lock was acquired in the specified mode',
+ 'locks', 'locks metrics', 'mongodb.locks_database', 'stacked'],
+ 'lines': [
+ ['Database_R', 'shared', 'incremental'],
+ ['Database_W', 'exclusive', 'incremental'],
+ ['Database_r', 'intent_shared', 'incremental'],
+ ['Database_w', 'intent_exclusive', 'incremental']
+ ]
+ },
+ 'locks_global': {
+ 'options': [None, 'Global lock. Number of times the lock was acquired in the specified mode',
+ 'locks', 'locks metrics', 'mongodb.locks_global', 'stacked'],
+ 'lines': [
+ ['Global_R', 'shared', 'incremental'],
+ ['Global_W', 'exclusive', 'incremental'],
+ ['Global_r', 'intent_shared', 'incremental'],
+ ['Global_w', 'intent_exclusive', 'incremental']
+ ]
+ },
+ 'locks_metadata': {
+ 'options': [None, 'Metadata lock. Number of times the lock was acquired in the specified mode',
+ 'locks', 'locks metrics', 'mongodb.locks_metadata', 'stacked'],
+ 'lines': [
+ ['Metadata_R', 'shared', 'incremental'],
+ ['Metadata_w', 'intent_exclusive', 'incremental']
+ ]
+ },
+ 'locks_oplog': {
+ 'options': [None, 'Lock on the oplog. Number of times the lock was acquired in the specified mode',
+ 'locks', 'locks metrics', 'mongodb.locks_oplog', 'stacked'],
+ 'lines': [
+ ['oplog_r', 'intent_shared', 'incremental'],
+ ['oplog_w', 'intent_exclusive', 'incremental']
+ ]
+ }
+}
+
+DEFAULT_HOST = '127.0.0.1'
+DEFAULT_PORT = 27017
+DEFAULT_TIMEOUT = 100
+DEFAULT_AUTHDB = 'admin'
+
+CONN_PARAM_HOST = 'host'
+CONN_PARAM_PORT = 'port'
+CONN_PARAM_SERVER_SELECTION_TIMEOUT_MS = 'serverselectiontimeoutms'
+CONN_PARAM_SSL_SSL = 'ssl'
+CONN_PARAM_SSL_CERT_REQS = 'ssl_cert_reqs'
+CONN_PARAM_SSL_CA_CERTS = 'ssl_ca_certs'
+CONN_PARAM_SSL_CRL_FILE = 'ssl_crlfile'
+CONN_PARAM_SSL_CERT_FILE = 'ssl_certfile'
+CONN_PARAM_SSL_KEY_FILE = 'ssl_keyfile'
+CONN_PARAM_SSL_PEM_PASSPHRASE = 'ssl_pem_passphrase'
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER[:]
+ self.definitions = deepcopy(CHARTS)
+ self.authdb = self.configuration.get('authdb', DEFAULT_AUTHDB)
+ self.user = self.configuration.get('user')
+ self.password = self.configuration.get('pass')
+ self.metrics_to_collect = deepcopy(DEFAULT_METRICS)
+ self.connection = None
+ self.do_replica = None
+ self.databases = list()
+
+ def check(self):
+ if not PYMONGO:
+ self.error('Pymongo package v2.4+ is needed to use mongodb.chart.py')
+ return False
+ self.connection, server_status, error = self._create_connection()
+ if error:
+ self.error(error)
+ return False
+
+ self.build_metrics_to_collect_(server_status)
+
+ try:
+ data = self._get_data()
+ except (LookupError, SyntaxError, AttributeError):
+ self.error('Type: %s, error: %s' % (str(exc_info()[0]), str(exc_info()[1])))
+ return False
+ if isinstance(data, dict) and data:
+ self._data_from_check = data
+ self.create_charts_(server_status)
+ return True
+ self.error('_get_data() returned no data or type is not <dict>')
+ return False
+
+ def build_metrics_to_collect_(self, server_status):
+
+ self.do_replica = 'repl' in server_status
+ if 'dur' in server_status:
+ self.metrics_to_collect.extend(DUR)
+ if 'tcmalloc' in server_status:
+ self.metrics_to_collect.extend(TCMALLOC)
+ if 'commands' in server_status['metrics']:
+ self.metrics_to_collect.extend(COMMANDS)
+ if 'wiredTiger' in server_status:
+ self.metrics_to_collect.extend(WIREDTIGER)
+ has_locks = 'locks' in server_status
+ if has_locks and 'Collection' in server_status['locks']:
+ self.metrics_to_collect.extend(LOCKS)
+
+ def create_charts_(self, server_status):
+
+ if 'dur' not in server_status:
+ self.order.remove('journaling_transactions')
+ self.order.remove('journaling_volume')
+
+ if 'backgroundFlushing' not in server_status:
+ self.order.remove('background_flush_average')
+ self.order.remove('background_flush_last')
+ self.order.remove('background_flush_rate')
+
+ if 'wiredTiger' not in server_status:
+ self.order.remove('wiredtiger_write')
+ self.order.remove('wiredtiger_read')
+ self.order.remove('wiredtiger_cache')
+
+ if 'tcmalloc' not in server_status:
+ self.order.remove('tcmalloc_generic')
+ self.order.remove('tcmalloc_metrics')
+
+ if 'commands' not in server_status['metrics']:
+ self.order.remove('command_total_rate')
+ self.order.remove('command_failed_rate')
+
+ has_no_locks = 'locks' not in server_status
+ if has_no_locks or 'Collection' not in server_status['locks']:
+ self.order.remove('locks_collection')
+ self.order.remove('locks_database')
+ self.order.remove('locks_global')
+ self.order.remove('locks_metadata')
+
+ if has_no_locks or 'oplog' not in server_status['locks']:
+ self.order.remove('locks_oplog')
+
+ for dbase in self.databases:
+ self.order.append('_'.join([dbase, 'dbstats']))
+ self.definitions['_'.join([dbase, 'dbstats'])] = {
+ 'options': [None, '%s: size of all documents, indexes, extents' % dbase, 'KB',
+ 'storage size metrics', 'mongodb.dbstats', 'line'],
+ 'lines': [
+ ['_'.join([dbase, 'dataSize']), 'documents', 'absolute', 1, 1024],
+ ['_'.join([dbase, 'indexSize']), 'indexes', 'absolute', 1, 1024],
+ ['_'.join([dbase, 'storageSize']), 'extents', 'absolute', 1, 1024]
+ ]}
+ self.definitions['dbstats_objects']['lines'].append(['_'.join([dbase, 'objects']), dbase, 'absolute'])
+
+ if self.do_replica:
+ def create_lines(hosts, string):
+ lines = list()
+ for host in hosts:
+ dim_id = '_'.join([host, string])
+ lines.append([dim_id, host, 'absolute', 1, 1000])
+ return lines
+
+ def create_state_lines(states):
+ lines = list()
+ for state, description in states:
+ dim_id = '_'.join([host, 'state', state])
+ lines.append([dim_id, description, 'absolute', 1, 1])
+ return lines
+
+ all_hosts = server_status['repl']['hosts'] + server_status['repl'].get('arbiters', list())
+ this_host = server_status['repl']['me']
+ other_hosts = [host for host in all_hosts if host != this_host]
+
+ if 'local' in self.databases:
+ self.order.append('oplog_window')
+ self.definitions['oplog_window'] = {
+ 'options': [None, 'Interval of time between the oldest and the latest entries in the oplog',
+ 'seconds', 'replication and oplog', 'mongodb.oplog_window', 'line'],
+ 'lines': [['timeDiff', 'window', 'absolute', 1, 1000]]}
+ # Create "heartbeat delay" chart
+ self.order.append('heartbeat_delay')
+ self.definitions['heartbeat_delay'] = {
+ 'options': [
+ None,
+ 'Time when last heartbeat was received from the replica set member (lastHeartbeatRecv)',
+ 'seconds ago', 'replication and oplog', 'mongodb.replication_heartbeat_delay', 'stacked'],
+ 'lines': create_lines(other_hosts, 'heartbeat_lag')}
+ # Create "optimedate delay" chart
+ self.order.append('optimedate_delay')
+ self.definitions['optimedate_delay'] = {
+ 'options': [None, 'Time when last entry from the oplog was applied (optimeDate)',
+ 'seconds ago', 'replication and oplog', 'mongodb.replication_optimedate_delay', 'stacked'],
+ 'lines': create_lines(all_hosts, 'optimedate')}
+ # Create "replica set members state" chart
+ for host in all_hosts:
+ chart_name = '_'.join([host, 'state'])
+ self.order.append(chart_name)
+ self.definitions[chart_name] = {
+ 'options': [None, 'Replica set member (%s) current state' % host, 'state',
+ 'replication and oplog', 'mongodb.replication_state', 'line'],
+ 'lines': create_state_lines(REPL_SET_STATES)}
+
+ def _get_raw_data(self):
+ raw_data = dict()
+
+ raw_data.update(self.get_server_status() or dict())
+ raw_data.update(self.get_db_stats() or dict())
+ raw_data.update(self.get_repl_set_get_status() or dict())
+ raw_data.update(self.get_get_replication_info() or dict())
+
+ return raw_data or None
+
+ def get_server_status(self):
+ raw_data = dict()
+ try:
+ raw_data['serverStatus'] = self.connection.admin.command('serverStatus')
+ except PyMongoError:
+ return None
+ else:
+ return raw_data
+
+ def get_db_stats(self):
+ if not self.databases:
+ return None
+
+ raw_data = dict()
+ raw_data['dbStats'] = dict()
+ try:
+ for dbase in self.databases:
+ raw_data['dbStats'][dbase] = self.connection[dbase].command('dbStats')
+ return raw_data
+ except PyMongoError:
+ return None
+
+ def get_repl_set_get_status(self):
+ if not self.do_replica:
+ return None
+
+ raw_data = dict()
+ try:
+ raw_data['replSetGetStatus'] = self.connection.admin.command('replSetGetStatus')
+ return raw_data
+ except PyMongoError:
+ return None
+
+ def get_get_replication_info(self):
+ if not (self.do_replica and 'local' in self.databases):
+ return None
+
+ raw_data = dict()
+ raw_data['getReplicationInfo'] = dict()
+ try:
+ raw_data['getReplicationInfo']['ASCENDING'] = self.connection.local.oplog.rs.find().sort(
+ '$natural', ASCENDING).limit(1)[0]
+ raw_data['getReplicationInfo']['DESCENDING'] = self.connection.local.oplog.rs.find().sort(
+ '$natural', DESCENDING).limit(1)[0]
+ return raw_data
+ except PyMongoError:
+ return None
+
+ def _get_data(self):
+ """
+ :return: dict
+ """
+ raw_data = self._get_raw_data()
+
+ if not raw_data:
+ return None
+
+ data = dict()
+ serverStatus = raw_data['serverStatus']
+ dbStats = raw_data.get('dbStats')
+ replSetGetStatus = raw_data.get('replSetGetStatus')
+ getReplicationInfo = raw_data.get('getReplicationInfo')
+ utc_now = datetime.utcnow()
+
+ # serverStatus
+ for metric, new_name, func in self.metrics_to_collect:
+ value = serverStatus
+ for key in metric.split('.'):
+ try:
+ value = value[key]
+ except KeyError:
+ break
+
+ if not isinstance(value, dict) and key:
+ data[new_name or key] = value if not func else func(value)
+
+ if 'mapped' in serverStatus['mem']:
+ data['nonmapped'] = data['virtual'] - serverStatus['mem'].get('mappedWithJournal', data['mapped'])
+
+ if data.get('maximum bytes configured'):
+ maximum = data['maximum bytes configured']
+ data['wiredTiger_percent_clean'] = int(data['bytes currently in the cache'] * 100 / maximum * 1000)
+ data['wiredTiger_percent_dirty'] = int(data['tracked dirty bytes in the cache'] * 100 / maximum * 1000)
+
+ # dbStats
+ if dbStats:
+ for dbase in dbStats:
+ for metric in DBSTATS:
+ key = '_'.join([dbase, metric])
+ data[key] = dbStats[dbase][metric]
+
+ # replSetGetStatus
+ if replSetGetStatus:
+ other_hosts = list()
+ members = replSetGetStatus['members']
+ unix_epoch = datetime(1970, 1, 1, 0, 0)
+
+ for member in members:
+ if not member.get('self'):
+ other_hosts.append(member)
+
+ # Replica set time diff between current time and time when last entry from the oplog was applied
+ if member.get('optimeDate', unix_epoch) != unix_epoch:
+ member_optimedate = member['name'] + '_optimedate'
+ delta = utc_now - member['optimeDate']
+ data[member_optimedate] = int(delta_calculation(delta=delta, multiplier=1000))
+
+ # Replica set members state
+ member_state = member['name'] + '_state'
+ for elem in REPL_SET_STATES:
+ state = elem[0]
+ data.update({'_'.join([member_state, state]): 0})
+ data.update({'_'.join([member_state, str(member['state'])]): member['state']})
+
+ # Heartbeat lag calculation
+ for other in other_hosts:
+ if other['lastHeartbeatRecv'] != unix_epoch:
+ node = other['name'] + '_heartbeat_lag'
+ delta = utc_now - other['lastHeartbeatRecv']
+ data[node] = int(delta_calculation(delta=delta, multiplier=1000))
+
+ if getReplicationInfo:
+ first_event = getReplicationInfo['ASCENDING']['ts'].as_datetime()
+ last_event = getReplicationInfo['DESCENDING']['ts'].as_datetime()
+ data['timeDiff'] = int(delta_calculation(delta=last_event - first_event, multiplier=1000))
+
+ return data
+
+ def build_ssl_connection_params(self):
+ conf = self.configuration
+
+ def cert_req(v):
+ if v is None:
+ return None
+ if not v:
+ return ssl.CERT_NONE
+ return ssl.CERT_REQUIRED
+
+ ssl_params = {
+ CONN_PARAM_SSL_SSL: conf.get(CONN_PARAM_SSL_SSL),
+ CONN_PARAM_SSL_CERT_REQS: cert_req(conf.get(CONN_PARAM_SSL_CERT_REQS)),
+ CONN_PARAM_SSL_CA_CERTS: conf.get(CONN_PARAM_SSL_CA_CERTS),
+ CONN_PARAM_SSL_CRL_FILE: conf.get(CONN_PARAM_SSL_CRL_FILE),
+ CONN_PARAM_SSL_CERT_FILE: conf.get(CONN_PARAM_SSL_CERT_FILE),
+ CONN_PARAM_SSL_KEY_FILE: conf.get(CONN_PARAM_SSL_KEY_FILE),
+ CONN_PARAM_SSL_PEM_PASSPHRASE: conf.get(CONN_PARAM_SSL_PEM_PASSPHRASE),
+ }
+
+ ssl_params = dict((k, v) for k, v in ssl_params.items() if v is not None)
+
+ return ssl_params
+
+ def build_connection_params(self):
+ conf = self.configuration
+ params = {
+ CONN_PARAM_HOST: conf.get(CONN_PARAM_HOST, DEFAULT_HOST),
+ CONN_PARAM_PORT: conf.get(CONN_PARAM_PORT, DEFAULT_PORT),
+ }
+ if hasattr(MongoClient, 'server_selection_timeout') or version_tuple[0] >= 4:
+ params[CONN_PARAM_SERVER_SELECTION_TIMEOUT_MS] = conf.get('timeout', DEFAULT_TIMEOUT)
+
+ params.update(self.build_ssl_connection_params())
+ return params
+
+ def _create_connection(self):
+ params = self.build_connection_params()
+ self.debug('creating connection, connection params: {0}'.format(sorted(params)))
+
+ try:
+ connection = MongoClient(**params)
+ if self.user and self.password:
+ self.debug('authenticating, user: {0}, password: {1}'.format(self.user, self.password))
+ getattr(connection, self.authdb).authenticate(name=self.user, password=self.password)
+ else:
+ self.debug('skip authenticating, user and password are not set')
+ # elif self.user:
+ # connection.admin.authenticate(name=self.user, mechanism='MONGODB-X509')
+ server_status = connection.admin.command('serverStatus')
+ except PyMongoError as error:
+ return None, None, str(error)
+ else:
+ try:
+ self.databases = connection.database_names()
+ except PyMongoError as error:
+ self.info('Can\'t collect databases: %s' % str(error))
+ return connection, server_status, None
+
+
+def delta_calculation(delta, multiplier=1):
+ if hasattr(delta, 'total_seconds'):
+ return delta.total_seconds() * multiplier
+ return (delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6 * multiplier
diff --git a/collectors/python.d.plugin/mongodb/mongodb.conf b/collectors/python.d.plugin/mongodb/mongodb.conf
new file mode 100644
index 0000000..9f660f5
--- /dev/null
+++ b/collectors/python.d.plugin/mongodb/mongodb.conf
@@ -0,0 +1,102 @@
+# netdata python.d.plugin configuration for mongodb
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, mongodb also supports the following:
+#
+# host: 'IP or HOSTNAME' # type <str> the host to connect to
+# port: PORT # type <int> the port to connect to
+#
+# in all cases, the following can also be set:
+#
+# authdb: 'dbname' # database to authenticate the user against,
+# # defaults to "admin".
+# user: 'username' # the mongodb username to use
+# pass: 'password' # the mongodb password to use
+#
+# SSL connection parameters (https://api.mongodb.com/python/current/examples/tls.html):
+#
+# ssl: yes # connect to the server using TLS
+# ssl_cert_reqs: yes # require a certificate from the server when TLS is enabled
+# ssl_ca_certs: '/path/to/ca.pem' # use a specific set of CA certificates
+# ssl_crlfile: '/path/to/crl.pem' # use a certificate revocation lists
+# ssl_certfile: '/path/to/client.pem' # use a client certificate
+# ssl_keyfile: '/path/to/key.pem' # use a specific client certificate key
+# ssl_pem_passphrase: 'passphrase' # use a passphrase to decrypt encrypted private keys
+#
+
+# ----------------------------------------------------------------------
+# to connect to the mongodb on localhost, without a password:
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+local:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 27017
+
+# authsample:
+# name : 'secure'
+# host : 'mongodb.example.com'
+# port : 27017
+# authdb : 'admin'
+# user : 'monitor'
+# pass : 'supersecret'
diff --git a/collectors/python.d.plugin/monit/Makefile.inc b/collectors/python.d.plugin/monit/Makefile.inc
new file mode 100644
index 0000000..4a3673f
--- /dev/null
+++ b/collectors/python.d.plugin/monit/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += monit/monit.chart.py
+dist_pythonconfig_DATA += monit/monit.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += monit/README.md monit/Makefile.inc
+
diff --git a/collectors/python.d.plugin/monit/README.md b/collectors/python.d.plugin/monit/README.md
new file mode 100644
index 0000000..1396025
--- /dev/null
+++ b/collectors/python.d.plugin/monit/README.md
@@ -0,0 +1,52 @@
+<!--
+title: "Monit monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/monit/README.md
+sidebar_label: "Monit"
+-->
+
+# Monit monitoring with Netdata
+
+Monit monitoring module. Data is grabbed from stats XML interface (exists for a long time, but not mentioned in official documentation). Mostly this plugin shows statuses of monit targets, i.e. [statuses of specified checks](https://mmonit.com/monit/documentation/monit.html#Service-checks).
+
+1. **Filesystems**
+
+ - Filesystems
+ - Directories
+ - Files
+ - Pipes
+
+2. **Applications**
+
+ - Processes (+threads/childs)
+ - Programs
+
+3. **Network**
+
+ - Hosts (+latency)
+ - Network interfaces
+
+## Configuration
+
+Edit the `python.d/monit.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/monit.conf
+```
+
+Sample:
+
+```yaml
+local:
+ name : 'local'
+ url : 'http://localhost:2812'
+ user: : admin
+ pass: : monit
+```
+
+If no configuration is given, module will attempt to connect to monit as `http://localhost:2812`.
+
+---
+
+
diff --git a/collectors/python.d.plugin/monit/monit.chart.py b/collectors/python.d.plugin/monit/monit.chart.py
new file mode 100644
index 0000000..bfc1823
--- /dev/null
+++ b/collectors/python.d.plugin/monit/monit.chart.py
@@ -0,0 +1,360 @@
+# -*- coding: utf-8 -*-
+# Description: monit netdata python.d module
+# Author: Evgeniy K. (n0guest)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import xml.etree.ElementTree as ET
+from collections import namedtuple
+
+from bases.FrameworkServices.UrlService import UrlService
+
+MonitType = namedtuple('MonitType', ('index', 'name'))
+
+# see enum Service_Type from monit.h (https://bitbucket.org/tildeslash/monit/src/master/src/monit.h)
+# typedef enum {
+# Service_Filesystem = 0,
+# Service_Directory,
+# Service_File,
+# Service_Process,
+# Service_Host,
+# Service_System,
+# Service_Fifo,
+# Service_Program,
+# Service_Net,
+# Service_Last = Service_Net
+# } __attribute__((__packed__)) Service_Type;
+
+TYPE_FILESYSTEM = MonitType(0, 'filesystem')
+TYPE_DIRECTORY = MonitType(1, 'directory')
+TYPE_FILE = MonitType(2, 'file')
+TYPE_PROCESS = MonitType(3, 'process')
+TYPE_HOST = MonitType(4, 'host')
+TYPE_SYSTEM = MonitType(5, 'system')
+TYPE_FIFO = MonitType(6, 'fifo')
+TYPE_PROGRAM = MonitType(7, 'program')
+TYPE_NET = MonitType(8, 'net')
+
+TYPES = (
+ TYPE_FILESYSTEM,
+ TYPE_DIRECTORY,
+ TYPE_FILE,
+ TYPE_PROCESS,
+ TYPE_HOST,
+ TYPE_SYSTEM,
+ TYPE_FIFO,
+ TYPE_PROGRAM,
+ TYPE_NET,
+)
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = [
+ 'filesystem',
+ 'directory',
+ 'file',
+ 'process',
+ 'process_uptime',
+ 'process_threads',
+ 'process_children',
+ 'host',
+ 'host_latency',
+ 'system',
+ 'fifo',
+ 'program',
+ 'net'
+]
+
+CHARTS = {
+ 'filesystem': {
+ 'options': ['filesystems', 'Filesystems', 'filesystems', 'filesystem', 'monit.filesystems', 'line'],
+ 'lines': []
+ },
+ 'directory': {
+ 'options': ['directories', 'Directories', 'directories', 'filesystem', 'monit.directories', 'line'],
+ 'lines': []
+ },
+ 'file': {
+ 'options': ['files', 'Files', 'files', 'filesystem', 'monit.files', 'line'],
+ 'lines': []
+ },
+ 'fifo': {
+ 'options': ['fifos', 'Pipes (fifo)', 'pipes', 'filesystem', 'monit.fifos', 'line'],
+ 'lines': []
+ },
+ 'program': {
+ 'options': ['programs', 'Programs statuses', 'programs', 'applications', 'monit.programs', 'line'],
+ 'lines': []
+ },
+ 'process': {
+ 'options': ['processes', 'Processes statuses', 'processes', 'applications', 'monit.services', 'line'],
+ 'lines': []
+ },
+ 'process_uptime': {
+ 'options': ['processes uptime', 'Processes uptime', 'seconds', 'applications',
+ 'monit.process_uptime', 'line', 'hidden'],
+ 'lines': []
+ },
+ 'process_threads': {
+ 'options': ['processes threads', 'Processes threads', 'threads', 'applications',
+ 'monit.process_threads', 'line'],
+ 'lines': []
+ },
+ 'process_children': {
+ 'options': ['processes childrens', 'Child processes', 'childrens', 'applications',
+ 'monit.process_childrens', 'line'],
+ 'lines': []
+ },
+ 'host': {
+ 'options': ['hosts', 'Hosts', 'hosts', 'network', 'monit.hosts', 'line'],
+ 'lines': []
+ },
+ 'host_latency': {
+ 'options': ['hosts latency', 'Hosts latency', 'milliseconds', 'network', 'monit.host_latency', 'line'],
+ 'lines': []
+ },
+ 'net': {
+ 'options': ['interfaces', 'Network interfaces and addresses', 'interfaces', 'network',
+ 'monit.networks', 'line'],
+ 'lines': []
+ },
+}
+
+
+class BaseMonitService(object):
+ def __init__(self, typ, name, status, monitor):
+ self.type = typ
+ self.name = name
+ self.status = status
+ self.monitor = monitor
+
+ def __repr__(self):
+ return 'MonitService({0}:{1})'.format(self.type.name, self.name)
+
+ def __eq__(self, other):
+ if not isinstance(other, BaseMonitService):
+ return False
+ return self.type == other.type and self.name == other.name
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ return hash(repr(self))
+
+ def is_running(self):
+ return self.status == '0' and self.monitor == '1'
+
+ def key(self):
+ return '{0}_{1}'.format(self.type.name, self.name)
+
+ def data(self):
+ return {self.key(): int(self.is_running())}
+
+
+class ProcessMonitService(BaseMonitService):
+ def __init__(self, typ, name, status, monitor):
+ super(ProcessMonitService, self).__init__(typ, name, status, monitor)
+ self.uptime = None
+ self.threads = None
+ self.children = None
+
+ def __eq__(self, other):
+ return super(ProcessMonitService, self).__eq__(other)
+
+ def __ne__(self, other):
+ return super(ProcessMonitService, self).__ne__(other)
+
+ def __hash__(self):
+ return super(ProcessMonitService, self).__hash__()
+
+ def uptime_key(self):
+ return 'process_uptime_{0}'.format(self.name)
+
+ def threads_key(self):
+ return 'process_threads_{0}'.format(self.name)
+
+ def children_key(self):
+ return 'process_children_{0}'.format(self.name)
+
+ def data(self):
+ base_data = super(ProcessMonitService, self).data()
+ # skipping bugged metrics with negative uptime (monit before v5.16)
+ uptime = self.uptime if self.uptime and int(self.uptime) >= 0 else None
+ data = {
+ self.uptime_key(): uptime,
+ self.threads_key(): self.threads,
+ self.children_key(): self.children,
+ }
+ data.update(base_data)
+
+ return data
+
+
+class HostMonitService(BaseMonitService):
+ def __init__(self, typ, name, status, monitor):
+ super(HostMonitService, self).__init__(typ, name, status, monitor)
+ self.latency = None
+
+ def __eq__(self, other):
+ return super(HostMonitService, self).__eq__(other)
+
+ def __ne__(self, other):
+ return super(HostMonitService, self).__ne__(other)
+
+ def __hash__(self):
+ return super(HostMonitService, self).__hash__()
+
+ def latency_key(self):
+ return 'host_latency_{0}'.format(self.name)
+
+ def data(self):
+ base_data = super(HostMonitService, self).data()
+ latency = float(self.latency) * 1000000 if self.latency else None
+ data = {self.latency_key(): latency}
+ data.update(base_data)
+
+ return data
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ base_url = self.configuration.get('url', "http://localhost:2812")
+ self.url = '{0}/_status?format=xml&level=full'.format(base_url)
+ self.active_services = list()
+
+ def parse(self, raw):
+ try:
+ root = ET.fromstring(raw)
+ except ET.ParseError:
+ self.error("URL {0} didn't return a valid XML page. Please check your settings.".format(self.url))
+ return None
+ return root
+
+ def _get_data(self):
+ raw = self._get_raw_data()
+ if not raw:
+ return None
+
+ root = self.parse(raw)
+ if root is None:
+ return None
+
+ services = self.get_services(root)
+ if not services:
+ return None
+
+ if len(self.charts) > 0:
+ self.update_charts(services)
+
+ data = dict()
+
+ for svc in services:
+ data.update(svc.data())
+
+ return data
+
+ def get_services(self, root):
+ services = list()
+
+ for typ in TYPES:
+ if typ == TYPE_SYSTEM:
+ self.debug("skipping service from '{0}' category, it's useless in graphs".format(TYPE_SYSTEM.name))
+ continue
+
+ xpath_query = "./service[@type='{0}']".format(typ.index)
+ self.debug('Searching for {0} as {1}'.format(typ.name, xpath_query))
+
+ for svc_root in root.findall(xpath_query):
+ svc = create_service(svc_root, typ)
+ self.debug('=> found {0} with type={1}, status={2}, monitoring={3}'.format(
+ svc.name, svc.type.name, svc.status, svc.monitor))
+
+ services.append(svc)
+
+ return services
+
+ def update_charts(self, services):
+ remove = [svc for svc in self.active_services if svc not in services]
+ add = [svc for svc in services if svc not in self.active_services]
+
+ self.remove_services_from_charts(remove)
+ self.add_services_to_charts(add)
+
+ self.active_services = services
+
+ def add_services_to_charts(self, services):
+ for svc in services:
+ if svc.type == TYPE_HOST:
+ self.charts['host_latency'].add_dimension([svc.latency_key(), svc.name, 'absolute', 1000, 1000000])
+ if svc.type == TYPE_PROCESS:
+ self.charts['process_uptime'].add_dimension([svc.uptime_key(), svc.name])
+ self.charts['process_threads'].add_dimension([svc.threads_key(), svc.name])
+ self.charts['process_children'].add_dimension([svc.children_key(), svc.name])
+ self.charts[svc.type.name].add_dimension([svc.key(), svc.name])
+
+ def remove_services_from_charts(self, services):
+ for svc in services:
+ if svc.type == TYPE_HOST:
+ self.charts['host_latency'].del_dimension(svc.latency_key(), False)
+ if svc.type == TYPE_PROCESS:
+ self.charts['process_uptime'].del_dimension(svc.uptime_key(), False)
+ self.charts['process_threads'].del_dimension(svc.threads_key(), False)
+ self.charts['process_children'].del_dimension(svc.children_key(), False)
+ self.charts[svc.type.name].del_dimension(svc.key(), False)
+
+
+def create_service(root, typ):
+ if typ == TYPE_HOST:
+ return create_host_service(root)
+ elif typ == TYPE_PROCESS:
+ return create_process_service(root)
+ return create_base_service(root, typ)
+
+
+def create_host_service(root):
+ svc = HostMonitService(
+ TYPE_HOST,
+ root.find('name').text,
+ root.find('status').text,
+ root.find('monitor').text,
+ )
+
+ latency = root.find('./icmp/responsetime')
+ if latency is not None:
+ svc.latency = latency.text
+
+ return svc
+
+
+def create_process_service(root):
+ svc = ProcessMonitService(
+ TYPE_PROCESS,
+ root.find('name').text,
+ root.find('status').text,
+ root.find('monitor').text,
+ )
+
+ uptime = root.find('uptime')
+ if uptime is not None:
+ svc.uptime = uptime.text
+
+ threads = root.find('threads')
+ if threads is not None:
+ svc.threads = threads.text
+
+ children = root.find('children')
+ if children is not None:
+ svc.children = children.text
+
+ return svc
+
+
+def create_base_service(root, typ):
+ return BaseMonitService(
+ typ,
+ root.find('name').text,
+ root.find('status').text,
+ root.find('monitor').text,
+ )
diff --git a/collectors/python.d.plugin/monit/monit.conf b/collectors/python.d.plugin/monit/monit.conf
new file mode 100644
index 0000000..9a3fb69
--- /dev/null
+++ b/collectors/python.d.plugin/monit/monit.conf
@@ -0,0 +1,86 @@
+# netdata python.d.plugin configuration for monit
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, this plugin also supports the following:
+#
+# url: 'URL' # the URL to fetch monit's status stats
+#
+# if the URL is password protected, the following are supported:
+#
+# user: 'username'
+# pass: 'password'
+#
+# Example
+#
+# local:
+# name : 'Local Monit'
+# url : 'http://localhost:2812'
+#
+# "local" will show up in Netdata logs. "Reverse Proxy" will show up in the menu
+# in the monit section.
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name : 'local'
+ url : 'http://localhost:2812'
diff --git a/collectors/python.d.plugin/nsd/Makefile.inc b/collectors/python.d.plugin/nsd/Makefile.inc
new file mode 100644
index 0000000..58e9fd6
--- /dev/null
+++ b/collectors/python.d.plugin/nsd/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += nsd/nsd.chart.py
+dist_pythonconfig_DATA += nsd/nsd.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += nsd/README.md nsd/Makefile.inc
+
diff --git a/collectors/python.d.plugin/nsd/README.md b/collectors/python.d.plugin/nsd/README.md
new file mode 100644
index 0000000..e5183ae
--- /dev/null
+++ b/collectors/python.d.plugin/nsd/README.md
@@ -0,0 +1,68 @@
+<!--
+title: "NSD monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/nsd/README.md
+sidebar_label: "NSD"
+-->
+
+# NSD monitoring with Netdata
+
+Uses the `nsd-control stats_noreset` command to provide `nsd` statistics.
+
+## Requirements
+
+- Version of `nsd` must be 4.0+
+- Netdata must have permissions to run `nsd-control stats_noreset`
+
+It produces:
+
+1. **Queries**
+
+ - queries
+
+2. **Zones**
+
+ - master
+ - slave
+
+3. **Protocol**
+
+ - udp
+ - udp6
+ - tcp
+ - tcp6
+
+4. **Query Type**
+
+ - A
+ - NS
+ - CNAME
+ - SOA
+ - PTR
+ - HINFO
+ - MX
+ - NAPTR
+ - TXT
+ - AAAA
+ - SRV
+ - ANY
+
+5. **Transfer**
+
+ - NOTIFY
+ - AXFR
+
+6. **Return Code**
+
+ - NOERROR
+ - FORMERR
+ - SERVFAIL
+ - NXDOMAIN
+ - NOTIMP
+ - REFUSED
+ - YXDOMAIN
+
+Configuration is not needed.
+
+---
+
+
diff --git a/collectors/python.d.plugin/nsd/nsd.chart.py b/collectors/python.d.plugin/nsd/nsd.chart.py
new file mode 100644
index 0000000..6f9b2ce
--- /dev/null
+++ b/collectors/python.d.plugin/nsd/nsd.chart.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# Description: NSD `nsd-control stats_noreset` netdata python.d module
+# Author: <383c57 at gmail.com>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import re
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+
+update_every = 30
+
+NSD_CONTROL_COMMAND = 'nsd-control stats_noreset'
+REGEX = re.compile(r'([A-Za-z0-9.]+)=(\d+)')
+
+ORDER = [
+ 'queries',
+ 'zones',
+ 'protocol',
+ 'type',
+ 'transfer',
+ 'rcode',
+]
+
+CHARTS = {
+ 'queries': {
+ 'options': [None, 'queries', 'queries/s', 'queries', 'nsd.queries', 'line'],
+ 'lines': [
+ ['num_queries', 'queries', 'incremental']
+ ]
+ },
+ 'zones': {
+ 'options': [None, 'zones', 'zones', 'zones', 'nsd.zones', 'stacked'],
+ 'lines': [
+ ['zone_master', 'master', 'absolute'],
+ ['zone_slave', 'slave', 'absolute']
+ ]
+ },
+ 'protocol': {
+ 'options': [None, 'protocol', 'queries/s', 'protocol', 'nsd.protocols', 'stacked'],
+ 'lines': [
+ ['num_udp', 'udp', 'incremental'],
+ ['num_udp6', 'udp6', 'incremental'],
+ ['num_tcp', 'tcp', 'incremental'],
+ ['num_tcp6', 'tcp6', 'incremental']
+ ]
+ },
+ 'type': {
+ 'options': [None, 'query type', 'queries/s', 'query type', 'nsd.type', 'stacked'],
+ 'lines': [
+ ['num_type_A', 'A', 'incremental'],
+ ['num_type_NS', 'NS', 'incremental'],
+ ['num_type_CNAME', 'CNAME', 'incremental'],
+ ['num_type_SOA', 'SOA', 'incremental'],
+ ['num_type_PTR', 'PTR', 'incremental'],
+ ['num_type_HINFO', 'HINFO', 'incremental'],
+ ['num_type_MX', 'MX', 'incremental'],
+ ['num_type_NAPTR', 'NAPTR', 'incremental'],
+ ['num_type_TXT', 'TXT', 'incremental'],
+ ['num_type_AAAA', 'AAAA', 'incremental'],
+ ['num_type_SRV', 'SRV', 'incremental'],
+ ['num_type_TYPE255', 'ANY', 'incremental']
+ ]
+ },
+ 'transfer': {
+ 'options': [None, 'transfer', 'queries/s', 'transfer', 'nsd.transfer', 'stacked'],
+ 'lines': [
+ ['num_opcode_NOTIFY', 'NOTIFY', 'incremental'],
+ ['num_type_TYPE252', 'AXFR', 'incremental']
+ ]
+ },
+ 'rcode': {
+ 'options': [None, 'return code', 'queries/s', 'return code', 'nsd.rcode', 'stacked'],
+ 'lines': [
+ ['num_rcode_NOERROR', 'NOERROR', 'incremental'],
+ ['num_rcode_FORMERR', 'FORMERR', 'incremental'],
+ ['num_rcode_SERVFAIL', 'SERVFAIL', 'incremental'],
+ ['num_rcode_NXDOMAIN', 'NXDOMAIN', 'incremental'],
+ ['num_rcode_NOTIMP', 'NOTIMP', 'incremental'],
+ ['num_rcode_REFUSED', 'REFUSED', 'incremental'],
+ ['num_rcode_YXDOMAIN', 'YXDOMAIN', 'incremental']
+ ]
+ }
+}
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ ExecutableService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.command = NSD_CONTROL_COMMAND
+
+ def _get_data(self):
+ lines = self._get_raw_data()
+ if not lines:
+ return None
+
+ stats = dict(
+ (k.replace('.', '_'), int(v)) for k, v in REGEX.findall(''.join(lines))
+ )
+ stats.setdefault('num_opcode_NOTIFY', 0)
+ stats.setdefault('num_type_TYPE252', 0)
+ stats.setdefault('num_type_TYPE255', 0)
+
+ return stats
diff --git a/collectors/python.d.plugin/nsd/nsd.conf b/collectors/python.d.plugin/nsd/nsd.conf
new file mode 100644
index 0000000..77a8a31
--- /dev/null
+++ b/collectors/python.d.plugin/nsd/nsd.conf
@@ -0,0 +1,91 @@
+# netdata python.d.plugin configuration for nsd
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# nsd-control is slow, so once every 30 seconds
+# update_every: 30
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, nsd also supports the following:
+#
+# command: 'nsd-control stats_noreset' # the command to run
+#
+
+# ----------------------------------------------------------------------
+# IMPORTANT Information
+#
+# Netdata must have permissions to run `nsd-control stats_noreset` command
+#
+# - Example-1 (use "sudo")
+# 1. sudoers (e.g. visudo -f /etc/sudoers.d/netdata)
+# Defaults:netdata !requiretty
+# netdata ALL=(ALL) NOPASSWD: /usr/sbin/nsd-control stats_noreset
+# 2. etc/netdata/python.d/nsd.conf
+# local:
+# update_every: 30
+# command: 'sudo /usr/sbin/nsd-control stats_noreset'
+#
+# - Example-2 (add "netdata" user to "nsd" group)
+# usermod -aG nsd netdata
+#
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+
+local:
+ update_every: 30
+ command: 'nsd-control stats_noreset'
diff --git a/collectors/python.d.plugin/ntpd/Makefile.inc b/collectors/python.d.plugin/ntpd/Makefile.inc
new file mode 100644
index 0000000..81210eb
--- /dev/null
+++ b/collectors/python.d.plugin/ntpd/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += ntpd/ntpd.chart.py
+dist_pythonconfig_DATA += ntpd/ntpd.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += ntpd/README.md ntpd/Makefile.inc
+
diff --git a/collectors/python.d.plugin/ntpd/README.md b/collectors/python.d.plugin/ntpd/README.md
new file mode 100644
index 0000000..9832707
--- /dev/null
+++ b/collectors/python.d.plugin/ntpd/README.md
@@ -0,0 +1,90 @@
+<!--
+title: "NTP daemon monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/ntpd/README.md
+sidebar_label: "NTP daemon"
+-->
+
+# NTP daemon monitoring with Netdata
+
+Monitors the system variables of the local `ntpd` daemon (optional incl. variables of the polled peers) using the NTP Control Message Protocol via UDP socket, similar to `ntpq`, the [standard NTP query program](http://doc.ntp.org/current-stable/ntpq.html).
+
+## Requirements
+
+- Version: `NTPv4`
+- Local interrogation allowed in `/etc/ntp.conf` (default):
+
+```
+# Local users may interrogate the ntp server more closely.
+restrict 127.0.0.1
+restrict ::1
+```
+
+It produces:
+
+1. system
+
+ - offset
+ - jitter
+ - frequency
+ - delay
+ - dispersion
+ - stratum
+ - tc
+ - precision
+
+2. peers
+
+ - offset
+ - delay
+ - dispersion
+ - jitter
+ - rootdelay
+ - rootdispersion
+ - stratum
+ - hmode
+ - pmode
+ - hpoll
+ - ppoll
+ - precision
+
+## Configuration
+
+Edit the `python.d/ntpd.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/ntpd.conf
+```
+
+Sample:
+
+```yaml
+update_every: 10
+
+host: 'localhost'
+port: '123'
+show_peers: yes
+# hide peers with source address in ranges 127.0.0.0/8 and 192.168.0.0/16
+peer_filter: '(127\..*)|(192\.168\..*)'
+# check for new/changed peers every 60 updates
+peer_rescan: 60
+```
+
+Sample (multiple jobs):
+
+Note: `ntp.conf` on the host `otherhost` must be configured to allow queries from our local host by including a line like `restrict <IP> nomodify notrap nopeer`.
+
+```yaml
+local:
+ host: 'localhost'
+
+otherhost:
+ host: 'otherhost'
+```
+
+If no configuration is given, module will attempt to connect to `ntpd` on `::1:123` or `127.0.0.1:123` and show charts for the systemvars. Use `show_peers: yes` to also show the charts for configured peers. Local peers in the range `127.0.0.0/8` are hidden by default, use `peer_filter: ''` to show all peers.
+
+---
+
+
diff --git a/collectors/python.d.plugin/ntpd/ntpd.chart.py b/collectors/python.d.plugin/ntpd/ntpd.chart.py
new file mode 100644
index 0000000..275d227
--- /dev/null
+++ b/collectors/python.d.plugin/ntpd/ntpd.chart.py
@@ -0,0 +1,385 @@
+# -*- coding: utf-8 -*-
+# Description: ntpd netdata python.d module
+# Author: Sven Mäder (rda0)
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import re
+import struct
+
+from bases.FrameworkServices.SocketService import SocketService
+
+# NTP Control Message Protocol constants
+MODE = 6
+HEADER_FORMAT = '!BBHHHHH'
+HEADER_LEN = 12
+OPCODES = {
+ 'readstat': 1,
+ 'readvar': 2
+}
+
+# Maximal dimension precision
+PRECISION = 1000000
+
+# Static charts
+ORDER = [
+ 'sys_offset',
+ 'sys_jitter',
+ 'sys_frequency',
+ 'sys_wander',
+ 'sys_rootdelay',
+ 'sys_rootdisp',
+ 'sys_stratum',
+ 'sys_tc',
+ 'sys_precision',
+ 'peer_offset',
+ 'peer_delay',
+ 'peer_dispersion',
+ 'peer_jitter',
+ 'peer_xleave',
+ 'peer_rootdelay',
+ 'peer_rootdisp',
+ 'peer_stratum',
+ 'peer_hmode',
+ 'peer_pmode',
+ 'peer_hpoll',
+ 'peer_ppoll',
+ 'peer_precision'
+]
+
+CHARTS = {
+ 'sys_offset': {
+ 'options': [None, 'Combined offset of server relative to this host', 'milliseconds',
+ 'system', 'ntpd.sys_offset', 'area'],
+ 'lines': [
+ ['offset', 'offset', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'sys_jitter': {
+ 'options': [None, 'Combined system jitter and clock jitter', 'milliseconds',
+ 'system', 'ntpd.sys_jitter', 'line'],
+ 'lines': [
+ ['sys_jitter', 'system', 'absolute', 1, PRECISION],
+ ['clk_jitter', 'clock', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'sys_frequency': {
+ 'options': [None, 'Frequency offset relative to hardware clock', 'ppm', 'system', 'ntpd.sys_frequency', 'area'],
+ 'lines': [
+ ['frequency', 'frequency', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'sys_wander': {
+ 'options': [None, 'Clock frequency wander', 'ppm', 'system', 'ntpd.sys_wander', 'area'],
+ 'lines': [
+ ['clk_wander', 'clock', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'sys_rootdelay': {
+ 'options': [None, 'Total roundtrip delay to the primary reference clock', 'milliseconds', 'system',
+ 'ntpd.sys_rootdelay', 'area'],
+ 'lines': [
+ ['rootdelay', 'delay', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'sys_rootdisp': {
+ 'options': [None, 'Total root dispersion to the primary reference clock', 'milliseconds', 'system',
+ 'ntpd.sys_rootdisp', 'area'],
+ 'lines': [
+ ['rootdisp', 'dispersion', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'sys_stratum': {
+ 'options': [None, 'Stratum (1-15)', 'stratum', 'system', 'ntpd.sys_stratum', 'line'],
+ 'lines': [
+ ['stratum', 'stratum', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'sys_tc': {
+ 'options': [None, 'Time constant and poll exponent (3-17)', 'log2 s', 'system', 'ntpd.sys_tc', 'line'],
+ 'lines': [
+ ['tc', 'current', 'absolute', 1, PRECISION],
+ ['mintc', 'minimum', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'sys_precision': {
+ 'options': [None, 'Precision', 'log2 s', 'system', 'ntpd.sys_precision', 'line'],
+ 'lines': [
+ ['precision', 'precision', 'absolute', 1, PRECISION]
+ ]
+ }
+}
+
+PEER_CHARTS = {
+ 'peer_offset': {
+ 'options': [None, 'Filter offset', 'milliseconds', 'peers', 'ntpd.peer_offset', 'line'],
+ 'lines': []
+ },
+ 'peer_delay': {
+ 'options': [None, 'Filter delay', 'milliseconds', 'peers', 'ntpd.peer_delay', 'line'],
+ 'lines': []
+ },
+ 'peer_dispersion': {
+ 'options': [None, 'Filter dispersion', 'milliseconds', 'peers', 'ntpd.peer_dispersion', 'line'],
+ 'lines': []
+ },
+ 'peer_jitter': {
+ 'options': [None, 'Filter jitter', 'milliseconds', 'peers', 'ntpd.peer_jitter', 'line'],
+ 'lines': []
+ },
+ 'peer_xleave': {
+ 'options': [None, 'Interleave delay', 'milliseconds', 'peers', 'ntpd.peer_xleave', 'line'],
+ 'lines': []
+ },
+ 'peer_rootdelay': {
+ 'options': [None, 'Total roundtrip delay to the primary reference clock', 'milliseconds', 'peers',
+ 'ntpd.peer_rootdelay', 'line'],
+ 'lines': []
+ },
+ 'peer_rootdisp': {
+ 'options': [None, 'Total root dispersion to the primary reference clock', 'ms', 'peers',
+ 'ntpd.peer_rootdisp', 'line'],
+ 'lines': []
+ },
+ 'peer_stratum': {
+ 'options': [None, 'Stratum (1-15)', 'stratum', 'peers', 'ntpd.peer_stratum', 'line'],
+ 'lines': []
+ },
+ 'peer_hmode': {
+ 'options': [None, 'Host mode (1-6)', 'hmode', 'peers', 'ntpd.peer_hmode', 'line'],
+ 'lines': []
+ },
+ 'peer_pmode': {
+ 'options': [None, 'Peer mode (1-5)', 'pmode', 'peers', 'ntpd.peer_pmode', 'line'],
+ 'lines': []
+ },
+ 'peer_hpoll': {
+ 'options': [None, 'Host poll exponent', 'log2 s', 'peers', 'ntpd.peer_hpoll', 'line'],
+ 'lines': []
+ },
+ 'peer_ppoll': {
+ 'options': [None, 'Peer poll exponent', 'log2 s', 'peers', 'ntpd.peer_ppoll', 'line'],
+ 'lines': []
+ },
+ 'peer_precision': {
+ 'options': [None, 'Precision', 'log2 s', 'peers', 'ntpd.peer_precision', 'line'],
+ 'lines': []
+ }
+}
+
+
+class Base:
+ regex = re.compile(r'([a-z_]+)=((?:-)?[0-9]+(?:\.[0-9]+)?)')
+
+ @staticmethod
+ def get_header(associd=0, operation='readvar'):
+ """
+ Constructs the NTP Control Message header:
+ 0 1 2 3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ |LI | VN |Mode |R|E|M| OpCode | Sequence Number |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Status | Association ID |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Offset | Count |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ """
+ version = 2
+ sequence = 1
+ status = 0
+ offset = 0
+ count = 0
+ header = struct.pack(HEADER_FORMAT, (version << 3 | MODE), OPCODES[operation],
+ sequence, status, associd, offset, count)
+ return header
+
+
+class System(Base):
+ def __init__(self):
+ self.request = self.get_header()
+
+ def get_data(self, raw):
+ """
+ Extracts key=value pairs with float/integer from ntp response packet data.
+ """
+ data = dict()
+ for key, value in self.regex.findall(raw):
+ data[key] = float(value) * PRECISION
+ return data
+
+
+class Peer(Base):
+ def __init__(self, idx, name):
+ self.id = idx
+ self.real_name = name
+ self.name = name.replace('.', '_')
+ self.request = self.get_header(self.id)
+
+ def get_data(self, raw):
+ """
+ Extracts key=value pairs with float/integer from ntp response packet data.
+ """
+ data = dict()
+ for key, value in self.regex.findall(raw):
+ dimension = '_'.join([self.name, key])
+ data[dimension] = float(value) * PRECISION
+ return data
+
+
+class Service(SocketService):
+ def __init__(self, configuration=None, name=None):
+ SocketService.__init__(self, configuration=configuration, name=name)
+ self.order = list(ORDER)
+ self.definitions = dict(CHARTS)
+ self.port = 'ntp'
+ self.dgram_socket = True
+ self.system = System()
+ self.peers = dict()
+ self.request = str()
+ self.retries = 0
+ self.show_peers = self.configuration.get('show_peers', False)
+ self.peer_rescan = self.configuration.get('peer_rescan', 60)
+ if self.show_peers:
+ self.definitions.update(PEER_CHARTS)
+
+ def check(self):
+ """
+ Checks if we can get valid systemvars.
+ If not, returns None to disable module.
+ """
+ self._parse_config()
+
+ peer_filter = self.configuration.get('peer_filter', r'127\..*')
+ try:
+ self.peer_filter = re.compile(r'^((0\.0\.0\.0)|({0}))$'.format(peer_filter))
+ except re.error as error:
+ self.error('Compile pattern error (peer_filter) : {0}'.format(error))
+ return None
+
+ self.request = self.system.request
+ raw_systemvars = self._get_raw_data()
+
+ if not self.system.get_data(raw_systemvars):
+ return None
+
+ return True
+
+ def get_data(self):
+ """
+ Gets systemvars data on each update.
+ Gets peervars data for all peers on each update.
+ """
+ data = dict()
+
+ self.request = self.system.request
+ raw = self._get_raw_data()
+ if not raw:
+ return None
+
+ data.update(self.system.get_data(raw))
+
+ if not self.show_peers:
+ return data
+
+ if not self.peers or self.runs_counter % self.peer_rescan == 0 or self.retries > 8:
+ self.find_new_peers()
+
+ for peer in self.peers.values():
+ self.request = peer.request
+ peer_data = peer.get_data(self._get_raw_data())
+ if peer_data:
+ data.update(peer_data)
+ else:
+ self.retries += 1
+
+ return data
+
+ def find_new_peers(self):
+ new_peers = dict((p.real_name, p) for p in self.get_peers())
+ if new_peers:
+
+ peers_to_remove = set(self.peers) - set(new_peers)
+ peers_to_add = set(new_peers) - set(self.peers)
+
+ for peer_name in peers_to_remove:
+ self.hide_old_peer_from_charts(self.peers[peer_name])
+ del self.peers[peer_name]
+
+ for peer_name in peers_to_add:
+ self.add_new_peer_to_charts(new_peers[peer_name])
+
+ self.peers.update(new_peers)
+ self.retries = 0
+
+ def add_new_peer_to_charts(self, peer):
+ for chart_id in set(self.charts.charts) & set(PEER_CHARTS):
+ dim_id = peer.name + chart_id[4:]
+ if dim_id not in self.charts[chart_id]:
+ self.charts[chart_id].add_dimension([dim_id, peer.real_name, 'absolute', 1, PRECISION])
+ else:
+ self.charts[chart_id].hide_dimension(dim_id, reverse=True)
+
+ def hide_old_peer_from_charts(self, peer):
+ for chart_id in set(self.charts.charts) & set(PEER_CHARTS):
+ dim_id = peer.name + chart_id[4:]
+ self.charts[chart_id].hide_dimension(dim_id)
+
+ def get_peers(self):
+ self.request = Base.get_header(operation='readstat')
+
+ raw_data = self._get_raw_data(raw=True)
+ if not raw_data:
+ return list()
+
+ peer_ids = self.get_peer_ids(raw_data)
+ if not peer_ids:
+ return list()
+
+ new_peers = list()
+ for peer_id in peer_ids:
+ self.request = Base.get_header(peer_id)
+ raw_peer_data = self._get_raw_data()
+ if not raw_peer_data:
+ continue
+ srcadr = re.search(r'(srcadr)=([^,]+)', raw_peer_data)
+ if not srcadr:
+ continue
+ srcadr = srcadr.group(2)
+ if self.peer_filter.search(srcadr):
+ continue
+ stratum = re.search(r'(stratum)=([^,]+)', raw_peer_data)
+ if not stratum:
+ continue
+ if int(stratum.group(2)) > 15:
+ continue
+
+ new_peer = Peer(idx=peer_id, name=srcadr)
+ new_peers.append(new_peer)
+ return new_peers
+
+ def get_peer_ids(self, res):
+ """
+ Unpack the NTP Control Message header
+ Get data length from header
+ Get list of association ids returned in the readstat response
+ """
+
+ try:
+ count = struct.unpack(HEADER_FORMAT, res[:HEADER_LEN])[6]
+ except struct.error as error:
+ self.error('error unpacking header: {0}'.format(error))
+ return None
+ if not count:
+ self.error('empty data field in NTP control packet')
+ return None
+
+ data_end = HEADER_LEN + count
+ data = res[HEADER_LEN:data_end]
+ data_format = ''.join(['!', 'H' * int(count / 2)])
+ try:
+ peer_ids = list(struct.unpack(data_format, data))[::2]
+ except struct.error as error:
+ self.error('error unpacking data: {0}'.format(error))
+ return None
+ return peer_ids
diff --git a/collectors/python.d.plugin/ntpd/ntpd.conf b/collectors/python.d.plugin/ntpd/ntpd.conf
new file mode 100644
index 0000000..80bd468
--- /dev/null
+++ b/collectors/python.d.plugin/ntpd/ntpd.conf
@@ -0,0 +1,89 @@
+# netdata python.d.plugin configuration for ntpd
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+#
+# Additionally to the above, ntp also supports the following:
+#
+# host: 'localhost' # the host to query
+# port: '123' # the UDP port where `ntpd` listens
+# show_peers: no # use `yes` to show peer charts. enabling this
+# # option is recommended only for debugging, as
+# # it could possibly imply memory leaks if the
+# # peers change frequently.
+# peer_filter: '127\..*' # regex to exclude peers
+# # by default local peers are hidden
+# # use `''` to show all peers.
+# peer_rescan: 60 # interval (>0) to check for new/changed peers
+# # use `1` to check on every update
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name: 'local'
+ host: 'localhost'
+ port: '123'
+ show_peers: no
+
+localhost_ipv4:
+ name: 'local'
+ host: '127.0.0.1'
+ port: '123'
+ show_peers: no
+
+localhost_ipv6:
+ name: 'local'
+ host: '::1'
+ port: '123'
+ show_peers: no
diff --git a/collectors/python.d.plugin/nvidia_smi/Makefile.inc b/collectors/python.d.plugin/nvidia_smi/Makefile.inc
new file mode 100644
index 0000000..52fb25a
--- /dev/null
+++ b/collectors/python.d.plugin/nvidia_smi/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += nvidia_smi/nvidia_smi.chart.py
+dist_pythonconfig_DATA += nvidia_smi/nvidia_smi.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += nvidia_smi/README.md nvidia_smi/Makefile.inc
+
diff --git a/collectors/python.d.plugin/nvidia_smi/README.md b/collectors/python.d.plugin/nvidia_smi/README.md
new file mode 100644
index 0000000..bb41694
--- /dev/null
+++ b/collectors/python.d.plugin/nvidia_smi/README.md
@@ -0,0 +1,66 @@
+<!--
+title: "Nvidia GPU monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/nvidia_smi/README.md
+sidebar_label: "Nvidia GPUs"
+-->
+
+# Nvidia GPU monitoring with Netdata
+
+Monitors performance metrics (memory usage, fan speed, pcie bandwidth utilization, temperature, etc.) using `nvidia-smi` cli tool.
+
+> **Warning**: this collector does not work when the Netdata Agent is [running in a container](https://learn.netdata.cloud/docs/agent/packaging/docker).
+
+
+## Requirements and Notes
+
+- You must have the `nvidia-smi` tool installed and your NVIDIA GPU(s) must support the tool. Mostly the newer high end models used for AI / ML and Crypto or Pro range, read more about [nvidia_smi](https://developer.nvidia.com/nvidia-system-management-interface).
+- You must enable this plugin, as its disabled by default due to minor performance issues:
+ ```bash
+ cd /etc/netdata # Replace this path with your Netdata config directory, if different
+ sudo ./edit-config python.d.conf
+ ```
+ Remove the '#' before nvidia_smi so it reads: `nvidia_smi: yes`.
+
+- On some systems when the GPU is idle the `nvidia-smi` tool unloads and there is added latency again when it is next queried. If you are running GPUs under constant workload this isn't likely to be an issue.
+- Currently the `nvidia-smi` tool is being queried via cli. Updating the plugin to use the nvidia c/c++ API directly should resolve this issue. See discussion here: <https://github.com/netdata/netdata/pull/4357>
+- Contributions are welcome.
+- Make sure `netdata` user can execute `/usr/bin/nvidia-smi` or wherever your binary is.
+- If `nvidia-smi` process [is not killed after netdata restart](https://github.com/netdata/netdata/issues/7143) you need to off `loop_mode`.
+- `poll_seconds` is how often in seconds the tool is polled for as an integer.
+
+## Charts
+
+It produces the following charts:
+
+- PCI Express Bandwidth Utilization in `KiB/s`
+- Fan Speed in `percentage`
+- GPU Utilization in `percentage`
+- Memory Bandwidth Utilization in `percentage`
+- Encoder/Decoder Utilization in `percentage`
+- Memory Usage in `MiB`
+- Temperature in `celsius`
+- Clock Frequencies in `MHz`
+- Power Utilization in `Watts`
+- Memory Used by Each Process in `MiB`
+- Memory Used by Each User in `MiB`
+- Number of User on GPU in `num`
+
+## Configuration
+
+Edit the `python.d/nvidia_smi.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/nvidia_smi.conf
+```
+
+Sample:
+
+```yaml
+loop_mode : yes
+poll_seconds : 1
+exclude_zero_memory_users : yes
+```
+
+
diff --git a/collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py b/collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py
new file mode 100644
index 0000000..23e90e6
--- /dev/null
+++ b/collectors/python.d.plugin/nvidia_smi/nvidia_smi.chart.py
@@ -0,0 +1,589 @@
+# -*- coding: utf-8 -*-
+# Description: nvidia-smi netdata python.d module
+# Original Author: Steven Noonan (tycho)
+# Author: Ilya Mashchenko (ilyam8)
+# User Memory Stat Author: Guido Scatena (scatenag)
+
+import os
+import pwd
+import subprocess
+import threading
+import xml.etree.ElementTree as et
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from bases.collection import find_binary
+
+disabled_by_default = True
+
+NVIDIA_SMI = 'nvidia-smi'
+
+EMPTY_ROW = ''
+EMPTY_ROW_LIMIT = 500
+POLLER_BREAK_ROW = '</nvidia_smi_log>'
+
+PCI_BANDWIDTH = 'pci_bandwidth'
+FAN_SPEED = 'fan_speed'
+GPU_UTIL = 'gpu_utilization'
+MEM_UTIL = 'mem_utilization'
+ENCODER_UTIL = 'encoder_utilization'
+MEM_USAGE = 'mem_usage'
+BAR_USAGE = 'bar1_mem_usage'
+TEMPERATURE = 'temperature'
+CLOCKS = 'clocks'
+POWER = 'power'
+POWER_STATE = 'power_state'
+PROCESSES_MEM = 'processes_mem'
+USER_MEM = 'user_mem'
+USER_NUM = 'user_num'
+
+ORDER = [
+ PCI_BANDWIDTH,
+ FAN_SPEED,
+ GPU_UTIL,
+ MEM_UTIL,
+ ENCODER_UTIL,
+ MEM_USAGE,
+ BAR_USAGE,
+ TEMPERATURE,
+ CLOCKS,
+ POWER,
+ POWER_STATE,
+ PROCESSES_MEM,
+ USER_MEM,
+ USER_NUM,
+]
+
+# https://docs.nvidia.com/gameworks/content/gameworkslibrary/coresdk/nvapi/group__gpupstate.html
+POWER_STATES = ['P' + str(i) for i in range(0, 16)]
+
+
+def gpu_charts(gpu):
+ fam = gpu.full_name()
+
+ charts = {
+ PCI_BANDWIDTH: {
+ 'options': [None, 'PCI Express Bandwidth Utilization', 'KiB/s', fam, 'nvidia_smi.pci_bandwidth', 'area'],
+ 'lines': [
+ ['rx_util', 'rx', 'absolute', 1, 1],
+ ['tx_util', 'tx', 'absolute', 1, -1],
+ ]
+ },
+ FAN_SPEED: {
+ 'options': [None, 'Fan Speed', 'percentage', fam, 'nvidia_smi.fan_speed', 'line'],
+ 'lines': [
+ ['fan_speed', 'speed'],
+ ]
+ },
+ GPU_UTIL: {
+ 'options': [None, 'GPU Utilization', 'percentage', fam, 'nvidia_smi.gpu_utilization', 'line'],
+ 'lines': [
+ ['gpu_util', 'utilization'],
+ ]
+ },
+ MEM_UTIL: {
+ 'options': [None, 'Memory Bandwidth Utilization', 'percentage', fam, 'nvidia_smi.mem_utilization', 'line'],
+ 'lines': [
+ ['memory_util', 'utilization'],
+ ]
+ },
+ ENCODER_UTIL: {
+ 'options': [None, 'Encoder/Decoder Utilization', 'percentage', fam, 'nvidia_smi.encoder_utilization',
+ 'line'],
+ 'lines': [
+ ['encoder_util', 'encoder'],
+ ['decoder_util', 'decoder'],
+ ]
+ },
+ MEM_USAGE: {
+ 'options': [None, 'Memory Usage', 'MiB', fam, 'nvidia_smi.memory_allocated', 'stacked'],
+ 'lines': [
+ ['fb_memory_free', 'free'],
+ ['fb_memory_used', 'used'],
+ ]
+ },
+ BAR_USAGE: {
+ 'options': [None, 'Bar1 Memory Usage', 'MiB', fam, 'nvidia_smi.bar1_memory_usage', 'stacked'],
+ 'lines': [
+ ['bar1_memory_free', 'free'],
+ ['bar1_memory_used', 'used'],
+ ]
+ },
+ TEMPERATURE: {
+ 'options': [None, 'Temperature', 'celsius', fam, 'nvidia_smi.temperature', 'line'],
+ 'lines': [
+ ['gpu_temp', 'temp'],
+ ]
+ },
+ CLOCKS: {
+ 'options': [None, 'Clock Frequencies', 'MHz', fam, 'nvidia_smi.clocks', 'line'],
+ 'lines': [
+ ['graphics_clock', 'graphics'],
+ ['video_clock', 'video'],
+ ['sm_clock', 'sm'],
+ ['mem_clock', 'mem'],
+ ]
+ },
+ POWER: {
+ 'options': [None, 'Power Utilization', 'Watts', fam, 'nvidia_smi.power', 'line'],
+ 'lines': [
+ ['power_draw', 'power', 'absolute', 1, 100],
+ ]
+ },
+ POWER_STATE: {
+ 'options': [None, 'Power State', 'state', fam, 'nvidia_smi.power_state', 'line'],
+ 'lines': [['power_state_' + v.lower(), v, 'absolute'] for v in POWER_STATES]
+ },
+ PROCESSES_MEM: {
+ 'options': [None, 'Memory Used by Each Process', 'MiB', fam, 'nvidia_smi.processes_mem', 'stacked'],
+ 'lines': []
+ },
+ USER_MEM: {
+ 'options': [None, 'Memory Used by Each User', 'MiB', fam, 'nvidia_smi.user_mem', 'stacked'],
+ 'lines': []
+ },
+ USER_NUM: {
+ 'options': [None, 'Number of User on GPU', 'num', fam, 'nvidia_smi.user_num', 'line'],
+ 'lines': [
+ ['user_num', 'users'],
+ ]
+ },
+ }
+
+ idx = gpu.num
+
+ order = ['gpu{0}_{1}'.format(idx, v) for v in ORDER]
+ charts = dict(('gpu{0}_{1}'.format(idx, k), v) for k, v in charts.items())
+
+ for chart in charts.values():
+ for line in chart['lines']:
+ line[0] = 'gpu{0}_{1}'.format(idx, line[0])
+
+ return order, charts
+
+
+class NvidiaSMI:
+ def __init__(self):
+ self.command = find_binary(NVIDIA_SMI)
+ self.active_proc = None
+
+ def run_once(self):
+ proc = subprocess.Popen([self.command, '-x', '-q'], stdout=subprocess.PIPE)
+ stdout, _ = proc.communicate()
+ return stdout
+
+ def run_loop(self, interval):
+ if self.active_proc:
+ self.kill()
+ proc = subprocess.Popen([self.command, '-x', '-q', '-l', str(interval)], stdout=subprocess.PIPE)
+ self.active_proc = proc
+ return proc.stdout
+
+ def kill(self):
+ if self.active_proc:
+ self.active_proc.kill()
+ self.active_proc = None
+
+
+class NvidiaSMIPoller(threading.Thread):
+ def __init__(self, poll_interval):
+ threading.Thread.__init__(self)
+ self.daemon = True
+
+ self.smi = NvidiaSMI()
+ self.interval = poll_interval
+
+ self.lock = threading.RLock()
+ self.last_data = str()
+ self.exit = False
+ self.empty_rows = 0
+ self.rows = list()
+
+ def has_smi(self):
+ return bool(self.smi.command)
+
+ def run_once(self):
+ return self.smi.run_once()
+
+ def run(self):
+ out = self.smi.run_loop(self.interval)
+
+ for row in out:
+ if self.exit or self.empty_rows > EMPTY_ROW_LIMIT:
+ break
+ self.process_row(row)
+ self.smi.kill()
+
+ def process_row(self, row):
+ row = row.decode()
+ self.empty_rows += (row == EMPTY_ROW)
+ self.rows.append(row)
+
+ if POLLER_BREAK_ROW in row:
+ self.lock.acquire()
+ self.last_data = '\n'.join(self.rows)
+ self.lock.release()
+
+ self.rows = list()
+ self.empty_rows = 0
+
+ def is_started(self):
+ return self.ident is not None
+
+ def shutdown(self):
+ self.exit = True
+
+ def data(self):
+ self.lock.acquire()
+ data = self.last_data
+ self.lock.release()
+ return data
+
+
+def handle_attr_error(method):
+ def on_call(*args, **kwargs):
+ try:
+ return method(*args, **kwargs)
+ except AttributeError:
+ return None
+
+ return on_call
+
+
+def handle_value_error(method):
+ def on_call(*args, **kwargs):
+ try:
+ return method(*args, **kwargs)
+ except ValueError:
+ return None
+
+ return on_call
+
+
+HOST_PREFIX = os.getenv('NETDATA_HOST_PREFIX')
+ETC_PASSWD_PATH = '/etc/passwd'
+PROC_PATH = '/proc'
+
+IS_INSIDE_DOCKER = False
+
+if HOST_PREFIX:
+ ETC_PASSWD_PATH = os.path.join(HOST_PREFIX, ETC_PASSWD_PATH[1:])
+ PROC_PATH = os.path.join(HOST_PREFIX, PROC_PATH[1:])
+ IS_INSIDE_DOCKER = True
+
+
+def read_passwd_file():
+ data = dict()
+ with open(ETC_PASSWD_PATH, 'r') as f:
+ for line in f:
+ line = line.strip()
+ if line.startswith("#"):
+ continue
+ fields = line.split(":")
+ # name, passwd, uid, gid, comment, home_dir, shell
+ if len(fields) != 7:
+ continue
+ # uid, guid
+ fields[2], fields[3] = int(fields[2]), int(fields[3])
+ data[fields[2]] = fields
+ return data
+
+
+def read_passwd_file_safe():
+ try:
+ if IS_INSIDE_DOCKER:
+ return read_passwd_file()
+ return dict((k[2], k) for k in pwd.getpwall())
+ except (OSError, IOError):
+ return dict()
+
+
+def get_username_by_pid_safe(pid, passwd_file):
+ path = os.path.join(PROC_PATH, pid)
+ try:
+ uid = os.stat(path).st_uid
+ except (OSError, IOError):
+ return ''
+ try:
+ if IS_INSIDE_DOCKER:
+ return passwd_file[uid][0]
+ return pwd.getpwuid(uid)[0]
+ except KeyError:
+ return str(uid)
+
+
+class GPU:
+ def __init__(self, num, root, exclude_zero_memory_users=False):
+ self.num = num
+ self.root = root
+ self.exclude_zero_memory_users = exclude_zero_memory_users
+
+ def id(self):
+ return self.root.get('id')
+
+ def name(self):
+ return self.root.find('product_name').text
+
+ def full_name(self):
+ return 'gpu{0} {1}'.format(self.num, self.name())
+
+ @handle_attr_error
+ def rx_util(self):
+ return self.root.find('pci').find('rx_util').text.split()[0]
+
+ @handle_attr_error
+ def tx_util(self):
+ return self.root.find('pci').find('tx_util').text.split()[0]
+
+ @handle_attr_error
+ def fan_speed(self):
+ return self.root.find('fan_speed').text.split()[0]
+
+ @handle_attr_error
+ def gpu_util(self):
+ return self.root.find('utilization').find('gpu_util').text.split()[0]
+
+ @handle_attr_error
+ def memory_util(self):
+ return self.root.find('utilization').find('memory_util').text.split()[0]
+
+ @handle_attr_error
+ def encoder_util(self):
+ return self.root.find('utilization').find('encoder_util').text.split()[0]
+
+ @handle_attr_error
+ def decoder_util(self):
+ return self.root.find('utilization').find('decoder_util').text.split()[0]
+
+ @handle_attr_error
+ def fb_memory_used(self):
+ return self.root.find('fb_memory_usage').find('used').text.split()[0]
+
+ @handle_attr_error
+ def fb_memory_free(self):
+ return self.root.find('fb_memory_usage').find('free').text.split()[0]
+
+ @handle_attr_error
+ def bar1_memory_used(self):
+ return self.root.find('bar1_memory_usage').find('used').text.split()[0]
+
+ @handle_attr_error
+ def bar1_memory_free(self):
+ return self.root.find('bar1_memory_usage').find('free').text.split()[0]
+
+ @handle_attr_error
+ def temperature(self):
+ return self.root.find('temperature').find('gpu_temp').text.split()[0]
+
+ @handle_attr_error
+ def graphics_clock(self):
+ return self.root.find('clocks').find('graphics_clock').text.split()[0]
+
+ @handle_attr_error
+ def video_clock(self):
+ return self.root.find('clocks').find('video_clock').text.split()[0]
+
+ @handle_attr_error
+ def sm_clock(self):
+ return self.root.find('clocks').find('sm_clock').text.split()[0]
+
+ @handle_attr_error
+ def mem_clock(self):
+ return self.root.find('clocks').find('mem_clock').text.split()[0]
+
+ @handle_attr_error
+ def power_state(self):
+ return str(self.root.find('power_readings').find('power_state').text.split()[0])
+
+ @handle_value_error
+ @handle_attr_error
+ def power_draw(self):
+ return float(self.root.find('power_readings').find('power_draw').text.split()[0]) * 100
+
+ @handle_attr_error
+ def processes(self):
+ processes_info = self.root.find('processes').findall('process_info')
+ if not processes_info:
+ return list()
+
+ passwd_file = read_passwd_file_safe()
+ processes = list()
+
+ for info in processes_info:
+ pid = info.find('pid').text
+ processes.append({
+ 'pid': int(pid),
+ 'process_name': info.find('process_name').text,
+ 'used_memory': int(info.find('used_memory').text.split()[0]),
+ 'username': get_username_by_pid_safe(pid, passwd_file),
+ })
+ return processes
+
+ def data(self):
+ data = {
+ 'rx_util': self.rx_util(),
+ 'tx_util': self.tx_util(),
+ 'fan_speed': self.fan_speed(),
+ 'gpu_util': self.gpu_util(),
+ 'memory_util': self.memory_util(),
+ 'encoder_util': self.encoder_util(),
+ 'decoder_util': self.decoder_util(),
+ 'fb_memory_used': self.fb_memory_used(),
+ 'fb_memory_free': self.fb_memory_free(),
+ 'bar1_memory_used': self.bar1_memory_used(),
+ 'bar1_memory_free': self.bar1_memory_free(),
+ 'gpu_temp': self.temperature(),
+ 'graphics_clock': self.graphics_clock(),
+ 'video_clock': self.video_clock(),
+ 'sm_clock': self.sm_clock(),
+ 'mem_clock': self.mem_clock(),
+ 'power_draw': self.power_draw(),
+ }
+
+ for v in POWER_STATES:
+ data['power_state_' + v.lower()] = 0
+ p_state = self.power_state()
+ if p_state:
+ data['power_state_' + p_state.lower()] = 1
+
+ processes = self.processes() or []
+ users = set()
+ for p in processes:
+ data['process_mem_{0}'.format(p['pid'])] = p['used_memory']
+ if p['username']:
+ if self.exclude_zero_memory_users and p['used_memory'] == 0:
+ continue
+ users.add(p['username'])
+ key = 'user_mem_{0}'.format(p['username'])
+ if key in data:
+ data[key] += p['used_memory']
+ else:
+ data[key] = p['used_memory']
+ data['user_num'] = len(users)
+
+ return dict(('gpu{0}_{1}'.format(self.num, k), v) for k, v in data.items())
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ super(Service, self).__init__(configuration=configuration, name=name)
+ self.order = list()
+ self.definitions = dict()
+ self.loop_mode = configuration.get('loop_mode', True)
+ poll = int(configuration.get('poll_seconds', self.get_update_every()))
+ self.exclude_zero_memory_users = configuration.get('exclude_zero_memory_users', False)
+ self.poller = NvidiaSMIPoller(poll)
+
+ def get_data_loop_mode(self):
+ if not self.poller.is_started():
+ self.poller.start()
+
+ if not self.poller.is_alive():
+ self.debug('poller is off')
+ return None
+
+ return self.poller.data()
+
+ def get_data_normal_mode(self):
+ return self.poller.run_once()
+
+ def get_data(self):
+ if self.loop_mode:
+ last_data = self.get_data_loop_mode()
+ else:
+ last_data = self.get_data_normal_mode()
+
+ if not last_data:
+ return None
+
+ parsed = self.parse_xml(last_data)
+ if parsed is None:
+ return None
+
+ data = dict()
+ for idx, root in enumerate(parsed.findall('gpu')):
+ gpu = GPU(idx, root, self.exclude_zero_memory_users)
+ gpu_data = gpu.data()
+ # self.debug(gpu_data)
+ gpu_data = dict((k, v) for k, v in gpu_data.items() if is_gpu_data_value_valid(v))
+ data.update(gpu_data)
+ self.update_processes_mem_chart(gpu)
+ self.update_processes_user_mem_chart(gpu)
+
+ return data or None
+
+ def update_processes_mem_chart(self, gpu):
+ ps = gpu.processes()
+ if not ps:
+ return
+ chart = self.charts['gpu{0}_{1}'.format(gpu.num, PROCESSES_MEM)]
+ active_dim_ids = []
+ for p in ps:
+ dim_id = 'gpu{0}_process_mem_{1}'.format(gpu.num, p['pid'])
+ active_dim_ids.append(dim_id)
+ if dim_id not in chart:
+ chart.add_dimension([dim_id, '{0} {1}'.format(p['pid'], p['process_name'])])
+ for dim in chart:
+ if dim.id not in active_dim_ids:
+ chart.del_dimension(dim.id, hide=False)
+
+ def update_processes_user_mem_chart(self, gpu):
+ ps = gpu.processes()
+ if not ps:
+ return
+ chart = self.charts['gpu{0}_{1}'.format(gpu.num, USER_MEM)]
+ active_dim_ids = []
+ for p in ps:
+ if not p.get('username'):
+ continue
+ dim_id = 'gpu{0}_user_mem_{1}'.format(gpu.num, p['username'])
+ active_dim_ids.append(dim_id)
+ if dim_id not in chart:
+ chart.add_dimension([dim_id, '{0}'.format(p['username'])])
+
+ for dim in chart:
+ if dim.id not in active_dim_ids:
+ chart.del_dimension(dim.id, hide=False)
+
+ def check(self):
+ if not self.poller.has_smi():
+ self.error("couldn't find '{0}' binary".format(NVIDIA_SMI))
+ return False
+
+ raw_data = self.poller.run_once()
+ if not raw_data:
+ self.error("failed to invoke '{0}' binary".format(NVIDIA_SMI))
+ return False
+
+ parsed = self.parse_xml(raw_data)
+ if parsed is None:
+ return False
+
+ gpus = parsed.findall('gpu')
+ if not gpus:
+ return False
+
+ self.create_charts(gpus)
+
+ return True
+
+ def parse_xml(self, data):
+ try:
+ return et.fromstring(data)
+ except et.ParseError as error:
+ self.error('xml parse failed: "{0}", error: {1}'.format(data, error))
+
+ return None
+
+ def create_charts(self, gpus):
+ for idx, root in enumerate(gpus):
+ order, charts = gpu_charts(GPU(idx, root))
+ self.order.extend(order)
+ self.definitions.update(charts)
+
+
+def is_gpu_data_value_valid(value):
+ try:
+ int(value)
+ except (TypeError, ValueError):
+ return False
+ return True
diff --git a/collectors/python.d.plugin/nvidia_smi/nvidia_smi.conf b/collectors/python.d.plugin/nvidia_smi/nvidia_smi.conf
new file mode 100644
index 0000000..3d2a30d
--- /dev/null
+++ b/collectors/python.d.plugin/nvidia_smi/nvidia_smi.conf
@@ -0,0 +1,68 @@
+# netdata python.d.plugin configuration for nvidia_smi
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, example also supports the following:
+#
+# loop_mode: yes/no # default is yes. If set to yes `nvidia-smi` is executed in a separate thread using `-l` option.
+# poll_seconds: SECONDS # default is 1. Sets the frequency of seconds the nvidia-smi tool is polled in loop mode.
+# exclude_zero_memory_users: yes/no # default is no. Whether to collect users metrics with 0Mb memory allocation.
+#
+# ----------------------------------------------------------------------
diff --git a/collectors/python.d.plugin/openldap/Makefile.inc b/collectors/python.d.plugin/openldap/Makefile.inc
new file mode 100644
index 0000000..dc947e2
--- /dev/null
+++ b/collectors/python.d.plugin/openldap/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += openldap/openldap.chart.py
+dist_pythonconfig_DATA += openldap/openldap.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += openldap/README.md openldap/Makefile.inc
+
diff --git a/collectors/python.d.plugin/openldap/README.md b/collectors/python.d.plugin/openldap/README.md
new file mode 100644
index 0000000..b0cd1db
--- /dev/null
+++ b/collectors/python.d.plugin/openldap/README.md
@@ -0,0 +1,79 @@
+<!--
+title: "OpenLDAP monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/openldap/README.md
+sidebar_label: "OpenLDAP"
+-->
+
+# OpenLDAP monitoring with Netdata
+
+Provides statistics information from openldap (slapd) server.
+Statistics are taken from LDAP monitoring interface. Manual page, slapd-monitor(5) is available.
+
+**Requirement:**
+
+- Follow instructions from <https://www.openldap.org/doc/admin24/monitoringslapd.html> to activate monitoring interface.
+- Install python ldap module `pip install ldap` or `yum install python-ldap`
+- Modify openldap.conf with your credentials
+
+### Module gives information with following charts:
+
+1. **connections**
+
+ - total connections number
+
+2. **Bytes**
+
+ - sent
+
+3. **operations**
+
+ - completed
+ - initiated
+
+4. **referrals**
+
+ - sent
+
+5. **entries**
+
+ - sent
+
+6. **ldap operations**
+
+ - bind
+ - search
+ - unbind
+ - add
+ - delete
+ - modify
+ - compare
+
+7. **waiters**
+
+ - read
+ - write
+
+## Configuration
+
+Edit the `python.d/openldap.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/openldap.conf
+```
+
+Sample:
+
+```yaml
+openldap:
+ name : 'local'
+ username : "cn=monitor,dc=superb,dc=eu"
+ password : "testpass"
+ server : 'localhost'
+ port : 389
+```
+
+---
+
+
diff --git a/collectors/python.d.plugin/openldap/openldap.chart.py b/collectors/python.d.plugin/openldap/openldap.chart.py
new file mode 100644
index 0000000..aba1439
--- /dev/null
+++ b/collectors/python.d.plugin/openldap/openldap.chart.py
@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+# Description: openldap netdata python.d module
+# Author: Manolis Kartsonakis (ekartsonakis)
+# SPDX-License-Identifier: GPL-3.0+
+
+try:
+ import ldap
+
+ HAS_LDAP = True
+except ImportError:
+ HAS_LDAP = False
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+DEFAULT_SERVER = 'localhost'
+DEFAULT_PORT = '389'
+DEFAULT_TLS = False
+DEFAULT_CERT_CHECK = True
+DEFAULT_TIMEOUT = 1
+DEFAULT_START_TLS = False
+
+ORDER = [
+ 'total_connections',
+ 'bytes_sent',
+ 'operations',
+ 'referrals_sent',
+ 'entries_sent',
+ 'ldap_operations',
+ 'waiters'
+]
+
+CHARTS = {
+ 'total_connections': {
+ 'options': [None, 'Total Connections', 'connections/s', 'ldap', 'openldap.total_connections', 'line'],
+ 'lines': [
+ ['total_connections', 'connections', 'incremental']
+ ]
+ },
+ 'bytes_sent': {
+ 'options': [None, 'Traffic', 'KiB/s', 'ldap', 'openldap.traffic_stats', 'line'],
+ 'lines': [
+ ['bytes_sent', 'sent', 'incremental', 1, 1024]
+ ]
+ },
+ 'operations': {
+ 'options': [None, 'Operations Status', 'ops/s', 'ldap', 'openldap.operations_status', 'line'],
+ 'lines': [
+ ['completed_operations', 'completed', 'incremental'],
+ ['initiated_operations', 'initiated', 'incremental']
+ ]
+ },
+ 'referrals_sent': {
+ 'options': [None, 'Referrals', 'referrals/s', 'ldap', 'openldap.referrals', 'line'],
+ 'lines': [
+ ['referrals_sent', 'sent', 'incremental']
+ ]
+ },
+ 'entries_sent': {
+ 'options': [None, 'Entries', 'entries/s', 'ldap', 'openldap.entries', 'line'],
+ 'lines': [
+ ['entries_sent', 'sent', 'incremental']
+ ]
+ },
+ 'ldap_operations': {
+ 'options': [None, 'Operations', 'ops/s', 'ldap', 'openldap.ldap_operations', 'line'],
+ 'lines': [
+ ['bind_operations', 'bind', 'incremental'],
+ ['search_operations', 'search', 'incremental'],
+ ['unbind_operations', 'unbind', 'incremental'],
+ ['add_operations', 'add', 'incremental'],
+ ['delete_operations', 'delete', 'incremental'],
+ ['modify_operations', 'modify', 'incremental'],
+ ['compare_operations', 'compare', 'incremental']
+ ]
+ },
+ 'waiters': {
+ 'options': [None, 'Waiters', 'waiters/s', 'ldap', 'openldap.waiters', 'line'],
+ 'lines': [
+ ['write_waiters', 'write', 'incremental'],
+ ['read_waiters', 'read', 'incremental']
+ ]
+ },
+}
+
+# Stuff to gather - make tuples of DN dn and attrib to get
+SEARCH_LIST = {
+ 'total_connections': (
+ 'cn=Total,cn=Connections,cn=Monitor', 'monitorCounter',
+ ),
+ 'bytes_sent': (
+ 'cn=Bytes,cn=Statistics,cn=Monitor', 'monitorCounter',
+ ),
+ 'completed_operations': (
+ 'cn=Operations,cn=Monitor', 'monitorOpCompleted',
+ ),
+ 'initiated_operations': (
+ 'cn=Operations,cn=Monitor', 'monitorOpInitiated',
+ ),
+ 'referrals_sent': (
+ 'cn=Referrals,cn=Statistics,cn=Monitor', 'monitorCounter',
+ ),
+ 'entries_sent': (
+ 'cn=Entries,cn=Statistics,cn=Monitor', 'monitorCounter',
+ ),
+ 'bind_operations': (
+ 'cn=Bind,cn=Operations,cn=Monitor', 'monitorOpCompleted',
+ ),
+ 'unbind_operations': (
+ 'cn=Unbind,cn=Operations,cn=Monitor', 'monitorOpCompleted',
+ ),
+ 'add_operations': (
+ 'cn=Add,cn=Operations,cn=Monitor', 'monitorOpInitiated',
+ ),
+ 'delete_operations': (
+ 'cn=Delete,cn=Operations,cn=Monitor', 'monitorOpCompleted',
+ ),
+ 'modify_operations': (
+ 'cn=Modify,cn=Operations,cn=Monitor', 'monitorOpCompleted',
+ ),
+ 'compare_operations': (
+ 'cn=Compare,cn=Operations,cn=Monitor', 'monitorOpCompleted',
+ ),
+ 'search_operations': (
+ 'cn=Search,cn=Operations,cn=Monitor', 'monitorOpCompleted',
+ ),
+ 'write_waiters': (
+ 'cn=Write,cn=Waiters,cn=Monitor', 'monitorCounter',
+ ),
+ 'read_waiters': (
+ 'cn=Read,cn=Waiters,cn=Monitor', 'monitorCounter',
+ ),
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.server = configuration.get('server', DEFAULT_SERVER)
+ self.port = configuration.get('port', DEFAULT_PORT)
+ self.username = configuration.get('username')
+ self.password = configuration.get('password')
+ self.timeout = configuration.get('timeout', DEFAULT_TIMEOUT)
+ self.use_tls = configuration.get('use_tls', DEFAULT_TLS)
+ self.cert_check = configuration.get('cert_check', DEFAULT_CERT_CHECK)
+ self.use_start_tls = configuration.get('use_start_tls', DEFAULT_START_TLS)
+ self.alive = False
+ self.conn = None
+
+ def disconnect(self):
+ if self.conn:
+ self.conn.unbind()
+ self.conn = None
+ self.alive = False
+
+ def connect(self):
+ try:
+ if self.use_tls:
+ self.conn = ldap.initialize('ldaps://%s:%s' % (self.server, self.port))
+ else:
+ self.conn = ldap.initialize('ldap://%s:%s' % (self.server, self.port))
+ self.conn.set_option(ldap.OPT_NETWORK_TIMEOUT, self.timeout)
+ if (self.use_tls or self.use_start_tls) and not self.cert_check:
+ self.conn.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+ if self.use_start_tls or self.use_tls:
+ self.conn.set_option(ldap.OPT_X_TLS_NEWCTX, 0)
+ if self.use_start_tls:
+ self.conn.protocol_version = ldap.VERSION3
+ self.conn.start_tls_s()
+ if self.username and self.password:
+ self.conn.simple_bind(self.username, self.password)
+ except ldap.LDAPError as error:
+ self.error(error)
+ return False
+
+ self.alive = True
+ return True
+
+ def reconnect(self):
+ self.disconnect()
+ return self.connect()
+
+ def check(self):
+ if not HAS_LDAP:
+ self.error("'python-ldap' package is needed")
+ return None
+
+ return self.connect() and self.get_data()
+
+ def get_data(self):
+ if not self.alive and not self.reconnect():
+ return None
+
+ data = dict()
+ for key in SEARCH_LIST:
+ dn = SEARCH_LIST[key][0]
+ attr = SEARCH_LIST[key][1]
+ try:
+ num = self.conn.search(dn, ldap.SCOPE_BASE, 'objectClass=*', [attr, ])
+ result_type, result_data = self.conn.result(num, 1)
+ except ldap.LDAPError as error:
+ self.error("Empty result. Check bind username/password. Message: ", error)
+ self.alive = False
+ return None
+
+ if result_type != 101:
+ continue
+
+ try:
+ data[key] = int(list(result_data[0][1].values())[0][0])
+ except (ValueError, IndexError) as error:
+ self.debug(error)
+ continue
+
+ return data
diff --git a/collectors/python.d.plugin/openldap/openldap.conf b/collectors/python.d.plugin/openldap/openldap.conf
new file mode 100644
index 0000000..5fd99a5
--- /dev/null
+++ b/collectors/python.d.plugin/openldap/openldap.conf
@@ -0,0 +1,75 @@
+# netdata python.d.plugin configuration for openldap
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# postfix is slow, so once every 10 seconds
+update_every: 10
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# ----------------------------------------------------------------------
+# OPENLDAP EXTRA PARAMETERS
+
+# Set here your LDAP connection settings
+
+#username : "cn=admin,dc=example,dc=com" # The bind user with right to access monitor statistics
+#password : "yourpass" # The password for the binded user
+#server : 'localhost' # The listening address of the LDAP server. In case of TLS, use the hostname which the certificate is published for.
+#port : 389 # The listening port of the LDAP server. Change to 636 port in case of TLS connection
+#use_tls : False # Make True if a TLS connection is used over ldaps://
+#use_start_tls: False # Make True if a TLS connection is used over ldap://
+#cert_check : True # False if you want to ignore certificate check
+#timeout : 1 # Seconds to timeout if no connection exi
diff --git a/collectors/python.d.plugin/oracledb/Makefile.inc b/collectors/python.d.plugin/oracledb/Makefile.inc
new file mode 100644
index 0000000..ea3a824
--- /dev/null
+++ b/collectors/python.d.plugin/oracledb/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += oracledb/oracledb.chart.py
+dist_pythonconfig_DATA += oracledb/oracledb.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += oracledb/README.md oracledb/Makefile.inc
+
diff --git a/collectors/python.d.plugin/oracledb/README.md b/collectors/python.d.plugin/oracledb/README.md
new file mode 100644
index 0000000..88024f8
--- /dev/null
+++ b/collectors/python.d.plugin/oracledb/README.md
@@ -0,0 +1,97 @@
+<!--
+title: "OracleDB monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/oracledb/README.md
+sidebar_label: "OracleDB"
+-->
+
+# OracleDB monitoring with Netdata
+
+Monitors the performance and health metrics of the Oracle database.
+
+## Requirements
+
+- `cx_Oracle` package.
+- Oracle Client (using `cx_Oracle` requires Oracle Client libraries to be installed).
+
+It produces following charts:
+
+- session activity
+ - Session Count
+ - Session Limit Usage
+ - Logons
+- disk activity
+ - Physical Disk Reads/Writes
+ - Sorts On Disk
+ - Full Table Scans
+- database and buffer activity
+ - Database Wait Time Ratio
+ - Shared Pool Free Memory
+ - In-Memory Sorts Ratio
+ - SQL Service Response Time
+ - User Rollbacks
+ - Enqueue Timeouts
+- cache
+ - Cache Hit Ratio
+ - Global Cache Blocks Events
+- activities
+ - Activities
+- wait time
+ - Wait Time
+- tablespace
+ - Size
+ - Usage
+ - Usage In Percent
+- allocated space
+ - Size
+ - Usage
+ - Usage In Percent
+
+## prerequisite
+
+To use the Oracle module do the following:
+
+1. Install `cx_Oracle` package ([link](https://cx-oracle.readthedocs.io/en/latest/user_guide/installation.html)).
+
+2. Install Oracle Client libraries
+ ([link](https://cx-oracle.readthedocs.io/en/latest/user_guide/installation.html#install-oracle-client)).
+
+3. Create a read-only `netdata` user with proper access to your Oracle Database Server.
+
+Connect to your Oracle database with an administrative user and execute:
+
+```
+ALTER SESSION SET "_ORACLE_SCRIPT"=true;
+
+CREATE USER netdata IDENTIFIED BY <PASSWORD>;
+
+GRANT CONNECT TO netdata;
+GRANT SELECT_CATALOG_ROLE TO netdata;
+```
+
+## Configuration
+
+Edit the `python.d/oracledb.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/oracledb.conf
+```
+
+```yaml
+local:
+ user: 'netdata'
+ password: 'secret'
+ server: 'localhost:1521'
+ service: 'XE'
+
+remote:
+ user: 'netdata'
+ password: 'secret'
+ server: '10.0.0.1:1521'
+ service: 'XE'
+```
+
+All parameters are required. Without them module will fail to start.
+
+
diff --git a/collectors/python.d.plugin/oracledb/oracledb.chart.py b/collectors/python.d.plugin/oracledb/oracledb.chart.py
new file mode 100644
index 0000000..28ef8db
--- /dev/null
+++ b/collectors/python.d.plugin/oracledb/oracledb.chart.py
@@ -0,0 +1,831 @@
+# -*- coding: utf-8 -*-
+# Description: oracledb netdata python.d module
+# Author: ilyam8 (Ilya Mashchenko)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from copy import deepcopy
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+try:
+ import cx_Oracle
+
+ HAS_ORACLE = True
+except ImportError:
+ HAS_ORACLE = False
+
+ORDER = [
+ 'session_count',
+ 'session_limit_usage',
+ 'logons',
+ 'physical_disk_read_write',
+ 'sorts_on_disk',
+ 'full_table_scans',
+ 'database_wait_time_ratio',
+ 'shared_pool_free_memory',
+ 'in_memory_sorts_ratio',
+ 'sql_service_response_time',
+ 'user_rollbacks',
+ 'enqueue_timeouts',
+ 'cache_hit_ratio',
+ 'global_cache_blocks',
+ 'activity',
+ 'wait_time',
+ 'tablespace_size',
+ 'tablespace_usage',
+ 'tablespace_usage_in_percent',
+ 'allocated_size',
+ 'allocated_usage',
+ 'allocated_usage_in_percent',
+]
+
+CHARTS = {
+ 'session_count': {
+ 'options': [None, 'Session Count', 'sessions', 'session activity', 'oracledb.session_count', 'line'],
+ 'lines': [
+ ['session_count', 'total', 'absolute', 1, 1000],
+ ['average_active_sessions', 'active', 'absolute', 1, 1000],
+ ]
+ },
+ 'session_limit_usage': {
+ 'options': [None, 'Session Limit Usage', '%', 'session activity', 'oracledb.session_limit_usage', 'area'],
+ 'lines': [
+ ['session_limit_percent', 'usage', 'absolute', 1, 1000],
+ ]
+ },
+ 'logons': {
+ 'options': [None, 'Logons', 'events/s', 'session activity', 'oracledb.logons', 'area'],
+ 'lines': [
+ ['logons_per_sec', 'logons', 'absolute', 1, 1000],
+ ]
+ },
+ 'physical_disk_read_write': {
+ 'options': [None, 'Physical Disk Reads/Writes', 'events/s', 'disk activity',
+ 'oracledb.physical_disk_read_writes', 'area'],
+ 'lines': [
+ ['physical_reads_per_sec', 'reads', 'absolute', 1, 1000],
+ ['physical_writes_per_sec', 'writes', 'absolute', -1, 1000],
+ ]
+ },
+ 'sorts_on_disk': {
+ 'options': [None, 'Sorts On Disk', 'events/s', 'disk activity', 'oracledb.sorts_on_disks', 'line'],
+ 'lines': [
+ ['disk_sort_per_sec', 'sorts', 'absolute', 1, 1000],
+ ]
+ },
+ 'full_table_scans': {
+ 'options': [None, 'Full Table Scans', 'events/s', 'disk activity', 'oracledb.full_table_scans', 'line'],
+ 'lines': [
+ ['long_table_scans_per_sec', 'full table scans', 'absolute', 1, 1000],
+ ]
+ },
+ 'database_wait_time_ratio': {
+ 'options': [None, 'Database Wait Time Ratio', '%', 'database and buffer activity',
+ 'oracledb.database_wait_time_ratio', 'line'],
+ 'lines': [
+ ['database_wait_time_ratio', 'wait time ratio', 'absolute', 1, 1000],
+ ]
+ },
+ 'shared_pool_free_memory': {
+ 'options': [None, 'Shared Pool Free Memory', '%', 'database and buffer activity',
+ 'oracledb.shared_pool_free_memory', 'line'],
+ 'lines': [
+ ['shared_pool_free_percent', 'free memory', 'absolute', 1, 1000],
+ ]
+ },
+ 'in_memory_sorts_ratio': {
+ 'options': [None, 'In-Memory Sorts Ratio', '%', 'database and buffer activity',
+ 'oracledb.in_memory_sorts_ratio', 'line'],
+ 'lines': [
+ ['memory_sorts_ratio', 'in-memory sorts', 'absolute', 1, 1000],
+ ]
+ },
+ 'sql_service_response_time': {
+ 'options': [None, 'SQL Service Response Time', 'seconds', 'database and buffer activity',
+ 'oracledb.sql_service_response_time', 'line'],
+ 'lines': [
+ ['sql_service_response_time', 'time', 'absolute', 1, 1000],
+ ]
+ },
+ 'user_rollbacks': {
+ 'options': [None, 'User Rollbacks', 'events/s', 'database and buffer activity',
+ 'oracledb.user_rollbacks', 'line'],
+ 'lines': [
+ ['user_rollbacks_per_sec', 'rollbacks', 'absolute', 1, 1000],
+ ]
+ },
+ 'enqueue_timeouts': {
+ 'options': [None, 'Enqueue Timeouts', 'events/s', 'database and buffer activity',
+ 'oracledb.enqueue_timeouts', 'line'],
+ 'lines': [
+ ['enqueue_timeouts_per_sec', 'enqueue timeouts', 'absolute', 1, 1000],
+ ]
+ },
+ 'cache_hit_ratio': {
+ 'options': [None, 'Cache Hit Ratio', '%', 'cache', 'oracledb.cache_hit_ration', 'stacked'],
+ 'lines': [
+ ['buffer_cache_hit_ratio', 'buffer', 'absolute', 1, 1000],
+ ['cursor_cache_hit_ratio', 'cursor', 'absolute', 1, 1000],
+ ['library_cache_hit_ratio', 'library', 'absolute', 1, 1000],
+ ['row_cache_hit_ratio', 'row', 'absolute', 1, 1000],
+ ]
+ },
+ 'global_cache_blocks': {
+ 'options': [None, 'Global Cache Blocks Events', 'events/s', 'cache', 'oracledb.global_cache_blocks', 'area'],
+ 'lines': [
+ ['global_cache_blocks_corrupted', 'corrupted', 'incremental', 1, 1000],
+ ['global_cache_blocks_lost', 'lost', 'incremental', 1, 1000],
+ ]
+ },
+ 'activity': {
+ 'options': [None, 'Activities', 'events/s', 'activities', 'oracledb.activity', 'stacked'],
+ 'lines': [
+ ['activity_parse_count_total', 'parse count', 'incremental', 1, 1000],
+ ['activity_execute_count', 'execute count', 'incremental', 1, 1000],
+ ['activity_user_commits', 'user commits', 'incremental', 1, 1000],
+ ['activity_user_rollbacks', 'user rollbacks', 'incremental', 1, 1000],
+ ]
+ },
+ 'wait_time': {
+ 'options': [None, 'Wait Time', 'ms', 'wait time', 'oracledb.wait_time', 'stacked'],
+ 'lines': [
+ ['wait_time_application', 'application', 'absolute', 1, 1000],
+ ['wait_time_configuration', 'configuration', 'absolute', 1, 1000],
+ ['wait_time_administrative', 'administrative', 'absolute', 1, 1000],
+ ['wait_time_concurrency', 'concurrency', 'absolute', 1, 1000],
+ ['wait_time_commit', 'commit', 'absolute', 1, 1000],
+ ['wait_time_network', 'network', 'absolute', 1, 1000],
+ ['wait_time_user_io', 'user I/O', 'absolute', 1, 1000],
+ ['wait_time_system_io', 'system I/O', 'absolute', 1, 1000],
+ ['wait_time_scheduler', 'scheduler', 'absolute', 1, 1000],
+ ['wait_time_other', 'other', 'absolute', 1, 1000],
+ ]
+ },
+ 'tablespace_size': {
+ 'options': [None, 'Size', 'KiB', 'tablespace', 'oracledb.tablespace_size', 'line'],
+ 'lines': [],
+ },
+ 'tablespace_usage': {
+ 'options': [None, 'Usage', 'KiB', 'tablespace', 'oracledb.tablespace_usage', 'line'],
+ 'lines': [],
+ },
+ 'tablespace_usage_in_percent': {
+ 'options': [None, 'Usage', '%', 'tablespace', 'oracledb.tablespace_usage_in_percent', 'line'],
+ 'lines': [],
+ },
+ 'allocated_size': {
+ 'options': [None, 'Size', 'B', 'tablespace', 'oracledb.allocated_size', 'line'],
+ 'lines': [],
+ },
+ 'allocated_usage': {
+ 'options': [None, 'Usage', 'B', 'tablespace', 'oracledb.allocated_usage', 'line'],
+ 'lines': [],
+ },
+ 'allocated_usage_in_percent': {
+ 'options': [None, 'Usage', '%', 'tablespace', 'oracledb.allocated_usage_in_percent', 'line'],
+ 'lines': [],
+ },
+}
+
+CX_CONNECT_STRING = "{0}/{1}@//{2}/{3}"
+
+QUERY_SYSTEM = '''
+SELECT
+ metric_name,
+ value
+FROM
+ gv$sysmetric
+ORDER BY
+ begin_time
+'''
+QUERY_TABLESPACE = '''
+SELECT
+ m.tablespace_name,
+ m.used_space * t.block_size AS used_bytes,
+ m.tablespace_size * t.block_size AS max_bytes,
+ m.used_percent
+FROM
+ dba_tablespace_usage_metrics m
+ JOIN dba_tablespaces t ON m.tablespace_name = t.tablespace_name
+'''
+QUERY_ALLOCATED = '''
+SELECT
+ nvl(b.tablespace_name,nvl(a.tablespace_name,'UNKNOWN')) tablespace_name,
+ bytes_alloc used_bytes,
+ bytes_alloc-nvl(bytes_free,0) max_bytes,
+ ((bytes_alloc-nvl(bytes_free,0))/ bytes_alloc)*100 used_percent
+FROM
+ (SELECT
+ sum(bytes) bytes_free,
+ tablespace_name
+ FROM sys.dba_free_space
+ GROUP BY tablespace_name
+ ) a,
+ (SELECT
+ sum(bytes) bytes_alloc,
+ tablespace_name
+ FROM sys.dba_data_files
+ GROUP BY tablespace_name
+ ) b
+WHERE a.tablespace_name (+) = b.tablespace_name
+'''
+QUERY_ACTIVITIES_COUNT = '''
+SELECT
+ name,
+ value
+FROM
+ v$sysstat
+WHERE
+ name IN (
+ 'parse count (total)',
+ 'execute count',
+ 'user commits',
+ 'user rollbacks'
+ )
+'''
+QUERY_WAIT_TIME = '''
+SELECT
+ n.wait_class,
+ round(m.time_waited / m.INTSIZE_CSEC, 3)
+FROM
+ v$waitclassmetric m,
+ v$system_wait_class n
+WHERE
+ m.wait_class_id = n.wait_class_id
+ AND n.wait_class != 'Idle'
+'''
+# QUERY_SESSION_COUNT = '''
+# SELECT
+# status,
+# type
+# FROM
+# v$session
+# GROUP BY
+# status,
+# type
+# '''
+# QUERY_PROCESSES_COUNT = '''
+# SELECT
+# COUNT(*)
+# FROM
+# v$process
+# '''
+# QUERY_PROCESS = '''
+# SELECT
+# program,
+# pga_used_mem,
+# pga_alloc_mem,
+# pga_freeable_mem,
+# pga_max_mem
+# FROM
+# gv$process
+# '''
+
+# PROCESS_METRICS = [
+# 'pga_used_memory',
+# 'pga_allocated_memory',
+# 'pga_freeable_memory',
+# 'pga_maximum_memory',
+# ]
+
+
+SYS_METRICS = {
+ 'Average Active Sessions': 'average_active_sessions',
+ 'Session Count': 'session_count',
+ 'Session Limit %': 'session_limit_percent',
+ 'Logons Per Sec': 'logons_per_sec',
+ 'Physical Reads Per Sec': 'physical_reads_per_sec',
+ 'Physical Writes Per Sec': 'physical_writes_per_sec',
+ 'Disk Sort Per Sec': 'disk_sort_per_sec',
+ 'Long Table Scans Per Sec': 'long_table_scans_per_sec',
+ 'Database Wait Time Ratio': 'database_wait_time_ratio',
+ 'Shared Pool Free %': 'shared_pool_free_percent',
+ 'Memory Sorts Ratio': 'memory_sorts_ratio',
+ 'SQL Service Response Time': 'sql_service_response_time',
+ 'User Rollbacks Per Sec': 'user_rollbacks_per_sec',
+ 'Enqueue Timeouts Per Sec': 'enqueue_timeouts_per_sec',
+ 'Buffer Cache Hit Ratio': 'buffer_cache_hit_ratio',
+ 'Cursor Cache Hit Ratio': 'cursor_cache_hit_ratio',
+ 'Library Cache Hit Ratio': 'library_cache_hit_ratio',
+ 'Row Cache Hit Ratio': 'row_cache_hit_ratio',
+ 'Global Cache Blocks Corrupted': 'global_cache_blocks_corrupted',
+ 'Global Cache Blocks Lost': 'global_cache_blocks_lost',
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = deepcopy(CHARTS)
+ self.user = configuration.get('user')
+ self.password = configuration.get('password')
+ self.server = configuration.get('server')
+ self.service = configuration.get('service')
+ self.alive = False
+ self.conn = None
+ self.active_tablespaces = set()
+
+ def connect(self):
+ if self.conn:
+ self.conn.close()
+ self.conn = None
+
+ try:
+ self.conn = cx_Oracle.connect(
+ CX_CONNECT_STRING.format(
+ self.user,
+ self.password,
+ self.server,
+ self.service,
+ ))
+ except cx_Oracle.DatabaseError as error:
+ self.error(error)
+ return False
+
+ self.alive = True
+ return True
+
+ def reconnect(self):
+ return self.connect()
+
+ def check(self):
+ if not HAS_ORACLE:
+ self.error("'cx_Oracle' package is needed to use oracledb module")
+ return False
+
+ if not all([
+ self.user,
+ self.password,
+ self.server,
+ self.service,
+ ]):
+ self.error("one of these parameters is not specified: user, password, server, service")
+ return False
+
+ if not self.connect():
+ return False
+
+ return bool(self.get_data())
+
+ def get_data(self):
+ if not self.alive and not self.reconnect():
+ return None
+
+ data = dict()
+
+ # SYSTEM
+ try:
+ rv = self.gather_system_metrics()
+ except cx_Oracle.Error as error:
+ self.error(error)
+ self.alive = False
+ return None
+ else:
+ for name, value in rv:
+ if name not in SYS_METRICS:
+ continue
+ data[SYS_METRICS[name]] = int(float(value) * 1000)
+
+ # ACTIVITIES COUNT
+ try:
+ rv = self.gather_activities_count()
+ except cx_Oracle.Error as error:
+ self.error(error)
+ self.alive = False
+ return None
+ else:
+ for name, amount in rv:
+ cleaned = name.replace(' ', '_').replace('(', '').replace(')', '')
+ new_name = 'activity_{0}'.format(cleaned)
+ data[new_name] = int(float(amount) * 1000)
+
+ # WAIT TIME
+ try:
+ rv = self.gather_wait_time_metrics()
+ except cx_Oracle.Error as error:
+ self.error(error)
+ self.alive = False
+ return None
+ else:
+ for name, amount in rv:
+ cleaned = name.replace(' ', '_').replace('/', '').lower()
+ new_name = 'wait_time_{0}'.format(cleaned)
+ data[new_name] = amount
+
+ # TABLESPACE
+ try:
+ rv = self.gather_tablespace_metrics()
+ except cx_Oracle.Error as error:
+ self.error(error)
+ self.alive = False
+ return None
+ else:
+ for name, offline, size, used, used_in_percent in rv:
+ # TODO: skip offline?
+ if not (not offline and self.charts):
+ continue
+ # TODO: remove inactive?
+ if name not in self.active_tablespaces:
+ self.active_tablespaces.add(name)
+ self.add_tablespace_to_charts(name)
+ data['{0}_tablespace_size'.format(name)] = int(size * 1000)
+ data['{0}_tablespace_used'.format(name)] = int(used * 1000)
+ data['{0}_tablespace_used_in_percent'.format(name)] = int(used_in_percent * 1000)
+
+ # ALLOCATED SPACE
+ try:
+ rv = self.gather_allocated_metrics()
+ except cx_Oracle.Error as error:
+ self.error(error)
+ self.alive = False
+ return None
+ else:
+ for name, offline, size, used, used_in_percent in rv:
+ # TODO: skip offline?
+ if not (not offline and self.charts):
+ continue
+ # TODO: remove inactive?
+ if name not in self.active_tablespaces:
+ self.active_tablespaces.add(name)
+ self.add_tablespace_to_charts(name)
+ data['{0}_allocated_size'.format(name)] = int(size * 1000)
+ data['{0}_allocated_used'.format(name)] = int(used * 1000)
+ data['{0}_allocated_used_in_percent'.format(name)] = int(used_in_percent * 1000)
+
+ return data or None
+
+ def gather_system_metrics(self):
+
+ """
+ :return:
+
+ [['Buffer Cache Hit Ratio', 100],
+ ['Memory Sorts Ratio', 100],
+ ['Redo Allocation Hit Ratio', 100],
+ ['User Transaction Per Sec', 0],
+ ['Physical Reads Per Sec', 0],
+ ['Physical Reads Per Txn', 0],
+ ['Physical Writes Per Sec', 0],
+ ['Physical Writes Per Txn', 0],
+ ['Physical Reads Direct Per Sec', 0],
+ ['Physical Reads Direct Per Txn', 0],
+ ['Physical Writes Direct Per Sec', 0],
+ ['Physical Writes Direct Per Txn', 0],
+ ['Physical Reads Direct Lobs Per Sec', 0],
+ ['Physical Reads Direct Lobs Per Txn', 0],
+ ['Physical Writes Direct Lobs Per Sec', 0],
+ ['Physical Writes Direct Lobs Per Txn', 0],
+ ['Redo Generated Per Sec', Decimal('4.66666666666667')],
+ ['Redo Generated Per Txn', 280],
+ ['Logons Per Sec', Decimal('0.0166666666666667')],
+ ['Logons Per Txn', 1],
+ ['Open Cursors Per Sec', 0.35],
+ ['Open Cursors Per Txn', 21],
+ ['User Commits Per Sec', 0],
+ ['User Commits Percentage', 0],
+ ['User Rollbacks Per Sec', 0],
+ ['User Rollbacks Percentage', 0],
+ ['User Calls Per Sec', Decimal('0.0333333333333333')],
+ ['User Calls Per Txn', 2],
+ ['Recursive Calls Per Sec', 14.15],
+ ['Recursive Calls Per Txn', 849],
+ ['Logical Reads Per Sec', Decimal('0.683333333333333')],
+ ['Logical Reads Per Txn', 41],
+ ['DBWR Checkpoints Per Sec', 0],
+ ['Background Checkpoints Per Sec', 0],
+ ['Redo Writes Per Sec', Decimal('0.0333333333333333')],
+ ['Redo Writes Per Txn', 2],
+ ['Long Table Scans Per Sec', 0],
+ ['Long Table Scans Per Txn', 0],
+ ['Total Table Scans Per Sec', Decimal('0.0166666666666667')],
+ ['Total Table Scans Per Txn', 1],
+ ['Full Index Scans Per Sec', 0],
+ ['Full Index Scans Per Txn', 0],
+ ['Total Index Scans Per Sec', Decimal('0.216666666666667')],
+ ['Total Index Scans Per Txn', 13],
+ ['Total Parse Count Per Sec', 0.35],
+ ['Total Parse Count Per Txn', 21],
+ ['Hard Parse Count Per Sec', 0],
+ ['Hard Parse Count Per Txn', 0],
+ ['Parse Failure Count Per Sec', 0],
+ ['Parse Failure Count Per Txn', 0],
+ ['Cursor Cache Hit Ratio', Decimal('52.3809523809524')],
+ ['Disk Sort Per Sec', 0],
+ ['Disk Sort Per Txn', 0],
+ ['Rows Per Sort', 8.6],
+ ['Execute Without Parse Ratio', Decimal('27.5862068965517')],
+ ['Soft Parse Ratio', 100],
+ ['User Calls Ratio', Decimal('0.235017626321974')],
+ ['Host CPU Utilization (%)', Decimal('0.124311845142959')],
+ ['Network Traffic Volume Per Sec', 0],
+ ['Enqueue Timeouts Per Sec', 0],
+ ['Enqueue Timeouts Per Txn', 0],
+ ['Enqueue Waits Per Sec', 0],
+ ['Enqueue Waits Per Txn', 0],
+ ['Enqueue Deadlocks Per Sec', 0],
+ ['Enqueue Deadlocks Per Txn', 0],
+ ['Enqueue Requests Per Sec', Decimal('216.683333333333')],
+ ['Enqueue Requests Per Txn', 13001],
+ ['DB Block Gets Per Sec', 0],
+ ['DB Block Gets Per Txn', 0],
+ ['Consistent Read Gets Per Sec', Decimal('0.683333333333333')],
+ ['Consistent Read Gets Per Txn', 41],
+ ['DB Block Changes Per Sec', 0],
+ ['DB Block Changes Per Txn', 0],
+ ['Consistent Read Changes Per Sec', 0],
+ ['Consistent Read Changes Per Txn', 0],
+ ['CPU Usage Per Sec', 0],
+ ['CPU Usage Per Txn', 0],
+ ['CR Blocks Created Per Sec', 0],
+ ['CR Blocks Created Per Txn', 0],
+ ['CR Undo Records Applied Per Sec', 0],
+ ['CR Undo Records Applied Per Txn', 0],
+ ['User Rollback UndoRec Applied Per Sec', 0],
+ ['User Rollback Undo Records Applied Per Txn', 0],
+ ['Leaf Node Splits Per Sec', 0],
+ ['Leaf Node Splits Per Txn', 0],
+ ['Branch Node Splits Per Sec', 0],
+ ['Branch Node Splits Per Txn', 0],
+ ['PX downgraded 1 to 25% Per Sec', 0],
+ ['PX downgraded 25 to 50% Per Sec', 0],
+ ['PX downgraded 50 to 75% Per Sec', 0],
+ ['PX downgraded 75 to 99% Per Sec', 0],
+ ['PX downgraded to serial Per Sec', 0],
+ ['Physical Read Total IO Requests Per Sec', Decimal('2.16666666666667')],
+ ['Physical Read Total Bytes Per Sec', Decimal('35498.6666666667')],
+ ['GC CR Block Received Per Second', 0],
+ ['GC CR Block Received Per Txn', 0],
+ ['GC Current Block Received Per Second', 0],
+ ['GC Current Block Received Per Txn', 0],
+ ['Global Cache Average CR Get Time', 0],
+ ['Global Cache Average Current Get Time', 0],
+ ['Physical Write Total IO Requests Per Sec', Decimal('0.966666666666667')],
+ ['Global Cache Blocks Corrupted', 0],
+ ['Global Cache Blocks Lost', 0],
+ ['Current Logons Count', 49],
+ ['Current Open Cursors Count', 64],
+ ['User Limit %', Decimal('0.00000114087015416959')],
+ ['SQL Service Response Time', 0],
+ ['Database Wait Time Ratio', 0],
+ ['Database CPU Time Ratio', 0],
+ ['Response Time Per Txn', 0],
+ ['Row Cache Hit Ratio', 100],
+ ['Row Cache Miss Ratio', 0],
+ ['Library Cache Hit Ratio', 100],
+ ['Library Cache Miss Ratio', 0],
+ ['Shared Pool Free %', Decimal('7.82380268491548')],
+ ['PGA Cache Hit %', Decimal('98.0399767109115')],
+ ['Process Limit %', Decimal('17.6666666666667')],
+ ['Session Limit %', Decimal('15.2542372881356')],
+ ['Executions Per Txn', 29],
+ ['Executions Per Sec', Decimal('0.483333333333333')],
+ ['Txns Per Logon', 0],
+ ['Database Time Per Sec', 0],
+ ['Physical Write Total Bytes Per Sec', 15308.8],
+ ['Physical Read IO Requests Per Sec', 0],
+ ['Physical Read Bytes Per Sec', 0],
+ ['Physical Write IO Requests Per Sec', 0],
+ ['Physical Write Bytes Per Sec', 0],
+ ['DB Block Changes Per User Call', 0],
+ ['DB Block Gets Per User Call', 0],
+ ['Executions Per User Call', 14.5],
+ ['Logical Reads Per User Call', 20.5],
+ ['Total Sorts Per User Call', 2.5],
+ ['Total Table Scans Per User Call', 0.5],
+ ['Current OS Load', 0.0390625],
+ ['Streams Pool Usage Percentage', 0],
+ ['PQ QC Session Count', 0],
+ ['PQ Slave Session Count', 0],
+ ['Queries parallelized Per Sec', 0],
+ ['DML statements parallelized Per Sec', 0],
+ ['DDL statements parallelized Per Sec', 0],
+ ['PX operations not downgraded Per Sec', 0],
+ ['Session Count', 72],
+ ['Average Synchronous Single-Block Read Latency', 0],
+ ['I/O Megabytes per Second', 0.05],
+ ['I/O Requests per Second', Decimal('3.13333333333333')],
+ ['Average Active Sessions', 0],
+ ['Active Serial Sessions', 1],
+ ['Active Parallel Sessions', 0],
+ ['Captured user calls', 0],
+ ['Replayed user calls', 0],
+ ['Workload Capture and Replay status', 0],
+ ['Background CPU Usage Per Sec', Decimal('1.22578833333333')],
+ ['Background Time Per Sec', 0.0147551],
+ ['Host CPU Usage Per Sec', Decimal('0.116666666666667')],
+ ['Cell Physical IO Interconnect Bytes', 3048448],
+ ['Temp Space Used', 0],
+ ['Total PGA Allocated', 200657920],
+ ['Total PGA Used by SQL Workareas', 0],
+ ['Run Queue Per Sec', 0],
+ ['VM in bytes Per Sec', 0],
+ ['VM out bytes Per Sec', 0]]
+ """
+
+ metrics = list()
+ with self.conn.cursor() as cursor:
+ cursor.execute(QUERY_SYSTEM)
+ for metric_name, value in cursor.fetchall():
+ metrics.append([metric_name, value])
+ return metrics
+
+ def gather_tablespace_metrics(self):
+ """
+ :return:
+
+ [['SYSTEM', 874250240.0, 3233169408.0, 27.040038107400033, 0],
+ ['SYSAUX', 498860032.0, 3233169408.0, 15.429443033997678, 0],
+ ['TEMP', 0.0, 3233177600.0, 0.0, 0],
+ ['USERS', 1048576.0, 3233169408.0, 0.03243182981397305, 0]]
+ """
+ metrics = list()
+ with self.conn.cursor() as cursor:
+ cursor.execute(QUERY_TABLESPACE)
+ for tablespace_name, used_bytes, max_bytes, used_percent in cursor.fetchall():
+ if used_bytes is None:
+ offline = True
+ used = 0
+ else:
+ offline = False
+ used = float(used_bytes)
+ if max_bytes is None:
+ size = 0
+ else:
+ size = float(max_bytes)
+ if used_percent is None:
+ used_percent = 0
+ else:
+ used_percent = float(used_percent)
+ metrics.append(
+ [
+ tablespace_name,
+ offline,
+ size,
+ used,
+ used_percent,
+ ]
+ )
+ return metrics
+
+ def gather_allocated_metrics(self):
+ """
+ :return:
+
+ [['SYSTEM', 874250240.0, 3233169408.0, 27.040038107400033, 0],
+ ['SYSAUX', 498860032.0, 3233169408.0, 15.429443033997678, 0],
+ ['TEMP', 0.0, 3233177600.0, 0.0, 0],
+ ['USERS', 1048576.0, 3233169408.0, 0.03243182981397305, 0]]
+ """
+ metrics = list()
+ with self.conn.cursor() as cursor:
+ cursor.execute(QUERY_ALLOCATED)
+ for tablespace_name, used_bytes, max_bytes, used_percent in cursor.fetchall():
+ if used_bytes is None:
+ offline = True
+ used = 0
+ else:
+ offline = False
+ used = float(used_bytes)
+ if max_bytes is None:
+ size = 0
+ else:
+ size = float(max_bytes)
+ if used_percent is None:
+ used_percent = 0
+ else:
+ used_percent = float(used_percent)
+ metrics.append(
+ [
+ tablespace_name,
+ offline,
+ size,
+ used,
+ used_percent,
+ ]
+ )
+ return metrics
+
+ def gather_wait_time_metrics(self):
+ """
+ :return:
+
+ [['Other', 0],
+ ['Application', 0],
+ ['Configuration', 0],
+ ['Administrative', 0],
+ ['Concurrency', 0],
+ ['Commit', 0],
+ ['Network', 0],
+ ['User I/O', 0],
+ ['System I/O', 0.002],
+ ['Scheduler', 0]]
+ """
+ metrics = list()
+ with self.conn.cursor() as cursor:
+ cursor.execute(QUERY_WAIT_TIME)
+ for wait_class_name, value in cursor.fetchall():
+ metrics.append([wait_class_name, value])
+ return metrics
+
+ def gather_activities_count(self):
+ """
+ :return:
+
+ [('user commits', 9104),
+ ('user rollbacks', 17),
+ ('parse count (total)', 483695),
+ ('execute count', 2020356)]
+ """
+ with self.conn.cursor() as cursor:
+ cursor.execute(QUERY_ACTIVITIES_COUNT)
+ return cursor.fetchall()
+
+ # def gather_process_metrics(self):
+ # """
+ # :return:
+ #
+ # [['PSEUDO', 'pga_used_memory', 0],
+ # ['PSEUDO', 'pga_allocated_memory', 0],
+ # ['PSEUDO', 'pga_freeable_memory', 0],
+ # ['PSEUDO', 'pga_maximum_memory', 0],
+ # ['oracle@localhost.localdomain (PMON)', 'pga_used_memory', 1793827],
+ # ['oracle@localhost.localdomain (PMON)', 'pga_allocated_memory', 1888651],
+ # ['oracle@localhost.localdomain (PMON)', 'pga_freeable_memory', 0],
+ # ['oracle@localhost.localdomain (PMON)', 'pga_maximum_memory', 1888651],
+ # ...
+ # ...
+ # """
+ #
+ # metrics = list()
+ # with self.conn.cursor() as cursor:
+ # cursor.execute(QUERY_PROCESS)
+ # for row in cursor.fetchall():
+ # for i, name in enumerate(PROCESS_METRICS, 1):
+ # metrics.append([row[0], name, row[i]])
+ # return metrics
+
+ # def gather_processes_count(self):
+ # with self.conn.cursor() as cursor:
+ # cursor.execute(QUERY_PROCESSES_COUNT)
+ # return cursor.fetchone()[0] # 53
+
+ # def gather_sessions_count(self):
+ # with self.conn.cursor() as cursor:
+ # cursor.execute(QUERY_SESSION_COUNT)
+ # total, active, inactive = 0, 0, 0
+ # for status, _ in cursor.fetchall():
+ # total += 1
+ # active += status == 'ACTIVE'
+ # inactive += status == 'INACTIVE'
+ # return [total, active, inactive]
+
+ def add_tablespace_to_charts(self, name):
+ self.charts['tablespace_size'].add_dimension(
+ [
+ '{0}_tablespace_size'.format(name),
+ name,
+ 'absolute',
+ 1,
+ 1024 * 1000,
+ ])
+ self.charts['tablespace_usage'].add_dimension(
+ [
+ '{0}_tablespace_used'.format(name),
+ name,
+ 'absolute',
+ 1,
+ 1024 * 1000,
+ ])
+ self.charts['tablespace_usage_in_percent'].add_dimension(
+ [
+ '{0}_tablespace_used_in_percent'.format(name),
+ name,
+ 'absolute',
+ 1,
+ 1000,
+ ])
+ self.charts['allocated_size'].add_dimension(
+ [
+ '{0}_allocated_size'.format(name),
+ name,
+ 'absolute',
+ 1,
+ 1000,
+ ])
+ self.charts['allocated_usage'].add_dimension(
+ [
+ '{0}_allocated_used'.format(name),
+ name,
+ 'absolute',
+ 1,
+ 1000,
+ ])
+ self.charts['allocated_usage_in_percent'].add_dimension(
+ [
+ '{0}_allocated_used_in_percent'.format(name),
+ name,
+ 'absolute',
+ 1,
+ 1000,
+ ])
diff --git a/collectors/python.d.plugin/oracledb/oracledb.conf b/collectors/python.d.plugin/oracledb/oracledb.conf
new file mode 100644
index 0000000..6257172
--- /dev/null
+++ b/collectors/python.d.plugin/oracledb/oracledb.conf
@@ -0,0 +1,84 @@
+# netdata python.d.plugin configuration for oracledb
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, oracledb also supports the following:
+#
+# user: username # the username for the user account. Required.
+# password: password # the password for the user account. Required.
+# server: localhost:1521 # the IP address or hostname of the Oracle Database Server. Required.
+# service: XE # the Oracle Database service name. Required. To view the services available on your server,
+# run this query: `SELECT value FROM v$parameter WHERE name='service_names'`.
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+#local:
+# user: 'netdata'
+# password: 'secret'
+# server: 'localhost:1521'
+# service: 'XE'
+
+#remote:
+# user: 'netdata'
+# password: 'secret'
+# server: '10.0.0.1:1521'
+# service: 'XE' \ No newline at end of file
diff --git a/collectors/python.d.plugin/pandas/Makefile.inc b/collectors/python.d.plugin/pandas/Makefile.inc
new file mode 100644
index 0000000..9f4f9b3
--- /dev/null
+++ b/collectors/python.d.plugin/pandas/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += pandas/pandas.chart.py
+dist_pythonconfig_DATA += pandas/pandas.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += pandas/README.md pandas/Makefile.inc
+
diff --git a/collectors/python.d.plugin/pandas/README.md b/collectors/python.d.plugin/pandas/README.md
new file mode 100644
index 0000000..1415494
--- /dev/null
+++ b/collectors/python.d.plugin/pandas/README.md
@@ -0,0 +1,92 @@
+<!--
+title: "Pandas"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/pandas/README.md
+-->
+
+# Pandas Netdata Collector
+
+<a href="https://pandas.pydata.org/" target="_blank">
+ <img src="https://pandas.pydata.org/docs/_static/pandas.svg" alt="Pandas" width="100px" height="50px" />
+ </a>
+
+A python collector using [pandas](https://pandas.pydata.org/) to pull data and do pandas based
+preprocessing before feeding to Netdata.
+
+## Requirements
+
+This collector depends on some Python (Python 3 only) packages that can usually be installed via `pip` or `pip3`.
+
+```bash
+sudo pip install pandas requests
+```
+
+## Configuration
+
+Below is an example configuration to query some json weather data from [Open-Meteo](https://open-meteo.com),
+do some data wrangling on it and save in format as expected by Netdata.
+
+```yaml
+# example pulling some hourly temperature data
+temperature:
+ name: "temperature"
+ update_every: 3
+ chart_configs:
+ - name: "temperature_by_city"
+ title: "Temperature By City"
+ family: "temperature.today"
+ context: "pandas.temperature"
+ type: "line"
+ units: "Celsius"
+ df_steps: >
+ pd.DataFrame.from_dict(
+ {city: requests.get(
+ f'https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lng}&hourly=temperature_2m'
+ ).json()['hourly']['temperature_2m']
+ for (city,lat,lng)
+ in [
+ ('dublin', 53.3441, -6.2675),
+ ('athens', 37.9792, 23.7166),
+ ('london', 51.5002, -0.1262),
+ ('berlin', 52.5235, 13.4115),
+ ('paris', 48.8567, 2.3510),
+ ]
+ }
+ ); # use dictionary comprehension to make multiple requests;
+ df.describe(); # get aggregate stats for each city;
+ df.transpose()[['mean', 'max', 'min']].reset_index(); # just take mean, min, max;
+ df.rename(columns={'index':'city'}); # some column renaming;
+ df.pivot(columns='city').mean().to_frame().reset_index(); # force to be one row per city;
+ df.rename(columns={0:'degrees'}); # some column renaming;
+ pd.concat([df, df['city']+'_'+df['level_0']], axis=1); # add new column combining city and summary measurement label;
+ df.rename(columns={0:'measurement'}); # some column renaming;
+ df[['measurement', 'degrees']].set_index('measurement'); # just take two columns we want;
+ df.sort_index(); # sort by city name;
+ df.transpose(); # transpose so its just one wide row;
+```
+
+`chart_configs` is a list of dictionary objects where each one defines the sequence of `df_steps` to be run using [`pandas`](https://pandas.pydata.org/),
+and the `name`, `title` etc to define the
+[CHART variables](https://learn.netdata.cloud/docs/agent/collectors/python.d.plugin#global-variables-order-and-chart)
+that will control how the results will look in netdata.
+
+The example configuration above would result in a `data` dictionary like the below being collected by Netdata
+at each time step. They keys in this dictionary will be the
+[dimension](https://learn.netdata.cloud/docs/agent/web#dimensions) names on the chart.
+
+```javascript
+{'athens_max': 26.2, 'athens_mean': 19.45952380952381, 'athens_min': 12.2, 'berlin_max': 17.4, 'berlin_mean': 10.764285714285714, 'berlin_min': 5.7, 'dublin_max': 15.3, 'dublin_mean': 12.008928571428571, 'dublin_min': 6.6, 'london_max': 18.9, 'london_mean': 12.510714285714286, 'london_min': 5.2, 'paris_max': 19.4, 'paris_mean': 12.054166666666665, 'paris_min': 4.8}
+```
+
+Which, given the above configuration would end up as a chart like below in Netdata.
+
+![pandas collector temperature example chart](https://user-images.githubusercontent.com/2178292/195075312-8ce8cf68-5172-48e3-af09-104ffecfcdd6.png)
+
+## Notes
+- Each line in `df_steps` must return a pandas
+[DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object (`df`) at each step.
+- You can use
+[this colab notebook](https://colab.research.google.com/drive/1VYrddSegZqGtkWGFuiUbMbUk5f3rW6Hi?usp=sharing)
+to mock up and work on your `df_steps` iteratively before adding them to your config.
+- This collector is expecting one row in the final pandas DataFrame. It is that first row that will be taken
+as the most recent values for each dimension on each chart using (`df.to_dict(orient='records')[0]`).
+See [pd.to_dict()](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_dict.html).
diff --git a/collectors/python.d.plugin/pandas/pandas.chart.py b/collectors/python.d.plugin/pandas/pandas.chart.py
new file mode 100644
index 0000000..8eb4452
--- /dev/null
+++ b/collectors/python.d.plugin/pandas/pandas.chart.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# Description: pandas netdata python.d module
+# Author: Andrew Maguire (andrewm4894)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import pandas as pd
+
+try:
+ import requests
+ HAS_REQUESTS = True
+except ImportError:
+ HAS_REQUESTS = False
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+ORDER = []
+
+CHARTS = {}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.chart_configs = self.configuration.get('chart_configs', None)
+ self.line_sep = self.configuration.get('line_sep', ';')
+
+ def run_code(self, df_steps):
+ """eval() each line of code and ensure the result is a pandas dataframe"""
+
+ # process each line of code
+ lines = df_steps.split(self.line_sep)
+ for line in lines:
+ line_clean = line.strip('\n').strip(' ')
+ if line_clean != '' and line_clean[0] != '#':
+ df = eval(line_clean)
+ assert isinstance(df, pd.DataFrame), 'The result of each evaluated line of `df_steps` must be of type `pd.DataFrame`'
+
+ # take top row of final df as data to be collected by netdata
+ data = df.to_dict(orient='records')[0]
+
+ return data
+
+ def check(self):
+ """ensure charts and dims all configured and that we can get data"""
+
+ if not HAS_REQUESTS:
+ self.warn('requests library could not be imported')
+
+ if not self.chart_configs:
+ self.error('chart_configs must be defined')
+
+ data = dict()
+
+ # add each chart as defined by the config
+ for chart_config in self.chart_configs:
+ if chart_config['name'] not in self.charts:
+ chart_template = {
+ 'options': [
+ chart_config['name'],
+ chart_config['title'],
+ chart_config['units'],
+ chart_config['family'],
+ chart_config['context'],
+ chart_config['type']
+ ],
+ 'lines': []
+ }
+ self.charts.add_chart([chart_config['name']] + chart_template['options'])
+
+ data_tmp = self.run_code(chart_config['df_steps'])
+ data.update(data_tmp)
+
+ for dim in data_tmp:
+ self.charts[chart_config['name']].add_dimension([dim, dim, 'absolute', 1, 1])
+
+ return True
+
+ def get_data(self):
+ """get data for each chart config"""
+
+ data = dict()
+
+ for chart_config in self.chart_configs:
+ data_tmp = self.run_code(chart_config['df_steps'])
+ data.update(data_tmp)
+
+ return data
diff --git a/collectors/python.d.plugin/pandas/pandas.conf b/collectors/python.d.plugin/pandas/pandas.conf
new file mode 100644
index 0000000..6684af9
--- /dev/null
+++ b/collectors/python.d.plugin/pandas/pandas.conf
@@ -0,0 +1,191 @@
+# netdata python.d.plugin configuration for pandas
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+update_every: 5
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear on the dashboard
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, example also supports the following:
+#
+# num_lines: 4 # the number of lines to create
+# lower: 0 # the lower bound of numbers to randomly sample from
+# upper: 100 # the upper bound of numbers to randomly sample from
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+
+# Some example configurations, enable this collector, uncomment and example below and restart netdata to enable.
+
+# example pulling some hourly temperature data, a chart for today forecast (mean,min,max) and another chart for current.
+# temperature:
+# name: "temperature"
+# update_every: 5
+# chart_configs:
+# - name: "temperature_forecast_by_city"
+# title: "Temperature By City - Today Forecast"
+# family: "temperature.today"
+# context: "pandas.temperature"
+# type: "line"
+# units: "Celsius"
+# df_steps: >
+# pd.DataFrame.from_dict(
+# {city: requests.get(f'https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lng}&hourly=temperature_2m').json()['hourly']['temperature_2m']
+# for (city,lat,lng)
+# in [
+# ('dublin', 53.3441, -6.2675),
+# ('athens', 37.9792, 23.7166),
+# ('london', 51.5002, -0.1262),
+# ('berlin', 52.5235, 13.4115),
+# ('paris', 48.8567, 2.3510),
+# ('madrid', 40.4167, -3.7033),
+# ('new_york', 40.71, -74.01),
+# ('los_angeles', 34.05, -118.24),
+# ]
+# }
+# );
+# df.describe(); # get aggregate stats for each city;
+# df.transpose()[['mean', 'max', 'min']].reset_index(); # just take mean, min, max;
+# df.rename(columns={'index':'city'}); # some column renaming;
+# df.pivot(columns='city').mean().to_frame().reset_index(); # force to be one row per city;
+# df.rename(columns={0:'degrees'}); # some column renaming;
+# pd.concat([df, df['city']+'_'+df['level_0']], axis=1); # add new column combining city and summary measurement label;
+# df.rename(columns={0:'measurement'}); # some column renaming;
+# df[['measurement', 'degrees']].set_index('measurement'); # just take two columns we want;
+# df.sort_index(); # sort by city name;
+# df.transpose(); # transpose so its just one wide row;
+# - name: "temperature_current_by_city"
+# title: "Temperature By City - Current"
+# family: "temperature.current"
+# context: "pandas.temperature"
+# type: "line"
+# units: "Celsius"
+# df_steps: >
+# pd.DataFrame.from_dict(
+# {city: requests.get(f'https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lng}&current_weather=true').json()['current_weather']
+# for (city,lat,lng)
+# in [
+# ('dublin', 53.3441, -6.2675),
+# ('athens', 37.9792, 23.7166),
+# ('london', 51.5002, -0.1262),
+# ('berlin', 52.5235, 13.4115),
+# ('paris', 48.8567, 2.3510),
+# ('madrid', 40.4167, -3.7033),
+# ('new_york', 40.71, -74.01),
+# ('los_angeles', 34.05, -118.24),
+# ]
+# }
+# );
+# df.transpose();
+# df[['temperature']];
+# df.transpose();
+
+# example showing a read_csv from a url and some light pandas data wrangling.
+# pull data in csv format from london demo server and then ratio of user cpus over system cpu averaged over last 60 seconds.
+# example_csv:
+# name: "example_csv"
+# update_every: 2
+# chart_configs:
+# - name: "london_system_cpu"
+# title: "London System CPU - Ratios"
+# family: "london_system_cpu"
+# context: "pandas"
+# type: "line"
+# units: "n"
+# df_steps: >
+# pd.read_csv('https://london.my-netdata.io/api/v1/data?chart=system.cpu&format=csv&after=-60', storage_options={'User-Agent': 'netdata'});
+# df.drop('time', axis=1);
+# df.mean().to_frame().transpose();
+# df.apply(lambda row: (row.user / row.system), axis = 1).to_frame();
+# df.rename(columns={0:'average_user_system_ratio'});
+# df*100;
+
+# example showing a read_json from a url and some light pandas data wrangling.
+# pull data in json format (using requests.get() if json data is too complex for pd.read_json() ) from london demo server and work out 'total_bandwidth'.
+# example_json:
+# name: "example_json"
+# update_every: 2
+# chart_configs:
+# - name: "london_system_net"
+# title: "London System Net - Total Bandwidth"
+# family: "london_system_net"
+# context: "pandas"
+# type: "area"
+# units: "kilobits/s"
+# df_steps: >
+# pd.DataFrame(requests.get('https://london.my-netdata.io/api/v1/data?chart=system.net&format=json&after=-1').json()['data'], columns=requests.get('https://london.my-netdata.io/api/v1/data?chart=system.net&format=json&after=-1').json()['labels']);
+# df.drop('time', axis=1);
+# abs(df);
+# df.sum(axis=1).to_frame();
+# df.rename(columns={0:'total_bandwidth'});
+
+# example showing a read_xml from a url and some light pandas data wrangling.
+# pull weather forecast data in xml format, use xpath to pull out temperature forecast.
+# example_xml:
+# name: "example_xml"
+# update_every: 2
+# line_sep: "|"
+# chart_configs:
+# - name: "temperature_forcast"
+# title: "Temperature Forecast"
+# family: "temp"
+# context: "pandas.temp"
+# type: "line"
+# units: "celsius"
+# df_steps: >
+# pd.read_xml('http://metwdb-openaccess.ichec.ie/metno-wdb2ts/locationforecast?lat=54.7210798611;long=-8.7237392806', xpath='./product/time[1]/location/temperature', parser='etree')|
+# df.rename(columns={'value': 'dublin'})|
+# df[['dublin']]| \ No newline at end of file
diff --git a/collectors/python.d.plugin/postfix/Makefile.inc b/collectors/python.d.plugin/postfix/Makefile.inc
new file mode 100644
index 0000000..f4091b2
--- /dev/null
+++ b/collectors/python.d.plugin/postfix/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += postfix/postfix.chart.py
+dist_pythonconfig_DATA += postfix/postfix.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += postfix/README.md postfix/Makefile.inc
+
diff --git a/collectors/python.d.plugin/postfix/README.md b/collectors/python.d.plugin/postfix/README.md
new file mode 100644
index 0000000..1a546c6
--- /dev/null
+++ b/collectors/python.d.plugin/postfix/README.md
@@ -0,0 +1,36 @@
+<!--
+title: "Postfix monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/postfix/README.md
+sidebar_label: "Postfix"
+-->
+
+# Postfix monitoring with Netdata
+
+Monitors MTA email queue statistics using [postqueue](http://www.postfix.org/postqueue.1.html) tool.
+
+The collector executes `postqueue -p` to get Postfix queue statistics.
+
+## Requirements
+
+Postfix has internal access controls that limit activities on the mail queue. By default, all users are allowed to view
+the queue. If your system is configured with stricter access controls, you need to grant the `netdata` user access to
+view the mail queue. In order to do it, add `netdata` to `authorized_mailq_users` in the `/etc/postfix/main.cf` file.
+
+See the `authorized_mailq_users` setting in
+the [Postfix documentation](https://www.postfix.org/postconf.5.html) for more details.
+
+## Charts
+
+It produces only two charts:
+
+1. **Postfix Queue Emails**
+
+ - emails
+
+2. **Postfix Queue Emails Size** in KB
+
+ - size
+
+## Configuration
+
+Configuration is not needed.
diff --git a/collectors/python.d.plugin/postfix/postfix.chart.py b/collectors/python.d.plugin/postfix/postfix.chart.py
new file mode 100644
index 0000000..b650514
--- /dev/null
+++ b/collectors/python.d.plugin/postfix/postfix.chart.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+# Description: postfix netdata python.d module
+# Author: Pawel Krupa (paulfantom)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+
+POSTQUEUE_COMMAND = 'postqueue -p'
+
+ORDER = [
+ 'qemails',
+ 'qsize',
+]
+
+CHARTS = {
+ 'qemails': {
+ 'options': [None, 'Postfix Queue Emails', 'emails', 'queue', 'postfix.qemails', 'line'],
+ 'lines': [
+ ['emails', None, 'absolute']
+ ]
+ },
+ 'qsize': {
+ 'options': [None, 'Postfix Queue Emails Size', 'KiB', 'queue', 'postfix.qsize', 'area'],
+ 'lines': [
+ ['size', None, 'absolute']
+ ]
+ }
+}
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ ExecutableService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.command = POSTQUEUE_COMMAND
+
+ def _get_data(self):
+ """
+ Format data received from shell command
+ :return: dict
+ """
+ try:
+ raw = self._get_raw_data()[-1].split(' ')
+ if raw[0] == 'Mail' and raw[1] == 'queue':
+ return {'emails': 0,
+ 'size': 0}
+
+ return {'emails': raw[4],
+ 'size': raw[1]}
+ except (ValueError, AttributeError):
+ return None
diff --git a/collectors/python.d.plugin/postfix/postfix.conf b/collectors/python.d.plugin/postfix/postfix.conf
new file mode 100644
index 0000000..a4d2472
--- /dev/null
+++ b/collectors/python.d.plugin/postfix/postfix.conf
@@ -0,0 +1,72 @@
+# netdata python.d.plugin configuration for postfix
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# postfix is slow, so once every 10 seconds
+update_every: 10
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, postfix also supports the following:
+#
+# command: 'postqueue -p' # the command to run
+#
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+
+local:
+ command: 'postqueue -p'
diff --git a/collectors/python.d.plugin/proxysql/Makefile.inc b/collectors/python.d.plugin/proxysql/Makefile.inc
new file mode 100644
index 0000000..66be372
--- /dev/null
+++ b/collectors/python.d.plugin/proxysql/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += proxysql/proxysql.chart.py
+dist_pythonconfig_DATA += proxysql/proxysql.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += proxysql/README.md proxysql/Makefile.inc
+
diff --git a/collectors/python.d.plugin/proxysql/README.md b/collectors/python.d.plugin/proxysql/README.md
new file mode 100644
index 0000000..8c6a394
--- /dev/null
+++ b/collectors/python.d.plugin/proxysql/README.md
@@ -0,0 +1,106 @@
+<!--
+title: "ProxySQL monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/proxysql/README.md
+sidebar_label: "ProxySQL"
+-->
+
+# ProxySQL monitoring with Netdata
+
+Monitors database backend and frontend performance metrics.
+
+## Requirements
+
+- python library [MySQLdb](https://github.com/PyMySQL/mysqlclient-python) (faster) or [PyMySQL](https://github.com/PyMySQL/PyMySQL) (slower)
+- `netdata` local user to connect to the ProxySQL server.
+
+To create the `netdata` user, follow [the documentation](https://github.com/sysown/proxysql/wiki/Users-configuration#creating-a-new-user).
+
+## Charts
+
+It produces:
+
+1. **Connections (frontend)**
+
+ - connected: number of frontend connections currently connected
+ - aborted: number of frontend connections aborted due to invalid credential or max_connections reached
+ - non_idle: number of frontend connections that are not currently idle
+ - created: number of frontend connections created
+
+2. **Questions (frontend)**
+
+ - questions: total number of queries sent from frontends
+ - slow_queries: number of queries that ran for longer than the threshold in milliseconds defined in global variable `mysql-long_query_time`
+
+3. **Overall Bandwidth (backends)**
+
+ - in
+ - out
+
+4. **Status (backends)**
+
+ - Backends
+ - `1=ONLINE`: backend server is fully operational
+ - `2=SHUNNED`: backend sever is temporarily taken out of use because of either too many connection errors in a time that was too short, or replication lag exceeded the allowed threshold
+ - `3=OFFLINE_SOFT`: when a server is put into OFFLINE_SOFT mode, new incoming connections aren't accepted anymore, while the existing connections are kept until they became inactive. In other words, connections are kept in use until the current transaction is completed. This allows to gracefully detach a backend
+ - `4=OFFLINE_HARD`: when a server is put into OFFLINE_HARD mode, the existing connections are dropped, while new incoming connections aren't accepted either. This is equivalent to deleting the server from a hostgroup, or temporarily taking it out of the hostgroup for maintenance work
+ - `-1`: Unknown status
+
+5. **Bandwidth (backends)**
+
+ - Backends
+ - in
+ - out
+
+6. **Queries (backends)**
+
+ - Backends
+ - queries
+
+7. **Latency (backends)**
+
+ - Backends
+ - ping time
+
+8. **Pool connections (backends)**
+
+ - Backends
+ - Used: The number of connections are currently used by ProxySQL for sending queries to the backend server.
+ - Free: The number of connections are currently free.
+ - Established/OK: The number of connections were established successfully.
+ - Error: The number of connections weren't established successfully.
+
+9. **Commands**
+
+ - Commands
+ - Count
+ - Duration (Total duration for each command)
+
+10. **Commands Histogram**
+
+ - Commands
+ - 100us, 500us, ..., 10s, inf: the total number of commands of the given type which executed within the specified time limit and the previous one.
+
+## Configuration
+
+Edit the `python.d/proxysql.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/proxysql.conf
+```
+
+```yaml
+tcpipv4:
+ name : 'local'
+ user : 'stats'
+ pass : 'stats'
+ host : '127.0.0.1'
+ port : '6032'
+```
+
+If no configuration is given, module will fail to run.
+
+---
+
+
diff --git a/collectors/python.d.plugin/proxysql/proxysql.chart.py b/collectors/python.d.plugin/proxysql/proxysql.chart.py
new file mode 100644
index 0000000..982c28e
--- /dev/null
+++ b/collectors/python.d.plugin/proxysql/proxysql.chart.py
@@ -0,0 +1,352 @@
+# -*- coding: utf-8 -*-
+# Description: Proxysql netdata python.d module
+# Author: Ali Borhani (alibo)
+# SPDX-License-Identifier: GPL-3.0+
+
+from bases.FrameworkServices.MySQLService import MySQLService
+
+
+def query(table, *params):
+ return 'SELECT {params} FROM {table}'.format(table=table, params=', '.join(params))
+
+
+# https://github.com/sysown/proxysql/blob/master/doc/admin_tables.md#stats_mysql_global
+QUERY_GLOBAL = query(
+ "stats_mysql_global",
+ "Variable_Name",
+ "Variable_Value"
+)
+
+# https://github.com/sysown/proxysql/blob/master/doc/admin_tables.md#stats_mysql_connection_pool
+QUERY_CONNECTION_POOL = query(
+ "stats_mysql_connection_pool",
+ "hostgroup",
+ "srv_host",
+ "srv_port",
+ "status",
+ "ConnUsed",
+ "ConnFree",
+ "ConnOK",
+ "ConnERR",
+ "Queries",
+ "Bytes_data_sent",
+ "Bytes_data_recv",
+ "Latency_us"
+)
+
+# https://github.com/sysown/proxysql/blob/master/doc/admin_tables.md#stats_mysql_commands_counters
+QUERY_COMMANDS = query(
+ "stats_mysql_commands_counters",
+ "Command",
+ "Total_Time_us",
+ "Total_cnt",
+ "cnt_100us",
+ "cnt_500us",
+ "cnt_1ms",
+ "cnt_5ms",
+ "cnt_10ms",
+ "cnt_50ms",
+ "cnt_100ms",
+ "cnt_500ms",
+ "cnt_1s",
+ "cnt_5s",
+ "cnt_10s",
+ "cnt_INFs"
+)
+
+GLOBAL_STATS = [
+ 'client_connections_aborted',
+ 'client_connections_connected',
+ 'client_connections_created',
+ 'client_connections_non_idle',
+ 'proxysql_uptime',
+ 'questions',
+ 'slow_queries'
+]
+
+CONNECTION_POOL_STATS = [
+ 'status',
+ 'connused',
+ 'connfree',
+ 'connok',
+ 'connerr',
+ 'queries',
+ 'bytes_data_sent',
+ 'bytes_data_recv',
+ 'latency_us'
+]
+
+ORDER = [
+ 'connections',
+ 'active_transactions',
+ 'questions',
+ 'pool_overall_net',
+ 'commands_count',
+ 'commands_duration',
+ 'pool_status',
+ 'pool_net',
+ 'pool_queries',
+ 'pool_latency',
+ 'pool_connection_used',
+ 'pool_connection_free',
+ 'pool_connection_ok',
+ 'pool_connection_error'
+]
+
+HISTOGRAM_ORDER = [
+ '100us',
+ '500us',
+ '1ms',
+ '5ms',
+ '10ms',
+ '50ms',
+ '100ms',
+ '500ms',
+ '1s',
+ '5s',
+ '10s',
+ 'inf'
+]
+
+STATUS = {
+ "ONLINE": 1,
+ "SHUNNED": 2,
+ "OFFLINE_SOFT": 3,
+ "OFFLINE_HARD": 4
+}
+
+CHARTS = {
+ 'pool_status': {
+ 'options': [None, 'ProxySQL Backend Status', 'status', 'status', 'proxysql.pool_status', 'line'],
+ 'lines': []
+ },
+ 'pool_net': {
+ 'options': [None, 'ProxySQL Backend Bandwidth', 'kilobits/s', 'bandwidth', 'proxysql.pool_net', 'area'],
+ 'lines': []
+ },
+ 'pool_overall_net': {
+ 'options': [None, 'ProxySQL Backend Overall Bandwidth', 'kilobits/s', 'overall_bandwidth',
+ 'proxysql.pool_overall_net', 'area'],
+ 'lines': [
+ ['bytes_data_recv', 'in', 'incremental', 8, 1000],
+ ['bytes_data_sent', 'out', 'incremental', -8, 1000]
+ ]
+ },
+ 'questions': {
+ 'options': [None, 'ProxySQL Frontend Questions', 'questions/s', 'questions', 'proxysql.questions', 'line'],
+ 'lines': [
+ ['questions', 'questions', 'incremental'],
+ ['slow_queries', 'slow_queries', 'incremental']
+ ]
+ },
+ 'pool_queries': {
+ 'options': [None, 'ProxySQL Backend Queries', 'queries/s', 'queries', 'proxysql.queries', 'line'],
+ 'lines': []
+ },
+ 'active_transactions': {
+ 'options': [None, 'ProxySQL Frontend Active Transactions', 'transactions/s', 'active_transactions',
+ 'proxysql.active_transactions', 'line'],
+ 'lines': [
+ ['active_transactions', 'active_transactions', 'absolute']
+ ]
+ },
+ 'pool_latency': {
+ 'options': [None, 'ProxySQL Backend Latency', 'milliseconds', 'latency', 'proxysql.latency', 'line'],
+ 'lines': []
+ },
+ 'connections': {
+ 'options': [None, 'ProxySQL Frontend Connections', 'connections/s', 'connections', 'proxysql.connections',
+ 'line'],
+ 'lines': [
+ ['client_connections_connected', 'connected', 'absolute'],
+ ['client_connections_created', 'created', 'incremental'],
+ ['client_connections_aborted', 'aborted', 'incremental'],
+ ['client_connections_non_idle', 'non_idle', 'absolute']
+ ]
+ },
+ 'pool_connection_used': {
+ 'options': [None, 'ProxySQL Used Connections', 'connections', 'pool_connections',
+ 'proxysql.pool_used_connections', 'line'],
+ 'lines': []
+ },
+ 'pool_connection_free': {
+ 'options': [None, 'ProxySQL Free Connections', 'connections', 'pool_connections',
+ 'proxysql.pool_free_connections', 'line'],
+ 'lines': []
+ },
+ 'pool_connection_ok': {
+ 'options': [None, 'ProxySQL Established Connections', 'connections', 'pool_connections',
+ 'proxysql.pool_ok_connections', 'line'],
+ 'lines': []
+ },
+ 'pool_connection_error': {
+ 'options': [None, 'ProxySQL Error Connections', 'connections', 'pool_connections',
+ 'proxysql.pool_error_connections', 'line'],
+ 'lines': []
+ },
+ 'commands_count': {
+ 'options': [None, 'ProxySQL Commands', 'commands', 'commands', 'proxysql.commands_count', 'line'],
+ 'lines': []
+ },
+ 'commands_duration': {
+ 'options': [None, 'ProxySQL Commands Duration', 'milliseconds', 'commands', 'proxysql.commands_duration',
+ 'line'],
+ 'lines': []
+ }
+}
+
+
+class Service(MySQLService):
+ def __init__(self, configuration=None, name=None):
+ MySQLService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.queries = dict(
+ global_status=QUERY_GLOBAL,
+ connection_pool_status=QUERY_CONNECTION_POOL,
+ commands_status=QUERY_COMMANDS
+ )
+
+ def _get_data(self):
+ raw_data = self._get_raw_data(description=True)
+
+ if not raw_data:
+ return None
+
+ to_netdata = dict()
+
+ if 'global_status' in raw_data:
+ global_status = dict(raw_data['global_status'][0])
+ for key in global_status:
+ if key.lower() in GLOBAL_STATS:
+ to_netdata[key.lower()] = global_status[key]
+
+ if 'connection_pool_status' in raw_data:
+
+ to_netdata['bytes_data_recv'] = 0
+ to_netdata['bytes_data_sent'] = 0
+
+ for record in raw_data['connection_pool_status'][0]:
+ backend = self.generate_backend(record)
+ name = self.generate_backend_name(backend)
+
+ for key in backend:
+ if key in CONNECTION_POOL_STATS:
+ if key == 'status':
+ backend[key] = self.convert_status(backend[key])
+
+ if len(self.charts) > 0:
+ if (name + '_status') not in self.charts['pool_status']:
+ self.add_backend_dimensions(name)
+
+ to_netdata["{0}_{1}".format(name, key)] = backend[key]
+
+ if key == 'bytes_data_recv':
+ to_netdata['bytes_data_recv'] += int(backend[key])
+
+ if key == 'bytes_data_sent':
+ to_netdata['bytes_data_sent'] += int(backend[key])
+
+ if 'commands_status' in raw_data:
+ for record in raw_data['commands_status'][0]:
+ cmd = self.generate_command_stats(record)
+ name = cmd['name']
+
+ if len(self.charts) > 0:
+ if (name + '_count') not in self.charts['commands_count']:
+ self.add_command_dimensions(name)
+ self.add_histogram_chart(cmd)
+
+ to_netdata[name + '_count'] = cmd['count']
+ to_netdata[name + '_duration'] = cmd['duration']
+ for histogram in cmd['histogram']:
+ dimId = 'commands_histogram_{0}_{1}'.format(name, histogram)
+ to_netdata[dimId] = cmd['histogram'][histogram]
+
+ return to_netdata or None
+
+ def add_backend_dimensions(self, name):
+ self.charts['pool_status'].add_dimension([name + '_status', name, 'absolute'])
+ self.charts['pool_net'].add_dimension([name + '_bytes_data_recv', 'from_' + name, 'incremental', 8, 1024])
+ self.charts['pool_net'].add_dimension([name + '_bytes_data_sent', 'to_' + name, 'incremental', -8, 1024])
+ self.charts['pool_queries'].add_dimension([name + '_queries', name, 'incremental'])
+ self.charts['pool_latency'].add_dimension([name + '_latency_us', name, 'absolute', 1, 1000])
+ self.charts['pool_connection_used'].add_dimension([name + '_connused', name, 'absolute'])
+ self.charts['pool_connection_free'].add_dimension([name + '_connfree', name, 'absolute'])
+ self.charts['pool_connection_ok'].add_dimension([name + '_connok', name, 'incremental'])
+ self.charts['pool_connection_error'].add_dimension([name + '_connerr', name, 'incremental'])
+
+ def add_command_dimensions(self, cmd):
+ self.charts['commands_count'].add_dimension([cmd + '_count', cmd, 'incremental'])
+ self.charts['commands_duration'].add_dimension([cmd + '_duration', cmd, 'incremental', 1, 1000])
+
+ def add_histogram_chart(self, cmd):
+ chart = self.charts.add_chart(self.histogram_chart(cmd))
+
+ for histogram in HISTOGRAM_ORDER:
+ dimId = 'commands_histogram_{0}_{1}'.format(cmd['name'], histogram)
+ chart.add_dimension([dimId, histogram, 'incremental'])
+
+ @staticmethod
+ def histogram_chart(cmd):
+ return [
+ 'commands_histogram_' + cmd['name'],
+ None,
+ 'ProxySQL {0} Command Histogram'.format(cmd['name'].title()),
+ 'commands',
+ 'commands_histogram',
+ 'proxysql.commands_histogram_' + cmd['name'],
+ 'stacked'
+ ]
+
+ @staticmethod
+ def generate_backend(data):
+ return {
+ 'hostgroup': data[0],
+ 'srv_host': data[1],
+ 'srv_port': data[2],
+ 'status': data[3],
+ 'connused': data[4],
+ 'connfree': data[5],
+ 'connok': data[6],
+ 'connerr': data[7],
+ 'queries': data[8],
+ 'bytes_data_sent': data[9],
+ 'bytes_data_recv': data[10],
+ 'latency_us': data[11]
+ }
+
+ @staticmethod
+ def generate_command_stats(data):
+ return {
+ 'name': data[0].lower(),
+ 'duration': data[1],
+ 'count': data[2],
+ 'histogram': {
+ '100us': data[3],
+ '500us': data[4],
+ '1ms': data[5],
+ '5ms': data[6],
+ '10ms': data[7],
+ '50ms': data[8],
+ '100ms': data[9],
+ '500ms': data[10],
+ '1s': data[11],
+ '5s': data[12],
+ '10s': data[13],
+ 'inf': data[14]
+ }
+ }
+
+ @staticmethod
+ def generate_backend_name(backend):
+ hostgroup = backend['hostgroup'].replace(' ', '_').lower()
+ host = backend['srv_host'].replace('.', '_')
+
+ return "{0}_{1}_{2}".format(hostgroup, host, backend['srv_port'])
+
+ @staticmethod
+ def convert_status(status):
+ if status in STATUS:
+ return STATUS[status]
+ return -1
diff --git a/collectors/python.d.plugin/proxysql/proxysql.conf b/collectors/python.d.plugin/proxysql/proxysql.conf
new file mode 100644
index 0000000..3c503a8
--- /dev/null
+++ b/collectors/python.d.plugin/proxysql/proxysql.conf
@@ -0,0 +1,116 @@
+# netdata python.d.plugin configuration for ProxySQL
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, proxysql also supports the following:
+#
+# host: 'IP or HOSTNAME' # the host to connect to
+# port: PORT # the port to connect to
+#
+# in all cases, the following can also be set:
+#
+# user: 'username' # the proxysql username to use
+# pass: 'password' # the proxysql password to use
+#
+
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+tcp:
+ name : 'local'
+ user : 'stats'
+ pass : 'stats'
+ host : 'localhost'
+ port : '6032'
+
+tcpipv4:
+ name : 'local'
+ user : 'stats'
+ pass : 'stats'
+ host : '127.0.0.1'
+ port : '6032'
+
+tcpipv6:
+ name : 'local'
+ user : 'stats'
+ pass : 'stats'
+ host : '::1'
+ port : '6032'
+
+tcp_admin:
+ name : 'local'
+ user : 'admin'
+ pass : 'admin'
+ host : 'localhost'
+ port : '6032'
+
+tcpipv4_admin:
+ name : 'local'
+ user : 'admin'
+ pass : 'admin'
+ host : '127.0.0.1'
+ port : '6032'
+
+tcpipv6_admin:
+ name : 'local'
+ user : 'admin'
+ pass : 'admin'
+ host : '::1'
+ port : '6032'
diff --git a/collectors/python.d.plugin/puppet/Makefile.inc b/collectors/python.d.plugin/puppet/Makefile.inc
new file mode 100644
index 0000000..fe94b92
--- /dev/null
+++ b/collectors/python.d.plugin/puppet/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += puppet/puppet.chart.py
+dist_pythonconfig_DATA += puppet/puppet.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += puppet/README.md puppet/Makefile.inc
+
diff --git a/collectors/python.d.plugin/puppet/README.md b/collectors/python.d.plugin/puppet/README.md
new file mode 100644
index 0000000..1b06d18
--- /dev/null
+++ b/collectors/python.d.plugin/puppet/README.md
@@ -0,0 +1,67 @@
+<!--
+title: "Puppet monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/puppet/README.md
+sidebar_label: "Puppet"
+-->
+
+# Puppet monitoring with Netdata
+
+Monitor status of Puppet Server and Puppet DB.
+
+Following charts are drawn:
+
+1. **JVM Heap**
+
+ - committed (allocated from OS)
+ - used (actual use)
+
+2. **JVM Non-Heap**
+
+ - committed (allocated from OS)
+ - used (actual use)
+
+3. **CPU Usage**
+
+ - execution
+ - GC (taken by garbage collection)
+
+4. **File Descriptors**
+
+ - max
+ - used
+
+## Configuration
+
+Edit the `python.d/puppet.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/puppet.conf
+```
+
+```yaml
+puppetdb:
+ url: 'https://fqdn.example.com:8081'
+ tls_cert_file: /path/to/client.crt
+ tls_key_file: /path/to/client.key
+ autodetection_retry: 1
+
+puppetserver:
+ url: 'https://fqdn.example.com:8140'
+ autodetection_retry: 1
+```
+
+When no configuration is given, module uses `https://fqdn.example.com:8140`.
+
+### notes
+
+- Exact Fully Qualified Domain Name of the node should be used.
+- Usually Puppet Server/DB startup time is VERY long. So, there should
+ be quite reasonable retry count.
+- Secure PuppetDB config may require client certificate. Not applies
+ to default PuppetDB configuration though.
+
+---
+
+
diff --git a/collectors/python.d.plugin/puppet/puppet.chart.py b/collectors/python.d.plugin/puppet/puppet.chart.py
new file mode 100644
index 0000000..f8adf60
--- /dev/null
+++ b/collectors/python.d.plugin/puppet/puppet.chart.py
@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+# Description: puppet netdata python.d module
+# Author: Andrey Galkin <andrey@futoin.org> (andvgal)
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This module should work both with OpenSource and PE versions
+# of PuppetServer and PuppetDB.
+#
+# NOTE: PuppetDB may be configured to require proper TLS
+# client certificate for security reasons. Use tls_key_file
+# and tls_cert_file options then.
+#
+
+import socket
+from json import loads
+
+from bases.FrameworkServices.UrlService import UrlService
+
+update_every = 5
+
+MiB = 1 << 20
+CPU_SCALE = 1000
+
+ORDER = [
+ 'jvm_heap',
+ 'jvm_nonheap',
+ 'cpu',
+ 'fd_open',
+]
+
+CHARTS = {
+ 'jvm_heap': {
+ 'options': [None, 'JVM Heap', 'MiB', 'resources', 'puppet.jvm', 'area'],
+ 'lines': [
+ ['jvm_heap_committed', 'committed', 'absolute', 1, MiB],
+ ['jvm_heap_used', 'used', 'absolute', 1, MiB],
+ ],
+ 'variables': [
+ ['jvm_heap_max'],
+ ['jvm_heap_init'],
+ ],
+ },
+ 'jvm_nonheap': {
+ 'options': [None, 'JVM Non-Heap', 'MiB', 'resources', 'puppet.jvm', 'area'],
+ 'lines': [
+ ['jvm_nonheap_committed', 'committed', 'absolute', 1, MiB],
+ ['jvm_nonheap_used', 'used', 'absolute', 1, MiB],
+ ],
+ 'variables': [
+ ['jvm_nonheap_max'],
+ ['jvm_nonheap_init'],
+ ],
+ },
+ 'cpu': {
+ 'options': [None, 'CPU usage', 'percentage', 'resources', 'puppet.cpu', 'stacked'],
+ 'lines': [
+ ['cpu_time', 'execution', 'absolute', 1, CPU_SCALE],
+ ['gc_time', 'GC', 'absolute', 1, CPU_SCALE],
+ ]
+ },
+ 'fd_open': {
+ 'options': [None, 'File Descriptors', 'descriptors', 'resources', 'puppet.fdopen', 'line'],
+ 'lines': [
+ ['fd_used', 'used', 'absolute'],
+ ],
+ 'variables': [
+ ['fd_max'],
+ ],
+ },
+}
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.url = 'https://{0}:8140'.format(socket.getfqdn())
+
+ def _get_data(self):
+ # NOTE: there are several ways to retrieve data
+ # 1. Only PE versions:
+ # https://puppet.com/docs/pe/2018.1/api_status/status_api_metrics_endpoints.html
+ # 2. Individual Metrics API (JMX):
+ # https://puppet.com/docs/pe/2018.1/api_status/metrics_api.html
+ # 3. Extended status at debug level:
+ # https://puppet.com/docs/pe/2018.1/api_status/status_api_json_endpoints.html
+ #
+ # For sake of simplicity and efficiency the status one is used..
+
+ raw_data = self._get_raw_data(self.url + '/status/v1/services?level=debug')
+
+ if raw_data is None:
+ return None
+
+ raw_data = loads(raw_data)
+ data = {}
+
+ try:
+ try:
+ jvm_metrics = raw_data['status-service']['status']['experimental']['jvm-metrics']
+ except KeyError:
+ jvm_metrics = raw_data['status-service']['status']['jvm-metrics']
+
+ heap_mem = jvm_metrics['heap-memory']
+ non_heap_mem = jvm_metrics['non-heap-memory']
+
+ for k in ['max', 'committed', 'used', 'init']:
+ data['jvm_heap_' + k] = heap_mem[k]
+ data['jvm_nonheap_' + k] = non_heap_mem[k]
+
+ fd_open = jvm_metrics['file-descriptors']
+ data['fd_max'] = fd_open['max']
+ data['fd_used'] = fd_open['used']
+
+ data['cpu_time'] = int(jvm_metrics['cpu-usage'] * CPU_SCALE)
+ data['gc_time'] = int(jvm_metrics['gc-cpu-usage'] * CPU_SCALE)
+ except KeyError:
+ pass
+
+ return data or None
diff --git a/collectors/python.d.plugin/puppet/puppet.conf b/collectors/python.d.plugin/puppet/puppet.conf
new file mode 100644
index 0000000..ff5c3d0
--- /dev/null
+++ b/collectors/python.d.plugin/puppet/puppet.conf
@@ -0,0 +1,94 @@
+# netdata python.d.plugin configuration for Puppet Server and Puppet DB
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# These configuration comes from UrlService base:
+# url: # HTTP or HTTPS URL
+# tls_verify: False # Control HTTPS server certificate verification
+# tls_ca_file: # Optional CA (bundle) file to use
+# tls_cert_file: # Optional client certificate file
+# tls_key_file: # Optional client key file
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+# puppet:
+# url: 'https://<FQDN>:8140'
+#
+
+#
+# Production configuration should look like below.
+#
+# NOTE: usually Puppet Server/DB startup time is VERY long. So, there should
+# be quite reasonable retry count.
+#
+# NOTE: secure PuppetDB config may require client certificate.
+# Not applies to default PuppetDB configuration though.
+#
+# puppetdb:
+# url: 'https://fqdn.example.com:8081'
+# tls_cert_file: /path/to/client.crt
+# tls_key_file: /path/to/client.key
+# autodetection_retry: 1
+#
+# puppetserver:
+# url: 'https://fqdn.example.com:8140'
+# autodetection_retry: 1
+#
diff --git a/collectors/python.d.plugin/python.d.conf b/collectors/python.d.plugin/python.d.conf
new file mode 100644
index 0000000..7b43ee2
--- /dev/null
+++ b/collectors/python.d.plugin/python.d.conf
@@ -0,0 +1,84 @@
+# netdata python.d.plugin configuration
+#
+# This file is in YaML format.
+# Generally the format is:
+#
+# name: value
+#
+
+# Enable / disable the whole python.d.plugin (all its modules)
+enabled: yes
+
+# ----------------------------------------------------------------------
+# Enable / Disable python.d.plugin modules
+#default_run: yes
+#
+# If "default_run" = "yes" the default for all modules is enabled (yes).
+# Setting any of these to "no" will disable it.
+#
+# If "default_run" = "no" the default for all modules is disabled (no).
+# Setting any of these to "yes" will enable it.
+
+# Enable / Disable explicit garbage collection (full collection run). Default is enabled.
+gc_run: yes
+
+# Garbage collection interval in seconds. Default is 300.
+gc_interval: 300
+
+# adaptec_raid: yes
+# alarms: yes
+# am2320: yes
+# anomalies: no
+# beanstalk: yes
+# bind_rndc: yes
+# boinc: yes
+# ceph: yes
+# changefinder: no
+# dockerd: yes
+# dovecot: yes
+
+# this is just an example
+example: no
+
+# exim: yes
+# fail2ban: yes
+# gearman: yes
+go_expvar: no
+
+# haproxy: yes
+# hddtemp: yes
+hpssa: no
+# icecast: yes
+# ipfs: yes
+# litespeed: yes
+logind: no
+# megacli: yes
+# memcached: yes
+# mongodb: yes
+# monit: yes
+# nvidia_smi: yes
+# nsd: yes
+# ntpd: yes
+# openldap: yes
+# oracledb: yes
+# pandas: yes
+# postfix: yes
+# proxysql: yes
+# puppet: yes
+# rabbitmq: yes
+# rethinkdbs: yes
+# retroshare: yes
+# riakkv: yes
+# samba: yes
+# sensors: yes
+# smartd_log: yes
+# spigotmc: yes
+# springboot: yes
+# squid: yes
+# traefik: yes
+# tomcat: yes
+# tor: yes
+# uwsgi: yes
+# varnish: yes
+# w1sensor: yes
+# zscores: no
diff --git a/collectors/python.d.plugin/python.d.plugin.in b/collectors/python.d.plugin/python.d.plugin.in
new file mode 100644
index 0000000..681ceb4
--- /dev/null
+++ b/collectors/python.d.plugin/python.d.plugin.in
@@ -0,0 +1,922 @@
+#!/usr/bin/env bash
+'''':;
+pybinary=$(which python3 || which python || which python2)
+filtered=()
+for arg in "$@"
+do
+ case $arg in
+ -p*) pybinary=${arg:2}
+ shift 1 ;;
+ *) filtered+=("$arg") ;;
+ esac
+done
+if [ "$pybinary" = "" ]
+then
+ echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM"
+ exit 1
+fi
+exec "$pybinary" "$0" "${filtered[@]}" # '''
+
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Pawel Krupa (paulfantom)
+# Author: Ilya Mashchenko (l2isbad)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import collections
+import copy
+import gc
+import json
+import os
+import pprint
+import re
+import sys
+import threading
+import time
+import types
+
+try:
+ from queue import Queue
+except ImportError:
+ from Queue import Queue
+
+PY_VERSION = sys.version_info[:2] # (major=3, minor=7, micro=3, releaselevel='final', serial=0)
+
+if PY_VERSION > (3, 1):
+ from importlib.machinery import SourceFileLoader
+else:
+ from imp import load_source as SourceFileLoader
+
+ENV_NETDATA_USER_CONFIG_DIR = 'NETDATA_USER_CONFIG_DIR'
+ENV_NETDATA_STOCK_CONFIG_DIR = 'NETDATA_STOCK_CONFIG_DIR'
+ENV_NETDATA_PLUGINS_DIR = 'NETDATA_PLUGINS_DIR'
+ENV_NETDATA_USER_PLUGINS_DIRS = 'NETDATA_USER_PLUGINS_DIRS'
+ENV_NETDATA_LIB_DIR = 'NETDATA_LIB_DIR'
+ENV_NETDATA_UPDATE_EVERY = 'NETDATA_UPDATE_EVERY'
+ENV_NETDATA_LOCK_DIR = 'NETDATA_LOCK_DIR'
+
+
+def add_pythond_packages():
+ pluginsd = os.getenv(ENV_NETDATA_PLUGINS_DIR, os.path.dirname(__file__))
+ pythond = os.path.abspath(pluginsd + '/../python.d')
+ packages = os.path.join(pythond, 'python_modules')
+ sys.path.append(packages)
+
+
+add_pythond_packages()
+
+from bases.collection import safe_print
+from bases.loggers import PythonDLogger
+from bases.loaders import load_config
+from third_party import filelock
+
+try:
+ from collections import OrderedDict
+except ImportError:
+ from third_party.ordereddict import OrderedDict
+
+
+def dirs():
+ var_lib = os.getenv(
+ ENV_NETDATA_LIB_DIR,
+ '@varlibdir_POST@',
+ )
+ plugin_user_config = os.getenv(
+ ENV_NETDATA_USER_CONFIG_DIR,
+ '@configdir_POST@',
+ )
+ plugin_stock_config = os.getenv(
+ ENV_NETDATA_STOCK_CONFIG_DIR,
+ '@libconfigdir_POST@',
+ )
+ pluginsd = os.getenv(
+ ENV_NETDATA_PLUGINS_DIR,
+ os.path.dirname(__file__),
+ )
+ locks = os.getenv(
+ ENV_NETDATA_LOCK_DIR,
+ os.path.join('@varlibdir_POST@', 'lock')
+ )
+ modules_user_config = os.path.join(plugin_user_config, 'python.d')
+ modules_stock_config = os.path.join(plugin_stock_config, 'python.d')
+ modules = os.path.abspath(pluginsd + '/../python.d')
+ user_modules = [os.path.join(p, 'python.d') for p in
+ os.getenv(ENV_NETDATA_USER_PLUGINS_DIRS, "").split(" ") if
+ p]
+
+ Dirs = collections.namedtuple(
+ 'Dirs',
+ [
+ 'plugin_user_config',
+ 'plugin_stock_config',
+ 'modules_user_config',
+ 'modules_stock_config',
+ 'modules',
+ 'user_modules',
+ 'var_lib',
+ 'locks',
+ ]
+ )
+ return Dirs(
+ plugin_user_config,
+ plugin_stock_config,
+ modules_user_config,
+ modules_stock_config,
+ modules,
+ user_modules,
+ var_lib,
+ locks,
+ )
+
+
+DIRS = dirs()
+
+IS_ATTY = sys.stdout.isatty() or sys.stderr.isatty()
+
+MODULE_SUFFIX = '.chart.py'
+
+
+def find_available_modules(*directories):
+ AvailableModule = collections.namedtuple(
+ 'AvailableModule',
+ [
+ 'filepath',
+ 'name',
+ ]
+ )
+ available = list()
+ for d in directories:
+ try:
+ if not os.path.isdir(d):
+ continue
+ files = sorted(os.listdir(d))
+ except OSError:
+ continue
+ modules = [m for m in files if m.endswith(MODULE_SUFFIX)]
+ available.extend([AvailableModule(os.path.join(d, m), m[:-len(MODULE_SUFFIX)]) for m in modules])
+
+ return available
+
+
+def available_modules():
+ obsolete = (
+ 'apache_cache', # replaced by web_log
+ 'cpuidle', # rewritten in C
+ 'cpufreq', # rewritten in C
+ 'gunicorn_log', # replaced by web_log
+ 'linux_power_supply', # rewritten in C
+ 'nginx_log', # replaced by web_log
+ 'mdstat', # rewritten in C
+ 'sslcheck', # rewritten in Go, memory leak bug https://github.com/netdata/netdata/issues/5624
+ 'unbound', # rewritten in Go
+ )
+
+ stock = [m for m in find_available_modules(DIRS.modules) if m.name not in obsolete]
+ user = find_available_modules(*DIRS.user_modules)
+
+ available, seen = list(), set()
+ for m in user + stock:
+ if m.name in seen:
+ continue
+ seen.add(m.name)
+ available.append(m)
+
+ return available
+
+
+AVAILABLE_MODULES = available_modules()
+
+JOB_BASE_CONF = {
+ 'update_every': int(os.getenv(ENV_NETDATA_UPDATE_EVERY, 1)),
+ 'priority': 60000,
+ 'autodetection_retry': 0,
+ 'chart_cleanup': 10,
+ 'penalty': True,
+ 'name': str(),
+}
+
+PLUGIN_BASE_CONF = {
+ 'enabled': True,
+ 'default_run': True,
+ 'gc_run': True,
+ 'gc_interval': 300,
+}
+
+
+def multi_path_find(name, *paths):
+ for path in paths:
+ abs_name = os.path.join(path, name)
+ if os.path.isfile(abs_name):
+ return abs_name
+ return str()
+
+
+def load_module(name, filepath):
+ module = SourceFileLoader('pythond_' + name, filepath)
+ if isinstance(module, types.ModuleType):
+ return module
+ return module.load_module()
+
+
+class ModuleConfig:
+ def __init__(self, name, config=None):
+ self.name = name
+ self.config = config or OrderedDict()
+
+ def load(self, abs_path):
+ self.config.update(load_config(abs_path) or dict())
+
+ def defaults(self):
+ keys = (
+ 'update_every',
+ 'priority',
+ 'autodetection_retry',
+ 'chart_cleanup',
+ 'penalty',
+ )
+ return dict((k, self.config[k]) for k in keys if k in self.config)
+
+ def create_job(self, job_name, job_config=None):
+ job_config = job_config or dict()
+
+ config = OrderedDict()
+ config.update(job_config)
+ config['job_name'] = job_name
+ for k, v in self.defaults().items():
+ config.setdefault(k, v)
+
+ return config
+
+ def job_names(self):
+ return [v for v in self.config if isinstance(self.config.get(v), dict)]
+
+ def single_job(self):
+ return [self.create_job(self.name, self.config)]
+
+ def multi_job(self):
+ return [self.create_job(n, self.config[n]) for n in self.job_names()]
+
+ def create_jobs(self):
+ return self.multi_job() or self.single_job()
+
+
+class JobsConfigsBuilder:
+ def __init__(self, config_dirs):
+ self.config_dirs = config_dirs
+ self.log = PythonDLogger()
+ self.job_defaults = None
+ self.module_defaults = None
+ self.min_update_every = None
+
+ def load_module_config(self, module_name):
+ name = '{0}.conf'.format(module_name)
+ self.log.debug("[{0}] looking for '{1}' in {2}".format(module_name, name, self.config_dirs))
+ config = ModuleConfig(module_name)
+
+ abs_path = multi_path_find(name, *self.config_dirs)
+ if not abs_path:
+ self.log.warning("[{0}] '{1}' was not found".format(module_name, name))
+ return config
+
+ self.log.debug("[{0}] loading '{1}'".format(module_name, abs_path))
+ try:
+ config.load(abs_path)
+ except Exception as error:
+ self.log.error("[{0}] error on loading '{1}' : {2}".format(module_name, abs_path, repr(error)))
+ return None
+
+ self.log.debug("[{0}] '{1}' is loaded".format(module_name, abs_path))
+ return config
+
+ @staticmethod
+ def apply_defaults(jobs, defaults):
+ if defaults is None:
+ return
+ for k, v in defaults.items():
+ for job in jobs:
+ job.setdefault(k, v)
+
+ def set_min_update_every(self, jobs, min_update_every):
+ if min_update_every is None:
+ return
+ for job in jobs:
+ if 'update_every' in job and job['update_every'] < self.min_update_every:
+ job['update_every'] = self.min_update_every
+
+ def build(self, module_name):
+ config = self.load_module_config(module_name)
+ if config is None:
+ return None
+
+ configs = config.create_jobs()
+ self.log.info("[{0}] built {1} job(s) configs".format(module_name, len(configs)))
+
+ self.apply_defaults(configs, self.module_defaults)
+ self.apply_defaults(configs, self.job_defaults)
+ self.set_min_update_every(configs, self.min_update_every)
+
+ return configs
+
+
+JOB_STATUS_ACTIVE = 'active'
+JOB_STATUS_RECOVERING = 'recovering'
+JOB_STATUS_DROPPED = 'dropped'
+JOB_STATUS_INIT = 'initial'
+
+
+class Job(threading.Thread):
+ inf = -1
+
+ def __init__(self, service, module_name, config):
+ threading.Thread.__init__(self)
+ self.daemon = True
+ self.service = service
+ self.module_name = module_name
+ self.config = config
+ self.real_name = config['job_name']
+ self.actual_name = config['override_name'] or self.real_name
+ self.autodetection_retry = config['autodetection_retry']
+ self.checks = self.inf
+ self.job = None
+ self.status = JOB_STATUS_INIT
+
+ def is_inited(self):
+ return self.job is not None
+
+ def init(self):
+ self.job = self.service(configuration=copy.deepcopy(self.config))
+
+ def full_name(self):
+ return self.job.name
+
+ def check(self):
+ ok = self.job.check()
+ self.checks -= self.checks != self.inf and not ok
+ return ok
+
+ def create(self):
+ self.job.create()
+
+ def need_to_recheck(self):
+ return self.autodetection_retry != 0 and self.checks != 0
+
+ def run(self):
+ self.job.run()
+
+
+class ModuleSrc:
+ def __init__(self, m):
+ self.name = m.name
+ self.filepath = m.filepath
+ self.src = None
+
+ def load(self):
+ self.src = load_module(self.name, self.filepath)
+
+ def get(self, key):
+ return getattr(self.src, key, None)
+
+ def service(self):
+ return self.get('Service')
+
+ def defaults(self):
+ keys = (
+ 'update_every',
+ 'priority',
+ 'autodetection_retry',
+ 'chart_cleanup',
+ 'penalty',
+ )
+ return dict((k, self.get(k)) for k in keys if self.get(k) is not None)
+
+ def is_disabled_by_default(self):
+ return bool(self.get('disabled_by_default'))
+
+
+class JobsStatuses:
+ def __init__(self):
+ self.items = OrderedDict()
+
+ def dump(self):
+ return json.dumps(self.items, indent=2)
+
+ def get(self, module_name, job_name):
+ if module_name not in self.items:
+ return None
+ return self.items[module_name].get(job_name)
+
+ def has(self, module_name, job_name):
+ return self.get(module_name, job_name) is not None
+
+ def from_file(self, path):
+ with open(path) as f:
+ data = json.load(f)
+ return self.from_json(data)
+
+ @staticmethod
+ def from_json(items):
+ if not isinstance(items, dict):
+ raise Exception('items obj has wrong type : {0}'.format(type(items)))
+ if not items:
+ return JobsStatuses()
+
+ v = OrderedDict()
+ for mod_name in sorted(items):
+ if not items[mod_name]:
+ continue
+ v[mod_name] = OrderedDict()
+ for job_name in sorted(items[mod_name]):
+ v[mod_name][job_name] = items[mod_name][job_name]
+
+ rv = JobsStatuses()
+ rv.items = v
+ return rv
+
+ @staticmethod
+ def from_jobs(jobs):
+ v = OrderedDict()
+ for job in jobs:
+ status = job.status
+ if status not in (JOB_STATUS_ACTIVE, JOB_STATUS_RECOVERING):
+ continue
+ if job.module_name not in v:
+ v[job.module_name] = OrderedDict()
+ v[job.module_name][job.real_name] = status
+
+ rv = JobsStatuses()
+ rv.items = v
+ return rv
+
+
+class StdoutSaver:
+ @staticmethod
+ def save(dump):
+ print(dump)
+
+
+class CachedFileSaver:
+ def __init__(self, path):
+ self.last_save_success = False
+ self.last_saved_dump = str()
+ self.path = path
+
+ def save(self, dump):
+ if self.last_save_success and self.last_saved_dump == dump:
+ return
+ try:
+ with open(self.path, 'w') as out:
+ out.write(dump)
+ except Exception:
+ self.last_save_success = False
+ raise
+ self.last_saved_dump = dump
+ self.last_save_success = True
+
+
+class PluginConfig(dict):
+ def __init__(self, *args):
+ dict.__init__(self, *args)
+
+ def is_module_explicitly_enabled(self, module_name):
+ return self._is_module_enabled(module_name, True)
+
+ def is_module_enabled(self, module_name):
+ return self._is_module_enabled(module_name, False)
+
+ def _is_module_enabled(self, module_name, explicit):
+ if module_name in self:
+ return self[module_name]
+ if explicit:
+ return False
+ return self['default_run']
+
+
+class FileLockRegistry:
+ def __init__(self, path):
+ self.path = path
+ self.locks = dict()
+
+ @staticmethod
+ def rename(name):
+ # go version name is 'docker'
+ if name.startswith("dockerd"):
+ name = "docker" + name[7:]
+ return name
+
+
+ def register(self, name):
+ name = self.rename(name)
+ if name in self.locks:
+ return
+ file = os.path.join(self.path, '{0}.collector.lock'.format(name))
+ lock = filelock.FileLock(file)
+ lock.acquire(timeout=0)
+ self.locks[name] = lock
+
+ def unregister(self, name):
+ name = self.rename(name)
+ if name not in self.locks:
+ return
+ lock = self.locks[name]
+ lock.release()
+ del self.locks[name]
+
+
+class DummyRegistry:
+ def register(self, name):
+ pass
+
+ def unregister(self, name):
+ pass
+
+
+class Plugin:
+ config_name = 'python.d.conf'
+ jobs_status_dump_name = 'pythond-jobs-statuses.json'
+
+ def __init__(self, modules_to_run, min_update_every, registry):
+ self.modules_to_run = modules_to_run
+ self.min_update_every = min_update_every
+ self.config = PluginConfig(PLUGIN_BASE_CONF)
+ self.log = PythonDLogger()
+ self.registry = registry
+ self.started_jobs = collections.defaultdict(dict)
+ self.jobs = list()
+ self.saver = None
+ self.runs = 0
+
+ def load_config_file(self, filepath, expected):
+ self.log.debug("looking for '{0}'".format(filepath))
+ if not os.path.isfile(filepath):
+ log = self.log.info if not expected else self.log.error
+ log("'{0}' was not found".format(filepath))
+ return dict()
+ try:
+ config = load_config(filepath)
+ except Exception as error:
+ self.log.error("error on loading '{0}' : {1}".format(filepath, repr(error)))
+ return dict()
+ self.log.debug("'{0}' is loaded".format(filepath))
+ return config
+
+ def load_config(self):
+ user_config = self.load_config_file(
+ filepath=os.path.join(DIRS.plugin_user_config, self.config_name),
+ expected=False,
+ )
+ stock_config = self.load_config_file(
+ filepath=os.path.join(DIRS.plugin_stock_config, self.config_name),
+ expected=True,
+ )
+ self.config.update(stock_config)
+ self.config.update(user_config)
+
+ def load_job_statuses(self):
+ self.log.debug("looking for '{0}' in {1}".format(self.jobs_status_dump_name, DIRS.var_lib))
+ abs_path = multi_path_find(self.jobs_status_dump_name, DIRS.var_lib)
+ if not abs_path:
+ self.log.warning("'{0}' was not found".format(self.jobs_status_dump_name))
+ return
+
+ self.log.debug("loading '{0}'".format(abs_path))
+ try:
+ statuses = JobsStatuses().from_file(abs_path)
+ except Exception as error:
+ self.log.error("[{0}] config file invalid YAML format: {1}".format(
+ module_name, ' '.join([v.strip() for v in str(error).split('\n')])))
+ return None
+ self.log.debug("'{0}' is loaded".format(abs_path))
+ return statuses
+
+ def create_jobs(self, job_statuses=None):
+ paths = [
+ DIRS.modules_user_config,
+ DIRS.modules_stock_config,
+ ]
+
+ builder = JobsConfigsBuilder(paths)
+ builder.job_defaults = JOB_BASE_CONF
+ builder.min_update_every = self.min_update_every
+
+ jobs = list()
+ for m in self.modules_to_run:
+ if not self.config.is_module_enabled(m.name):
+ self.log.info("[{0}] is disabled in the configuration file, skipping it".format(m.name))
+ continue
+
+ src = ModuleSrc(m)
+ try:
+ src.load()
+ except Exception as error:
+ self.log.warning("[{0}] error on loading source : {1}, skipping it".format(m.name, repr(error)))
+ continue
+ self.log.debug("[{0}] loaded module source : '{1}'".format(m.name, m.filepath))
+
+ if not (src.service() and callable(src.service())):
+ self.log.warning("[{0}] has no callable Service object, skipping it".format(m.name))
+ continue
+
+ if src.is_disabled_by_default() and not self.config.is_module_explicitly_enabled(m.name):
+ self.log.info("[{0}] is disabled by default, skipping it".format(m.name))
+ continue
+
+ builder.module_defaults = src.defaults()
+ configs = builder.build(m.name)
+ if not configs:
+ self.log.info("[{0}] has no job configs, skipping it".format(m.name))
+ continue
+
+ for config in configs:
+ config['job_name'] = re.sub(r'\s+', '_', config['job_name'])
+ config['override_name'] = re.sub(r'\s+', '_', config.pop('name'))
+
+ job = Job(src.service(), m.name, config)
+
+ was_previously_active = job_statuses and job_statuses.has(job.module_name, job.real_name)
+ if was_previously_active and job.autodetection_retry == 0:
+ self.log.debug('{0}[{1}] was previously active, applying recovering settings'.format(
+ job.module_name, job.real_name))
+ job.checks = 11
+ job.autodetection_retry = 30
+
+ jobs.append(job)
+
+ return jobs
+
+ def setup(self):
+ self.load_config()
+
+ if not self.config['enabled']:
+ self.log.info('disabled in the configuration file')
+ return False
+
+ statuses = self.load_job_statuses()
+
+ self.jobs = self.create_jobs(statuses)
+ if not self.jobs:
+ self.log.info('no jobs to run')
+ return False
+
+ if not IS_ATTY:
+ abs_path = os.path.join(DIRS.var_lib, self.jobs_status_dump_name)
+ self.saver = CachedFileSaver(abs_path)
+ return True
+
+ def start_jobs(self, *jobs):
+ for job in jobs:
+ if job.status not in (JOB_STATUS_INIT, JOB_STATUS_RECOVERING):
+ continue
+
+ if job.actual_name in self.started_jobs[job.module_name]:
+ self.log.info('{0}[{1}] : already served by another job, skipping it'.format(
+ job.module_name, job.real_name))
+ job.status = JOB_STATUS_DROPPED
+ continue
+
+ if not job.is_inited():
+ try:
+ job.init()
+ except Exception as error:
+ self.log.warning("{0}[{1}] : unhandled exception on init : {2}, skipping the job".format(
+ job.module_name, job.real_name, repr(error)))
+ job.status = JOB_STATUS_DROPPED
+ continue
+
+ try:
+ ok = job.check()
+ except Exception as error:
+ self.log.warning("{0}[{1}] : unhandled exception on check : {2}, skipping the job".format(
+ job.module_name, job.real_name, repr(error)))
+ job.status = JOB_STATUS_DROPPED
+ continue
+ if not ok:
+ self.log.info('{0}[{1}] : check failed'.format(job.module_name, job.real_name))
+ job.status = JOB_STATUS_RECOVERING if job.need_to_recheck() else JOB_STATUS_DROPPED
+ continue
+ self.log.info('{0}[{1}] : check success'.format(job.module_name, job.real_name))
+
+ try:
+ self.registry.register(job.full_name())
+ except filelock.Timeout as error:
+ self.log.info('{0}[{1}] : already registered by another process, skipping the job ({2})'.format(
+ job.module_name, job.real_name, error))
+ job.status = JOB_STATUS_DROPPED
+ continue
+ except Exception as error:
+ self.log.warning('{0}[{1}] : registration failed: {2}, skipping the job'.format(
+ job.module_name, job.real_name, error))
+ job.status = JOB_STATUS_DROPPED
+ continue
+
+ try:
+ job.create()
+ except Exception as error:
+ self.log.warning("{0}[{1}] : unhandled exception on create : {2}, skipping the job".format(
+ job.module_name, job.real_name, repr(error)))
+ job.status = JOB_STATUS_DROPPED
+ try:
+ self.registry.unregister(job.full_name())
+ except Exception as error:
+ self.log.warning('{0}[{1}] : deregistration failed: {2}'.format(
+ job.module_name, job.real_name, error))
+ continue
+
+ self.started_jobs[job.module_name] = job.actual_name
+ job.status = JOB_STATUS_ACTIVE
+ job.start()
+
+ @staticmethod
+ def keep_alive():
+ if not IS_ATTY:
+ safe_print('\n')
+
+ def garbage_collection(self):
+ if self.config['gc_run'] and self.runs % self.config['gc_interval'] == 0:
+ v = gc.collect()
+ self.log.debug('GC collection run result: {0}'.format(v))
+
+ def restart_recovering_jobs(self):
+ for job in self.jobs:
+ if job.status != JOB_STATUS_RECOVERING:
+ continue
+ if self.runs % job.autodetection_retry != 0:
+ continue
+ self.start_jobs(job)
+
+ def cleanup_jobs(self):
+ self.jobs = [j for j in self.jobs if j.status != JOB_STATUS_DROPPED]
+
+ def have_alive_jobs(self):
+ return next(
+ (True for job in self.jobs if job.status in (JOB_STATUS_RECOVERING, JOB_STATUS_ACTIVE)),
+ False,
+ )
+
+ def save_job_statuses(self):
+ if self.saver is None:
+ return
+ if self.runs % 10 != 0:
+ return
+ dump = JobsStatuses().from_jobs(self.jobs).dump()
+ try:
+ self.saver.save(dump)
+ except Exception as error:
+ self.log.error("error on saving jobs statuses dump : {0}".format(repr(error)))
+
+ def serve_once(self):
+ if not self.have_alive_jobs():
+ self.log.info('no jobs to serve')
+ return False
+
+ time.sleep(1)
+ self.runs += 1
+
+ self.keep_alive()
+ self.garbage_collection()
+ self.cleanup_jobs()
+ self.restart_recovering_jobs()
+ self.save_job_statuses()
+ return True
+
+ def serve(self):
+ while self.serve_once():
+ pass
+
+ def run(self):
+ self.start_jobs(*self.jobs)
+ self.serve()
+
+
+def parse_command_line():
+ opts = sys.argv[:][1:]
+
+ debug = False
+ trace = False
+ nolock = False
+ update_every = 1
+ modules_to_run = list()
+
+ def find_first_positive_int(values):
+ return next((v for v in values if v.isdigit() and int(v) >= 1), None)
+
+ u = find_first_positive_int(opts)
+ if u is not None:
+ update_every = int(u)
+ opts.remove(u)
+ if 'debug' in opts:
+ debug = True
+ opts.remove('debug')
+ if 'trace' in opts:
+ trace = True
+ opts.remove('trace')
+ if 'nolock' in opts:
+ nolock = True
+ opts.remove('nolock')
+ if opts:
+ modules_to_run = list(opts)
+
+ cmd = collections.namedtuple(
+ 'CMD',
+ [
+ 'update_every',
+ 'debug',
+ 'trace',
+ 'nolock',
+ 'modules_to_run',
+ ])
+ return cmd(
+ update_every,
+ debug,
+ trace,
+ nolock,
+ modules_to_run,
+ )
+
+
+def guess_module(modules, *names):
+ def guess(n):
+ found = None
+ for i, _ in enumerate(n):
+ cur = [x for x in modules if x.startswith(name[:i + 1])]
+ if not cur:
+ return found
+ found = cur
+ return found
+
+ guessed = list()
+ for name in names:
+ name = name.lower()
+ m = guess(name)
+ if m:
+ guessed.extend(m)
+ return sorted(set(guessed))
+
+
+def disable():
+ if not IS_ATTY:
+ safe_print('DISABLE')
+ exit(0)
+
+
+def get_modules_to_run(cmd):
+ if not cmd.modules_to_run:
+ return AVAILABLE_MODULES
+
+ modules_to_run, seen = list(), set()
+ for m in AVAILABLE_MODULES:
+ if m.name not in cmd.modules_to_run or m.name in seen:
+ continue
+ seen.add(m.name)
+ modules_to_run.append(m)
+
+ return modules_to_run
+
+
+def main():
+ cmd = parse_command_line()
+ log = PythonDLogger()
+
+ if cmd.debug:
+ log.logger.severity = 'DEBUG'
+ if cmd.trace:
+ log.log_traceback = True
+
+ log.info('using python v{0}'.format(PY_VERSION[0]))
+
+ if DIRS.locks and not cmd.nolock:
+ registry = FileLockRegistry(DIRS.locks)
+ else:
+ registry = DummyRegistry()
+
+ unique_avail_module_names = set([m.name for m in AVAILABLE_MODULES])
+ unknown = set(cmd.modules_to_run) - unique_avail_module_names
+ if unknown:
+ log.error('unknown modules : {0}'.format(sorted(list(unknown))))
+ guessed = guess_module(unique_avail_module_names, *cmd.modules_to_run)
+ if guessed:
+ log.info('probably you meant : \n{0}'.format(pprint.pformat(guessed, width=1)))
+ return
+
+ p = Plugin(
+ get_modules_to_run(cmd),
+ cmd.update_every,
+ registry,
+ )
+
+ # cheap attempt to reduce chance of python.d job running before go.d
+ # TODO: better implementation needed
+ if not IS_ATTY:
+ time.sleep(1.5)
+
+ try:
+ if not p.setup():
+ return
+ p.run()
+ except KeyboardInterrupt:
+ pass
+ log.info('exiting from main...')
+
+
+if __name__ == "__main__":
+ main()
+ disable()
diff --git a/collectors/python.d.plugin/python_modules/__init__.py b/collectors/python.d.plugin/python_modules/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/__init__.py
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py
new file mode 100644
index 0000000..a74b423
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Pawel Krupa (paulfantom)
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+
+from subprocess import Popen, PIPE
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from bases.collection import find_binary
+
+
+class ExecutableService(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.command = None
+
+ def _get_raw_data(self, stderr=False, command=None):
+ """
+ Get raw data from executed command
+ :return: <list>
+ """
+ command = command or self.command
+ self.debug("Executing command '{0}'".format(' '.join(command)))
+ try:
+ p = Popen(command, stdout=PIPE, stderr=PIPE)
+ except Exception as error:
+ self.error('Executing command {0} resulted in error: {1}'.format(command, error))
+ return None
+
+ data = list()
+ std = p.stderr if stderr else p.stdout
+ for line in std:
+ try:
+ data.append(line.decode('utf-8'))
+ except (TypeError, UnicodeDecodeError):
+ continue
+
+ return data
+
+ def check(self):
+ """
+ Parse basic configuration, check if command is whitelisted and is returning values
+ :return: <boolean>
+ """
+ # Preference: 1. "command" from configuration file 2. "command" from plugin (if specified)
+ if 'command' in self.configuration:
+ self.command = self.configuration['command']
+
+ # "command" must be: 1.not None 2. type <str>
+ if not (self.command and isinstance(self.command, str)):
+ self.error('Command is not defined or command type is not <str>')
+ return False
+
+ # Split "command" into: 1. command <str> 2. options <list>
+ command, opts = self.command.split()[0], self.command.split()[1:]
+
+ # Check for "bad" symbols in options. No pipes, redirects etc.
+ opts_list = ['&', '|', ';', '>', '<']
+ bad_opts = set(''.join(opts)) & set(opts_list)
+ if bad_opts:
+ self.error("Bad command argument(s): {opts}".format(opts=bad_opts))
+ return False
+
+ # Find absolute path ('echo' => '/bin/echo')
+ if '/' not in command:
+ command = find_binary(command)
+ if not command:
+ self.error('Can\'t locate "{command}" binary'.format(command=self.command))
+ return False
+ # Check if binary exist and executable
+ else:
+ if not os.access(command, os.X_OK):
+ self.error('"{binary}" is not executable'.format(binary=command))
+ return False
+
+ self.command = [command] + opts if opts else [command]
+
+ try:
+ data = self._get_data()
+ except Exception as error:
+ self.error('_get_data() failed. Command: {command}. Error: {error}'.format(command=self.command,
+ error=error))
+ return False
+
+ if isinstance(data, dict) and data:
+ return True
+ self.error('Command "{command}" returned no data'.format(command=self.command))
+ return False
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/LogService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/LogService.py
new file mode 100644
index 0000000..a55e33f
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/LogService.py
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Pawel Krupa (paulfantom)
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from glob import glob
+import sys
+import os
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+
+class LogService(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.log_path = self.configuration.get('path')
+ self.__glob_path = self.log_path
+ self._last_position = 0
+ self.__re_find = dict(current=0, run=0, maximum=60)
+ self.__open_args = {'errors': 'replace'} if sys.version_info[0] > 2 else {}
+
+ def _get_raw_data(self):
+ """
+ Get log lines since last poll
+ :return: list
+ """
+ lines = list()
+ try:
+ if self.__re_find['current'] == self.__re_find['run']:
+ self._find_recent_log_file()
+ size = os.path.getsize(self.log_path)
+ if size == self._last_position:
+ self.__re_find['current'] += 1
+ return list() # return empty list if nothing has changed
+ elif size < self._last_position:
+ self._last_position = 0 # read from beginning if file has shrunk
+
+ with open(self.log_path, **self.__open_args) as fp:
+ fp.seek(self._last_position)
+ for line in fp:
+ lines.append(line)
+ self._last_position = fp.tell()
+ self.__re_find['current'] = 0
+ except (OSError, IOError) as error:
+ self.__re_find['current'] += 1
+ self.error(str(error))
+
+ return lines or None
+
+ def _find_recent_log_file(self):
+ """
+ :return:
+ """
+ self.__re_find['run'] = self.__re_find['maximum']
+ self.__re_find['current'] = 0
+ self.__glob_path = self.__glob_path or self.log_path # workaround for modules w/o config files
+ path_list = glob(self.__glob_path)
+ if path_list:
+ self.log_path = max(path_list)
+ return True
+ return False
+
+ def check(self):
+ """
+ Parse basic configuration and check if log file exists
+ :return: boolean
+ """
+ if not self.log_path:
+ self.error('No path to log specified')
+ return None
+
+ if self._find_recent_log_file() and os.access(self.log_path, os.R_OK) and os.path.isfile(self.log_path):
+ return True
+ self.error('Cannot access {0}'.format(self.log_path))
+ return False
+
+ def create(self):
+ # set cursor at last byte of log file
+ self._last_position = os.path.getsize(self.log_path)
+ status = SimpleService.create(self)
+ return status
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py
new file mode 100644
index 0000000..7f5c7d2
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py
@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from sys import exc_info
+
+try:
+ import MySQLdb
+
+ PY_MYSQL = True
+except ImportError:
+ try:
+ import pymysql as MySQLdb
+
+ PY_MYSQL = True
+ except ImportError:
+ PY_MYSQL = False
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+
+class MySQLService(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.__connection = None
+ self.__conn_properties = dict()
+ self.extra_conn_properties = dict()
+ self.__queries = self.configuration.get('queries', dict())
+ self.queries = dict()
+
+ def __connect(self):
+ try:
+ connection = MySQLdb.connect(connect_timeout=self.update_every, **self.__conn_properties)
+ except (MySQLdb.MySQLError, TypeError, AttributeError) as error:
+ return None, str(error)
+ else:
+ return connection, None
+
+ def check(self):
+ def get_connection_properties(conf, extra_conf):
+ properties = dict()
+ if conf.get('user'):
+ properties['user'] = conf['user']
+ if conf.get('pass'):
+ properties['passwd'] = conf['pass']
+
+ if conf.get('socket'):
+ properties['unix_socket'] = conf['socket']
+ elif conf.get('host'):
+ properties['host'] = conf['host']
+ properties['port'] = int(conf.get('port', 3306))
+ elif conf.get('my.cnf'):
+ properties['read_default_file'] = conf['my.cnf']
+
+ if conf.get('ssl'):
+ properties['ssl'] = conf['ssl']
+
+ if isinstance(extra_conf, dict) and extra_conf:
+ properties.update(extra_conf)
+
+ return properties or None
+
+ def is_valid_queries_dict(raw_queries, log_error):
+ """
+ :param raw_queries: dict:
+ :param log_error: function:
+ :return: dict or None
+
+ raw_queries is valid when: type <dict> and not empty after is_valid_query(for all queries)
+ """
+
+ def is_valid_query(query):
+ return all([isinstance(query, str),
+ query.startswith(('SELECT', 'select', 'SHOW', 'show'))])
+
+ if hasattr(raw_queries, 'keys') and raw_queries:
+ valid_queries = dict([(n, q) for n, q in raw_queries.items() if is_valid_query(q)])
+ bad_queries = set(raw_queries) - set(valid_queries)
+
+ if bad_queries:
+ log_error('Removed query(s): {queries}'.format(queries=bad_queries))
+ return valid_queries
+ else:
+ log_error('Unsupported "queries" format. Must be not empty <dict>')
+ return None
+
+ if not PY_MYSQL:
+ self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin')
+ return False
+
+ # Preference: 1. "queries" from the configuration file 2. "queries" from the module
+ self.queries = self.__queries or self.queries
+ # Check if "self.queries" exist, not empty and all queries are in valid format
+ self.queries = is_valid_queries_dict(self.queries, self.error)
+ if not self.queries:
+ return None
+
+ # Get connection properties
+ self.__conn_properties = get_connection_properties(self.configuration, self.extra_conn_properties)
+ if not self.__conn_properties:
+ self.error('Connection properties are missing')
+ return False
+
+ # Create connection to the database
+ self.__connection, error = self.__connect()
+ if error:
+ self.error('Can\'t establish connection to MySQL: {error}'.format(error=error))
+ return False
+
+ try:
+ data = self._get_data()
+ except Exception as error:
+ self.error('_get_data() failed. Error: {error}'.format(error=error))
+ return False
+
+ if isinstance(data, dict) and data:
+ return True
+ self.error("_get_data() returned no data or type is not <dict>")
+ return False
+
+ def _get_raw_data(self, description=None):
+ """
+ Get raw data from MySQL server
+ :return: dict: fetchall() or (fetchall(), description)
+ """
+
+ if not self.__connection:
+ self.__connection, error = self.__connect()
+ if error:
+ return None
+
+ raw_data = dict()
+ queries = dict(self.queries)
+ try:
+ cursor = self.__connection.cursor()
+ for name, query in queries.items():
+ try:
+ cursor.execute(query)
+ except (MySQLdb.ProgrammingError, MySQLdb.OperationalError) as error:
+ if self.__is_error_critical(err_class=exc_info()[0], err_text=str(error)):
+ cursor.close()
+ raise RuntimeError
+ self.error('Removed query: {name}[{query}]. Error: error'.format(name=name,
+ query=query,
+ error=error))
+ self.queries.pop(name)
+ continue
+ else:
+ raw_data[name] = (cursor.fetchall(), cursor.description) if description else cursor.fetchall()
+ cursor.close()
+ self.__connection.commit()
+ except (MySQLdb.MySQLError, RuntimeError, TypeError, AttributeError):
+ self.__connection.close()
+ self.__connection = None
+ return None
+ else:
+ return raw_data or None
+
+ @staticmethod
+ def __is_error_critical(err_class, err_text):
+ return err_class == MySQLdb.OperationalError and all(['denied' not in err_text,
+ 'Unknown column' not in err_text])
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py
new file mode 100644
index 0000000..a7acc23
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py
@@ -0,0 +1,261 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Pawel Krupa (paulfantom)
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+
+from bases.charts import Charts, ChartError, create_runtime_chart
+from bases.collection import safe_print
+from bases.loggers import PythonDLimitedLogger
+from third_party.monotonic import monotonic
+from time import sleep, time
+
+RUNTIME_CHART_UPDATE = 'BEGIN netdata.runtime_{job_name} {since_last}\n' \
+ 'SET run_time = {elapsed}\n' \
+ 'END\n'
+
+PENALTY_EVERY = 5
+MAX_PENALTY = 10 * 60 # 10 minutes
+
+ND_INTERNAL_MONITORING_DISABLED = os.getenv("NETDATA_INTERNALS_MONITORING") == "NO"
+
+
+class RuntimeCounters:
+ def __init__(self, configuration):
+ """
+ :param configuration: <dict>
+ """
+ self.update_every = int(configuration.pop('update_every'))
+ self.do_penalty = configuration.pop('penalty')
+
+ self.start_mono = 0
+ self.start_real = 0
+ self.retries = 0
+ self.penalty = 0
+ self.elapsed = 0
+ self.prev_update = 0
+
+ self.runs = 1
+
+ def calc_next(self):
+ self.start_mono = monotonic()
+ return self.start_mono - (self.start_mono % self.update_every) + self.update_every + self.penalty
+
+ def sleep_until_next(self):
+ next_time = self.calc_next()
+ while self.start_mono < next_time:
+ sleep(next_time - self.start_mono)
+ self.start_mono = monotonic()
+ self.start_real = time()
+
+ def handle_retries(self):
+ self.retries += 1
+ if self.do_penalty and self.retries % PENALTY_EVERY == 0:
+ self.penalty = round(min(self.retries * self.update_every / 2, MAX_PENALTY))
+
+
+def clean_module_name(name):
+ if name.startswith('pythond_'):
+ return name[8:]
+ return name
+
+
+class SimpleService(PythonDLimitedLogger, object):
+ """
+ Prototype of Service class.
+ Implemented basic functionality to run jobs by `python.d.plugin`
+ """
+
+ def __init__(self, configuration, name=''):
+ """
+ :param configuration: <dict>
+ :param name: <str>
+ """
+ PythonDLimitedLogger.__init__(self)
+ self.configuration = configuration
+ self.order = list()
+ self.definitions = dict()
+
+ self.module_name = clean_module_name(self.__module__)
+ self.job_name = configuration.pop('job_name')
+ self.actual_job_name = self.job_name or self.module_name
+ self.override_name = configuration.pop('override_name')
+ self.fake_name = None
+
+ self._runtime_counters = RuntimeCounters(configuration=configuration)
+ self.charts = Charts(job_name=self.actual_name,
+ actual_job_name=self.actual_job_name,
+ priority=configuration.pop('priority'),
+ cleanup=configuration.pop('chart_cleanup'),
+ get_update_every=self.get_update_every,
+ module_name=self.module_name)
+
+ def __repr__(self):
+ return '<{cls_bases}: {name}>'.format(cls_bases=', '.join(c.__name__ for c in self.__class__.__bases__),
+ name=self.name)
+
+ @property
+ def name(self):
+ name = self.override_name or self.job_name
+ if name and name != self.module_name:
+ return '_'.join([self.module_name, name])
+ return self.module_name
+
+ def actual_name(self):
+ return self.fake_name or self.name
+
+ @property
+ def runs_counter(self):
+ return self._runtime_counters.runs
+
+ @property
+ def update_every(self):
+ return self._runtime_counters.update_every
+
+ @update_every.setter
+ def update_every(self, value):
+ """
+ :param value: <int>
+ :return:
+ """
+ self._runtime_counters.update_every = value
+
+ def get_update_every(self):
+ return self.update_every
+
+ def check(self):
+ """
+ check() prototype
+ :return: boolean
+ """
+ self.debug("job doesn't implement check() method. Using default which simply invokes get_data().")
+ data = self.get_data()
+ if data and isinstance(data, dict):
+ return True
+ self.debug('returned value is wrong: {0}'.format(data))
+ return False
+
+ @create_runtime_chart
+ def create(self):
+ for chart_name in self.order:
+ chart_config = self.definitions.get(chart_name)
+
+ if not chart_config:
+ self.debug("create() => [NOT ADDED] chart '{chart_name}' not in definitions. "
+ "Skipping it.".format(chart_name=chart_name))
+ continue
+
+ # create chart
+ chart_params = [chart_name] + chart_config['options']
+ try:
+ self.charts.add_chart(params=chart_params)
+ except ChartError as error:
+ self.error("create() => [NOT ADDED] (chart '{chart}': {error})".format(chart=chart_name,
+ error=error))
+ continue
+
+ # add dimensions to chart
+ for dimension in chart_config['lines']:
+ try:
+ self.charts[chart_name].add_dimension(dimension)
+ except ChartError as error:
+ self.error("create() => [NOT ADDED] (dimension '{dimension}': {error})".format(dimension=dimension,
+ error=error))
+ continue
+
+ # add variables to chart
+ if 'variables' in chart_config:
+ for variable in chart_config['variables']:
+ try:
+ self.charts[chart_name].add_variable(variable)
+ except ChartError as error:
+ self.error("create() => [NOT ADDED] (variable '{var}': {error})".format(var=variable,
+ error=error))
+ continue
+
+ del self.order
+ del self.definitions
+
+ # True if job has at least 1 chart else False
+ return bool(self.charts)
+
+ def run(self):
+ """
+ Runs job in thread. Handles retries.
+ Exits when job failed or timed out.
+ :return: None
+ """
+ job = self._runtime_counters
+ self.debug('started, update frequency: {freq}'.format(freq=job.update_every))
+
+ while True:
+ job.sleep_until_next()
+
+ since = 0
+ if job.prev_update:
+ since = int((job.start_real - job.prev_update) * 1e6)
+
+ try:
+ updated = self.update(interval=since)
+ except Exception as error:
+ self.error('update() unhandled exception: {error}'.format(error=error))
+ updated = False
+
+ job.runs += 1
+
+ if not updated:
+ job.handle_retries()
+ else:
+ job.elapsed = int((monotonic() - job.start_mono) * 1e3)
+ job.prev_update = job.start_real
+ job.retries, job.penalty = 0, 0
+ if not ND_INTERNAL_MONITORING_DISABLED:
+ safe_print(RUNTIME_CHART_UPDATE.format(job_name=self.name,
+ since_last=since,
+ elapsed=job.elapsed))
+ self.debug('update => [{status}] (elapsed time: {elapsed}, failed retries in a row: {retries})'.format(
+ status='OK' if updated else 'FAILED',
+ elapsed=job.elapsed if updated else '-',
+ retries=job.retries))
+
+ def update(self, interval):
+ """
+ :return:
+ """
+ data = self.get_data()
+ if not data:
+ self.debug('get_data() returned no data')
+ return False
+ elif not isinstance(data, dict):
+ self.debug('get_data() returned incorrect type data')
+ return False
+
+ updated = False
+
+ for chart in self.charts:
+ if chart.flags.obsoleted:
+ if chart.can_be_updated(data):
+ chart.refresh()
+ else:
+ continue
+ elif self.charts.cleanup and chart.penalty >= self.charts.cleanup:
+ chart.obsolete()
+ self.info("chart '{0}' was suppressed due to non updating".format(chart.name))
+ continue
+
+ ok = chart.update(data, interval)
+ if ok:
+ updated = True
+
+ if not updated:
+ self.debug('none of the charts has been updated')
+
+ return updated
+
+ def get_data(self):
+ return self._get_data()
+
+ def _get_data(self):
+ raise NotImplementedError
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py
new file mode 100644
index 0000000..d6c7550
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py
@@ -0,0 +1,336 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Pawel Krupa (paulfantom)
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import errno
+import socket
+
+try:
+ import ssl
+except ImportError:
+ _TLS_SUPPORT = False
+else:
+ _TLS_SUPPORT = True
+
+if _TLS_SUPPORT:
+ try:
+ PROTOCOL_TLS = ssl.PROTOCOL_TLS
+ except AttributeError:
+ PROTOCOL_TLS = ssl.PROTOCOL_SSLv23
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+
+DEFAULT_CONNECT_TIMEOUT = 2.0
+DEFAULT_READ_TIMEOUT = 2.0
+DEFAULT_WRITE_TIMEOUT = 2.0
+
+
+class SocketService(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ self._sock = None
+ self._keep_alive = False
+ self.host = 'localhost'
+ self.port = None
+ self.unix_socket = None
+ self.dgram_socket = False
+ self.request = ''
+ self.tls = False
+ self.cert = None
+ self.key = None
+ self.__socket_config = None
+ self.__empty_request = "".encode()
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.connect_timeout = configuration.get('connect_timeout', DEFAULT_CONNECT_TIMEOUT)
+ self.read_timeout = configuration.get('read_timeout', DEFAULT_READ_TIMEOUT)
+ self.write_timeout = configuration.get('write_timeout', DEFAULT_WRITE_TIMEOUT)
+
+ def _socket_error(self, message=None):
+ if self.unix_socket is not None:
+ self.error('unix socket "{socket}": {message}'.format(socket=self.unix_socket,
+ message=message))
+ else:
+ if self.__socket_config is not None:
+ _, _, _, _, sa = self.__socket_config
+ self.error('socket to "{address}" port {port}: {message}'.format(address=sa[0],
+ port=sa[1],
+ message=message))
+ else:
+ self.error('unknown socket: {0}'.format(message))
+
+ def _connect2socket(self, res=None):
+ """
+ Connect to a socket, passing the result of getaddrinfo()
+ :return: boolean
+ """
+ if res is None:
+ res = self.__socket_config
+ if res is None:
+ self.error("Cannot create socket to 'None':")
+ return False
+
+ af, sock_type, proto, _, sa = res
+ try:
+ self.debug('Creating socket to "{address}", port {port}'.format(address=sa[0], port=sa[1]))
+ self._sock = socket.socket(af, sock_type, proto)
+ except socket.error as error:
+ self.error('Failed to create socket "{address}", port {port}, error: {error}'.format(address=sa[0],
+ port=sa[1],
+ error=error))
+ self._sock = None
+ self.__socket_config = None
+ return False
+
+ if self.tls:
+ try:
+ self.debug('Encapsulating socket with TLS')
+ self.debug('Using keyfile: {0}, certfile: {1}, cert_reqs: {2}, ssl_version: {3}'.format(
+ self.key, self.cert, ssl.CERT_NONE, PROTOCOL_TLS
+ ))
+ self._sock = ssl.wrap_socket(self._sock,
+ keyfile=self.key,
+ certfile=self.cert,
+ server_side=False,
+ cert_reqs=ssl.CERT_NONE,
+ ssl_version=PROTOCOL_TLS,
+ )
+ except (socket.error, ssl.SSLError, IOError, OSError) as error:
+ self.error('failed to wrap socket : {0}'.format(repr(error)))
+ self._disconnect()
+ self.__socket_config = None
+ return False
+
+ try:
+ self.debug('connecting socket to "{address}", port {port}'.format(address=sa[0], port=sa[1]))
+ self._sock.settimeout(self.connect_timeout)
+ self.debug('set socket connect timeout to: {0}'.format(self._sock.gettimeout()))
+ self._sock.connect(sa)
+ except (socket.error, ssl.SSLError) as error:
+ self.error('Failed to connect to "{address}", port {port}, error: {error}'.format(address=sa[0],
+ port=sa[1],
+ error=error))
+ self._disconnect()
+ self.__socket_config = None
+ return False
+
+ self.debug('connected to "{address}", port {port}'.format(address=sa[0], port=sa[1]))
+ self.__socket_config = res
+ return True
+
+ def _connect2unixsocket(self):
+ """
+ Connect to a unix socket, given its filename
+ :return: boolean
+ """
+ if self.unix_socket is None:
+ self.error("cannot connect to unix socket 'None'")
+ return False
+
+ try:
+ self.debug('attempting DGRAM unix socket "{0}"'.format(self.unix_socket))
+ self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+ self._sock.settimeout(self.connect_timeout)
+ self.debug('set socket connect timeout to: {0}'.format(self._sock.gettimeout()))
+ self._sock.connect(self.unix_socket)
+ self.debug('connected DGRAM unix socket "{0}"'.format(self.unix_socket))
+ return True
+ except socket.error as error:
+ self.debug('Failed to connect DGRAM unix socket "{socket}": {error}'.format(socket=self.unix_socket,
+ error=error))
+
+ try:
+ self.debug('attempting STREAM unix socket "{0}"'.format(self.unix_socket))
+ self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ self._sock.settimeout(self.connect_timeout)
+ self.debug('set socket connect timeout to: {0}'.format(self._sock.gettimeout()))
+ self._sock.connect(self.unix_socket)
+ self.debug('connected STREAM unix socket "{0}"'.format(self.unix_socket))
+ return True
+ except socket.error as error:
+ self.debug('Failed to connect STREAM unix socket "{socket}": {error}'.format(socket=self.unix_socket,
+ error=error))
+ self._sock = None
+ return False
+
+ def _connect(self):
+ """
+ Recreate socket and connect to it since sockets cannot be reused after closing
+ Available configurations are IPv6, IPv4 or UNIX socket
+ :return:
+ """
+ try:
+ if self.unix_socket is not None:
+ self._connect2unixsocket()
+
+ else:
+ if self.__socket_config is not None:
+ self._connect2socket()
+ else:
+ if self.dgram_socket:
+ sock_type = socket.SOCK_DGRAM
+ else:
+ sock_type = socket.SOCK_STREAM
+ for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, sock_type):
+ if self._connect2socket(res):
+ break
+
+ except Exception as error:
+ self.error('unhandled exception during connect : {0}'.format(repr(error)))
+ self._sock = None
+ self.__socket_config = None
+
+ def _disconnect(self):
+ """
+ Close socket connection
+ :return:
+ """
+ if self._sock is not None:
+ try:
+ self.debug('closing socket')
+ self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all
+ self._sock.close()
+ except Exception as error:
+ if not (hasattr(error, 'errno') and error.errno == errno.ENOTCONN):
+ self.error(error)
+ self._sock = None
+
+ def _send(self, request=None):
+ """
+ Send request.
+ :return: boolean
+ """
+ # Send request if it is needed
+ if self.request != self.__empty_request:
+ try:
+ self.debug('set socket write timeout to: {0}'.format(self._sock.gettimeout()))
+ self._sock.settimeout(self.write_timeout)
+ self.debug('sending request: {0}'.format(request or self.request))
+ self._sock.send(request or self.request)
+ except Exception as error:
+ self._socket_error('error sending request: {0}'.format(error))
+ self._disconnect()
+ return False
+ return True
+
+ def _receive(self, raw=False):
+ """
+ Receive data from socket
+ :param raw: set `True` to return bytes
+ :type raw: bool
+ :return: decoded str or raw bytes
+ :rtype: str/bytes
+ """
+ data = "" if not raw else b""
+ while True:
+ self.debug('receiving response')
+ try:
+ self.debug('set socket read timeout to: {0}'.format(self._sock.gettimeout()))
+ self._sock.settimeout(self.read_timeout)
+ buf = self._sock.recv(4096)
+ except Exception as error:
+ self._socket_error('failed to receive response: {0}'.format(error))
+ self._disconnect()
+ break
+
+ if buf is None or len(buf) == 0: # handle server disconnect
+ if data == "" or data == b"":
+ self._socket_error('unexpectedly disconnected')
+ else:
+ self.debug('server closed the connection')
+ self._disconnect()
+ break
+
+ self.debug('received data')
+ data += buf.decode('utf-8', 'ignore') if not raw else buf
+ if self._check_raw_data(data):
+ break
+
+ self.debug(u'final response: {0}'.format(data if not raw else u'binary data'))
+ return data
+
+ def _get_raw_data(self, raw=False, request=None):
+ """
+ Get raw data with low-level "socket" module.
+ :param raw: set `True` to return bytes
+ :type raw: bool
+ :return: decoded data (str) or raw data (bytes)
+ :rtype: str/bytes
+ """
+ if self._sock is None:
+ self._connect()
+ if self._sock is None:
+ return None
+
+ # Send request if it is needed
+ if not self._send(request):
+ return None
+
+ data = self._receive(raw)
+
+ if not self._keep_alive:
+ self._disconnect()
+
+ return data
+
+ @staticmethod
+ def _check_raw_data(data):
+ """
+ Check if all data has been gathered from socket
+ :param data: str
+ :return: boolean
+ """
+ return bool(data)
+
+ def _parse_config(self):
+ """
+ Parse configuration data
+ :return: boolean
+ """
+ try:
+ self.unix_socket = str(self.configuration['socket'])
+ except (KeyError, TypeError):
+ self.debug('No unix socket specified. Trying TCP/IP socket.')
+ self.unix_socket = None
+ try:
+ self.host = str(self.configuration['host'])
+ except (KeyError, TypeError):
+ self.debug('No host specified. Using: "{0}"'.format(self.host))
+ try:
+ self.port = int(self.configuration['port'])
+ except (KeyError, TypeError):
+ self.debug('No port specified. Using: "{0}"'.format(self.port))
+
+ self.tls = bool(self.configuration.get('tls', self.tls))
+ if self.tls and not _TLS_SUPPORT:
+ self.warning('TLS requested but no TLS module found, disabling TLS support.')
+ self.tls = False
+ if _TLS_SUPPORT and not self.tls:
+ self.debug('No TLS preference specified, not using TLS.')
+
+ if self.tls and _TLS_SUPPORT:
+ self.key = self.configuration.get('tls_key_file')
+ self.cert = self.configuration.get('tls_cert_file')
+ if not self.cert:
+ # If there's not a valid certificate, clear the key too.
+ self.debug('No valid TLS client certificate configuration found.')
+ self.key = None
+ self.cert = None
+ elif not self.key:
+ # If a key isn't listed, the config may still be
+ # valid, because there may be a key attached to the
+ # certificate.
+ self.info('No TLS client key specified, assuming it\'s attached to the certificate.')
+ self.key = None
+
+ try:
+ self.request = str(self.configuration['request'])
+ except (KeyError, TypeError):
+ self.debug('No request specified. Using: "{0}"'.format(self.request))
+
+ self.request = self.request.encode()
+
+ def check(self):
+ self._parse_config()
+ return SimpleService.check(self)
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py
new file mode 100644
index 0000000..1faf036
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py
@@ -0,0 +1,207 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Pawel Krupa (paulfantom)
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import urllib3
+
+from distutils.version import StrictVersion as version
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+try:
+ urllib3.disable_warnings()
+except AttributeError:
+ pass
+
+# https://github.com/urllib3/urllib3/blob/master/CHANGES.rst#19-2014-07-04
+# New retry logic and urllib3.util.retry.Retry configuration object. (Issue https://github.com/urllib3/urllib3/pull/326)
+URLLIB3_MIN_REQUIRED_VERSION = '1.9'
+URLLIB3_VERSION = urllib3.__version__
+URLLIB3 = 'urllib3'
+
+
+def version_check():
+ if version(URLLIB3_VERSION) >= version(URLLIB3_MIN_REQUIRED_VERSION):
+ return
+
+ err = '{0} version: {1}, minimum required version: {2}, please upgrade'.format(
+ URLLIB3,
+ URLLIB3_VERSION,
+ URLLIB3_MIN_REQUIRED_VERSION,
+ )
+ raise Exception(err)
+
+
+class UrlService(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ version_check()
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.debug("{0} version: {1}".format(URLLIB3, URLLIB3_VERSION))
+ self.url = self.configuration.get('url')
+ self.user = self.configuration.get('user')
+ self.password = self.configuration.get('pass')
+ self.proxy_user = self.configuration.get('proxy_user')
+ self.proxy_password = self.configuration.get('proxy_pass')
+ self.proxy_url = self.configuration.get('proxy_url')
+ self.method = self.configuration.get('method', 'GET')
+ self.header = self.configuration.get('header')
+ self.body = self.configuration.get('body')
+ self.request_timeout = self.configuration.get('timeout', 1)
+ self.respect_retry_after_header = self.configuration.get('respect_retry_after_header')
+ self.tls_verify = self.configuration.get('tls_verify')
+ self.tls_ca_file = self.configuration.get('tls_ca_file')
+ self.tls_key_file = self.configuration.get('tls_key_file')
+ self.tls_cert_file = self.configuration.get('tls_cert_file')
+ self._manager = None
+
+ def __make_headers(self, **header_kw):
+ user = header_kw.get('user') or self.user
+ password = header_kw.get('pass') or self.password
+ proxy_user = header_kw.get('proxy_user') or self.proxy_user
+ proxy_password = header_kw.get('proxy_pass') or self.proxy_password
+ custom_header = header_kw.get('header') or self.header
+ header_params = dict(keep_alive=True)
+ proxy_header_params = dict()
+ if user and password:
+ header_params['basic_auth'] = '{user}:{password}'.format(user=user,
+ password=password)
+ if proxy_user and proxy_password:
+ proxy_header_params['proxy_basic_auth'] = '{user}:{password}'.format(user=proxy_user,
+ password=proxy_password)
+ try:
+ header, proxy_header = urllib3.make_headers(**header_params), urllib3.make_headers(**proxy_header_params)
+ except TypeError as error:
+ self.error('build_header() error: {error}'.format(error=error))
+ return None, None
+ else:
+ header.update(custom_header or dict())
+ return header, proxy_header
+
+ def _build_manager(self, **header_kw):
+ header, proxy_header = self.__make_headers(**header_kw)
+ if header is None or proxy_header is None:
+ return None
+ proxy_url = header_kw.get('proxy_url') or self.proxy_url
+ if proxy_url:
+ manager = urllib3.ProxyManager
+ params = dict(proxy_url=proxy_url, headers=header, proxy_headers=proxy_header)
+ else:
+ manager = urllib3.PoolManager
+ params = dict(headers=header)
+ tls_cert_file = self.tls_cert_file
+ if tls_cert_file:
+ params['cert_file'] = tls_cert_file
+ # NOTE: key_file is useless without cert_file, but
+ # cert_file may include the key as well.
+ tls_key_file = self.tls_key_file
+ if tls_key_file:
+ params['key_file'] = tls_key_file
+ tls_ca_file = self.tls_ca_file
+ if tls_ca_file:
+ params['ca_certs'] = tls_ca_file
+ try:
+ url = header_kw.get('url') or self.url
+ is_https = url.startswith('https')
+ if skip_tls_verify(is_https, self.tls_verify, tls_ca_file):
+ params['ca_certs'] = None
+ params['cert_reqs'] = 'CERT_NONE'
+ if is_https:
+ params['assert_hostname'] = False
+ return manager(**params)
+ except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error:
+ self.error('build_manager() error:', str(error))
+ return None
+
+ def _get_raw_data(self, url=None, manager=None, **kwargs):
+ """
+ Get raw data from http request
+ :return: str
+ """
+ try:
+ response = self._do_request(url, manager, **kwargs)
+ except Exception as error:
+ self.error('Url: {url}. Error: {error}'.format(url=url or self.url, error=error))
+ return None
+
+ if response.status == 200:
+ if isinstance(response.data, str):
+ return response.data
+ return response.data.decode(errors='ignore')
+ else:
+ self.debug('Url: {url}. Http response status code: {code}'.format(url=url or self.url, code=response.status))
+ return None
+
+ def _get_raw_data_with_status(self, url=None, manager=None, retries=1, redirect=True, **kwargs):
+ """
+ Get status and response body content from http request. Does not catch exceptions
+ :return: int, str
+ """
+ response = self._do_request(url, manager, retries, redirect, **kwargs)
+
+ if isinstance(response.data, str):
+ return response.status, response.data
+ return response.status, response.data.decode(errors='ignore')
+
+ def _do_request(self, url=None, manager=None, retries=1, redirect=True, **kwargs):
+ """
+ Get response from http request. Does not catch exceptions
+ :return: HTTPResponse
+ """
+ url = url or self.url
+ manager = manager or self._manager
+ retry = urllib3.Retry(retries)
+ if hasattr(retry, 'respect_retry_after_header'):
+ retry.respect_retry_after_header = bool(self.respect_retry_after_header)
+
+ if self.body:
+ kwargs['body'] = self.body
+
+ response = manager.request(
+ method=self.method,
+ url=url,
+ timeout=self.request_timeout,
+ retries=retry,
+ headers=manager.headers,
+ redirect=redirect,
+ **kwargs
+ )
+ return response
+
+ def check(self):
+ """
+ Format configuration data and try to connect to server
+ :return: boolean
+ """
+ if not (self.url and isinstance(self.url, str)):
+ self.error('URL is not defined or type is not <str>')
+ return False
+
+ self._manager = self._build_manager()
+ if not self._manager:
+ return False
+
+ try:
+ data = self._get_data()
+ except Exception as error:
+ self.error('_get_data() failed. Url: {url}. Error: {error}'.format(url=self.url, error=error))
+ return False
+
+ if isinstance(data, dict) and data:
+ return True
+ self.error('_get_data() returned no data or type is not <dict>')
+ return False
+
+
+def skip_tls_verify(is_https, tls_verify, tls_ca_file):
+ # default 'tls_verify' value is None
+ # logic is:
+ # - never skip if there is 'tls_ca_file' file
+ # - skip by default for https
+ # - do not skip by default for http
+ if tls_ca_file:
+ return False
+ if is_https and not tls_verify:
+ return True
+ return tls_verify is False
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/__init__.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/__init__.py
diff --git a/collectors/python.d.plugin/python_modules/bases/__init__.py b/collectors/python.d.plugin/python_modules/bases/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/__init__.py
diff --git a/collectors/python.d.plugin/python_modules/bases/charts.py b/collectors/python.d.plugin/python_modules/bases/charts.py
new file mode 100644
index 0000000..203ad16
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/charts.py
@@ -0,0 +1,431 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+
+from bases.collection import safe_print
+
+CHART_PARAMS = ['type', 'id', 'name', 'title', 'units', 'family', 'context', 'chart_type', 'hidden']
+DIMENSION_PARAMS = ['id', 'name', 'algorithm', 'multiplier', 'divisor', 'hidden']
+VARIABLE_PARAMS = ['id', 'value']
+
+CHART_TYPES = ['line', 'area', 'stacked']
+DIMENSION_ALGORITHMS = ['absolute', 'incremental', 'percentage-of-absolute-row', 'percentage-of-incremental-row']
+
+CHART_BEGIN = 'BEGIN {type}.{id} {since_last}\n'
+CHART_CREATE = "CHART {type}.{id} '{name}' '{title}' '{units}' '{family}' '{context}' " \
+ "{chart_type} {priority} {update_every} '{hidden}' 'python.d.plugin' '{module_name}'\n"
+CHART_OBSOLETE = "CHART {type}.{id} '{name}' '{title}' '{units}' '{family}' '{context}' " \
+ "{chart_type} {priority} {update_every} '{hidden} obsolete'\n"
+
+CLABEL_COLLECT_JOB = "CLABEL '_collect_job' '{actual_job_name}' '0'\n"
+CLABEL_COMMIT = "CLABEL_COMMIT\n"
+
+DIMENSION_CREATE = "DIMENSION '{id}' '{name}' {algorithm} {multiplier} {divisor} '{hidden} {obsolete}'\n"
+DIMENSION_SET = "SET '{id}' = {value}\n"
+
+CHART_VARIABLE_SET = "VARIABLE CHART '{id}' = {value}\n"
+
+# 1 is label source auto
+# https://github.com/netdata/netdata/blob/cc2586de697702f86a3c34e60e23652dd4ddcb42/database/rrd.h#L205
+RUNTIME_CHART_CREATE = "CHART netdata.runtime_{job_name} '' 'Execution time' 'ms' 'python.d' " \
+ "netdata.pythond_runtime line 145000 {update_every} '' 'python.d.plugin' '{module_name}'\n" \
+ "CLABEL '_collect_job' '{actual_job_name}' '1'\n" \
+ "CLABEL_COMMIT\n" \
+ "DIMENSION run_time 'run time' absolute 1 1\n"
+
+ND_INTERNAL_MONITORING_DISABLED = os.getenv("NETDATA_INTERNALS_MONITORING") == "NO"
+
+
+def create_runtime_chart(func):
+ """
+ Calls a wrapped function, then prints runtime chart to stdout.
+
+ Used as a decorator for SimpleService.create() method.
+ The whole point of making 'create runtime chart' functionality as a decorator was
+ to help users who re-implements create() in theirs classes.
+
+ :param func: class method
+ :return:
+ """
+
+ def wrapper(*args, **kwargs):
+ self = args[0]
+ if not ND_INTERNAL_MONITORING_DISABLED:
+ chart = RUNTIME_CHART_CREATE.format(
+ job_name=self.name,
+ actual_job_name=self.actual_job_name,
+ update_every=self._runtime_counters.update_every,
+ module_name=self.module_name,
+ )
+ safe_print(chart)
+ ok = func(*args, **kwargs)
+ return ok
+
+ return wrapper
+
+
+class ChartError(Exception):
+ """Base-class for all exceptions raised by this module"""
+
+
+class DuplicateItemError(ChartError):
+ """Occurs when user re-adds a chart or a dimension that has already been added"""
+
+
+class ItemTypeError(ChartError):
+ """Occurs when user passes value of wrong type to Chart, Dimension or ChartVariable class"""
+
+
+class ItemValueError(ChartError):
+ """Occurs when user passes inappropriate value to Chart, Dimension or ChartVariable class"""
+
+
+class Charts:
+ """Represent a collection of charts
+
+ All charts stored in a dict.
+ Chart is a instance of Chart class.
+ Charts adding must be done using Charts.add_chart() method only"""
+
+ def __init__(self, job_name, actual_job_name, priority, cleanup, get_update_every, module_name):
+ """
+ :param job_name: <bound method>
+ :param priority: <int>
+ :param get_update_every: <bound method>
+ """
+ self.job_name = job_name
+ self.actual_job_name = actual_job_name
+ self.priority = priority
+ self.cleanup = cleanup
+ self.get_update_every = get_update_every
+ self.module_name = module_name
+ self.charts = dict()
+
+ def __len__(self):
+ return len(self.charts)
+
+ def __iter__(self):
+ return iter(self.charts.values())
+
+ def __repr__(self):
+ return 'Charts({0})'.format(self)
+
+ def __str__(self):
+ return str([chart for chart in self.charts])
+
+ def __contains__(self, item):
+ return item in self.charts
+
+ def __getitem__(self, item):
+ return self.charts[item]
+
+ def __delitem__(self, key):
+ del self.charts[key]
+
+ def __bool__(self):
+ return bool(self.charts)
+
+ def __nonzero__(self):
+ return self.__bool__()
+
+ def add_chart(self, params):
+ """
+ Create Chart instance and add it to the dict
+
+ Manually adds job name, priority and update_every to params.
+ :param params: <list>
+ :return:
+ """
+ params = [self.job_name()] + params
+ new_chart = Chart(params)
+
+ new_chart.params['update_every'] = self.get_update_every()
+ new_chart.params['priority'] = self.priority
+ new_chart.params['module_name'] = self.module_name
+ new_chart.params['actual_job_name'] = self.actual_job_name
+
+ self.priority += 1
+ self.charts[new_chart.id] = new_chart
+
+ return new_chart
+
+ def active_charts(self):
+ return [chart.id for chart in self if not chart.flags.obsoleted]
+
+
+class Chart:
+ """Represent a chart"""
+
+ def __init__(self, params):
+ """
+ :param params: <list>
+ """
+ if not isinstance(params, list):
+ raise ItemTypeError("'chart' must be a list type")
+ if not len(params) >= 8:
+ raise ItemValueError("invalid value for 'chart', must be {0}".format(CHART_PARAMS))
+
+ self.params = dict(zip(CHART_PARAMS, (p or str() for p in params)))
+ self.name = '{type}.{id}'.format(type=self.params['type'],
+ id=self.params['id'])
+ if self.params.get('chart_type') not in CHART_TYPES:
+ self.params['chart_type'] = 'absolute'
+ hidden = str(self.params.get('hidden', ''))
+ self.params['hidden'] = 'hidden' if hidden == 'hidden' else ''
+
+ self.dimensions = list()
+ self.variables = set()
+ self.flags = ChartFlags()
+ self.penalty = 0
+
+ def __getattr__(self, item):
+ try:
+ return self.params[item]
+ except KeyError:
+ raise AttributeError("'{instance}' has no attribute '{attr}'".format(instance=repr(self),
+ attr=item))
+
+ def __repr__(self):
+ return 'Chart({0})'.format(self.id)
+
+ def __str__(self):
+ return self.id
+
+ def __iter__(self):
+ return iter(self.dimensions)
+
+ def __contains__(self, item):
+ return item in [dimension.id for dimension in self.dimensions]
+
+ def add_variable(self, variable):
+ """
+ :param variable: <list>
+ :return:
+ """
+ self.variables.add(ChartVariable(variable))
+
+ def add_dimension(self, dimension):
+ """
+ :param dimension: <list>
+ :return:
+ """
+ dim = Dimension(dimension)
+
+ if dim.id in self:
+ raise DuplicateItemError("'{dimension}' already in '{chart}' dimensions".format(dimension=dim.id,
+ chart=self.name))
+ self.refresh()
+ self.dimensions.append(dim)
+ return dim
+
+ def del_dimension(self, dimension_id, hide=True):
+ if dimension_id not in self:
+ return
+ idx = self.dimensions.index(dimension_id)
+ dimension = self.dimensions[idx]
+ if hide:
+ dimension.params['hidden'] = 'hidden'
+ dimension.params['obsolete'] = 'obsolete'
+ self.create()
+ self.dimensions.remove(dimension)
+
+ def hide_dimension(self, dimension_id, reverse=False):
+ if dimension_id not in self:
+ return
+ idx = self.dimensions.index(dimension_id)
+ dimension = self.dimensions[idx]
+ dimension.params['hidden'] = 'hidden' if not reverse else str()
+ self.refresh()
+
+ def create(self):
+ """
+ :return:
+ """
+ chart = CHART_CREATE.format(**self.params)
+ labels = CLABEL_COLLECT_JOB.format(**self.params) + CLABEL_COMMIT
+ dimensions = ''.join([dimension.create() for dimension in self.dimensions])
+ variables = ''.join([var.set(var.value) for var in self.variables if var])
+
+ self.flags.push = False
+ self.flags.created = True
+
+ safe_print(chart + labels + dimensions + variables)
+
+ def can_be_updated(self, data):
+ for dim in self.dimensions:
+ if dim.get_value(data) is not None:
+ return True
+ return False
+
+ def update(self, data, interval):
+ updated_dimensions, updated_variables = str(), str()
+
+ for dim in self.dimensions:
+ value = dim.get_value(data)
+ if value is not None:
+ updated_dimensions += dim.set(value)
+
+ for var in self.variables:
+ value = var.get_value(data)
+ if value is not None:
+ updated_variables += var.set(value)
+
+ if updated_dimensions:
+ since_last = interval if self.flags.updated else 0
+
+ if self.flags.push:
+ self.create()
+
+ chart_begin = CHART_BEGIN.format(type=self.type, id=self.id, since_last=since_last)
+ safe_print(chart_begin, updated_dimensions, updated_variables, 'END\n')
+
+ self.flags.updated = True
+ self.penalty = 0
+ else:
+ self.penalty += 1
+ self.flags.updated = False
+
+ return bool(updated_dimensions)
+
+ def obsolete(self):
+ self.flags.obsoleted = True
+ if self.flags.created:
+ safe_print(CHART_OBSOLETE.format(**self.params))
+
+ def refresh(self):
+ self.penalty = 0
+ self.flags.push = True
+ self.flags.obsoleted = False
+
+
+class Dimension:
+ """Represent a dimension"""
+
+ def __init__(self, params):
+ """
+ :param params: <list>
+ """
+ if not isinstance(params, list):
+ raise ItemTypeError("'dimension' must be a list type")
+ if not params:
+ raise ItemValueError("invalid value for 'dimension', must be {0}".format(DIMENSION_PARAMS))
+
+ self.params = dict(zip(DIMENSION_PARAMS, (p or str() for p in params)))
+ self.params['name'] = self.params.get('name') or self.params['id']
+
+ if self.params.get('algorithm') not in DIMENSION_ALGORITHMS:
+ self.params['algorithm'] = 'absolute'
+ if not isinstance(self.params.get('multiplier'), int):
+ self.params['multiplier'] = 1
+ if not isinstance(self.params.get('divisor'), int):
+ self.params['divisor'] = 1
+ self.params.setdefault('hidden', '')
+ self.params.setdefault('obsolete', '')
+
+ def __getattr__(self, item):
+ try:
+ return self.params[item]
+ except KeyError:
+ raise AttributeError("'{instance}' has no attribute '{attr}'".format(instance=repr(self),
+ attr=item))
+
+ def __repr__(self):
+ return 'Dimension({0})'.format(self.id)
+
+ def __str__(self):
+ return self.id
+
+ def __eq__(self, other):
+ if not isinstance(other, Dimension):
+ return self.id == other
+ return self.id == other.id
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ return hash(repr(self))
+
+ def create(self):
+ return DIMENSION_CREATE.format(**self.params)
+
+ def set(self, value):
+ """
+ :param value: <str>: must be a digit
+ :return:
+ """
+ return DIMENSION_SET.format(id=self.id,
+ value=value)
+
+ def get_value(self, data):
+ try:
+ return int(data[self.id])
+ except (KeyError, TypeError):
+ return None
+
+
+class ChartVariable:
+ """Represent a chart variable"""
+
+ def __init__(self, params):
+ """
+ :param params: <list>
+ """
+ if not isinstance(params, list):
+ raise ItemTypeError("'variable' must be a list type")
+ if not params:
+ raise ItemValueError("invalid value for 'variable' must be: {0}".format(VARIABLE_PARAMS))
+
+ self.params = dict(zip(VARIABLE_PARAMS, params))
+ self.params.setdefault('value', None)
+
+ def __getattr__(self, item):
+ try:
+ return self.params[item]
+ except KeyError:
+ raise AttributeError("'{instance}' has no attribute '{attr}'".format(instance=repr(self),
+ attr=item))
+
+ def __bool__(self):
+ return self.value is not None
+
+ def __nonzero__(self):
+ return self.__bool__()
+
+ def __repr__(self):
+ return 'ChartVariable({0})'.format(self.id)
+
+ def __str__(self):
+ return self.id
+
+ def __eq__(self, other):
+ if isinstance(other, ChartVariable):
+ return self.id == other.id
+ return False
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ return hash(repr(self))
+
+ def set(self, value):
+ return CHART_VARIABLE_SET.format(id=self.id,
+ value=value)
+
+ def get_value(self, data):
+ try:
+ return int(data[self.id])
+ except (KeyError, TypeError):
+ return None
+
+
+class ChartFlags:
+ def __init__(self):
+ self.push = True
+ self.created = False
+ self.updated = False
+ self.obsoleted = False
diff --git a/collectors/python.d.plugin/python_modules/bases/collection.py b/collectors/python.d.plugin/python_modules/bases/collection.py
new file mode 100644
index 0000000..93bf8cf
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/collection.py
@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+
+from threading import Lock
+
+PATH = os.getenv('PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin').split(':')
+
+CHART_BEGIN = 'BEGIN {0} {1}\n'
+CHART_CREATE = "CHART {0} '{1}' '{2}' '{3}' '{4}' '{5}' {6} {7} {8}\n"
+DIMENSION_CREATE = "DIMENSION '{0}' '{1}' {2} {3} {4} '{5}'\n"
+DIMENSION_SET = "SET '{0}' = {1}\n"
+
+print_lock = Lock()
+
+
+def setdefault_values(config, base_dict):
+ for key, value in base_dict.items():
+ config.setdefault(key, value)
+ return config
+
+
+def run_and_exit(func):
+ def wrapper(*args, **kwargs):
+ func(*args, **kwargs)
+ exit(1)
+
+ return wrapper
+
+
+def on_try_except_finally(on_except=(None,), on_finally=(None,)):
+ except_func = on_except[0]
+ finally_func = on_finally[0]
+
+ def decorator(func):
+ def wrapper(*args, **kwargs):
+ try:
+ func(*args, **kwargs)
+ except Exception:
+ if except_func:
+ except_func(*on_except[1:])
+ finally:
+ if finally_func:
+ finally_func(*on_finally[1:])
+
+ return wrapper
+
+ return decorator
+
+
+def static_vars(**kwargs):
+ def decorate(func):
+ for k in kwargs:
+ setattr(func, k, kwargs[k])
+ return func
+
+ return decorate
+
+
+@on_try_except_finally(on_except=(exit, 1))
+def safe_print(*msg):
+ """
+ :param msg:
+ :return:
+ """
+ print_lock.acquire()
+ print(''.join(msg))
+ print_lock.release()
+
+
+def find_binary(binary):
+ """
+ :param binary: <str>
+ :return:
+ """
+ for directory in PATH:
+ binary_name = os.path.join(directory, binary)
+ if os.path.isfile(binary_name) and os.access(binary_name, os.X_OK):
+ return binary_name
+ return None
+
+
+def read_last_line(f):
+ with open(f, 'rb') as opened:
+ opened.seek(-2, 2)
+ while opened.read(1) != b'\n':
+ opened.seek(-2, 1)
+ if opened.tell() == 0:
+ break
+ result = opened.readline()
+ return result.decode()
+
+
+def unicode_str(arg):
+ """Return the argument as a unicode string.
+
+ The `unicode` function has been removed from Python3 and `str` takes its
+ place. This function is a helper which will try using Python 2's `unicode`
+ and if it doesn't exist, assume we're using Python 3 and use `str`.
+
+ :param arg:
+ :return: <str>
+ """
+ # TODO: fix
+ try:
+ # https://github.com/netdata/netdata/issues/7613
+ if isinstance(arg, unicode):
+ return arg
+ return unicode(arg, errors='ignore')
+ # https://github.com/netdata/netdata/issues/7642
+ except TypeError:
+ return unicode(arg)
+ except NameError:
+ return str(arg)
diff --git a/collectors/python.d.plugin/python_modules/bases/loaders.py b/collectors/python.d.plugin/python_modules/bases/loaders.py
new file mode 100644
index 0000000..095f3a3
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/loaders.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+
+from sys import version_info
+
+PY_VERSION = version_info[:2]
+
+try:
+ if PY_VERSION > (3, 1):
+ from pyyaml3 import SafeLoader as YamlSafeLoader
+ else:
+ from pyyaml2 import SafeLoader as YamlSafeLoader
+except ImportError:
+ from yaml import SafeLoader as YamlSafeLoader
+
+
+try:
+ from collections import OrderedDict
+except ImportError:
+ from third_party.ordereddict import OrderedDict
+
+
+DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' if PY_VERSION > (3, 1) else u'tag:yaml.org,2002:map'
+
+
+def dict_constructor(loader, node):
+ return OrderedDict(loader.construct_pairs(node))
+
+
+YamlSafeLoader.add_constructor(DEFAULT_MAPPING_TAG, dict_constructor)
+
+
+def load_yaml(stream):
+ loader = YamlSafeLoader(stream)
+ try:
+ return loader.get_single_data()
+ finally:
+ loader.dispose()
+
+
+def load_config(file_name):
+ with open(file_name, 'r') as stream:
+ return load_yaml(stream)
diff --git a/collectors/python.d.plugin/python_modules/bases/loggers.py b/collectors/python.d.plugin/python_modules/bases/loggers.py
new file mode 100644
index 0000000..47f196a
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/bases/loggers.py
@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+# Description:
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import logging
+import traceback
+
+from sys import exc_info
+
+try:
+ from time import monotonic as time
+except ImportError:
+ from time import time
+
+from bases.collection import on_try_except_finally, unicode_str
+
+
+LOGGING_LEVELS = {'CRITICAL': 50,
+ 'ERROR': 40,
+ 'WARNING': 30,
+ 'INFO': 20,
+ 'DEBUG': 10,
+ 'NOTSET': 0}
+
+DEFAULT_LOG_LINE_FORMAT = '%(asctime)s: %(name)s %(levelname)s : %(message)s'
+DEFAULT_LOG_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
+
+PYTHON_D_LOG_LINE_FORMAT = '%(asctime)s: %(name)s %(levelname)s: %(module_name)s[%(job_name)s] : %(message)s'
+PYTHON_D_LOG_NAME = 'python.d'
+
+
+def limiter(log_max_count=30, allowed_in_seconds=60):
+ def on_decorator(func):
+
+ def on_call(*args):
+ current_time = args[0]._runtime_counters.start_mono
+ lc = args[0]._logger_counters
+
+ if lc.logged and lc.logged % log_max_count == 0:
+ if current_time - lc.time_to_compare <= allowed_in_seconds:
+ lc.dropped += 1
+ return
+ lc.time_to_compare = current_time
+
+ lc.logged += 1
+ func(*args)
+
+ return on_call
+ return on_decorator
+
+
+def add_traceback(func):
+ def on_call(*args):
+ self = args[0]
+
+ if not self.log_traceback:
+ func(*args)
+ else:
+ if exc_info()[0]:
+ func(*args)
+ func(self, traceback.format_exc())
+ else:
+ func(*args)
+
+ return on_call
+
+
+class LoggerCounters:
+ def __init__(self):
+ self.logged = 0
+ self.dropped = 0
+ self.time_to_compare = time()
+
+ def __repr__(self):
+ return 'LoggerCounter(logged: {logged}, dropped: {dropped})'.format(logged=self.logged,
+ dropped=self.dropped)
+
+
+class BaseLogger(object):
+ def __init__(self, logger_name, log_fmt=DEFAULT_LOG_LINE_FORMAT, date_fmt=DEFAULT_LOG_TIME_FORMAT,
+ handler=logging.StreamHandler):
+ """
+ :param logger_name: <str>
+ :param log_fmt: <str>
+ :param date_fmt: <str>
+ :param handler: <logging handler>
+ """
+ self.logger = logging.getLogger(logger_name)
+ if not self.has_handlers():
+ self.severity = 'INFO'
+ self.logger.addHandler(handler())
+ self.set_formatter(fmt=log_fmt, date_fmt=date_fmt)
+
+ def __repr__(self):
+ return '<Logger: {name})>'.format(name=self.logger.name)
+
+ def set_formatter(self, fmt, date_fmt=DEFAULT_LOG_TIME_FORMAT):
+ """
+ :param fmt: <str>
+ :param date_fmt: <str>
+ :return:
+ """
+ if self.has_handlers():
+ self.logger.handlers[0].setFormatter(logging.Formatter(fmt=fmt, datefmt=date_fmt))
+
+ def has_handlers(self):
+ return self.logger.handlers
+
+ @property
+ def severity(self):
+ return self.logger.getEffectiveLevel()
+
+ @severity.setter
+ def severity(self, level):
+ """
+ :param level: <str> or <int>
+ :return:
+ """
+ if level in LOGGING_LEVELS:
+ self.logger.setLevel(LOGGING_LEVELS[level])
+
+ def debug(self, *msg, **kwargs):
+ self.logger.debug(' '.join(map(unicode_str, msg)), **kwargs)
+
+ def info(self, *msg, **kwargs):
+ self.logger.info(' '.join(map(unicode_str, msg)), **kwargs)
+
+ def warning(self, *msg, **kwargs):
+ self.logger.warning(' '.join(map(unicode_str, msg)), **kwargs)
+
+ def error(self, *msg, **kwargs):
+ self.logger.error(' '.join(map(unicode_str, msg)), **kwargs)
+
+ def alert(self, *msg, **kwargs):
+ self.logger.critical(' '.join(map(unicode_str, msg)), **kwargs)
+
+ @on_try_except_finally(on_finally=(exit, 1))
+ def fatal(self, *msg, **kwargs):
+ self.logger.critical(' '.join(map(unicode_str, msg)), **kwargs)
+
+
+class PythonDLogger(object):
+ def __init__(self, logger_name=PYTHON_D_LOG_NAME, log_fmt=PYTHON_D_LOG_LINE_FORMAT):
+ """
+ :param logger_name: <str>
+ :param log_fmt: <str>
+ """
+ self.logger = BaseLogger(logger_name, log_fmt=log_fmt)
+ self.module_name = 'plugin'
+ self.job_name = 'main'
+ self._logger_counters = LoggerCounters()
+
+ _LOG_TRACEBACK = False
+
+ @property
+ def log_traceback(self):
+ return PythonDLogger._LOG_TRACEBACK
+
+ @log_traceback.setter
+ def log_traceback(self, value):
+ PythonDLogger._LOG_TRACEBACK = value
+
+ def debug(self, *msg):
+ self.logger.debug(*msg, extra={'module_name': self.module_name,
+ 'job_name': self.job_name or self.module_name})
+
+ def info(self, *msg):
+ self.logger.info(*msg, extra={'module_name': self.module_name,
+ 'job_name': self.job_name or self.module_name})
+
+ def warning(self, *msg):
+ self.logger.warning(*msg, extra={'module_name': self.module_name,
+ 'job_name': self.job_name or self.module_name})
+
+ @add_traceback
+ def error(self, *msg):
+ self.logger.error(*msg, extra={'module_name': self.module_name,
+ 'job_name': self.job_name or self.module_name})
+
+ @add_traceback
+ def alert(self, *msg):
+ self.logger.alert(*msg, extra={'module_name': self.module_name,
+ 'job_name': self.job_name or self.module_name})
+
+ def fatal(self, *msg):
+ self.logger.fatal(*msg, extra={'module_name': self.module_name,
+ 'job_name': self.job_name or self.module_name})
+
+
+class PythonDLimitedLogger(PythonDLogger):
+ @limiter()
+ def info(self, *msg):
+ PythonDLogger.info(self, *msg)
+
+ @limiter()
+ def warning(self, *msg):
+ PythonDLogger.warning(self, *msg)
+
+ @limiter()
+ def error(self, *msg):
+ PythonDLogger.error(self, *msg)
+
+ @limiter()
+ def alert(self, *msg):
+ PythonDLogger.alert(self, *msg)
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/__init__.py b/collectors/python.d.plugin/python_modules/pyyaml2/__init__.py
new file mode 100644
index 0000000..4d560e4
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/__init__.py
@@ -0,0 +1,316 @@
+# SPDX-License-Identifier: MIT
+
+from error import *
+
+from tokens import *
+from events import *
+from nodes import *
+
+from loader import *
+from dumper import *
+
+__version__ = '3.11'
+
+try:
+ from cyaml import *
+ __with_libyaml__ = True
+except ImportError:
+ __with_libyaml__ = False
+
+def scan(stream, Loader=Loader):
+ """
+ Scan a YAML stream and produce scanning tokens.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_token():
+ yield loader.get_token()
+ finally:
+ loader.dispose()
+
+def parse(stream, Loader=Loader):
+ """
+ Parse a YAML stream and produce parsing events.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_event():
+ yield loader.get_event()
+ finally:
+ loader.dispose()
+
+def compose(stream, Loader=Loader):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding representation tree.
+ """
+ loader = Loader(stream)
+ try:
+ return loader.get_single_node()
+ finally:
+ loader.dispose()
+
+def compose_all(stream, Loader=Loader):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding representation trees.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_node():
+ yield loader.get_node()
+ finally:
+ loader.dispose()
+
+def load(stream, Loader=Loader):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding Python object.
+ """
+ loader = Loader(stream)
+ try:
+ return loader.get_single_data()
+ finally:
+ loader.dispose()
+
+def load_all(stream, Loader=Loader):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding Python objects.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_data():
+ yield loader.get_data()
+ finally:
+ loader.dispose()
+
+def safe_load(stream):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding Python object.
+ Resolve only basic YAML tags.
+ """
+ return load(stream, SafeLoader)
+
+def safe_load_all(stream):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding Python objects.
+ Resolve only basic YAML tags.
+ """
+ return load_all(stream, SafeLoader)
+
+def emit(events, stream=None, Dumper=Dumper,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None):
+ """
+ Emit YAML parsing events into a stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ from StringIO import StringIO
+ stream = StringIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ try:
+ for event in events:
+ dumper.emit(event)
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def serialize_all(nodes, stream=None, Dumper=Dumper,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding='utf-8', explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ """
+ Serialize a sequence of representation trees into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ if encoding is None:
+ from StringIO import StringIO
+ else:
+ from cStringIO import StringIO
+ stream = StringIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break,
+ encoding=encoding, version=version, tags=tags,
+ explicit_start=explicit_start, explicit_end=explicit_end)
+ try:
+ dumper.open()
+ for node in nodes:
+ dumper.serialize(node)
+ dumper.close()
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def serialize(node, stream=None, Dumper=Dumper, **kwds):
+ """
+ Serialize a representation tree into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ return serialize_all([node], stream, Dumper=Dumper, **kwds)
+
+def dump_all(documents, stream=None, Dumper=Dumper,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding='utf-8', explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ """
+ Serialize a sequence of Python objects into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ if encoding is None:
+ from StringIO import StringIO
+ else:
+ from cStringIO import StringIO
+ stream = StringIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, default_style=default_style,
+ default_flow_style=default_flow_style,
+ canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break,
+ encoding=encoding, version=version, tags=tags,
+ explicit_start=explicit_start, explicit_end=explicit_end)
+ try:
+ dumper.open()
+ for data in documents:
+ dumper.represent(data)
+ dumper.close()
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def dump(data, stream=None, Dumper=Dumper, **kwds):
+ """
+ Serialize a Python object into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all([data], stream, Dumper=Dumper, **kwds)
+
+def safe_dump_all(documents, stream=None, **kwds):
+ """
+ Serialize a sequence of Python objects into a YAML stream.
+ Produce only basic YAML tags.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all(documents, stream, Dumper=SafeDumper, **kwds)
+
+def safe_dump(data, stream=None, **kwds):
+ """
+ Serialize a Python object into a YAML stream.
+ Produce only basic YAML tags.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all([data], stream, Dumper=SafeDumper, **kwds)
+
+def add_implicit_resolver(tag, regexp, first=None,
+ Loader=Loader, Dumper=Dumper):
+ """
+ Add an implicit scalar detector.
+ If an implicit scalar value matches the given regexp,
+ the corresponding tag is assigned to the scalar.
+ first is a sequence of possible initial characters or None.
+ """
+ Loader.add_implicit_resolver(tag, regexp, first)
+ Dumper.add_implicit_resolver(tag, regexp, first)
+
+def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper):
+ """
+ Add a path based resolver for the given tag.
+ A path is a list of keys that forms a path
+ to a node in the representation tree.
+ Keys can be string values, integers, or None.
+ """
+ Loader.add_path_resolver(tag, path, kind)
+ Dumper.add_path_resolver(tag, path, kind)
+
+def add_constructor(tag, constructor, Loader=Loader):
+ """
+ Add a constructor for the given tag.
+ Constructor is a function that accepts a Loader instance
+ and a node object and produces the corresponding Python object.
+ """
+ Loader.add_constructor(tag, constructor)
+
+def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader):
+ """
+ Add a multi-constructor for the given tag prefix.
+ Multi-constructor is called for a node if its tag starts with tag_prefix.
+ Multi-constructor accepts a Loader instance, a tag suffix,
+ and a node object and produces the corresponding Python object.
+ """
+ Loader.add_multi_constructor(tag_prefix, multi_constructor)
+
+def add_representer(data_type, representer, Dumper=Dumper):
+ """
+ Add a representer for the given type.
+ Representer is a function accepting a Dumper instance
+ and an instance of the given data type
+ and producing the corresponding representation node.
+ """
+ Dumper.add_representer(data_type, representer)
+
+def add_multi_representer(data_type, multi_representer, Dumper=Dumper):
+ """
+ Add a representer for the given type.
+ Multi-representer is a function accepting a Dumper instance
+ and an instance of the given data type or subtype
+ and producing the corresponding representation node.
+ """
+ Dumper.add_multi_representer(data_type, multi_representer)
+
+class YAMLObjectMetaclass(type):
+ """
+ The metaclass for YAMLObject.
+ """
+ def __init__(cls, name, bases, kwds):
+ super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
+ if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
+ cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
+ cls.yaml_dumper.add_representer(cls, cls.to_yaml)
+
+class YAMLObject(object):
+ """
+ An object that can dump itself to a YAML stream
+ and load itself from a YAML stream.
+ """
+
+ __metaclass__ = YAMLObjectMetaclass
+ __slots__ = () # no direct instantiation, so allow immutable subclasses
+
+ yaml_loader = Loader
+ yaml_dumper = Dumper
+
+ yaml_tag = None
+ yaml_flow_style = None
+
+ def from_yaml(cls, loader, node):
+ """
+ Convert a representation node to a Python object.
+ """
+ return loader.construct_yaml_object(node, cls)
+ from_yaml = classmethod(from_yaml)
+
+ def to_yaml(cls, dumper, data):
+ """
+ Convert a Python object to a representation node.
+ """
+ return dumper.represent_yaml_object(cls.yaml_tag, data, cls,
+ flow_style=cls.yaml_flow_style)
+ to_yaml = classmethod(to_yaml)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/composer.py b/collectors/python.d.plugin/python_modules/pyyaml2/composer.py
new file mode 100644
index 0000000..6b41b80
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/composer.py
@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['Composer', 'ComposerError']
+
+from error import MarkedYAMLError
+from events import *
+from nodes import *
+
+class ComposerError(MarkedYAMLError):
+ pass
+
+class Composer(object):
+
+ def __init__(self):
+ self.anchors = {}
+
+ def check_node(self):
+ # Drop the STREAM-START event.
+ if self.check_event(StreamStartEvent):
+ self.get_event()
+
+ # If there are more documents available?
+ return not self.check_event(StreamEndEvent)
+
+ def get_node(self):
+ # Get the root node of the next document.
+ if not self.check_event(StreamEndEvent):
+ return self.compose_document()
+
+ def get_single_node(self):
+ # Drop the STREAM-START event.
+ self.get_event()
+
+ # Compose a document if the stream is not empty.
+ document = None
+ if not self.check_event(StreamEndEvent):
+ document = self.compose_document()
+
+ # Ensure that the stream contains no more documents.
+ if not self.check_event(StreamEndEvent):
+ event = self.get_event()
+ raise ComposerError("expected a single document in the stream",
+ document.start_mark, "but found another document",
+ event.start_mark)
+
+ # Drop the STREAM-END event.
+ self.get_event()
+
+ return document
+
+ def compose_document(self):
+ # Drop the DOCUMENT-START event.
+ self.get_event()
+
+ # Compose the root node.
+ node = self.compose_node(None, None)
+
+ # Drop the DOCUMENT-END event.
+ self.get_event()
+
+ self.anchors = {}
+ return node
+
+ def compose_node(self, parent, index):
+ if self.check_event(AliasEvent):
+ event = self.get_event()
+ anchor = event.anchor
+ if anchor not in self.anchors:
+ raise ComposerError(None, None, "found undefined alias %r"
+ % anchor.encode('utf-8'), event.start_mark)
+ return self.anchors[anchor]
+ event = self.peek_event()
+ anchor = event.anchor
+ if anchor is not None:
+ if anchor in self.anchors:
+ raise ComposerError("found duplicate anchor %r; first occurence"
+ % anchor.encode('utf-8'), self.anchors[anchor].start_mark,
+ "second occurence", event.start_mark)
+ self.descend_resolver(parent, index)
+ if self.check_event(ScalarEvent):
+ node = self.compose_scalar_node(anchor)
+ elif self.check_event(SequenceStartEvent):
+ node = self.compose_sequence_node(anchor)
+ elif self.check_event(MappingStartEvent):
+ node = self.compose_mapping_node(anchor)
+ self.ascend_resolver()
+ return node
+
+ def compose_scalar_node(self, anchor):
+ event = self.get_event()
+ tag = event.tag
+ if tag is None or tag == u'!':
+ tag = self.resolve(ScalarNode, event.value, event.implicit)
+ node = ScalarNode(tag, event.value,
+ event.start_mark, event.end_mark, style=event.style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ return node
+
+ def compose_sequence_node(self, anchor):
+ start_event = self.get_event()
+ tag = start_event.tag
+ if tag is None or tag == u'!':
+ tag = self.resolve(SequenceNode, None, start_event.implicit)
+ node = SequenceNode(tag, [],
+ start_event.start_mark, None,
+ flow_style=start_event.flow_style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ index = 0
+ while not self.check_event(SequenceEndEvent):
+ node.value.append(self.compose_node(node, index))
+ index += 1
+ end_event = self.get_event()
+ node.end_mark = end_event.end_mark
+ return node
+
+ def compose_mapping_node(self, anchor):
+ start_event = self.get_event()
+ tag = start_event.tag
+ if tag is None or tag == u'!':
+ tag = self.resolve(MappingNode, None, start_event.implicit)
+ node = MappingNode(tag, [],
+ start_event.start_mark, None,
+ flow_style=start_event.flow_style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ while not self.check_event(MappingEndEvent):
+ #key_event = self.peek_event()
+ item_key = self.compose_node(node, None)
+ #if item_key in node.value:
+ # raise ComposerError("while composing a mapping", start_event.start_mark,
+ # "found duplicate key", key_event.start_mark)
+ item_value = self.compose_node(node, item_key)
+ #node.value[item_key] = item_value
+ node.value.append((item_key, item_value))
+ end_event = self.get_event()
+ node.end_mark = end_event.end_mark
+ return node
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/constructor.py b/collectors/python.d.plugin/python_modules/pyyaml2/constructor.py
new file mode 100644
index 0000000..8ad1b90
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/constructor.py
@@ -0,0 +1,676 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
+ 'ConstructorError']
+
+from error import *
+from nodes import *
+
+import datetime
+
+import binascii, re, sys, types
+
+class ConstructorError(MarkedYAMLError):
+ pass
+
+class BaseConstructor(object):
+
+ yaml_constructors = {}
+ yaml_multi_constructors = {}
+
+ def __init__(self):
+ self.constructed_objects = {}
+ self.recursive_objects = {}
+ self.state_generators = []
+ self.deep_construct = False
+
+ def check_data(self):
+ # If there are more documents available?
+ return self.check_node()
+
+ def get_data(self):
+ # Construct and return the next document.
+ if self.check_node():
+ return self.construct_document(self.get_node())
+
+ def get_single_data(self):
+ # Ensure that the stream contains a single document and construct it.
+ node = self.get_single_node()
+ if node is not None:
+ return self.construct_document(node)
+ return None
+
+ def construct_document(self, node):
+ data = self.construct_object(node)
+ while self.state_generators:
+ state_generators = self.state_generators
+ self.state_generators = []
+ for generator in state_generators:
+ for dummy in generator:
+ pass
+ self.constructed_objects = {}
+ self.recursive_objects = {}
+ self.deep_construct = False
+ return data
+
+ def construct_object(self, node, deep=False):
+ if node in self.constructed_objects:
+ return self.constructed_objects[node]
+ if deep:
+ old_deep = self.deep_construct
+ self.deep_construct = True
+ if node in self.recursive_objects:
+ raise ConstructorError(None, None,
+ "found unconstructable recursive node", node.start_mark)
+ self.recursive_objects[node] = None
+ constructor = None
+ tag_suffix = None
+ if node.tag in self.yaml_constructors:
+ constructor = self.yaml_constructors[node.tag]
+ else:
+ for tag_prefix in self.yaml_multi_constructors:
+ if node.tag.startswith(tag_prefix):
+ tag_suffix = node.tag[len(tag_prefix):]
+ constructor = self.yaml_multi_constructors[tag_prefix]
+ break
+ else:
+ if None in self.yaml_multi_constructors:
+ tag_suffix = node.tag
+ constructor = self.yaml_multi_constructors[None]
+ elif None in self.yaml_constructors:
+ constructor = self.yaml_constructors[None]
+ elif isinstance(node, ScalarNode):
+ constructor = self.__class__.construct_scalar
+ elif isinstance(node, SequenceNode):
+ constructor = self.__class__.construct_sequence
+ elif isinstance(node, MappingNode):
+ constructor = self.__class__.construct_mapping
+ if tag_suffix is None:
+ data = constructor(self, node)
+ else:
+ data = constructor(self, tag_suffix, node)
+ if isinstance(data, types.GeneratorType):
+ generator = data
+ data = generator.next()
+ if self.deep_construct:
+ for dummy in generator:
+ pass
+ else:
+ self.state_generators.append(generator)
+ self.constructed_objects[node] = data
+ del self.recursive_objects[node]
+ if deep:
+ self.deep_construct = old_deep
+ return data
+
+ def construct_scalar(self, node):
+ if not isinstance(node, ScalarNode):
+ raise ConstructorError(None, None,
+ "expected a scalar node, but found %s" % node.id,
+ node.start_mark)
+ return node.value
+
+ def construct_sequence(self, node, deep=False):
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError(None, None,
+ "expected a sequence node, but found %s" % node.id,
+ node.start_mark)
+ return [self.construct_object(child, deep=deep)
+ for child in node.value]
+
+ def construct_mapping(self, node, deep=False):
+ if not isinstance(node, MappingNode):
+ raise ConstructorError(None, None,
+ "expected a mapping node, but found %s" % node.id,
+ node.start_mark)
+ mapping = {}
+ for key_node, value_node in node.value:
+ key = self.construct_object(key_node, deep=deep)
+ try:
+ hash(key)
+ except TypeError, exc:
+ raise ConstructorError("while constructing a mapping", node.start_mark,
+ "found unacceptable key (%s)" % exc, key_node.start_mark)
+ value = self.construct_object(value_node, deep=deep)
+ mapping[key] = value
+ return mapping
+
+ def construct_pairs(self, node, deep=False):
+ if not isinstance(node, MappingNode):
+ raise ConstructorError(None, None,
+ "expected a mapping node, but found %s" % node.id,
+ node.start_mark)
+ pairs = []
+ for key_node, value_node in node.value:
+ key = self.construct_object(key_node, deep=deep)
+ value = self.construct_object(value_node, deep=deep)
+ pairs.append((key, value))
+ return pairs
+
+ def add_constructor(cls, tag, constructor):
+ if not 'yaml_constructors' in cls.__dict__:
+ cls.yaml_constructors = cls.yaml_constructors.copy()
+ cls.yaml_constructors[tag] = constructor
+ add_constructor = classmethod(add_constructor)
+
+ def add_multi_constructor(cls, tag_prefix, multi_constructor):
+ if not 'yaml_multi_constructors' in cls.__dict__:
+ cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
+ cls.yaml_multi_constructors[tag_prefix] = multi_constructor
+ add_multi_constructor = classmethod(add_multi_constructor)
+
+class SafeConstructor(BaseConstructor):
+
+ def construct_scalar(self, node):
+ if isinstance(node, MappingNode):
+ for key_node, value_node in node.value:
+ if key_node.tag == u'tag:yaml.org,2002:value':
+ return self.construct_scalar(value_node)
+ return BaseConstructor.construct_scalar(self, node)
+
+ def flatten_mapping(self, node):
+ merge = []
+ index = 0
+ while index < len(node.value):
+ key_node, value_node = node.value[index]
+ if key_node.tag == u'tag:yaml.org,2002:merge':
+ del node.value[index]
+ if isinstance(value_node, MappingNode):
+ self.flatten_mapping(value_node)
+ merge.extend(value_node.value)
+ elif isinstance(value_node, SequenceNode):
+ submerge = []
+ for subnode in value_node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing a mapping",
+ node.start_mark,
+ "expected a mapping for merging, but found %s"
+ % subnode.id, subnode.start_mark)
+ self.flatten_mapping(subnode)
+ submerge.append(subnode.value)
+ submerge.reverse()
+ for value in submerge:
+ merge.extend(value)
+ else:
+ raise ConstructorError("while constructing a mapping", node.start_mark,
+ "expected a mapping or list of mappings for merging, but found %s"
+ % value_node.id, value_node.start_mark)
+ elif key_node.tag == u'tag:yaml.org,2002:value':
+ key_node.tag = u'tag:yaml.org,2002:str'
+ index += 1
+ else:
+ index += 1
+ if merge:
+ node.value = merge + node.value
+
+ def construct_mapping(self, node, deep=False):
+ if isinstance(node, MappingNode):
+ self.flatten_mapping(node)
+ return BaseConstructor.construct_mapping(self, node, deep=deep)
+
+ def construct_yaml_null(self, node):
+ self.construct_scalar(node)
+ return None
+
+ bool_values = {
+ u'yes': True,
+ u'no': False,
+ u'true': True,
+ u'false': False,
+ u'on': True,
+ u'off': False,
+ }
+
+ def construct_yaml_bool(self, node):
+ value = self.construct_scalar(node)
+ return self.bool_values[value.lower()]
+
+ def construct_yaml_int(self, node):
+ value = str(self.construct_scalar(node))
+ value = value.replace('_', '')
+ sign = +1
+ if value[0] == '-':
+ sign = -1
+ if value[0] in '+-':
+ value = value[1:]
+ if value == '0':
+ return 0
+ elif value.startswith('0b'):
+ return sign*int(value[2:], 2)
+ elif value.startswith('0x'):
+ return sign*int(value[2:], 16)
+ elif value[0] == '0':
+ return sign*int(value, 8)
+ elif ':' in value:
+ digits = [int(part) for part in value.split(':')]
+ digits.reverse()
+ base = 1
+ value = 0
+ for digit in digits:
+ value += digit*base
+ base *= 60
+ return sign*value
+ else:
+ return sign*int(value)
+
+ inf_value = 1e300
+ while inf_value != inf_value*inf_value:
+ inf_value *= inf_value
+ nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99).
+
+ def construct_yaml_float(self, node):
+ value = str(self.construct_scalar(node))
+ value = value.replace('_', '').lower()
+ sign = +1
+ if value[0] == '-':
+ sign = -1
+ if value[0] in '+-':
+ value = value[1:]
+ if value == '.inf':
+ return sign*self.inf_value
+ elif value == '.nan':
+ return self.nan_value
+ elif ':' in value:
+ digits = [float(part) for part in value.split(':')]
+ digits.reverse()
+ base = 1
+ value = 0.0
+ for digit in digits:
+ value += digit*base
+ base *= 60
+ return sign*value
+ else:
+ return sign*float(value)
+
+ def construct_yaml_binary(self, node):
+ value = self.construct_scalar(node)
+ try:
+ return str(value).decode('base64')
+ except (binascii.Error, UnicodeEncodeError), exc:
+ raise ConstructorError(None, None,
+ "failed to decode base64 data: %s" % exc, node.start_mark)
+
+ timestamp_regexp = re.compile(
+ ur'''^(?P<year>[0-9][0-9][0-9][0-9])
+ -(?P<month>[0-9][0-9]?)
+ -(?P<day>[0-9][0-9]?)
+ (?:(?:[Tt]|[ \t]+)
+ (?P<hour>[0-9][0-9]?)
+ :(?P<minute>[0-9][0-9])
+ :(?P<second>[0-9][0-9])
+ (?:\.(?P<fraction>[0-9]*))?
+ (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)
+ (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X)
+
+ def construct_yaml_timestamp(self, node):
+ value = self.construct_scalar(node)
+ match = self.timestamp_regexp.match(node.value)
+ values = match.groupdict()
+ year = int(values['year'])
+ month = int(values['month'])
+ day = int(values['day'])
+ if not values['hour']:
+ return datetime.date(year, month, day)
+ hour = int(values['hour'])
+ minute = int(values['minute'])
+ second = int(values['second'])
+ fraction = 0
+ if values['fraction']:
+ fraction = values['fraction'][:6]
+ while len(fraction) < 6:
+ fraction += '0'
+ fraction = int(fraction)
+ delta = None
+ if values['tz_sign']:
+ tz_hour = int(values['tz_hour'])
+ tz_minute = int(values['tz_minute'] or 0)
+ delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
+ if values['tz_sign'] == '-':
+ delta = -delta
+ data = datetime.datetime(year, month, day, hour, minute, second, fraction)
+ if delta:
+ data -= delta
+ return data
+
+ def construct_yaml_omap(self, node):
+ # Note: we do not check for duplicate keys, because it's too
+ # CPU-expensive.
+ omap = []
+ yield omap
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a sequence, but found %s" % node.id, node.start_mark)
+ for subnode in node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a mapping of length 1, but found %s" % subnode.id,
+ subnode.start_mark)
+ if len(subnode.value) != 1:
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a single mapping item, but found %d items" % len(subnode.value),
+ subnode.start_mark)
+ key_node, value_node = subnode.value[0]
+ key = self.construct_object(key_node)
+ value = self.construct_object(value_node)
+ omap.append((key, value))
+
+ def construct_yaml_pairs(self, node):
+ # Note: the same code as `construct_yaml_omap`.
+ pairs = []
+ yield pairs
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a sequence, but found %s" % node.id, node.start_mark)
+ for subnode in node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a mapping of length 1, but found %s" % subnode.id,
+ subnode.start_mark)
+ if len(subnode.value) != 1:
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a single mapping item, but found %d items" % len(subnode.value),
+ subnode.start_mark)
+ key_node, value_node = subnode.value[0]
+ key = self.construct_object(key_node)
+ value = self.construct_object(value_node)
+ pairs.append((key, value))
+
+ def construct_yaml_set(self, node):
+ data = set()
+ yield data
+ value = self.construct_mapping(node)
+ data.update(value)
+
+ def construct_yaml_str(self, node):
+ value = self.construct_scalar(node)
+ try:
+ return value.encode('ascii')
+ except UnicodeEncodeError:
+ return value
+
+ def construct_yaml_seq(self, node):
+ data = []
+ yield data
+ data.extend(self.construct_sequence(node))
+
+ def construct_yaml_map(self, node):
+ data = {}
+ yield data
+ value = self.construct_mapping(node)
+ data.update(value)
+
+ def construct_yaml_object(self, node, cls):
+ data = cls.__new__(cls)
+ yield data
+ if hasattr(data, '__setstate__'):
+ state = self.construct_mapping(node, deep=True)
+ data.__setstate__(state)
+ else:
+ state = self.construct_mapping(node)
+ data.__dict__.update(state)
+
+ def construct_undefined(self, node):
+ raise ConstructorError(None, None,
+ "could not determine a constructor for the tag %r" % node.tag.encode('utf-8'),
+ node.start_mark)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:null',
+ SafeConstructor.construct_yaml_null)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:bool',
+ SafeConstructor.construct_yaml_bool)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:int',
+ SafeConstructor.construct_yaml_int)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:float',
+ SafeConstructor.construct_yaml_float)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:binary',
+ SafeConstructor.construct_yaml_binary)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:timestamp',
+ SafeConstructor.construct_yaml_timestamp)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:omap',
+ SafeConstructor.construct_yaml_omap)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:pairs',
+ SafeConstructor.construct_yaml_pairs)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:set',
+ SafeConstructor.construct_yaml_set)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:str',
+ SafeConstructor.construct_yaml_str)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:seq',
+ SafeConstructor.construct_yaml_seq)
+
+SafeConstructor.add_constructor(
+ u'tag:yaml.org,2002:map',
+ SafeConstructor.construct_yaml_map)
+
+SafeConstructor.add_constructor(None,
+ SafeConstructor.construct_undefined)
+
+class Constructor(SafeConstructor):
+
+ def construct_python_str(self, node):
+ return self.construct_scalar(node).encode('utf-8')
+
+ def construct_python_unicode(self, node):
+ return self.construct_scalar(node)
+
+ def construct_python_long(self, node):
+ return long(self.construct_yaml_int(node))
+
+ def construct_python_complex(self, node):
+ return complex(self.construct_scalar(node))
+
+ def construct_python_tuple(self, node):
+ return tuple(self.construct_sequence(node))
+
+ def find_python_module(self, name, mark):
+ if not name:
+ raise ConstructorError("while constructing a Python module", mark,
+ "expected non-empty name appended to the tag", mark)
+ try:
+ __import__(name)
+ except ImportError, exc:
+ raise ConstructorError("while constructing a Python module", mark,
+ "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark)
+ return sys.modules[name]
+
+ def find_python_name(self, name, mark):
+ if not name:
+ raise ConstructorError("while constructing a Python object", mark,
+ "expected non-empty name appended to the tag", mark)
+ if u'.' in name:
+ module_name, object_name = name.rsplit('.', 1)
+ else:
+ module_name = '__builtin__'
+ object_name = name
+ try:
+ __import__(module_name)
+ except ImportError, exc:
+ raise ConstructorError("while constructing a Python object", mark,
+ "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark)
+ module = sys.modules[module_name]
+ if not hasattr(module, object_name):
+ raise ConstructorError("while constructing a Python object", mark,
+ "cannot find %r in the module %r" % (object_name.encode('utf-8'),
+ module.__name__), mark)
+ return getattr(module, object_name)
+
+ def construct_python_name(self, suffix, node):
+ value = self.construct_scalar(node)
+ if value:
+ raise ConstructorError("while constructing a Python name", node.start_mark,
+ "expected the empty value, but found %r" % value.encode('utf-8'),
+ node.start_mark)
+ return self.find_python_name(suffix, node.start_mark)
+
+ def construct_python_module(self, suffix, node):
+ value = self.construct_scalar(node)
+ if value:
+ raise ConstructorError("while constructing a Python module", node.start_mark,
+ "expected the empty value, but found %r" % value.encode('utf-8'),
+ node.start_mark)
+ return self.find_python_module(suffix, node.start_mark)
+
+ class classobj: pass
+
+ def make_python_instance(self, suffix, node,
+ args=None, kwds=None, newobj=False):
+ if not args:
+ args = []
+ if not kwds:
+ kwds = {}
+ cls = self.find_python_name(suffix, node.start_mark)
+ if newobj and isinstance(cls, type(self.classobj)) \
+ and not args and not kwds:
+ instance = self.classobj()
+ instance.__class__ = cls
+ return instance
+ elif newobj and isinstance(cls, type):
+ return cls.__new__(cls, *args, **kwds)
+ else:
+ return cls(*args, **kwds)
+
+ def set_python_instance_state(self, instance, state):
+ if hasattr(instance, '__setstate__'):
+ instance.__setstate__(state)
+ else:
+ slotstate = {}
+ if isinstance(state, tuple) and len(state) == 2:
+ state, slotstate = state
+ if hasattr(instance, '__dict__'):
+ instance.__dict__.update(state)
+ elif state:
+ slotstate.update(state)
+ for key, value in slotstate.items():
+ setattr(object, key, value)
+
+ def construct_python_object(self, suffix, node):
+ # Format:
+ # !!python/object:module.name { ... state ... }
+ instance = self.make_python_instance(suffix, node, newobj=True)
+ yield instance
+ deep = hasattr(instance, '__setstate__')
+ state = self.construct_mapping(node, deep=deep)
+ self.set_python_instance_state(instance, state)
+
+ def construct_python_object_apply(self, suffix, node, newobj=False):
+ # Format:
+ # !!python/object/apply # (or !!python/object/new)
+ # args: [ ... arguments ... ]
+ # kwds: { ... keywords ... }
+ # state: ... state ...
+ # listitems: [ ... listitems ... ]
+ # dictitems: { ... dictitems ... }
+ # or short format:
+ # !!python/object/apply [ ... arguments ... ]
+ # The difference between !!python/object/apply and !!python/object/new
+ # is how an object is created, check make_python_instance for details.
+ if isinstance(node, SequenceNode):
+ args = self.construct_sequence(node, deep=True)
+ kwds = {}
+ state = {}
+ listitems = []
+ dictitems = {}
+ else:
+ value = self.construct_mapping(node, deep=True)
+ args = value.get('args', [])
+ kwds = value.get('kwds', {})
+ state = value.get('state', {})
+ listitems = value.get('listitems', [])
+ dictitems = value.get('dictitems', {})
+ instance = self.make_python_instance(suffix, node, args, kwds, newobj)
+ if state:
+ self.set_python_instance_state(instance, state)
+ if listitems:
+ instance.extend(listitems)
+ if dictitems:
+ for key in dictitems:
+ instance[key] = dictitems[key]
+ return instance
+
+ def construct_python_object_new(self, suffix, node):
+ return self.construct_python_object_apply(suffix, node, newobj=True)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/none',
+ Constructor.construct_yaml_null)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/bool',
+ Constructor.construct_yaml_bool)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/str',
+ Constructor.construct_python_str)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/unicode',
+ Constructor.construct_python_unicode)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/int',
+ Constructor.construct_yaml_int)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/long',
+ Constructor.construct_python_long)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/float',
+ Constructor.construct_yaml_float)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/complex',
+ Constructor.construct_python_complex)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/list',
+ Constructor.construct_yaml_seq)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/tuple',
+ Constructor.construct_python_tuple)
+
+Constructor.add_constructor(
+ u'tag:yaml.org,2002:python/dict',
+ Constructor.construct_yaml_map)
+
+Constructor.add_multi_constructor(
+ u'tag:yaml.org,2002:python/name:',
+ Constructor.construct_python_name)
+
+Constructor.add_multi_constructor(
+ u'tag:yaml.org,2002:python/module:',
+ Constructor.construct_python_module)
+
+Constructor.add_multi_constructor(
+ u'tag:yaml.org,2002:python/object:',
+ Constructor.construct_python_object)
+
+Constructor.add_multi_constructor(
+ u'tag:yaml.org,2002:python/object/apply:',
+ Constructor.construct_python_object_apply)
+
+Constructor.add_multi_constructor(
+ u'tag:yaml.org,2002:python/object/new:',
+ Constructor.construct_python_object_new)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/cyaml.py b/collectors/python.d.plugin/python_modules/pyyaml2/cyaml.py
new file mode 100644
index 0000000..2858ab4
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/cyaml.py
@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader',
+ 'CBaseDumper', 'CSafeDumper', 'CDumper']
+
+from _yaml import CParser, CEmitter
+
+from constructor import *
+
+from serializer import *
+from representer import *
+
+from resolver import *
+
+class CBaseLoader(CParser, BaseConstructor, BaseResolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ BaseConstructor.__init__(self)
+ BaseResolver.__init__(self)
+
+class CSafeLoader(CParser, SafeConstructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ SafeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class CLoader(CParser, Constructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ Constructor.__init__(self)
+ Resolver.__init__(self)
+
+class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ SafeRepresenter.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class CDumper(CEmitter, Serializer, Representer, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/dumper.py b/collectors/python.d.plugin/python_modules/pyyaml2/dumper.py
new file mode 100644
index 0000000..3685cbe
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/dumper.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseDumper', 'SafeDumper', 'Dumper']
+
+from emitter import *
+from serializer import *
+from representer import *
+from resolver import *
+
+class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ SafeRepresenter.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class Dumper(Emitter, Serializer, Representer, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/emitter.py b/collectors/python.d.plugin/python_modules/pyyaml2/emitter.py
new file mode 100644
index 0000000..9a460a0
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/emitter.py
@@ -0,0 +1,1141 @@
+# SPDX-License-Identifier: MIT
+
+# Emitter expects events obeying the following grammar:
+# stream ::= STREAM-START document* STREAM-END
+# document ::= DOCUMENT-START node DOCUMENT-END
+# node ::= SCALAR | sequence | mapping
+# sequence ::= SEQUENCE-START node* SEQUENCE-END
+# mapping ::= MAPPING-START (node node)* MAPPING-END
+
+__all__ = ['Emitter', 'EmitterError']
+
+from error import YAMLError
+from events import *
+
+class EmitterError(YAMLError):
+ pass
+
+class ScalarAnalysis(object):
+ def __init__(self, scalar, empty, multiline,
+ allow_flow_plain, allow_block_plain,
+ allow_single_quoted, allow_double_quoted,
+ allow_block):
+ self.scalar = scalar
+ self.empty = empty
+ self.multiline = multiline
+ self.allow_flow_plain = allow_flow_plain
+ self.allow_block_plain = allow_block_plain
+ self.allow_single_quoted = allow_single_quoted
+ self.allow_double_quoted = allow_double_quoted
+ self.allow_block = allow_block
+
+class Emitter(object):
+
+ DEFAULT_TAG_PREFIXES = {
+ u'!' : u'!',
+ u'tag:yaml.org,2002:' : u'!!',
+ }
+
+ def __init__(self, stream, canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None):
+
+ # The stream should have the methods `write` and possibly `flush`.
+ self.stream = stream
+
+ # Encoding can be overriden by STREAM-START.
+ self.encoding = None
+
+ # Emitter is a state machine with a stack of states to handle nested
+ # structures.
+ self.states = []
+ self.state = self.expect_stream_start
+
+ # Current event and the event queue.
+ self.events = []
+ self.event = None
+
+ # The current indentation level and the stack of previous indents.
+ self.indents = []
+ self.indent = None
+
+ # Flow level.
+ self.flow_level = 0
+
+ # Contexts.
+ self.root_context = False
+ self.sequence_context = False
+ self.mapping_context = False
+ self.simple_key_context = False
+
+ # Characteristics of the last emitted character:
+ # - current position.
+ # - is it a whitespace?
+ # - is it an indention character
+ # (indentation space, '-', '?', or ':')?
+ self.line = 0
+ self.column = 0
+ self.whitespace = True
+ self.indention = True
+
+ # Whether the document requires an explicit document indicator
+ self.open_ended = False
+
+ # Formatting details.
+ self.canonical = canonical
+ self.allow_unicode = allow_unicode
+ self.best_indent = 2
+ if indent and 1 < indent < 10:
+ self.best_indent = indent
+ self.best_width = 80
+ if width and width > self.best_indent*2:
+ self.best_width = width
+ self.best_line_break = u'\n'
+ if line_break in [u'\r', u'\n', u'\r\n']:
+ self.best_line_break = line_break
+
+ # Tag prefixes.
+ self.tag_prefixes = None
+
+ # Prepared anchor and tag.
+ self.prepared_anchor = None
+ self.prepared_tag = None
+
+ # Scalar analysis and style.
+ self.analysis = None
+ self.style = None
+
+ def dispose(self):
+ # Reset the state attributes (to clear self-references)
+ self.states = []
+ self.state = None
+
+ def emit(self, event):
+ self.events.append(event)
+ while not self.need_more_events():
+ self.event = self.events.pop(0)
+ self.state()
+ self.event = None
+
+ # In some cases, we wait for a few next events before emitting.
+
+ def need_more_events(self):
+ if not self.events:
+ return True
+ event = self.events[0]
+ if isinstance(event, DocumentStartEvent):
+ return self.need_events(1)
+ elif isinstance(event, SequenceStartEvent):
+ return self.need_events(2)
+ elif isinstance(event, MappingStartEvent):
+ return self.need_events(3)
+ else:
+ return False
+
+ def need_events(self, count):
+ level = 0
+ for event in self.events[1:]:
+ if isinstance(event, (DocumentStartEvent, CollectionStartEvent)):
+ level += 1
+ elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)):
+ level -= 1
+ elif isinstance(event, StreamEndEvent):
+ level = -1
+ if level < 0:
+ return False
+ return (len(self.events) < count+1)
+
+ def increase_indent(self, flow=False, indentless=False):
+ self.indents.append(self.indent)
+ if self.indent is None:
+ if flow:
+ self.indent = self.best_indent
+ else:
+ self.indent = 0
+ elif not indentless:
+ self.indent += self.best_indent
+
+ # States.
+
+ # Stream handlers.
+
+ def expect_stream_start(self):
+ if isinstance(self.event, StreamStartEvent):
+ if self.event.encoding and not getattr(self.stream, 'encoding', None):
+ self.encoding = self.event.encoding
+ self.write_stream_start()
+ self.state = self.expect_first_document_start
+ else:
+ raise EmitterError("expected StreamStartEvent, but got %s"
+ % self.event)
+
+ def expect_nothing(self):
+ raise EmitterError("expected nothing, but got %s" % self.event)
+
+ # Document handlers.
+
+ def expect_first_document_start(self):
+ return self.expect_document_start(first=True)
+
+ def expect_document_start(self, first=False):
+ if isinstance(self.event, DocumentStartEvent):
+ if (self.event.version or self.event.tags) and self.open_ended:
+ self.write_indicator(u'...', True)
+ self.write_indent()
+ if self.event.version:
+ version_text = self.prepare_version(self.event.version)
+ self.write_version_directive(version_text)
+ self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
+ if self.event.tags:
+ handles = self.event.tags.keys()
+ handles.sort()
+ for handle in handles:
+ prefix = self.event.tags[handle]
+ self.tag_prefixes[prefix] = handle
+ handle_text = self.prepare_tag_handle(handle)
+ prefix_text = self.prepare_tag_prefix(prefix)
+ self.write_tag_directive(handle_text, prefix_text)
+ implicit = (first and not self.event.explicit and not self.canonical
+ and not self.event.version and not self.event.tags
+ and not self.check_empty_document())
+ if not implicit:
+ self.write_indent()
+ self.write_indicator(u'---', True)
+ if self.canonical:
+ self.write_indent()
+ self.state = self.expect_document_root
+ elif isinstance(self.event, StreamEndEvent):
+ if self.open_ended:
+ self.write_indicator(u'...', True)
+ self.write_indent()
+ self.write_stream_end()
+ self.state = self.expect_nothing
+ else:
+ raise EmitterError("expected DocumentStartEvent, but got %s"
+ % self.event)
+
+ def expect_document_end(self):
+ if isinstance(self.event, DocumentEndEvent):
+ self.write_indent()
+ if self.event.explicit:
+ self.write_indicator(u'...', True)
+ self.write_indent()
+ self.flush_stream()
+ self.state = self.expect_document_start
+ else:
+ raise EmitterError("expected DocumentEndEvent, but got %s"
+ % self.event)
+
+ def expect_document_root(self):
+ self.states.append(self.expect_document_end)
+ self.expect_node(root=True)
+
+ # Node handlers.
+
+ def expect_node(self, root=False, sequence=False, mapping=False,
+ simple_key=False):
+ self.root_context = root
+ self.sequence_context = sequence
+ self.mapping_context = mapping
+ self.simple_key_context = simple_key
+ if isinstance(self.event, AliasEvent):
+ self.expect_alias()
+ elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)):
+ self.process_anchor(u'&')
+ self.process_tag()
+ if isinstance(self.event, ScalarEvent):
+ self.expect_scalar()
+ elif isinstance(self.event, SequenceStartEvent):
+ if self.flow_level or self.canonical or self.event.flow_style \
+ or self.check_empty_sequence():
+ self.expect_flow_sequence()
+ else:
+ self.expect_block_sequence()
+ elif isinstance(self.event, MappingStartEvent):
+ if self.flow_level or self.canonical or self.event.flow_style \
+ or self.check_empty_mapping():
+ self.expect_flow_mapping()
+ else:
+ self.expect_block_mapping()
+ else:
+ raise EmitterError("expected NodeEvent, but got %s" % self.event)
+
+ def expect_alias(self):
+ if self.event.anchor is None:
+ raise EmitterError("anchor is not specified for alias")
+ self.process_anchor(u'*')
+ self.state = self.states.pop()
+
+ def expect_scalar(self):
+ self.increase_indent(flow=True)
+ self.process_scalar()
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+
+ # Flow sequence handlers.
+
+ def expect_flow_sequence(self):
+ self.write_indicator(u'[', True, whitespace=True)
+ self.flow_level += 1
+ self.increase_indent(flow=True)
+ self.state = self.expect_first_flow_sequence_item
+
+ def expect_first_flow_sequence_item(self):
+ if isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ self.write_indicator(u']', False)
+ self.state = self.states.pop()
+ else:
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.states.append(self.expect_flow_sequence_item)
+ self.expect_node(sequence=True)
+
+ def expect_flow_sequence_item(self):
+ if isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ if self.canonical:
+ self.write_indicator(u',', False)
+ self.write_indent()
+ self.write_indicator(u']', False)
+ self.state = self.states.pop()
+ else:
+ self.write_indicator(u',', False)
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.states.append(self.expect_flow_sequence_item)
+ self.expect_node(sequence=True)
+
+ # Flow mapping handlers.
+
+ def expect_flow_mapping(self):
+ self.write_indicator(u'{', True, whitespace=True)
+ self.flow_level += 1
+ self.increase_indent(flow=True)
+ self.state = self.expect_first_flow_mapping_key
+
+ def expect_first_flow_mapping_key(self):
+ if isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ self.write_indicator(u'}', False)
+ self.state = self.states.pop()
+ else:
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ if not self.canonical and self.check_simple_key():
+ self.states.append(self.expect_flow_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator(u'?', True)
+ self.states.append(self.expect_flow_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_key(self):
+ if isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ if self.canonical:
+ self.write_indicator(u',', False)
+ self.write_indent()
+ self.write_indicator(u'}', False)
+ self.state = self.states.pop()
+ else:
+ self.write_indicator(u',', False)
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ if not self.canonical and self.check_simple_key():
+ self.states.append(self.expect_flow_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator(u'?', True)
+ self.states.append(self.expect_flow_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_simple_value(self):
+ self.write_indicator(u':', False)
+ self.states.append(self.expect_flow_mapping_key)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_value(self):
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.write_indicator(u':', True)
+ self.states.append(self.expect_flow_mapping_key)
+ self.expect_node(mapping=True)
+
+ # Block sequence handlers.
+
+ def expect_block_sequence(self):
+ indentless = (self.mapping_context and not self.indention)
+ self.increase_indent(flow=False, indentless=indentless)
+ self.state = self.expect_first_block_sequence_item
+
+ def expect_first_block_sequence_item(self):
+ return self.expect_block_sequence_item(first=True)
+
+ def expect_block_sequence_item(self, first=False):
+ if not first and isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+ else:
+ self.write_indent()
+ self.write_indicator(u'-', True, indention=True)
+ self.states.append(self.expect_block_sequence_item)
+ self.expect_node(sequence=True)
+
+ # Block mapping handlers.
+
+ def expect_block_mapping(self):
+ self.increase_indent(flow=False)
+ self.state = self.expect_first_block_mapping_key
+
+ def expect_first_block_mapping_key(self):
+ return self.expect_block_mapping_key(first=True)
+
+ def expect_block_mapping_key(self, first=False):
+ if not first and isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+ else:
+ self.write_indent()
+ if self.check_simple_key():
+ self.states.append(self.expect_block_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator(u'?', True, indention=True)
+ self.states.append(self.expect_block_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_block_mapping_simple_value(self):
+ self.write_indicator(u':', False)
+ self.states.append(self.expect_block_mapping_key)
+ self.expect_node(mapping=True)
+
+ def expect_block_mapping_value(self):
+ self.write_indent()
+ self.write_indicator(u':', True, indention=True)
+ self.states.append(self.expect_block_mapping_key)
+ self.expect_node(mapping=True)
+
+ # Checkers.
+
+ def check_empty_sequence(self):
+ return (isinstance(self.event, SequenceStartEvent) and self.events
+ and isinstance(self.events[0], SequenceEndEvent))
+
+ def check_empty_mapping(self):
+ return (isinstance(self.event, MappingStartEvent) and self.events
+ and isinstance(self.events[0], MappingEndEvent))
+
+ def check_empty_document(self):
+ if not isinstance(self.event, DocumentStartEvent) or not self.events:
+ return False
+ event = self.events[0]
+ return (isinstance(event, ScalarEvent) and event.anchor is None
+ and event.tag is None and event.implicit and event.value == u'')
+
+ def check_simple_key(self):
+ length = 0
+ if isinstance(self.event, NodeEvent) and self.event.anchor is not None:
+ if self.prepared_anchor is None:
+ self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+ length += len(self.prepared_anchor)
+ if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \
+ and self.event.tag is not None:
+ if self.prepared_tag is None:
+ self.prepared_tag = self.prepare_tag(self.event.tag)
+ length += len(self.prepared_tag)
+ if isinstance(self.event, ScalarEvent):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ length += len(self.analysis.scalar)
+ return (length < 128 and (isinstance(self.event, AliasEvent)
+ or (isinstance(self.event, ScalarEvent)
+ and not self.analysis.empty and not self.analysis.multiline)
+ or self.check_empty_sequence() or self.check_empty_mapping()))
+
+ # Anchor, Tag, and Scalar processors.
+
+ def process_anchor(self, indicator):
+ if self.event.anchor is None:
+ self.prepared_anchor = None
+ return
+ if self.prepared_anchor is None:
+ self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+ if self.prepared_anchor:
+ self.write_indicator(indicator+self.prepared_anchor, True)
+ self.prepared_anchor = None
+
+ def process_tag(self):
+ tag = self.event.tag
+ if isinstance(self.event, ScalarEvent):
+ if self.style is None:
+ self.style = self.choose_scalar_style()
+ if ((not self.canonical or tag is None) and
+ ((self.style == '' and self.event.implicit[0])
+ or (self.style != '' and self.event.implicit[1]))):
+ self.prepared_tag = None
+ return
+ if self.event.implicit[0] and tag is None:
+ tag = u'!'
+ self.prepared_tag = None
+ else:
+ if (not self.canonical or tag is None) and self.event.implicit:
+ self.prepared_tag = None
+ return
+ if tag is None:
+ raise EmitterError("tag is not specified")
+ if self.prepared_tag is None:
+ self.prepared_tag = self.prepare_tag(tag)
+ if self.prepared_tag:
+ self.write_indicator(self.prepared_tag, True)
+ self.prepared_tag = None
+
+ def choose_scalar_style(self):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ if self.event.style == '"' or self.canonical:
+ return '"'
+ if not self.event.style and self.event.implicit[0]:
+ if (not (self.simple_key_context and
+ (self.analysis.empty or self.analysis.multiline))
+ and (self.flow_level and self.analysis.allow_flow_plain
+ or (not self.flow_level and self.analysis.allow_block_plain))):
+ return ''
+ if self.event.style and self.event.style in '|>':
+ if (not self.flow_level and not self.simple_key_context
+ and self.analysis.allow_block):
+ return self.event.style
+ if not self.event.style or self.event.style == '\'':
+ if (self.analysis.allow_single_quoted and
+ not (self.simple_key_context and self.analysis.multiline)):
+ return '\''
+ return '"'
+
+ def process_scalar(self):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ if self.style is None:
+ self.style = self.choose_scalar_style()
+ split = (not self.simple_key_context)
+ #if self.analysis.multiline and split \
+ # and (not self.style or self.style in '\'\"'):
+ # self.write_indent()
+ if self.style == '"':
+ self.write_double_quoted(self.analysis.scalar, split)
+ elif self.style == '\'':
+ self.write_single_quoted(self.analysis.scalar, split)
+ elif self.style == '>':
+ self.write_folded(self.analysis.scalar)
+ elif self.style == '|':
+ self.write_literal(self.analysis.scalar)
+ else:
+ self.write_plain(self.analysis.scalar, split)
+ self.analysis = None
+ self.style = None
+
+ # Analyzers.
+
+ def prepare_version(self, version):
+ major, minor = version
+ if major != 1:
+ raise EmitterError("unsupported YAML version: %d.%d" % (major, minor))
+ return u'%d.%d' % (major, minor)
+
+ def prepare_tag_handle(self, handle):
+ if not handle:
+ raise EmitterError("tag handle must not be empty")
+ if handle[0] != u'!' or handle[-1] != u'!':
+ raise EmitterError("tag handle must start and end with '!': %r"
+ % (handle.encode('utf-8')))
+ for ch in handle[1:-1]:
+ if not (u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
+ or ch in u'-_'):
+ raise EmitterError("invalid character %r in the tag handle: %r"
+ % (ch.encode('utf-8'), handle.encode('utf-8')))
+ return handle
+
+ def prepare_tag_prefix(self, prefix):
+ if not prefix:
+ raise EmitterError("tag prefix must not be empty")
+ chunks = []
+ start = end = 0
+ if prefix[0] == u'!':
+ end = 1
+ while end < len(prefix):
+ ch = prefix[end]
+ if u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
+ or ch in u'-;/?!:@&=+$,_.~*\'()[]':
+ end += 1
+ else:
+ if start < end:
+ chunks.append(prefix[start:end])
+ start = end = end+1
+ data = ch.encode('utf-8')
+ for ch in data:
+ chunks.append(u'%%%02X' % ord(ch))
+ if start < end:
+ chunks.append(prefix[start:end])
+ return u''.join(chunks)
+
+ def prepare_tag(self, tag):
+ if not tag:
+ raise EmitterError("tag must not be empty")
+ if tag == u'!':
+ return tag
+ handle = None
+ suffix = tag
+ prefixes = self.tag_prefixes.keys()
+ prefixes.sort()
+ for prefix in prefixes:
+ if tag.startswith(prefix) \
+ and (prefix == u'!' or len(prefix) < len(tag)):
+ handle = self.tag_prefixes[prefix]
+ suffix = tag[len(prefix):]
+ chunks = []
+ start = end = 0
+ while end < len(suffix):
+ ch = suffix[end]
+ if u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
+ or ch in u'-;/?:@&=+$,_.~*\'()[]' \
+ or (ch == u'!' and handle != u'!'):
+ end += 1
+ else:
+ if start < end:
+ chunks.append(suffix[start:end])
+ start = end = end+1
+ data = ch.encode('utf-8')
+ for ch in data:
+ chunks.append(u'%%%02X' % ord(ch))
+ if start < end:
+ chunks.append(suffix[start:end])
+ suffix_text = u''.join(chunks)
+ if handle:
+ return u'%s%s' % (handle, suffix_text)
+ else:
+ return u'!<%s>' % suffix_text
+
+ def prepare_anchor(self, anchor):
+ if not anchor:
+ raise EmitterError("anchor must not be empty")
+ for ch in anchor:
+ if not (u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
+ or ch in u'-_'):
+ raise EmitterError("invalid character %r in the anchor: %r"
+ % (ch.encode('utf-8'), anchor.encode('utf-8')))
+ return anchor
+
+ def analyze_scalar(self, scalar):
+
+ # Empty scalar is a special case.
+ if not scalar:
+ return ScalarAnalysis(scalar=scalar, empty=True, multiline=False,
+ allow_flow_plain=False, allow_block_plain=True,
+ allow_single_quoted=True, allow_double_quoted=True,
+ allow_block=False)
+
+ # Indicators and special characters.
+ block_indicators = False
+ flow_indicators = False
+ line_breaks = False
+ special_characters = False
+
+ # Important whitespace combinations.
+ leading_space = False
+ leading_break = False
+ trailing_space = False
+ trailing_break = False
+ break_space = False
+ space_break = False
+
+ # Check document indicators.
+ if scalar.startswith(u'---') or scalar.startswith(u'...'):
+ block_indicators = True
+ flow_indicators = True
+
+ # First character or preceded by a whitespace.
+ preceeded_by_whitespace = True
+
+ # Last character or followed by a whitespace.
+ followed_by_whitespace = (len(scalar) == 1 or
+ scalar[1] in u'\0 \t\r\n\x85\u2028\u2029')
+
+ # The previous character is a space.
+ previous_space = False
+
+ # The previous character is a break.
+ previous_break = False
+
+ index = 0
+ while index < len(scalar):
+ ch = scalar[index]
+
+ # Check for indicators.
+ if index == 0:
+ # Leading indicators are special characters.
+ if ch in u'#,[]{}&*!|>\'\"%@`':
+ flow_indicators = True
+ block_indicators = True
+ if ch in u'?:':
+ flow_indicators = True
+ if followed_by_whitespace:
+ block_indicators = True
+ if ch == u'-' and followed_by_whitespace:
+ flow_indicators = True
+ block_indicators = True
+ else:
+ # Some indicators cannot appear within a scalar as well.
+ if ch in u',?[]{}':
+ flow_indicators = True
+ if ch == u':':
+ flow_indicators = True
+ if followed_by_whitespace:
+ block_indicators = True
+ if ch == u'#' and preceeded_by_whitespace:
+ flow_indicators = True
+ block_indicators = True
+
+ # Check for line breaks, special, and unicode characters.
+ if ch in u'\n\x85\u2028\u2029':
+ line_breaks = True
+ if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'):
+ if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF'
+ or u'\uE000' <= ch <= u'\uFFFD') and ch != u'\uFEFF':
+ unicode_characters = True
+ if not self.allow_unicode:
+ special_characters = True
+ else:
+ special_characters = True
+
+ # Detect important whitespace combinations.
+ if ch == u' ':
+ if index == 0:
+ leading_space = True
+ if index == len(scalar)-1:
+ trailing_space = True
+ if previous_break:
+ break_space = True
+ previous_space = True
+ previous_break = False
+ elif ch in u'\n\x85\u2028\u2029':
+ if index == 0:
+ leading_break = True
+ if index == len(scalar)-1:
+ trailing_break = True
+ if previous_space:
+ space_break = True
+ previous_space = False
+ previous_break = True
+ else:
+ previous_space = False
+ previous_break = False
+
+ # Prepare for the next character.
+ index += 1
+ preceeded_by_whitespace = (ch in u'\0 \t\r\n\x85\u2028\u2029')
+ followed_by_whitespace = (index+1 >= len(scalar) or
+ scalar[index+1] in u'\0 \t\r\n\x85\u2028\u2029')
+
+ # Let's decide what styles are allowed.
+ allow_flow_plain = True
+ allow_block_plain = True
+ allow_single_quoted = True
+ allow_double_quoted = True
+ allow_block = True
+
+ # Leading and trailing whitespaces are bad for plain scalars.
+ if (leading_space or leading_break
+ or trailing_space or trailing_break):
+ allow_flow_plain = allow_block_plain = False
+
+ # We do not permit trailing spaces for block scalars.
+ if trailing_space:
+ allow_block = False
+
+ # Spaces at the beginning of a new line are only acceptable for block
+ # scalars.
+ if break_space:
+ allow_flow_plain = allow_block_plain = allow_single_quoted = False
+
+ # Spaces followed by breaks, as well as special character are only
+ # allowed for double quoted scalars.
+ if space_break or special_characters:
+ allow_flow_plain = allow_block_plain = \
+ allow_single_quoted = allow_block = False
+
+ # Although the plain scalar writer supports breaks, we never emit
+ # multiline plain scalars.
+ if line_breaks:
+ allow_flow_plain = allow_block_plain = False
+
+ # Flow indicators are forbidden for flow plain scalars.
+ if flow_indicators:
+ allow_flow_plain = False
+
+ # Block indicators are forbidden for block plain scalars.
+ if block_indicators:
+ allow_block_plain = False
+
+ return ScalarAnalysis(scalar=scalar,
+ empty=False, multiline=line_breaks,
+ allow_flow_plain=allow_flow_plain,
+ allow_block_plain=allow_block_plain,
+ allow_single_quoted=allow_single_quoted,
+ allow_double_quoted=allow_double_quoted,
+ allow_block=allow_block)
+
+ # Writers.
+
+ def flush_stream(self):
+ if hasattr(self.stream, 'flush'):
+ self.stream.flush()
+
+ def write_stream_start(self):
+ # Write BOM if needed.
+ if self.encoding and self.encoding.startswith('utf-16'):
+ self.stream.write(u'\uFEFF'.encode(self.encoding))
+
+ def write_stream_end(self):
+ self.flush_stream()
+
+ def write_indicator(self, indicator, need_whitespace,
+ whitespace=False, indention=False):
+ if self.whitespace or not need_whitespace:
+ data = indicator
+ else:
+ data = u' '+indicator
+ self.whitespace = whitespace
+ self.indention = self.indention and indention
+ self.column += len(data)
+ self.open_ended = False
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_indent(self):
+ indent = self.indent or 0
+ if not self.indention or self.column > indent \
+ or (self.column == indent and not self.whitespace):
+ self.write_line_break()
+ if self.column < indent:
+ self.whitespace = True
+ data = u' '*(indent-self.column)
+ self.column = indent
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_line_break(self, data=None):
+ if data is None:
+ data = self.best_line_break
+ self.whitespace = True
+ self.indention = True
+ self.line += 1
+ self.column = 0
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_version_directive(self, version_text):
+ data = u'%%YAML %s' % version_text
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_line_break()
+
+ def write_tag_directive(self, handle_text, prefix_text):
+ data = u'%%TAG %s %s' % (handle_text, prefix_text)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_line_break()
+
+ # Scalar streams.
+
+ def write_single_quoted(self, text, split=True):
+ self.write_indicator(u'\'', True)
+ spaces = False
+ breaks = False
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if spaces:
+ if ch is None or ch != u' ':
+ if start+1 == end and self.column > self.best_width and split \
+ and start != 0 and end != len(text):
+ self.write_indent()
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ elif breaks:
+ if ch is None or ch not in u'\n\x85\u2028\u2029':
+ if text[start] == u'\n':
+ self.write_line_break()
+ for br in text[start:end]:
+ if br == u'\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ self.write_indent()
+ start = end
+ else:
+ if ch is None or ch in u' \n\x85\u2028\u2029' or ch == u'\'':
+ if start < end:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch == u'\'':
+ data = u'\'\''
+ self.column += 2
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end + 1
+ if ch is not None:
+ spaces = (ch == u' ')
+ breaks = (ch in u'\n\x85\u2028\u2029')
+ end += 1
+ self.write_indicator(u'\'', False)
+
+ ESCAPE_REPLACEMENTS = {
+ u'\0': u'0',
+ u'\x07': u'a',
+ u'\x08': u'b',
+ u'\x09': u't',
+ u'\x0A': u'n',
+ u'\x0B': u'v',
+ u'\x0C': u'f',
+ u'\x0D': u'r',
+ u'\x1B': u'e',
+ u'\"': u'\"',
+ u'\\': u'\\',
+ u'\x85': u'N',
+ u'\xA0': u'_',
+ u'\u2028': u'L',
+ u'\u2029': u'P',
+ }
+
+ def write_double_quoted(self, text, split=True):
+ self.write_indicator(u'"', True)
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if ch is None or ch in u'"\\\x85\u2028\u2029\uFEFF' \
+ or not (u'\x20' <= ch <= u'\x7E'
+ or (self.allow_unicode
+ and (u'\xA0' <= ch <= u'\uD7FF'
+ or u'\uE000' <= ch <= u'\uFFFD'))):
+ if start < end:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch is not None:
+ if ch in self.ESCAPE_REPLACEMENTS:
+ data = u'\\'+self.ESCAPE_REPLACEMENTS[ch]
+ elif ch <= u'\xFF':
+ data = u'\\x%02X' % ord(ch)
+ elif ch <= u'\uFFFF':
+ data = u'\\u%04X' % ord(ch)
+ else:
+ data = u'\\U%08X' % ord(ch)
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end+1
+ if 0 < end < len(text)-1 and (ch == u' ' or start >= end) \
+ and self.column+(end-start) > self.best_width and split:
+ data = text[start:end]+u'\\'
+ if start < end:
+ start = end
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ if text[start] == u' ':
+ data = u'\\'
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ end += 1
+ self.write_indicator(u'"', False)
+
+ def determine_block_hints(self, text):
+ hints = u''
+ if text:
+ if text[0] in u' \n\x85\u2028\u2029':
+ hints += unicode(self.best_indent)
+ if text[-1] not in u'\n\x85\u2028\u2029':
+ hints += u'-'
+ elif len(text) == 1 or text[-2] in u'\n\x85\u2028\u2029':
+ hints += u'+'
+ return hints
+
+ def write_folded(self, text):
+ hints = self.determine_block_hints(text)
+ self.write_indicator(u'>'+hints, True)
+ if hints[-1:] == u'+':
+ self.open_ended = True
+ self.write_line_break()
+ leading_space = True
+ spaces = False
+ breaks = True
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if breaks:
+ if ch is None or ch not in u'\n\x85\u2028\u2029':
+ if not leading_space and ch is not None and ch != u' ' \
+ and text[start] == u'\n':
+ self.write_line_break()
+ leading_space = (ch == u' ')
+ for br in text[start:end]:
+ if br == u'\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ if ch is not None:
+ self.write_indent()
+ start = end
+ elif spaces:
+ if ch != u' ':
+ if start+1 == end and self.column > self.best_width:
+ self.write_indent()
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ else:
+ if ch is None or ch in u' \n\x85\u2028\u2029':
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ if ch is None:
+ self.write_line_break()
+ start = end
+ if ch is not None:
+ breaks = (ch in u'\n\x85\u2028\u2029')
+ spaces = (ch == u' ')
+ end += 1
+
+ def write_literal(self, text):
+ hints = self.determine_block_hints(text)
+ self.write_indicator(u'|'+hints, True)
+ if hints[-1:] == u'+':
+ self.open_ended = True
+ self.write_line_break()
+ breaks = True
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if breaks:
+ if ch is None or ch not in u'\n\x85\u2028\u2029':
+ for br in text[start:end]:
+ if br == u'\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ if ch is not None:
+ self.write_indent()
+ start = end
+ else:
+ if ch is None or ch in u'\n\x85\u2028\u2029':
+ data = text[start:end]
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ if ch is None:
+ self.write_line_break()
+ start = end
+ if ch is not None:
+ breaks = (ch in u'\n\x85\u2028\u2029')
+ end += 1
+
+ def write_plain(self, text, split=True):
+ if self.root_context:
+ self.open_ended = True
+ if not text:
+ return
+ if not self.whitespace:
+ data = u' '
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.whitespace = False
+ self.indention = False
+ spaces = False
+ breaks = False
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if spaces:
+ if ch != u' ':
+ if start+1 == end and self.column > self.best_width and split:
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ elif breaks:
+ if ch not in u'\n\x85\u2028\u2029':
+ if text[start] == u'\n':
+ self.write_line_break()
+ for br in text[start:end]:
+ if br == u'\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ start = end
+ else:
+ if ch is None or ch in u' \n\x85\u2028\u2029':
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch is not None:
+ spaces = (ch == u' ')
+ breaks = (ch in u'\n\x85\u2028\u2029')
+ end += 1
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/error.py b/collectors/python.d.plugin/python_modules/pyyaml2/error.py
new file mode 100644
index 0000000..5466be7
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/error.py
@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError']
+
+class Mark(object):
+
+ def __init__(self, name, index, line, column, buffer, pointer):
+ self.name = name
+ self.index = index
+ self.line = line
+ self.column = column
+ self.buffer = buffer
+ self.pointer = pointer
+
+ def get_snippet(self, indent=4, max_length=75):
+ if self.buffer is None:
+ return None
+ head = ''
+ start = self.pointer
+ while start > 0 and self.buffer[start-1] not in u'\0\r\n\x85\u2028\u2029':
+ start -= 1
+ if self.pointer-start > max_length/2-1:
+ head = ' ... '
+ start += 5
+ break
+ tail = ''
+ end = self.pointer
+ while end < len(self.buffer) and self.buffer[end] not in u'\0\r\n\x85\u2028\u2029':
+ end += 1
+ if end-self.pointer > max_length/2-1:
+ tail = ' ... '
+ end -= 5
+ break
+ snippet = self.buffer[start:end].encode('utf-8')
+ return ' '*indent + head + snippet + tail + '\n' \
+ + ' '*(indent+self.pointer-start+len(head)) + '^'
+
+ def __str__(self):
+ snippet = self.get_snippet()
+ where = " in \"%s\", line %d, column %d" \
+ % (self.name, self.line+1, self.column+1)
+ if snippet is not None:
+ where += ":\n"+snippet
+ return where
+
+class YAMLError(Exception):
+ pass
+
+class MarkedYAMLError(YAMLError):
+
+ def __init__(self, context=None, context_mark=None,
+ problem=None, problem_mark=None, note=None):
+ self.context = context
+ self.context_mark = context_mark
+ self.problem = problem
+ self.problem_mark = problem_mark
+ self.note = note
+
+ def __str__(self):
+ lines = []
+ if self.context is not None:
+ lines.append(self.context)
+ if self.context_mark is not None \
+ and (self.problem is None or self.problem_mark is None
+ or self.context_mark.name != self.problem_mark.name
+ or self.context_mark.line != self.problem_mark.line
+ or self.context_mark.column != self.problem_mark.column):
+ lines.append(str(self.context_mark))
+ if self.problem is not None:
+ lines.append(self.problem)
+ if self.problem_mark is not None:
+ lines.append(str(self.problem_mark))
+ if self.note is not None:
+ lines.append(self.note)
+ return '\n'.join(lines)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/events.py b/collectors/python.d.plugin/python_modules/pyyaml2/events.py
new file mode 100644
index 0000000..283452a
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/events.py
@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: MIT
+
+# Abstract classes.
+
+class Event(object):
+ def __init__(self, start_mark=None, end_mark=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ attributes = [key for key in ['anchor', 'tag', 'implicit', 'value']
+ if hasattr(self, key)]
+ arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+ for key in attributes])
+ return '%s(%s)' % (self.__class__.__name__, arguments)
+
+class NodeEvent(Event):
+ def __init__(self, anchor, start_mark=None, end_mark=None):
+ self.anchor = anchor
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class CollectionStartEvent(NodeEvent):
+ def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None,
+ flow_style=None):
+ self.anchor = anchor
+ self.tag = tag
+ self.implicit = implicit
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.flow_style = flow_style
+
+class CollectionEndEvent(Event):
+ pass
+
+# Implementations.
+
+class StreamStartEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None, encoding=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.encoding = encoding
+
+class StreamEndEvent(Event):
+ pass
+
+class DocumentStartEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None,
+ explicit=None, version=None, tags=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.explicit = explicit
+ self.version = version
+ self.tags = tags
+
+class DocumentEndEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None,
+ explicit=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.explicit = explicit
+
+class AliasEvent(NodeEvent):
+ pass
+
+class ScalarEvent(NodeEvent):
+ def __init__(self, anchor, tag, implicit, value,
+ start_mark=None, end_mark=None, style=None):
+ self.anchor = anchor
+ self.tag = tag
+ self.implicit = implicit
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
+class SequenceStartEvent(CollectionStartEvent):
+ pass
+
+class SequenceEndEvent(CollectionEndEvent):
+ pass
+
+class MappingStartEvent(CollectionStartEvent):
+ pass
+
+class MappingEndEvent(CollectionEndEvent):
+ pass
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/loader.py b/collectors/python.d.plugin/python_modules/pyyaml2/loader.py
new file mode 100644
index 0000000..1c19553
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/loader.py
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseLoader', 'SafeLoader', 'Loader']
+
+from reader import *
+from scanner import *
+from parser import *
+from composer import *
+from constructor import *
+from resolver import *
+
+class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ BaseConstructor.__init__(self)
+ BaseResolver.__init__(self)
+
+class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ SafeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ Constructor.__init__(self)
+ Resolver.__init__(self)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/nodes.py b/collectors/python.d.plugin/python_modules/pyyaml2/nodes.py
new file mode 100644
index 0000000..ed2a1b4
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/nodes.py
@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: MIT
+
+class Node(object):
+ def __init__(self, tag, value, start_mark, end_mark):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ value = self.value
+ #if isinstance(value, list):
+ # if len(value) == 0:
+ # value = '<empty>'
+ # elif len(value) == 1:
+ # value = '<1 item>'
+ # else:
+ # value = '<%d items>' % len(value)
+ #else:
+ # if len(value) > 75:
+ # value = repr(value[:70]+u' ... ')
+ # else:
+ # value = repr(value)
+ value = repr(value)
+ return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value)
+
+class ScalarNode(Node):
+ id = 'scalar'
+ def __init__(self, tag, value,
+ start_mark=None, end_mark=None, style=None):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
+class CollectionNode(Node):
+ def __init__(self, tag, value,
+ start_mark=None, end_mark=None, flow_style=None):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.flow_style = flow_style
+
+class SequenceNode(CollectionNode):
+ id = 'sequence'
+
+class MappingNode(CollectionNode):
+ id = 'mapping'
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/parser.py b/collectors/python.d.plugin/python_modules/pyyaml2/parser.py
new file mode 100644
index 0000000..97ba083
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/parser.py
@@ -0,0 +1,590 @@
+# SPDX-License-Identifier: MIT
+
+# The following YAML grammar is LL(1) and is parsed by a recursive descent
+# parser.
+#
+# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
+# implicit_document ::= block_node DOCUMENT-END*
+# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
+# block_node_or_indentless_sequence ::=
+# ALIAS
+# | properties (block_content | indentless_block_sequence)?
+# | block_content
+# | indentless_block_sequence
+# block_node ::= ALIAS
+# | properties block_content?
+# | block_content
+# flow_node ::= ALIAS
+# | properties flow_content?
+# | flow_content
+# properties ::= TAG ANCHOR? | ANCHOR TAG?
+# block_content ::= block_collection | flow_collection | SCALAR
+# flow_content ::= flow_collection | SCALAR
+# block_collection ::= block_sequence | block_mapping
+# flow_collection ::= flow_sequence | flow_mapping
+# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+# indentless_sequence ::= (BLOCK-ENTRY block_node?)+
+# block_mapping ::= BLOCK-MAPPING_START
+# ((KEY block_node_or_indentless_sequence?)?
+# (VALUE block_node_or_indentless_sequence?)?)*
+# BLOCK-END
+# flow_sequence ::= FLOW-SEQUENCE-START
+# (flow_sequence_entry FLOW-ENTRY)*
+# flow_sequence_entry?
+# FLOW-SEQUENCE-END
+# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+# flow_mapping ::= FLOW-MAPPING-START
+# (flow_mapping_entry FLOW-ENTRY)*
+# flow_mapping_entry?
+# FLOW-MAPPING-END
+# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+#
+# FIRST sets:
+#
+# stream: { STREAM-START }
+# explicit_document: { DIRECTIVE DOCUMENT-START }
+# implicit_document: FIRST(block_node)
+# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_sequence: { BLOCK-SEQUENCE-START }
+# block_mapping: { BLOCK-MAPPING-START }
+# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
+# indentless_sequence: { ENTRY }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_sequence: { FLOW-SEQUENCE-START }
+# flow_mapping: { FLOW-MAPPING-START }
+# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+
+__all__ = ['Parser', 'ParserError']
+
+from error import MarkedYAMLError
+from tokens import *
+from events import *
+from scanner import *
+
+class ParserError(MarkedYAMLError):
+ pass
+
+class Parser(object):
+ # Since writing a recursive-descendant parser is a straightforward task, we
+ # do not give many comments here.
+
+ DEFAULT_TAGS = {
+ u'!': u'!',
+ u'!!': u'tag:yaml.org,2002:',
+ }
+
+ def __init__(self):
+ self.current_event = None
+ self.yaml_version = None
+ self.tag_handles = {}
+ self.states = []
+ self.marks = []
+ self.state = self.parse_stream_start
+
+ def dispose(self):
+ # Reset the state attributes (to clear self-references)
+ self.states = []
+ self.state = None
+
+ def check_event(self, *choices):
+ # Check the type of the next event.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ if self.current_event is not None:
+ if not choices:
+ return True
+ for choice in choices:
+ if isinstance(self.current_event, choice):
+ return True
+ return False
+
+ def peek_event(self):
+ # Get the next event.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ return self.current_event
+
+ def get_event(self):
+ # Get the next event and proceed further.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ value = self.current_event
+ self.current_event = None
+ return value
+
+ # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
+ # implicit_document ::= block_node DOCUMENT-END*
+ # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
+
+ def parse_stream_start(self):
+
+ # Parse the stream start.
+ token = self.get_token()
+ event = StreamStartEvent(token.start_mark, token.end_mark,
+ encoding=token.encoding)
+
+ # Prepare the next state.
+ self.state = self.parse_implicit_document_start
+
+ return event
+
+ def parse_implicit_document_start(self):
+
+ # Parse an implicit document.
+ if not self.check_token(DirectiveToken, DocumentStartToken,
+ StreamEndToken):
+ self.tag_handles = self.DEFAULT_TAGS
+ token = self.peek_token()
+ start_mark = end_mark = token.start_mark
+ event = DocumentStartEvent(start_mark, end_mark,
+ explicit=False)
+
+ # Prepare the next state.
+ self.states.append(self.parse_document_end)
+ self.state = self.parse_block_node
+
+ return event
+
+ else:
+ return self.parse_document_start()
+
+ def parse_document_start(self):
+
+ # Parse any extra document end indicators.
+ while self.check_token(DocumentEndToken):
+ self.get_token()
+
+ # Parse an explicit document.
+ if not self.check_token(StreamEndToken):
+ token = self.peek_token()
+ start_mark = token.start_mark
+ version, tags = self.process_directives()
+ if not self.check_token(DocumentStartToken):
+ raise ParserError(None, None,
+ "expected '<document start>', but found %r"
+ % self.peek_token().id,
+ self.peek_token().start_mark)
+ token = self.get_token()
+ end_mark = token.end_mark
+ event = DocumentStartEvent(start_mark, end_mark,
+ explicit=True, version=version, tags=tags)
+ self.states.append(self.parse_document_end)
+ self.state = self.parse_document_content
+ else:
+ # Parse the end of the stream.
+ token = self.get_token()
+ event = StreamEndEvent(token.start_mark, token.end_mark)
+ assert not self.states
+ assert not self.marks
+ self.state = None
+ return event
+
+ def parse_document_end(self):
+
+ # Parse the document end.
+ token = self.peek_token()
+ start_mark = end_mark = token.start_mark
+ explicit = False
+ if self.check_token(DocumentEndToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ explicit = True
+ event = DocumentEndEvent(start_mark, end_mark,
+ explicit=explicit)
+
+ # Prepare the next state.
+ self.state = self.parse_document_start
+
+ return event
+
+ def parse_document_content(self):
+ if self.check_token(DirectiveToken,
+ DocumentStartToken, DocumentEndToken, StreamEndToken):
+ event = self.process_empty_scalar(self.peek_token().start_mark)
+ self.state = self.states.pop()
+ return event
+ else:
+ return self.parse_block_node()
+
+ def process_directives(self):
+ self.yaml_version = None
+ self.tag_handles = {}
+ while self.check_token(DirectiveToken):
+ token = self.get_token()
+ if token.name == u'YAML':
+ if self.yaml_version is not None:
+ raise ParserError(None, None,
+ "found duplicate YAML directive", token.start_mark)
+ major, minor = token.value
+ if major != 1:
+ raise ParserError(None, None,
+ "found incompatible YAML document (version 1.* is required)",
+ token.start_mark)
+ self.yaml_version = token.value
+ elif token.name == u'TAG':
+ handle, prefix = token.value
+ if handle in self.tag_handles:
+ raise ParserError(None, None,
+ "duplicate tag handle %r" % handle.encode('utf-8'),
+ token.start_mark)
+ self.tag_handles[handle] = prefix
+ if self.tag_handles:
+ value = self.yaml_version, self.tag_handles.copy()
+ else:
+ value = self.yaml_version, None
+ for key in self.DEFAULT_TAGS:
+ if key not in self.tag_handles:
+ self.tag_handles[key] = self.DEFAULT_TAGS[key]
+ return value
+
+ # block_node_or_indentless_sequence ::= ALIAS
+ # | properties (block_content | indentless_block_sequence)?
+ # | block_content
+ # | indentless_block_sequence
+ # block_node ::= ALIAS
+ # | properties block_content?
+ # | block_content
+ # flow_node ::= ALIAS
+ # | properties flow_content?
+ # | flow_content
+ # properties ::= TAG ANCHOR? | ANCHOR TAG?
+ # block_content ::= block_collection | flow_collection | SCALAR
+ # flow_content ::= flow_collection | SCALAR
+ # block_collection ::= block_sequence | block_mapping
+ # flow_collection ::= flow_sequence | flow_mapping
+
+ def parse_block_node(self):
+ return self.parse_node(block=True)
+
+ def parse_flow_node(self):
+ return self.parse_node()
+
+ def parse_block_node_or_indentless_sequence(self):
+ return self.parse_node(block=True, indentless_sequence=True)
+
+ def parse_node(self, block=False, indentless_sequence=False):
+ if self.check_token(AliasToken):
+ token = self.get_token()
+ event = AliasEvent(token.value, token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ else:
+ anchor = None
+ tag = None
+ start_mark = end_mark = tag_mark = None
+ if self.check_token(AnchorToken):
+ token = self.get_token()
+ start_mark = token.start_mark
+ end_mark = token.end_mark
+ anchor = token.value
+ if self.check_token(TagToken):
+ token = self.get_token()
+ tag_mark = token.start_mark
+ end_mark = token.end_mark
+ tag = token.value
+ elif self.check_token(TagToken):
+ token = self.get_token()
+ start_mark = tag_mark = token.start_mark
+ end_mark = token.end_mark
+ tag = token.value
+ if self.check_token(AnchorToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ anchor = token.value
+ if tag is not None:
+ handle, suffix = tag
+ if handle is not None:
+ if handle not in self.tag_handles:
+ raise ParserError("while parsing a node", start_mark,
+ "found undefined tag handle %r" % handle.encode('utf-8'),
+ tag_mark)
+ tag = self.tag_handles[handle]+suffix
+ else:
+ tag = suffix
+ #if tag == u'!':
+ # raise ParserError("while parsing a node", start_mark,
+ # "found non-specific tag '!'", tag_mark,
+ # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
+ if start_mark is None:
+ start_mark = end_mark = self.peek_token().start_mark
+ event = None
+ implicit = (tag is None or tag == u'!')
+ if indentless_sequence and self.check_token(BlockEntryToken):
+ end_mark = self.peek_token().end_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark)
+ self.state = self.parse_indentless_sequence_entry
+ else:
+ if self.check_token(ScalarToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ if (token.plain and tag is None) or tag == u'!':
+ implicit = (True, False)
+ elif tag is None:
+ implicit = (False, True)
+ else:
+ implicit = (False, False)
+ event = ScalarEvent(anchor, tag, implicit, token.value,
+ start_mark, end_mark, style=token.style)
+ self.state = self.states.pop()
+ elif self.check_token(FlowSequenceStartToken):
+ end_mark = self.peek_token().end_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=True)
+ self.state = self.parse_flow_sequence_first_entry
+ elif self.check_token(FlowMappingStartToken):
+ end_mark = self.peek_token().end_mark
+ event = MappingStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=True)
+ self.state = self.parse_flow_mapping_first_key
+ elif block and self.check_token(BlockSequenceStartToken):
+ end_mark = self.peek_token().start_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=False)
+ self.state = self.parse_block_sequence_first_entry
+ elif block and self.check_token(BlockMappingStartToken):
+ end_mark = self.peek_token().start_mark
+ event = MappingStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=False)
+ self.state = self.parse_block_mapping_first_key
+ elif anchor is not None or tag is not None:
+ # Empty scalars are allowed even if a tag or an anchor is
+ # specified.
+ event = ScalarEvent(anchor, tag, (implicit, False), u'',
+ start_mark, end_mark)
+ self.state = self.states.pop()
+ else:
+ if block:
+ node = 'block'
+ else:
+ node = 'flow'
+ token = self.peek_token()
+ raise ParserError("while parsing a %s node" % node, start_mark,
+ "expected the node content, but found %r" % token.id,
+ token.start_mark)
+ return event
+
+ # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+
+ def parse_block_sequence_first_entry(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_block_sequence_entry()
+
+ def parse_block_sequence_entry(self):
+ if self.check_token(BlockEntryToken):
+ token = self.get_token()
+ if not self.check_token(BlockEntryToken, BlockEndToken):
+ self.states.append(self.parse_block_sequence_entry)
+ return self.parse_block_node()
+ else:
+ self.state = self.parse_block_sequence_entry
+ return self.process_empty_scalar(token.end_mark)
+ if not self.check_token(BlockEndToken):
+ token = self.peek_token()
+ raise ParserError("while parsing a block collection", self.marks[-1],
+ "expected <block end>, but found %r" % token.id, token.start_mark)
+ token = self.get_token()
+ event = SequenceEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ # indentless_sequence ::= (BLOCK-ENTRY block_node?)+
+
+ def parse_indentless_sequence_entry(self):
+ if self.check_token(BlockEntryToken):
+ token = self.get_token()
+ if not self.check_token(BlockEntryToken,
+ KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_indentless_sequence_entry)
+ return self.parse_block_node()
+ else:
+ self.state = self.parse_indentless_sequence_entry
+ return self.process_empty_scalar(token.end_mark)
+ token = self.peek_token()
+ event = SequenceEndEvent(token.start_mark, token.start_mark)
+ self.state = self.states.pop()
+ return event
+
+ # block_mapping ::= BLOCK-MAPPING_START
+ # ((KEY block_node_or_indentless_sequence?)?
+ # (VALUE block_node_or_indentless_sequence?)?)*
+ # BLOCK-END
+
+ def parse_block_mapping_first_key(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_block_mapping_key()
+
+ def parse_block_mapping_key(self):
+ if self.check_token(KeyToken):
+ token = self.get_token()
+ if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_block_mapping_value)
+ return self.parse_block_node_or_indentless_sequence()
+ else:
+ self.state = self.parse_block_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+ if not self.check_token(BlockEndToken):
+ token = self.peek_token()
+ raise ParserError("while parsing a block mapping", self.marks[-1],
+ "expected <block end>, but found %r" % token.id, token.start_mark)
+ token = self.get_token()
+ event = MappingEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_block_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_block_mapping_key)
+ return self.parse_block_node_or_indentless_sequence()
+ else:
+ self.state = self.parse_block_mapping_key
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_block_mapping_key
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ # flow_sequence ::= FLOW-SEQUENCE-START
+ # (flow_sequence_entry FLOW-ENTRY)*
+ # flow_sequence_entry?
+ # FLOW-SEQUENCE-END
+ # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+ #
+ # Note that while production rules for both flow_sequence_entry and
+ # flow_mapping_entry are equal, their interpretations are different.
+ # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
+ # generate an inline mapping (set syntax).
+
+ def parse_flow_sequence_first_entry(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_flow_sequence_entry(first=True)
+
+ def parse_flow_sequence_entry(self, first=False):
+ if not self.check_token(FlowSequenceEndToken):
+ if not first:
+ if self.check_token(FlowEntryToken):
+ self.get_token()
+ else:
+ token = self.peek_token()
+ raise ParserError("while parsing a flow sequence", self.marks[-1],
+ "expected ',' or ']', but got %r" % token.id, token.start_mark)
+
+ if self.check_token(KeyToken):
+ token = self.peek_token()
+ event = MappingStartEvent(None, None, True,
+ token.start_mark, token.end_mark,
+ flow_style=True)
+ self.state = self.parse_flow_sequence_entry_mapping_key
+ return event
+ elif not self.check_token(FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry)
+ return self.parse_flow_node()
+ token = self.get_token()
+ event = SequenceEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_flow_sequence_entry_mapping_key(self):
+ token = self.get_token()
+ if not self.check_token(ValueToken,
+ FlowEntryToken, FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry_mapping_value)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+
+ def parse_flow_sequence_entry_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry_mapping_end)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_end
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_end
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ def parse_flow_sequence_entry_mapping_end(self):
+ self.state = self.parse_flow_sequence_entry
+ token = self.peek_token()
+ return MappingEndEvent(token.start_mark, token.start_mark)
+
+ # flow_mapping ::= FLOW-MAPPING-START
+ # (flow_mapping_entry FLOW-ENTRY)*
+ # flow_mapping_entry?
+ # FLOW-MAPPING-END
+ # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+
+ def parse_flow_mapping_first_key(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_flow_mapping_key(first=True)
+
+ def parse_flow_mapping_key(self, first=False):
+ if not self.check_token(FlowMappingEndToken):
+ if not first:
+ if self.check_token(FlowEntryToken):
+ self.get_token()
+ else:
+ token = self.peek_token()
+ raise ParserError("while parsing a flow mapping", self.marks[-1],
+ "expected ',' or '}', but got %r" % token.id, token.start_mark)
+ if self.check_token(KeyToken):
+ token = self.get_token()
+ if not self.check_token(ValueToken,
+ FlowEntryToken, FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_value)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+ elif not self.check_token(FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_empty_value)
+ return self.parse_flow_node()
+ token = self.get_token()
+ event = MappingEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_flow_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(FlowEntryToken, FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_key)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_mapping_key
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_flow_mapping_key
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ def parse_flow_mapping_empty_value(self):
+ self.state = self.parse_flow_mapping_key
+ return self.process_empty_scalar(self.peek_token().start_mark)
+
+ def process_empty_scalar(self, mark):
+ return ScalarEvent(None, None, (True, False), u'', mark, mark)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/reader.py b/collectors/python.d.plugin/python_modules/pyyaml2/reader.py
new file mode 100644
index 0000000..8d42295
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/reader.py
@@ -0,0 +1,191 @@
+# SPDX-License-Identifier: MIT
+# This module contains abstractions for the input stream. You don't have to
+# looks further, there are no pretty code.
+#
+# We define two classes here.
+#
+# Mark(source, line, column)
+# It's just a record and its only use is producing nice error messages.
+# Parser does not use it for any other purposes.
+#
+# Reader(source, data)
+# Reader determines the encoding of `data` and converts it to unicode.
+# Reader provides the following methods and attributes:
+# reader.peek(length=1) - return the next `length` characters
+# reader.forward(length=1) - move the current position to `length` characters.
+# reader.index - the number of the current character.
+# reader.line, stream.column - the line and the column of the current character.
+
+__all__ = ['Reader', 'ReaderError']
+
+from error import YAMLError, Mark
+
+import codecs, re
+
+class ReaderError(YAMLError):
+
+ def __init__(self, name, position, character, encoding, reason):
+ self.name = name
+ self.character = character
+ self.position = position
+ self.encoding = encoding
+ self.reason = reason
+
+ def __str__(self):
+ if isinstance(self.character, str):
+ return "'%s' codec can't decode byte #x%02x: %s\n" \
+ " in \"%s\", position %d" \
+ % (self.encoding, ord(self.character), self.reason,
+ self.name, self.position)
+ else:
+ return "unacceptable character #x%04x: %s\n" \
+ " in \"%s\", position %d" \
+ % (self.character, self.reason,
+ self.name, self.position)
+
+class Reader(object):
+ # Reader:
+ # - determines the data encoding and converts it to unicode,
+ # - checks if characters are in allowed range,
+ # - adds '\0' to the end.
+
+ # Reader accepts
+ # - a `str` object,
+ # - a `unicode` object,
+ # - a file-like object with its `read` method returning `str`,
+ # - a file-like object with its `read` method returning `unicode`.
+
+ # Yeah, it's ugly and slow.
+
+ def __init__(self, stream):
+ self.name = None
+ self.stream = None
+ self.stream_pointer = 0
+ self.eof = True
+ self.buffer = u''
+ self.pointer = 0
+ self.raw_buffer = None
+ self.raw_decode = None
+ self.encoding = None
+ self.index = 0
+ self.line = 0
+ self.column = 0
+ if isinstance(stream, unicode):
+ self.name = "<unicode string>"
+ self.check_printable(stream)
+ self.buffer = stream+u'\0'
+ elif isinstance(stream, str):
+ self.name = "<string>"
+ self.raw_buffer = stream
+ self.determine_encoding()
+ else:
+ self.stream = stream
+ self.name = getattr(stream, 'name', "<file>")
+ self.eof = False
+ self.raw_buffer = ''
+ self.determine_encoding()
+
+ def peek(self, index=0):
+ try:
+ return self.buffer[self.pointer+index]
+ except IndexError:
+ self.update(index+1)
+ return self.buffer[self.pointer+index]
+
+ def prefix(self, length=1):
+ if self.pointer+length >= len(self.buffer):
+ self.update(length)
+ return self.buffer[self.pointer:self.pointer+length]
+
+ def forward(self, length=1):
+ if self.pointer+length+1 >= len(self.buffer):
+ self.update(length+1)
+ while length:
+ ch = self.buffer[self.pointer]
+ self.pointer += 1
+ self.index += 1
+ if ch in u'\n\x85\u2028\u2029' \
+ or (ch == u'\r' and self.buffer[self.pointer] != u'\n'):
+ self.line += 1
+ self.column = 0
+ elif ch != u'\uFEFF':
+ self.column += 1
+ length -= 1
+
+ def get_mark(self):
+ if self.stream is None:
+ return Mark(self.name, self.index, self.line, self.column,
+ self.buffer, self.pointer)
+ else:
+ return Mark(self.name, self.index, self.line, self.column,
+ None, None)
+
+ def determine_encoding(self):
+ while not self.eof and len(self.raw_buffer) < 2:
+ self.update_raw()
+ if not isinstance(self.raw_buffer, unicode):
+ if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
+ self.raw_decode = codecs.utf_16_le_decode
+ self.encoding = 'utf-16-le'
+ elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
+ self.raw_decode = codecs.utf_16_be_decode
+ self.encoding = 'utf-16-be'
+ else:
+ self.raw_decode = codecs.utf_8_decode
+ self.encoding = 'utf-8'
+ self.update(1)
+
+ NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]')
+ def check_printable(self, data):
+ match = self.NON_PRINTABLE.search(data)
+ if match:
+ character = match.group()
+ position = self.index+(len(self.buffer)-self.pointer)+match.start()
+ raise ReaderError(self.name, position, ord(character),
+ 'unicode', "special characters are not allowed")
+
+ def update(self, length):
+ if self.raw_buffer is None:
+ return
+ self.buffer = self.buffer[self.pointer:]
+ self.pointer = 0
+ while len(self.buffer) < length:
+ if not self.eof:
+ self.update_raw()
+ if self.raw_decode is not None:
+ try:
+ data, converted = self.raw_decode(self.raw_buffer,
+ 'strict', self.eof)
+ except UnicodeDecodeError, exc:
+ character = exc.object[exc.start]
+ if self.stream is not None:
+ position = self.stream_pointer-len(self.raw_buffer)+exc.start
+ else:
+ position = exc.start
+ raise ReaderError(self.name, position, character,
+ exc.encoding, exc.reason)
+ else:
+ data = self.raw_buffer
+ converted = len(data)
+ self.check_printable(data)
+ self.buffer += data
+ self.raw_buffer = self.raw_buffer[converted:]
+ if self.eof:
+ self.buffer += u'\0'
+ self.raw_buffer = None
+ break
+
+ def update_raw(self, size=1024):
+ data = self.stream.read(size)
+ if data:
+ self.raw_buffer += data
+ self.stream_pointer += len(data)
+ else:
+ self.eof = True
+
+#try:
+# import psyco
+# psyco.bind(Reader)
+#except ImportError:
+# pass
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/representer.py b/collectors/python.d.plugin/python_modules/pyyaml2/representer.py
new file mode 100644
index 0000000..0a1404e
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/representer.py
@@ -0,0 +1,485 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer',
+ 'RepresenterError']
+
+from error import *
+from nodes import *
+
+import datetime
+
+import sys, copy_reg, types
+
+class RepresenterError(YAMLError):
+ pass
+
+class BaseRepresenter(object):
+
+ yaml_representers = {}
+ yaml_multi_representers = {}
+
+ def __init__(self, default_style=None, default_flow_style=None):
+ self.default_style = default_style
+ self.default_flow_style = default_flow_style
+ self.represented_objects = {}
+ self.object_keeper = []
+ self.alias_key = None
+
+ def represent(self, data):
+ node = self.represent_data(data)
+ self.serialize(node)
+ self.represented_objects = {}
+ self.object_keeper = []
+ self.alias_key = None
+
+ def get_classobj_bases(self, cls):
+ bases = [cls]
+ for base in cls.__bases__:
+ bases.extend(self.get_classobj_bases(base))
+ return bases
+
+ def represent_data(self, data):
+ if self.ignore_aliases(data):
+ self.alias_key = None
+ else:
+ self.alias_key = id(data)
+ if self.alias_key is not None:
+ if self.alias_key in self.represented_objects:
+ node = self.represented_objects[self.alias_key]
+ #if node is None:
+ # raise RepresenterError("recursive objects are not allowed: %r" % data)
+ return node
+ #self.represented_objects[alias_key] = None
+ self.object_keeper.append(data)
+ data_types = type(data).__mro__
+ if type(data) is types.InstanceType:
+ data_types = self.get_classobj_bases(data.__class__)+list(data_types)
+ if data_types[0] in self.yaml_representers:
+ node = self.yaml_representers[data_types[0]](self, data)
+ else:
+ for data_type in data_types:
+ if data_type in self.yaml_multi_representers:
+ node = self.yaml_multi_representers[data_type](self, data)
+ break
+ else:
+ if None in self.yaml_multi_representers:
+ node = self.yaml_multi_representers[None](self, data)
+ elif None in self.yaml_representers:
+ node = self.yaml_representers[None](self, data)
+ else:
+ node = ScalarNode(None, unicode(data))
+ #if alias_key is not None:
+ # self.represented_objects[alias_key] = node
+ return node
+
+ def add_representer(cls, data_type, representer):
+ if not 'yaml_representers' in cls.__dict__:
+ cls.yaml_representers = cls.yaml_representers.copy()
+ cls.yaml_representers[data_type] = representer
+ add_representer = classmethod(add_representer)
+
+ def add_multi_representer(cls, data_type, representer):
+ if not 'yaml_multi_representers' in cls.__dict__:
+ cls.yaml_multi_representers = cls.yaml_multi_representers.copy()
+ cls.yaml_multi_representers[data_type] = representer
+ add_multi_representer = classmethod(add_multi_representer)
+
+ def represent_scalar(self, tag, value, style=None):
+ if style is None:
+ style = self.default_style
+ node = ScalarNode(tag, value, style=style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ return node
+
+ def represent_sequence(self, tag, sequence, flow_style=None):
+ value = []
+ node = SequenceNode(tag, value, flow_style=flow_style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ best_style = True
+ for item in sequence:
+ node_item = self.represent_data(item)
+ if not (isinstance(node_item, ScalarNode) and not node_item.style):
+ best_style = False
+ value.append(node_item)
+ if flow_style is None:
+ if self.default_flow_style is not None:
+ node.flow_style = self.default_flow_style
+ else:
+ node.flow_style = best_style
+ return node
+
+ def represent_mapping(self, tag, mapping, flow_style=None):
+ value = []
+ node = MappingNode(tag, value, flow_style=flow_style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ best_style = True
+ if hasattr(mapping, 'items'):
+ mapping = mapping.items()
+ mapping.sort()
+ for item_key, item_value in mapping:
+ node_key = self.represent_data(item_key)
+ node_value = self.represent_data(item_value)
+ if not (isinstance(node_key, ScalarNode) and not node_key.style):
+ best_style = False
+ if not (isinstance(node_value, ScalarNode) and not node_value.style):
+ best_style = False
+ value.append((node_key, node_value))
+ if flow_style is None:
+ if self.default_flow_style is not None:
+ node.flow_style = self.default_flow_style
+ else:
+ node.flow_style = best_style
+ return node
+
+ def ignore_aliases(self, data):
+ return False
+
+class SafeRepresenter(BaseRepresenter):
+
+ def ignore_aliases(self, data):
+ if data in [None, ()]:
+ return True
+ if isinstance(data, (str, unicode, bool, int, float)):
+ return True
+
+ def represent_none(self, data):
+ return self.represent_scalar(u'tag:yaml.org,2002:null',
+ u'null')
+
+ def represent_str(self, data):
+ tag = None
+ style = None
+ try:
+ data = unicode(data, 'ascii')
+ tag = u'tag:yaml.org,2002:str'
+ except UnicodeDecodeError:
+ try:
+ data = unicode(data, 'utf-8')
+ tag = u'tag:yaml.org,2002:str'
+ except UnicodeDecodeError:
+ data = data.encode('base64')
+ tag = u'tag:yaml.org,2002:binary'
+ style = '|'
+ return self.represent_scalar(tag, data, style=style)
+
+ def represent_unicode(self, data):
+ return self.represent_scalar(u'tag:yaml.org,2002:str', data)
+
+ def represent_bool(self, data):
+ if data:
+ value = u'true'
+ else:
+ value = u'false'
+ return self.represent_scalar(u'tag:yaml.org,2002:bool', value)
+
+ def represent_int(self, data):
+ return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data))
+
+ def represent_long(self, data):
+ return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data))
+
+ inf_value = 1e300
+ while repr(inf_value) != repr(inf_value*inf_value):
+ inf_value *= inf_value
+
+ def represent_float(self, data):
+ if data != data or (data == 0.0 and data == 1.0):
+ value = u'.nan'
+ elif data == self.inf_value:
+ value = u'.inf'
+ elif data == -self.inf_value:
+ value = u'-.inf'
+ else:
+ value = unicode(repr(data)).lower()
+ # Note that in some cases `repr(data)` represents a float number
+ # without the decimal parts. For instance:
+ # >>> repr(1e17)
+ # '1e17'
+ # Unfortunately, this is not a valid float representation according
+ # to the definition of the `!!float` tag. We fix this by adding
+ # '.0' before the 'e' symbol.
+ if u'.' not in value and u'e' in value:
+ value = value.replace(u'e', u'.0e', 1)
+ return self.represent_scalar(u'tag:yaml.org,2002:float', value)
+
+ def represent_list(self, data):
+ #pairs = (len(data) > 0 and isinstance(data, list))
+ #if pairs:
+ # for item in data:
+ # if not isinstance(item, tuple) or len(item) != 2:
+ # pairs = False
+ # break
+ #if not pairs:
+ return self.represent_sequence(u'tag:yaml.org,2002:seq', data)
+ #value = []
+ #for item_key, item_value in data:
+ # value.append(self.represent_mapping(u'tag:yaml.org,2002:map',
+ # [(item_key, item_value)]))
+ #return SequenceNode(u'tag:yaml.org,2002:pairs', value)
+
+ def represent_dict(self, data):
+ return self.represent_mapping(u'tag:yaml.org,2002:map', data)
+
+ def represent_set(self, data):
+ value = {}
+ for key in data:
+ value[key] = None
+ return self.represent_mapping(u'tag:yaml.org,2002:set', value)
+
+ def represent_date(self, data):
+ value = unicode(data.isoformat())
+ return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value)
+
+ def represent_datetime(self, data):
+ value = unicode(data.isoformat(' '))
+ return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value)
+
+ def represent_yaml_object(self, tag, data, cls, flow_style=None):
+ if hasattr(data, '__getstate__'):
+ state = data.__getstate__()
+ else:
+ state = data.__dict__.copy()
+ return self.represent_mapping(tag, state, flow_style=flow_style)
+
+ def represent_undefined(self, data):
+ raise RepresenterError("cannot represent an object: %s" % data)
+
+SafeRepresenter.add_representer(type(None),
+ SafeRepresenter.represent_none)
+
+SafeRepresenter.add_representer(str,
+ SafeRepresenter.represent_str)
+
+SafeRepresenter.add_representer(unicode,
+ SafeRepresenter.represent_unicode)
+
+SafeRepresenter.add_representer(bool,
+ SafeRepresenter.represent_bool)
+
+SafeRepresenter.add_representer(int,
+ SafeRepresenter.represent_int)
+
+SafeRepresenter.add_representer(long,
+ SafeRepresenter.represent_long)
+
+SafeRepresenter.add_representer(float,
+ SafeRepresenter.represent_float)
+
+SafeRepresenter.add_representer(list,
+ SafeRepresenter.represent_list)
+
+SafeRepresenter.add_representer(tuple,
+ SafeRepresenter.represent_list)
+
+SafeRepresenter.add_representer(dict,
+ SafeRepresenter.represent_dict)
+
+SafeRepresenter.add_representer(set,
+ SafeRepresenter.represent_set)
+
+SafeRepresenter.add_representer(datetime.date,
+ SafeRepresenter.represent_date)
+
+SafeRepresenter.add_representer(datetime.datetime,
+ SafeRepresenter.represent_datetime)
+
+SafeRepresenter.add_representer(None,
+ SafeRepresenter.represent_undefined)
+
+class Representer(SafeRepresenter):
+
+ def represent_str(self, data):
+ tag = None
+ style = None
+ try:
+ data = unicode(data, 'ascii')
+ tag = u'tag:yaml.org,2002:str'
+ except UnicodeDecodeError:
+ try:
+ data = unicode(data, 'utf-8')
+ tag = u'tag:yaml.org,2002:python/str'
+ except UnicodeDecodeError:
+ data = data.encode('base64')
+ tag = u'tag:yaml.org,2002:binary'
+ style = '|'
+ return self.represent_scalar(tag, data, style=style)
+
+ def represent_unicode(self, data):
+ tag = None
+ try:
+ data.encode('ascii')
+ tag = u'tag:yaml.org,2002:python/unicode'
+ except UnicodeEncodeError:
+ tag = u'tag:yaml.org,2002:str'
+ return self.represent_scalar(tag, data)
+
+ def represent_long(self, data):
+ tag = u'tag:yaml.org,2002:int'
+ if int(data) is not data:
+ tag = u'tag:yaml.org,2002:python/long'
+ return self.represent_scalar(tag, unicode(data))
+
+ def represent_complex(self, data):
+ if data.imag == 0.0:
+ data = u'%r' % data.real
+ elif data.real == 0.0:
+ data = u'%rj' % data.imag
+ elif data.imag > 0:
+ data = u'%r+%rj' % (data.real, data.imag)
+ else:
+ data = u'%r%rj' % (data.real, data.imag)
+ return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data)
+
+ def represent_tuple(self, data):
+ return self.represent_sequence(u'tag:yaml.org,2002:python/tuple', data)
+
+ def represent_name(self, data):
+ name = u'%s.%s' % (data.__module__, data.__name__)
+ return self.represent_scalar(u'tag:yaml.org,2002:python/name:'+name, u'')
+
+ def represent_module(self, data):
+ return self.represent_scalar(
+ u'tag:yaml.org,2002:python/module:'+data.__name__, u'')
+
+ def represent_instance(self, data):
+ # For instances of classic classes, we use __getinitargs__ and
+ # __getstate__ to serialize the data.
+
+ # If data.__getinitargs__ exists, the object must be reconstructed by
+ # calling cls(**args), where args is a tuple returned by
+ # __getinitargs__. Otherwise, the cls.__init__ method should never be
+ # called and the class instance is created by instantiating a trivial
+ # class and assigning to the instance's __class__ variable.
+
+ # If data.__getstate__ exists, it returns the state of the object.
+ # Otherwise, the state of the object is data.__dict__.
+
+ # We produce either a !!python/object or !!python/object/new node.
+ # If data.__getinitargs__ does not exist and state is a dictionary, we
+ # produce a !!python/object node . Otherwise we produce a
+ # !!python/object/new node.
+
+ cls = data.__class__
+ class_name = u'%s.%s' % (cls.__module__, cls.__name__)
+ args = None
+ state = None
+ if hasattr(data, '__getinitargs__'):
+ args = list(data.__getinitargs__())
+ if hasattr(data, '__getstate__'):
+ state = data.__getstate__()
+ else:
+ state = data.__dict__
+ if args is None and isinstance(state, dict):
+ return self.represent_mapping(
+ u'tag:yaml.org,2002:python/object:'+class_name, state)
+ if isinstance(state, dict) and not state:
+ return self.represent_sequence(
+ u'tag:yaml.org,2002:python/object/new:'+class_name, args)
+ value = {}
+ if args:
+ value['args'] = args
+ value['state'] = state
+ return self.represent_mapping(
+ u'tag:yaml.org,2002:python/object/new:'+class_name, value)
+
+ def represent_object(self, data):
+ # We use __reduce__ API to save the data. data.__reduce__ returns
+ # a tuple of length 2-5:
+ # (function, args, state, listitems, dictitems)
+
+ # For reconstructing, we calls function(*args), then set its state,
+ # listitems, and dictitems if they are not None.
+
+ # A special case is when function.__name__ == '__newobj__'. In this
+ # case we create the object with args[0].__new__(*args).
+
+ # Another special case is when __reduce__ returns a string - we don't
+ # support it.
+
+ # We produce a !!python/object, !!python/object/new or
+ # !!python/object/apply node.
+
+ cls = type(data)
+ if cls in copy_reg.dispatch_table:
+ reduce = copy_reg.dispatch_table[cls](data)
+ elif hasattr(data, '__reduce_ex__'):
+ reduce = data.__reduce_ex__(2)
+ elif hasattr(data, '__reduce__'):
+ reduce = data.__reduce__()
+ else:
+ raise RepresenterError("cannot represent object: %r" % data)
+ reduce = (list(reduce)+[None]*5)[:5]
+ function, args, state, listitems, dictitems = reduce
+ args = list(args)
+ if state is None:
+ state = {}
+ if listitems is not None:
+ listitems = list(listitems)
+ if dictitems is not None:
+ dictitems = dict(dictitems)
+ if function.__name__ == '__newobj__':
+ function = args[0]
+ args = args[1:]
+ tag = u'tag:yaml.org,2002:python/object/new:'
+ newobj = True
+ else:
+ tag = u'tag:yaml.org,2002:python/object/apply:'
+ newobj = False
+ function_name = u'%s.%s' % (function.__module__, function.__name__)
+ if not args and not listitems and not dictitems \
+ and isinstance(state, dict) and newobj:
+ return self.represent_mapping(
+ u'tag:yaml.org,2002:python/object:'+function_name, state)
+ if not listitems and not dictitems \
+ and isinstance(state, dict) and not state:
+ return self.represent_sequence(tag+function_name, args)
+ value = {}
+ if args:
+ value['args'] = args
+ if state or not isinstance(state, dict):
+ value['state'] = state
+ if listitems:
+ value['listitems'] = listitems
+ if dictitems:
+ value['dictitems'] = dictitems
+ return self.represent_mapping(tag+function_name, value)
+
+Representer.add_representer(str,
+ Representer.represent_str)
+
+Representer.add_representer(unicode,
+ Representer.represent_unicode)
+
+Representer.add_representer(long,
+ Representer.represent_long)
+
+Representer.add_representer(complex,
+ Representer.represent_complex)
+
+Representer.add_representer(tuple,
+ Representer.represent_tuple)
+
+Representer.add_representer(type,
+ Representer.represent_name)
+
+Representer.add_representer(types.ClassType,
+ Representer.represent_name)
+
+Representer.add_representer(types.FunctionType,
+ Representer.represent_name)
+
+Representer.add_representer(types.BuiltinFunctionType,
+ Representer.represent_name)
+
+Representer.add_representer(types.ModuleType,
+ Representer.represent_module)
+
+Representer.add_multi_representer(types.InstanceType,
+ Representer.represent_instance)
+
+Representer.add_multi_representer(object,
+ Representer.represent_object)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/resolver.py b/collectors/python.d.plugin/python_modules/pyyaml2/resolver.py
new file mode 100644
index 0000000..49922de
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/resolver.py
@@ -0,0 +1,225 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseResolver', 'Resolver']
+
+from error import *
+from nodes import *
+
+import re
+
+class ResolverError(YAMLError):
+ pass
+
+class BaseResolver(object):
+
+ DEFAULT_SCALAR_TAG = u'tag:yaml.org,2002:str'
+ DEFAULT_SEQUENCE_TAG = u'tag:yaml.org,2002:seq'
+ DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map'
+
+ yaml_implicit_resolvers = {}
+ yaml_path_resolvers = {}
+
+ def __init__(self):
+ self.resolver_exact_paths = []
+ self.resolver_prefix_paths = []
+
+ def add_implicit_resolver(cls, tag, regexp, first):
+ if not 'yaml_implicit_resolvers' in cls.__dict__:
+ cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()
+ if first is None:
+ first = [None]
+ for ch in first:
+ cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
+ add_implicit_resolver = classmethod(add_implicit_resolver)
+
+ def add_path_resolver(cls, tag, path, kind=None):
+ # Note: `add_path_resolver` is experimental. The API could be changed.
+ # `new_path` is a pattern that is matched against the path from the
+ # root to the node that is being considered. `node_path` elements are
+ # tuples `(node_check, index_check)`. `node_check` is a node class:
+ # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None`
+ # matches any kind of a node. `index_check` could be `None`, a boolean
+ # value, a string value, or a number. `None` and `False` match against
+ # any _value_ of sequence and mapping nodes. `True` matches against
+ # any _key_ of a mapping node. A string `index_check` matches against
+ # a mapping value that corresponds to a scalar key which content is
+ # equal to the `index_check` value. An integer `index_check` matches
+ # against a sequence value with the index equal to `index_check`.
+ if not 'yaml_path_resolvers' in cls.__dict__:
+ cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy()
+ new_path = []
+ for element in path:
+ if isinstance(element, (list, tuple)):
+ if len(element) == 2:
+ node_check, index_check = element
+ elif len(element) == 1:
+ node_check = element[0]
+ index_check = True
+ else:
+ raise ResolverError("Invalid path element: %s" % element)
+ else:
+ node_check = None
+ index_check = element
+ if node_check is str:
+ node_check = ScalarNode
+ elif node_check is list:
+ node_check = SequenceNode
+ elif node_check is dict:
+ node_check = MappingNode
+ elif node_check not in [ScalarNode, SequenceNode, MappingNode] \
+ and not isinstance(node_check, basestring) \
+ and node_check is not None:
+ raise ResolverError("Invalid node checker: %s" % node_check)
+ if not isinstance(index_check, (basestring, int)) \
+ and index_check is not None:
+ raise ResolverError("Invalid index checker: %s" % index_check)
+ new_path.append((node_check, index_check))
+ if kind is str:
+ kind = ScalarNode
+ elif kind is list:
+ kind = SequenceNode
+ elif kind is dict:
+ kind = MappingNode
+ elif kind not in [ScalarNode, SequenceNode, MappingNode] \
+ and kind is not None:
+ raise ResolverError("Invalid node kind: %s" % kind)
+ cls.yaml_path_resolvers[tuple(new_path), kind] = tag
+ add_path_resolver = classmethod(add_path_resolver)
+
+ def descend_resolver(self, current_node, current_index):
+ if not self.yaml_path_resolvers:
+ return
+ exact_paths = {}
+ prefix_paths = []
+ if current_node:
+ depth = len(self.resolver_prefix_paths)
+ for path, kind in self.resolver_prefix_paths[-1]:
+ if self.check_resolver_prefix(depth, path, kind,
+ current_node, current_index):
+ if len(path) > depth:
+ prefix_paths.append((path, kind))
+ else:
+ exact_paths[kind] = self.yaml_path_resolvers[path, kind]
+ else:
+ for path, kind in self.yaml_path_resolvers:
+ if not path:
+ exact_paths[kind] = self.yaml_path_resolvers[path, kind]
+ else:
+ prefix_paths.append((path, kind))
+ self.resolver_exact_paths.append(exact_paths)
+ self.resolver_prefix_paths.append(prefix_paths)
+
+ def ascend_resolver(self):
+ if not self.yaml_path_resolvers:
+ return
+ self.resolver_exact_paths.pop()
+ self.resolver_prefix_paths.pop()
+
+ def check_resolver_prefix(self, depth, path, kind,
+ current_node, current_index):
+ node_check, index_check = path[depth-1]
+ if isinstance(node_check, basestring):
+ if current_node.tag != node_check:
+ return
+ elif node_check is not None:
+ if not isinstance(current_node, node_check):
+ return
+ if index_check is True and current_index is not None:
+ return
+ if (index_check is False or index_check is None) \
+ and current_index is None:
+ return
+ if isinstance(index_check, basestring):
+ if not (isinstance(current_index, ScalarNode)
+ and index_check == current_index.value):
+ return
+ elif isinstance(index_check, int) and not isinstance(index_check, bool):
+ if index_check != current_index:
+ return
+ return True
+
+ def resolve(self, kind, value, implicit):
+ if kind is ScalarNode and implicit[0]:
+ if value == u'':
+ resolvers = self.yaml_implicit_resolvers.get(u'', [])
+ else:
+ resolvers = self.yaml_implicit_resolvers.get(value[0], [])
+ resolvers += self.yaml_implicit_resolvers.get(None, [])
+ for tag, regexp in resolvers:
+ if regexp.match(value):
+ return tag
+ implicit = implicit[1]
+ if self.yaml_path_resolvers:
+ exact_paths = self.resolver_exact_paths[-1]
+ if kind in exact_paths:
+ return exact_paths[kind]
+ if None in exact_paths:
+ return exact_paths[None]
+ if kind is ScalarNode:
+ return self.DEFAULT_SCALAR_TAG
+ elif kind is SequenceNode:
+ return self.DEFAULT_SEQUENCE_TAG
+ elif kind is MappingNode:
+ return self.DEFAULT_MAPPING_TAG
+
+class Resolver(BaseResolver):
+ pass
+
+Resolver.add_implicit_resolver(
+ u'tag:yaml.org,2002:bool',
+ re.compile(ur'''^(?:yes|Yes|YES|no|No|NO
+ |true|True|TRUE|false|False|FALSE
+ |on|On|ON|off|Off|OFF)$''', re.X),
+ list(u'yYnNtTfFoO'))
+
+Resolver.add_implicit_resolver(
+ u'tag:yaml.org,2002:float',
+ re.compile(ur'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?
+ |\.[0-9_]+(?:[eE][-+][0-9]+)?
+ |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
+ |[-+]?\.(?:inf|Inf|INF)
+ |\.(?:nan|NaN|NAN))$''', re.X),
+ list(u'-+0123456789.'))
+
+Resolver.add_implicit_resolver(
+ u'tag:yaml.org,2002:int',
+ re.compile(ur'''^(?:[-+]?0b[0-1_]+
+ |[-+]?0[0-7_]+
+ |[-+]?(?:0|[1-9][0-9_]*)
+ |[-+]?0x[0-9a-fA-F_]+
+ |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X),
+ list(u'-+0123456789'))
+
+Resolver.add_implicit_resolver(
+ u'tag:yaml.org,2002:merge',
+ re.compile(ur'^(?:<<)$'),
+ [u'<'])
+
+Resolver.add_implicit_resolver(
+ u'tag:yaml.org,2002:null',
+ re.compile(ur'''^(?: ~
+ |null|Null|NULL
+ | )$''', re.X),
+ [u'~', u'n', u'N', u''])
+
+Resolver.add_implicit_resolver(
+ u'tag:yaml.org,2002:timestamp',
+ re.compile(ur'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
+ |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
+ (?:[Tt]|[ \t]+)[0-9][0-9]?
+ :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)?
+ (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
+ list(u'0123456789'))
+
+Resolver.add_implicit_resolver(
+ u'tag:yaml.org,2002:value',
+ re.compile(ur'^(?:=)$'),
+ [u'='])
+
+# The following resolver is only for documentation purposes. It cannot work
+# because plain scalars cannot start with '!', '&', or '*'.
+Resolver.add_implicit_resolver(
+ u'tag:yaml.org,2002:yaml',
+ re.compile(ur'^(?:!|&|\*)$'),
+ list(u'!&*'))
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/scanner.py b/collectors/python.d.plugin/python_modules/pyyaml2/scanner.py
new file mode 100644
index 0000000..971da61
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/scanner.py
@@ -0,0 +1,1458 @@
+# SPDX-License-Identifier: MIT
+
+# Scanner produces tokens of the following types:
+# STREAM-START
+# STREAM-END
+# DIRECTIVE(name, value)
+# DOCUMENT-START
+# DOCUMENT-END
+# BLOCK-SEQUENCE-START
+# BLOCK-MAPPING-START
+# BLOCK-END
+# FLOW-SEQUENCE-START
+# FLOW-MAPPING-START
+# FLOW-SEQUENCE-END
+# FLOW-MAPPING-END
+# BLOCK-ENTRY
+# FLOW-ENTRY
+# KEY
+# VALUE
+# ALIAS(value)
+# ANCHOR(value)
+# TAG(value)
+# SCALAR(value, plain, style)
+#
+# Read comments in the Scanner code for more details.
+#
+
+__all__ = ['Scanner', 'ScannerError']
+
+from error import MarkedYAMLError
+from tokens import *
+
+class ScannerError(MarkedYAMLError):
+ pass
+
+class SimpleKey(object):
+ # See below simple keys treatment.
+
+ def __init__(self, token_number, required, index, line, column, mark):
+ self.token_number = token_number
+ self.required = required
+ self.index = index
+ self.line = line
+ self.column = column
+ self.mark = mark
+
+class Scanner(object):
+
+ def __init__(self):
+ """Initialize the scanner."""
+ # It is assumed that Scanner and Reader will have a common descendant.
+ # Reader do the dirty work of checking for BOM and converting the
+ # input data to Unicode. It also adds NUL to the end.
+ #
+ # Reader supports the following methods
+ # self.peek(i=0) # peek the next i-th character
+ # self.prefix(l=1) # peek the next l characters
+ # self.forward(l=1) # read the next l characters and move the pointer.
+
+ # Had we reached the end of the stream?
+ self.done = False
+
+ # The number of unclosed '{' and '['. `flow_level == 0` means block
+ # context.
+ self.flow_level = 0
+
+ # List of processed tokens that are not yet emitted.
+ self.tokens = []
+
+ # Add the STREAM-START token.
+ self.fetch_stream_start()
+
+ # Number of tokens that were emitted through the `get_token` method.
+ self.tokens_taken = 0
+
+ # The current indentation level.
+ self.indent = -1
+
+ # Past indentation levels.
+ self.indents = []
+
+ # Variables related to simple keys treatment.
+
+ # A simple key is a key that is not denoted by the '?' indicator.
+ # Example of simple keys:
+ # ---
+ # block simple key: value
+ # ? not a simple key:
+ # : { flow simple key: value }
+ # We emit the KEY token before all keys, so when we find a potential
+ # simple key, we try to locate the corresponding ':' indicator.
+ # Simple keys should be limited to a single line and 1024 characters.
+
+ # Can a simple key start at the current position? A simple key may
+ # start:
+ # - at the beginning of the line, not counting indentation spaces
+ # (in block context),
+ # - after '{', '[', ',' (in the flow context),
+ # - after '?', ':', '-' (in the block context).
+ # In the block context, this flag also signifies if a block collection
+ # may start at the current position.
+ self.allow_simple_key = True
+
+ # Keep track of possible simple keys. This is a dictionary. The key
+ # is `flow_level`; there can be no more that one possible simple key
+ # for each level. The value is a SimpleKey record:
+ # (token_number, required, index, line, column, mark)
+ # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
+ # '[', or '{' tokens.
+ self.possible_simple_keys = {}
+
+ # Public methods.
+
+ def check_token(self, *choices):
+ # Check if the next token is one of the given types.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ if not choices:
+ return True
+ for choice in choices:
+ if isinstance(self.tokens[0], choice):
+ return True
+ return False
+
+ def peek_token(self):
+ # Return the next token, but do not delete if from the queue.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ return self.tokens[0]
+
+ def get_token(self):
+ # Return the next token.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ self.tokens_taken += 1
+ return self.tokens.pop(0)
+
+ # Private methods.
+
+ def need_more_tokens(self):
+ if self.done:
+ return False
+ if not self.tokens:
+ return True
+ # The current token may be a potential simple key, so we
+ # need to look further.
+ self.stale_possible_simple_keys()
+ if self.next_possible_simple_key() == self.tokens_taken:
+ return True
+
+ def fetch_more_tokens(self):
+
+ # Eat whitespaces and comments until we reach the next token.
+ self.scan_to_next_token()
+
+ # Remove obsolete possible simple keys.
+ self.stale_possible_simple_keys()
+
+ # Compare the current indentation and column. It may add some tokens
+ # and decrease the current indentation level.
+ self.unwind_indent(self.column)
+
+ # Peek the next character.
+ ch = self.peek()
+
+ # Is it the end of stream?
+ if ch == u'\0':
+ return self.fetch_stream_end()
+
+ # Is it a directive?
+ if ch == u'%' and self.check_directive():
+ return self.fetch_directive()
+
+ # Is it the document start?
+ if ch == u'-' and self.check_document_start():
+ return self.fetch_document_start()
+
+ # Is it the document end?
+ if ch == u'.' and self.check_document_end():
+ return self.fetch_document_end()
+
+ # TODO: support for BOM within a stream.
+ #if ch == u'\uFEFF':
+ # return self.fetch_bom() <-- issue BOMToken
+
+ # Note: the order of the following checks is NOT significant.
+
+ # Is it the flow sequence start indicator?
+ if ch == u'[':
+ return self.fetch_flow_sequence_start()
+
+ # Is it the flow mapping start indicator?
+ if ch == u'{':
+ return self.fetch_flow_mapping_start()
+
+ # Is it the flow sequence end indicator?
+ if ch == u']':
+ return self.fetch_flow_sequence_end()
+
+ # Is it the flow mapping end indicator?
+ if ch == u'}':
+ return self.fetch_flow_mapping_end()
+
+ # Is it the flow entry indicator?
+ if ch == u',':
+ return self.fetch_flow_entry()
+
+ # Is it the block entry indicator?
+ if ch == u'-' and self.check_block_entry():
+ return self.fetch_block_entry()
+
+ # Is it the key indicator?
+ if ch == u'?' and self.check_key():
+ return self.fetch_key()
+
+ # Is it the value indicator?
+ if ch == u':' and self.check_value():
+ return self.fetch_value()
+
+ # Is it an alias?
+ if ch == u'*':
+ return self.fetch_alias()
+
+ # Is it an anchor?
+ if ch == u'&':
+ return self.fetch_anchor()
+
+ # Is it a tag?
+ if ch == u'!':
+ return self.fetch_tag()
+
+ # Is it a literal scalar?
+ if ch == u'|' and not self.flow_level:
+ return self.fetch_literal()
+
+ # Is it a folded scalar?
+ if ch == u'>' and not self.flow_level:
+ return self.fetch_folded()
+
+ # Is it a single quoted scalar?
+ if ch == u'\'':
+ return self.fetch_single()
+
+ # Is it a double quoted scalar?
+ if ch == u'\"':
+ return self.fetch_double()
+
+ # It must be a plain scalar then.
+ if self.check_plain():
+ return self.fetch_plain()
+
+ # No? It's an error. Let's produce a nice error message.
+ raise ScannerError("while scanning for the next token", None,
+ "found character %r that cannot start any token"
+ % ch.encode('utf-8'), self.get_mark())
+
+ # Simple keys treatment.
+
+ def next_possible_simple_key(self):
+ # Return the number of the nearest possible simple key. Actually we
+ # don't need to loop through the whole dictionary. We may replace it
+ # with the following code:
+ # if not self.possible_simple_keys:
+ # return None
+ # return self.possible_simple_keys[
+ # min(self.possible_simple_keys.keys())].token_number
+ min_token_number = None
+ for level in self.possible_simple_keys:
+ key = self.possible_simple_keys[level]
+ if min_token_number is None or key.token_number < min_token_number:
+ min_token_number = key.token_number
+ return min_token_number
+
+ def stale_possible_simple_keys(self):
+ # Remove entries that are no longer possible simple keys. According to
+ # the YAML specification, simple keys
+ # - should be limited to a single line,
+ # - should be no longer than 1024 characters.
+ # Disabling this procedure will allow simple keys of any length and
+ # height (may cause problems if indentation is broken though).
+ for level in self.possible_simple_keys.keys():
+ key = self.possible_simple_keys[level]
+ if key.line != self.line \
+ or self.index-key.index > 1024:
+ if key.required:
+ raise ScannerError("while scanning a simple key", key.mark,
+ "could not found expected ':'", self.get_mark())
+ del self.possible_simple_keys[level]
+
+ def save_possible_simple_key(self):
+ # The next token may start a simple key. We check if it's possible
+ # and save its position. This function is called for
+ # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
+
+ # Check if a simple key is required at the current position.
+ required = not self.flow_level and self.indent == self.column
+
+ # A simple key is required only if it is the first token in the current
+ # line. Therefore it is always allowed.
+ assert self.allow_simple_key or not required
+
+ # The next token might be a simple key. Let's save it's number and
+ # position.
+ if self.allow_simple_key:
+ self.remove_possible_simple_key()
+ token_number = self.tokens_taken+len(self.tokens)
+ key = SimpleKey(token_number, required,
+ self.index, self.line, self.column, self.get_mark())
+ self.possible_simple_keys[self.flow_level] = key
+
+ def remove_possible_simple_key(self):
+ # Remove the saved possible key position at the current flow level.
+ if self.flow_level in self.possible_simple_keys:
+ key = self.possible_simple_keys[self.flow_level]
+
+ if key.required:
+ raise ScannerError("while scanning a simple key", key.mark,
+ "could not found expected ':'", self.get_mark())
+
+ del self.possible_simple_keys[self.flow_level]
+
+ # Indentation functions.
+
+ def unwind_indent(self, column):
+
+ ## In flow context, tokens should respect indentation.
+ ## Actually the condition should be `self.indent >= column` according to
+ ## the spec. But this condition will prohibit intuitively correct
+ ## constructions such as
+ ## key : {
+ ## }
+ #if self.flow_level and self.indent > column:
+ # raise ScannerError(None, None,
+ # "invalid intendation or unclosed '[' or '{'",
+ # self.get_mark())
+
+ # In the flow context, indentation is ignored. We make the scanner less
+ # restrictive then specification requires.
+ if self.flow_level:
+ return
+
+ # In block context, we may need to issue the BLOCK-END tokens.
+ while self.indent > column:
+ mark = self.get_mark()
+ self.indent = self.indents.pop()
+ self.tokens.append(BlockEndToken(mark, mark))
+
+ def add_indent(self, column):
+ # Check if we need to increase indentation.
+ if self.indent < column:
+ self.indents.append(self.indent)
+ self.indent = column
+ return True
+ return False
+
+ # Fetchers.
+
+ def fetch_stream_start(self):
+ # We always add STREAM-START as the first token and STREAM-END as the
+ # last token.
+
+ # Read the token.
+ mark = self.get_mark()
+
+ # Add STREAM-START.
+ self.tokens.append(StreamStartToken(mark, mark,
+ encoding=self.encoding))
+
+
+ def fetch_stream_end(self):
+
+ # Set the current intendation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+ self.possible_simple_keys = {}
+
+ # Read the token.
+ mark = self.get_mark()
+
+ # Add STREAM-END.
+ self.tokens.append(StreamEndToken(mark, mark))
+
+ # The steam is finished.
+ self.done = True
+
+ def fetch_directive(self):
+
+ # Set the current intendation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+
+ # Scan and add DIRECTIVE.
+ self.tokens.append(self.scan_directive())
+
+ def fetch_document_start(self):
+ self.fetch_document_indicator(DocumentStartToken)
+
+ def fetch_document_end(self):
+ self.fetch_document_indicator(DocumentEndToken)
+
+ def fetch_document_indicator(self, TokenClass):
+
+ # Set the current intendation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys. Note that there could not be a block collection
+ # after '---'.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+
+ # Add DOCUMENT-START or DOCUMENT-END.
+ start_mark = self.get_mark()
+ self.forward(3)
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_sequence_start(self):
+ self.fetch_flow_collection_start(FlowSequenceStartToken)
+
+ def fetch_flow_mapping_start(self):
+ self.fetch_flow_collection_start(FlowMappingStartToken)
+
+ def fetch_flow_collection_start(self, TokenClass):
+
+ # '[' and '{' may start a simple key.
+ self.save_possible_simple_key()
+
+ # Increase the flow level.
+ self.flow_level += 1
+
+ # Simple keys are allowed after '[' and '{'.
+ self.allow_simple_key = True
+
+ # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_sequence_end(self):
+ self.fetch_flow_collection_end(FlowSequenceEndToken)
+
+ def fetch_flow_mapping_end(self):
+ self.fetch_flow_collection_end(FlowMappingEndToken)
+
+ def fetch_flow_collection_end(self, TokenClass):
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Decrease the flow level.
+ self.flow_level -= 1
+
+ # No simple keys after ']' or '}'.
+ self.allow_simple_key = False
+
+ # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_entry(self):
+
+ # Simple keys are allowed after ','.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add FLOW-ENTRY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(FlowEntryToken(start_mark, end_mark))
+
+ def fetch_block_entry(self):
+
+ # Block context needs additional checks.
+ if not self.flow_level:
+
+ # Are we allowed to start a new entry?
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "sequence entries are not allowed here",
+ self.get_mark())
+
+ # We may need to add BLOCK-SEQUENCE-START.
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockSequenceStartToken(mark, mark))
+
+ # It's an error for the block entry to occur in the flow context,
+ # but we let the parser detect this.
+ else:
+ pass
+
+ # Simple keys are allowed after '-'.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add BLOCK-ENTRY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(BlockEntryToken(start_mark, end_mark))
+
+ def fetch_key(self):
+
+ # Block context needs additional checks.
+ if not self.flow_level:
+
+ # Are we allowed to start a key (not nessesary a simple)?
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "mapping keys are not allowed here",
+ self.get_mark())
+
+ # We may need to add BLOCK-MAPPING-START.
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockMappingStartToken(mark, mark))
+
+ # Simple keys are allowed after '?' in the block context.
+ self.allow_simple_key = not self.flow_level
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add KEY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(KeyToken(start_mark, end_mark))
+
+ def fetch_value(self):
+
+ # Do we determine a simple key?
+ if self.flow_level in self.possible_simple_keys:
+
+ # Add KEY.
+ key = self.possible_simple_keys[self.flow_level]
+ del self.possible_simple_keys[self.flow_level]
+ self.tokens.insert(key.token_number-self.tokens_taken,
+ KeyToken(key.mark, key.mark))
+
+ # If this key starts a new block mapping, we need to add
+ # BLOCK-MAPPING-START.
+ if not self.flow_level:
+ if self.add_indent(key.column):
+ self.tokens.insert(key.token_number-self.tokens_taken,
+ BlockMappingStartToken(key.mark, key.mark))
+
+ # There cannot be two simple keys one after another.
+ self.allow_simple_key = False
+
+ # It must be a part of a complex key.
+ else:
+
+ # Block context needs additional checks.
+ # (Do we really need them? They will be catched by the parser
+ # anyway.)
+ if not self.flow_level:
+
+ # We are allowed to start a complex value if and only if
+ # we can start a simple key.
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "mapping values are not allowed here",
+ self.get_mark())
+
+ # If this value starts a new block mapping, we need to add
+ # BLOCK-MAPPING-START. It will be detected as an error later by
+ # the parser.
+ if not self.flow_level:
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockMappingStartToken(mark, mark))
+
+ # Simple keys are allowed after ':' in the block context.
+ self.allow_simple_key = not self.flow_level
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add VALUE.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(ValueToken(start_mark, end_mark))
+
+ def fetch_alias(self):
+
+ # ALIAS could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after ALIAS.
+ self.allow_simple_key = False
+
+ # Scan and add ALIAS.
+ self.tokens.append(self.scan_anchor(AliasToken))
+
+ def fetch_anchor(self):
+
+ # ANCHOR could start a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after ANCHOR.
+ self.allow_simple_key = False
+
+ # Scan and add ANCHOR.
+ self.tokens.append(self.scan_anchor(AnchorToken))
+
+ def fetch_tag(self):
+
+ # TAG could start a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after TAG.
+ self.allow_simple_key = False
+
+ # Scan and add TAG.
+ self.tokens.append(self.scan_tag())
+
+ def fetch_literal(self):
+ self.fetch_block_scalar(style='|')
+
+ def fetch_folded(self):
+ self.fetch_block_scalar(style='>')
+
+ def fetch_block_scalar(self, style):
+
+ # A simple key may follow a block scalar.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Scan and add SCALAR.
+ self.tokens.append(self.scan_block_scalar(style))
+
+ def fetch_single(self):
+ self.fetch_flow_scalar(style='\'')
+
+ def fetch_double(self):
+ self.fetch_flow_scalar(style='"')
+
+ def fetch_flow_scalar(self, style):
+
+ # A flow scalar could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after flow scalars.
+ self.allow_simple_key = False
+
+ # Scan and add SCALAR.
+ self.tokens.append(self.scan_flow_scalar(style))
+
+ def fetch_plain(self):
+
+ # A plain scalar could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after plain scalars. But note that `scan_plain` will
+ # change this flag if the scan is finished at the beginning of the
+ # line.
+ self.allow_simple_key = False
+
+ # Scan and add SCALAR. May change `allow_simple_key`.
+ self.tokens.append(self.scan_plain())
+
+ # Checkers.
+
+ def check_directive(self):
+
+ # DIRECTIVE: ^ '%' ...
+ # The '%' indicator is already checked.
+ if self.column == 0:
+ return True
+
+ def check_document_start(self):
+
+ # DOCUMENT-START: ^ '---' (' '|'\n')
+ if self.column == 0:
+ if self.prefix(3) == u'---' \
+ and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
+ return True
+
+ def check_document_end(self):
+
+ # DOCUMENT-END: ^ '...' (' '|'\n')
+ if self.column == 0:
+ if self.prefix(3) == u'...' \
+ and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
+ return True
+
+ def check_block_entry(self):
+
+ # BLOCK-ENTRY: '-' (' '|'\n')
+ return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
+
+ def check_key(self):
+
+ # KEY(flow context): '?'
+ if self.flow_level:
+ return True
+
+ # KEY(block context): '?' (' '|'\n')
+ else:
+ return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
+
+ def check_value(self):
+
+ # VALUE(flow context): ':'
+ if self.flow_level:
+ return True
+
+ # VALUE(block context): ':' (' '|'\n')
+ else:
+ return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
+
+ def check_plain(self):
+
+ # A plain scalar may start with any non-space character except:
+ # '-', '?', ':', ',', '[', ']', '{', '}',
+ # '#', '&', '*', '!', '|', '>', '\'', '\"',
+ # '%', '@', '`'.
+ #
+ # It may also start with
+ # '-', '?', ':'
+ # if it is followed by a non-space character.
+ #
+ # Note that we limit the last rule to the block context (except the
+ # '-' character) because we want the flow context to be space
+ # independent.
+ ch = self.peek()
+ return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \
+ or (self.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029'
+ and (ch == u'-' or (not self.flow_level and ch in u'?:')))
+
+ # Scanners.
+
+ def scan_to_next_token(self):
+ # We ignore spaces, line breaks and comments.
+ # If we find a line break in the block context, we set the flag
+ # `allow_simple_key` on.
+ # The byte order mark is stripped if it's the first character in the
+ # stream. We do not yet support BOM inside the stream as the
+ # specification requires. Any such mark will be considered as a part
+ # of the document.
+ #
+ # TODO: We need to make tab handling rules more sane. A good rule is
+ # Tabs cannot precede tokens
+ # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
+ # KEY(block), VALUE(block), BLOCK-ENTRY
+ # So the checking code is
+ # if <TAB>:
+ # self.allow_simple_keys = False
+ # We also need to add the check for `allow_simple_keys == True` to
+ # `unwind_indent` before issuing BLOCK-END.
+ # Scanners for block, flow, and plain scalars need to be modified.
+
+ if self.index == 0 and self.peek() == u'\uFEFF':
+ self.forward()
+ found = False
+ while not found:
+ while self.peek() == u' ':
+ self.forward()
+ if self.peek() == u'#':
+ while self.peek() not in u'\0\r\n\x85\u2028\u2029':
+ self.forward()
+ if self.scan_line_break():
+ if not self.flow_level:
+ self.allow_simple_key = True
+ else:
+ found = True
+
+ def scan_directive(self):
+ # See the specification for details.
+ start_mark = self.get_mark()
+ self.forward()
+ name = self.scan_directive_name(start_mark)
+ value = None
+ if name == u'YAML':
+ value = self.scan_yaml_directive_value(start_mark)
+ end_mark = self.get_mark()
+ elif name == u'TAG':
+ value = self.scan_tag_directive_value(start_mark)
+ end_mark = self.get_mark()
+ else:
+ end_mark = self.get_mark()
+ while self.peek() not in u'\0\r\n\x85\u2028\u2029':
+ self.forward()
+ self.scan_directive_ignored_line(start_mark)
+ return DirectiveToken(name, value, start_mark, end_mark)
+
+ def scan_directive_name(self, start_mark):
+ # See the specification for details.
+ length = 0
+ ch = self.peek(length)
+ while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
+ or ch in u'-_':
+ length += 1
+ ch = self.peek(length)
+ if not length:
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch.encode('utf-8'), self.get_mark())
+ value = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch not in u'\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch.encode('utf-8'), self.get_mark())
+ return value
+
+ def scan_yaml_directive_value(self, start_mark):
+ # See the specification for details.
+ while self.peek() == u' ':
+ self.forward()
+ major = self.scan_yaml_directive_number(start_mark)
+ if self.peek() != '.':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit or '.', but found %r"
+ % self.peek().encode('utf-8'),
+ self.get_mark())
+ self.forward()
+ minor = self.scan_yaml_directive_number(start_mark)
+ if self.peek() not in u'\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit or ' ', but found %r"
+ % self.peek().encode('utf-8'),
+ self.get_mark())
+ return (major, minor)
+
+ def scan_yaml_directive_number(self, start_mark):
+ # See the specification for details.
+ ch = self.peek()
+ if not (u'0' <= ch <= u'9'):
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit, but found %r" % ch.encode('utf-8'),
+ self.get_mark())
+ length = 0
+ while u'0' <= self.peek(length) <= u'9':
+ length += 1
+ value = int(self.prefix(length))
+ self.forward(length)
+ return value
+
+ def scan_tag_directive_value(self, start_mark):
+ # See the specification for details.
+ while self.peek() == u' ':
+ self.forward()
+ handle = self.scan_tag_directive_handle(start_mark)
+ while self.peek() == u' ':
+ self.forward()
+ prefix = self.scan_tag_directive_prefix(start_mark)
+ return (handle, prefix)
+
+ def scan_tag_directive_handle(self, start_mark):
+ # See the specification for details.
+ value = self.scan_tag_handle('directive', start_mark)
+ ch = self.peek()
+ if ch != u' ':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected ' ', but found %r" % ch.encode('utf-8'),
+ self.get_mark())
+ return value
+
+ def scan_tag_directive_prefix(self, start_mark):
+ # See the specification for details.
+ value = self.scan_tag_uri('directive', start_mark)
+ ch = self.peek()
+ if ch not in u'\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected ' ', but found %r" % ch.encode('utf-8'),
+ self.get_mark())
+ return value
+
+ def scan_directive_ignored_line(self, start_mark):
+ # See the specification for details.
+ while self.peek() == u' ':
+ self.forward()
+ if self.peek() == u'#':
+ while self.peek() not in u'\0\r\n\x85\u2028\u2029':
+ self.forward()
+ ch = self.peek()
+ if ch not in u'\0\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a comment or a line break, but found %r"
+ % ch.encode('utf-8'), self.get_mark())
+ self.scan_line_break()
+
+ def scan_anchor(self, TokenClass):
+ # The specification does not restrict characters for anchors and
+ # aliases. This may lead to problems, for instance, the document:
+ # [ *alias, value ]
+ # can be interpteted in two ways, as
+ # [ "value" ]
+ # and
+ # [ *alias , "value" ]
+ # Therefore we restrict aliases to numbers and ASCII letters.
+ start_mark = self.get_mark()
+ indicator = self.peek()
+ if indicator == u'*':
+ name = 'alias'
+ else:
+ name = 'anchor'
+ self.forward()
+ length = 0
+ ch = self.peek(length)
+ while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
+ or ch in u'-_':
+ length += 1
+ ch = self.peek(length)
+ if not length:
+ raise ScannerError("while scanning an %s" % name, start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch.encode('utf-8'), self.get_mark())
+ value = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
+ raise ScannerError("while scanning an %s" % name, start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch.encode('utf-8'), self.get_mark())
+ end_mark = self.get_mark()
+ return TokenClass(value, start_mark, end_mark)
+
+ def scan_tag(self):
+ # See the specification for details.
+ start_mark = self.get_mark()
+ ch = self.peek(1)
+ if ch == u'<':
+ handle = None
+ self.forward(2)
+ suffix = self.scan_tag_uri('tag', start_mark)
+ if self.peek() != u'>':
+ raise ScannerError("while parsing a tag", start_mark,
+ "expected '>', but found %r" % self.peek().encode('utf-8'),
+ self.get_mark())
+ self.forward()
+ elif ch in u'\0 \t\r\n\x85\u2028\u2029':
+ handle = None
+ suffix = u'!'
+ self.forward()
+ else:
+ length = 1
+ use_handle = False
+ while ch not in u'\0 \r\n\x85\u2028\u2029':
+ if ch == u'!':
+ use_handle = True
+ break
+ length += 1
+ ch = self.peek(length)
+ handle = u'!'
+ if use_handle:
+ handle = self.scan_tag_handle('tag', start_mark)
+ else:
+ handle = u'!'
+ self.forward()
+ suffix = self.scan_tag_uri('tag', start_mark)
+ ch = self.peek()
+ if ch not in u'\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a tag", start_mark,
+ "expected ' ', but found %r" % ch.encode('utf-8'),
+ self.get_mark())
+ value = (handle, suffix)
+ end_mark = self.get_mark()
+ return TagToken(value, start_mark, end_mark)
+
+ def scan_block_scalar(self, style):
+ # See the specification for details.
+
+ if style == '>':
+ folded = True
+ else:
+ folded = False
+
+ chunks = []
+ start_mark = self.get_mark()
+
+ # Scan the header.
+ self.forward()
+ chomping, increment = self.scan_block_scalar_indicators(start_mark)
+ self.scan_block_scalar_ignored_line(start_mark)
+
+ # Determine the indentation level and go to the first non-empty line.
+ min_indent = self.indent+1
+ if min_indent < 1:
+ min_indent = 1
+ if increment is None:
+ breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
+ indent = max(min_indent, max_indent)
+ else:
+ indent = min_indent+increment-1
+ breaks, end_mark = self.scan_block_scalar_breaks(indent)
+ line_break = u''
+
+ # Scan the inner part of the block scalar.
+ while self.column == indent and self.peek() != u'\0':
+ chunks.extend(breaks)
+ leading_non_space = self.peek() not in u' \t'
+ length = 0
+ while self.peek(length) not in u'\0\r\n\x85\u2028\u2029':
+ length += 1
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ line_break = self.scan_line_break()
+ breaks, end_mark = self.scan_block_scalar_breaks(indent)
+ if self.column == indent and self.peek() != u'\0':
+
+ # Unfortunately, folding rules are ambiguous.
+ #
+ # This is the folding according to the specification:
+
+ if folded and line_break == u'\n' \
+ and leading_non_space and self.peek() not in u' \t':
+ if not breaks:
+ chunks.append(u' ')
+ else:
+ chunks.append(line_break)
+
+ # This is Clark Evans's interpretation (also in the spec
+ # examples):
+ #
+ #if folded and line_break == u'\n':
+ # if not breaks:
+ # if self.peek() not in ' \t':
+ # chunks.append(u' ')
+ # else:
+ # chunks.append(line_break)
+ #else:
+ # chunks.append(line_break)
+ else:
+ break
+
+ # Chomp the tail.
+ if chomping is not False:
+ chunks.append(line_break)
+ if chomping is True:
+ chunks.extend(breaks)
+
+ # We are done.
+ return ScalarToken(u''.join(chunks), False, start_mark, end_mark,
+ style)
+
+ def scan_block_scalar_indicators(self, start_mark):
+ # See the specification for details.
+ chomping = None
+ increment = None
+ ch = self.peek()
+ if ch in u'+-':
+ if ch == '+':
+ chomping = True
+ else:
+ chomping = False
+ self.forward()
+ ch = self.peek()
+ if ch in u'0123456789':
+ increment = int(ch)
+ if increment == 0:
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected indentation indicator in the range 1-9, but found 0",
+ self.get_mark())
+ self.forward()
+ elif ch in u'0123456789':
+ increment = int(ch)
+ if increment == 0:
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected indentation indicator in the range 1-9, but found 0",
+ self.get_mark())
+ self.forward()
+ ch = self.peek()
+ if ch in u'+-':
+ if ch == '+':
+ chomping = True
+ else:
+ chomping = False
+ self.forward()
+ ch = self.peek()
+ if ch not in u'\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected chomping or indentation indicators, but found %r"
+ % ch.encode('utf-8'), self.get_mark())
+ return chomping, increment
+
+ def scan_block_scalar_ignored_line(self, start_mark):
+ # See the specification for details.
+ while self.peek() == u' ':
+ self.forward()
+ if self.peek() == u'#':
+ while self.peek() not in u'\0\r\n\x85\u2028\u2029':
+ self.forward()
+ ch = self.peek()
+ if ch not in u'\0\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected a comment or a line break, but found %r"
+ % ch.encode('utf-8'), self.get_mark())
+ self.scan_line_break()
+
+ def scan_block_scalar_indentation(self):
+ # See the specification for details.
+ chunks = []
+ max_indent = 0
+ end_mark = self.get_mark()
+ while self.peek() in u' \r\n\x85\u2028\u2029':
+ if self.peek() != u' ':
+ chunks.append(self.scan_line_break())
+ end_mark = self.get_mark()
+ else:
+ self.forward()
+ if self.column > max_indent:
+ max_indent = self.column
+ return chunks, max_indent, end_mark
+
+ def scan_block_scalar_breaks(self, indent):
+ # See the specification for details.
+ chunks = []
+ end_mark = self.get_mark()
+ while self.column < indent and self.peek() == u' ':
+ self.forward()
+ while self.peek() in u'\r\n\x85\u2028\u2029':
+ chunks.append(self.scan_line_break())
+ end_mark = self.get_mark()
+ while self.column < indent and self.peek() == u' ':
+ self.forward()
+ return chunks, end_mark
+
+ def scan_flow_scalar(self, style):
+ # See the specification for details.
+ # Note that we loose indentation rules for quoted scalars. Quoted
+ # scalars don't need to adhere indentation because " and ' clearly
+ # mark the beginning and the end of them. Therefore we are less
+ # restrictive then the specification requires. We only need to check
+ # that document separators are not included in scalars.
+ if style == '"':
+ double = True
+ else:
+ double = False
+ chunks = []
+ start_mark = self.get_mark()
+ quote = self.peek()
+ self.forward()
+ chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
+ while self.peek() != quote:
+ chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
+ chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
+ self.forward()
+ end_mark = self.get_mark()
+ return ScalarToken(u''.join(chunks), False, start_mark, end_mark,
+ style)
+
+ ESCAPE_REPLACEMENTS = {
+ u'0': u'\0',
+ u'a': u'\x07',
+ u'b': u'\x08',
+ u't': u'\x09',
+ u'\t': u'\x09',
+ u'n': u'\x0A',
+ u'v': u'\x0B',
+ u'f': u'\x0C',
+ u'r': u'\x0D',
+ u'e': u'\x1B',
+ u' ': u'\x20',
+ u'\"': u'\"',
+ u'\\': u'\\',
+ u'N': u'\x85',
+ u'_': u'\xA0',
+ u'L': u'\u2028',
+ u'P': u'\u2029',
+ }
+
+ ESCAPE_CODES = {
+ u'x': 2,
+ u'u': 4,
+ u'U': 8,
+ }
+
+ def scan_flow_scalar_non_spaces(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ while True:
+ length = 0
+ while self.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029':
+ length += 1
+ if length:
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ ch = self.peek()
+ if not double and ch == u'\'' and self.peek(1) == u'\'':
+ chunks.append(u'\'')
+ self.forward(2)
+ elif (double and ch == u'\'') or (not double and ch in u'\"\\'):
+ chunks.append(ch)
+ self.forward()
+ elif double and ch == u'\\':
+ self.forward()
+ ch = self.peek()
+ if ch in self.ESCAPE_REPLACEMENTS:
+ chunks.append(self.ESCAPE_REPLACEMENTS[ch])
+ self.forward()
+ elif ch in self.ESCAPE_CODES:
+ length = self.ESCAPE_CODES[ch]
+ self.forward()
+ for k in range(length):
+ if self.peek(k) not in u'0123456789ABCDEFabcdef':
+ raise ScannerError("while scanning a double-quoted scalar", start_mark,
+ "expected escape sequence of %d hexdecimal numbers, but found %r" %
+ (length, self.peek(k).encode('utf-8')), self.get_mark())
+ code = int(self.prefix(length), 16)
+ chunks.append(unichr(code))
+ self.forward(length)
+ elif ch in u'\r\n\x85\u2028\u2029':
+ self.scan_line_break()
+ chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
+ else:
+ raise ScannerError("while scanning a double-quoted scalar", start_mark,
+ "found unknown escape character %r" % ch.encode('utf-8'), self.get_mark())
+ else:
+ return chunks
+
+ def scan_flow_scalar_spaces(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ length = 0
+ while self.peek(length) in u' \t':
+ length += 1
+ whitespaces = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch == u'\0':
+ raise ScannerError("while scanning a quoted scalar", start_mark,
+ "found unexpected end of stream", self.get_mark())
+ elif ch in u'\r\n\x85\u2028\u2029':
+ line_break = self.scan_line_break()
+ breaks = self.scan_flow_scalar_breaks(double, start_mark)
+ if line_break != u'\n':
+ chunks.append(line_break)
+ elif not breaks:
+ chunks.append(u' ')
+ chunks.extend(breaks)
+ else:
+ chunks.append(whitespaces)
+ return chunks
+
+ def scan_flow_scalar_breaks(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ while True:
+ # Instead of checking indentation, we check for document
+ # separators.
+ prefix = self.prefix(3)
+ if (prefix == u'---' or prefix == u'...') \
+ and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a quoted scalar", start_mark,
+ "found unexpected document separator", self.get_mark())
+ while self.peek() in u' \t':
+ self.forward()
+ if self.peek() in u'\r\n\x85\u2028\u2029':
+ chunks.append(self.scan_line_break())
+ else:
+ return chunks
+
+ def scan_plain(self):
+ # See the specification for details.
+ # We add an additional restriction for the flow context:
+ # plain scalars in the flow context cannot contain ',', ':' and '?'.
+ # We also keep track of the `allow_simple_key` flag here.
+ # Indentation rules are loosed for the flow context.
+ chunks = []
+ start_mark = self.get_mark()
+ end_mark = start_mark
+ indent = self.indent+1
+ # We allow zero indentation for scalars, but then we need to check for
+ # document separators at the beginning of the line.
+ #if indent == 0:
+ # indent = 1
+ spaces = []
+ while True:
+ length = 0
+ if self.peek() == u'#':
+ break
+ while True:
+ ch = self.peek(length)
+ if ch in u'\0 \t\r\n\x85\u2028\u2029' \
+ or (not self.flow_level and ch == u':' and
+ self.peek(length+1) in u'\0 \t\r\n\x85\u2028\u2029') \
+ or (self.flow_level and ch in u',:?[]{}'):
+ break
+ length += 1
+ # It's not clear what we should do with ':' in the flow context.
+ if (self.flow_level and ch == u':'
+ and self.peek(length+1) not in u'\0 \t\r\n\x85\u2028\u2029,[]{}'):
+ self.forward(length)
+ raise ScannerError("while scanning a plain scalar", start_mark,
+ "found unexpected ':'", self.get_mark(),
+ "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
+ if length == 0:
+ break
+ self.allow_simple_key = False
+ chunks.extend(spaces)
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ end_mark = self.get_mark()
+ spaces = self.scan_plain_spaces(indent, start_mark)
+ if not spaces or self.peek() == u'#' \
+ or (not self.flow_level and self.column < indent):
+ break
+ return ScalarToken(u''.join(chunks), True, start_mark, end_mark)
+
+ def scan_plain_spaces(self, indent, start_mark):
+ # See the specification for details.
+ # The specification is really confusing about tabs in plain scalars.
+ # We just forbid them completely. Do not use tabs in YAML!
+ chunks = []
+ length = 0
+ while self.peek(length) in u' ':
+ length += 1
+ whitespaces = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch in u'\r\n\x85\u2028\u2029':
+ line_break = self.scan_line_break()
+ self.allow_simple_key = True
+ prefix = self.prefix(3)
+ if (prefix == u'---' or prefix == u'...') \
+ and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
+ return
+ breaks = []
+ while self.peek() in u' \r\n\x85\u2028\u2029':
+ if self.peek() == ' ':
+ self.forward()
+ else:
+ breaks.append(self.scan_line_break())
+ prefix = self.prefix(3)
+ if (prefix == u'---' or prefix == u'...') \
+ and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
+ return
+ if line_break != u'\n':
+ chunks.append(line_break)
+ elif not breaks:
+ chunks.append(u' ')
+ chunks.extend(breaks)
+ elif whitespaces:
+ chunks.append(whitespaces)
+ return chunks
+
+ def scan_tag_handle(self, name, start_mark):
+ # See the specification for details.
+ # For some strange reasons, the specification does not allow '_' in
+ # tag handles. I have allowed it anyway.
+ ch = self.peek()
+ if ch != u'!':
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected '!', but found %r" % ch.encode('utf-8'),
+ self.get_mark())
+ length = 1
+ ch = self.peek(length)
+ if ch != u' ':
+ while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
+ or ch in u'-_':
+ length += 1
+ ch = self.peek(length)
+ if ch != u'!':
+ self.forward(length)
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected '!', but found %r" % ch.encode('utf-8'),
+ self.get_mark())
+ length += 1
+ value = self.prefix(length)
+ self.forward(length)
+ return value
+
+ def scan_tag_uri(self, name, start_mark):
+ # See the specification for details.
+ # Note: we do not check if URI is well-formed.
+ chunks = []
+ length = 0
+ ch = self.peek(length)
+ while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
+ or ch in u'-;/?:@&=+$,_.!~*\'()[]%':
+ if ch == u'%':
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ length = 0
+ chunks.append(self.scan_uri_escapes(name, start_mark))
+ else:
+ length += 1
+ ch = self.peek(length)
+ if length:
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ length = 0
+ if not chunks:
+ raise ScannerError("while parsing a %s" % name, start_mark,
+ "expected URI, but found %r" % ch.encode('utf-8'),
+ self.get_mark())
+ return u''.join(chunks)
+
+ def scan_uri_escapes(self, name, start_mark):
+ # See the specification for details.
+ bytes = []
+ mark = self.get_mark()
+ while self.peek() == u'%':
+ self.forward()
+ for k in range(2):
+ if self.peek(k) not in u'0123456789ABCDEFabcdef':
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected URI escape sequence of 2 hexdecimal numbers, but found %r" %
+ (self.peek(k).encode('utf-8')), self.get_mark())
+ bytes.append(chr(int(self.prefix(2), 16)))
+ self.forward(2)
+ try:
+ value = unicode(''.join(bytes), 'utf-8')
+ except UnicodeDecodeError, exc:
+ raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
+ return value
+
+ def scan_line_break(self):
+ # Transforms:
+ # '\r\n' : '\n'
+ # '\r' : '\n'
+ # '\n' : '\n'
+ # '\x85' : '\n'
+ # '\u2028' : '\u2028'
+ # '\u2029 : '\u2029'
+ # default : ''
+ ch = self.peek()
+ if ch in u'\r\n\x85':
+ if self.prefix(2) == u'\r\n':
+ self.forward(2)
+ else:
+ self.forward()
+ return u'\n'
+ elif ch in u'\u2028\u2029':
+ self.forward()
+ return ch
+ return u''
+
+#try:
+# import psyco
+# psyco.bind(Scanner)
+#except ImportError:
+# pass
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/serializer.py b/collectors/python.d.plugin/python_modules/pyyaml2/serializer.py
new file mode 100644
index 0000000..15fdbb0
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/serializer.py
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['Serializer', 'SerializerError']
+
+from error import YAMLError
+from events import *
+from nodes import *
+
+class SerializerError(YAMLError):
+ pass
+
+class Serializer(object):
+
+ ANCHOR_TEMPLATE = u'id%03d'
+
+ def __init__(self, encoding=None,
+ explicit_start=None, explicit_end=None, version=None, tags=None):
+ self.use_encoding = encoding
+ self.use_explicit_start = explicit_start
+ self.use_explicit_end = explicit_end
+ self.use_version = version
+ self.use_tags = tags
+ self.serialized_nodes = {}
+ self.anchors = {}
+ self.last_anchor_id = 0
+ self.closed = None
+
+ def open(self):
+ if self.closed is None:
+ self.emit(StreamStartEvent(encoding=self.use_encoding))
+ self.closed = False
+ elif self.closed:
+ raise SerializerError("serializer is closed")
+ else:
+ raise SerializerError("serializer is already opened")
+
+ def close(self):
+ if self.closed is None:
+ raise SerializerError("serializer is not opened")
+ elif not self.closed:
+ self.emit(StreamEndEvent())
+ self.closed = True
+
+ #def __del__(self):
+ # self.close()
+
+ def serialize(self, node):
+ if self.closed is None:
+ raise SerializerError("serializer is not opened")
+ elif self.closed:
+ raise SerializerError("serializer is closed")
+ self.emit(DocumentStartEvent(explicit=self.use_explicit_start,
+ version=self.use_version, tags=self.use_tags))
+ self.anchor_node(node)
+ self.serialize_node(node, None, None)
+ self.emit(DocumentEndEvent(explicit=self.use_explicit_end))
+ self.serialized_nodes = {}
+ self.anchors = {}
+ self.last_anchor_id = 0
+
+ def anchor_node(self, node):
+ if node in self.anchors:
+ if self.anchors[node] is None:
+ self.anchors[node] = self.generate_anchor(node)
+ else:
+ self.anchors[node] = None
+ if isinstance(node, SequenceNode):
+ for item in node.value:
+ self.anchor_node(item)
+ elif isinstance(node, MappingNode):
+ for key, value in node.value:
+ self.anchor_node(key)
+ self.anchor_node(value)
+
+ def generate_anchor(self, node):
+ self.last_anchor_id += 1
+ return self.ANCHOR_TEMPLATE % self.last_anchor_id
+
+ def serialize_node(self, node, parent, index):
+ alias = self.anchors[node]
+ if node in self.serialized_nodes:
+ self.emit(AliasEvent(alias))
+ else:
+ self.serialized_nodes[node] = True
+ self.descend_resolver(parent, index)
+ if isinstance(node, ScalarNode):
+ detected_tag = self.resolve(ScalarNode, node.value, (True, False))
+ default_tag = self.resolve(ScalarNode, node.value, (False, True))
+ implicit = (node.tag == detected_tag), (node.tag == default_tag)
+ self.emit(ScalarEvent(alias, node.tag, implicit, node.value,
+ style=node.style))
+ elif isinstance(node, SequenceNode):
+ implicit = (node.tag
+ == self.resolve(SequenceNode, node.value, True))
+ self.emit(SequenceStartEvent(alias, node.tag, implicit,
+ flow_style=node.flow_style))
+ index = 0
+ for item in node.value:
+ self.serialize_node(item, node, index)
+ index += 1
+ self.emit(SequenceEndEvent())
+ elif isinstance(node, MappingNode):
+ implicit = (node.tag
+ == self.resolve(MappingNode, node.value, True))
+ self.emit(MappingStartEvent(alias, node.tag, implicit,
+ flow_style=node.flow_style))
+ for key, value in node.value:
+ self.serialize_node(key, node, None)
+ self.serialize_node(value, node, key)
+ self.emit(MappingEndEvent())
+ self.ascend_resolver()
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml2/tokens.py b/collectors/python.d.plugin/python_modules/pyyaml2/tokens.py
new file mode 100644
index 0000000..c5c4fb1
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml2/tokens.py
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: MIT
+
+class Token(object):
+ def __init__(self, start_mark, end_mark):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ attributes = [key for key in self.__dict__
+ if not key.endswith('_mark')]
+ attributes.sort()
+ arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+ for key in attributes])
+ return '%s(%s)' % (self.__class__.__name__, arguments)
+
+#class BOMToken(Token):
+# id = '<byte order mark>'
+
+class DirectiveToken(Token):
+ id = '<directive>'
+ def __init__(self, name, value, start_mark, end_mark):
+ self.name = name
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class DocumentStartToken(Token):
+ id = '<document start>'
+
+class DocumentEndToken(Token):
+ id = '<document end>'
+
+class StreamStartToken(Token):
+ id = '<stream start>'
+ def __init__(self, start_mark=None, end_mark=None,
+ encoding=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.encoding = encoding
+
+class StreamEndToken(Token):
+ id = '<stream end>'
+
+class BlockSequenceStartToken(Token):
+ id = '<block sequence start>'
+
+class BlockMappingStartToken(Token):
+ id = '<block mapping start>'
+
+class BlockEndToken(Token):
+ id = '<block end>'
+
+class FlowSequenceStartToken(Token):
+ id = '['
+
+class FlowMappingStartToken(Token):
+ id = '{'
+
+class FlowSequenceEndToken(Token):
+ id = ']'
+
+class FlowMappingEndToken(Token):
+ id = '}'
+
+class KeyToken(Token):
+ id = '?'
+
+class ValueToken(Token):
+ id = ':'
+
+class BlockEntryToken(Token):
+ id = '-'
+
+class FlowEntryToken(Token):
+ id = ','
+
+class AliasToken(Token):
+ id = '<alias>'
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class AnchorToken(Token):
+ id = '<anchor>'
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class TagToken(Token):
+ id = '<tag>'
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class ScalarToken(Token):
+ id = '<scalar>'
+ def __init__(self, value, plain, start_mark, end_mark, style=None):
+ self.value = value
+ self.plain = plain
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/__init__.py b/collectors/python.d.plugin/python_modules/pyyaml3/__init__.py
new file mode 100644
index 0000000..a884b33
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/__init__.py
@@ -0,0 +1,313 @@
+# SPDX-License-Identifier: MIT
+
+from .error import *
+
+from .tokens import *
+from .events import *
+from .nodes import *
+
+from .loader import *
+from .dumper import *
+
+__version__ = '3.11'
+try:
+ from .cyaml import *
+ __with_libyaml__ = True
+except ImportError:
+ __with_libyaml__ = False
+
+import io
+
+def scan(stream, Loader=Loader):
+ """
+ Scan a YAML stream and produce scanning tokens.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_token():
+ yield loader.get_token()
+ finally:
+ loader.dispose()
+
+def parse(stream, Loader=Loader):
+ """
+ Parse a YAML stream and produce parsing events.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_event():
+ yield loader.get_event()
+ finally:
+ loader.dispose()
+
+def compose(stream, Loader=Loader):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding representation tree.
+ """
+ loader = Loader(stream)
+ try:
+ return loader.get_single_node()
+ finally:
+ loader.dispose()
+
+def compose_all(stream, Loader=Loader):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding representation trees.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_node():
+ yield loader.get_node()
+ finally:
+ loader.dispose()
+
+def load(stream, Loader=Loader):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding Python object.
+ """
+ loader = Loader(stream)
+ try:
+ return loader.get_single_data()
+ finally:
+ loader.dispose()
+
+def load_all(stream, Loader=Loader):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding Python objects.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_data():
+ yield loader.get_data()
+ finally:
+ loader.dispose()
+
+def safe_load(stream):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding Python object.
+ Resolve only basic YAML tags.
+ """
+ return load(stream, SafeLoader)
+
+def safe_load_all(stream):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding Python objects.
+ Resolve only basic YAML tags.
+ """
+ return load_all(stream, SafeLoader)
+
+def emit(events, stream=None, Dumper=Dumper,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None):
+ """
+ Emit YAML parsing events into a stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ stream = io.StringIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ try:
+ for event in events:
+ dumper.emit(event)
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def serialize_all(nodes, stream=None, Dumper=Dumper,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ """
+ Serialize a sequence of representation trees into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ if encoding is None:
+ stream = io.StringIO()
+ else:
+ stream = io.BytesIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break,
+ encoding=encoding, version=version, tags=tags,
+ explicit_start=explicit_start, explicit_end=explicit_end)
+ try:
+ dumper.open()
+ for node in nodes:
+ dumper.serialize(node)
+ dumper.close()
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def serialize(node, stream=None, Dumper=Dumper, **kwds):
+ """
+ Serialize a representation tree into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ return serialize_all([node], stream, Dumper=Dumper, **kwds)
+
+def dump_all(documents, stream=None, Dumper=Dumper,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ """
+ Serialize a sequence of Python objects into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ if encoding is None:
+ stream = io.StringIO()
+ else:
+ stream = io.BytesIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, default_style=default_style,
+ default_flow_style=default_flow_style,
+ canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break,
+ encoding=encoding, version=version, tags=tags,
+ explicit_start=explicit_start, explicit_end=explicit_end)
+ try:
+ dumper.open()
+ for data in documents:
+ dumper.represent(data)
+ dumper.close()
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def dump(data, stream=None, Dumper=Dumper, **kwds):
+ """
+ Serialize a Python object into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all([data], stream, Dumper=Dumper, **kwds)
+
+def safe_dump_all(documents, stream=None, **kwds):
+ """
+ Serialize a sequence of Python objects into a YAML stream.
+ Produce only basic YAML tags.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all(documents, stream, Dumper=SafeDumper, **kwds)
+
+def safe_dump(data, stream=None, **kwds):
+ """
+ Serialize a Python object into a YAML stream.
+ Produce only basic YAML tags.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all([data], stream, Dumper=SafeDumper, **kwds)
+
+def add_implicit_resolver(tag, regexp, first=None,
+ Loader=Loader, Dumper=Dumper):
+ """
+ Add an implicit scalar detector.
+ If an implicit scalar value matches the given regexp,
+ the corresponding tag is assigned to the scalar.
+ first is a sequence of possible initial characters or None.
+ """
+ Loader.add_implicit_resolver(tag, regexp, first)
+ Dumper.add_implicit_resolver(tag, regexp, first)
+
+def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper):
+ """
+ Add a path based resolver for the given tag.
+ A path is a list of keys that forms a path
+ to a node in the representation tree.
+ Keys can be string values, integers, or None.
+ """
+ Loader.add_path_resolver(tag, path, kind)
+ Dumper.add_path_resolver(tag, path, kind)
+
+def add_constructor(tag, constructor, Loader=Loader):
+ """
+ Add a constructor for the given tag.
+ Constructor is a function that accepts a Loader instance
+ and a node object and produces the corresponding Python object.
+ """
+ Loader.add_constructor(tag, constructor)
+
+def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader):
+ """
+ Add a multi-constructor for the given tag prefix.
+ Multi-constructor is called for a node if its tag starts with tag_prefix.
+ Multi-constructor accepts a Loader instance, a tag suffix,
+ and a node object and produces the corresponding Python object.
+ """
+ Loader.add_multi_constructor(tag_prefix, multi_constructor)
+
+def add_representer(data_type, representer, Dumper=Dumper):
+ """
+ Add a representer for the given type.
+ Representer is a function accepting a Dumper instance
+ and an instance of the given data type
+ and producing the corresponding representation node.
+ """
+ Dumper.add_representer(data_type, representer)
+
+def add_multi_representer(data_type, multi_representer, Dumper=Dumper):
+ """
+ Add a representer for the given type.
+ Multi-representer is a function accepting a Dumper instance
+ and an instance of the given data type or subtype
+ and producing the corresponding representation node.
+ """
+ Dumper.add_multi_representer(data_type, multi_representer)
+
+class YAMLObjectMetaclass(type):
+ """
+ The metaclass for YAMLObject.
+ """
+ def __init__(cls, name, bases, kwds):
+ super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
+ if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
+ cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
+ cls.yaml_dumper.add_representer(cls, cls.to_yaml)
+
+class YAMLObject(metaclass=YAMLObjectMetaclass):
+ """
+ An object that can dump itself to a YAML stream
+ and load itself from a YAML stream.
+ """
+
+ __slots__ = () # no direct instantiation, so allow immutable subclasses
+
+ yaml_loader = Loader
+ yaml_dumper = Dumper
+
+ yaml_tag = None
+ yaml_flow_style = None
+
+ @classmethod
+ def from_yaml(cls, loader, node):
+ """
+ Convert a representation node to a Python object.
+ """
+ return loader.construct_yaml_object(node, cls)
+
+ @classmethod
+ def to_yaml(cls, dumper, data):
+ """
+ Convert a Python object to a representation node.
+ """
+ return dumper.represent_yaml_object(cls.yaml_tag, data, cls,
+ flow_style=cls.yaml_flow_style)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/composer.py b/collectors/python.d.plugin/python_modules/pyyaml3/composer.py
new file mode 100644
index 0000000..c418bba
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/composer.py
@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['Composer', 'ComposerError']
+
+from .error import MarkedYAMLError
+from .events import *
+from .nodes import *
+
+class ComposerError(MarkedYAMLError):
+ pass
+
+class Composer:
+
+ def __init__(self):
+ self.anchors = {}
+
+ def check_node(self):
+ # Drop the STREAM-START event.
+ if self.check_event(StreamStartEvent):
+ self.get_event()
+
+ # If there are more documents available?
+ return not self.check_event(StreamEndEvent)
+
+ def get_node(self):
+ # Get the root node of the next document.
+ if not self.check_event(StreamEndEvent):
+ return self.compose_document()
+
+ def get_single_node(self):
+ # Drop the STREAM-START event.
+ self.get_event()
+
+ # Compose a document if the stream is not empty.
+ document = None
+ if not self.check_event(StreamEndEvent):
+ document = self.compose_document()
+
+ # Ensure that the stream contains no more documents.
+ if not self.check_event(StreamEndEvent):
+ event = self.get_event()
+ raise ComposerError("expected a single document in the stream",
+ document.start_mark, "but found another document",
+ event.start_mark)
+
+ # Drop the STREAM-END event.
+ self.get_event()
+
+ return document
+
+ def compose_document(self):
+ # Drop the DOCUMENT-START event.
+ self.get_event()
+
+ # Compose the root node.
+ node = self.compose_node(None, None)
+
+ # Drop the DOCUMENT-END event.
+ self.get_event()
+
+ self.anchors = {}
+ return node
+
+ def compose_node(self, parent, index):
+ if self.check_event(AliasEvent):
+ event = self.get_event()
+ anchor = event.anchor
+ if anchor not in self.anchors:
+ raise ComposerError(None, None, "found undefined alias %r"
+ % anchor, event.start_mark)
+ return self.anchors[anchor]
+ event = self.peek_event()
+ anchor = event.anchor
+ if anchor is not None:
+ if anchor in self.anchors:
+ raise ComposerError("found duplicate anchor %r; first occurence"
+ % anchor, self.anchors[anchor].start_mark,
+ "second occurence", event.start_mark)
+ self.descend_resolver(parent, index)
+ if self.check_event(ScalarEvent):
+ node = self.compose_scalar_node(anchor)
+ elif self.check_event(SequenceStartEvent):
+ node = self.compose_sequence_node(anchor)
+ elif self.check_event(MappingStartEvent):
+ node = self.compose_mapping_node(anchor)
+ self.ascend_resolver()
+ return node
+
+ def compose_scalar_node(self, anchor):
+ event = self.get_event()
+ tag = event.tag
+ if tag is None or tag == '!':
+ tag = self.resolve(ScalarNode, event.value, event.implicit)
+ node = ScalarNode(tag, event.value,
+ event.start_mark, event.end_mark, style=event.style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ return node
+
+ def compose_sequence_node(self, anchor):
+ start_event = self.get_event()
+ tag = start_event.tag
+ if tag is None or tag == '!':
+ tag = self.resolve(SequenceNode, None, start_event.implicit)
+ node = SequenceNode(tag, [],
+ start_event.start_mark, None,
+ flow_style=start_event.flow_style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ index = 0
+ while not self.check_event(SequenceEndEvent):
+ node.value.append(self.compose_node(node, index))
+ index += 1
+ end_event = self.get_event()
+ node.end_mark = end_event.end_mark
+ return node
+
+ def compose_mapping_node(self, anchor):
+ start_event = self.get_event()
+ tag = start_event.tag
+ if tag is None or tag == '!':
+ tag = self.resolve(MappingNode, None, start_event.implicit)
+ node = MappingNode(tag, [],
+ start_event.start_mark, None,
+ flow_style=start_event.flow_style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ while not self.check_event(MappingEndEvent):
+ #key_event = self.peek_event()
+ item_key = self.compose_node(node, None)
+ #if item_key in node.value:
+ # raise ComposerError("while composing a mapping", start_event.start_mark,
+ # "found duplicate key", key_event.start_mark)
+ item_value = self.compose_node(node, item_key)
+ #node.value[item_key] = item_value
+ node.value.append((item_key, item_value))
+ end_event = self.get_event()
+ node.end_mark = end_event.end_mark
+ return node
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/constructor.py b/collectors/python.d.plugin/python_modules/pyyaml3/constructor.py
new file mode 100644
index 0000000..ee09a7a
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/constructor.py
@@ -0,0 +1,687 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
+ 'ConstructorError']
+
+from .error import *
+from .nodes import *
+
+import collections, datetime, base64, binascii, re, sys, types
+
+class ConstructorError(MarkedYAMLError):
+ pass
+
+class BaseConstructor:
+
+ yaml_constructors = {}
+ yaml_multi_constructors = {}
+
+ def __init__(self):
+ self.constructed_objects = {}
+ self.recursive_objects = {}
+ self.state_generators = []
+ self.deep_construct = False
+
+ def check_data(self):
+ # If there are more documents available?
+ return self.check_node()
+
+ def get_data(self):
+ # Construct and return the next document.
+ if self.check_node():
+ return self.construct_document(self.get_node())
+
+ def get_single_data(self):
+ # Ensure that the stream contains a single document and construct it.
+ node = self.get_single_node()
+ if node is not None:
+ return self.construct_document(node)
+ return None
+
+ def construct_document(self, node):
+ data = self.construct_object(node)
+ while self.state_generators:
+ state_generators = self.state_generators
+ self.state_generators = []
+ for generator in state_generators:
+ for dummy in generator:
+ pass
+ self.constructed_objects = {}
+ self.recursive_objects = {}
+ self.deep_construct = False
+ return data
+
+ def construct_object(self, node, deep=False):
+ if node in self.constructed_objects:
+ return self.constructed_objects[node]
+ if deep:
+ old_deep = self.deep_construct
+ self.deep_construct = True
+ if node in self.recursive_objects:
+ raise ConstructorError(None, None,
+ "found unconstructable recursive node", node.start_mark)
+ self.recursive_objects[node] = None
+ constructor = None
+ tag_suffix = None
+ if node.tag in self.yaml_constructors:
+ constructor = self.yaml_constructors[node.tag]
+ else:
+ for tag_prefix in self.yaml_multi_constructors:
+ if node.tag.startswith(tag_prefix):
+ tag_suffix = node.tag[len(tag_prefix):]
+ constructor = self.yaml_multi_constructors[tag_prefix]
+ break
+ else:
+ if None in self.yaml_multi_constructors:
+ tag_suffix = node.tag
+ constructor = self.yaml_multi_constructors[None]
+ elif None in self.yaml_constructors:
+ constructor = self.yaml_constructors[None]
+ elif isinstance(node, ScalarNode):
+ constructor = self.__class__.construct_scalar
+ elif isinstance(node, SequenceNode):
+ constructor = self.__class__.construct_sequence
+ elif isinstance(node, MappingNode):
+ constructor = self.__class__.construct_mapping
+ if tag_suffix is None:
+ data = constructor(self, node)
+ else:
+ data = constructor(self, tag_suffix, node)
+ if isinstance(data, types.GeneratorType):
+ generator = data
+ data = next(generator)
+ if self.deep_construct:
+ for dummy in generator:
+ pass
+ else:
+ self.state_generators.append(generator)
+ self.constructed_objects[node] = data
+ del self.recursive_objects[node]
+ if deep:
+ self.deep_construct = old_deep
+ return data
+
+ def construct_scalar(self, node):
+ if not isinstance(node, ScalarNode):
+ raise ConstructorError(None, None,
+ "expected a scalar node, but found %s" % node.id,
+ node.start_mark)
+ return node.value
+
+ def construct_sequence(self, node, deep=False):
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError(None, None,
+ "expected a sequence node, but found %s" % node.id,
+ node.start_mark)
+ return [self.construct_object(child, deep=deep)
+ for child in node.value]
+
+ def construct_mapping(self, node, deep=False):
+ if not isinstance(node, MappingNode):
+ raise ConstructorError(None, None,
+ "expected a mapping node, but found %s" % node.id,
+ node.start_mark)
+ mapping = {}
+ for key_node, value_node in node.value:
+ key = self.construct_object(key_node, deep=deep)
+ if not isinstance(key, collections.Hashable):
+ raise ConstructorError("while constructing a mapping", node.start_mark,
+ "found unhashable key", key_node.start_mark)
+ value = self.construct_object(value_node, deep=deep)
+ mapping[key] = value
+ return mapping
+
+ def construct_pairs(self, node, deep=False):
+ if not isinstance(node, MappingNode):
+ raise ConstructorError(None, None,
+ "expected a mapping node, but found %s" % node.id,
+ node.start_mark)
+ pairs = []
+ for key_node, value_node in node.value:
+ key = self.construct_object(key_node, deep=deep)
+ value = self.construct_object(value_node, deep=deep)
+ pairs.append((key, value))
+ return pairs
+
+ @classmethod
+ def add_constructor(cls, tag, constructor):
+ if not 'yaml_constructors' in cls.__dict__:
+ cls.yaml_constructors = cls.yaml_constructors.copy()
+ cls.yaml_constructors[tag] = constructor
+
+ @classmethod
+ def add_multi_constructor(cls, tag_prefix, multi_constructor):
+ if not 'yaml_multi_constructors' in cls.__dict__:
+ cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
+ cls.yaml_multi_constructors[tag_prefix] = multi_constructor
+
+class SafeConstructor(BaseConstructor):
+
+ def construct_scalar(self, node):
+ if isinstance(node, MappingNode):
+ for key_node, value_node in node.value:
+ if key_node.tag == 'tag:yaml.org,2002:value':
+ return self.construct_scalar(value_node)
+ return super().construct_scalar(node)
+
+ def flatten_mapping(self, node):
+ merge = []
+ index = 0
+ while index < len(node.value):
+ key_node, value_node = node.value[index]
+ if key_node.tag == 'tag:yaml.org,2002:merge':
+ del node.value[index]
+ if isinstance(value_node, MappingNode):
+ self.flatten_mapping(value_node)
+ merge.extend(value_node.value)
+ elif isinstance(value_node, SequenceNode):
+ submerge = []
+ for subnode in value_node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing a mapping",
+ node.start_mark,
+ "expected a mapping for merging, but found %s"
+ % subnode.id, subnode.start_mark)
+ self.flatten_mapping(subnode)
+ submerge.append(subnode.value)
+ submerge.reverse()
+ for value in submerge:
+ merge.extend(value)
+ else:
+ raise ConstructorError("while constructing a mapping", node.start_mark,
+ "expected a mapping or list of mappings for merging, but found %s"
+ % value_node.id, value_node.start_mark)
+ elif key_node.tag == 'tag:yaml.org,2002:value':
+ key_node.tag = 'tag:yaml.org,2002:str'
+ index += 1
+ else:
+ index += 1
+ if merge:
+ node.value = merge + node.value
+
+ def construct_mapping(self, node, deep=False):
+ if isinstance(node, MappingNode):
+ self.flatten_mapping(node)
+ return super().construct_mapping(node, deep=deep)
+
+ def construct_yaml_null(self, node):
+ self.construct_scalar(node)
+ return None
+
+ bool_values = {
+ 'yes': True,
+ 'no': False,
+ 'true': True,
+ 'false': False,
+ 'on': True,
+ 'off': False,
+ }
+
+ def construct_yaml_bool(self, node):
+ value = self.construct_scalar(node)
+ return self.bool_values[value.lower()]
+
+ def construct_yaml_int(self, node):
+ value = self.construct_scalar(node)
+ value = value.replace('_', '')
+ sign = +1
+ if value[0] == '-':
+ sign = -1
+ if value[0] in '+-':
+ value = value[1:]
+ if value == '0':
+ return 0
+ elif value.startswith('0b'):
+ return sign*int(value[2:], 2)
+ elif value.startswith('0x'):
+ return sign*int(value[2:], 16)
+ elif value[0] == '0':
+ return sign*int(value, 8)
+ elif ':' in value:
+ digits = [int(part) for part in value.split(':')]
+ digits.reverse()
+ base = 1
+ value = 0
+ for digit in digits:
+ value += digit*base
+ base *= 60
+ return sign*value
+ else:
+ return sign*int(value)
+
+ inf_value = 1e300
+ while inf_value != inf_value*inf_value:
+ inf_value *= inf_value
+ nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99).
+
+ def construct_yaml_float(self, node):
+ value = self.construct_scalar(node)
+ value = value.replace('_', '').lower()
+ sign = +1
+ if value[0] == '-':
+ sign = -1
+ if value[0] in '+-':
+ value = value[1:]
+ if value == '.inf':
+ return sign*self.inf_value
+ elif value == '.nan':
+ return self.nan_value
+ elif ':' in value:
+ digits = [float(part) for part in value.split(':')]
+ digits.reverse()
+ base = 1
+ value = 0.0
+ for digit in digits:
+ value += digit*base
+ base *= 60
+ return sign*value
+ else:
+ return sign*float(value)
+
+ def construct_yaml_binary(self, node):
+ try:
+ value = self.construct_scalar(node).encode('ascii')
+ except UnicodeEncodeError as exc:
+ raise ConstructorError(None, None,
+ "failed to convert base64 data into ascii: %s" % exc,
+ node.start_mark)
+ try:
+ if hasattr(base64, 'decodebytes'):
+ return base64.decodebytes(value)
+ else:
+ return base64.decodestring(value)
+ except binascii.Error as exc:
+ raise ConstructorError(None, None,
+ "failed to decode base64 data: %s" % exc, node.start_mark)
+
+ timestamp_regexp = re.compile(
+ r'''^(?P<year>[0-9][0-9][0-9][0-9])
+ -(?P<month>[0-9][0-9]?)
+ -(?P<day>[0-9][0-9]?)
+ (?:(?:[Tt]|[ \t]+)
+ (?P<hour>[0-9][0-9]?)
+ :(?P<minute>[0-9][0-9])
+ :(?P<second>[0-9][0-9])
+ (?:\.(?P<fraction>[0-9]*))?
+ (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)
+ (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X)
+
+ def construct_yaml_timestamp(self, node):
+ value = self.construct_scalar(node)
+ match = self.timestamp_regexp.match(node.value)
+ values = match.groupdict()
+ year = int(values['year'])
+ month = int(values['month'])
+ day = int(values['day'])
+ if not values['hour']:
+ return datetime.date(year, month, day)
+ hour = int(values['hour'])
+ minute = int(values['minute'])
+ second = int(values['second'])
+ fraction = 0
+ if values['fraction']:
+ fraction = values['fraction'][:6]
+ while len(fraction) < 6:
+ fraction += '0'
+ fraction = int(fraction)
+ delta = None
+ if values['tz_sign']:
+ tz_hour = int(values['tz_hour'])
+ tz_minute = int(values['tz_minute'] or 0)
+ delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
+ if values['tz_sign'] == '-':
+ delta = -delta
+ data = datetime.datetime(year, month, day, hour, minute, second, fraction)
+ if delta:
+ data -= delta
+ return data
+
+ def construct_yaml_omap(self, node):
+ # Note: we do not check for duplicate keys, because it's too
+ # CPU-expensive.
+ omap = []
+ yield omap
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a sequence, but found %s" % node.id, node.start_mark)
+ for subnode in node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a mapping of length 1, but found %s" % subnode.id,
+ subnode.start_mark)
+ if len(subnode.value) != 1:
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a single mapping item, but found %d items" % len(subnode.value),
+ subnode.start_mark)
+ key_node, value_node = subnode.value[0]
+ key = self.construct_object(key_node)
+ value = self.construct_object(value_node)
+ omap.append((key, value))
+
+ def construct_yaml_pairs(self, node):
+ # Note: the same code as `construct_yaml_omap`.
+ pairs = []
+ yield pairs
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a sequence, but found %s" % node.id, node.start_mark)
+ for subnode in node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a mapping of length 1, but found %s" % subnode.id,
+ subnode.start_mark)
+ if len(subnode.value) != 1:
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a single mapping item, but found %d items" % len(subnode.value),
+ subnode.start_mark)
+ key_node, value_node = subnode.value[0]
+ key = self.construct_object(key_node)
+ value = self.construct_object(value_node)
+ pairs.append((key, value))
+
+ def construct_yaml_set(self, node):
+ data = set()
+ yield data
+ value = self.construct_mapping(node)
+ data.update(value)
+
+ def construct_yaml_str(self, node):
+ return self.construct_scalar(node)
+
+ def construct_yaml_seq(self, node):
+ data = []
+ yield data
+ data.extend(self.construct_sequence(node))
+
+ def construct_yaml_map(self, node):
+ data = {}
+ yield data
+ value = self.construct_mapping(node)
+ data.update(value)
+
+ def construct_yaml_object(self, node, cls):
+ data = cls.__new__(cls)
+ yield data
+ if hasattr(data, '__setstate__'):
+ state = self.construct_mapping(node, deep=True)
+ data.__setstate__(state)
+ else:
+ state = self.construct_mapping(node)
+ data.__dict__.update(state)
+
+ def construct_undefined(self, node):
+ raise ConstructorError(None, None,
+ "could not determine a constructor for the tag %r" % node.tag,
+ node.start_mark)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:null',
+ SafeConstructor.construct_yaml_null)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:bool',
+ SafeConstructor.construct_yaml_bool)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:int',
+ SafeConstructor.construct_yaml_int)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:float',
+ SafeConstructor.construct_yaml_float)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:binary',
+ SafeConstructor.construct_yaml_binary)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:timestamp',
+ SafeConstructor.construct_yaml_timestamp)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:omap',
+ SafeConstructor.construct_yaml_omap)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:pairs',
+ SafeConstructor.construct_yaml_pairs)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:set',
+ SafeConstructor.construct_yaml_set)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:str',
+ SafeConstructor.construct_yaml_str)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:seq',
+ SafeConstructor.construct_yaml_seq)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:map',
+ SafeConstructor.construct_yaml_map)
+
+SafeConstructor.add_constructor(None,
+ SafeConstructor.construct_undefined)
+
+class Constructor(SafeConstructor):
+
+ def construct_python_str(self, node):
+ return self.construct_scalar(node)
+
+ def construct_python_unicode(self, node):
+ return self.construct_scalar(node)
+
+ def construct_python_bytes(self, node):
+ try:
+ value = self.construct_scalar(node).encode('ascii')
+ except UnicodeEncodeError as exc:
+ raise ConstructorError(None, None,
+ "failed to convert base64 data into ascii: %s" % exc,
+ node.start_mark)
+ try:
+ if hasattr(base64, 'decodebytes'):
+ return base64.decodebytes(value)
+ else:
+ return base64.decodestring(value)
+ except binascii.Error as exc:
+ raise ConstructorError(None, None,
+ "failed to decode base64 data: %s" % exc, node.start_mark)
+
+ def construct_python_long(self, node):
+ return self.construct_yaml_int(node)
+
+ def construct_python_complex(self, node):
+ return complex(self.construct_scalar(node))
+
+ def construct_python_tuple(self, node):
+ return tuple(self.construct_sequence(node))
+
+ def find_python_module(self, name, mark):
+ if not name:
+ raise ConstructorError("while constructing a Python module", mark,
+ "expected non-empty name appended to the tag", mark)
+ try:
+ __import__(name)
+ except ImportError as exc:
+ raise ConstructorError("while constructing a Python module", mark,
+ "cannot find module %r (%s)" % (name, exc), mark)
+ return sys.modules[name]
+
+ def find_python_name(self, name, mark):
+ if not name:
+ raise ConstructorError("while constructing a Python object", mark,
+ "expected non-empty name appended to the tag", mark)
+ if '.' in name:
+ module_name, object_name = name.rsplit('.', 1)
+ else:
+ module_name = 'builtins'
+ object_name = name
+ try:
+ __import__(module_name)
+ except ImportError as exc:
+ raise ConstructorError("while constructing a Python object", mark,
+ "cannot find module %r (%s)" % (module_name, exc), mark)
+ module = sys.modules[module_name]
+ if not hasattr(module, object_name):
+ raise ConstructorError("while constructing a Python object", mark,
+ "cannot find %r in the module %r"
+ % (object_name, module.__name__), mark)
+ return getattr(module, object_name)
+
+ def construct_python_name(self, suffix, node):
+ value = self.construct_scalar(node)
+ if value:
+ raise ConstructorError("while constructing a Python name", node.start_mark,
+ "expected the empty value, but found %r" % value, node.start_mark)
+ return self.find_python_name(suffix, node.start_mark)
+
+ def construct_python_module(self, suffix, node):
+ value = self.construct_scalar(node)
+ if value:
+ raise ConstructorError("while constructing a Python module", node.start_mark,
+ "expected the empty value, but found %r" % value, node.start_mark)
+ return self.find_python_module(suffix, node.start_mark)
+
+ def make_python_instance(self, suffix, node,
+ args=None, kwds=None, newobj=False):
+ if not args:
+ args = []
+ if not kwds:
+ kwds = {}
+ cls = self.find_python_name(suffix, node.start_mark)
+ if newobj and isinstance(cls, type):
+ return cls.__new__(cls, *args, **kwds)
+ else:
+ return cls(*args, **kwds)
+
+ def set_python_instance_state(self, instance, state):
+ if hasattr(instance, '__setstate__'):
+ instance.__setstate__(state)
+ else:
+ slotstate = {}
+ if isinstance(state, tuple) and len(state) == 2:
+ state, slotstate = state
+ if hasattr(instance, '__dict__'):
+ instance.__dict__.update(state)
+ elif state:
+ slotstate.update(state)
+ for key, value in slotstate.items():
+ setattr(object, key, value)
+
+ def construct_python_object(self, suffix, node):
+ # Format:
+ # !!python/object:module.name { ... state ... }
+ instance = self.make_python_instance(suffix, node, newobj=True)
+ yield instance
+ deep = hasattr(instance, '__setstate__')
+ state = self.construct_mapping(node, deep=deep)
+ self.set_python_instance_state(instance, state)
+
+ def construct_python_object_apply(self, suffix, node, newobj=False):
+ # Format:
+ # !!python/object/apply # (or !!python/object/new)
+ # args: [ ... arguments ... ]
+ # kwds: { ... keywords ... }
+ # state: ... state ...
+ # listitems: [ ... listitems ... ]
+ # dictitems: { ... dictitems ... }
+ # or short format:
+ # !!python/object/apply [ ... arguments ... ]
+ # The difference between !!python/object/apply and !!python/object/new
+ # is how an object is created, check make_python_instance for details.
+ if isinstance(node, SequenceNode):
+ args = self.construct_sequence(node, deep=True)
+ kwds = {}
+ state = {}
+ listitems = []
+ dictitems = {}
+ else:
+ value = self.construct_mapping(node, deep=True)
+ args = value.get('args', [])
+ kwds = value.get('kwds', {})
+ state = value.get('state', {})
+ listitems = value.get('listitems', [])
+ dictitems = value.get('dictitems', {})
+ instance = self.make_python_instance(suffix, node, args, kwds, newobj)
+ if state:
+ self.set_python_instance_state(instance, state)
+ if listitems:
+ instance.extend(listitems)
+ if dictitems:
+ for key in dictitems:
+ instance[key] = dictitems[key]
+ return instance
+
+ def construct_python_object_new(self, suffix, node):
+ return self.construct_python_object_apply(suffix, node, newobj=True)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/none',
+ Constructor.construct_yaml_null)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/bool',
+ Constructor.construct_yaml_bool)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/str',
+ Constructor.construct_python_str)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/unicode',
+ Constructor.construct_python_unicode)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/bytes',
+ Constructor.construct_python_bytes)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/int',
+ Constructor.construct_yaml_int)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/long',
+ Constructor.construct_python_long)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/float',
+ Constructor.construct_yaml_float)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/complex',
+ Constructor.construct_python_complex)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/list',
+ Constructor.construct_yaml_seq)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/tuple',
+ Constructor.construct_python_tuple)
+
+Constructor.add_constructor(
+ 'tag:yaml.org,2002:python/dict',
+ Constructor.construct_yaml_map)
+
+Constructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/name:',
+ Constructor.construct_python_name)
+
+Constructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/module:',
+ Constructor.construct_python_module)
+
+Constructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/object:',
+ Constructor.construct_python_object)
+
+Constructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/object/apply:',
+ Constructor.construct_python_object_apply)
+
+Constructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/object/new:',
+ Constructor.construct_python_object_new)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/cyaml.py b/collectors/python.d.plugin/python_modules/pyyaml3/cyaml.py
new file mode 100644
index 0000000..e6c16d8
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/cyaml.py
@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader',
+ 'CBaseDumper', 'CSafeDumper', 'CDumper']
+
+from _yaml import CParser, CEmitter
+
+from .constructor import *
+
+from .serializer import *
+from .representer import *
+
+from .resolver import *
+
+class CBaseLoader(CParser, BaseConstructor, BaseResolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ BaseConstructor.__init__(self)
+ BaseResolver.__init__(self)
+
+class CSafeLoader(CParser, SafeConstructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ SafeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class CLoader(CParser, Constructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ Constructor.__init__(self)
+ Resolver.__init__(self)
+
+class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ SafeRepresenter.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class CDumper(CEmitter, Serializer, Representer, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/dumper.py b/collectors/python.d.plugin/python_modules/pyyaml3/dumper.py
new file mode 100644
index 0000000..ba590c6
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/dumper.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseDumper', 'SafeDumper', 'Dumper']
+
+from .emitter import *
+from .serializer import *
+from .representer import *
+from .resolver import *
+
+class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ SafeRepresenter.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class Dumper(Emitter, Serializer, Representer, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/emitter.py b/collectors/python.d.plugin/python_modules/pyyaml3/emitter.py
new file mode 100644
index 0000000..d4be65a
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/emitter.py
@@ -0,0 +1,1138 @@
+# SPDX-License-Identifier: MIT
+
+# Emitter expects events obeying the following grammar:
+# stream ::= STREAM-START document* STREAM-END
+# document ::= DOCUMENT-START node DOCUMENT-END
+# node ::= SCALAR | sequence | mapping
+# sequence ::= SEQUENCE-START node* SEQUENCE-END
+# mapping ::= MAPPING-START (node node)* MAPPING-END
+
+__all__ = ['Emitter', 'EmitterError']
+
+from .error import YAMLError
+from .events import *
+
+class EmitterError(YAMLError):
+ pass
+
+class ScalarAnalysis:
+ def __init__(self, scalar, empty, multiline,
+ allow_flow_plain, allow_block_plain,
+ allow_single_quoted, allow_double_quoted,
+ allow_block):
+ self.scalar = scalar
+ self.empty = empty
+ self.multiline = multiline
+ self.allow_flow_plain = allow_flow_plain
+ self.allow_block_plain = allow_block_plain
+ self.allow_single_quoted = allow_single_quoted
+ self.allow_double_quoted = allow_double_quoted
+ self.allow_block = allow_block
+
+class Emitter:
+
+ DEFAULT_TAG_PREFIXES = {
+ '!' : '!',
+ 'tag:yaml.org,2002:' : '!!',
+ }
+
+ def __init__(self, stream, canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None):
+
+ # The stream should have the methods `write` and possibly `flush`.
+ self.stream = stream
+
+ # Encoding can be overriden by STREAM-START.
+ self.encoding = None
+
+ # Emitter is a state machine with a stack of states to handle nested
+ # structures.
+ self.states = []
+ self.state = self.expect_stream_start
+
+ # Current event and the event queue.
+ self.events = []
+ self.event = None
+
+ # The current indentation level and the stack of previous indents.
+ self.indents = []
+ self.indent = None
+
+ # Flow level.
+ self.flow_level = 0
+
+ # Contexts.
+ self.root_context = False
+ self.sequence_context = False
+ self.mapping_context = False
+ self.simple_key_context = False
+
+ # Characteristics of the last emitted character:
+ # - current position.
+ # - is it a whitespace?
+ # - is it an indention character
+ # (indentation space, '-', '?', or ':')?
+ self.line = 0
+ self.column = 0
+ self.whitespace = True
+ self.indention = True
+
+ # Whether the document requires an explicit document indicator
+ self.open_ended = False
+
+ # Formatting details.
+ self.canonical = canonical
+ self.allow_unicode = allow_unicode
+ self.best_indent = 2
+ if indent and 1 < indent < 10:
+ self.best_indent = indent
+ self.best_width = 80
+ if width and width > self.best_indent*2:
+ self.best_width = width
+ self.best_line_break = '\n'
+ if line_break in ['\r', '\n', '\r\n']:
+ self.best_line_break = line_break
+
+ # Tag prefixes.
+ self.tag_prefixes = None
+
+ # Prepared anchor and tag.
+ self.prepared_anchor = None
+ self.prepared_tag = None
+
+ # Scalar analysis and style.
+ self.analysis = None
+ self.style = None
+
+ def dispose(self):
+ # Reset the state attributes (to clear self-references)
+ self.states = []
+ self.state = None
+
+ def emit(self, event):
+ self.events.append(event)
+ while not self.need_more_events():
+ self.event = self.events.pop(0)
+ self.state()
+ self.event = None
+
+ # In some cases, we wait for a few next events before emitting.
+
+ def need_more_events(self):
+ if not self.events:
+ return True
+ event = self.events[0]
+ if isinstance(event, DocumentStartEvent):
+ return self.need_events(1)
+ elif isinstance(event, SequenceStartEvent):
+ return self.need_events(2)
+ elif isinstance(event, MappingStartEvent):
+ return self.need_events(3)
+ else:
+ return False
+
+ def need_events(self, count):
+ level = 0
+ for event in self.events[1:]:
+ if isinstance(event, (DocumentStartEvent, CollectionStartEvent)):
+ level += 1
+ elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)):
+ level -= 1
+ elif isinstance(event, StreamEndEvent):
+ level = -1
+ if level < 0:
+ return False
+ return (len(self.events) < count+1)
+
+ def increase_indent(self, flow=False, indentless=False):
+ self.indents.append(self.indent)
+ if self.indent is None:
+ if flow:
+ self.indent = self.best_indent
+ else:
+ self.indent = 0
+ elif not indentless:
+ self.indent += self.best_indent
+
+ # States.
+
+ # Stream handlers.
+
+ def expect_stream_start(self):
+ if isinstance(self.event, StreamStartEvent):
+ if self.event.encoding and not hasattr(self.stream, 'encoding'):
+ self.encoding = self.event.encoding
+ self.write_stream_start()
+ self.state = self.expect_first_document_start
+ else:
+ raise EmitterError("expected StreamStartEvent, but got %s"
+ % self.event)
+
+ def expect_nothing(self):
+ raise EmitterError("expected nothing, but got %s" % self.event)
+
+ # Document handlers.
+
+ def expect_first_document_start(self):
+ return self.expect_document_start(first=True)
+
+ def expect_document_start(self, first=False):
+ if isinstance(self.event, DocumentStartEvent):
+ if (self.event.version or self.event.tags) and self.open_ended:
+ self.write_indicator('...', True)
+ self.write_indent()
+ if self.event.version:
+ version_text = self.prepare_version(self.event.version)
+ self.write_version_directive(version_text)
+ self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
+ if self.event.tags:
+ handles = sorted(self.event.tags.keys())
+ for handle in handles:
+ prefix = self.event.tags[handle]
+ self.tag_prefixes[prefix] = handle
+ handle_text = self.prepare_tag_handle(handle)
+ prefix_text = self.prepare_tag_prefix(prefix)
+ self.write_tag_directive(handle_text, prefix_text)
+ implicit = (first and not self.event.explicit and not self.canonical
+ and not self.event.version and not self.event.tags
+ and not self.check_empty_document())
+ if not implicit:
+ self.write_indent()
+ self.write_indicator('---', True)
+ if self.canonical:
+ self.write_indent()
+ self.state = self.expect_document_root
+ elif isinstance(self.event, StreamEndEvent):
+ if self.open_ended:
+ self.write_indicator('...', True)
+ self.write_indent()
+ self.write_stream_end()
+ self.state = self.expect_nothing
+ else:
+ raise EmitterError("expected DocumentStartEvent, but got %s"
+ % self.event)
+
+ def expect_document_end(self):
+ if isinstance(self.event, DocumentEndEvent):
+ self.write_indent()
+ if self.event.explicit:
+ self.write_indicator('...', True)
+ self.write_indent()
+ self.flush_stream()
+ self.state = self.expect_document_start
+ else:
+ raise EmitterError("expected DocumentEndEvent, but got %s"
+ % self.event)
+
+ def expect_document_root(self):
+ self.states.append(self.expect_document_end)
+ self.expect_node(root=True)
+
+ # Node handlers.
+
+ def expect_node(self, root=False, sequence=False, mapping=False,
+ simple_key=False):
+ self.root_context = root
+ self.sequence_context = sequence
+ self.mapping_context = mapping
+ self.simple_key_context = simple_key
+ if isinstance(self.event, AliasEvent):
+ self.expect_alias()
+ elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)):
+ self.process_anchor('&')
+ self.process_tag()
+ if isinstance(self.event, ScalarEvent):
+ self.expect_scalar()
+ elif isinstance(self.event, SequenceStartEvent):
+ if self.flow_level or self.canonical or self.event.flow_style \
+ or self.check_empty_sequence():
+ self.expect_flow_sequence()
+ else:
+ self.expect_block_sequence()
+ elif isinstance(self.event, MappingStartEvent):
+ if self.flow_level or self.canonical or self.event.flow_style \
+ or self.check_empty_mapping():
+ self.expect_flow_mapping()
+ else:
+ self.expect_block_mapping()
+ else:
+ raise EmitterError("expected NodeEvent, but got %s" % self.event)
+
+ def expect_alias(self):
+ if self.event.anchor is None:
+ raise EmitterError("anchor is not specified for alias")
+ self.process_anchor('*')
+ self.state = self.states.pop()
+
+ def expect_scalar(self):
+ self.increase_indent(flow=True)
+ self.process_scalar()
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+
+ # Flow sequence handlers.
+
+ def expect_flow_sequence(self):
+ self.write_indicator('[', True, whitespace=True)
+ self.flow_level += 1
+ self.increase_indent(flow=True)
+ self.state = self.expect_first_flow_sequence_item
+
+ def expect_first_flow_sequence_item(self):
+ if isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ self.write_indicator(']', False)
+ self.state = self.states.pop()
+ else:
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.states.append(self.expect_flow_sequence_item)
+ self.expect_node(sequence=True)
+
+ def expect_flow_sequence_item(self):
+ if isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ if self.canonical:
+ self.write_indicator(',', False)
+ self.write_indent()
+ self.write_indicator(']', False)
+ self.state = self.states.pop()
+ else:
+ self.write_indicator(',', False)
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.states.append(self.expect_flow_sequence_item)
+ self.expect_node(sequence=True)
+
+ # Flow mapping handlers.
+
+ def expect_flow_mapping(self):
+ self.write_indicator('{', True, whitespace=True)
+ self.flow_level += 1
+ self.increase_indent(flow=True)
+ self.state = self.expect_first_flow_mapping_key
+
+ def expect_first_flow_mapping_key(self):
+ if isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ self.write_indicator('}', False)
+ self.state = self.states.pop()
+ else:
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ if not self.canonical and self.check_simple_key():
+ self.states.append(self.expect_flow_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator('?', True)
+ self.states.append(self.expect_flow_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_key(self):
+ if isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ if self.canonical:
+ self.write_indicator(',', False)
+ self.write_indent()
+ self.write_indicator('}', False)
+ self.state = self.states.pop()
+ else:
+ self.write_indicator(',', False)
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ if not self.canonical and self.check_simple_key():
+ self.states.append(self.expect_flow_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator('?', True)
+ self.states.append(self.expect_flow_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_simple_value(self):
+ self.write_indicator(':', False)
+ self.states.append(self.expect_flow_mapping_key)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_value(self):
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.write_indicator(':', True)
+ self.states.append(self.expect_flow_mapping_key)
+ self.expect_node(mapping=True)
+
+ # Block sequence handlers.
+
+ def expect_block_sequence(self):
+ indentless = (self.mapping_context and not self.indention)
+ self.increase_indent(flow=False, indentless=indentless)
+ self.state = self.expect_first_block_sequence_item
+
+ def expect_first_block_sequence_item(self):
+ return self.expect_block_sequence_item(first=True)
+
+ def expect_block_sequence_item(self, first=False):
+ if not first and isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+ else:
+ self.write_indent()
+ self.write_indicator('-', True, indention=True)
+ self.states.append(self.expect_block_sequence_item)
+ self.expect_node(sequence=True)
+
+ # Block mapping handlers.
+
+ def expect_block_mapping(self):
+ self.increase_indent(flow=False)
+ self.state = self.expect_first_block_mapping_key
+
+ def expect_first_block_mapping_key(self):
+ return self.expect_block_mapping_key(first=True)
+
+ def expect_block_mapping_key(self, first=False):
+ if not first and isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+ else:
+ self.write_indent()
+ if self.check_simple_key():
+ self.states.append(self.expect_block_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator('?', True, indention=True)
+ self.states.append(self.expect_block_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_block_mapping_simple_value(self):
+ self.write_indicator(':', False)
+ self.states.append(self.expect_block_mapping_key)
+ self.expect_node(mapping=True)
+
+ def expect_block_mapping_value(self):
+ self.write_indent()
+ self.write_indicator(':', True, indention=True)
+ self.states.append(self.expect_block_mapping_key)
+ self.expect_node(mapping=True)
+
+ # Checkers.
+
+ def check_empty_sequence(self):
+ return (isinstance(self.event, SequenceStartEvent) and self.events
+ and isinstance(self.events[0], SequenceEndEvent))
+
+ def check_empty_mapping(self):
+ return (isinstance(self.event, MappingStartEvent) and self.events
+ and isinstance(self.events[0], MappingEndEvent))
+
+ def check_empty_document(self):
+ if not isinstance(self.event, DocumentStartEvent) or not self.events:
+ return False
+ event = self.events[0]
+ return (isinstance(event, ScalarEvent) and event.anchor is None
+ and event.tag is None and event.implicit and event.value == '')
+
+ def check_simple_key(self):
+ length = 0
+ if isinstance(self.event, NodeEvent) and self.event.anchor is not None:
+ if self.prepared_anchor is None:
+ self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+ length += len(self.prepared_anchor)
+ if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \
+ and self.event.tag is not None:
+ if self.prepared_tag is None:
+ self.prepared_tag = self.prepare_tag(self.event.tag)
+ length += len(self.prepared_tag)
+ if isinstance(self.event, ScalarEvent):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ length += len(self.analysis.scalar)
+ return (length < 128 and (isinstance(self.event, AliasEvent)
+ or (isinstance(self.event, ScalarEvent)
+ and not self.analysis.empty and not self.analysis.multiline)
+ or self.check_empty_sequence() or self.check_empty_mapping()))
+
+ # Anchor, Tag, and Scalar processors.
+
+ def process_anchor(self, indicator):
+ if self.event.anchor is None:
+ self.prepared_anchor = None
+ return
+ if self.prepared_anchor is None:
+ self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+ if self.prepared_anchor:
+ self.write_indicator(indicator+self.prepared_anchor, True)
+ self.prepared_anchor = None
+
+ def process_tag(self):
+ tag = self.event.tag
+ if isinstance(self.event, ScalarEvent):
+ if self.style is None:
+ self.style = self.choose_scalar_style()
+ if ((not self.canonical or tag is None) and
+ ((self.style == '' and self.event.implicit[0])
+ or (self.style != '' and self.event.implicit[1]))):
+ self.prepared_tag = None
+ return
+ if self.event.implicit[0] and tag is None:
+ tag = '!'
+ self.prepared_tag = None
+ else:
+ if (not self.canonical or tag is None) and self.event.implicit:
+ self.prepared_tag = None
+ return
+ if tag is None:
+ raise EmitterError("tag is not specified")
+ if self.prepared_tag is None:
+ self.prepared_tag = self.prepare_tag(tag)
+ if self.prepared_tag:
+ self.write_indicator(self.prepared_tag, True)
+ self.prepared_tag = None
+
+ def choose_scalar_style(self):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ if self.event.style == '"' or self.canonical:
+ return '"'
+ if not self.event.style and self.event.implicit[0]:
+ if (not (self.simple_key_context and
+ (self.analysis.empty or self.analysis.multiline))
+ and (self.flow_level and self.analysis.allow_flow_plain
+ or (not self.flow_level and self.analysis.allow_block_plain))):
+ return ''
+ if self.event.style and self.event.style in '|>':
+ if (not self.flow_level and not self.simple_key_context
+ and self.analysis.allow_block):
+ return self.event.style
+ if not self.event.style or self.event.style == '\'':
+ if (self.analysis.allow_single_quoted and
+ not (self.simple_key_context and self.analysis.multiline)):
+ return '\''
+ return '"'
+
+ def process_scalar(self):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ if self.style is None:
+ self.style = self.choose_scalar_style()
+ split = (not self.simple_key_context)
+ #if self.analysis.multiline and split \
+ # and (not self.style or self.style in '\'\"'):
+ # self.write_indent()
+ if self.style == '"':
+ self.write_double_quoted(self.analysis.scalar, split)
+ elif self.style == '\'':
+ self.write_single_quoted(self.analysis.scalar, split)
+ elif self.style == '>':
+ self.write_folded(self.analysis.scalar)
+ elif self.style == '|':
+ self.write_literal(self.analysis.scalar)
+ else:
+ self.write_plain(self.analysis.scalar, split)
+ self.analysis = None
+ self.style = None
+
+ # Analyzers.
+
+ def prepare_version(self, version):
+ major, minor = version
+ if major != 1:
+ raise EmitterError("unsupported YAML version: %d.%d" % (major, minor))
+ return '%d.%d' % (major, minor)
+
+ def prepare_tag_handle(self, handle):
+ if not handle:
+ raise EmitterError("tag handle must not be empty")
+ if handle[0] != '!' or handle[-1] != '!':
+ raise EmitterError("tag handle must start and end with '!': %r" % handle)
+ for ch in handle[1:-1]:
+ if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_'):
+ raise EmitterError("invalid character %r in the tag handle: %r"
+ % (ch, handle))
+ return handle
+
+ def prepare_tag_prefix(self, prefix):
+ if not prefix:
+ raise EmitterError("tag prefix must not be empty")
+ chunks = []
+ start = end = 0
+ if prefix[0] == '!':
+ end = 1
+ while end < len(prefix):
+ ch = prefix[end]
+ if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-;/?!:@&=+$,_.~*\'()[]':
+ end += 1
+ else:
+ if start < end:
+ chunks.append(prefix[start:end])
+ start = end = end+1
+ data = ch.encode('utf-8')
+ for ch in data:
+ chunks.append('%%%02X' % ord(ch))
+ if start < end:
+ chunks.append(prefix[start:end])
+ return ''.join(chunks)
+
+ def prepare_tag(self, tag):
+ if not tag:
+ raise EmitterError("tag must not be empty")
+ if tag == '!':
+ return tag
+ handle = None
+ suffix = tag
+ prefixes = sorted(self.tag_prefixes.keys())
+ for prefix in prefixes:
+ if tag.startswith(prefix) \
+ and (prefix == '!' or len(prefix) < len(tag)):
+ handle = self.tag_prefixes[prefix]
+ suffix = tag[len(prefix):]
+ chunks = []
+ start = end = 0
+ while end < len(suffix):
+ ch = suffix[end]
+ if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-;/?:@&=+$,_.~*\'()[]' \
+ or (ch == '!' and handle != '!'):
+ end += 1
+ else:
+ if start < end:
+ chunks.append(suffix[start:end])
+ start = end = end+1
+ data = ch.encode('utf-8')
+ for ch in data:
+ chunks.append('%%%02X' % ord(ch))
+ if start < end:
+ chunks.append(suffix[start:end])
+ suffix_text = ''.join(chunks)
+ if handle:
+ return '%s%s' % (handle, suffix_text)
+ else:
+ return '!<%s>' % suffix_text
+
+ def prepare_anchor(self, anchor):
+ if not anchor:
+ raise EmitterError("anchor must not be empty")
+ for ch in anchor:
+ if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_'):
+ raise EmitterError("invalid character %r in the anchor: %r"
+ % (ch, anchor))
+ return anchor
+
+ def analyze_scalar(self, scalar):
+
+ # Empty scalar is a special case.
+ if not scalar:
+ return ScalarAnalysis(scalar=scalar, empty=True, multiline=False,
+ allow_flow_plain=False, allow_block_plain=True,
+ allow_single_quoted=True, allow_double_quoted=True,
+ allow_block=False)
+
+ # Indicators and special characters.
+ block_indicators = False
+ flow_indicators = False
+ line_breaks = False
+ special_characters = False
+
+ # Important whitespace combinations.
+ leading_space = False
+ leading_break = False
+ trailing_space = False
+ trailing_break = False
+ break_space = False
+ space_break = False
+
+ # Check document indicators.
+ if scalar.startswith('---') or scalar.startswith('...'):
+ block_indicators = True
+ flow_indicators = True
+
+ # First character or preceded by a whitespace.
+ preceeded_by_whitespace = True
+
+ # Last character or followed by a whitespace.
+ followed_by_whitespace = (len(scalar) == 1 or
+ scalar[1] in '\0 \t\r\n\x85\u2028\u2029')
+
+ # The previous character is a space.
+ previous_space = False
+
+ # The previous character is a break.
+ previous_break = False
+
+ index = 0
+ while index < len(scalar):
+ ch = scalar[index]
+
+ # Check for indicators.
+ if index == 0:
+ # Leading indicators are special characters.
+ if ch in '#,[]{}&*!|>\'\"%@`':
+ flow_indicators = True
+ block_indicators = True
+ if ch in '?:':
+ flow_indicators = True
+ if followed_by_whitespace:
+ block_indicators = True
+ if ch == '-' and followed_by_whitespace:
+ flow_indicators = True
+ block_indicators = True
+ else:
+ # Some indicators cannot appear within a scalar as well.
+ if ch in ',?[]{}':
+ flow_indicators = True
+ if ch == ':':
+ flow_indicators = True
+ if followed_by_whitespace:
+ block_indicators = True
+ if ch == '#' and preceeded_by_whitespace:
+ flow_indicators = True
+ block_indicators = True
+
+ # Check for line breaks, special, and unicode characters.
+ if ch in '\n\x85\u2028\u2029':
+ line_breaks = True
+ if not (ch == '\n' or '\x20' <= ch <= '\x7E'):
+ if (ch == '\x85' or '\xA0' <= ch <= '\uD7FF'
+ or '\uE000' <= ch <= '\uFFFD') and ch != '\uFEFF':
+ unicode_characters = True
+ if not self.allow_unicode:
+ special_characters = True
+ else:
+ special_characters = True
+
+ # Detect important whitespace combinations.
+ if ch == ' ':
+ if index == 0:
+ leading_space = True
+ if index == len(scalar)-1:
+ trailing_space = True
+ if previous_break:
+ break_space = True
+ previous_space = True
+ previous_break = False
+ elif ch in '\n\x85\u2028\u2029':
+ if index == 0:
+ leading_break = True
+ if index == len(scalar)-1:
+ trailing_break = True
+ if previous_space:
+ space_break = True
+ previous_space = False
+ previous_break = True
+ else:
+ previous_space = False
+ previous_break = False
+
+ # Prepare for the next character.
+ index += 1
+ preceeded_by_whitespace = (ch in '\0 \t\r\n\x85\u2028\u2029')
+ followed_by_whitespace = (index+1 >= len(scalar) or
+ scalar[index+1] in '\0 \t\r\n\x85\u2028\u2029')
+
+ # Let's decide what styles are allowed.
+ allow_flow_plain = True
+ allow_block_plain = True
+ allow_single_quoted = True
+ allow_double_quoted = True
+ allow_block = True
+
+ # Leading and trailing whitespaces are bad for plain scalars.
+ if (leading_space or leading_break
+ or trailing_space or trailing_break):
+ allow_flow_plain = allow_block_plain = False
+
+ # We do not permit trailing spaces for block scalars.
+ if trailing_space:
+ allow_block = False
+
+ # Spaces at the beginning of a new line are only acceptable for block
+ # scalars.
+ if break_space:
+ allow_flow_plain = allow_block_plain = allow_single_quoted = False
+
+ # Spaces followed by breaks, as well as special character are only
+ # allowed for double quoted scalars.
+ if space_break or special_characters:
+ allow_flow_plain = allow_block_plain = \
+ allow_single_quoted = allow_block = False
+
+ # Although the plain scalar writer supports breaks, we never emit
+ # multiline plain scalars.
+ if line_breaks:
+ allow_flow_plain = allow_block_plain = False
+
+ # Flow indicators are forbidden for flow plain scalars.
+ if flow_indicators:
+ allow_flow_plain = False
+
+ # Block indicators are forbidden for block plain scalars.
+ if block_indicators:
+ allow_block_plain = False
+
+ return ScalarAnalysis(scalar=scalar,
+ empty=False, multiline=line_breaks,
+ allow_flow_plain=allow_flow_plain,
+ allow_block_plain=allow_block_plain,
+ allow_single_quoted=allow_single_quoted,
+ allow_double_quoted=allow_double_quoted,
+ allow_block=allow_block)
+
+ # Writers.
+
+ def flush_stream(self):
+ if hasattr(self.stream, 'flush'):
+ self.stream.flush()
+
+ def write_stream_start(self):
+ # Write BOM if needed.
+ if self.encoding and self.encoding.startswith('utf-16'):
+ self.stream.write('\uFEFF'.encode(self.encoding))
+
+ def write_stream_end(self):
+ self.flush_stream()
+
+ def write_indicator(self, indicator, need_whitespace,
+ whitespace=False, indention=False):
+ if self.whitespace or not need_whitespace:
+ data = indicator
+ else:
+ data = ' '+indicator
+ self.whitespace = whitespace
+ self.indention = self.indention and indention
+ self.column += len(data)
+ self.open_ended = False
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_indent(self):
+ indent = self.indent or 0
+ if not self.indention or self.column > indent \
+ or (self.column == indent and not self.whitespace):
+ self.write_line_break()
+ if self.column < indent:
+ self.whitespace = True
+ data = ' '*(indent-self.column)
+ self.column = indent
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_line_break(self, data=None):
+ if data is None:
+ data = self.best_line_break
+ self.whitespace = True
+ self.indention = True
+ self.line += 1
+ self.column = 0
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_version_directive(self, version_text):
+ data = '%%YAML %s' % version_text
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_line_break()
+
+ def write_tag_directive(self, handle_text, prefix_text):
+ data = '%%TAG %s %s' % (handle_text, prefix_text)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_line_break()
+
+ # Scalar streams.
+
+ def write_single_quoted(self, text, split=True):
+ self.write_indicator('\'', True)
+ spaces = False
+ breaks = False
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if spaces:
+ if ch is None or ch != ' ':
+ if start+1 == end and self.column > self.best_width and split \
+ and start != 0 and end != len(text):
+ self.write_indent()
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ elif breaks:
+ if ch is None or ch not in '\n\x85\u2028\u2029':
+ if text[start] == '\n':
+ self.write_line_break()
+ for br in text[start:end]:
+ if br == '\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ self.write_indent()
+ start = end
+ else:
+ if ch is None or ch in ' \n\x85\u2028\u2029' or ch == '\'':
+ if start < end:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch == '\'':
+ data = '\'\''
+ self.column += 2
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end + 1
+ if ch is not None:
+ spaces = (ch == ' ')
+ breaks = (ch in '\n\x85\u2028\u2029')
+ end += 1
+ self.write_indicator('\'', False)
+
+ ESCAPE_REPLACEMENTS = {
+ '\0': '0',
+ '\x07': 'a',
+ '\x08': 'b',
+ '\x09': 't',
+ '\x0A': 'n',
+ '\x0B': 'v',
+ '\x0C': 'f',
+ '\x0D': 'r',
+ '\x1B': 'e',
+ '\"': '\"',
+ '\\': '\\',
+ '\x85': 'N',
+ '\xA0': '_',
+ '\u2028': 'L',
+ '\u2029': 'P',
+ }
+
+ def write_double_quoted(self, text, split=True):
+ self.write_indicator('"', True)
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if ch is None or ch in '"\\\x85\u2028\u2029\uFEFF' \
+ or not ('\x20' <= ch <= '\x7E'
+ or (self.allow_unicode
+ and ('\xA0' <= ch <= '\uD7FF'
+ or '\uE000' <= ch <= '\uFFFD'))):
+ if start < end:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch is not None:
+ if ch in self.ESCAPE_REPLACEMENTS:
+ data = '\\'+self.ESCAPE_REPLACEMENTS[ch]
+ elif ch <= '\xFF':
+ data = '\\x%02X' % ord(ch)
+ elif ch <= '\uFFFF':
+ data = '\\u%04X' % ord(ch)
+ else:
+ data = '\\U%08X' % ord(ch)
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end+1
+ if 0 < end < len(text)-1 and (ch == ' ' or start >= end) \
+ and self.column+(end-start) > self.best_width and split:
+ data = text[start:end]+'\\'
+ if start < end:
+ start = end
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ if text[start] == ' ':
+ data = '\\'
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ end += 1
+ self.write_indicator('"', False)
+
+ def determine_block_hints(self, text):
+ hints = ''
+ if text:
+ if text[0] in ' \n\x85\u2028\u2029':
+ hints += str(self.best_indent)
+ if text[-1] not in '\n\x85\u2028\u2029':
+ hints += '-'
+ elif len(text) == 1 or text[-2] in '\n\x85\u2028\u2029':
+ hints += '+'
+ return hints
+
+ def write_folded(self, text):
+ hints = self.determine_block_hints(text)
+ self.write_indicator('>'+hints, True)
+ if hints[-1:] == '+':
+ self.open_ended = True
+ self.write_line_break()
+ leading_space = True
+ spaces = False
+ breaks = True
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if breaks:
+ if ch is None or ch not in '\n\x85\u2028\u2029':
+ if not leading_space and ch is not None and ch != ' ' \
+ and text[start] == '\n':
+ self.write_line_break()
+ leading_space = (ch == ' ')
+ for br in text[start:end]:
+ if br == '\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ if ch is not None:
+ self.write_indent()
+ start = end
+ elif spaces:
+ if ch != ' ':
+ if start+1 == end and self.column > self.best_width:
+ self.write_indent()
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ else:
+ if ch is None or ch in ' \n\x85\u2028\u2029':
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ if ch is None:
+ self.write_line_break()
+ start = end
+ if ch is not None:
+ breaks = (ch in '\n\x85\u2028\u2029')
+ spaces = (ch == ' ')
+ end += 1
+
+ def write_literal(self, text):
+ hints = self.determine_block_hints(text)
+ self.write_indicator('|'+hints, True)
+ if hints[-1:] == '+':
+ self.open_ended = True
+ self.write_line_break()
+ breaks = True
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if breaks:
+ if ch is None or ch not in '\n\x85\u2028\u2029':
+ for br in text[start:end]:
+ if br == '\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ if ch is not None:
+ self.write_indent()
+ start = end
+ else:
+ if ch is None or ch in '\n\x85\u2028\u2029':
+ data = text[start:end]
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ if ch is None:
+ self.write_line_break()
+ start = end
+ if ch is not None:
+ breaks = (ch in '\n\x85\u2028\u2029')
+ end += 1
+
+ def write_plain(self, text, split=True):
+ if self.root_context:
+ self.open_ended = True
+ if not text:
+ return
+ if not self.whitespace:
+ data = ' '
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.whitespace = False
+ self.indention = False
+ spaces = False
+ breaks = False
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if spaces:
+ if ch != ' ':
+ if start+1 == end and self.column > self.best_width and split:
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ elif breaks:
+ if ch not in '\n\x85\u2028\u2029':
+ if text[start] == '\n':
+ self.write_line_break()
+ for br in text[start:end]:
+ if br == '\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ start = end
+ else:
+ if ch is None or ch in ' \n\x85\u2028\u2029':
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch is not None:
+ spaces = (ch == ' ')
+ breaks = (ch in '\n\x85\u2028\u2029')
+ end += 1
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/error.py b/collectors/python.d.plugin/python_modules/pyyaml3/error.py
new file mode 100644
index 0000000..5fec7d4
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/error.py
@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError']
+
+class Mark:
+
+ def __init__(self, name, index, line, column, buffer, pointer):
+ self.name = name
+ self.index = index
+ self.line = line
+ self.column = column
+ self.buffer = buffer
+ self.pointer = pointer
+
+ def get_snippet(self, indent=4, max_length=75):
+ if self.buffer is None:
+ return None
+ head = ''
+ start = self.pointer
+ while start > 0 and self.buffer[start-1] not in '\0\r\n\x85\u2028\u2029':
+ start -= 1
+ if self.pointer-start > max_length/2-1:
+ head = ' ... '
+ start += 5
+ break
+ tail = ''
+ end = self.pointer
+ while end < len(self.buffer) and self.buffer[end] not in '\0\r\n\x85\u2028\u2029':
+ end += 1
+ if end-self.pointer > max_length/2-1:
+ tail = ' ... '
+ end -= 5
+ break
+ snippet = self.buffer[start:end]
+ return ' '*indent + head + snippet + tail + '\n' \
+ + ' '*(indent+self.pointer-start+len(head)) + '^'
+
+ def __str__(self):
+ snippet = self.get_snippet()
+ where = " in \"%s\", line %d, column %d" \
+ % (self.name, self.line+1, self.column+1)
+ if snippet is not None:
+ where += ":\n"+snippet
+ return where
+
+class YAMLError(Exception):
+ pass
+
+class MarkedYAMLError(YAMLError):
+
+ def __init__(self, context=None, context_mark=None,
+ problem=None, problem_mark=None, note=None):
+ self.context = context
+ self.context_mark = context_mark
+ self.problem = problem
+ self.problem_mark = problem_mark
+ self.note = note
+
+ def __str__(self):
+ lines = []
+ if self.context is not None:
+ lines.append(self.context)
+ if self.context_mark is not None \
+ and (self.problem is None or self.problem_mark is None
+ or self.context_mark.name != self.problem_mark.name
+ or self.context_mark.line != self.problem_mark.line
+ or self.context_mark.column != self.problem_mark.column):
+ lines.append(str(self.context_mark))
+ if self.problem is not None:
+ lines.append(self.problem)
+ if self.problem_mark is not None:
+ lines.append(str(self.problem_mark))
+ if self.note is not None:
+ lines.append(self.note)
+ return '\n'.join(lines)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/events.py b/collectors/python.d.plugin/python_modules/pyyaml3/events.py
new file mode 100644
index 0000000..283452a
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/events.py
@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: MIT
+
+# Abstract classes.
+
+class Event(object):
+ def __init__(self, start_mark=None, end_mark=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ attributes = [key for key in ['anchor', 'tag', 'implicit', 'value']
+ if hasattr(self, key)]
+ arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+ for key in attributes])
+ return '%s(%s)' % (self.__class__.__name__, arguments)
+
+class NodeEvent(Event):
+ def __init__(self, anchor, start_mark=None, end_mark=None):
+ self.anchor = anchor
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class CollectionStartEvent(NodeEvent):
+ def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None,
+ flow_style=None):
+ self.anchor = anchor
+ self.tag = tag
+ self.implicit = implicit
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.flow_style = flow_style
+
+class CollectionEndEvent(Event):
+ pass
+
+# Implementations.
+
+class StreamStartEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None, encoding=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.encoding = encoding
+
+class StreamEndEvent(Event):
+ pass
+
+class DocumentStartEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None,
+ explicit=None, version=None, tags=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.explicit = explicit
+ self.version = version
+ self.tags = tags
+
+class DocumentEndEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None,
+ explicit=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.explicit = explicit
+
+class AliasEvent(NodeEvent):
+ pass
+
+class ScalarEvent(NodeEvent):
+ def __init__(self, anchor, tag, implicit, value,
+ start_mark=None, end_mark=None, style=None):
+ self.anchor = anchor
+ self.tag = tag
+ self.implicit = implicit
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
+class SequenceStartEvent(CollectionStartEvent):
+ pass
+
+class SequenceEndEvent(CollectionEndEvent):
+ pass
+
+class MappingStartEvent(CollectionStartEvent):
+ pass
+
+class MappingEndEvent(CollectionEndEvent):
+ pass
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/loader.py b/collectors/python.d.plugin/python_modules/pyyaml3/loader.py
new file mode 100644
index 0000000..7ef6cf8
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/loader.py
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseLoader', 'SafeLoader', 'Loader']
+
+from .reader import *
+from .scanner import *
+from .parser import *
+from .composer import *
+from .constructor import *
+from .resolver import *
+
+class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ BaseConstructor.__init__(self)
+ BaseResolver.__init__(self)
+
+class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ SafeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ Constructor.__init__(self)
+ Resolver.__init__(self)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/nodes.py b/collectors/python.d.plugin/python_modules/pyyaml3/nodes.py
new file mode 100644
index 0000000..ed2a1b4
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/nodes.py
@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: MIT
+
+class Node(object):
+ def __init__(self, tag, value, start_mark, end_mark):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ value = self.value
+ #if isinstance(value, list):
+ # if len(value) == 0:
+ # value = '<empty>'
+ # elif len(value) == 1:
+ # value = '<1 item>'
+ # else:
+ # value = '<%d items>' % len(value)
+ #else:
+ # if len(value) > 75:
+ # value = repr(value[:70]+u' ... ')
+ # else:
+ # value = repr(value)
+ value = repr(value)
+ return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value)
+
+class ScalarNode(Node):
+ id = 'scalar'
+ def __init__(self, tag, value,
+ start_mark=None, end_mark=None, style=None):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
+class CollectionNode(Node):
+ def __init__(self, tag, value,
+ start_mark=None, end_mark=None, flow_style=None):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.flow_style = flow_style
+
+class SequenceNode(CollectionNode):
+ id = 'sequence'
+
+class MappingNode(CollectionNode):
+ id = 'mapping'
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/parser.py b/collectors/python.d.plugin/python_modules/pyyaml3/parser.py
new file mode 100644
index 0000000..bcec7f9
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/parser.py
@@ -0,0 +1,590 @@
+# SPDX-License-Identifier: MIT
+
+# The following YAML grammar is LL(1) and is parsed by a recursive descent
+# parser.
+#
+# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
+# implicit_document ::= block_node DOCUMENT-END*
+# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
+# block_node_or_indentless_sequence ::=
+# ALIAS
+# | properties (block_content | indentless_block_sequence)?
+# | block_content
+# | indentless_block_sequence
+# block_node ::= ALIAS
+# | properties block_content?
+# | block_content
+# flow_node ::= ALIAS
+# | properties flow_content?
+# | flow_content
+# properties ::= TAG ANCHOR? | ANCHOR TAG?
+# block_content ::= block_collection | flow_collection | SCALAR
+# flow_content ::= flow_collection | SCALAR
+# block_collection ::= block_sequence | block_mapping
+# flow_collection ::= flow_sequence | flow_mapping
+# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+# indentless_sequence ::= (BLOCK-ENTRY block_node?)+
+# block_mapping ::= BLOCK-MAPPING_START
+# ((KEY block_node_or_indentless_sequence?)?
+# (VALUE block_node_or_indentless_sequence?)?)*
+# BLOCK-END
+# flow_sequence ::= FLOW-SEQUENCE-START
+# (flow_sequence_entry FLOW-ENTRY)*
+# flow_sequence_entry?
+# FLOW-SEQUENCE-END
+# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+# flow_mapping ::= FLOW-MAPPING-START
+# (flow_mapping_entry FLOW-ENTRY)*
+# flow_mapping_entry?
+# FLOW-MAPPING-END
+# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+#
+# FIRST sets:
+#
+# stream: { STREAM-START }
+# explicit_document: { DIRECTIVE DOCUMENT-START }
+# implicit_document: FIRST(block_node)
+# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_sequence: { BLOCK-SEQUENCE-START }
+# block_mapping: { BLOCK-MAPPING-START }
+# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
+# indentless_sequence: { ENTRY }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_sequence: { FLOW-SEQUENCE-START }
+# flow_mapping: { FLOW-MAPPING-START }
+# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+
+__all__ = ['Parser', 'ParserError']
+
+from .error import MarkedYAMLError
+from .tokens import *
+from .events import *
+from .scanner import *
+
+class ParserError(MarkedYAMLError):
+ pass
+
+class Parser:
+ # Since writing a recursive-descendant parser is a straightforward task, we
+ # do not give many comments here.
+
+ DEFAULT_TAGS = {
+ '!': '!',
+ '!!': 'tag:yaml.org,2002:',
+ }
+
+ def __init__(self):
+ self.current_event = None
+ self.yaml_version = None
+ self.tag_handles = {}
+ self.states = []
+ self.marks = []
+ self.state = self.parse_stream_start
+
+ def dispose(self):
+ # Reset the state attributes (to clear self-references)
+ self.states = []
+ self.state = None
+
+ def check_event(self, *choices):
+ # Check the type of the next event.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ if self.current_event is not None:
+ if not choices:
+ return True
+ for choice in choices:
+ if isinstance(self.current_event, choice):
+ return True
+ return False
+
+ def peek_event(self):
+ # Get the next event.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ return self.current_event
+
+ def get_event(self):
+ # Get the next event and proceed further.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ value = self.current_event
+ self.current_event = None
+ return value
+
+ # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
+ # implicit_document ::= block_node DOCUMENT-END*
+ # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
+
+ def parse_stream_start(self):
+
+ # Parse the stream start.
+ token = self.get_token()
+ event = StreamStartEvent(token.start_mark, token.end_mark,
+ encoding=token.encoding)
+
+ # Prepare the next state.
+ self.state = self.parse_implicit_document_start
+
+ return event
+
+ def parse_implicit_document_start(self):
+
+ # Parse an implicit document.
+ if not self.check_token(DirectiveToken, DocumentStartToken,
+ StreamEndToken):
+ self.tag_handles = self.DEFAULT_TAGS
+ token = self.peek_token()
+ start_mark = end_mark = token.start_mark
+ event = DocumentStartEvent(start_mark, end_mark,
+ explicit=False)
+
+ # Prepare the next state.
+ self.states.append(self.parse_document_end)
+ self.state = self.parse_block_node
+
+ return event
+
+ else:
+ return self.parse_document_start()
+
+ def parse_document_start(self):
+
+ # Parse any extra document end indicators.
+ while self.check_token(DocumentEndToken):
+ self.get_token()
+
+ # Parse an explicit document.
+ if not self.check_token(StreamEndToken):
+ token = self.peek_token()
+ start_mark = token.start_mark
+ version, tags = self.process_directives()
+ if not self.check_token(DocumentStartToken):
+ raise ParserError(None, None,
+ "expected '<document start>', but found %r"
+ % self.peek_token().id,
+ self.peek_token().start_mark)
+ token = self.get_token()
+ end_mark = token.end_mark
+ event = DocumentStartEvent(start_mark, end_mark,
+ explicit=True, version=version, tags=tags)
+ self.states.append(self.parse_document_end)
+ self.state = self.parse_document_content
+ else:
+ # Parse the end of the stream.
+ token = self.get_token()
+ event = StreamEndEvent(token.start_mark, token.end_mark)
+ assert not self.states
+ assert not self.marks
+ self.state = None
+ return event
+
+ def parse_document_end(self):
+
+ # Parse the document end.
+ token = self.peek_token()
+ start_mark = end_mark = token.start_mark
+ explicit = False
+ if self.check_token(DocumentEndToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ explicit = True
+ event = DocumentEndEvent(start_mark, end_mark,
+ explicit=explicit)
+
+ # Prepare the next state.
+ self.state = self.parse_document_start
+
+ return event
+
+ def parse_document_content(self):
+ if self.check_token(DirectiveToken,
+ DocumentStartToken, DocumentEndToken, StreamEndToken):
+ event = self.process_empty_scalar(self.peek_token().start_mark)
+ self.state = self.states.pop()
+ return event
+ else:
+ return self.parse_block_node()
+
+ def process_directives(self):
+ self.yaml_version = None
+ self.tag_handles = {}
+ while self.check_token(DirectiveToken):
+ token = self.get_token()
+ if token.name == 'YAML':
+ if self.yaml_version is not None:
+ raise ParserError(None, None,
+ "found duplicate YAML directive", token.start_mark)
+ major, minor = token.value
+ if major != 1:
+ raise ParserError(None, None,
+ "found incompatible YAML document (version 1.* is required)",
+ token.start_mark)
+ self.yaml_version = token.value
+ elif token.name == 'TAG':
+ handle, prefix = token.value
+ if handle in self.tag_handles:
+ raise ParserError(None, None,
+ "duplicate tag handle %r" % handle,
+ token.start_mark)
+ self.tag_handles[handle] = prefix
+ if self.tag_handles:
+ value = self.yaml_version, self.tag_handles.copy()
+ else:
+ value = self.yaml_version, None
+ for key in self.DEFAULT_TAGS:
+ if key not in self.tag_handles:
+ self.tag_handles[key] = self.DEFAULT_TAGS[key]
+ return value
+
+ # block_node_or_indentless_sequence ::= ALIAS
+ # | properties (block_content | indentless_block_sequence)?
+ # | block_content
+ # | indentless_block_sequence
+ # block_node ::= ALIAS
+ # | properties block_content?
+ # | block_content
+ # flow_node ::= ALIAS
+ # | properties flow_content?
+ # | flow_content
+ # properties ::= TAG ANCHOR? | ANCHOR TAG?
+ # block_content ::= block_collection | flow_collection | SCALAR
+ # flow_content ::= flow_collection | SCALAR
+ # block_collection ::= block_sequence | block_mapping
+ # flow_collection ::= flow_sequence | flow_mapping
+
+ def parse_block_node(self):
+ return self.parse_node(block=True)
+
+ def parse_flow_node(self):
+ return self.parse_node()
+
+ def parse_block_node_or_indentless_sequence(self):
+ return self.parse_node(block=True, indentless_sequence=True)
+
+ def parse_node(self, block=False, indentless_sequence=False):
+ if self.check_token(AliasToken):
+ token = self.get_token()
+ event = AliasEvent(token.value, token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ else:
+ anchor = None
+ tag = None
+ start_mark = end_mark = tag_mark = None
+ if self.check_token(AnchorToken):
+ token = self.get_token()
+ start_mark = token.start_mark
+ end_mark = token.end_mark
+ anchor = token.value
+ if self.check_token(TagToken):
+ token = self.get_token()
+ tag_mark = token.start_mark
+ end_mark = token.end_mark
+ tag = token.value
+ elif self.check_token(TagToken):
+ token = self.get_token()
+ start_mark = tag_mark = token.start_mark
+ end_mark = token.end_mark
+ tag = token.value
+ if self.check_token(AnchorToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ anchor = token.value
+ if tag is not None:
+ handle, suffix = tag
+ if handle is not None:
+ if handle not in self.tag_handles:
+ raise ParserError("while parsing a node", start_mark,
+ "found undefined tag handle %r" % handle,
+ tag_mark)
+ tag = self.tag_handles[handle]+suffix
+ else:
+ tag = suffix
+ #if tag == '!':
+ # raise ParserError("while parsing a node", start_mark,
+ # "found non-specific tag '!'", tag_mark,
+ # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
+ if start_mark is None:
+ start_mark = end_mark = self.peek_token().start_mark
+ event = None
+ implicit = (tag is None or tag == '!')
+ if indentless_sequence and self.check_token(BlockEntryToken):
+ end_mark = self.peek_token().end_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark)
+ self.state = self.parse_indentless_sequence_entry
+ else:
+ if self.check_token(ScalarToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ if (token.plain and tag is None) or tag == '!':
+ implicit = (True, False)
+ elif tag is None:
+ implicit = (False, True)
+ else:
+ implicit = (False, False)
+ event = ScalarEvent(anchor, tag, implicit, token.value,
+ start_mark, end_mark, style=token.style)
+ self.state = self.states.pop()
+ elif self.check_token(FlowSequenceStartToken):
+ end_mark = self.peek_token().end_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=True)
+ self.state = self.parse_flow_sequence_first_entry
+ elif self.check_token(FlowMappingStartToken):
+ end_mark = self.peek_token().end_mark
+ event = MappingStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=True)
+ self.state = self.parse_flow_mapping_first_key
+ elif block and self.check_token(BlockSequenceStartToken):
+ end_mark = self.peek_token().start_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=False)
+ self.state = self.parse_block_sequence_first_entry
+ elif block and self.check_token(BlockMappingStartToken):
+ end_mark = self.peek_token().start_mark
+ event = MappingStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=False)
+ self.state = self.parse_block_mapping_first_key
+ elif anchor is not None or tag is not None:
+ # Empty scalars are allowed even if a tag or an anchor is
+ # specified.
+ event = ScalarEvent(anchor, tag, (implicit, False), '',
+ start_mark, end_mark)
+ self.state = self.states.pop()
+ else:
+ if block:
+ node = 'block'
+ else:
+ node = 'flow'
+ token = self.peek_token()
+ raise ParserError("while parsing a %s node" % node, start_mark,
+ "expected the node content, but found %r" % token.id,
+ token.start_mark)
+ return event
+
+ # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+
+ def parse_block_sequence_first_entry(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_block_sequence_entry()
+
+ def parse_block_sequence_entry(self):
+ if self.check_token(BlockEntryToken):
+ token = self.get_token()
+ if not self.check_token(BlockEntryToken, BlockEndToken):
+ self.states.append(self.parse_block_sequence_entry)
+ return self.parse_block_node()
+ else:
+ self.state = self.parse_block_sequence_entry
+ return self.process_empty_scalar(token.end_mark)
+ if not self.check_token(BlockEndToken):
+ token = self.peek_token()
+ raise ParserError("while parsing a block collection", self.marks[-1],
+ "expected <block end>, but found %r" % token.id, token.start_mark)
+ token = self.get_token()
+ event = SequenceEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ # indentless_sequence ::= (BLOCK-ENTRY block_node?)+
+
+ def parse_indentless_sequence_entry(self):
+ if self.check_token(BlockEntryToken):
+ token = self.get_token()
+ if not self.check_token(BlockEntryToken,
+ KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_indentless_sequence_entry)
+ return self.parse_block_node()
+ else:
+ self.state = self.parse_indentless_sequence_entry
+ return self.process_empty_scalar(token.end_mark)
+ token = self.peek_token()
+ event = SequenceEndEvent(token.start_mark, token.start_mark)
+ self.state = self.states.pop()
+ return event
+
+ # block_mapping ::= BLOCK-MAPPING_START
+ # ((KEY block_node_or_indentless_sequence?)?
+ # (VALUE block_node_or_indentless_sequence?)?)*
+ # BLOCK-END
+
+ def parse_block_mapping_first_key(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_block_mapping_key()
+
+ def parse_block_mapping_key(self):
+ if self.check_token(KeyToken):
+ token = self.get_token()
+ if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_block_mapping_value)
+ return self.parse_block_node_or_indentless_sequence()
+ else:
+ self.state = self.parse_block_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+ if not self.check_token(BlockEndToken):
+ token = self.peek_token()
+ raise ParserError("while parsing a block mapping", self.marks[-1],
+ "expected <block end>, but found %r" % token.id, token.start_mark)
+ token = self.get_token()
+ event = MappingEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_block_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_block_mapping_key)
+ return self.parse_block_node_or_indentless_sequence()
+ else:
+ self.state = self.parse_block_mapping_key
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_block_mapping_key
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ # flow_sequence ::= FLOW-SEQUENCE-START
+ # (flow_sequence_entry FLOW-ENTRY)*
+ # flow_sequence_entry?
+ # FLOW-SEQUENCE-END
+ # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+ #
+ # Note that while production rules for both flow_sequence_entry and
+ # flow_mapping_entry are equal, their interpretations are different.
+ # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
+ # generate an inline mapping (set syntax).
+
+ def parse_flow_sequence_first_entry(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_flow_sequence_entry(first=True)
+
+ def parse_flow_sequence_entry(self, first=False):
+ if not self.check_token(FlowSequenceEndToken):
+ if not first:
+ if self.check_token(FlowEntryToken):
+ self.get_token()
+ else:
+ token = self.peek_token()
+ raise ParserError("while parsing a flow sequence", self.marks[-1],
+ "expected ',' or ']', but got %r" % token.id, token.start_mark)
+
+ if self.check_token(KeyToken):
+ token = self.peek_token()
+ event = MappingStartEvent(None, None, True,
+ token.start_mark, token.end_mark,
+ flow_style=True)
+ self.state = self.parse_flow_sequence_entry_mapping_key
+ return event
+ elif not self.check_token(FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry)
+ return self.parse_flow_node()
+ token = self.get_token()
+ event = SequenceEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_flow_sequence_entry_mapping_key(self):
+ token = self.get_token()
+ if not self.check_token(ValueToken,
+ FlowEntryToken, FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry_mapping_value)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+
+ def parse_flow_sequence_entry_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry_mapping_end)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_end
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_end
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ def parse_flow_sequence_entry_mapping_end(self):
+ self.state = self.parse_flow_sequence_entry
+ token = self.peek_token()
+ return MappingEndEvent(token.start_mark, token.start_mark)
+
+ # flow_mapping ::= FLOW-MAPPING-START
+ # (flow_mapping_entry FLOW-ENTRY)*
+ # flow_mapping_entry?
+ # FLOW-MAPPING-END
+ # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+
+ def parse_flow_mapping_first_key(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_flow_mapping_key(first=True)
+
+ def parse_flow_mapping_key(self, first=False):
+ if not self.check_token(FlowMappingEndToken):
+ if not first:
+ if self.check_token(FlowEntryToken):
+ self.get_token()
+ else:
+ token = self.peek_token()
+ raise ParserError("while parsing a flow mapping", self.marks[-1],
+ "expected ',' or '}', but got %r" % token.id, token.start_mark)
+ if self.check_token(KeyToken):
+ token = self.get_token()
+ if not self.check_token(ValueToken,
+ FlowEntryToken, FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_value)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+ elif not self.check_token(FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_empty_value)
+ return self.parse_flow_node()
+ token = self.get_token()
+ event = MappingEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_flow_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(FlowEntryToken, FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_key)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_mapping_key
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_flow_mapping_key
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ def parse_flow_mapping_empty_value(self):
+ self.state = self.parse_flow_mapping_key
+ return self.process_empty_scalar(self.peek_token().start_mark)
+
+ def process_empty_scalar(self, mark):
+ return ScalarEvent(None, None, (True, False), '', mark, mark)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/reader.py b/collectors/python.d.plugin/python_modules/pyyaml3/reader.py
new file mode 100644
index 0000000..0a515fd
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/reader.py
@@ -0,0 +1,193 @@
+# SPDX-License-Identifier: MIT
+# This module contains abstractions for the input stream. You don't have to
+# looks further, there are no pretty code.
+#
+# We define two classes here.
+#
+# Mark(source, line, column)
+# It's just a record and its only use is producing nice error messages.
+# Parser does not use it for any other purposes.
+#
+# Reader(source, data)
+# Reader determines the encoding of `data` and converts it to unicode.
+# Reader provides the following methods and attributes:
+# reader.peek(length=1) - return the next `length` characters
+# reader.forward(length=1) - move the current position to `length` characters.
+# reader.index - the number of the current character.
+# reader.line, stream.column - the line and the column of the current character.
+
+__all__ = ['Reader', 'ReaderError']
+
+from .error import YAMLError, Mark
+
+import codecs, re
+
+class ReaderError(YAMLError):
+
+ def __init__(self, name, position, character, encoding, reason):
+ self.name = name
+ self.character = character
+ self.position = position
+ self.encoding = encoding
+ self.reason = reason
+
+ def __str__(self):
+ if isinstance(self.character, bytes):
+ return "'%s' codec can't decode byte #x%02x: %s\n" \
+ " in \"%s\", position %d" \
+ % (self.encoding, ord(self.character), self.reason,
+ self.name, self.position)
+ else:
+ return "unacceptable character #x%04x: %s\n" \
+ " in \"%s\", position %d" \
+ % (self.character, self.reason,
+ self.name, self.position)
+
+class Reader(object):
+ # Reader:
+ # - determines the data encoding and converts it to a unicode string,
+ # - checks if characters are in allowed range,
+ # - adds '\0' to the end.
+
+ # Reader accepts
+ # - a `bytes` object,
+ # - a `str` object,
+ # - a file-like object with its `read` method returning `str`,
+ # - a file-like object with its `read` method returning `unicode`.
+
+ # Yeah, it's ugly and slow.
+
+ def __init__(self, stream):
+ self.name = None
+ self.stream = None
+ self.stream_pointer = 0
+ self.eof = True
+ self.buffer = ''
+ self.pointer = 0
+ self.raw_buffer = None
+ self.raw_decode = None
+ self.encoding = None
+ self.index = 0
+ self.line = 0
+ self.column = 0
+ if isinstance(stream, str):
+ self.name = "<unicode string>"
+ self.check_printable(stream)
+ self.buffer = stream+'\0'
+ elif isinstance(stream, bytes):
+ self.name = "<byte string>"
+ self.raw_buffer = stream
+ self.determine_encoding()
+ else:
+ self.stream = stream
+ self.name = getattr(stream, 'name', "<file>")
+ self.eof = False
+ self.raw_buffer = None
+ self.determine_encoding()
+
+ def peek(self, index=0):
+ try:
+ return self.buffer[self.pointer+index]
+ except IndexError:
+ self.update(index+1)
+ return self.buffer[self.pointer+index]
+
+ def prefix(self, length=1):
+ if self.pointer+length >= len(self.buffer):
+ self.update(length)
+ return self.buffer[self.pointer:self.pointer+length]
+
+ def forward(self, length=1):
+ if self.pointer+length+1 >= len(self.buffer):
+ self.update(length+1)
+ while length:
+ ch = self.buffer[self.pointer]
+ self.pointer += 1
+ self.index += 1
+ if ch in '\n\x85\u2028\u2029' \
+ or (ch == '\r' and self.buffer[self.pointer] != '\n'):
+ self.line += 1
+ self.column = 0
+ elif ch != '\uFEFF':
+ self.column += 1
+ length -= 1
+
+ def get_mark(self):
+ if self.stream is None:
+ return Mark(self.name, self.index, self.line, self.column,
+ self.buffer, self.pointer)
+ else:
+ return Mark(self.name, self.index, self.line, self.column,
+ None, None)
+
+ def determine_encoding(self):
+ while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2):
+ self.update_raw()
+ if isinstance(self.raw_buffer, bytes):
+ if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
+ self.raw_decode = codecs.utf_16_le_decode
+ self.encoding = 'utf-16-le'
+ elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
+ self.raw_decode = codecs.utf_16_be_decode
+ self.encoding = 'utf-16-be'
+ else:
+ self.raw_decode = codecs.utf_8_decode
+ self.encoding = 'utf-8'
+ self.update(1)
+
+ NON_PRINTABLE = re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]')
+ def check_printable(self, data):
+ match = self.NON_PRINTABLE.search(data)
+ if match:
+ character = match.group()
+ position = self.index+(len(self.buffer)-self.pointer)+match.start()
+ raise ReaderError(self.name, position, ord(character),
+ 'unicode', "special characters are not allowed")
+
+ def update(self, length):
+ if self.raw_buffer is None:
+ return
+ self.buffer = self.buffer[self.pointer:]
+ self.pointer = 0
+ while len(self.buffer) < length:
+ if not self.eof:
+ self.update_raw()
+ if self.raw_decode is not None:
+ try:
+ data, converted = self.raw_decode(self.raw_buffer,
+ 'strict', self.eof)
+ except UnicodeDecodeError as exc:
+ character = self.raw_buffer[exc.start]
+ if self.stream is not None:
+ position = self.stream_pointer-len(self.raw_buffer)+exc.start
+ else:
+ position = exc.start
+ raise ReaderError(self.name, position, character,
+ exc.encoding, exc.reason)
+ else:
+ data = self.raw_buffer
+ converted = len(data)
+ self.check_printable(data)
+ self.buffer += data
+ self.raw_buffer = self.raw_buffer[converted:]
+ if self.eof:
+ self.buffer += '\0'
+ self.raw_buffer = None
+ break
+
+ def update_raw(self, size=4096):
+ data = self.stream.read(size)
+ if self.raw_buffer is None:
+ self.raw_buffer = data
+ else:
+ self.raw_buffer += data
+ self.stream_pointer += len(data)
+ if not data:
+ self.eof = True
+
+#try:
+# import psyco
+# psyco.bind(Reader)
+#except ImportError:
+# pass
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/representer.py b/collectors/python.d.plugin/python_modules/pyyaml3/representer.py
new file mode 100644
index 0000000..756a18d
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/representer.py
@@ -0,0 +1,375 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer',
+ 'RepresenterError']
+
+from .error import *
+from .nodes import *
+
+import datetime, sys, copyreg, types, base64
+
+class RepresenterError(YAMLError):
+ pass
+
+class BaseRepresenter:
+
+ yaml_representers = {}
+ yaml_multi_representers = {}
+
+ def __init__(self, default_style=None, default_flow_style=None):
+ self.default_style = default_style
+ self.default_flow_style = default_flow_style
+ self.represented_objects = {}
+ self.object_keeper = []
+ self.alias_key = None
+
+ def represent(self, data):
+ node = self.represent_data(data)
+ self.serialize(node)
+ self.represented_objects = {}
+ self.object_keeper = []
+ self.alias_key = None
+
+ def represent_data(self, data):
+ if self.ignore_aliases(data):
+ self.alias_key = None
+ else:
+ self.alias_key = id(data)
+ if self.alias_key is not None:
+ if self.alias_key in self.represented_objects:
+ node = self.represented_objects[self.alias_key]
+ #if node is None:
+ # raise RepresenterError("recursive objects are not allowed: %r" % data)
+ return node
+ #self.represented_objects[alias_key] = None
+ self.object_keeper.append(data)
+ data_types = type(data).__mro__
+ if data_types[0] in self.yaml_representers:
+ node = self.yaml_representers[data_types[0]](self, data)
+ else:
+ for data_type in data_types:
+ if data_type in self.yaml_multi_representers:
+ node = self.yaml_multi_representers[data_type](self, data)
+ break
+ else:
+ if None in self.yaml_multi_representers:
+ node = self.yaml_multi_representers[None](self, data)
+ elif None in self.yaml_representers:
+ node = self.yaml_representers[None](self, data)
+ else:
+ node = ScalarNode(None, str(data))
+ #if alias_key is not None:
+ # self.represented_objects[alias_key] = node
+ return node
+
+ @classmethod
+ def add_representer(cls, data_type, representer):
+ if not 'yaml_representers' in cls.__dict__:
+ cls.yaml_representers = cls.yaml_representers.copy()
+ cls.yaml_representers[data_type] = representer
+
+ @classmethod
+ def add_multi_representer(cls, data_type, representer):
+ if not 'yaml_multi_representers' in cls.__dict__:
+ cls.yaml_multi_representers = cls.yaml_multi_representers.copy()
+ cls.yaml_multi_representers[data_type] = representer
+
+ def represent_scalar(self, tag, value, style=None):
+ if style is None:
+ style = self.default_style
+ node = ScalarNode(tag, value, style=style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ return node
+
+ def represent_sequence(self, tag, sequence, flow_style=None):
+ value = []
+ node = SequenceNode(tag, value, flow_style=flow_style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ best_style = True
+ for item in sequence:
+ node_item = self.represent_data(item)
+ if not (isinstance(node_item, ScalarNode) and not node_item.style):
+ best_style = False
+ value.append(node_item)
+ if flow_style is None:
+ if self.default_flow_style is not None:
+ node.flow_style = self.default_flow_style
+ else:
+ node.flow_style = best_style
+ return node
+
+ def represent_mapping(self, tag, mapping, flow_style=None):
+ value = []
+ node = MappingNode(tag, value, flow_style=flow_style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ best_style = True
+ if hasattr(mapping, 'items'):
+ mapping = list(mapping.items())
+ try:
+ mapping = sorted(mapping)
+ except TypeError:
+ pass
+ for item_key, item_value in mapping:
+ node_key = self.represent_data(item_key)
+ node_value = self.represent_data(item_value)
+ if not (isinstance(node_key, ScalarNode) and not node_key.style):
+ best_style = False
+ if not (isinstance(node_value, ScalarNode) and not node_value.style):
+ best_style = False
+ value.append((node_key, node_value))
+ if flow_style is None:
+ if self.default_flow_style is not None:
+ node.flow_style = self.default_flow_style
+ else:
+ node.flow_style = best_style
+ return node
+
+ def ignore_aliases(self, data):
+ return False
+
+class SafeRepresenter(BaseRepresenter):
+
+ def ignore_aliases(self, data):
+ if data in [None, ()]:
+ return True
+ if isinstance(data, (str, bytes, bool, int, float)):
+ return True
+
+ def represent_none(self, data):
+ return self.represent_scalar('tag:yaml.org,2002:null', 'null')
+
+ def represent_str(self, data):
+ return self.represent_scalar('tag:yaml.org,2002:str', data)
+
+ def represent_binary(self, data):
+ if hasattr(base64, 'encodebytes'):
+ data = base64.encodebytes(data).decode('ascii')
+ else:
+ data = base64.encodestring(data).decode('ascii')
+ return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|')
+
+ def represent_bool(self, data):
+ if data:
+ value = 'true'
+ else:
+ value = 'false'
+ return self.represent_scalar('tag:yaml.org,2002:bool', value)
+
+ def represent_int(self, data):
+ return self.represent_scalar('tag:yaml.org,2002:int', str(data))
+
+ inf_value = 1e300
+ while repr(inf_value) != repr(inf_value*inf_value):
+ inf_value *= inf_value
+
+ def represent_float(self, data):
+ if data != data or (data == 0.0 and data == 1.0):
+ value = '.nan'
+ elif data == self.inf_value:
+ value = '.inf'
+ elif data == -self.inf_value:
+ value = '-.inf'
+ else:
+ value = repr(data).lower()
+ # Note that in some cases `repr(data)` represents a float number
+ # without the decimal parts. For instance:
+ # >>> repr(1e17)
+ # '1e17'
+ # Unfortunately, this is not a valid float representation according
+ # to the definition of the `!!float` tag. We fix this by adding
+ # '.0' before the 'e' symbol.
+ if '.' not in value and 'e' in value:
+ value = value.replace('e', '.0e', 1)
+ return self.represent_scalar('tag:yaml.org,2002:float', value)
+
+ def represent_list(self, data):
+ #pairs = (len(data) > 0 and isinstance(data, list))
+ #if pairs:
+ # for item in data:
+ # if not isinstance(item, tuple) or len(item) != 2:
+ # pairs = False
+ # break
+ #if not pairs:
+ return self.represent_sequence('tag:yaml.org,2002:seq', data)
+ #value = []
+ #for item_key, item_value in data:
+ # value.append(self.represent_mapping(u'tag:yaml.org,2002:map',
+ # [(item_key, item_value)]))
+ #return SequenceNode(u'tag:yaml.org,2002:pairs', value)
+
+ def represent_dict(self, data):
+ return self.represent_mapping('tag:yaml.org,2002:map', data)
+
+ def represent_set(self, data):
+ value = {}
+ for key in data:
+ value[key] = None
+ return self.represent_mapping('tag:yaml.org,2002:set', value)
+
+ def represent_date(self, data):
+ value = data.isoformat()
+ return self.represent_scalar('tag:yaml.org,2002:timestamp', value)
+
+ def represent_datetime(self, data):
+ value = data.isoformat(' ')
+ return self.represent_scalar('tag:yaml.org,2002:timestamp', value)
+
+ def represent_yaml_object(self, tag, data, cls, flow_style=None):
+ if hasattr(data, '__getstate__'):
+ state = data.__getstate__()
+ else:
+ state = data.__dict__.copy()
+ return self.represent_mapping(tag, state, flow_style=flow_style)
+
+ def represent_undefined(self, data):
+ raise RepresenterError("cannot represent an object: %s" % data)
+
+SafeRepresenter.add_representer(type(None),
+ SafeRepresenter.represent_none)
+
+SafeRepresenter.add_representer(str,
+ SafeRepresenter.represent_str)
+
+SafeRepresenter.add_representer(bytes,
+ SafeRepresenter.represent_binary)
+
+SafeRepresenter.add_representer(bool,
+ SafeRepresenter.represent_bool)
+
+SafeRepresenter.add_representer(int,
+ SafeRepresenter.represent_int)
+
+SafeRepresenter.add_representer(float,
+ SafeRepresenter.represent_float)
+
+SafeRepresenter.add_representer(list,
+ SafeRepresenter.represent_list)
+
+SafeRepresenter.add_representer(tuple,
+ SafeRepresenter.represent_list)
+
+SafeRepresenter.add_representer(dict,
+ SafeRepresenter.represent_dict)
+
+SafeRepresenter.add_representer(set,
+ SafeRepresenter.represent_set)
+
+SafeRepresenter.add_representer(datetime.date,
+ SafeRepresenter.represent_date)
+
+SafeRepresenter.add_representer(datetime.datetime,
+ SafeRepresenter.represent_datetime)
+
+SafeRepresenter.add_representer(None,
+ SafeRepresenter.represent_undefined)
+
+class Representer(SafeRepresenter):
+
+ def represent_complex(self, data):
+ if data.imag == 0.0:
+ data = '%r' % data.real
+ elif data.real == 0.0:
+ data = '%rj' % data.imag
+ elif data.imag > 0:
+ data = '%r+%rj' % (data.real, data.imag)
+ else:
+ data = '%r%rj' % (data.real, data.imag)
+ return self.represent_scalar('tag:yaml.org,2002:python/complex', data)
+
+ def represent_tuple(self, data):
+ return self.represent_sequence('tag:yaml.org,2002:python/tuple', data)
+
+ def represent_name(self, data):
+ name = '%s.%s' % (data.__module__, data.__name__)
+ return self.represent_scalar('tag:yaml.org,2002:python/name:'+name, '')
+
+ def represent_module(self, data):
+ return self.represent_scalar(
+ 'tag:yaml.org,2002:python/module:'+data.__name__, '')
+
+ def represent_object(self, data):
+ # We use __reduce__ API to save the data. data.__reduce__ returns
+ # a tuple of length 2-5:
+ # (function, args, state, listitems, dictitems)
+
+ # For reconstructing, we calls function(*args), then set its state,
+ # listitems, and dictitems if they are not None.
+
+ # A special case is when function.__name__ == '__newobj__'. In this
+ # case we create the object with args[0].__new__(*args).
+
+ # Another special case is when __reduce__ returns a string - we don't
+ # support it.
+
+ # We produce a !!python/object, !!python/object/new or
+ # !!python/object/apply node.
+
+ cls = type(data)
+ if cls in copyreg.dispatch_table:
+ reduce = copyreg.dispatch_table[cls](data)
+ elif hasattr(data, '__reduce_ex__'):
+ reduce = data.__reduce_ex__(2)
+ elif hasattr(data, '__reduce__'):
+ reduce = data.__reduce__()
+ else:
+ raise RepresenterError("cannot represent object: %r" % data)
+ reduce = (list(reduce)+[None]*5)[:5]
+ function, args, state, listitems, dictitems = reduce
+ args = list(args)
+ if state is None:
+ state = {}
+ if listitems is not None:
+ listitems = list(listitems)
+ if dictitems is not None:
+ dictitems = dict(dictitems)
+ if function.__name__ == '__newobj__':
+ function = args[0]
+ args = args[1:]
+ tag = 'tag:yaml.org,2002:python/object/new:'
+ newobj = True
+ else:
+ tag = 'tag:yaml.org,2002:python/object/apply:'
+ newobj = False
+ function_name = '%s.%s' % (function.__module__, function.__name__)
+ if not args and not listitems and not dictitems \
+ and isinstance(state, dict) and newobj:
+ return self.represent_mapping(
+ 'tag:yaml.org,2002:python/object:'+function_name, state)
+ if not listitems and not dictitems \
+ and isinstance(state, dict) and not state:
+ return self.represent_sequence(tag+function_name, args)
+ value = {}
+ if args:
+ value['args'] = args
+ if state or not isinstance(state, dict):
+ value['state'] = state
+ if listitems:
+ value['listitems'] = listitems
+ if dictitems:
+ value['dictitems'] = dictitems
+ return self.represent_mapping(tag+function_name, value)
+
+Representer.add_representer(complex,
+ Representer.represent_complex)
+
+Representer.add_representer(tuple,
+ Representer.represent_tuple)
+
+Representer.add_representer(type,
+ Representer.represent_name)
+
+Representer.add_representer(types.FunctionType,
+ Representer.represent_name)
+
+Representer.add_representer(types.BuiltinFunctionType,
+ Representer.represent_name)
+
+Representer.add_representer(types.ModuleType,
+ Representer.represent_module)
+
+Representer.add_multi_representer(object,
+ Representer.represent_object)
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/resolver.py b/collectors/python.d.plugin/python_modules/pyyaml3/resolver.py
new file mode 100644
index 0000000..50945e0
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/resolver.py
@@ -0,0 +1,225 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['BaseResolver', 'Resolver']
+
+from .error import *
+from .nodes import *
+
+import re
+
+class ResolverError(YAMLError):
+ pass
+
+class BaseResolver:
+
+ DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str'
+ DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq'
+ DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map'
+
+ yaml_implicit_resolvers = {}
+ yaml_path_resolvers = {}
+
+ def __init__(self):
+ self.resolver_exact_paths = []
+ self.resolver_prefix_paths = []
+
+ @classmethod
+ def add_implicit_resolver(cls, tag, regexp, first):
+ if not 'yaml_implicit_resolvers' in cls.__dict__:
+ cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()
+ if first is None:
+ first = [None]
+ for ch in first:
+ cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
+
+ @classmethod
+ def add_path_resolver(cls, tag, path, kind=None):
+ # Note: `add_path_resolver` is experimental. The API could be changed.
+ # `new_path` is a pattern that is matched against the path from the
+ # root to the node that is being considered. `node_path` elements are
+ # tuples `(node_check, index_check)`. `node_check` is a node class:
+ # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None`
+ # matches any kind of a node. `index_check` could be `None`, a boolean
+ # value, a string value, or a number. `None` and `False` match against
+ # any _value_ of sequence and mapping nodes. `True` matches against
+ # any _key_ of a mapping node. A string `index_check` matches against
+ # a mapping value that corresponds to a scalar key which content is
+ # equal to the `index_check` value. An integer `index_check` matches
+ # against a sequence value with the index equal to `index_check`.
+ if not 'yaml_path_resolvers' in cls.__dict__:
+ cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy()
+ new_path = []
+ for element in path:
+ if isinstance(element, (list, tuple)):
+ if len(element) == 2:
+ node_check, index_check = element
+ elif len(element) == 1:
+ node_check = element[0]
+ index_check = True
+ else:
+ raise ResolverError("Invalid path element: %s" % element)
+ else:
+ node_check = None
+ index_check = element
+ if node_check is str:
+ node_check = ScalarNode
+ elif node_check is list:
+ node_check = SequenceNode
+ elif node_check is dict:
+ node_check = MappingNode
+ elif node_check not in [ScalarNode, SequenceNode, MappingNode] \
+ and not isinstance(node_check, str) \
+ and node_check is not None:
+ raise ResolverError("Invalid node checker: %s" % node_check)
+ if not isinstance(index_check, (str, int)) \
+ and index_check is not None:
+ raise ResolverError("Invalid index checker: %s" % index_check)
+ new_path.append((node_check, index_check))
+ if kind is str:
+ kind = ScalarNode
+ elif kind is list:
+ kind = SequenceNode
+ elif kind is dict:
+ kind = MappingNode
+ elif kind not in [ScalarNode, SequenceNode, MappingNode] \
+ and kind is not None:
+ raise ResolverError("Invalid node kind: %s" % kind)
+ cls.yaml_path_resolvers[tuple(new_path), kind] = tag
+
+ def descend_resolver(self, current_node, current_index):
+ if not self.yaml_path_resolvers:
+ return
+ exact_paths = {}
+ prefix_paths = []
+ if current_node:
+ depth = len(self.resolver_prefix_paths)
+ for path, kind in self.resolver_prefix_paths[-1]:
+ if self.check_resolver_prefix(depth, path, kind,
+ current_node, current_index):
+ if len(path) > depth:
+ prefix_paths.append((path, kind))
+ else:
+ exact_paths[kind] = self.yaml_path_resolvers[path, kind]
+ else:
+ for path, kind in self.yaml_path_resolvers:
+ if not path:
+ exact_paths[kind] = self.yaml_path_resolvers[path, kind]
+ else:
+ prefix_paths.append((path, kind))
+ self.resolver_exact_paths.append(exact_paths)
+ self.resolver_prefix_paths.append(prefix_paths)
+
+ def ascend_resolver(self):
+ if not self.yaml_path_resolvers:
+ return
+ self.resolver_exact_paths.pop()
+ self.resolver_prefix_paths.pop()
+
+ def check_resolver_prefix(self, depth, path, kind,
+ current_node, current_index):
+ node_check, index_check = path[depth-1]
+ if isinstance(node_check, str):
+ if current_node.tag != node_check:
+ return
+ elif node_check is not None:
+ if not isinstance(current_node, node_check):
+ return
+ if index_check is True and current_index is not None:
+ return
+ if (index_check is False or index_check is None) \
+ and current_index is None:
+ return
+ if isinstance(index_check, str):
+ if not (isinstance(current_index, ScalarNode)
+ and index_check == current_index.value):
+ return
+ elif isinstance(index_check, int) and not isinstance(index_check, bool):
+ if index_check != current_index:
+ return
+ return True
+
+ def resolve(self, kind, value, implicit):
+ if kind is ScalarNode and implicit[0]:
+ if value == '':
+ resolvers = self.yaml_implicit_resolvers.get('', [])
+ else:
+ resolvers = self.yaml_implicit_resolvers.get(value[0], [])
+ resolvers += self.yaml_implicit_resolvers.get(None, [])
+ for tag, regexp in resolvers:
+ if regexp.match(value):
+ return tag
+ implicit = implicit[1]
+ if self.yaml_path_resolvers:
+ exact_paths = self.resolver_exact_paths[-1]
+ if kind in exact_paths:
+ return exact_paths[kind]
+ if None in exact_paths:
+ return exact_paths[None]
+ if kind is ScalarNode:
+ return self.DEFAULT_SCALAR_TAG
+ elif kind is SequenceNode:
+ return self.DEFAULT_SEQUENCE_TAG
+ elif kind is MappingNode:
+ return self.DEFAULT_MAPPING_TAG
+
+class Resolver(BaseResolver):
+ pass
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:bool',
+ re.compile(r'''^(?:yes|Yes|YES|no|No|NO
+ |true|True|TRUE|false|False|FALSE
+ |on|On|ON|off|Off|OFF)$''', re.X),
+ list('yYnNtTfFoO'))
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:float',
+ re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?
+ |\.[0-9_]+(?:[eE][-+][0-9]+)?
+ |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
+ |[-+]?\.(?:inf|Inf|INF)
+ |\.(?:nan|NaN|NAN))$''', re.X),
+ list('-+0123456789.'))
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:int',
+ re.compile(r'''^(?:[-+]?0b[0-1_]+
+ |[-+]?0[0-7_]+
+ |[-+]?(?:0|[1-9][0-9_]*)
+ |[-+]?0x[0-9a-fA-F_]+
+ |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X),
+ list('-+0123456789'))
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:merge',
+ re.compile(r'^(?:<<)$'),
+ ['<'])
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:null',
+ re.compile(r'''^(?: ~
+ |null|Null|NULL
+ | )$''', re.X),
+ ['~', 'n', 'N', ''])
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:timestamp',
+ re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
+ |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
+ (?:[Tt]|[ \t]+)[0-9][0-9]?
+ :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)?
+ (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
+ list('0123456789'))
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:value',
+ re.compile(r'^(?:=)$'),
+ ['='])
+
+# The following resolver is only for documentation purposes. It cannot work
+# because plain scalars cannot start with '!', '&', or '*'.
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:yaml',
+ re.compile(r'^(?:!|&|\*)$'),
+ list('!&*'))
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/scanner.py b/collectors/python.d.plugin/python_modules/pyyaml3/scanner.py
new file mode 100644
index 0000000..b55854e
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/scanner.py
@@ -0,0 +1,1449 @@
+# SPDX-License-Identifier: MIT
+
+# Scanner produces tokens of the following types:
+# STREAM-START
+# STREAM-END
+# DIRECTIVE(name, value)
+# DOCUMENT-START
+# DOCUMENT-END
+# BLOCK-SEQUENCE-START
+# BLOCK-MAPPING-START
+# BLOCK-END
+# FLOW-SEQUENCE-START
+# FLOW-MAPPING-START
+# FLOW-SEQUENCE-END
+# FLOW-MAPPING-END
+# BLOCK-ENTRY
+# FLOW-ENTRY
+# KEY
+# VALUE
+# ALIAS(value)
+# ANCHOR(value)
+# TAG(value)
+# SCALAR(value, plain, style)
+#
+# Read comments in the Scanner code for more details.
+#
+
+__all__ = ['Scanner', 'ScannerError']
+
+from .error import MarkedYAMLError
+from .tokens import *
+
+class ScannerError(MarkedYAMLError):
+ pass
+
+class SimpleKey:
+ # See below simple keys treatment.
+
+ def __init__(self, token_number, required, index, line, column, mark):
+ self.token_number = token_number
+ self.required = required
+ self.index = index
+ self.line = line
+ self.column = column
+ self.mark = mark
+
+class Scanner:
+
+ def __init__(self):
+ """Initialize the scanner."""
+ # It is assumed that Scanner and Reader will have a common descendant.
+ # Reader do the dirty work of checking for BOM and converting the
+ # input data to Unicode. It also adds NUL to the end.
+ #
+ # Reader supports the following methods
+ # self.peek(i=0) # peek the next i-th character
+ # self.prefix(l=1) # peek the next l characters
+ # self.forward(l=1) # read the next l characters and move the pointer.
+
+ # Had we reached the end of the stream?
+ self.done = False
+
+ # The number of unclosed '{' and '['. `flow_level == 0` means block
+ # context.
+ self.flow_level = 0
+
+ # List of processed tokens that are not yet emitted.
+ self.tokens = []
+
+ # Add the STREAM-START token.
+ self.fetch_stream_start()
+
+ # Number of tokens that were emitted through the `get_token` method.
+ self.tokens_taken = 0
+
+ # The current indentation level.
+ self.indent = -1
+
+ # Past indentation levels.
+ self.indents = []
+
+ # Variables related to simple keys treatment.
+
+ # A simple key is a key that is not denoted by the '?' indicator.
+ # Example of simple keys:
+ # ---
+ # block simple key: value
+ # ? not a simple key:
+ # : { flow simple key: value }
+ # We emit the KEY token before all keys, so when we find a potential
+ # simple key, we try to locate the corresponding ':' indicator.
+ # Simple keys should be limited to a single line and 1024 characters.
+
+ # Can a simple key start at the current position? A simple key may
+ # start:
+ # - at the beginning of the line, not counting indentation spaces
+ # (in block context),
+ # - after '{', '[', ',' (in the flow context),
+ # - after '?', ':', '-' (in the block context).
+ # In the block context, this flag also signifies if a block collection
+ # may start at the current position.
+ self.allow_simple_key = True
+
+ # Keep track of possible simple keys. This is a dictionary. The key
+ # is `flow_level`; there can be no more that one possible simple key
+ # for each level. The value is a SimpleKey record:
+ # (token_number, required, index, line, column, mark)
+ # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
+ # '[', or '{' tokens.
+ self.possible_simple_keys = {}
+
+ # Public methods.
+
+ def check_token(self, *choices):
+ # Check if the next token is one of the given types.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ if not choices:
+ return True
+ for choice in choices:
+ if isinstance(self.tokens[0], choice):
+ return True
+ return False
+
+ def peek_token(self):
+ # Return the next token, but do not delete if from the queue.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ return self.tokens[0]
+
+ def get_token(self):
+ # Return the next token.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ self.tokens_taken += 1
+ return self.tokens.pop(0)
+
+ # Private methods.
+
+ def need_more_tokens(self):
+ if self.done:
+ return False
+ if not self.tokens:
+ return True
+ # The current token may be a potential simple key, so we
+ # need to look further.
+ self.stale_possible_simple_keys()
+ if self.next_possible_simple_key() == self.tokens_taken:
+ return True
+
+ def fetch_more_tokens(self):
+
+ # Eat whitespaces and comments until we reach the next token.
+ self.scan_to_next_token()
+
+ # Remove obsolete possible simple keys.
+ self.stale_possible_simple_keys()
+
+ # Compare the current indentation and column. It may add some tokens
+ # and decrease the current indentation level.
+ self.unwind_indent(self.column)
+
+ # Peek the next character.
+ ch = self.peek()
+
+ # Is it the end of stream?
+ if ch == '\0':
+ return self.fetch_stream_end()
+
+ # Is it a directive?
+ if ch == '%' and self.check_directive():
+ return self.fetch_directive()
+
+ # Is it the document start?
+ if ch == '-' and self.check_document_start():
+ return self.fetch_document_start()
+
+ # Is it the document end?
+ if ch == '.' and self.check_document_end():
+ return self.fetch_document_end()
+
+ # TODO: support for BOM within a stream.
+ #if ch == '\uFEFF':
+ # return self.fetch_bom() <-- issue BOMToken
+
+ # Note: the order of the following checks is NOT significant.
+
+ # Is it the flow sequence start indicator?
+ if ch == '[':
+ return self.fetch_flow_sequence_start()
+
+ # Is it the flow mapping start indicator?
+ if ch == '{':
+ return self.fetch_flow_mapping_start()
+
+ # Is it the flow sequence end indicator?
+ if ch == ']':
+ return self.fetch_flow_sequence_end()
+
+ # Is it the flow mapping end indicator?
+ if ch == '}':
+ return self.fetch_flow_mapping_end()
+
+ # Is it the flow entry indicator?
+ if ch == ',':
+ return self.fetch_flow_entry()
+
+ # Is it the block entry indicator?
+ if ch == '-' and self.check_block_entry():
+ return self.fetch_block_entry()
+
+ # Is it the key indicator?
+ if ch == '?' and self.check_key():
+ return self.fetch_key()
+
+ # Is it the value indicator?
+ if ch == ':' and self.check_value():
+ return self.fetch_value()
+
+ # Is it an alias?
+ if ch == '*':
+ return self.fetch_alias()
+
+ # Is it an anchor?
+ if ch == '&':
+ return self.fetch_anchor()
+
+ # Is it a tag?
+ if ch == '!':
+ return self.fetch_tag()
+
+ # Is it a literal scalar?
+ if ch == '|' and not self.flow_level:
+ return self.fetch_literal()
+
+ # Is it a folded scalar?
+ if ch == '>' and not self.flow_level:
+ return self.fetch_folded()
+
+ # Is it a single quoted scalar?
+ if ch == '\'':
+ return self.fetch_single()
+
+ # Is it a double quoted scalar?
+ if ch == '\"':
+ return self.fetch_double()
+
+ # It must be a plain scalar then.
+ if self.check_plain():
+ return self.fetch_plain()
+
+ # No? It's an error. Let's produce a nice error message.
+ raise ScannerError("while scanning for the next token", None,
+ "found character %r that cannot start any token" % ch,
+ self.get_mark())
+
+ # Simple keys treatment.
+
+ def next_possible_simple_key(self):
+ # Return the number of the nearest possible simple key. Actually we
+ # don't need to loop through the whole dictionary. We may replace it
+ # with the following code:
+ # if not self.possible_simple_keys:
+ # return None
+ # return self.possible_simple_keys[
+ # min(self.possible_simple_keys.keys())].token_number
+ min_token_number = None
+ for level in self.possible_simple_keys:
+ key = self.possible_simple_keys[level]
+ if min_token_number is None or key.token_number < min_token_number:
+ min_token_number = key.token_number
+ return min_token_number
+
+ def stale_possible_simple_keys(self):
+ # Remove entries that are no longer possible simple keys. According to
+ # the YAML specification, simple keys
+ # - should be limited to a single line,
+ # - should be no longer than 1024 characters.
+ # Disabling this procedure will allow simple keys of any length and
+ # height (may cause problems if indentation is broken though).
+ for level in list(self.possible_simple_keys):
+ key = self.possible_simple_keys[level]
+ if key.line != self.line \
+ or self.index-key.index > 1024:
+ if key.required:
+ raise ScannerError("while scanning a simple key", key.mark,
+ "could not found expected ':'", self.get_mark())
+ del self.possible_simple_keys[level]
+
+ def save_possible_simple_key(self):
+ # The next token may start a simple key. We check if it's possible
+ # and save its position. This function is called for
+ # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
+
+ # Check if a simple key is required at the current position.
+ required = not self.flow_level and self.indent == self.column
+
+ # A simple key is required only if it is the first token in the current
+ # line. Therefore it is always allowed.
+ assert self.allow_simple_key or not required
+
+ # The next token might be a simple key. Let's save it's number and
+ # position.
+ if self.allow_simple_key:
+ self.remove_possible_simple_key()
+ token_number = self.tokens_taken+len(self.tokens)
+ key = SimpleKey(token_number, required,
+ self.index, self.line, self.column, self.get_mark())
+ self.possible_simple_keys[self.flow_level] = key
+
+ def remove_possible_simple_key(self):
+ # Remove the saved possible key position at the current flow level.
+ if self.flow_level in self.possible_simple_keys:
+ key = self.possible_simple_keys[self.flow_level]
+
+ if key.required:
+ raise ScannerError("while scanning a simple key", key.mark,
+ "could not found expected ':'", self.get_mark())
+
+ del self.possible_simple_keys[self.flow_level]
+
+ # Indentation functions.
+
+ def unwind_indent(self, column):
+
+ ## In flow context, tokens should respect indentation.
+ ## Actually the condition should be `self.indent >= column` according to
+ ## the spec. But this condition will prohibit intuitively correct
+ ## constructions such as
+ ## key : {
+ ## }
+ #if self.flow_level and self.indent > column:
+ # raise ScannerError(None, None,
+ # "invalid intendation or unclosed '[' or '{'",
+ # self.get_mark())
+
+ # In the flow context, indentation is ignored. We make the scanner less
+ # restrictive then specification requires.
+ if self.flow_level:
+ return
+
+ # In block context, we may need to issue the BLOCK-END tokens.
+ while self.indent > column:
+ mark = self.get_mark()
+ self.indent = self.indents.pop()
+ self.tokens.append(BlockEndToken(mark, mark))
+
+ def add_indent(self, column):
+ # Check if we need to increase indentation.
+ if self.indent < column:
+ self.indents.append(self.indent)
+ self.indent = column
+ return True
+ return False
+
+ # Fetchers.
+
+ def fetch_stream_start(self):
+ # We always add STREAM-START as the first token and STREAM-END as the
+ # last token.
+
+ # Read the token.
+ mark = self.get_mark()
+
+ # Add STREAM-START.
+ self.tokens.append(StreamStartToken(mark, mark,
+ encoding=self.encoding))
+
+
+ def fetch_stream_end(self):
+
+ # Set the current intendation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+ self.possible_simple_keys = {}
+
+ # Read the token.
+ mark = self.get_mark()
+
+ # Add STREAM-END.
+ self.tokens.append(StreamEndToken(mark, mark))
+
+ # The steam is finished.
+ self.done = True
+
+ def fetch_directive(self):
+
+ # Set the current intendation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+
+ # Scan and add DIRECTIVE.
+ self.tokens.append(self.scan_directive())
+
+ def fetch_document_start(self):
+ self.fetch_document_indicator(DocumentStartToken)
+
+ def fetch_document_end(self):
+ self.fetch_document_indicator(DocumentEndToken)
+
+ def fetch_document_indicator(self, TokenClass):
+
+ # Set the current intendation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys. Note that there could not be a block collection
+ # after '---'.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+
+ # Add DOCUMENT-START or DOCUMENT-END.
+ start_mark = self.get_mark()
+ self.forward(3)
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_sequence_start(self):
+ self.fetch_flow_collection_start(FlowSequenceStartToken)
+
+ def fetch_flow_mapping_start(self):
+ self.fetch_flow_collection_start(FlowMappingStartToken)
+
+ def fetch_flow_collection_start(self, TokenClass):
+
+ # '[' and '{' may start a simple key.
+ self.save_possible_simple_key()
+
+ # Increase the flow level.
+ self.flow_level += 1
+
+ # Simple keys are allowed after '[' and '{'.
+ self.allow_simple_key = True
+
+ # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_sequence_end(self):
+ self.fetch_flow_collection_end(FlowSequenceEndToken)
+
+ def fetch_flow_mapping_end(self):
+ self.fetch_flow_collection_end(FlowMappingEndToken)
+
+ def fetch_flow_collection_end(self, TokenClass):
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Decrease the flow level.
+ self.flow_level -= 1
+
+ # No simple keys after ']' or '}'.
+ self.allow_simple_key = False
+
+ # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_entry(self):
+
+ # Simple keys are allowed after ','.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add FLOW-ENTRY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(FlowEntryToken(start_mark, end_mark))
+
+ def fetch_block_entry(self):
+
+ # Block context needs additional checks.
+ if not self.flow_level:
+
+ # Are we allowed to start a new entry?
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "sequence entries are not allowed here",
+ self.get_mark())
+
+ # We may need to add BLOCK-SEQUENCE-START.
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockSequenceStartToken(mark, mark))
+
+ # It's an error for the block entry to occur in the flow context,
+ # but we let the parser detect this.
+ else:
+ pass
+
+ # Simple keys are allowed after '-'.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add BLOCK-ENTRY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(BlockEntryToken(start_mark, end_mark))
+
+ def fetch_key(self):
+
+ # Block context needs additional checks.
+ if not self.flow_level:
+
+ # Are we allowed to start a key (not nessesary a simple)?
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "mapping keys are not allowed here",
+ self.get_mark())
+
+ # We may need to add BLOCK-MAPPING-START.
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockMappingStartToken(mark, mark))
+
+ # Simple keys are allowed after '?' in the block context.
+ self.allow_simple_key = not self.flow_level
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add KEY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(KeyToken(start_mark, end_mark))
+
+ def fetch_value(self):
+
+ # Do we determine a simple key?
+ if self.flow_level in self.possible_simple_keys:
+
+ # Add KEY.
+ key = self.possible_simple_keys[self.flow_level]
+ del self.possible_simple_keys[self.flow_level]
+ self.tokens.insert(key.token_number-self.tokens_taken,
+ KeyToken(key.mark, key.mark))
+
+ # If this key starts a new block mapping, we need to add
+ # BLOCK-MAPPING-START.
+ if not self.flow_level:
+ if self.add_indent(key.column):
+ self.tokens.insert(key.token_number-self.tokens_taken,
+ BlockMappingStartToken(key.mark, key.mark))
+
+ # There cannot be two simple keys one after another.
+ self.allow_simple_key = False
+
+ # It must be a part of a complex key.
+ else:
+
+ # Block context needs additional checks.
+ # (Do we really need them? They will be catched by the parser
+ # anyway.)
+ if not self.flow_level:
+
+ # We are allowed to start a complex value if and only if
+ # we can start a simple key.
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "mapping values are not allowed here",
+ self.get_mark())
+
+ # If this value starts a new block mapping, we need to add
+ # BLOCK-MAPPING-START. It will be detected as an error later by
+ # the parser.
+ if not self.flow_level:
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockMappingStartToken(mark, mark))
+
+ # Simple keys are allowed after ':' in the block context.
+ self.allow_simple_key = not self.flow_level
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add VALUE.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(ValueToken(start_mark, end_mark))
+
+ def fetch_alias(self):
+
+ # ALIAS could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after ALIAS.
+ self.allow_simple_key = False
+
+ # Scan and add ALIAS.
+ self.tokens.append(self.scan_anchor(AliasToken))
+
+ def fetch_anchor(self):
+
+ # ANCHOR could start a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after ANCHOR.
+ self.allow_simple_key = False
+
+ # Scan and add ANCHOR.
+ self.tokens.append(self.scan_anchor(AnchorToken))
+
+ def fetch_tag(self):
+
+ # TAG could start a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after TAG.
+ self.allow_simple_key = False
+
+ # Scan and add TAG.
+ self.tokens.append(self.scan_tag())
+
+ def fetch_literal(self):
+ self.fetch_block_scalar(style='|')
+
+ def fetch_folded(self):
+ self.fetch_block_scalar(style='>')
+
+ def fetch_block_scalar(self, style):
+
+ # A simple key may follow a block scalar.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Scan and add SCALAR.
+ self.tokens.append(self.scan_block_scalar(style))
+
+ def fetch_single(self):
+ self.fetch_flow_scalar(style='\'')
+
+ def fetch_double(self):
+ self.fetch_flow_scalar(style='"')
+
+ def fetch_flow_scalar(self, style):
+
+ # A flow scalar could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after flow scalars.
+ self.allow_simple_key = False
+
+ # Scan and add SCALAR.
+ self.tokens.append(self.scan_flow_scalar(style))
+
+ def fetch_plain(self):
+
+ # A plain scalar could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after plain scalars. But note that `scan_plain` will
+ # change this flag if the scan is finished at the beginning of the
+ # line.
+ self.allow_simple_key = False
+
+ # Scan and add SCALAR. May change `allow_simple_key`.
+ self.tokens.append(self.scan_plain())
+
+ # Checkers.
+
+ def check_directive(self):
+
+ # DIRECTIVE: ^ '%' ...
+ # The '%' indicator is already checked.
+ if self.column == 0:
+ return True
+
+ def check_document_start(self):
+
+ # DOCUMENT-START: ^ '---' (' '|'\n')
+ if self.column == 0:
+ if self.prefix(3) == '---' \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ return True
+
+ def check_document_end(self):
+
+ # DOCUMENT-END: ^ '...' (' '|'\n')
+ if self.column == 0:
+ if self.prefix(3) == '...' \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ return True
+
+ def check_block_entry(self):
+
+ # BLOCK-ENTRY: '-' (' '|'\n')
+ return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+ def check_key(self):
+
+ # KEY(flow context): '?'
+ if self.flow_level:
+ return True
+
+ # KEY(block context): '?' (' '|'\n')
+ else:
+ return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+ def check_value(self):
+
+ # VALUE(flow context): ':'
+ if self.flow_level:
+ return True
+
+ # VALUE(block context): ':' (' '|'\n')
+ else:
+ return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+ def check_plain(self):
+
+ # A plain scalar may start with any non-space character except:
+ # '-', '?', ':', ',', '[', ']', '{', '}',
+ # '#', '&', '*', '!', '|', '>', '\'', '\"',
+ # '%', '@', '`'.
+ #
+ # It may also start with
+ # '-', '?', ':'
+ # if it is followed by a non-space character.
+ #
+ # Note that we limit the last rule to the block context (except the
+ # '-' character) because we want the flow context to be space
+ # independent.
+ ch = self.peek()
+ return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \
+ or (self.peek(1) not in '\0 \t\r\n\x85\u2028\u2029'
+ and (ch == '-' or (not self.flow_level and ch in '?:')))
+
+ # Scanners.
+
+ def scan_to_next_token(self):
+ # We ignore spaces, line breaks and comments.
+ # If we find a line break in the block context, we set the flag
+ # `allow_simple_key` on.
+ # The byte order mark is stripped if it's the first character in the
+ # stream. We do not yet support BOM inside the stream as the
+ # specification requires. Any such mark will be considered as a part
+ # of the document.
+ #
+ # TODO: We need to make tab handling rules more sane. A good rule is
+ # Tabs cannot precede tokens
+ # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
+ # KEY(block), VALUE(block), BLOCK-ENTRY
+ # So the checking code is
+ # if <TAB>:
+ # self.allow_simple_keys = False
+ # We also need to add the check for `allow_simple_keys == True` to
+ # `unwind_indent` before issuing BLOCK-END.
+ # Scanners for block, flow, and plain scalars need to be modified.
+
+ if self.index == 0 and self.peek() == '\uFEFF':
+ self.forward()
+ found = False
+ while not found:
+ while self.peek() == ' ':
+ self.forward()
+ if self.peek() == '#':
+ while self.peek() not in '\0\r\n\x85\u2028\u2029':
+ self.forward()
+ if self.scan_line_break():
+ if not self.flow_level:
+ self.allow_simple_key = True
+ else:
+ found = True
+
+ def scan_directive(self):
+ # See the specification for details.
+ start_mark = self.get_mark()
+ self.forward()
+ name = self.scan_directive_name(start_mark)
+ value = None
+ if name == 'YAML':
+ value = self.scan_yaml_directive_value(start_mark)
+ end_mark = self.get_mark()
+ elif name == 'TAG':
+ value = self.scan_tag_directive_value(start_mark)
+ end_mark = self.get_mark()
+ else:
+ end_mark = self.get_mark()
+ while self.peek() not in '\0\r\n\x85\u2028\u2029':
+ self.forward()
+ self.scan_directive_ignored_line(start_mark)
+ return DirectiveToken(name, value, start_mark, end_mark)
+
+ def scan_directive_name(self, start_mark):
+ # See the specification for details.
+ length = 0
+ ch = self.peek(length)
+ while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_':
+ length += 1
+ ch = self.peek(length)
+ if not length:
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch, self.get_mark())
+ value = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch, self.get_mark())
+ return value
+
+ def scan_yaml_directive_value(self, start_mark):
+ # See the specification for details.
+ while self.peek() == ' ':
+ self.forward()
+ major = self.scan_yaml_directive_number(start_mark)
+ if self.peek() != '.':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit or '.', but found %r" % self.peek(),
+ self.get_mark())
+ self.forward()
+ minor = self.scan_yaml_directive_number(start_mark)
+ if self.peek() not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit or ' ', but found %r" % self.peek(),
+ self.get_mark())
+ return (major, minor)
+
+ def scan_yaml_directive_number(self, start_mark):
+ # See the specification for details.
+ ch = self.peek()
+ if not ('0' <= ch <= '9'):
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit, but found %r" % ch, self.get_mark())
+ length = 0
+ while '0' <= self.peek(length) <= '9':
+ length += 1
+ value = int(self.prefix(length))
+ self.forward(length)
+ return value
+
+ def scan_tag_directive_value(self, start_mark):
+ # See the specification for details.
+ while self.peek() == ' ':
+ self.forward()
+ handle = self.scan_tag_directive_handle(start_mark)
+ while self.peek() == ' ':
+ self.forward()
+ prefix = self.scan_tag_directive_prefix(start_mark)
+ return (handle, prefix)
+
+ def scan_tag_directive_handle(self, start_mark):
+ # See the specification for details.
+ value = self.scan_tag_handle('directive', start_mark)
+ ch = self.peek()
+ if ch != ' ':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected ' ', but found %r" % ch, self.get_mark())
+ return value
+
+ def scan_tag_directive_prefix(self, start_mark):
+ # See the specification for details.
+ value = self.scan_tag_uri('directive', start_mark)
+ ch = self.peek()
+ if ch not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected ' ', but found %r" % ch, self.get_mark())
+ return value
+
+ def scan_directive_ignored_line(self, start_mark):
+ # See the specification for details.
+ while self.peek() == ' ':
+ self.forward()
+ if self.peek() == '#':
+ while self.peek() not in '\0\r\n\x85\u2028\u2029':
+ self.forward()
+ ch = self.peek()
+ if ch not in '\0\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a comment or a line break, but found %r"
+ % ch, self.get_mark())
+ self.scan_line_break()
+
+ def scan_anchor(self, TokenClass):
+ # The specification does not restrict characters for anchors and
+ # aliases. This may lead to problems, for instance, the document:
+ # [ *alias, value ]
+ # can be interpteted in two ways, as
+ # [ "value" ]
+ # and
+ # [ *alias , "value" ]
+ # Therefore we restrict aliases to numbers and ASCII letters.
+ start_mark = self.get_mark()
+ indicator = self.peek()
+ if indicator == '*':
+ name = 'alias'
+ else:
+ name = 'anchor'
+ self.forward()
+ length = 0
+ ch = self.peek(length)
+ while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_':
+ length += 1
+ ch = self.peek(length)
+ if not length:
+ raise ScannerError("while scanning an %s" % name, start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch, self.get_mark())
+ value = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch not in '\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
+ raise ScannerError("while scanning an %s" % name, start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch, self.get_mark())
+ end_mark = self.get_mark()
+ return TokenClass(value, start_mark, end_mark)
+
+ def scan_tag(self):
+ # See the specification for details.
+ start_mark = self.get_mark()
+ ch = self.peek(1)
+ if ch == '<':
+ handle = None
+ self.forward(2)
+ suffix = self.scan_tag_uri('tag', start_mark)
+ if self.peek() != '>':
+ raise ScannerError("while parsing a tag", start_mark,
+ "expected '>', but found %r" % self.peek(),
+ self.get_mark())
+ self.forward()
+ elif ch in '\0 \t\r\n\x85\u2028\u2029':
+ handle = None
+ suffix = '!'
+ self.forward()
+ else:
+ length = 1
+ use_handle = False
+ while ch not in '\0 \r\n\x85\u2028\u2029':
+ if ch == '!':
+ use_handle = True
+ break
+ length += 1
+ ch = self.peek(length)
+ handle = '!'
+ if use_handle:
+ handle = self.scan_tag_handle('tag', start_mark)
+ else:
+ handle = '!'
+ self.forward()
+ suffix = self.scan_tag_uri('tag', start_mark)
+ ch = self.peek()
+ if ch not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a tag", start_mark,
+ "expected ' ', but found %r" % ch, self.get_mark())
+ value = (handle, suffix)
+ end_mark = self.get_mark()
+ return TagToken(value, start_mark, end_mark)
+
+ def scan_block_scalar(self, style):
+ # See the specification for details.
+
+ if style == '>':
+ folded = True
+ else:
+ folded = False
+
+ chunks = []
+ start_mark = self.get_mark()
+
+ # Scan the header.
+ self.forward()
+ chomping, increment = self.scan_block_scalar_indicators(start_mark)
+ self.scan_block_scalar_ignored_line(start_mark)
+
+ # Determine the indentation level and go to the first non-empty line.
+ min_indent = self.indent+1
+ if min_indent < 1:
+ min_indent = 1
+ if increment is None:
+ breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
+ indent = max(min_indent, max_indent)
+ else:
+ indent = min_indent+increment-1
+ breaks, end_mark = self.scan_block_scalar_breaks(indent)
+ line_break = ''
+
+ # Scan the inner part of the block scalar.
+ while self.column == indent and self.peek() != '\0':
+ chunks.extend(breaks)
+ leading_non_space = self.peek() not in ' \t'
+ length = 0
+ while self.peek(length) not in '\0\r\n\x85\u2028\u2029':
+ length += 1
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ line_break = self.scan_line_break()
+ breaks, end_mark = self.scan_block_scalar_breaks(indent)
+ if self.column == indent and self.peek() != '\0':
+
+ # Unfortunately, folding rules are ambiguous.
+ #
+ # This is the folding according to the specification:
+
+ if folded and line_break == '\n' \
+ and leading_non_space and self.peek() not in ' \t':
+ if not breaks:
+ chunks.append(' ')
+ else:
+ chunks.append(line_break)
+
+ # This is Clark Evans's interpretation (also in the spec
+ # examples):
+ #
+ #if folded and line_break == '\n':
+ # if not breaks:
+ # if self.peek() not in ' \t':
+ # chunks.append(' ')
+ # else:
+ # chunks.append(line_break)
+ #else:
+ # chunks.append(line_break)
+ else:
+ break
+
+ # Chomp the tail.
+ if chomping is not False:
+ chunks.append(line_break)
+ if chomping is True:
+ chunks.extend(breaks)
+
+ # We are done.
+ return ScalarToken(''.join(chunks), False, start_mark, end_mark,
+ style)
+
+ def scan_block_scalar_indicators(self, start_mark):
+ # See the specification for details.
+ chomping = None
+ increment = None
+ ch = self.peek()
+ if ch in '+-':
+ if ch == '+':
+ chomping = True
+ else:
+ chomping = False
+ self.forward()
+ ch = self.peek()
+ if ch in '0123456789':
+ increment = int(ch)
+ if increment == 0:
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected indentation indicator in the range 1-9, but found 0",
+ self.get_mark())
+ self.forward()
+ elif ch in '0123456789':
+ increment = int(ch)
+ if increment == 0:
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected indentation indicator in the range 1-9, but found 0",
+ self.get_mark())
+ self.forward()
+ ch = self.peek()
+ if ch in '+-':
+ if ch == '+':
+ chomping = True
+ else:
+ chomping = False
+ self.forward()
+ ch = self.peek()
+ if ch not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected chomping or indentation indicators, but found %r"
+ % ch, self.get_mark())
+ return chomping, increment
+
+ def scan_block_scalar_ignored_line(self, start_mark):
+ # See the specification for details.
+ while self.peek() == ' ':
+ self.forward()
+ if self.peek() == '#':
+ while self.peek() not in '\0\r\n\x85\u2028\u2029':
+ self.forward()
+ ch = self.peek()
+ if ch not in '\0\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected a comment or a line break, but found %r" % ch,
+ self.get_mark())
+ self.scan_line_break()
+
+ def scan_block_scalar_indentation(self):
+ # See the specification for details.
+ chunks = []
+ max_indent = 0
+ end_mark = self.get_mark()
+ while self.peek() in ' \r\n\x85\u2028\u2029':
+ if self.peek() != ' ':
+ chunks.append(self.scan_line_break())
+ end_mark = self.get_mark()
+ else:
+ self.forward()
+ if self.column > max_indent:
+ max_indent = self.column
+ return chunks, max_indent, end_mark
+
+ def scan_block_scalar_breaks(self, indent):
+ # See the specification for details.
+ chunks = []
+ end_mark = self.get_mark()
+ while self.column < indent and self.peek() == ' ':
+ self.forward()
+ while self.peek() in '\r\n\x85\u2028\u2029':
+ chunks.append(self.scan_line_break())
+ end_mark = self.get_mark()
+ while self.column < indent and self.peek() == ' ':
+ self.forward()
+ return chunks, end_mark
+
+ def scan_flow_scalar(self, style):
+ # See the specification for details.
+ # Note that we loose indentation rules for quoted scalars. Quoted
+ # scalars don't need to adhere indentation because " and ' clearly
+ # mark the beginning and the end of them. Therefore we are less
+ # restrictive then the specification requires. We only need to check
+ # that document separators are not included in scalars.
+ if style == '"':
+ double = True
+ else:
+ double = False
+ chunks = []
+ start_mark = self.get_mark()
+ quote = self.peek()
+ self.forward()
+ chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
+ while self.peek() != quote:
+ chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
+ chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
+ self.forward()
+ end_mark = self.get_mark()
+ return ScalarToken(''.join(chunks), False, start_mark, end_mark,
+ style)
+
+ ESCAPE_REPLACEMENTS = {
+ '0': '\0',
+ 'a': '\x07',
+ 'b': '\x08',
+ 't': '\x09',
+ '\t': '\x09',
+ 'n': '\x0A',
+ 'v': '\x0B',
+ 'f': '\x0C',
+ 'r': '\x0D',
+ 'e': '\x1B',
+ ' ': '\x20',
+ '\"': '\"',
+ '\\': '\\',
+ 'N': '\x85',
+ '_': '\xA0',
+ 'L': '\u2028',
+ 'P': '\u2029',
+ }
+
+ ESCAPE_CODES = {
+ 'x': 2,
+ 'u': 4,
+ 'U': 8,
+ }
+
+ def scan_flow_scalar_non_spaces(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ while True:
+ length = 0
+ while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029':
+ length += 1
+ if length:
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ ch = self.peek()
+ if not double and ch == '\'' and self.peek(1) == '\'':
+ chunks.append('\'')
+ self.forward(2)
+ elif (double and ch == '\'') or (not double and ch in '\"\\'):
+ chunks.append(ch)
+ self.forward()
+ elif double and ch == '\\':
+ self.forward()
+ ch = self.peek()
+ if ch in self.ESCAPE_REPLACEMENTS:
+ chunks.append(self.ESCAPE_REPLACEMENTS[ch])
+ self.forward()
+ elif ch in self.ESCAPE_CODES:
+ length = self.ESCAPE_CODES[ch]
+ self.forward()
+ for k in range(length):
+ if self.peek(k) not in '0123456789ABCDEFabcdef':
+ raise ScannerError("while scanning a double-quoted scalar", start_mark,
+ "expected escape sequence of %d hexdecimal numbers, but found %r" %
+ (length, self.peek(k)), self.get_mark())
+ code = int(self.prefix(length), 16)
+ chunks.append(chr(code))
+ self.forward(length)
+ elif ch in '\r\n\x85\u2028\u2029':
+ self.scan_line_break()
+ chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
+ else:
+ raise ScannerError("while scanning a double-quoted scalar", start_mark,
+ "found unknown escape character %r" % ch, self.get_mark())
+ else:
+ return chunks
+
+ def scan_flow_scalar_spaces(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ length = 0
+ while self.peek(length) in ' \t':
+ length += 1
+ whitespaces = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch == '\0':
+ raise ScannerError("while scanning a quoted scalar", start_mark,
+ "found unexpected end of stream", self.get_mark())
+ elif ch in '\r\n\x85\u2028\u2029':
+ line_break = self.scan_line_break()
+ breaks = self.scan_flow_scalar_breaks(double, start_mark)
+ if line_break != '\n':
+ chunks.append(line_break)
+ elif not breaks:
+ chunks.append(' ')
+ chunks.extend(breaks)
+ else:
+ chunks.append(whitespaces)
+ return chunks
+
+ def scan_flow_scalar_breaks(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ while True:
+ # Instead of checking indentation, we check for document
+ # separators.
+ prefix = self.prefix(3)
+ if (prefix == '---' or prefix == '...') \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a quoted scalar", start_mark,
+ "found unexpected document separator", self.get_mark())
+ while self.peek() in ' \t':
+ self.forward()
+ if self.peek() in '\r\n\x85\u2028\u2029':
+ chunks.append(self.scan_line_break())
+ else:
+ return chunks
+
+ def scan_plain(self):
+ # See the specification for details.
+ # We add an additional restriction for the flow context:
+ # plain scalars in the flow context cannot contain ',', ':' and '?'.
+ # We also keep track of the `allow_simple_key` flag here.
+ # Indentation rules are loosed for the flow context.
+ chunks = []
+ start_mark = self.get_mark()
+ end_mark = start_mark
+ indent = self.indent+1
+ # We allow zero indentation for scalars, but then we need to check for
+ # document separators at the beginning of the line.
+ #if indent == 0:
+ # indent = 1
+ spaces = []
+ while True:
+ length = 0
+ if self.peek() == '#':
+ break
+ while True:
+ ch = self.peek(length)
+ if ch in '\0 \t\r\n\x85\u2028\u2029' \
+ or (not self.flow_level and ch == ':' and
+ self.peek(length+1) in '\0 \t\r\n\x85\u2028\u2029') \
+ or (self.flow_level and ch in ',:?[]{}'):
+ break
+ length += 1
+ # It's not clear what we should do with ':' in the flow context.
+ if (self.flow_level and ch == ':'
+ and self.peek(length+1) not in '\0 \t\r\n\x85\u2028\u2029,[]{}'):
+ self.forward(length)
+ raise ScannerError("while scanning a plain scalar", start_mark,
+ "found unexpected ':'", self.get_mark(),
+ "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
+ if length == 0:
+ break
+ self.allow_simple_key = False
+ chunks.extend(spaces)
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ end_mark = self.get_mark()
+ spaces = self.scan_plain_spaces(indent, start_mark)
+ if not spaces or self.peek() == '#' \
+ or (not self.flow_level and self.column < indent):
+ break
+ return ScalarToken(''.join(chunks), True, start_mark, end_mark)
+
+ def scan_plain_spaces(self, indent, start_mark):
+ # See the specification for details.
+ # The specification is really confusing about tabs in plain scalars.
+ # We just forbid them completely. Do not use tabs in YAML!
+ chunks = []
+ length = 0
+ while self.peek(length) in ' ':
+ length += 1
+ whitespaces = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch in '\r\n\x85\u2028\u2029':
+ line_break = self.scan_line_break()
+ self.allow_simple_key = True
+ prefix = self.prefix(3)
+ if (prefix == '---' or prefix == '...') \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ return
+ breaks = []
+ while self.peek() in ' \r\n\x85\u2028\u2029':
+ if self.peek() == ' ':
+ self.forward()
+ else:
+ breaks.append(self.scan_line_break())
+ prefix = self.prefix(3)
+ if (prefix == '---' or prefix == '...') \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ return
+ if line_break != '\n':
+ chunks.append(line_break)
+ elif not breaks:
+ chunks.append(' ')
+ chunks.extend(breaks)
+ elif whitespaces:
+ chunks.append(whitespaces)
+ return chunks
+
+ def scan_tag_handle(self, name, start_mark):
+ # See the specification for details.
+ # For some strange reasons, the specification does not allow '_' in
+ # tag handles. I have allowed it anyway.
+ ch = self.peek()
+ if ch != '!':
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected '!', but found %r" % ch, self.get_mark())
+ length = 1
+ ch = self.peek(length)
+ if ch != ' ':
+ while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_':
+ length += 1
+ ch = self.peek(length)
+ if ch != '!':
+ self.forward(length)
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected '!', but found %r" % ch, self.get_mark())
+ length += 1
+ value = self.prefix(length)
+ self.forward(length)
+ return value
+
+ def scan_tag_uri(self, name, start_mark):
+ # See the specification for details.
+ # Note: we do not check if URI is well-formed.
+ chunks = []
+ length = 0
+ ch = self.peek(length)
+ while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-;/?:@&=+$,_.!~*\'()[]%':
+ if ch == '%':
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ length = 0
+ chunks.append(self.scan_uri_escapes(name, start_mark))
+ else:
+ length += 1
+ ch = self.peek(length)
+ if length:
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ length = 0
+ if not chunks:
+ raise ScannerError("while parsing a %s" % name, start_mark,
+ "expected URI, but found %r" % ch, self.get_mark())
+ return ''.join(chunks)
+
+ def scan_uri_escapes(self, name, start_mark):
+ # See the specification for details.
+ codes = []
+ mark = self.get_mark()
+ while self.peek() == '%':
+ self.forward()
+ for k in range(2):
+ if self.peek(k) not in '0123456789ABCDEFabcdef':
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected URI escape sequence of 2 hexdecimal numbers, but found %r"
+ % self.peek(k), self.get_mark())
+ codes.append(int(self.prefix(2), 16))
+ self.forward(2)
+ try:
+ value = bytes(codes).decode('utf-8')
+ except UnicodeDecodeError as exc:
+ raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
+ return value
+
+ def scan_line_break(self):
+ # Transforms:
+ # '\r\n' : '\n'
+ # '\r' : '\n'
+ # '\n' : '\n'
+ # '\x85' : '\n'
+ # '\u2028' : '\u2028'
+ # '\u2029 : '\u2029'
+ # default : ''
+ ch = self.peek()
+ if ch in '\r\n\x85':
+ if self.prefix(2) == '\r\n':
+ self.forward(2)
+ else:
+ self.forward()
+ return '\n'
+ elif ch in '\u2028\u2029':
+ self.forward()
+ return ch
+ return ''
+
+#try:
+# import psyco
+# psyco.bind(Scanner)
+#except ImportError:
+# pass
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/serializer.py b/collectors/python.d.plugin/python_modules/pyyaml3/serializer.py
new file mode 100644
index 0000000..1ba2f7f
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/serializer.py
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: MIT
+
+__all__ = ['Serializer', 'SerializerError']
+
+from .error import YAMLError
+from .events import *
+from .nodes import *
+
+class SerializerError(YAMLError):
+ pass
+
+class Serializer:
+
+ ANCHOR_TEMPLATE = 'id%03d'
+
+ def __init__(self, encoding=None,
+ explicit_start=None, explicit_end=None, version=None, tags=None):
+ self.use_encoding = encoding
+ self.use_explicit_start = explicit_start
+ self.use_explicit_end = explicit_end
+ self.use_version = version
+ self.use_tags = tags
+ self.serialized_nodes = {}
+ self.anchors = {}
+ self.last_anchor_id = 0
+ self.closed = None
+
+ def open(self):
+ if self.closed is None:
+ self.emit(StreamStartEvent(encoding=self.use_encoding))
+ self.closed = False
+ elif self.closed:
+ raise SerializerError("serializer is closed")
+ else:
+ raise SerializerError("serializer is already opened")
+
+ def close(self):
+ if self.closed is None:
+ raise SerializerError("serializer is not opened")
+ elif not self.closed:
+ self.emit(StreamEndEvent())
+ self.closed = True
+
+ #def __del__(self):
+ # self.close()
+
+ def serialize(self, node):
+ if self.closed is None:
+ raise SerializerError("serializer is not opened")
+ elif self.closed:
+ raise SerializerError("serializer is closed")
+ self.emit(DocumentStartEvent(explicit=self.use_explicit_start,
+ version=self.use_version, tags=self.use_tags))
+ self.anchor_node(node)
+ self.serialize_node(node, None, None)
+ self.emit(DocumentEndEvent(explicit=self.use_explicit_end))
+ self.serialized_nodes = {}
+ self.anchors = {}
+ self.last_anchor_id = 0
+
+ def anchor_node(self, node):
+ if node in self.anchors:
+ if self.anchors[node] is None:
+ self.anchors[node] = self.generate_anchor(node)
+ else:
+ self.anchors[node] = None
+ if isinstance(node, SequenceNode):
+ for item in node.value:
+ self.anchor_node(item)
+ elif isinstance(node, MappingNode):
+ for key, value in node.value:
+ self.anchor_node(key)
+ self.anchor_node(value)
+
+ def generate_anchor(self, node):
+ self.last_anchor_id += 1
+ return self.ANCHOR_TEMPLATE % self.last_anchor_id
+
+ def serialize_node(self, node, parent, index):
+ alias = self.anchors[node]
+ if node in self.serialized_nodes:
+ self.emit(AliasEvent(alias))
+ else:
+ self.serialized_nodes[node] = True
+ self.descend_resolver(parent, index)
+ if isinstance(node, ScalarNode):
+ detected_tag = self.resolve(ScalarNode, node.value, (True, False))
+ default_tag = self.resolve(ScalarNode, node.value, (False, True))
+ implicit = (node.tag == detected_tag), (node.tag == default_tag)
+ self.emit(ScalarEvent(alias, node.tag, implicit, node.value,
+ style=node.style))
+ elif isinstance(node, SequenceNode):
+ implicit = (node.tag
+ == self.resolve(SequenceNode, node.value, True))
+ self.emit(SequenceStartEvent(alias, node.tag, implicit,
+ flow_style=node.flow_style))
+ index = 0
+ for item in node.value:
+ self.serialize_node(item, node, index)
+ index += 1
+ self.emit(SequenceEndEvent())
+ elif isinstance(node, MappingNode):
+ implicit = (node.tag
+ == self.resolve(MappingNode, node.value, True))
+ self.emit(MappingStartEvent(alias, node.tag, implicit,
+ flow_style=node.flow_style))
+ for key, value in node.value:
+ self.serialize_node(key, node, None)
+ self.serialize_node(value, node, key)
+ self.emit(MappingEndEvent())
+ self.ascend_resolver()
+
diff --git a/collectors/python.d.plugin/python_modules/pyyaml3/tokens.py b/collectors/python.d.plugin/python_modules/pyyaml3/tokens.py
new file mode 100644
index 0000000..c5c4fb1
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/pyyaml3/tokens.py
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: MIT
+
+class Token(object):
+ def __init__(self, start_mark, end_mark):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ attributes = [key for key in self.__dict__
+ if not key.endswith('_mark')]
+ attributes.sort()
+ arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+ for key in attributes])
+ return '%s(%s)' % (self.__class__.__name__, arguments)
+
+#class BOMToken(Token):
+# id = '<byte order mark>'
+
+class DirectiveToken(Token):
+ id = '<directive>'
+ def __init__(self, name, value, start_mark, end_mark):
+ self.name = name
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class DocumentStartToken(Token):
+ id = '<document start>'
+
+class DocumentEndToken(Token):
+ id = '<document end>'
+
+class StreamStartToken(Token):
+ id = '<stream start>'
+ def __init__(self, start_mark=None, end_mark=None,
+ encoding=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.encoding = encoding
+
+class StreamEndToken(Token):
+ id = '<stream end>'
+
+class BlockSequenceStartToken(Token):
+ id = '<block sequence start>'
+
+class BlockMappingStartToken(Token):
+ id = '<block mapping start>'
+
+class BlockEndToken(Token):
+ id = '<block end>'
+
+class FlowSequenceStartToken(Token):
+ id = '['
+
+class FlowMappingStartToken(Token):
+ id = '{'
+
+class FlowSequenceEndToken(Token):
+ id = ']'
+
+class FlowMappingEndToken(Token):
+ id = '}'
+
+class KeyToken(Token):
+ id = '?'
+
+class ValueToken(Token):
+ id = ':'
+
+class BlockEntryToken(Token):
+ id = '-'
+
+class FlowEntryToken(Token):
+ id = ','
+
+class AliasToken(Token):
+ id = '<alias>'
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class AnchorToken(Token):
+ id = '<anchor>'
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class TagToken(Token):
+ id = '<tag>'
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class ScalarToken(Token):
+ id = '<scalar>'
+ def __init__(self, value, plain, start_mark, end_mark, style=None):
+ self.value = value
+ self.plain = plain
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
diff --git a/collectors/python.d.plugin/python_modules/third_party/__init__.py b/collectors/python.d.plugin/python_modules/third_party/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/third_party/__init__.py
diff --git a/collectors/python.d.plugin/python_modules/third_party/boinc_client.py b/collectors/python.d.plugin/python_modules/third_party/boinc_client.py
new file mode 100644
index 0000000..ec21779
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/third_party/boinc_client.py
@@ -0,0 +1,515 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# client.py - Somewhat higher-level GUI_RPC API for BOINC core client
+#
+# Copyright (C) 2013 Rodrigo Silva (MestreLion) <linux@rodrigosilva.com>
+# Copyright (C) 2017 Austin S. Hemmelgarn
+#
+# SPDX-License-Identifier: GPL-3.0
+
+# Based on client/boinc_cmd.cpp
+
+import hashlib
+import socket
+import sys
+import time
+from functools import total_ordering
+from xml.etree import ElementTree
+
+GUI_RPC_PASSWD_FILE = "/var/lib/boinc/gui_rpc_auth.cfg"
+
+GUI_RPC_HOSTNAME = None # localhost
+GUI_RPC_PORT = 31416
+GUI_RPC_TIMEOUT = 1
+
+class Rpc(object):
+ ''' Class to perform GUI RPC calls to a BOINC core client.
+ Usage in a context manager ('with' block) is recommended to ensure
+ disconnect() is called. Using the same instance for all calls is also
+ recommended so it reuses the same socket connection
+ '''
+ def __init__(self, hostname="", port=0, timeout=0, text_output=False):
+ self.hostname = hostname
+ self.port = port
+ self.timeout = timeout
+ self.sock = None
+ self.text_output = text_output
+
+ @property
+ def sockargs(self):
+ return (self.hostname, self.port, self.timeout)
+
+ def __enter__(self): self.connect(*self.sockargs); return self
+ def __exit__(self, *args): self.disconnect()
+
+ def connect(self, hostname="", port=0, timeout=0):
+ ''' Connect to (hostname, port) with timeout in seconds.
+ Hostname defaults to None (localhost), and port to 31416
+ Calling multiple times will disconnect previous connection (if any),
+ and (re-)connect to host.
+ '''
+ if self.sock:
+ self.disconnect()
+
+ self.hostname = hostname or GUI_RPC_HOSTNAME
+ self.port = port or GUI_RPC_PORT
+ self.timeout = timeout or GUI_RPC_TIMEOUT
+
+ self.sock = socket.create_connection(self.sockargs[0:2], self.sockargs[2])
+
+ def disconnect(self):
+ ''' Disconnect from host. Calling multiple times is OK (idempotent)
+ '''
+ if self.sock:
+ self.sock.close()
+ self.sock = None
+
+ def call(self, request, text_output=None):
+ ''' Do an RPC call. Pack and send the XML request and return the
+ unpacked reply. request can be either plain XML text or a
+ xml.etree.ElementTree.Element object. Return ElementTree.Element
+ or XML text according to text_output flag.
+ Will auto-connect if not connected.
+ '''
+ if text_output is None:
+ text_output = self.text_output
+
+ if not self.sock:
+ self.connect(*self.sockargs)
+
+ if not isinstance(request, ElementTree.Element):
+ request = ElementTree.fromstring(request)
+
+ # pack request
+ end = '\003'
+ if sys.version_info[0] < 3:
+ req = "<boinc_gui_rpc_request>\n{0}\n</boinc_gui_rpc_request>\n{1}".format(ElementTree.tostring(request).replace(' />', '/>'), end)
+ else:
+ req = "<boinc_gui_rpc_request>\n{0}\n</boinc_gui_rpc_request>\n{1}".format(ElementTree.tostring(request, encoding='unicode').replace(' />', '/>'), end).encode()
+
+ try:
+ self.sock.sendall(req)
+ except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+ raise
+
+ req = ""
+ while True:
+ try:
+ buf = self.sock.recv(8192)
+ if not buf:
+ raise socket.error("No data from socket")
+ if sys.version_info[0] >= 3:
+ buf = buf.decode()
+ except socket.error:
+ raise
+ n = buf.find(end)
+ if not n == -1: break
+ req += buf
+ req += buf[:n]
+
+ # unpack reply (remove root tag, ie: first and last lines)
+ req = '\n'.join(req.strip().rsplit('\n')[1:-1])
+
+ if text_output:
+ return req
+ else:
+ return ElementTree.fromstring(req)
+
+def setattrs_from_xml(obj, xml, attrfuncdict={}):
+ ''' Helper to set values for attributes of a class instance by mapping
+ matching tags from a XML file.
+ attrfuncdict is a dict of functions to customize value data type of
+ each attribute. It falls back to simple int/float/bool/str detection
+ based on values defined in __init__(). This would not be needed if
+ Boinc used standard RPC protocol, which includes data type in XML.
+ '''
+ if not isinstance(xml, ElementTree.Element):
+ xml = ElementTree.fromstring(xml)
+ for e in list(xml):
+ if hasattr(obj, e.tag):
+ attr = getattr(obj, e.tag)
+ attrfunc = attrfuncdict.get(e.tag, None)
+ if attrfunc is None:
+ if isinstance(attr, bool): attrfunc = parse_bool
+ elif isinstance(attr, int): attrfunc = parse_int
+ elif isinstance(attr, float): attrfunc = parse_float
+ elif isinstance(attr, str): attrfunc = parse_str
+ elif isinstance(attr, list): attrfunc = parse_list
+ else: attrfunc = lambda x: x
+ setattr(obj, e.tag, attrfunc(e))
+ else:
+ pass
+ #print "class missing attribute '%s': %r" % (e.tag, obj)
+ return obj
+
+
+def parse_bool(e):
+ ''' Helper to convert ElementTree.Element.text to boolean.
+ Treat '<foo/>' (and '<foo>[[:blank:]]</foo>') as True
+ Treat '0' and 'false' as False
+ '''
+ if e.text is None:
+ return True
+ else:
+ return bool(e.text) and not e.text.strip().lower() in ('0', 'false')
+
+
+def parse_int(e):
+ ''' Helper to convert ElementTree.Element.text to integer.
+ Treat '<foo/>' (and '<foo></foo>') as 0
+ '''
+ # int(float()) allows casting to int a value expressed as float in XML
+ return 0 if e.text is None else int(float(e.text.strip()))
+
+
+def parse_float(e):
+ ''' Helper to convert ElementTree.Element.text to float. '''
+ return 0.0 if e.text is None else float(e.text.strip())
+
+
+def parse_str(e):
+ ''' Helper to convert ElementTree.Element.text to string. '''
+ return "" if e.text is None else e.text.strip()
+
+
+def parse_list(e):
+ ''' Helper to convert ElementTree.Element to list. For now, simply return
+ the list of root element's children
+ '''
+ return list(e)
+
+
+class Enum(object):
+ UNKNOWN = -1 # Not in original API
+
+ @classmethod
+ def name(cls, value):
+ ''' Quick-and-dirty fallback for getting the "name" of an enum item '''
+
+ # value as string, if it matches an enum attribute.
+ # Allows short usage as Enum.name("VALUE") besides Enum.name(Enum.VALUE)
+ if hasattr(cls, str(value)):
+ return cls.name(getattr(cls, value, None))
+
+ # value not handled in subclass name()
+ for k, v in cls.__dict__.items():
+ if v == value:
+ return k.lower().replace('_', ' ')
+
+ # value not found
+ return cls.name(Enum.UNKNOWN)
+
+
+class CpuSched(Enum):
+ ''' values of ACTIVE_TASK::scheduler_state and ACTIVE_TASK::next_scheduler_state
+ "SCHEDULED" is synonymous with "executing" except when CPU throttling
+ is in use.
+ '''
+ UNINITIALIZED = 0
+ PREEMPTED = 1
+ SCHEDULED = 2
+
+
+class ResultState(Enum):
+ ''' Values of RESULT::state in client.
+ THESE MUST BE IN NUMERICAL ORDER
+ (because of the > comparison in RESULT::computing_done())
+ see html/inc/common_defs.inc
+ '''
+ NEW = 0
+ #// New result
+ FILES_DOWNLOADING = 1
+ #// Input files for result (WU, app version) are being downloaded
+ FILES_DOWNLOADED = 2
+ #// Files are downloaded, result can be (or is being) computed
+ COMPUTE_ERROR = 3
+ #// computation failed; no file upload
+ FILES_UPLOADING = 4
+ #// Output files for result are being uploaded
+ FILES_UPLOADED = 5
+ #// Files are uploaded, notify scheduling server at some point
+ ABORTED = 6
+ #// result was aborted
+ UPLOAD_FAILED = 7
+ #// some output file permanent failure
+
+
+class Process(Enum):
+ ''' values of ACTIVE_TASK::task_state '''
+ UNINITIALIZED = 0
+ #// process doesn't exist yet
+ EXECUTING = 1
+ #// process is running, as far as we know
+ SUSPENDED = 9
+ #// we've sent it a "suspend" message
+ ABORT_PENDING = 5
+ #// process exceeded limits; send "abort" message, waiting to exit
+ QUIT_PENDING = 8
+ #// we've sent it a "quit" message, waiting to exit
+ COPY_PENDING = 10
+ #// waiting for async file copies to finish
+
+
+class _Struct(object):
+ ''' base helper class with common methods for all classes derived from
+ BOINC's C++ structs
+ '''
+ @classmethod
+ def parse(cls, xml):
+ return setattrs_from_xml(cls(), xml)
+
+ def __str__(self, indent=0):
+ buf = '{0}{1}:\n'.format('\t' * indent, self.__class__.__name__)
+ for attr in self.__dict__:
+ value = getattr(self, attr)
+ if isinstance(value, list):
+ buf += '{0}\t{1} [\n'.format('\t' * indent, attr)
+ for v in value: buf += '\t\t{0}\t\t,\n'.format(v)
+ buf += '\t]\n'
+ else:
+ buf += '{0}\t{1}\t{2}\n'.format('\t' * indent,
+ attr,
+ value.__str__(indent+2)
+ if isinstance(value, _Struct)
+ else repr(value))
+ return buf
+
+
+@total_ordering
+class VersionInfo(_Struct):
+ def __init__(self, major=0, minor=0, release=0):
+ self.major = major
+ self.minor = minor
+ self.release = release
+
+ @property
+ def _tuple(self):
+ return (self.major, self.minor, self.release)
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self._tuple == other._tuple
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __gt__(self, other):
+ if not isinstance(other, self.__class__):
+ return NotImplemented
+ return self._tuple > other._tuple
+
+ def __str__(self):
+ return "{0}.{1}.{2}".format(self.major, self.minor, self.release)
+
+ def __repr__(self):
+ return "{0}{1}".format(self.__class__.__name__, self._tuple)
+
+
+class Result(_Struct):
+ ''' Also called "task" in some contexts '''
+ def __init__(self):
+ # Names and values follow lib/gui_rpc_client.h @ RESULT
+ # Order too, except when grouping contradicts client/result.cpp
+ # RESULT::write_gui(), then XML order is used.
+
+ self.name = ""
+ self.wu_name = ""
+ self.version_num = 0
+ #// identifies the app used
+ self.plan_class = ""
+ self.project_url = "" # from PROJECT.master_url
+ self.report_deadline = 0.0 # seconds since epoch
+ self.received_time = 0.0 # seconds since epoch
+ #// when we got this from server
+ self.ready_to_report = False
+ #// we're ready to report this result to the server;
+ #// either computation is done and all the files have been uploaded
+ #// or there was an error
+ self.got_server_ack = False
+ #// we've received the ack for this result from the server
+ self.final_cpu_time = 0.0
+ self.final_elapsed_time = 0.0
+ self.state = ResultState.NEW
+ self.estimated_cpu_time_remaining = 0.0
+ #// actually, estimated elapsed time remaining
+ self.exit_status = 0
+ #// return value from the application
+ self.suspended_via_gui = False
+ self.project_suspended_via_gui = False
+ self.edf_scheduled = False
+ #// temporary used to tell GUI that this result is deadline-scheduled
+ self.coproc_missing = False
+ #// a coproc needed by this job is missing
+ #// (e.g. because user removed their GPU board).
+ self.scheduler_wait = False
+ self.scheduler_wait_reason = ""
+ self.network_wait = False
+ self.resources = ""
+ #// textual description of resources used
+
+ #// the following defined if active
+ # XML is generated in client/app.cpp ACTIVE_TASK::write_gui()
+ self.active_task = False
+ self.active_task_state = Process.UNINITIALIZED
+ self.app_version_num = 0
+ self.slot = -1
+ self.pid = 0
+ self.scheduler_state = CpuSched.UNINITIALIZED
+ self.checkpoint_cpu_time = 0.0
+ self.current_cpu_time = 0.0
+ self.fraction_done = 0.0
+ self.elapsed_time = 0.0
+ self.swap_size = 0
+ self.working_set_size_smoothed = 0.0
+ self.too_large = False
+ self.needs_shmem = False
+ self.graphics_exec_path = ""
+ self.web_graphics_url = ""
+ self.remote_desktop_addr = ""
+ self.slot_path = ""
+ #// only present if graphics_exec_path is
+
+ # The following are not in original API, but are present in RPC XML reply
+ self.completed_time = 0.0
+ #// time when ready_to_report was set
+ self.report_immediately = False
+ self.working_set_size = 0
+ self.page_fault_rate = 0.0
+ #// derived by higher-level code
+
+ # The following are in API, but are NEVER in RPC XML reply. Go figure
+ self.signal = 0
+
+ self.app = None # APP*
+ self.wup = None # WORKUNIT*
+ self.project = None # PROJECT*
+ self.avp = None # APP_VERSION*
+
+ @classmethod
+ def parse(cls, xml):
+ if not isinstance(xml, ElementTree.Element):
+ xml = ElementTree.fromstring(xml)
+
+ # parse main XML
+ result = super(Result, cls).parse(xml)
+
+ # parse '<active_task>' children
+ active_task = xml.find('active_task')
+ if active_task is None:
+ result.active_task = False # already the default after __init__()
+ else:
+ result.active_task = True # already the default after main parse
+ result = setattrs_from_xml(result, active_task)
+
+ #// if CPU time is nonzero but elapsed time is zero,
+ #// we must be talking to an old client.
+ #// Set elapsed = CPU
+ #// (easier to deal with this here than in the manager)
+ if result.current_cpu_time != 0 and result.elapsed_time == 0:
+ result.elapsed_time = result.current_cpu_time
+
+ if result.final_cpu_time != 0 and result.final_elapsed_time == 0:
+ result.final_elapsed_time = result.final_cpu_time
+
+ return result
+
+ def __str__(self):
+ buf = '{0}:\n'.format(self.__class__.__name__)
+ for attr in self.__dict__:
+ value = getattr(self, attr)
+ if attr in ['received_time', 'report_deadline']:
+ value = time.ctime(value)
+ buf += '\t{0}\t{1}\n'.format(attr, value)
+ return buf
+
+
+class BoincClient(object):
+
+ def __init__(self, host="", port=0, passwd=None):
+ self.hostname = host
+ self.port = port
+ self.passwd = passwd
+ self.rpc = Rpc(text_output=False)
+ self.version = None
+ self.authorized = False
+
+ # Informative, not authoritative. Records status of *last* RPC call,
+ # but does not infer success about the *next* one.
+ # Thus, it should be read *after* an RPC call, not prior to one
+ self.connected = False
+
+ def __enter__(self): self.connect(); return self
+ def __exit__(self, *args): self.disconnect()
+
+ def connect(self):
+ try:
+ self.rpc.connect(self.hostname, self.port)
+ self.connected = True
+ except socket.error:
+ self.connected = False
+ return
+ self.authorized = self.authorize(self.passwd)
+ self.version = self.exchange_versions()
+
+ def disconnect(self):
+ self.rpc.disconnect()
+
+ def authorize(self, password):
+ ''' Request authorization. If password is None and we are connecting
+ to localhost, try to read password from the local config file
+ GUI_RPC_PASSWD_FILE. If file can't be read (not found or no
+ permission to read), try to authorize with a blank password.
+ If authorization is requested and fails, all subsequent calls
+ will be refused with socket.error 'Connection reset by peer' (104).
+ Since most local calls do no require authorization, do not attempt
+ it if you're not sure about the password.
+ '''
+ if password is None and not self.hostname:
+ password = read_gui_rpc_password() or ""
+ nonce = self.rpc.call('<auth1/>').text
+ authhash = hashlib.md5('{0}{1}'.format(nonce, password).encode()).hexdigest().lower()
+ reply = self.rpc.call('<auth2><nonce_hash>{0}</nonce_hash></auth2>'.format(authhash))
+
+ if reply.tag == 'authorized':
+ return True
+ else:
+ return False
+
+ def exchange_versions(self):
+ ''' Return VersionInfo instance with core client version info '''
+ return VersionInfo.parse(self.rpc.call('<exchange_versions/>'))
+
+ def get_tasks(self):
+ ''' Same as get_results(active_only=False) '''
+ return self.get_results(False)
+
+ def get_results(self, active_only=False):
+ ''' Get a list of results.
+ Those that are in progress will have information such as CPU time
+ and fraction done. Each result includes a name;
+ Use CC_STATE::lookup_result() to find this result in the current static state;
+ if it's not there, call get_state() again.
+ '''
+ reply = self.rpc.call("<get_results><active_only>{0}</active_only></get_results>".format(1 if active_only else 0))
+ if not reply.tag == 'results':
+ return []
+
+ results = []
+ for item in list(reply):
+ results.append(Result.parse(item))
+
+ return results
+
+
+def read_gui_rpc_password():
+ ''' Read password string from GUI_RPC_PASSWD_FILE file, trim the last CR
+ (if any), and return it
+ '''
+ try:
+ with open(GUI_RPC_PASSWD_FILE, 'r') as f:
+ buf = f.read()
+ if buf.endswith('\n'): return buf[:-1] # trim last CR
+ else: return buf
+ except IOError:
+ # Permission denied or File not found.
+ pass
diff --git a/collectors/python.d.plugin/python_modules/third_party/filelock.py b/collectors/python.d.plugin/python_modules/third_party/filelock.py
new file mode 100644
index 0000000..4c98167
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/third_party/filelock.py
@@ -0,0 +1,451 @@
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# For more information, please refer to <http://unlicense.org>
+
+"""
+A platform independent file lock that supports the with-statement.
+"""
+
+
+# Modules
+# ------------------------------------------------
+import logging
+import os
+import threading
+import time
+try:
+ import warnings
+except ImportError:
+ warnings = None
+
+try:
+ import msvcrt
+except ImportError:
+ msvcrt = None
+
+try:
+ import fcntl
+except ImportError:
+ fcntl = None
+
+
+# Backward compatibility
+# ------------------------------------------------
+try:
+ TimeoutError
+except NameError:
+ TimeoutError = OSError
+
+
+# Data
+# ------------------------------------------------
+__all__ = [
+ "Timeout",
+ "BaseFileLock",
+ "WindowsFileLock",
+ "UnixFileLock",
+ "SoftFileLock",
+ "FileLock"
+]
+
+__version__ = "3.0.12"
+
+
+_logger = None
+def logger():
+ """Returns the logger instance used in this module."""
+ global _logger
+ _logger = _logger or logging.getLogger(__name__)
+ return _logger
+
+
+# Exceptions
+# ------------------------------------------------
+class Timeout(TimeoutError):
+ """
+ Raised when the lock could not be acquired in *timeout*
+ seconds.
+ """
+
+ def __init__(self, lock_file):
+ """
+ """
+ #: The path of the file lock.
+ self.lock_file = lock_file
+ return None
+
+ def __str__(self):
+ temp = "The file lock '{}' could not be acquired."\
+ .format(self.lock_file)
+ return temp
+
+
+# Classes
+# ------------------------------------------------
+
+# This is a helper class which is returned by :meth:`BaseFileLock.acquire`
+# and wraps the lock to make sure __enter__ is not called twice when entering
+# the with statement.
+# If we would simply return *self*, the lock would be acquired again
+# in the *__enter__* method of the BaseFileLock, but not released again
+# automatically.
+#
+# :seealso: issue #37 (memory leak)
+class _Acquire_ReturnProxy(object):
+
+ def __init__(self, lock):
+ self.lock = lock
+ return None
+
+ def __enter__(self):
+ return self.lock
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.lock.release()
+ return None
+
+
+class BaseFileLock(object):
+ """
+ Implements the base class of a file lock.
+ """
+
+ def __init__(self, lock_file, timeout = -1):
+ """
+ """
+ # The path to the lock file.
+ self._lock_file = lock_file
+
+ # The file descriptor for the *_lock_file* as it is returned by the
+ # os.open() function.
+ # This file lock is only NOT None, if the object currently holds the
+ # lock.
+ self._lock_file_fd = None
+
+ # The default timeout value.
+ self.timeout = timeout
+
+ # We use this lock primarily for the lock counter.
+ self._thread_lock = threading.Lock()
+
+ # The lock counter is used for implementing the nested locking
+ # mechanism. Whenever the lock is acquired, the counter is increased and
+ # the lock is only released, when this value is 0 again.
+ self._lock_counter = 0
+ return None
+
+ @property
+ def lock_file(self):
+ """
+ The path to the lock file.
+ """
+ return self._lock_file
+
+ @property
+ def timeout(self):
+ """
+ You can set a default timeout for the filelock. It will be used as
+ fallback value in the acquire method, if no timeout value (*None*) is
+ given.
+
+ If you want to disable the timeout, set it to a negative value.
+
+ A timeout of 0 means, that there is exactly one attempt to acquire the
+ file lock.
+
+ .. versionadded:: 2.0.0
+ """
+ return self._timeout
+
+ @timeout.setter
+ def timeout(self, value):
+ """
+ """
+ self._timeout = float(value)
+ return None
+
+ # Platform dependent locking
+ # --------------------------------------------
+
+ def _acquire(self):
+ """
+ Platform dependent. If the file lock could be
+ acquired, self._lock_file_fd holds the file descriptor
+ of the lock file.
+ """
+ raise NotImplementedError()
+
+ def _release(self):
+ """
+ Releases the lock and sets self._lock_file_fd to None.
+ """
+ raise NotImplementedError()
+
+ # Platform independent methods
+ # --------------------------------------------
+
+ @property
+ def is_locked(self):
+ """
+ True, if the object holds the file lock.
+
+ .. versionchanged:: 2.0.0
+
+ This was previously a method and is now a property.
+ """
+ return self._lock_file_fd is not None
+
+ def acquire(self, timeout=None, poll_intervall=0.05):
+ """
+ Acquires the file lock or fails with a :exc:`Timeout` error.
+
+ .. code-block:: python
+
+ # You can use this method in the context manager (recommended)
+ with lock.acquire():
+ pass
+
+ # Or use an equivalent try-finally construct:
+ lock.acquire()
+ try:
+ pass
+ finally:
+ lock.release()
+
+ :arg float timeout:
+ The maximum time waited for the file lock.
+ If ``timeout < 0``, there is no timeout and this method will
+ block until the lock could be acquired.
+ If ``timeout`` is None, the default :attr:`~timeout` is used.
+
+ :arg float poll_intervall:
+ We check once in *poll_intervall* seconds if we can acquire the
+ file lock.
+
+ :raises Timeout:
+ if the lock could not be acquired in *timeout* seconds.
+
+ .. versionchanged:: 2.0.0
+
+ This method returns now a *proxy* object instead of *self*,
+ so that it can be used in a with statement without side effects.
+ """
+ # Use the default timeout, if no timeout is provided.
+ if timeout is None:
+ timeout = self.timeout
+
+ # Increment the number right at the beginning.
+ # We can still undo it, if something fails.
+ with self._thread_lock:
+ self._lock_counter += 1
+
+ lock_id = id(self)
+ lock_filename = self._lock_file
+ start_time = time.time()
+ try:
+ while True:
+ with self._thread_lock:
+ if not self.is_locked:
+ logger().debug('Attempting to acquire lock %s on %s', lock_id, lock_filename)
+ self._acquire()
+
+ if self.is_locked:
+ logger().info('Lock %s acquired on %s', lock_id, lock_filename)
+ break
+ elif timeout >= 0 and time.time() - start_time > timeout:
+ logger().debug('Timeout on acquiring lock %s on %s', lock_id, lock_filename)
+ raise Timeout(self._lock_file)
+ else:
+ logger().debug(
+ 'Lock %s not acquired on %s, waiting %s seconds ...',
+ lock_id, lock_filename, poll_intervall
+ )
+ time.sleep(poll_intervall)
+ except:
+ # Something did go wrong, so decrement the counter.
+ with self._thread_lock:
+ self._lock_counter = max(0, self._lock_counter - 1)
+
+ raise
+ return _Acquire_ReturnProxy(lock = self)
+
+ def release(self, force = False):
+ """
+ Releases the file lock.
+
+ Please note, that the lock is only completly released, if the lock
+ counter is 0.
+
+ Also note, that the lock file itself is not automatically deleted.
+
+ :arg bool force:
+ If true, the lock counter is ignored and the lock is released in
+ every case.
+ """
+ with self._thread_lock:
+
+ if self.is_locked:
+ self._lock_counter -= 1
+
+ if self._lock_counter == 0 or force:
+ lock_id = id(self)
+ lock_filename = self._lock_file
+
+ logger().debug('Attempting to release lock %s on %s', lock_id, lock_filename)
+ self._release()
+ self._lock_counter = 0
+ logger().info('Lock %s released on %s', lock_id, lock_filename)
+
+ return None
+
+ def __enter__(self):
+ self.acquire()
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.release()
+ return None
+
+ def __del__(self):
+ self.release(force = True)
+ return None
+
+
+# Windows locking mechanism
+# ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+class WindowsFileLock(BaseFileLock):
+ """
+ Uses the :func:`msvcrt.locking` function to hard lock the lock file on
+ windows systems.
+ """
+
+ def _acquire(self):
+ open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC
+
+ try:
+ fd = os.open(self._lock_file, open_mode)
+ except OSError:
+ pass
+ else:
+ try:
+ msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
+ except (IOError, OSError):
+ os.close(fd)
+ else:
+ self._lock_file_fd = fd
+ return None
+
+ def _release(self):
+ fd = self._lock_file_fd
+ self._lock_file_fd = None
+ msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
+ os.close(fd)
+
+ try:
+ os.remove(self._lock_file)
+ # Probably another instance of the application
+ # that acquired the file lock.
+ except OSError:
+ pass
+ return None
+
+# Unix locking mechanism
+# ~~~~~~~~~~~~~~~~~~~~~~
+
+class UnixFileLock(BaseFileLock):
+ """
+ Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.
+ """
+
+ def _acquire(self):
+ open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC
+ fd = os.open(self._lock_file, open_mode)
+
+ try:
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except (IOError, OSError):
+ os.close(fd)
+ else:
+ self._lock_file_fd = fd
+ return None
+
+ def _release(self):
+ # Do not remove the lockfile:
+ #
+ # https://github.com/benediktschmitt/py-filelock/issues/31
+ # https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
+ fd = self._lock_file_fd
+ self._lock_file_fd = None
+ fcntl.flock(fd, fcntl.LOCK_UN)
+ os.close(fd)
+ return None
+
+# Soft lock
+# ~~~~~~~~~
+
+class SoftFileLock(BaseFileLock):
+ """
+ Simply watches the existence of the lock file.
+ """
+
+ def _acquire(self):
+ open_mode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
+ try:
+ fd = os.open(self._lock_file, open_mode)
+ except (IOError, OSError):
+ pass
+ else:
+ self._lock_file_fd = fd
+ return None
+
+ def _release(self):
+ os.close(self._lock_file_fd)
+ self._lock_file_fd = None
+
+ try:
+ os.remove(self._lock_file)
+ # The file is already deleted and that's what we want.
+ except OSError:
+ pass
+ return None
+
+
+# Platform filelock
+# ~~~~~~~~~~~~~~~~~
+
+#: Alias for the lock, which should be used for the current platform. On
+#: Windows, this is an alias for :class:`WindowsFileLock`, on Unix for
+#: :class:`UnixFileLock` and otherwise for :class:`SoftFileLock`.
+FileLock = None
+
+if msvcrt:
+ FileLock = WindowsFileLock
+elif fcntl:
+ FileLock = UnixFileLock
+else:
+ FileLock = SoftFileLock
+
+ if warnings is not None:
+ warnings.warn("only soft file lock is available")
diff --git a/collectors/python.d.plugin/python_modules/third_party/lm_sensors.py b/collectors/python.d.plugin/python_modules/third_party/lm_sensors.py
new file mode 100644
index 0000000..f873eac
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/third_party/lm_sensors.py
@@ -0,0 +1,327 @@
+# SPDX-License-Identifier: LGPL-2.1
+"""
+@package sensors.py
+Python Bindings for libsensors3
+
+use the documentation of libsensors for the low level API.
+see example.py for high level API usage.
+
+@author: Pavel Rojtberg (http://www.rojtberg.net)
+@see: https://github.com/paroj/sensors.py
+@copyright: LGPLv2 (same as libsensors) <http://opensource.org/licenses/LGPL-2.1>
+"""
+
+from ctypes import *
+import ctypes.util
+
+_libc = cdll.LoadLibrary(ctypes.util.find_library("c"))
+# see https://github.com/paroj/sensors.py/issues/1
+_libc.free.argtypes = [c_void_p]
+
+_hdl = cdll.LoadLibrary(ctypes.util.find_library("sensors"))
+
+version = c_char_p.in_dll(_hdl, "libsensors_version").value.decode("ascii")
+
+
+class SensorsError(Exception):
+ pass
+
+
+class ErrorWildcards(SensorsError):
+ pass
+
+
+class ErrorNoEntry(SensorsError):
+ pass
+
+
+class ErrorAccessRead(SensorsError, OSError):
+ pass
+
+
+class ErrorKernel(SensorsError, OSError):
+ pass
+
+
+class ErrorDivZero(SensorsError, ZeroDivisionError):
+ pass
+
+
+class ErrorChipName(SensorsError):
+ pass
+
+
+class ErrorBusName(SensorsError):
+ pass
+
+
+class ErrorParse(SensorsError):
+ pass
+
+
+class ErrorAccessWrite(SensorsError, OSError):
+ pass
+
+
+class ErrorIO(SensorsError, IOError):
+ pass
+
+
+class ErrorRecursion(SensorsError):
+ pass
+
+
+_ERR_MAP = {
+ 1: ErrorWildcards,
+ 2: ErrorNoEntry,
+ 3: ErrorAccessRead,
+ 4: ErrorKernel,
+ 5: ErrorDivZero,
+ 6: ErrorChipName,
+ 7: ErrorBusName,
+ 8: ErrorParse,
+ 9: ErrorAccessWrite,
+ 10: ErrorIO,
+ 11: ErrorRecursion
+}
+
+
+def raise_sensor_error(errno, message=''):
+ raise _ERR_MAP[abs(errno)](message)
+
+
+class bus_id(Structure):
+ _fields_ = [("type", c_short),
+ ("nr", c_short)]
+
+
+class chip_name(Structure):
+ _fields_ = [("prefix", c_char_p),
+ ("bus", bus_id),
+ ("addr", c_int),
+ ("path", c_char_p)]
+
+
+class feature(Structure):
+ _fields_ = [("name", c_char_p),
+ ("number", c_int),
+ ("type", c_int)]
+
+ # sensors_feature_type
+ IN = 0x00
+ FAN = 0x01
+ TEMP = 0x02
+ POWER = 0x03
+ ENERGY = 0x04
+ CURR = 0x05
+ HUMIDITY = 0x06
+ MAX_MAIN = 0x7
+ VID = 0x10
+ INTRUSION = 0x11
+ MAX_OTHER = 0x12
+ BEEP_ENABLE = 0x18
+
+
+class subfeature(Structure):
+ _fields_ = [("name", c_char_p),
+ ("number", c_int),
+ ("type", c_int),
+ ("mapping", c_int),
+ ("flags", c_uint)]
+
+
+_hdl.sensors_get_detected_chips.restype = POINTER(chip_name)
+_hdl.sensors_get_features.restype = POINTER(feature)
+_hdl.sensors_get_all_subfeatures.restype = POINTER(subfeature)
+_hdl.sensors_get_label.restype = c_void_p # return pointer instead of str so we can free it
+_hdl.sensors_get_adapter_name.restype = c_char_p # docs do not say whether to free this or not
+_hdl.sensors_strerror.restype = c_char_p
+
+### RAW API ###
+MODE_R = 1
+MODE_W = 2
+COMPUTE_MAPPING = 4
+
+
+def init(cfg_file=None):
+ file = _libc.fopen(cfg_file.encode("utf-8"), "r") if cfg_file is not None else None
+
+ result = _hdl.sensors_init(file)
+ if result != 0:
+ raise_sensor_error(result, "sensors_init failed")
+
+ if file is not None:
+ _libc.fclose(file)
+
+
+def cleanup():
+ _hdl.sensors_cleanup()
+
+
+def parse_chip_name(orig_name):
+ ret = chip_name()
+ err = _hdl.sensors_parse_chip_name(orig_name.encode("utf-8"), byref(ret))
+
+ if err < 0:
+ raise_sensor_error(err, strerror(err))
+
+ return ret
+
+
+def strerror(errnum):
+ return _hdl.sensors_strerror(errnum).decode("utf-8")
+
+
+def free_chip_name(chip):
+ _hdl.sensors_free_chip_name(byref(chip))
+
+
+def get_detected_chips(match, nr):
+ """
+ @return: (chip, next nr to query)
+ """
+ _nr = c_int(nr)
+
+ if match is not None:
+ match = byref(match)
+
+ chip = _hdl.sensors_get_detected_chips(match, byref(_nr))
+ chip = chip.contents if bool(chip) else None
+ return chip, _nr.value
+
+
+def chip_snprintf_name(chip, buffer_size=200):
+ """
+ @param buffer_size defaults to the size used in the sensors utility
+ """
+ ret = create_string_buffer(buffer_size)
+ err = _hdl.sensors_snprintf_chip_name(ret, buffer_size, byref(chip))
+
+ if err < 0:
+ raise_sensor_error(err, strerror(err))
+
+ return ret.value.decode("utf-8")
+
+
+def do_chip_sets(chip):
+ """
+ @attention this function was not tested
+ """
+ err = _hdl.sensors_do_chip_sets(byref(chip))
+ if err < 0:
+ raise_sensor_error(err, strerror(err))
+
+
+def get_adapter_name(bus):
+ return _hdl.sensors_get_adapter_name(byref(bus)).decode("utf-8")
+
+
+def get_features(chip, nr):
+ """
+ @return: (feature, next nr to query)
+ """
+ _nr = c_int(nr)
+ feature = _hdl.sensors_get_features(byref(chip), byref(_nr))
+ feature = feature.contents if bool(feature) else None
+ return feature, _nr.value
+
+
+def get_label(chip, feature):
+ ptr = _hdl.sensors_get_label(byref(chip), byref(feature))
+ val = cast(ptr, c_char_p).value.decode("utf-8")
+ _libc.free(ptr)
+ return val
+
+
+def get_all_subfeatures(chip, feature, nr):
+ """
+ @return: (subfeature, next nr to query)
+ """
+ _nr = c_int(nr)
+ subfeature = _hdl.sensors_get_all_subfeatures(byref(chip), byref(feature), byref(_nr))
+ subfeature = subfeature.contents if bool(subfeature) else None
+ return subfeature, _nr.value
+
+
+def get_value(chip, subfeature_nr):
+ val = c_double()
+ err = _hdl.sensors_get_value(byref(chip), subfeature_nr, byref(val))
+ if err < 0:
+ raise_sensor_error(err, strerror(err))
+ return val.value
+
+
+def set_value(chip, subfeature_nr, value):
+ """
+ @attention this function was not tested
+ """
+ val = c_double(value)
+ err = _hdl.sensors_set_value(byref(chip), subfeature_nr, byref(val))
+ if err < 0:
+ raise_sensor_error(err, strerror(err))
+
+
+### Convenience API ###
+class ChipIterator:
+ def __init__(self, match=None):
+ self.match = parse_chip_name(match) if match is not None else None
+ self.nr = 0
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ chip, self.nr = get_detected_chips(self.match, self.nr)
+
+ if chip is None:
+ raise StopIteration
+
+ return chip
+
+ def __del__(self):
+ if self.match is not None:
+ free_chip_name(self.match)
+
+ def next(self): # python2 compability
+ return self.__next__()
+
+
+class FeatureIterator:
+ def __init__(self, chip):
+ self.chip = chip
+ self.nr = 0
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ feature, self.nr = get_features(self.chip, self.nr)
+
+ if feature is None:
+ raise StopIteration
+
+ return feature
+
+ def next(self): # python2 compability
+ return self.__next__()
+
+
+class SubFeatureIterator:
+ def __init__(self, chip, feature):
+ self.chip = chip
+ self.feature = feature
+ self.nr = 0
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ subfeature, self.nr = get_all_subfeatures(self.chip, self.feature, self.nr)
+
+ if subfeature is None:
+ raise StopIteration
+
+ return subfeature
+
+ def next(self): # python2 compability
+ return self.__next__()
diff --git a/collectors/python.d.plugin/python_modules/third_party/mcrcon.py b/collectors/python.d.plugin/python_modules/third_party/mcrcon.py
new file mode 100644
index 0000000..a65a304
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/third_party/mcrcon.py
@@ -0,0 +1,74 @@
+# Minecraft Remote Console module.
+#
+# Copyright (C) 2015 Barnaby Gale
+#
+# SPDX-License-Identifier: MIT
+
+import socket
+import select
+import struct
+import time
+
+
+class MCRconException(Exception):
+ pass
+
+
+class MCRcon(object):
+ socket = None
+
+ def connect(self, host, port, password):
+ if self.socket is not None:
+ raise MCRconException("Already connected")
+ self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.socket.settimeout(0.9)
+ self.socket.connect((host, port))
+ self.send(3, password)
+
+ def disconnect(self):
+ if self.socket is None:
+ raise MCRconException("Already disconnected")
+ self.socket.close()
+ self.socket = None
+
+ def read(self, length):
+ data = b""
+ while len(data) < length:
+ data += self.socket.recv(length - len(data))
+ return data
+
+ def send(self, out_type, out_data):
+ if self.socket is None:
+ raise MCRconException("Must connect before sending data")
+
+ # Send a request packet
+ out_payload = struct.pack('<ii', 0, out_type) + out_data.encode('utf8') + b'\x00\x00'
+ out_length = struct.pack('<i', len(out_payload))
+ self.socket.send(out_length + out_payload)
+
+ # Read response packets
+ in_data = ""
+ while True:
+ # Read a packet
+ in_length, = struct.unpack('<i', self.read(4))
+ in_payload = self.read(in_length)
+ in_id = struct.unpack('<ii', in_payload[:8])
+ in_data_partial, in_padding = in_payload[8:-2], in_payload[-2:]
+
+ # Sanity checks
+ if in_padding != b'\x00\x00':
+ raise MCRconException("Incorrect padding")
+ if in_id == -1:
+ raise MCRconException("Login failed")
+
+ # Record the response
+ in_data += in_data_partial.decode('utf8')
+
+ # If there's nothing more to receive, return the response
+ if len(select.select([self.socket], [], [], 0)[0]) == 0:
+ return in_data
+
+ def command(self, command):
+ result = self.send(2, command)
+ time.sleep(0.003) # MC-72390 workaround
+ return result
diff --git a/collectors/python.d.plugin/python_modules/third_party/monotonic.py b/collectors/python.d.plugin/python_modules/third_party/monotonic.py
new file mode 100644
index 0000000..4ebd556
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/third_party/monotonic.py
@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+#
+# SPDX-License-Identifier: Apache-2.0
+"""
+ monotonic
+ ~~~~~~~~~
+
+ This module provides a ``monotonic()`` function which returns the
+ value (in fractional seconds) of a clock which never goes backwards.
+
+ On Python 3.3 or newer, ``monotonic`` will be an alias of
+ ``time.monotonic`` from the standard library. On older versions,
+ it will fall back to an equivalent implementation:
+
+ +-------------+----------------------------------------+
+ | Linux, BSD | ``clock_gettime(3)`` |
+ +-------------+----------------------------------------+
+ | Windows | ``GetTickCount`` or ``GetTickCount64`` |
+ +-------------+----------------------------------------+
+ | OS X | ``mach_absolute_time`` |
+ +-------------+----------------------------------------+
+
+ If no suitable implementation exists for the current platform,
+ attempting to import this module (or to import from it) will
+ cause a ``RuntimeError`` exception to be raised.
+
+
+ Copyright 2014, 2015, 2016 Ori Livneh <ori@wikimedia.org>
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+"""
+import time
+
+
+__all__ = ('monotonic',)
+
+
+try:
+ monotonic = time.monotonic
+except AttributeError:
+ import ctypes
+ import ctypes.util
+ import os
+ import sys
+ import threading
+
+
+ def clock_clock_gettime_c_library():
+ return ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True).clock_gettime
+
+
+ def clock_clock_gettime_rt_library():
+ return ctypes.CDLL(ctypes.util.find_library('rt'), use_errno=True).clock_gettime
+
+
+ def clock_clock_gettime_c_library_synology6():
+ return ctypes.CDLL('/usr/lib/libc.so.6', use_errno=True).clock_gettime
+
+
+ def clock_clock_gettime_rt_library_synology6():
+ return ctypes.CDLL('/usr/lib/librt.so.1', use_errno=True).clock_gettime
+
+
+ def clock_gettime_linux():
+ # see https://github.com/netdata/netdata/issues/7976
+ order = [
+ clock_clock_gettime_c_library,
+ clock_clock_gettime_rt_library,
+ clock_clock_gettime_c_library_synology6,
+ clock_clock_gettime_rt_library_synology6,
+ ]
+
+ for gettime in order:
+ try:
+ return gettime()
+ except (RuntimeError, AttributeError, OSError):
+ continue
+ raise RuntimeError('can not find c and rt libraries')
+
+
+ try:
+ if sys.platform == 'darwin': # OS X, iOS
+ # See Technical Q&A QA1398 of the Mac Developer Library:
+ # <https://developer.apple.com/library/mac/qa/qa1398/>
+ libc = ctypes.CDLL('/usr/lib/libc.dylib', use_errno=True)
+
+ class mach_timebase_info_data_t(ctypes.Structure):
+ """System timebase info. Defined in <mach/mach_time.h>."""
+ _fields_ = (('numer', ctypes.c_uint32),
+ ('denom', ctypes.c_uint32))
+
+ mach_absolute_time = libc.mach_absolute_time
+ mach_absolute_time.restype = ctypes.c_uint64
+
+ timebase = mach_timebase_info_data_t()
+ libc.mach_timebase_info(ctypes.byref(timebase))
+ ticks_per_second = timebase.numer / timebase.denom * 1.0e9
+
+ def monotonic():
+ """Monotonic clock, cannot go backward."""
+ return mach_absolute_time() / ticks_per_second
+
+ elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
+ if sys.platform.startswith('cygwin'):
+ # Note: cygwin implements clock_gettime (CLOCK_MONOTONIC = 4) since
+ # version 1.7.6. Using raw WinAPI for maximum version compatibility.
+
+ # Ugly hack using the wrong calling convention (in 32-bit mode)
+ # because ctypes has no windll under cygwin (and it also seems that
+ # the code letting you select stdcall in _ctypes doesn't exist under
+ # the preprocessor definitions relevant to cygwin).
+ # This is 'safe' because:
+ # 1. The ABI of GetTickCount and GetTickCount64 is identical for
+ # both calling conventions because they both have no parameters.
+ # 2. libffi masks the problem because after making the call it doesn't
+ # touch anything through esp and epilogue code restores a correct
+ # esp from ebp afterwards.
+ try:
+ kernel32 = ctypes.cdll.kernel32
+ except OSError: # 'No such file or directory'
+ kernel32 = ctypes.cdll.LoadLibrary('kernel32.dll')
+ else:
+ kernel32 = ctypes.windll.kernel32
+
+ GetTickCount64 = getattr(kernel32, 'GetTickCount64', None)
+ if GetTickCount64:
+ # Windows Vista / Windows Server 2008 or newer.
+ GetTickCount64.restype = ctypes.c_ulonglong
+
+ def monotonic():
+ """Monotonic clock, cannot go backward."""
+ return GetTickCount64() / 1000.0
+
+ else:
+ # Before Windows Vista.
+ GetTickCount = kernel32.GetTickCount
+ GetTickCount.restype = ctypes.c_uint32
+
+ get_tick_count_lock = threading.Lock()
+ get_tick_count_last_sample = 0
+ get_tick_count_wraparounds = 0
+
+ def monotonic():
+ """Monotonic clock, cannot go backward."""
+ global get_tick_count_last_sample
+ global get_tick_count_wraparounds
+
+ with get_tick_count_lock:
+ current_sample = GetTickCount()
+ if current_sample < get_tick_count_last_sample:
+ get_tick_count_wraparounds += 1
+ get_tick_count_last_sample = current_sample
+
+ final_milliseconds = get_tick_count_wraparounds << 32
+ final_milliseconds += get_tick_count_last_sample
+ return final_milliseconds / 1000.0
+
+ else:
+ clock_gettime = clock_gettime_linux()
+
+ class timespec(ctypes.Structure):
+ """Time specification, as described in clock_gettime(3)."""
+ _fields_ = (('tv_sec', ctypes.c_long),
+ ('tv_nsec', ctypes.c_long))
+
+ if sys.platform.startswith('linux'):
+ CLOCK_MONOTONIC = 1
+ elif sys.platform.startswith('freebsd'):
+ CLOCK_MONOTONIC = 4
+ elif sys.platform.startswith('sunos5'):
+ CLOCK_MONOTONIC = 4
+ elif 'bsd' in sys.platform:
+ CLOCK_MONOTONIC = 3
+ elif sys.platform.startswith('aix'):
+ CLOCK_MONOTONIC = ctypes.c_longlong(10)
+
+ def monotonic():
+ """Monotonic clock, cannot go backward."""
+ ts = timespec()
+ if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(ts)):
+ errno = ctypes.get_errno()
+ raise OSError(errno, os.strerror(errno))
+ return ts.tv_sec + ts.tv_nsec / 1.0e9
+
+ # Perform a sanity-check.
+ if monotonic() - monotonic() > 0:
+ raise ValueError('monotonic() is not monotonic!')
+
+ except Exception as e:
+ raise RuntimeError('no suitable implementation for this system: ' + repr(e))
diff --git a/collectors/python.d.plugin/python_modules/third_party/ordereddict.py b/collectors/python.d.plugin/python_modules/third_party/ordereddict.py
new file mode 100644
index 0000000..589401b
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/third_party/ordereddict.py
@@ -0,0 +1,110 @@
+# Copyright (c) 2009 Raymond Hettinger
+#
+# SPDX-License-Identifier: MIT
+
+from UserDict import DictMixin
+
+
+class OrderedDict(dict, DictMixin):
+
+ def __init__(self, *args, **kwds):
+ if len(args) > 1:
+ raise TypeError('expected at most 1 arguments, got %d' % len(args))
+ try:
+ self.__end
+ except AttributeError:
+ self.clear()
+ self.update(*args, **kwds)
+
+ def clear(self):
+ self.__end = end = []
+ end += [None, end, end] # sentinel node for doubly linked list
+ self.__map = {} # key --> [key, prev, next]
+ dict.clear(self)
+
+ def __setitem__(self, key, value):
+ if key not in self:
+ end = self.__end
+ curr = end[1]
+ curr[2] = end[1] = self.__map[key] = [key, curr, end]
+ dict.__setitem__(self, key, value)
+
+ def __delitem__(self, key):
+ dict.__delitem__(self, key)
+ key, prev, next = self.__map.pop(key)
+ prev[2] = next
+ next[1] = prev
+
+ def __iter__(self):
+ end = self.__end
+ curr = end[2]
+ while curr is not end:
+ yield curr[0]
+ curr = curr[2]
+
+ def __reversed__(self):
+ end = self.__end
+ curr = end[1]
+ while curr is not end:
+ yield curr[0]
+ curr = curr[1]
+
+ def popitem(self, last=True):
+ if not self:
+ raise KeyError('dictionary is empty')
+ if last:
+ key = reversed(self).next()
+ else:
+ key = iter(self).next()
+ value = self.pop(key)
+ return key, value
+
+ def __reduce__(self):
+ items = [[k, self[k]] for k in self]
+ tmp = self.__map, self.__end
+ del self.__map, self.__end
+ inst_dict = vars(self).copy()
+ self.__map, self.__end = tmp
+ if inst_dict:
+ return self.__class__, (items,), inst_dict
+ return self.__class__, (items,)
+
+ def keys(self):
+ return list(self)
+
+ setdefault = DictMixin.setdefault
+ update = DictMixin.update
+ pop = DictMixin.pop
+ values = DictMixin.values
+ items = DictMixin.items
+ iterkeys = DictMixin.iterkeys
+ itervalues = DictMixin.itervalues
+ iteritems = DictMixin.iteritems
+
+ def __repr__(self):
+ if not self:
+ return '%s()' % (self.__class__.__name__,)
+ return '%s(%r)' % (self.__class__.__name__, self.items())
+
+ def copy(self):
+ return self.__class__(self)
+
+ @classmethod
+ def fromkeys(cls, iterable, value=None):
+ d = cls()
+ for key in iterable:
+ d[key] = value
+ return d
+
+ def __eq__(self, other):
+ if isinstance(other, OrderedDict):
+ if len(self) != len(other):
+ return False
+ for p, q in zip(self.items(), other.items()):
+ if p != q:
+ return False
+ return True
+ return dict.__eq__(self, other)
+
+ def __ne__(self, other):
+ return not self == other
diff --git a/collectors/python.d.plugin/python_modules/urllib3/__init__.py b/collectors/python.d.plugin/python_modules/urllib3/__init__.py
new file mode 100644
index 0000000..3add848
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/__init__.py
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: MIT
+"""
+urllib3 - Thread-safe connection pooling and re-using.
+"""
+
+from __future__ import absolute_import
+import warnings
+
+from .connectionpool import (
+ HTTPConnectionPool,
+ HTTPSConnectionPool,
+ connection_from_url
+)
+
+from . import exceptions
+from .filepost import encode_multipart_formdata
+from .poolmanager import PoolManager, ProxyManager, proxy_from_url
+from .response import HTTPResponse
+from .util.request import make_headers
+from .util.url import get_host
+from .util.timeout import Timeout
+from .util.retry import Retry
+
+
+# Set default logging handler to avoid "No handler found" warnings.
+import logging
+try: # Python 2.7+
+ from logging import NullHandler
+except ImportError:
+ class NullHandler(logging.Handler):
+ def emit(self, record):
+ pass
+
+__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
+__license__ = 'MIT'
+__version__ = '1.21.1'
+
+__all__ = (
+ 'HTTPConnectionPool',
+ 'HTTPSConnectionPool',
+ 'PoolManager',
+ 'ProxyManager',
+ 'HTTPResponse',
+ 'Retry',
+ 'Timeout',
+ 'add_stderr_logger',
+ 'connection_from_url',
+ 'disable_warnings',
+ 'encode_multipart_formdata',
+ 'get_host',
+ 'make_headers',
+ 'proxy_from_url',
+)
+
+logging.getLogger(__name__).addHandler(NullHandler())
+
+
+def add_stderr_logger(level=logging.DEBUG):
+ """
+ Helper for quickly adding a StreamHandler to the logger. Useful for
+ debugging.
+
+ Returns the handler after adding it.
+ """
+ # This method needs to be in this __init__.py to get the __name__ correct
+ # even if urllib3 is vendored within another package.
+ logger = logging.getLogger(__name__)
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
+ logger.addHandler(handler)
+ logger.setLevel(level)
+ logger.debug('Added a stderr logging handler to logger: %s', __name__)
+ return handler
+
+
+# ... Clean up.
+del NullHandler
+
+
+# All warning filters *must* be appended unless you're really certain that they
+# shouldn't be: otherwise, it's very hard for users to use most Python
+# mechanisms to silence them.
+# SecurityWarning's always go off by default.
+warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
+# SubjectAltNameWarning's should go off once per host
+warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True)
+# InsecurePlatformWarning's don't vary between requests, so we keep it default.
+warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
+ append=True)
+# SNIMissingWarnings should go off only once.
+warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True)
+
+
+def disable_warnings(category=exceptions.HTTPWarning):
+ """
+ Helper for quickly disabling all urllib3 warnings.
+ """
+ warnings.simplefilter('ignore', category)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/_collections.py b/collectors/python.d.plugin/python_modules/urllib3/_collections.py
new file mode 100644
index 0000000..2a6b3ec
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/_collections.py
@@ -0,0 +1,320 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+
+try:
+ from collections import Mapping, MutableMapping
+except ImportError:
+ from collections.abc import Mapping, MutableMapping
+
+try:
+ from threading import RLock
+except ImportError: # Platform-specific: No threads available
+ class RLock:
+ def __enter__(self):
+ pass
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ pass
+
+
+try: # Python 2.7+
+ from collections import OrderedDict
+except ImportError:
+ from .packages.ordered_dict import OrderedDict
+from .packages.six import iterkeys, itervalues, PY3
+
+
+__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
+
+
+_Null = object()
+
+
+class RecentlyUsedContainer(MutableMapping):
+ """
+ Provides a thread-safe dict-like container which maintains up to
+ ``maxsize`` keys while throwing away the least-recently-used keys beyond
+ ``maxsize``.
+
+ :param maxsize:
+ Maximum number of recent elements to retain.
+
+ :param dispose_func:
+ Every time an item is evicted from the container,
+ ``dispose_func(value)`` is called. Callback which will get called
+ """
+
+ ContainerCls = OrderedDict
+
+ def __init__(self, maxsize=10, dispose_func=None):
+ self._maxsize = maxsize
+ self.dispose_func = dispose_func
+
+ self._container = self.ContainerCls()
+ self.lock = RLock()
+
+ def __getitem__(self, key):
+ # Re-insert the item, moving it to the end of the eviction line.
+ with self.lock:
+ item = self._container.pop(key)
+ self._container[key] = item
+ return item
+
+ def __setitem__(self, key, value):
+ evicted_value = _Null
+ with self.lock:
+ # Possibly evict the existing value of 'key'
+ evicted_value = self._container.get(key, _Null)
+ self._container[key] = value
+
+ # If we didn't evict an existing value, we might have to evict the
+ # least recently used item from the beginning of the container.
+ if len(self._container) > self._maxsize:
+ _key, evicted_value = self._container.popitem(last=False)
+
+ if self.dispose_func and evicted_value is not _Null:
+ self.dispose_func(evicted_value)
+
+ def __delitem__(self, key):
+ with self.lock:
+ value = self._container.pop(key)
+
+ if self.dispose_func:
+ self.dispose_func(value)
+
+ def __len__(self):
+ with self.lock:
+ return len(self._container)
+
+ def __iter__(self):
+ raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')
+
+ def clear(self):
+ with self.lock:
+ # Copy pointers to all values, then wipe the mapping
+ values = list(itervalues(self._container))
+ self._container.clear()
+
+ if self.dispose_func:
+ for value in values:
+ self.dispose_func(value)
+
+ def keys(self):
+ with self.lock:
+ return list(iterkeys(self._container))
+
+
+class HTTPHeaderDict(MutableMapping):
+ """
+ :param headers:
+ An iterable of field-value pairs. Must not contain multiple field names
+ when compared case-insensitively.
+
+ :param kwargs:
+ Additional field-value pairs to pass in to ``dict.update``.
+
+ A ``dict`` like container for storing HTTP Headers.
+
+ Field names are stored and compared case-insensitively in compliance with
+ RFC 7230. Iteration provides the first case-sensitive key seen for each
+ case-insensitive pair.
+
+ Using ``__setitem__`` syntax overwrites fields that compare equal
+ case-insensitively in order to maintain ``dict``'s api. For fields that
+ compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
+ in a loop.
+
+ If multiple fields that are equal case-insensitively are passed to the
+ constructor or ``.update``, the behavior is undefined and some will be
+ lost.
+
+ >>> headers = HTTPHeaderDict()
+ >>> headers.add('Set-Cookie', 'foo=bar')
+ >>> headers.add('set-cookie', 'baz=quxx')
+ >>> headers['content-length'] = '7'
+ >>> headers['SET-cookie']
+ 'foo=bar, baz=quxx'
+ >>> headers['Content-Length']
+ '7'
+ """
+
+ def __init__(self, headers=None, **kwargs):
+ super(HTTPHeaderDict, self).__init__()
+ self._container = OrderedDict()
+ if headers is not None:
+ if isinstance(headers, HTTPHeaderDict):
+ self._copy_from(headers)
+ else:
+ self.extend(headers)
+ if kwargs:
+ self.extend(kwargs)
+
+ def __setitem__(self, key, val):
+ self._container[key.lower()] = [key, val]
+ return self._container[key.lower()]
+
+ def __getitem__(self, key):
+ val = self._container[key.lower()]
+ return ', '.join(val[1:])
+
+ def __delitem__(self, key):
+ del self._container[key.lower()]
+
+ def __contains__(self, key):
+ return key.lower() in self._container
+
+ def __eq__(self, other):
+ if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
+ return False
+ if not isinstance(other, type(self)):
+ other = type(self)(other)
+ return (dict((k.lower(), v) for k, v in self.itermerged()) ==
+ dict((k.lower(), v) for k, v in other.itermerged()))
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ if not PY3: # Python 2
+ iterkeys = MutableMapping.iterkeys
+ itervalues = MutableMapping.itervalues
+
+ __marker = object()
+
+ def __len__(self):
+ return len(self._container)
+
+ def __iter__(self):
+ # Only provide the originally cased names
+ for vals in self._container.values():
+ yield vals[0]
+
+ def pop(self, key, default=__marker):
+ '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
+ If key is not found, d is returned if given, otherwise KeyError is raised.
+ '''
+ # Using the MutableMapping function directly fails due to the private marker.
+ # Using ordinary dict.pop would expose the internal structures.
+ # So let's reinvent the wheel.
+ try:
+ value = self[key]
+ except KeyError:
+ if default is self.__marker:
+ raise
+ return default
+ else:
+ del self[key]
+ return value
+
+ def discard(self, key):
+ try:
+ del self[key]
+ except KeyError:
+ pass
+
+ def add(self, key, val):
+ """Adds a (name, value) pair, doesn't overwrite the value if it already
+ exists.
+
+ >>> headers = HTTPHeaderDict(foo='bar')
+ >>> headers.add('Foo', 'baz')
+ >>> headers['foo']
+ 'bar, baz'
+ """
+ key_lower = key.lower()
+ new_vals = [key, val]
+ # Keep the common case aka no item present as fast as possible
+ vals = self._container.setdefault(key_lower, new_vals)
+ if new_vals is not vals:
+ vals.append(val)
+
+ def extend(self, *args, **kwargs):
+ """Generic import function for any type of header-like object.
+ Adapted version of MutableMapping.update in order to insert items
+ with self.add instead of self.__setitem__
+ """
+ if len(args) > 1:
+ raise TypeError("extend() takes at most 1 positional "
+ "arguments ({0} given)".format(len(args)))
+ other = args[0] if len(args) >= 1 else ()
+
+ if isinstance(other, HTTPHeaderDict):
+ for key, val in other.iteritems():
+ self.add(key, val)
+ elif isinstance(other, Mapping):
+ for key in other:
+ self.add(key, other[key])
+ elif hasattr(other, "keys"):
+ for key in other.keys():
+ self.add(key, other[key])
+ else:
+ for key, value in other:
+ self.add(key, value)
+
+ for key, value in kwargs.items():
+ self.add(key, value)
+
+ def getlist(self, key):
+ """Returns a list of all the values for the named field. Returns an
+ empty list if the key doesn't exist."""
+ try:
+ vals = self._container[key.lower()]
+ except KeyError:
+ return []
+ else:
+ return vals[1:]
+
+ # Backwards compatibility for httplib
+ getheaders = getlist
+ getallmatchingheaders = getlist
+ iget = getlist
+
+ def __repr__(self):
+ return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
+
+ def _copy_from(self, other):
+ for key in other:
+ val = other.getlist(key)
+ if isinstance(val, list):
+ # Don't need to convert tuples
+ val = list(val)
+ self._container[key.lower()] = [key] + val
+
+ def copy(self):
+ clone = type(self)()
+ clone._copy_from(self)
+ return clone
+
+ def iteritems(self):
+ """Iterate over all header lines, including duplicate ones."""
+ for key in self:
+ vals = self._container[key.lower()]
+ for val in vals[1:]:
+ yield vals[0], val
+
+ def itermerged(self):
+ """Iterate over all headers, merging duplicate ones together."""
+ for key in self:
+ val = self._container[key.lower()]
+ yield val[0], ', '.join(val[1:])
+
+ def items(self):
+ return list(self.iteritems())
+
+ @classmethod
+ def from_httplib(cls, message): # Python 2
+ """Read headers from a Python 2 httplib message object."""
+ # python2.7 does not expose a proper API for exporting multiheaders
+ # efficiently. This function re-reads raw lines from the message
+ # object and extracts the multiheaders properly.
+ headers = []
+
+ for line in message.headers:
+ if line.startswith((' ', '\t')):
+ key, value = headers[-1]
+ headers[-1] = (key, value + '\r\n' + line.rstrip())
+ continue
+
+ key, value = line.split(':', 1)
+ headers.append((key, value.strip()))
+
+ return cls(headers)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/connection.py b/collectors/python.d.plugin/python_modules/urllib3/connection.py
new file mode 100644
index 0000000..f757493
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/connection.py
@@ -0,0 +1,374 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+import datetime
+import logging
+import os
+import sys
+import socket
+from socket import error as SocketError, timeout as SocketTimeout
+import warnings
+from .packages import six
+from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection
+from .packages.six.moves.http_client import HTTPException # noqa: F401
+
+try: # Compiled with SSL?
+ import ssl
+ BaseSSLError = ssl.SSLError
+except (ImportError, AttributeError): # Platform-specific: No SSL.
+ ssl = None
+
+ class BaseSSLError(BaseException):
+ pass
+
+
+try: # Python 3:
+ # Not a no-op, we're adding this to the namespace so it can be imported.
+ ConnectionError = ConnectionError
+except NameError: # Python 2:
+ class ConnectionError(Exception):
+ pass
+
+
+from .exceptions import (
+ NewConnectionError,
+ ConnectTimeoutError,
+ SubjectAltNameWarning,
+ SystemTimeWarning,
+)
+from .packages.ssl_match_hostname import match_hostname, CertificateError
+
+from .util.ssl_ import (
+ resolve_cert_reqs,
+ resolve_ssl_version,
+ assert_fingerprint,
+ create_urllib3_context,
+ ssl_wrap_socket
+)
+
+
+from .util import connection
+
+from ._collections import HTTPHeaderDict
+
+log = logging.getLogger(__name__)
+
+port_by_scheme = {
+ 'http': 80,
+ 'https': 443,
+}
+
+# When updating RECENT_DATE, move it to
+# within two years of the current date, and no
+# earlier than 6 months ago.
+RECENT_DATE = datetime.date(2016, 1, 1)
+
+
+class DummyConnection(object):
+ """Used to detect a failed ConnectionCls import."""
+ pass
+
+
+class HTTPConnection(_HTTPConnection, object):
+ """
+ Based on httplib.HTTPConnection but provides an extra constructor
+ backwards-compatibility layer between older and newer Pythons.
+
+ Additional keyword parameters are used to configure attributes of the connection.
+ Accepted parameters include:
+
+ - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool`
+ - ``source_address``: Set the source address for the current connection.
+
+ .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x
+
+ - ``socket_options``: Set specific options on the underlying socket. If not specified, then
+ defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
+ Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
+
+ For example, if you wish to enable TCP Keep Alive in addition to the defaults,
+ you might pass::
+
+ HTTPConnection.default_socket_options + [
+ (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
+ ]
+
+ Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
+ """
+
+ default_port = port_by_scheme['http']
+
+ #: Disable Nagle's algorithm by default.
+ #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
+ default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
+
+ #: Whether this connection verifies the host's certificate.
+ is_verified = False
+
+ def __init__(self, *args, **kw):
+ if six.PY3: # Python 3
+ kw.pop('strict', None)
+
+ # Pre-set source_address in case we have an older Python like 2.6.
+ self.source_address = kw.get('source_address')
+
+ if sys.version_info < (2, 7): # Python 2.6
+ # _HTTPConnection on Python 2.6 will balk at this keyword arg, but
+ # not newer versions. We can still use it when creating a
+ # connection though, so we pop it *after* we have saved it as
+ # self.source_address.
+ kw.pop('source_address', None)
+
+ #: The socket options provided by the user. If no options are
+ #: provided, we use the default options.
+ self.socket_options = kw.pop('socket_options', self.default_socket_options)
+
+ # Superclass also sets self.source_address in Python 2.7+.
+ _HTTPConnection.__init__(self, *args, **kw)
+
+ def _new_conn(self):
+ """ Establish a socket connection and set nodelay settings on it.
+
+ :return: New socket connection.
+ """
+ extra_kw = {}
+ if self.source_address:
+ extra_kw['source_address'] = self.source_address
+
+ if self.socket_options:
+ extra_kw['socket_options'] = self.socket_options
+
+ try:
+ conn = connection.create_connection(
+ (self.host, self.port), self.timeout, **extra_kw)
+
+ except SocketTimeout as e:
+ raise ConnectTimeoutError(
+ self, "Connection to %s timed out. (connect timeout=%s)" %
+ (self.host, self.timeout))
+
+ except SocketError as e:
+ raise NewConnectionError(
+ self, "Failed to establish a new connection: %s" % e)
+
+ return conn
+
+ def _prepare_conn(self, conn):
+ self.sock = conn
+ # the _tunnel_host attribute was added in python 2.6.3 (via
+ # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do
+ # not have them.
+ if getattr(self, '_tunnel_host', None):
+ # TODO: Fix tunnel so it doesn't depend on self.sock state.
+ self._tunnel()
+ # Mark this connection as not reusable
+ self.auto_open = 0
+
+ def connect(self):
+ conn = self._new_conn()
+ self._prepare_conn(conn)
+
+ def request_chunked(self, method, url, body=None, headers=None):
+ """
+ Alternative to the common request method, which sends the
+ body with chunked encoding and not as one block
+ """
+ headers = HTTPHeaderDict(headers if headers is not None else {})
+ skip_accept_encoding = 'accept-encoding' in headers
+ skip_host = 'host' in headers
+ self.putrequest(
+ method,
+ url,
+ skip_accept_encoding=skip_accept_encoding,
+ skip_host=skip_host
+ )
+ for header, value in headers.items():
+ self.putheader(header, value)
+ if 'transfer-encoding' not in headers:
+ self.putheader('Transfer-Encoding', 'chunked')
+ self.endheaders()
+
+ if body is not None:
+ stringish_types = six.string_types + (six.binary_type,)
+ if isinstance(body, stringish_types):
+ body = (body,)
+ for chunk in body:
+ if not chunk:
+ continue
+ if not isinstance(chunk, six.binary_type):
+ chunk = chunk.encode('utf8')
+ len_str = hex(len(chunk))[2:]
+ self.send(len_str.encode('utf-8'))
+ self.send(b'\r\n')
+ self.send(chunk)
+ self.send(b'\r\n')
+
+ # After the if clause, to always have a closed body
+ self.send(b'0\r\n\r\n')
+
+
+class HTTPSConnection(HTTPConnection):
+ default_port = port_by_scheme['https']
+
+ ssl_version = None
+
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ ssl_context=None, **kw):
+
+ HTTPConnection.__init__(self, host, port, strict=strict,
+ timeout=timeout, **kw)
+
+ self.key_file = key_file
+ self.cert_file = cert_file
+ self.ssl_context = ssl_context
+
+ # Required property for Google AppEngine 1.9.0 which otherwise causes
+ # HTTPS requests to go out as HTTP. (See Issue #356)
+ self._protocol = 'https'
+
+ def connect(self):
+ conn = self._new_conn()
+ self._prepare_conn(conn)
+
+ if self.ssl_context is None:
+ self.ssl_context = create_urllib3_context(
+ ssl_version=resolve_ssl_version(None),
+ cert_reqs=resolve_cert_reqs(None),
+ )
+
+ self.sock = ssl_wrap_socket(
+ sock=conn,
+ keyfile=self.key_file,
+ certfile=self.cert_file,
+ ssl_context=self.ssl_context,
+ )
+
+
+class VerifiedHTTPSConnection(HTTPSConnection):
+ """
+ Based on httplib.HTTPSConnection but wraps the socket with
+ SSL certification.
+ """
+ cert_reqs = None
+ ca_certs = None
+ ca_cert_dir = None
+ ssl_version = None
+ assert_fingerprint = None
+
+ def set_cert(self, key_file=None, cert_file=None,
+ cert_reqs=None, ca_certs=None,
+ assert_hostname=None, assert_fingerprint=None,
+ ca_cert_dir=None):
+ """
+ This method should only be called once, before the connection is used.
+ """
+ # If cert_reqs is not provided, we can try to guess. If the user gave
+ # us a cert database, we assume they want to use it: otherwise, if
+ # they gave us an SSL Context object we should use whatever is set for
+ # it.
+ if cert_reqs is None:
+ if ca_certs or ca_cert_dir:
+ cert_reqs = 'CERT_REQUIRED'
+ elif self.ssl_context is not None:
+ cert_reqs = self.ssl_context.verify_mode
+
+ self.key_file = key_file
+ self.cert_file = cert_file
+ self.cert_reqs = cert_reqs
+ self.assert_hostname = assert_hostname
+ self.assert_fingerprint = assert_fingerprint
+ self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
+ self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
+
+ def connect(self):
+ # Add certificate verification
+ conn = self._new_conn()
+
+ hostname = self.host
+ if getattr(self, '_tunnel_host', None):
+ # _tunnel_host was added in Python 2.6.3
+ # (See: http://hg.python.org/cpython/rev/0f57b30a152f)
+
+ self.sock = conn
+ # Calls self._set_hostport(), so self.host is
+ # self._tunnel_host below.
+ self._tunnel()
+ # Mark this connection as not reusable
+ self.auto_open = 0
+
+ # Override the host with the one we're requesting data from.
+ hostname = self._tunnel_host
+
+ is_time_off = datetime.date.today() < RECENT_DATE
+ if is_time_off:
+ warnings.warn((
+ 'System time is way off (before {0}). This will probably '
+ 'lead to SSL verification errors').format(RECENT_DATE),
+ SystemTimeWarning
+ )
+
+ # Wrap socket using verification with the root certs in
+ # trusted_root_certs
+ if self.ssl_context is None:
+ self.ssl_context = create_urllib3_context(
+ ssl_version=resolve_ssl_version(self.ssl_version),
+ cert_reqs=resolve_cert_reqs(self.cert_reqs),
+ )
+
+ context = self.ssl_context
+ context.verify_mode = resolve_cert_reqs(self.cert_reqs)
+ self.sock = ssl_wrap_socket(
+ sock=conn,
+ keyfile=self.key_file,
+ certfile=self.cert_file,
+ ca_certs=self.ca_certs,
+ ca_cert_dir=self.ca_cert_dir,
+ server_hostname=hostname,
+ ssl_context=context)
+
+ if self.assert_fingerprint:
+ assert_fingerprint(self.sock.getpeercert(binary_form=True),
+ self.assert_fingerprint)
+ elif context.verify_mode != ssl.CERT_NONE \
+ and not getattr(context, 'check_hostname', False) \
+ and self.assert_hostname is not False:
+ # While urllib3 attempts to always turn off hostname matching from
+ # the TLS library, this cannot always be done. So we check whether
+ # the TLS Library still thinks it's matching hostnames.
+ cert = self.sock.getpeercert()
+ if not cert.get('subjectAltName', ()):
+ warnings.warn((
+ 'Certificate for {0} has no `subjectAltName`, falling back to check for a '
+ '`commonName` for now. This feature is being removed by major browsers and '
+ 'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 '
+ 'for details.)'.format(hostname)),
+ SubjectAltNameWarning
+ )
+ _match_hostname(cert, self.assert_hostname or hostname)
+
+ self.is_verified = (
+ context.verify_mode == ssl.CERT_REQUIRED or
+ self.assert_fingerprint is not None
+ )
+
+
+def _match_hostname(cert, asserted_hostname):
+ try:
+ match_hostname(cert, asserted_hostname)
+ except CertificateError as e:
+ log.error(
+ 'Certificate did not match expected hostname: %s. '
+ 'Certificate: %s', asserted_hostname, cert
+ )
+ # Add cert to exception and reraise so client code can inspect
+ # the cert when catching the exception, if they want to
+ e._peer_cert = cert
+ raise
+
+
+if ssl:
+ # Make a copy for testing.
+ UnverifiedHTTPSConnection = HTTPSConnection
+ HTTPSConnection = VerifiedHTTPSConnection
+else:
+ HTTPSConnection = DummyConnection
diff --git a/collectors/python.d.plugin/python_modules/urllib3/connectionpool.py b/collectors/python.d.plugin/python_modules/urllib3/connectionpool.py
new file mode 100644
index 0000000..90e4c86
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/connectionpool.py
@@ -0,0 +1,900 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+import errno
+import logging
+import sys
+import warnings
+
+from socket import error as SocketError, timeout as SocketTimeout
+import socket
+
+
+from .exceptions import (
+ ClosedPoolError,
+ ProtocolError,
+ EmptyPoolError,
+ HeaderParsingError,
+ HostChangedError,
+ LocationValueError,
+ MaxRetryError,
+ ProxyError,
+ ReadTimeoutError,
+ SSLError,
+ TimeoutError,
+ InsecureRequestWarning,
+ NewConnectionError,
+)
+from .packages.ssl_match_hostname import CertificateError
+from .packages import six
+from .packages.six.moves import queue
+from .connection import (
+ port_by_scheme,
+ DummyConnection,
+ HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
+ HTTPException, BaseSSLError,
+)
+from .request import RequestMethods
+from .response import HTTPResponse
+
+from .util.connection import is_connection_dropped
+from .util.request import set_file_position
+from .util.response import assert_header_parsing
+from .util.retry import Retry
+from .util.timeout import Timeout
+from .util.url import get_host, Url
+
+
+if six.PY2:
+ # Queue is imported for side effects on MS Windows
+ import Queue as _unused_module_Queue # noqa: F401
+
+xrange = six.moves.xrange
+
+log = logging.getLogger(__name__)
+
+_Default = object()
+
+
+# Pool objects
+class ConnectionPool(object):
+ """
+ Base class for all connection pools, such as
+ :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
+ """
+
+ scheme = None
+ QueueCls = queue.LifoQueue
+
+ def __init__(self, host, port=None):
+ if not host:
+ raise LocationValueError("No host specified.")
+
+ self.host = _ipv6_host(host).lower()
+ self.port = port
+
+ def __str__(self):
+ return '%s(host=%r, port=%r)' % (type(self).__name__,
+ self.host, self.port)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+ # Return False to re-raise any potential exceptions
+ return False
+
+ def close(self):
+ """
+ Close all pooled connections and disable the pool.
+ """
+ pass
+
+
+# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
+_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK])
+
+
+class HTTPConnectionPool(ConnectionPool, RequestMethods):
+ """
+ Thread-safe connection pool for one host.
+
+ :param host:
+ Host used for this HTTP Connection (e.g. "localhost"), passed into
+ :class:`httplib.HTTPConnection`.
+
+ :param port:
+ Port used for this HTTP Connection (None is equivalent to 80), passed
+ into :class:`httplib.HTTPConnection`.
+
+ :param strict:
+ Causes BadStatusLine to be raised if the status line can't be parsed
+ as a valid HTTP/1.0 or 1.1 status line, passed into
+ :class:`httplib.HTTPConnection`.
+
+ .. note::
+ Only works in Python 2. This parameter is ignored in Python 3.
+
+ :param timeout:
+ Socket timeout in seconds for each individual connection. This can
+ be a float or integer, which sets the timeout for the HTTP request,
+ or an instance of :class:`urllib3.util.Timeout` which gives you more
+ fine-grained control over request timeouts. After the constructor has
+ been parsed, this is always a `urllib3.util.Timeout` object.
+
+ :param maxsize:
+ Number of connections to save that can be reused. More than 1 is useful
+ in multithreaded situations. If ``block`` is set to False, more
+ connections will be created but they will not be saved once they've
+ been used.
+
+ :param block:
+ If set to True, no more than ``maxsize`` connections will be used at
+ a time. When no free connections are available, the call will block
+ until a connection has been released. This is a useful side effect for
+ particular multithreaded situations where one does not want to use more
+ than maxsize connections per host to prevent flooding.
+
+ :param headers:
+ Headers to include with all requests, unless other headers are given
+ explicitly.
+
+ :param retries:
+ Retry configuration to use by default with requests in this pool.
+
+ :param _proxy:
+ Parsed proxy URL, should not be used directly, instead, see
+ :class:`urllib3.connectionpool.ProxyManager`"
+
+ :param _proxy_headers:
+ A dictionary with proxy headers, should not be used directly,
+ instead, see :class:`urllib3.connectionpool.ProxyManager`"
+
+ :param \\**conn_kw:
+ Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
+ :class:`urllib3.connection.HTTPSConnection` instances.
+ """
+
+ scheme = 'http'
+ ConnectionCls = HTTPConnection
+ ResponseCls = HTTPResponse
+
+ def __init__(self, host, port=None, strict=False,
+ timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False,
+ headers=None, retries=None,
+ _proxy=None, _proxy_headers=None,
+ **conn_kw):
+ ConnectionPool.__init__(self, host, port)
+ RequestMethods.__init__(self, headers)
+
+ self.strict = strict
+
+ if not isinstance(timeout, Timeout):
+ timeout = Timeout.from_float(timeout)
+
+ if retries is None:
+ retries = Retry.DEFAULT
+
+ self.timeout = timeout
+ self.retries = retries
+
+ self.pool = self.QueueCls(maxsize)
+ self.block = block
+
+ self.proxy = _proxy
+ self.proxy_headers = _proxy_headers or {}
+
+ # Fill the queue up so that doing get() on it will block properly
+ for _ in xrange(maxsize):
+ self.pool.put(None)
+
+ # These are mostly for testing and debugging purposes.
+ self.num_connections = 0
+ self.num_requests = 0
+ self.conn_kw = conn_kw
+
+ if self.proxy:
+ # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
+ # We cannot know if the user has added default socket options, so we cannot replace the
+ # list.
+ self.conn_kw.setdefault('socket_options', [])
+
+ def _new_conn(self):
+ """
+ Return a fresh :class:`HTTPConnection`.
+ """
+ self.num_connections += 1
+ log.debug("Starting new HTTP connection (%d): %s",
+ self.num_connections, self.host)
+
+ conn = self.ConnectionCls(host=self.host, port=self.port,
+ timeout=self.timeout.connect_timeout,
+ strict=self.strict, **self.conn_kw)
+ return conn
+
+ def _get_conn(self, timeout=None):
+ """
+ Get a connection. Will return a pooled connection if one is available.
+
+ If no connections are available and :prop:`.block` is ``False``, then a
+ fresh connection is returned.
+
+ :param timeout:
+ Seconds to wait before giving up and raising
+ :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
+ :prop:`.block` is ``True``.
+ """
+ conn = None
+ try:
+ conn = self.pool.get(block=self.block, timeout=timeout)
+
+ except AttributeError: # self.pool is None
+ raise ClosedPoolError(self, "Pool is closed.")
+
+ except queue.Empty:
+ if self.block:
+ raise EmptyPoolError(self,
+ "Pool reached maximum size and no more "
+ "connections are allowed.")
+ pass # Oh well, we'll create a new connection then
+
+ # If this is a persistent connection, check if it got disconnected
+ if conn and is_connection_dropped(conn):
+ log.debug("Resetting dropped connection: %s", self.host)
+ conn.close()
+ if getattr(conn, 'auto_open', 1) == 0:
+ # This is a proxied connection that has been mutated by
+ # httplib._tunnel() and cannot be reused (since it would
+ # attempt to bypass the proxy)
+ conn = None
+
+ return conn or self._new_conn()
+
+ def _put_conn(self, conn):
+ """
+ Put a connection back into the pool.
+
+ :param conn:
+ Connection object for the current host and port as returned by
+ :meth:`._new_conn` or :meth:`._get_conn`.
+
+ If the pool is already full, the connection is closed and discarded
+ because we exceeded maxsize. If connections are discarded frequently,
+ then maxsize should be increased.
+
+ If the pool is closed, then the connection will be closed and discarded.
+ """
+ try:
+ self.pool.put(conn, block=False)
+ return # Everything is dandy, done.
+ except AttributeError:
+ # self.pool is None.
+ pass
+ except queue.Full:
+ # This should never happen if self.block == True
+ log.warning(
+ "Connection pool is full, discarding connection: %s",
+ self.host)
+
+ # Connection never got put back into the pool, close it.
+ if conn:
+ conn.close()
+
+ def _validate_conn(self, conn):
+ """
+ Called right before a request is made, after the socket is created.
+ """
+ pass
+
+ def _prepare_proxy(self, conn):
+ # Nothing to do for HTTP connections.
+ pass
+
+ def _get_timeout(self, timeout):
+ """ Helper that always returns a :class:`urllib3.util.Timeout` """
+ if timeout is _Default:
+ return self.timeout.clone()
+
+ if isinstance(timeout, Timeout):
+ return timeout.clone()
+ else:
+ # User passed us an int/float. This is for backwards compatibility,
+ # can be removed later
+ return Timeout.from_float(timeout)
+
+ def _raise_timeout(self, err, url, timeout_value):
+ """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
+
+ if isinstance(err, SocketTimeout):
+ raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
+ # See the above comment about EAGAIN in Python 3. In Python 2 we have
+ # to specifically catch it and throw the timeout error
+ if hasattr(err, 'errno') and err.errno in _blocking_errnos:
+ raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
+ # Catch possible read timeouts thrown as SSL errors. If not the
+ # case, rethrow the original. We need to do this because of:
+ # http://bugs.python.org/issue10272
+ if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6
+ raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
+ def _make_request(self, conn, method, url, timeout=_Default, chunked=False,
+ **httplib_request_kw):
+ """
+ Perform a request on a given urllib connection object taken from our
+ pool.
+
+ :param conn:
+ a connection from one of our connection pools
+
+ :param timeout:
+ Socket timeout in seconds for the request. This can be a
+ float or integer, which will set the same timeout value for
+ the socket connect and the socket read, or an instance of
+ :class:`urllib3.util.Timeout`, which gives you more fine-grained
+ control over your timeouts.
+ """
+ self.num_requests += 1
+
+ timeout_obj = self._get_timeout(timeout)
+ timeout_obj.start_connect()
+ conn.timeout = timeout_obj.connect_timeout
+
+ # Trigger any extra validation we need to do.
+ try:
+ self._validate_conn(conn)
+ except (SocketTimeout, BaseSSLError) as e:
+ # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
+ self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
+ raise
+
+ # conn.request() calls httplib.*.request, not the method in
+ # urllib3.request. It also calls makefile (recv) on the socket.
+ if chunked:
+ conn.request_chunked(method, url, **httplib_request_kw)
+ else:
+ conn.request(method, url, **httplib_request_kw)
+
+ # Reset the timeout for the recv() on the socket
+ read_timeout = timeout_obj.read_timeout
+
+ # App Engine doesn't have a sock attr
+ if getattr(conn, 'sock', None):
+ # In Python 3 socket.py will catch EAGAIN and return None when you
+ # try and read into the file pointer created by http.client, which
+ # instead raises a BadStatusLine exception. Instead of catching
+ # the exception and assuming all BadStatusLine exceptions are read
+ # timeouts, check for a zero timeout before making the request.
+ if read_timeout == 0:
+ raise ReadTimeoutError(
+ self, url, "Read timed out. (read timeout=%s)" % read_timeout)
+ if read_timeout is Timeout.DEFAULT_TIMEOUT:
+ conn.sock.settimeout(socket.getdefaulttimeout())
+ else: # None or a value
+ conn.sock.settimeout(read_timeout)
+
+ # Receive the response from the server
+ try:
+ try: # Python 2.7, use buffering of HTTP responses
+ httplib_response = conn.getresponse(buffering=True)
+ except TypeError: # Python 2.6 and older, Python 3
+ try:
+ httplib_response = conn.getresponse()
+ except Exception as e:
+ # Remove the TypeError from the exception chain in Python 3;
+ # otherwise it looks like a programming error was the cause.
+ six.raise_from(e, None)
+ except (SocketTimeout, BaseSSLError, SocketError) as e:
+ self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
+ raise
+
+ # AppEngine doesn't have a version attr.
+ http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
+ log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port,
+ method, url, http_version, httplib_response.status,
+ httplib_response.length)
+
+ try:
+ assert_header_parsing(httplib_response.msg)
+ except HeaderParsingError as hpe: # Platform-specific: Python 3
+ log.warning(
+ 'Failed to parse headers (url=%s): %s',
+ self._absolute_url(url), hpe, exc_info=True)
+
+ return httplib_response
+
+ def _absolute_url(self, path):
+ return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url
+
+ def close(self):
+ """
+ Close all pooled connections and disable the pool.
+ """
+ # Disable access to the pool
+ old_pool, self.pool = self.pool, None
+
+ try:
+ while True:
+ conn = old_pool.get(block=False)
+ if conn:
+ conn.close()
+
+ except queue.Empty:
+ pass # Done.
+
+ def is_same_host(self, url):
+ """
+ Check if the given ``url`` is a member of the same host as this
+ connection pool.
+ """
+ if url.startswith('/'):
+ return True
+
+ # TODO: Add optional support for socket.gethostbyname checking.
+ scheme, host, port = get_host(url)
+
+ host = _ipv6_host(host).lower()
+
+ # Use explicit default port for comparison when none is given
+ if self.port and not port:
+ port = port_by_scheme.get(scheme)
+ elif not self.port and port == port_by_scheme.get(scheme):
+ port = None
+
+ return (scheme, host, port) == (self.scheme, self.host, self.port)
+
+ def urlopen(self, method, url, body=None, headers=None, retries=None,
+ redirect=True, assert_same_host=True, timeout=_Default,
+ pool_timeout=None, release_conn=None, chunked=False,
+ body_pos=None, **response_kw):
+ """
+ Get a connection from the pool and perform an HTTP request. This is the
+ lowest level call for making a request, so you'll need to specify all
+ the raw details.
+
+ .. note::
+
+ More commonly, it's appropriate to use a convenience method provided
+ by :class:`.RequestMethods`, such as :meth:`request`.
+
+ .. note::
+
+ `release_conn` will only behave as expected if
+ `preload_content=False` because we want to make
+ `preload_content=False` the default behaviour someday soon without
+ breaking backwards compatibility.
+
+ :param method:
+ HTTP request method (such as GET, POST, PUT, etc.)
+
+ :param body:
+ Data to send in the request body (useful for creating
+ POST requests, see HTTPConnectionPool.post_url for
+ more convenience).
+
+ :param headers:
+ Dictionary of custom headers to send, such as User-Agent,
+ If-None-Match, etc. If None, pool headers are used. If provided,
+ these headers completely replace any pool-specific headers.
+
+ :param retries:
+ Configure the number of retries to allow before raising a
+ :class:`~urllib3.exceptions.MaxRetryError` exception.
+
+ Pass ``None`` to retry until you receive a response. Pass a
+ :class:`~urllib3.util.retry.Retry` object for fine-grained control
+ over different types of retries.
+ Pass an integer number to retry connection errors that many times,
+ but no other types of errors. Pass zero to never retry.
+
+ If ``False``, then retries are disabled and any exception is raised
+ immediately. Also, instead of raising a MaxRetryError on redirects,
+ the redirect response will be returned.
+
+ :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
+
+ :param redirect:
+ If True, automatically handle redirects (status codes 301, 302,
+ 303, 307, 308). Each redirect counts as a retry. Disabling retries
+ will disable redirect, too.
+
+ :param assert_same_host:
+ If ``True``, will make sure that the host of the pool requests is
+ consistent else will raise HostChangedError. When False, you can
+ use the pool on an HTTP proxy and request foreign hosts.
+
+ :param timeout:
+ If specified, overrides the default timeout for this one
+ request. It may be a float (in seconds) or an instance of
+ :class:`urllib3.util.Timeout`.
+
+ :param pool_timeout:
+ If set and the pool is set to block=True, then this method will
+ block for ``pool_timeout`` seconds and raise EmptyPoolError if no
+ connection is available within the time period.
+
+ :param release_conn:
+ If False, then the urlopen call will not release the connection
+ back into the pool once a response is received (but will release if
+ you read the entire contents of the response such as when
+ `preload_content=True`). This is useful if you're not preloading
+ the response's content immediately. You will need to call
+ ``r.release_conn()`` on the response ``r`` to return the connection
+ back into the pool. If None, it takes the value of
+ ``response_kw.get('preload_content', True)``.
+
+ :param chunked:
+ If True, urllib3 will send the body using chunked transfer
+ encoding. Otherwise, urllib3 will send the body using the standard
+ content-length form. Defaults to False.
+
+ :param int body_pos:
+ Position to seek to in file-like body in the event of a retry or
+ redirect. Typically this won't need to be set because urllib3 will
+ auto-populate the value when needed.
+
+ :param \\**response_kw:
+ Additional parameters are passed to
+ :meth:`urllib3.response.HTTPResponse.from_httplib`
+ """
+ if headers is None:
+ headers = self.headers
+
+ if not isinstance(retries, Retry):
+ retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
+
+ if release_conn is None:
+ release_conn = response_kw.get('preload_content', True)
+
+ # Check host
+ if assert_same_host and not self.is_same_host(url):
+ raise HostChangedError(self, url, retries)
+
+ conn = None
+
+ # Track whether `conn` needs to be released before
+ # returning/raising/recursing. Update this variable if necessary, and
+ # leave `release_conn` constant throughout the function. That way, if
+ # the function recurses, the original value of `release_conn` will be
+ # passed down into the recursive call, and its value will be respected.
+ #
+ # See issue #651 [1] for details.
+ #
+ # [1] <https://github.com/shazow/urllib3/issues/651>
+ release_this_conn = release_conn
+
+ # Merge the proxy headers. Only do this in HTTP. We have to copy the
+ # headers dict so we can safely change it without those changes being
+ # reflected in anyone else's copy.
+ if self.scheme == 'http':
+ headers = headers.copy()
+ headers.update(self.proxy_headers)
+
+ # Must keep the exception bound to a separate variable or else Python 3
+ # complains about UnboundLocalError.
+ err = None
+
+ # Keep track of whether we cleanly exited the except block. This
+ # ensures we do proper cleanup in finally.
+ clean_exit = False
+
+ # Rewind body position, if needed. Record current position
+ # for future rewinds in the event of a redirect/retry.
+ body_pos = set_file_position(body, body_pos)
+
+ try:
+ # Request a connection from the queue.
+ timeout_obj = self._get_timeout(timeout)
+ conn = self._get_conn(timeout=pool_timeout)
+
+ conn.timeout = timeout_obj.connect_timeout
+
+ is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None)
+ if is_new_proxy_conn:
+ self._prepare_proxy(conn)
+
+ # Make the request on the httplib connection object.
+ httplib_response = self._make_request(conn, method, url,
+ timeout=timeout_obj,
+ body=body, headers=headers,
+ chunked=chunked)
+
+ # If we're going to release the connection in ``finally:``, then
+ # the response doesn't need to know about the connection. Otherwise
+ # it will also try to release it and we'll have a double-release
+ # mess.
+ response_conn = conn if not release_conn else None
+
+ # Pass method to Response for length checking
+ response_kw['request_method'] = method
+
+ # Import httplib's response into our own wrapper object
+ response = self.ResponseCls.from_httplib(httplib_response,
+ pool=self,
+ connection=response_conn,
+ retries=retries,
+ **response_kw)
+
+ # Everything went great!
+ clean_exit = True
+
+ except queue.Empty:
+ # Timed out by queue.
+ raise EmptyPoolError(self, "No pool connections are available.")
+
+ except (BaseSSLError, CertificateError) as e:
+ # Close the connection. If a connection is reused on which there
+ # was a Certificate error, the next request will certainly raise
+ # another Certificate error.
+ clean_exit = False
+ raise SSLError(e)
+
+ except SSLError:
+ # Treat SSLError separately from BaseSSLError to preserve
+ # traceback.
+ clean_exit = False
+ raise
+
+ except (TimeoutError, HTTPException, SocketError, ProtocolError) as e:
+ # Discard the connection for these exceptions. It will be
+ # be replaced during the next _get_conn() call.
+ clean_exit = False
+
+ if isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
+ e = ProxyError('Cannot connect to proxy.', e)
+ elif isinstance(e, (SocketError, HTTPException)):
+ e = ProtocolError('Connection aborted.', e)
+
+ retries = retries.increment(method, url, error=e, _pool=self,
+ _stacktrace=sys.exc_info()[2])
+ retries.sleep()
+
+ # Keep track of the error for the retry warning.
+ err = e
+
+ finally:
+ if not clean_exit:
+ # We hit some kind of exception, handled or otherwise. We need
+ # to throw the connection away unless explicitly told not to.
+ # Close the connection, set the variable to None, and make sure
+ # we put the None back in the pool to avoid leaking it.
+ conn = conn and conn.close()
+ release_this_conn = True
+
+ if release_this_conn:
+ # Put the connection back to be reused. If the connection is
+ # expired then it will be None, which will get replaced with a
+ # fresh connection during _get_conn.
+ self._put_conn(conn)
+
+ if not conn:
+ # Try again
+ log.warning("Retrying (%r) after connection "
+ "broken by '%r': %s", retries, err, url)
+ return self.urlopen(method, url, body, headers, retries,
+ redirect, assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn, body_pos=body_pos,
+ **response_kw)
+
+ # Handle redirect?
+ redirect_location = redirect and response.get_redirect_location()
+ if redirect_location:
+ if response.status == 303:
+ method = 'GET'
+
+ try:
+ retries = retries.increment(method, url, response=response, _pool=self)
+ except MaxRetryError:
+ if retries.raise_on_redirect:
+ # Release the connection for this response, since we're not
+ # returning it to be released manually.
+ response.release_conn()
+ raise
+ return response
+
+ retries.sleep_for_retry(response)
+ log.debug("Redirecting %s -> %s", url, redirect_location)
+ return self.urlopen(
+ method, redirect_location, body, headers,
+ retries=retries, redirect=redirect,
+ assert_same_host=assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn, body_pos=body_pos,
+ **response_kw)
+
+ # Check if we should retry the HTTP response.
+ has_retry_after = bool(response.getheader('Retry-After'))
+ if retries.is_retry(method, response.status, has_retry_after):
+ try:
+ retries = retries.increment(method, url, response=response, _pool=self)
+ except MaxRetryError:
+ if retries.raise_on_status:
+ # Release the connection for this response, since we're not
+ # returning it to be released manually.
+ response.release_conn()
+ raise
+ return response
+ retries.sleep(response)
+ log.debug("Retry: %s", url)
+ return self.urlopen(
+ method, url, body, headers,
+ retries=retries, redirect=redirect,
+ assert_same_host=assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn,
+ body_pos=body_pos, **response_kw)
+
+ return response
+
+
+class HTTPSConnectionPool(HTTPConnectionPool):
+ """
+ Same as :class:`.HTTPConnectionPool`, but HTTPS.
+
+ When Python is compiled with the :mod:`ssl` module, then
+ :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
+ instead of :class:`.HTTPSConnection`.
+
+ :class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``,
+ ``assert_hostname`` and ``host`` in this order to verify connections.
+ If ``assert_hostname`` is False, no verification is done.
+
+ The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
+ ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is
+ available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
+ the connection socket into an SSL socket.
+ """
+
+ scheme = 'https'
+ ConnectionCls = HTTPSConnection
+
+ def __init__(self, host, port=None,
+ strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1,
+ block=False, headers=None, retries=None,
+ _proxy=None, _proxy_headers=None,
+ key_file=None, cert_file=None, cert_reqs=None,
+ ca_certs=None, ssl_version=None,
+ assert_hostname=None, assert_fingerprint=None,
+ ca_cert_dir=None, **conn_kw):
+
+ HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize,
+ block, headers, retries, _proxy, _proxy_headers,
+ **conn_kw)
+
+ if ca_certs and cert_reqs is None:
+ cert_reqs = 'CERT_REQUIRED'
+
+ self.key_file = key_file
+ self.cert_file = cert_file
+ self.cert_reqs = cert_reqs
+ self.ca_certs = ca_certs
+ self.ca_cert_dir = ca_cert_dir
+ self.ssl_version = ssl_version
+ self.assert_hostname = assert_hostname
+ self.assert_fingerprint = assert_fingerprint
+
+ def _prepare_conn(self, conn):
+ """
+ Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
+ and establish the tunnel if proxy is used.
+ """
+
+ if isinstance(conn, VerifiedHTTPSConnection):
+ conn.set_cert(key_file=self.key_file,
+ cert_file=self.cert_file,
+ cert_reqs=self.cert_reqs,
+ ca_certs=self.ca_certs,
+ ca_cert_dir=self.ca_cert_dir,
+ assert_hostname=self.assert_hostname,
+ assert_fingerprint=self.assert_fingerprint)
+ conn.ssl_version = self.ssl_version
+ return conn
+
+ def _prepare_proxy(self, conn):
+ """
+ Establish tunnel connection early, because otherwise httplib
+ would improperly set Host: header to proxy's IP:port.
+ """
+ # Python 2.7+
+ try:
+ set_tunnel = conn.set_tunnel
+ except AttributeError: # Platform-specific: Python 2.6
+ set_tunnel = conn._set_tunnel
+
+ if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older
+ set_tunnel(self.host, self.port)
+ else:
+ set_tunnel(self.host, self.port, self.proxy_headers)
+
+ conn.connect()
+
+ def _new_conn(self):
+ """
+ Return a fresh :class:`httplib.HTTPSConnection`.
+ """
+ self.num_connections += 1
+ log.debug("Starting new HTTPS connection (%d): %s",
+ self.num_connections, self.host)
+
+ if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
+ raise SSLError("Can't connect to HTTPS URL because the SSL "
+ "module is not available.")
+
+ actual_host = self.host
+ actual_port = self.port
+ if self.proxy is not None:
+ actual_host = self.proxy.host
+ actual_port = self.proxy.port
+
+ conn = self.ConnectionCls(host=actual_host, port=actual_port,
+ timeout=self.timeout.connect_timeout,
+ strict=self.strict, **self.conn_kw)
+
+ return self._prepare_conn(conn)
+
+ def _validate_conn(self, conn):
+ """
+ Called right before a request is made, after the socket is created.
+ """
+ super(HTTPSConnectionPool, self)._validate_conn(conn)
+
+ # Force connect early to allow us to validate the connection.
+ if not getattr(conn, 'sock', None): # AppEngine might not have `.sock`
+ conn.connect()
+
+ if not conn.is_verified:
+ warnings.warn((
+ 'Unverified HTTPS request is being made. '
+ 'Adding certificate verification is strongly advised. See: '
+ 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
+ '#ssl-warnings'),
+ InsecureRequestWarning)
+
+
+def connection_from_url(url, **kw):
+ """
+ Given a url, return an :class:`.ConnectionPool` instance of its host.
+
+ This is a shortcut for not having to parse out the scheme, host, and port
+ of the url before creating an :class:`.ConnectionPool` instance.
+
+ :param url:
+ Absolute URL string that must include the scheme. Port is optional.
+
+ :param \\**kw:
+ Passes additional parameters to the constructor of the appropriate
+ :class:`.ConnectionPool`. Useful for specifying things like
+ timeout, maxsize, headers, etc.
+
+ Example::
+
+ >>> conn = connection_from_url('http://google.com/')
+ >>> r = conn.request('GET', '/')
+ """
+ scheme, host, port = get_host(url)
+ port = port or port_by_scheme.get(scheme, 80)
+ if scheme == 'https':
+ return HTTPSConnectionPool(host, port=port, **kw)
+ else:
+ return HTTPConnectionPool(host, port=port, **kw)
+
+
+def _ipv6_host(host):
+ """
+ Process IPv6 address literals
+ """
+
+ # httplib doesn't like it when we include brackets in IPv6 addresses
+ # Specifically, if we include brackets but also pass the port then
+ # httplib crazily doubles up the square brackets on the Host header.
+ # Instead, we need to make sure we never pass ``None`` as the port.
+ # However, for backward compatibility reasons we can't actually
+ # *assert* that. See http://bugs.python.org/issue28539
+ #
+ # Also if an IPv6 address literal has a zone identifier, the
+ # percent sign might be URIencoded, convert it back into ASCII
+ if host.startswith('[') and host.endswith(']'):
+ host = host.replace('%25', '%').strip('[]')
+ return host
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/__init__.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/__init__.py
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/__init__.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/__init__.py
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/bindings.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/bindings.py
new file mode 100644
index 0000000..bb82667
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/bindings.py
@@ -0,0 +1,591 @@
+# SPDX-License-Identifier: MIT
+"""
+This module uses ctypes to bind a whole bunch of functions and constants from
+SecureTransport. The goal here is to provide the low-level API to
+SecureTransport. These are essentially the C-level functions and constants, and
+they're pretty gross to work with.
+
+This code is a bastardised version of the code found in Will Bond's oscrypto
+library. An enormous debt is owed to him for blazing this trail for us. For
+that reason, this code should be considered to be covered both by urllib3's
+license and by oscrypto's:
+
+ Copyright (c) 2015-2016 Will Bond <will@wbond.net>
+
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.
+"""
+from __future__ import absolute_import
+
+import platform
+from ctypes.util import find_library
+from ctypes import (
+ c_void_p, c_int32, c_char_p, c_size_t, c_byte, c_uint32, c_ulong, c_long,
+ c_bool
+)
+from ctypes import CDLL, POINTER, CFUNCTYPE
+
+
+security_path = find_library('Security')
+if not security_path:
+ raise ImportError('The library Security could not be found')
+
+
+core_foundation_path = find_library('CoreFoundation')
+if not core_foundation_path:
+ raise ImportError('The library CoreFoundation could not be found')
+
+
+version = platform.mac_ver()[0]
+version_info = tuple(map(int, version.split('.')))
+if version_info < (10, 8):
+ raise OSError(
+ 'Only OS X 10.8 and newer are supported, not %s.%s' % (
+ version_info[0], version_info[1]
+ )
+ )
+
+Security = CDLL(security_path, use_errno=True)
+CoreFoundation = CDLL(core_foundation_path, use_errno=True)
+
+Boolean = c_bool
+CFIndex = c_long
+CFStringEncoding = c_uint32
+CFData = c_void_p
+CFString = c_void_p
+CFArray = c_void_p
+CFMutableArray = c_void_p
+CFDictionary = c_void_p
+CFError = c_void_p
+CFType = c_void_p
+CFTypeID = c_ulong
+
+CFTypeRef = POINTER(CFType)
+CFAllocatorRef = c_void_p
+
+OSStatus = c_int32
+
+CFDataRef = POINTER(CFData)
+CFStringRef = POINTER(CFString)
+CFArrayRef = POINTER(CFArray)
+CFMutableArrayRef = POINTER(CFMutableArray)
+CFDictionaryRef = POINTER(CFDictionary)
+CFArrayCallBacks = c_void_p
+CFDictionaryKeyCallBacks = c_void_p
+CFDictionaryValueCallBacks = c_void_p
+
+SecCertificateRef = POINTER(c_void_p)
+SecExternalFormat = c_uint32
+SecExternalItemType = c_uint32
+SecIdentityRef = POINTER(c_void_p)
+SecItemImportExportFlags = c_uint32
+SecItemImportExportKeyParameters = c_void_p
+SecKeychainRef = POINTER(c_void_p)
+SSLProtocol = c_uint32
+SSLCipherSuite = c_uint32
+SSLContextRef = POINTER(c_void_p)
+SecTrustRef = POINTER(c_void_p)
+SSLConnectionRef = c_uint32
+SecTrustResultType = c_uint32
+SecTrustOptionFlags = c_uint32
+SSLProtocolSide = c_uint32
+SSLConnectionType = c_uint32
+SSLSessionOption = c_uint32
+
+
+try:
+ Security.SecItemImport.argtypes = [
+ CFDataRef,
+ CFStringRef,
+ POINTER(SecExternalFormat),
+ POINTER(SecExternalItemType),
+ SecItemImportExportFlags,
+ POINTER(SecItemImportExportKeyParameters),
+ SecKeychainRef,
+ POINTER(CFArrayRef),
+ ]
+ Security.SecItemImport.restype = OSStatus
+
+ Security.SecCertificateGetTypeID.argtypes = []
+ Security.SecCertificateGetTypeID.restype = CFTypeID
+
+ Security.SecIdentityGetTypeID.argtypes = []
+ Security.SecIdentityGetTypeID.restype = CFTypeID
+
+ Security.SecKeyGetTypeID.argtypes = []
+ Security.SecKeyGetTypeID.restype = CFTypeID
+
+ Security.SecCertificateCreateWithData.argtypes = [
+ CFAllocatorRef,
+ CFDataRef
+ ]
+ Security.SecCertificateCreateWithData.restype = SecCertificateRef
+
+ Security.SecCertificateCopyData.argtypes = [
+ SecCertificateRef
+ ]
+ Security.SecCertificateCopyData.restype = CFDataRef
+
+ Security.SecCopyErrorMessageString.argtypes = [
+ OSStatus,
+ c_void_p
+ ]
+ Security.SecCopyErrorMessageString.restype = CFStringRef
+
+ Security.SecIdentityCreateWithCertificate.argtypes = [
+ CFTypeRef,
+ SecCertificateRef,
+ POINTER(SecIdentityRef)
+ ]
+ Security.SecIdentityCreateWithCertificate.restype = OSStatus
+
+ Security.SecKeychainCreate.argtypes = [
+ c_char_p,
+ c_uint32,
+ c_void_p,
+ Boolean,
+ c_void_p,
+ POINTER(SecKeychainRef)
+ ]
+ Security.SecKeychainCreate.restype = OSStatus
+
+ Security.SecKeychainDelete.argtypes = [
+ SecKeychainRef
+ ]
+ Security.SecKeychainDelete.restype = OSStatus
+
+ Security.SecPKCS12Import.argtypes = [
+ CFDataRef,
+ CFDictionaryRef,
+ POINTER(CFArrayRef)
+ ]
+ Security.SecPKCS12Import.restype = OSStatus
+
+ SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t))
+ SSLWriteFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t))
+
+ Security.SSLSetIOFuncs.argtypes = [
+ SSLContextRef,
+ SSLReadFunc,
+ SSLWriteFunc
+ ]
+ Security.SSLSetIOFuncs.restype = OSStatus
+
+ Security.SSLSetPeerID.argtypes = [
+ SSLContextRef,
+ c_char_p,
+ c_size_t
+ ]
+ Security.SSLSetPeerID.restype = OSStatus
+
+ Security.SSLSetCertificate.argtypes = [
+ SSLContextRef,
+ CFArrayRef
+ ]
+ Security.SSLSetCertificate.restype = OSStatus
+
+ Security.SSLSetCertificateAuthorities.argtypes = [
+ SSLContextRef,
+ CFTypeRef,
+ Boolean
+ ]
+ Security.SSLSetCertificateAuthorities.restype = OSStatus
+
+ Security.SSLSetConnection.argtypes = [
+ SSLContextRef,
+ SSLConnectionRef
+ ]
+ Security.SSLSetConnection.restype = OSStatus
+
+ Security.SSLSetPeerDomainName.argtypes = [
+ SSLContextRef,
+ c_char_p,
+ c_size_t
+ ]
+ Security.SSLSetPeerDomainName.restype = OSStatus
+
+ Security.SSLHandshake.argtypes = [
+ SSLContextRef
+ ]
+ Security.SSLHandshake.restype = OSStatus
+
+ Security.SSLRead.argtypes = [
+ SSLContextRef,
+ c_char_p,
+ c_size_t,
+ POINTER(c_size_t)
+ ]
+ Security.SSLRead.restype = OSStatus
+
+ Security.SSLWrite.argtypes = [
+ SSLContextRef,
+ c_char_p,
+ c_size_t,
+ POINTER(c_size_t)
+ ]
+ Security.SSLWrite.restype = OSStatus
+
+ Security.SSLClose.argtypes = [
+ SSLContextRef
+ ]
+ Security.SSLClose.restype = OSStatus
+
+ Security.SSLGetNumberSupportedCiphers.argtypes = [
+ SSLContextRef,
+ POINTER(c_size_t)
+ ]
+ Security.SSLGetNumberSupportedCiphers.restype = OSStatus
+
+ Security.SSLGetSupportedCiphers.argtypes = [
+ SSLContextRef,
+ POINTER(SSLCipherSuite),
+ POINTER(c_size_t)
+ ]
+ Security.SSLGetSupportedCiphers.restype = OSStatus
+
+ Security.SSLSetEnabledCiphers.argtypes = [
+ SSLContextRef,
+ POINTER(SSLCipherSuite),
+ c_size_t
+ ]
+ Security.SSLSetEnabledCiphers.restype = OSStatus
+
+ Security.SSLGetNumberEnabledCiphers.argtype = [
+ SSLContextRef,
+ POINTER(c_size_t)
+ ]
+ Security.SSLGetNumberEnabledCiphers.restype = OSStatus
+
+ Security.SSLGetEnabledCiphers.argtypes = [
+ SSLContextRef,
+ POINTER(SSLCipherSuite),
+ POINTER(c_size_t)
+ ]
+ Security.SSLGetEnabledCiphers.restype = OSStatus
+
+ Security.SSLGetNegotiatedCipher.argtypes = [
+ SSLContextRef,
+ POINTER(SSLCipherSuite)
+ ]
+ Security.SSLGetNegotiatedCipher.restype = OSStatus
+
+ Security.SSLGetNegotiatedProtocolVersion.argtypes = [
+ SSLContextRef,
+ POINTER(SSLProtocol)
+ ]
+ Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus
+
+ Security.SSLCopyPeerTrust.argtypes = [
+ SSLContextRef,
+ POINTER(SecTrustRef)
+ ]
+ Security.SSLCopyPeerTrust.restype = OSStatus
+
+ Security.SecTrustSetAnchorCertificates.argtypes = [
+ SecTrustRef,
+ CFArrayRef
+ ]
+ Security.SecTrustSetAnchorCertificates.restype = OSStatus
+
+ Security.SecTrustSetAnchorCertificatesOnly.argstypes = [
+ SecTrustRef,
+ Boolean
+ ]
+ Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus
+
+ Security.SecTrustEvaluate.argtypes = [
+ SecTrustRef,
+ POINTER(SecTrustResultType)
+ ]
+ Security.SecTrustEvaluate.restype = OSStatus
+
+ Security.SecTrustGetCertificateCount.argtypes = [
+ SecTrustRef
+ ]
+ Security.SecTrustGetCertificateCount.restype = CFIndex
+
+ Security.SecTrustGetCertificateAtIndex.argtypes = [
+ SecTrustRef,
+ CFIndex
+ ]
+ Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef
+
+ Security.SSLCreateContext.argtypes = [
+ CFAllocatorRef,
+ SSLProtocolSide,
+ SSLConnectionType
+ ]
+ Security.SSLCreateContext.restype = SSLContextRef
+
+ Security.SSLSetSessionOption.argtypes = [
+ SSLContextRef,
+ SSLSessionOption,
+ Boolean
+ ]
+ Security.SSLSetSessionOption.restype = OSStatus
+
+ Security.SSLSetProtocolVersionMin.argtypes = [
+ SSLContextRef,
+ SSLProtocol
+ ]
+ Security.SSLSetProtocolVersionMin.restype = OSStatus
+
+ Security.SSLSetProtocolVersionMax.argtypes = [
+ SSLContextRef,
+ SSLProtocol
+ ]
+ Security.SSLSetProtocolVersionMax.restype = OSStatus
+
+ Security.SecCopyErrorMessageString.argtypes = [
+ OSStatus,
+ c_void_p
+ ]
+ Security.SecCopyErrorMessageString.restype = CFStringRef
+
+ Security.SSLReadFunc = SSLReadFunc
+ Security.SSLWriteFunc = SSLWriteFunc
+ Security.SSLContextRef = SSLContextRef
+ Security.SSLProtocol = SSLProtocol
+ Security.SSLCipherSuite = SSLCipherSuite
+ Security.SecIdentityRef = SecIdentityRef
+ Security.SecKeychainRef = SecKeychainRef
+ Security.SecTrustRef = SecTrustRef
+ Security.SecTrustResultType = SecTrustResultType
+ Security.SecExternalFormat = SecExternalFormat
+ Security.OSStatus = OSStatus
+
+ Security.kSecImportExportPassphrase = CFStringRef.in_dll(
+ Security, 'kSecImportExportPassphrase'
+ )
+ Security.kSecImportItemIdentity = CFStringRef.in_dll(
+ Security, 'kSecImportItemIdentity'
+ )
+
+ # CoreFoundation time!
+ CoreFoundation.CFRetain.argtypes = [
+ CFTypeRef
+ ]
+ CoreFoundation.CFRetain.restype = CFTypeRef
+
+ CoreFoundation.CFRelease.argtypes = [
+ CFTypeRef
+ ]
+ CoreFoundation.CFRelease.restype = None
+
+ CoreFoundation.CFGetTypeID.argtypes = [
+ CFTypeRef
+ ]
+ CoreFoundation.CFGetTypeID.restype = CFTypeID
+
+ CoreFoundation.CFStringCreateWithCString.argtypes = [
+ CFAllocatorRef,
+ c_char_p,
+ CFStringEncoding
+ ]
+ CoreFoundation.CFStringCreateWithCString.restype = CFStringRef
+
+ CoreFoundation.CFStringGetCStringPtr.argtypes = [
+ CFStringRef,
+ CFStringEncoding
+ ]
+ CoreFoundation.CFStringGetCStringPtr.restype = c_char_p
+
+ CoreFoundation.CFStringGetCString.argtypes = [
+ CFStringRef,
+ c_char_p,
+ CFIndex,
+ CFStringEncoding
+ ]
+ CoreFoundation.CFStringGetCString.restype = c_bool
+
+ CoreFoundation.CFDataCreate.argtypes = [
+ CFAllocatorRef,
+ c_char_p,
+ CFIndex
+ ]
+ CoreFoundation.CFDataCreate.restype = CFDataRef
+
+ CoreFoundation.CFDataGetLength.argtypes = [
+ CFDataRef
+ ]
+ CoreFoundation.CFDataGetLength.restype = CFIndex
+
+ CoreFoundation.CFDataGetBytePtr.argtypes = [
+ CFDataRef
+ ]
+ CoreFoundation.CFDataGetBytePtr.restype = c_void_p
+
+ CoreFoundation.CFDictionaryCreate.argtypes = [
+ CFAllocatorRef,
+ POINTER(CFTypeRef),
+ POINTER(CFTypeRef),
+ CFIndex,
+ CFDictionaryKeyCallBacks,
+ CFDictionaryValueCallBacks
+ ]
+ CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef
+
+ CoreFoundation.CFDictionaryGetValue.argtypes = [
+ CFDictionaryRef,
+ CFTypeRef
+ ]
+ CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef
+
+ CoreFoundation.CFArrayCreate.argtypes = [
+ CFAllocatorRef,
+ POINTER(CFTypeRef),
+ CFIndex,
+ CFArrayCallBacks,
+ ]
+ CoreFoundation.CFArrayCreate.restype = CFArrayRef
+
+ CoreFoundation.CFArrayCreateMutable.argtypes = [
+ CFAllocatorRef,
+ CFIndex,
+ CFArrayCallBacks
+ ]
+ CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef
+
+ CoreFoundation.CFArrayAppendValue.argtypes = [
+ CFMutableArrayRef,
+ c_void_p
+ ]
+ CoreFoundation.CFArrayAppendValue.restype = None
+
+ CoreFoundation.CFArrayGetCount.argtypes = [
+ CFArrayRef
+ ]
+ CoreFoundation.CFArrayGetCount.restype = CFIndex
+
+ CoreFoundation.CFArrayGetValueAtIndex.argtypes = [
+ CFArrayRef,
+ CFIndex
+ ]
+ CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p
+
+ CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll(
+ CoreFoundation, 'kCFAllocatorDefault'
+ )
+ CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(CoreFoundation, 'kCFTypeArrayCallBacks')
+ CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll(
+ CoreFoundation, 'kCFTypeDictionaryKeyCallBacks'
+ )
+ CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll(
+ CoreFoundation, 'kCFTypeDictionaryValueCallBacks'
+ )
+
+ CoreFoundation.CFTypeRef = CFTypeRef
+ CoreFoundation.CFArrayRef = CFArrayRef
+ CoreFoundation.CFStringRef = CFStringRef
+ CoreFoundation.CFDictionaryRef = CFDictionaryRef
+
+except (AttributeError):
+ raise ImportError('Error initializing ctypes')
+
+
+class CFConst(object):
+ """
+ A class object that acts as essentially a namespace for CoreFoundation
+ constants.
+ """
+ kCFStringEncodingUTF8 = CFStringEncoding(0x08000100)
+
+
+class SecurityConst(object):
+ """
+ A class object that acts as essentially a namespace for Security constants.
+ """
+ kSSLSessionOptionBreakOnServerAuth = 0
+
+ kSSLProtocol2 = 1
+ kSSLProtocol3 = 2
+ kTLSProtocol1 = 4
+ kTLSProtocol11 = 7
+ kTLSProtocol12 = 8
+
+ kSSLClientSide = 1
+ kSSLStreamType = 0
+
+ kSecFormatPEMSequence = 10
+
+ kSecTrustResultInvalid = 0
+ kSecTrustResultProceed = 1
+ # This gap is present on purpose: this was kSecTrustResultConfirm, which
+ # is deprecated.
+ kSecTrustResultDeny = 3
+ kSecTrustResultUnspecified = 4
+ kSecTrustResultRecoverableTrustFailure = 5
+ kSecTrustResultFatalTrustFailure = 6
+ kSecTrustResultOtherError = 7
+
+ errSSLProtocol = -9800
+ errSSLWouldBlock = -9803
+ errSSLClosedGraceful = -9805
+ errSSLClosedNoNotify = -9816
+ errSSLClosedAbort = -9806
+
+ errSSLXCertChainInvalid = -9807
+ errSSLCrypto = -9809
+ errSSLInternal = -9810
+ errSSLCertExpired = -9814
+ errSSLCertNotYetValid = -9815
+ errSSLUnknownRootCert = -9812
+ errSSLNoRootCert = -9813
+ errSSLHostNameMismatch = -9843
+ errSSLPeerHandshakeFail = -9824
+ errSSLPeerUserCancelled = -9839
+ errSSLWeakPeerEphemeralDHKey = -9850
+ errSSLServerAuthCompleted = -9841
+ errSSLRecordOverflow = -9847
+
+ errSecVerifyFailed = -67808
+ errSecNoTrustSettings = -25263
+ errSecItemNotFound = -25300
+ errSecInvalidTrustSettings = -25262
+
+ # Cipher suites. We only pick the ones our default cipher string allows.
+ TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C
+ TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030
+ TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B
+ TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F
+ TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3
+ TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F
+ TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2
+ TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E
+ TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024
+ TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028
+ TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A
+ TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014
+ TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B
+ TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A
+ TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039
+ TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038
+ TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023
+ TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027
+ TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009
+ TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013
+ TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067
+ TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040
+ TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033
+ TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032
+ TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D
+ TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C
+ TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D
+ TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C
+ TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035
+ TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/low_level.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/low_level.py
new file mode 100644
index 0000000..0f79a13
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/_securetransport/low_level.py
@@ -0,0 +1,344 @@
+# SPDX-License-Identifier: MIT
+"""
+Low-level helpers for the SecureTransport bindings.
+
+These are Python functions that are not directly related to the high-level APIs
+but are necessary to get them to work. They include a whole bunch of low-level
+CoreFoundation messing about and memory management. The concerns in this module
+are almost entirely about trying to avoid memory leaks and providing
+appropriate and useful assistance to the higher-level code.
+"""
+import base64
+import ctypes
+import itertools
+import re
+import os
+import ssl
+import tempfile
+
+from .bindings import Security, CoreFoundation, CFConst
+
+
+# This regular expression is used to grab PEM data out of a PEM bundle.
+_PEM_CERTS_RE = re.compile(
+ b"-----BEGIN CERTIFICATE-----\n(.*?)\n-----END CERTIFICATE-----", re.DOTALL
+)
+
+
+def _cf_data_from_bytes(bytestring):
+ """
+ Given a bytestring, create a CFData object from it. This CFData object must
+ be CFReleased by the caller.
+ """
+ return CoreFoundation.CFDataCreate(
+ CoreFoundation.kCFAllocatorDefault, bytestring, len(bytestring)
+ )
+
+
+def _cf_dictionary_from_tuples(tuples):
+ """
+ Given a list of Python tuples, create an associated CFDictionary.
+ """
+ dictionary_size = len(tuples)
+
+ # We need to get the dictionary keys and values out in the same order.
+ keys = (t[0] for t in tuples)
+ values = (t[1] for t in tuples)
+ cf_keys = (CoreFoundation.CFTypeRef * dictionary_size)(*keys)
+ cf_values = (CoreFoundation.CFTypeRef * dictionary_size)(*values)
+
+ return CoreFoundation.CFDictionaryCreate(
+ CoreFoundation.kCFAllocatorDefault,
+ cf_keys,
+ cf_values,
+ dictionary_size,
+ CoreFoundation.kCFTypeDictionaryKeyCallBacks,
+ CoreFoundation.kCFTypeDictionaryValueCallBacks,
+ )
+
+
+def _cf_string_to_unicode(value):
+ """
+ Creates a Unicode string from a CFString object. Used entirely for error
+ reporting.
+
+ Yes, it annoys me quite a lot that this function is this complex.
+ """
+ value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p))
+
+ string = CoreFoundation.CFStringGetCStringPtr(
+ value_as_void_p,
+ CFConst.kCFStringEncodingUTF8
+ )
+ if string is None:
+ buffer = ctypes.create_string_buffer(1024)
+ result = CoreFoundation.CFStringGetCString(
+ value_as_void_p,
+ buffer,
+ 1024,
+ CFConst.kCFStringEncodingUTF8
+ )
+ if not result:
+ raise OSError('Error copying C string from CFStringRef')
+ string = buffer.value
+ if string is not None:
+ string = string.decode('utf-8')
+ return string
+
+
+def _assert_no_error(error, exception_class=None):
+ """
+ Checks the return code and throws an exception if there is an error to
+ report
+ """
+ if error == 0:
+ return
+
+ cf_error_string = Security.SecCopyErrorMessageString(error, None)
+ output = _cf_string_to_unicode(cf_error_string)
+ CoreFoundation.CFRelease(cf_error_string)
+
+ if output is None or output == u'':
+ output = u'OSStatus %s' % error
+
+ if exception_class is None:
+ exception_class = ssl.SSLError
+
+ raise exception_class(output)
+
+
+def _cert_array_from_pem(pem_bundle):
+ """
+ Given a bundle of certs in PEM format, turns them into a CFArray of certs
+ that can be used to validate a cert chain.
+ """
+ der_certs = [
+ base64.b64decode(match.group(1))
+ for match in _PEM_CERTS_RE.finditer(pem_bundle)
+ ]
+ if not der_certs:
+ raise ssl.SSLError("No root certificates specified")
+
+ cert_array = CoreFoundation.CFArrayCreateMutable(
+ CoreFoundation.kCFAllocatorDefault,
+ 0,
+ ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks)
+ )
+ if not cert_array:
+ raise ssl.SSLError("Unable to allocate memory!")
+
+ try:
+ for der_bytes in der_certs:
+ certdata = _cf_data_from_bytes(der_bytes)
+ if not certdata:
+ raise ssl.SSLError("Unable to allocate memory!")
+ cert = Security.SecCertificateCreateWithData(
+ CoreFoundation.kCFAllocatorDefault, certdata
+ )
+ CoreFoundation.CFRelease(certdata)
+ if not cert:
+ raise ssl.SSLError("Unable to build cert object!")
+
+ CoreFoundation.CFArrayAppendValue(cert_array, cert)
+ CoreFoundation.CFRelease(cert)
+ except Exception:
+ # We need to free the array before the exception bubbles further.
+ # We only want to do that if an error occurs: otherwise, the caller
+ # should free.
+ CoreFoundation.CFRelease(cert_array)
+
+ return cert_array
+
+
+def _is_cert(item):
+ """
+ Returns True if a given CFTypeRef is a certificate.
+ """
+ expected = Security.SecCertificateGetTypeID()
+ return CoreFoundation.CFGetTypeID(item) == expected
+
+
+def _is_identity(item):
+ """
+ Returns True if a given CFTypeRef is an identity.
+ """
+ expected = Security.SecIdentityGetTypeID()
+ return CoreFoundation.CFGetTypeID(item) == expected
+
+
+def _temporary_keychain():
+ """
+ This function creates a temporary Mac keychain that we can use to work with
+ credentials. This keychain uses a one-time password and a temporary file to
+ store the data. We expect to have one keychain per socket. The returned
+ SecKeychainRef must be freed by the caller, including calling
+ SecKeychainDelete.
+
+ Returns a tuple of the SecKeychainRef and the path to the temporary
+ directory that contains it.
+ """
+ # Unfortunately, SecKeychainCreate requires a path to a keychain. This
+ # means we cannot use mkstemp to use a generic temporary file. Instead,
+ # we're going to create a temporary directory and a filename to use there.
+ # This filename will be 8 random bytes expanded into base64. We also need
+ # some random bytes to password-protect the keychain we're creating, so we
+ # ask for 40 random bytes.
+ random_bytes = os.urandom(40)
+ filename = base64.b64encode(random_bytes[:8]).decode('utf-8')
+ password = base64.b64encode(random_bytes[8:]) # Must be valid UTF-8
+ tempdirectory = tempfile.mkdtemp()
+
+ keychain_path = os.path.join(tempdirectory, filename).encode('utf-8')
+
+ # We now want to create the keychain itself.
+ keychain = Security.SecKeychainRef()
+ status = Security.SecKeychainCreate(
+ keychain_path,
+ len(password),
+ password,
+ False,
+ None,
+ ctypes.byref(keychain)
+ )
+ _assert_no_error(status)
+
+ # Having created the keychain, we want to pass it off to the caller.
+ return keychain, tempdirectory
+
+
+def _load_items_from_file(keychain, path):
+ """
+ Given a single file, loads all the trust objects from it into arrays and
+ the keychain.
+ Returns a tuple of lists: the first list is a list of identities, the
+ second a list of certs.
+ """
+ certificates = []
+ identities = []
+ result_array = None
+
+ with open(path, 'rb') as f:
+ raw_filedata = f.read()
+
+ try:
+ filedata = CoreFoundation.CFDataCreate(
+ CoreFoundation.kCFAllocatorDefault,
+ raw_filedata,
+ len(raw_filedata)
+ )
+ result_array = CoreFoundation.CFArrayRef()
+ result = Security.SecItemImport(
+ filedata, # cert data
+ None, # Filename, leaving it out for now
+ None, # What the type of the file is, we don't care
+ None, # what's in the file, we don't care
+ 0, # import flags
+ None, # key params, can include passphrase in the future
+ keychain, # The keychain to insert into
+ ctypes.byref(result_array) # Results
+ )
+ _assert_no_error(result)
+
+ # A CFArray is not very useful to us as an intermediary
+ # representation, so we are going to extract the objects we want
+ # and then free the array. We don't need to keep hold of keys: the
+ # keychain already has them!
+ result_count = CoreFoundation.CFArrayGetCount(result_array)
+ for index in range(result_count):
+ item = CoreFoundation.CFArrayGetValueAtIndex(
+ result_array, index
+ )
+ item = ctypes.cast(item, CoreFoundation.CFTypeRef)
+
+ if _is_cert(item):
+ CoreFoundation.CFRetain(item)
+ certificates.append(item)
+ elif _is_identity(item):
+ CoreFoundation.CFRetain(item)
+ identities.append(item)
+ finally:
+ if result_array:
+ CoreFoundation.CFRelease(result_array)
+
+ CoreFoundation.CFRelease(filedata)
+
+ return (identities, certificates)
+
+
+def _load_client_cert_chain(keychain, *paths):
+ """
+ Load certificates and maybe keys from a number of files. Has the end goal
+ of returning a CFArray containing one SecIdentityRef, and then zero or more
+ SecCertificateRef objects, suitable for use as a client certificate trust
+ chain.
+ """
+ # Ok, the strategy.
+ #
+ # This relies on knowing that macOS will not give you a SecIdentityRef
+ # unless you have imported a key into a keychain. This is a somewhat
+ # artificial limitation of macOS (for example, it doesn't necessarily
+ # affect iOS), but there is nothing inside Security.framework that lets you
+ # get a SecIdentityRef without having a key in a keychain.
+ #
+ # So the policy here is we take all the files and iterate them in order.
+ # Each one will use SecItemImport to have one or more objects loaded from
+ # it. We will also point at a keychain that macOS can use to work with the
+ # private key.
+ #
+ # Once we have all the objects, we'll check what we actually have. If we
+ # already have a SecIdentityRef in hand, fab: we'll use that. Otherwise,
+ # we'll take the first certificate (which we assume to be our leaf) and
+ # ask the keychain to give us a SecIdentityRef with that cert's associated
+ # key.
+ #
+ # We'll then return a CFArray containing the trust chain: one
+ # SecIdentityRef and then zero-or-more SecCertificateRef objects. The
+ # responsibility for freeing this CFArray will be with the caller. This
+ # CFArray must remain alive for the entire connection, so in practice it
+ # will be stored with a single SSLSocket, along with the reference to the
+ # keychain.
+ certificates = []
+ identities = []
+
+ # Filter out bad paths.
+ paths = (path for path in paths if path)
+
+ try:
+ for file_path in paths:
+ new_identities, new_certs = _load_items_from_file(
+ keychain, file_path
+ )
+ identities.extend(new_identities)
+ certificates.extend(new_certs)
+
+ # Ok, we have everything. The question is: do we have an identity? If
+ # not, we want to grab one from the first cert we have.
+ if not identities:
+ new_identity = Security.SecIdentityRef()
+ status = Security.SecIdentityCreateWithCertificate(
+ keychain,
+ certificates[0],
+ ctypes.byref(new_identity)
+ )
+ _assert_no_error(status)
+ identities.append(new_identity)
+
+ # We now want to release the original certificate, as we no longer
+ # need it.
+ CoreFoundation.CFRelease(certificates.pop(0))
+
+ # We now need to build a new CFArray that holds the trust chain.
+ trust_chain = CoreFoundation.CFArrayCreateMutable(
+ CoreFoundation.kCFAllocatorDefault,
+ 0,
+ ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks),
+ )
+ for item in itertools.chain(identities, certificates):
+ # ArrayAppendValue does a CFRetain on the item. That's fine,
+ # because the finally block will release our other refs to them.
+ CoreFoundation.CFArrayAppendValue(trust_chain, item)
+
+ return trust_chain
+ finally:
+ for obj in itertools.chain(identities, certificates):
+ CoreFoundation.CFRelease(obj)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/appengine.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/appengine.py
new file mode 100644
index 0000000..e74589f
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/appengine.py
@@ -0,0 +1,297 @@
+# SPDX-License-Identifier: MIT
+"""
+This module provides a pool manager that uses Google App Engine's
+`URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_.
+
+Example usage::
+
+ from urllib3 import PoolManager
+ from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox
+
+ if is_appengine_sandbox():
+ # AppEngineManager uses AppEngine's URLFetch API behind the scenes
+ http = AppEngineManager()
+ else:
+ # PoolManager uses a socket-level API behind the scenes
+ http = PoolManager()
+
+ r = http.request('GET', 'https://google.com/')
+
+There are `limitations <https://cloud.google.com/appengine/docs/python/\
+urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be
+the best choice for your application. There are three options for using
+urllib3 on Google App Engine:
+
+1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is
+ cost-effective in many circumstances as long as your usage is within the
+ limitations.
+2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets.
+ Sockets also have `limitations and restrictions
+ <https://cloud.google.com/appengine/docs/python/sockets/\
+ #limitations-and-restrictions>`_ and have a lower free quota than URLFetch.
+ To use sockets, be sure to specify the following in your ``app.yaml``::
+
+ env_variables:
+ GAE_USE_SOCKETS_HTTPLIB : 'true'
+
+3. If you are using `App Engine Flexible
+<https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard
+:class:`PoolManager` without any configuration or special environment variables.
+"""
+
+from __future__ import absolute_import
+import logging
+import os
+import warnings
+from ..packages.six.moves.urllib.parse import urljoin
+
+from ..exceptions import (
+ HTTPError,
+ HTTPWarning,
+ MaxRetryError,
+ ProtocolError,
+ TimeoutError,
+ SSLError
+)
+
+from ..packages.six import BytesIO
+from ..request import RequestMethods
+from ..response import HTTPResponse
+from ..util.timeout import Timeout
+from ..util.retry import Retry
+
+try:
+ from google.appengine.api import urlfetch
+except ImportError:
+ urlfetch = None
+
+
+log = logging.getLogger(__name__)
+
+
+class AppEnginePlatformWarning(HTTPWarning):
+ pass
+
+
+class AppEnginePlatformError(HTTPError):
+ pass
+
+
+class AppEngineManager(RequestMethods):
+ """
+ Connection manager for Google App Engine sandbox applications.
+
+ This manager uses the URLFetch service directly instead of using the
+ emulated httplib, and is subject to URLFetch limitations as described in
+ the App Engine documentation `here
+ <https://cloud.google.com/appengine/docs/python/urlfetch>`_.
+
+ Notably it will raise an :class:`AppEnginePlatformError` if:
+ * URLFetch is not available.
+ * If you attempt to use this on App Engine Flexible, as full socket
+ support is available.
+ * If a request size is more than 10 megabytes.
+ * If a response size is more than 32 megabtyes.
+ * If you use an unsupported request method such as OPTIONS.
+
+ Beyond those cases, it will raise normal urllib3 errors.
+ """
+
+ def __init__(self, headers=None, retries=None, validate_certificate=True,
+ urlfetch_retries=True):
+ if not urlfetch:
+ raise AppEnginePlatformError(
+ "URLFetch is not available in this environment.")
+
+ if is_prod_appengine_mvms():
+ raise AppEnginePlatformError(
+ "Use normal urllib3.PoolManager instead of AppEngineManager"
+ "on Managed VMs, as using URLFetch is not necessary in "
+ "this environment.")
+
+ warnings.warn(
+ "urllib3 is using URLFetch on Google App Engine sandbox instead "
+ "of sockets. To use sockets directly instead of URLFetch see "
+ "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.",
+ AppEnginePlatformWarning)
+
+ RequestMethods.__init__(self, headers)
+ self.validate_certificate = validate_certificate
+ self.urlfetch_retries = urlfetch_retries
+
+ self.retries = retries or Retry.DEFAULT
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ # Return False to re-raise any potential exceptions
+ return False
+
+ def urlopen(self, method, url, body=None, headers=None,
+ retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT,
+ **response_kw):
+
+ retries = self._get_retries(retries, redirect)
+
+ try:
+ follow_redirects = (
+ redirect and
+ retries.redirect != 0 and
+ retries.total)
+ response = urlfetch.fetch(
+ url,
+ payload=body,
+ method=method,
+ headers=headers or {},
+ allow_truncated=False,
+ follow_redirects=self.urlfetch_retries and follow_redirects,
+ deadline=self._get_absolute_timeout(timeout),
+ validate_certificate=self.validate_certificate,
+ )
+ except urlfetch.DeadlineExceededError as e:
+ raise TimeoutError(self, e)
+
+ except urlfetch.InvalidURLError as e:
+ if 'too large' in str(e):
+ raise AppEnginePlatformError(
+ "URLFetch request too large, URLFetch only "
+ "supports requests up to 10mb in size.", e)
+ raise ProtocolError(e)
+
+ except urlfetch.DownloadError as e:
+ if 'Too many redirects' in str(e):
+ raise MaxRetryError(self, url, reason=e)
+ raise ProtocolError(e)
+
+ except urlfetch.ResponseTooLargeError as e:
+ raise AppEnginePlatformError(
+ "URLFetch response too large, URLFetch only supports"
+ "responses up to 32mb in size.", e)
+
+ except urlfetch.SSLCertificateError as e:
+ raise SSLError(e)
+
+ except urlfetch.InvalidMethodError as e:
+ raise AppEnginePlatformError(
+ "URLFetch does not support method: %s" % method, e)
+
+ http_response = self._urlfetch_response_to_http_response(
+ response, retries=retries, **response_kw)
+
+ # Handle redirect?
+ redirect_location = redirect and http_response.get_redirect_location()
+ if redirect_location:
+ # Check for redirect response
+ if (self.urlfetch_retries and retries.raise_on_redirect):
+ raise MaxRetryError(self, url, "too many redirects")
+ else:
+ if http_response.status == 303:
+ method = 'GET'
+
+ try:
+ retries = retries.increment(method, url, response=http_response, _pool=self)
+ except MaxRetryError:
+ if retries.raise_on_redirect:
+ raise MaxRetryError(self, url, "too many redirects")
+ return http_response
+
+ retries.sleep_for_retry(http_response)
+ log.debug("Redirecting %s -> %s", url, redirect_location)
+ redirect_url = urljoin(url, redirect_location)
+ return self.urlopen(
+ method, redirect_url, body, headers,
+ retries=retries, redirect=redirect,
+ timeout=timeout, **response_kw)
+
+ # Check if we should retry the HTTP response.
+ has_retry_after = bool(http_response.getheader('Retry-After'))
+ if retries.is_retry(method, http_response.status, has_retry_after):
+ retries = retries.increment(
+ method, url, response=http_response, _pool=self)
+ log.debug("Retry: %s", url)
+ retries.sleep(http_response)
+ return self.urlopen(
+ method, url,
+ body=body, headers=headers,
+ retries=retries, redirect=redirect,
+ timeout=timeout, **response_kw)
+
+ return http_response
+
+ def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw):
+
+ if is_prod_appengine():
+ # Production GAE handles deflate encoding automatically, but does
+ # not remove the encoding header.
+ content_encoding = urlfetch_resp.headers.get('content-encoding')
+
+ if content_encoding == 'deflate':
+ del urlfetch_resp.headers['content-encoding']
+
+ transfer_encoding = urlfetch_resp.headers.get('transfer-encoding')
+ # We have a full response's content,
+ # so let's make sure we don't report ourselves as chunked data.
+ if transfer_encoding == 'chunked':
+ encodings = transfer_encoding.split(",")
+ encodings.remove('chunked')
+ urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings)
+
+ return HTTPResponse(
+ # In order for decoding to work, we must present the content as
+ # a file-like object.
+ body=BytesIO(urlfetch_resp.content),
+ headers=urlfetch_resp.headers,
+ status=urlfetch_resp.status_code,
+ **response_kw
+ )
+
+ def _get_absolute_timeout(self, timeout):
+ if timeout is Timeout.DEFAULT_TIMEOUT:
+ return None # Defer to URLFetch's default.
+ if isinstance(timeout, Timeout):
+ if timeout._read is not None or timeout._connect is not None:
+ warnings.warn(
+ "URLFetch does not support granular timeout settings, "
+ "reverting to total or default URLFetch timeout.",
+ AppEnginePlatformWarning)
+ return timeout.total
+ return timeout
+
+ def _get_retries(self, retries, redirect):
+ if not isinstance(retries, Retry):
+ retries = Retry.from_int(
+ retries, redirect=redirect, default=self.retries)
+
+ if retries.connect or retries.read or retries.redirect:
+ warnings.warn(
+ "URLFetch only supports total retries and does not "
+ "recognize connect, read, or redirect retry parameters.",
+ AppEnginePlatformWarning)
+
+ return retries
+
+
+def is_appengine():
+ return (is_local_appengine() or
+ is_prod_appengine() or
+ is_prod_appengine_mvms())
+
+
+def is_appengine_sandbox():
+ return is_appengine() and not is_prod_appengine_mvms()
+
+
+def is_local_appengine():
+ return ('APPENGINE_RUNTIME' in os.environ and
+ 'Development/' in os.environ['SERVER_SOFTWARE'])
+
+
+def is_prod_appengine():
+ return ('APPENGINE_RUNTIME' in os.environ and
+ 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and
+ not is_prod_appengine_mvms())
+
+
+def is_prod_appengine_mvms():
+ return os.environ.get('GAE_VM', False) == 'true'
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/ntlmpool.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/ntlmpool.py
new file mode 100644
index 0000000..3f8c9eb
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/ntlmpool.py
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: MIT
+"""
+NTLM authenticating pool, contributed by erikcederstran
+
+Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10
+"""
+from __future__ import absolute_import
+
+from logging import getLogger
+from ntlm import ntlm
+
+from .. import HTTPSConnectionPool
+from ..packages.six.moves.http_client import HTTPSConnection
+
+
+log = getLogger(__name__)
+
+
+class NTLMConnectionPool(HTTPSConnectionPool):
+ """
+ Implements an NTLM authentication version of an urllib3 connection pool
+ """
+
+ scheme = 'https'
+
+ def __init__(self, user, pw, authurl, *args, **kwargs):
+ """
+ authurl is a random URL on the server that is protected by NTLM.
+ user is the Windows user, probably in the DOMAIN\\username format.
+ pw is the password for the user.
+ """
+ super(NTLMConnectionPool, self).__init__(*args, **kwargs)
+ self.authurl = authurl
+ self.rawuser = user
+ user_parts = user.split('\\', 1)
+ self.domain = user_parts[0].upper()
+ self.user = user_parts[1]
+ self.pw = pw
+
+ def _new_conn(self):
+ # Performs the NTLM handshake that secures the connection. The socket
+ # must be kept open while requests are performed.
+ self.num_connections += 1
+ log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s',
+ self.num_connections, self.host, self.authurl)
+
+ headers = {}
+ headers['Connection'] = 'Keep-Alive'
+ req_header = 'Authorization'
+ resp_header = 'www-authenticate'
+
+ conn = HTTPSConnection(host=self.host, port=self.port)
+
+ # Send negotiation message
+ headers[req_header] = (
+ 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
+ log.debug('Request headers: %s', headers)
+ conn.request('GET', self.authurl, None, headers)
+ res = conn.getresponse()
+ reshdr = dict(res.getheaders())
+ log.debug('Response status: %s %s', res.status, res.reason)
+ log.debug('Response headers: %s', reshdr)
+ log.debug('Response data: %s [...]', res.read(100))
+
+ # Remove the reference to the socket, so that it can not be closed by
+ # the response object (we want to keep the socket open)
+ res.fp = None
+
+ # Server should respond with a challenge message
+ auth_header_values = reshdr[resp_header].split(', ')
+ auth_header_value = None
+ for s in auth_header_values:
+ if s[:5] == 'NTLM ':
+ auth_header_value = s[5:]
+ if auth_header_value is None:
+ raise Exception('Unexpected %s response header: %s' %
+ (resp_header, reshdr[resp_header]))
+
+ # Send authentication message
+ ServerChallenge, NegotiateFlags = \
+ ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
+ auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge,
+ self.user,
+ self.domain,
+ self.pw,
+ NegotiateFlags)
+ headers[req_header] = 'NTLM %s' % auth_msg
+ log.debug('Request headers: %s', headers)
+ conn.request('GET', self.authurl, None, headers)
+ res = conn.getresponse()
+ log.debug('Response status: %s %s', res.status, res.reason)
+ log.debug('Response headers: %s', dict(res.getheaders()))
+ log.debug('Response data: %s [...]', res.read()[:100])
+ if res.status != 200:
+ if res.status == 401:
+ raise Exception('Server rejected request: wrong '
+ 'username or password')
+ raise Exception('Wrong server response: %s %s' %
+ (res.status, res.reason))
+
+ res.fp = None
+ log.debug('Connection established')
+ return conn
+
+ def urlopen(self, method, url, body=None, headers=None, retries=3,
+ redirect=True, assert_same_host=True):
+ if headers is None:
+ headers = {}
+ headers['Connection'] = 'Keep-Alive'
+ return super(NTLMConnectionPool, self).urlopen(method, url, body,
+ headers, retries,
+ redirect,
+ assert_same_host)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/pyopenssl.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/pyopenssl.py
new file mode 100644
index 0000000..8d37350
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/pyopenssl.py
@@ -0,0 +1,458 @@
+# SPDX-License-Identifier: MIT
+"""
+SSL with SNI_-support for Python 2. Follow these instructions if you would
+like to verify SSL certificates in Python 2. Note, the default libraries do
+*not* do certificate checking; you need to do additional work to validate
+certificates yourself.
+
+This needs the following packages installed:
+
+* pyOpenSSL (tested with 16.0.0)
+* cryptography (minimum 1.3.4, from pyopenssl)
+* idna (minimum 2.0, from cryptography)
+
+However, pyopenssl depends on cryptography, which depends on idna, so while we
+use all three directly here we end up having relatively few packages required.
+
+You can install them with the following command:
+
+ pip install pyopenssl cryptography idna
+
+To activate certificate checking, call
+:func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code
+before you begin making HTTP requests. This can be done in a ``sitecustomize``
+module, or at any other time before your application begins using ``urllib3``,
+like this::
+
+ try:
+ import urllib3.contrib.pyopenssl
+ urllib3.contrib.pyopenssl.inject_into_urllib3()
+ except ImportError:
+ pass
+
+Now you can use :mod:`urllib3` as you normally would, and it will support SNI
+when the required modules are installed.
+
+Activating this module also has the positive side effect of disabling SSL/TLS
+compression in Python 2 (see `CRIME attack`_).
+
+If you want to configure the default list of supported cipher suites, you can
+set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
+
+.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
+.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
+"""
+from __future__ import absolute_import
+
+import OpenSSL.SSL
+from cryptography import x509
+from cryptography.hazmat.backends.openssl import backend as openssl_backend
+from cryptography.hazmat.backends.openssl.x509 import _Certificate
+
+from socket import timeout, error as SocketError
+from io import BytesIO
+
+try: # Platform-specific: Python 2
+ from socket import _fileobject
+except ImportError: # Platform-specific: Python 3
+ _fileobject = None
+ from ..packages.backports.makefile import backport_makefile
+
+import logging
+import ssl
+
+try:
+ import six
+except ImportError:
+ from ..packages import six
+
+import sys
+
+from .. import util
+
+__all__ = ['inject_into_urllib3', 'extract_from_urllib3']
+
+# SNI always works.
+HAS_SNI = True
+
+# Map from urllib3 to PyOpenSSL compatible parameter-values.
+_openssl_versions = {
+ ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD,
+ ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
+}
+
+if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'):
+ _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
+
+if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'):
+ _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
+
+try:
+ _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD})
+except AttributeError:
+ pass
+
+_stdlib_to_openssl_verify = {
+ ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
+ ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
+ ssl.CERT_REQUIRED:
+ OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
+}
+_openssl_to_stdlib_verify = dict(
+ (v, k) for k, v in _stdlib_to_openssl_verify.items()
+)
+
+# OpenSSL will only write 16K at a time
+SSL_WRITE_BLOCKSIZE = 16384
+
+orig_util_HAS_SNI = util.HAS_SNI
+orig_util_SSLContext = util.ssl_.SSLContext
+
+
+log = logging.getLogger(__name__)
+
+
+def inject_into_urllib3():
+ 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'
+
+ _validate_dependencies_met()
+
+ util.ssl_.SSLContext = PyOpenSSLContext
+ util.HAS_SNI = HAS_SNI
+ util.ssl_.HAS_SNI = HAS_SNI
+ util.IS_PYOPENSSL = True
+ util.ssl_.IS_PYOPENSSL = True
+
+
+def extract_from_urllib3():
+ 'Undo monkey-patching by :func:`inject_into_urllib3`.'
+
+ util.ssl_.SSLContext = orig_util_SSLContext
+ util.HAS_SNI = orig_util_HAS_SNI
+ util.ssl_.HAS_SNI = orig_util_HAS_SNI
+ util.IS_PYOPENSSL = False
+ util.ssl_.IS_PYOPENSSL = False
+
+
+def _validate_dependencies_met():
+ """
+ Verifies that PyOpenSSL's package-level dependencies have been met.
+ Throws `ImportError` if they are not met.
+ """
+ # Method added in `cryptography==1.1`; not available in older versions
+ from cryptography.x509.extensions import Extensions
+ if getattr(Extensions, "get_extension_for_class", None) is None:
+ raise ImportError("'cryptography' module missing required functionality. "
+ "Try upgrading to v1.3.4 or newer.")
+
+ # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
+ # attribute is only present on those versions.
+ from OpenSSL.crypto import X509
+ x509 = X509()
+ if getattr(x509, "_x509", None) is None:
+ raise ImportError("'pyOpenSSL' module missing required functionality. "
+ "Try upgrading to v0.14 or newer.")
+
+
+def _dnsname_to_stdlib(name):
+ """
+ Converts a dNSName SubjectAlternativeName field to the form used by the
+ standard library on the given Python version.
+
+ Cryptography produces a dNSName as a unicode string that was idna-decoded
+ from ASCII bytes. We need to idna-encode that string to get it back, and
+ then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib
+ uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8).
+ """
+ def idna_encode(name):
+ """
+ Borrowed wholesale from the Python Cryptography Project. It turns out
+ that we can't just safely call `idna.encode`: it can explode for
+ wildcard names. This avoids that problem.
+ """
+ import idna
+
+ for prefix in [u'*.', u'.']:
+ if name.startswith(prefix):
+ name = name[len(prefix):]
+ return prefix.encode('ascii') + idna.encode(name)
+ return idna.encode(name)
+
+ name = idna_encode(name)
+ if sys.version_info >= (3, 0):
+ name = name.decode('utf-8')
+ return name
+
+
+def get_subj_alt_name(peer_cert):
+ """
+ Given an PyOpenSSL certificate, provides all the subject alternative names.
+ """
+ # Pass the cert to cryptography, which has much better APIs for this.
+ # This is technically using private APIs, but should work across all
+ # relevant versions until PyOpenSSL gets something proper for this.
+ cert = _Certificate(openssl_backend, peer_cert._x509)
+
+ # We want to find the SAN extension. Ask Cryptography to locate it (it's
+ # faster than looping in Python)
+ try:
+ ext = cert.extensions.get_extension_for_class(
+ x509.SubjectAlternativeName
+ ).value
+ except x509.ExtensionNotFound:
+ # No such extension, return the empty list.
+ return []
+ except (x509.DuplicateExtension, x509.UnsupportedExtension,
+ x509.UnsupportedGeneralNameType, UnicodeError) as e:
+ # A problem has been found with the quality of the certificate. Assume
+ # no SAN field is present.
+ log.warning(
+ "A problem was encountered with the certificate that prevented "
+ "urllib3 from finding the SubjectAlternativeName field. This can "
+ "affect certificate validation. The error was %s",
+ e,
+ )
+ return []
+
+ # We want to return dNSName and iPAddress fields. We need to cast the IPs
+ # back to strings because the match_hostname function wants them as
+ # strings.
+ # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8
+ # decoded. This is pretty frustrating, but that's what the standard library
+ # does with certificates, and so we need to attempt to do the same.
+ names = [
+ ('DNS', _dnsname_to_stdlib(name))
+ for name in ext.get_values_for_type(x509.DNSName)
+ ]
+ names.extend(
+ ('IP Address', str(name))
+ for name in ext.get_values_for_type(x509.IPAddress)
+ )
+
+ return names
+
+
+class WrappedSocket(object):
+ '''API-compatibility wrapper for Python OpenSSL's Connection-class.
+
+ Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
+ collector of pypy.
+ '''
+
+ def __init__(self, connection, socket, suppress_ragged_eofs=True):
+ self.connection = connection
+ self.socket = socket
+ self.suppress_ragged_eofs = suppress_ragged_eofs
+ self._makefile_refs = 0
+ self._closed = False
+
+ def fileno(self):
+ return self.socket.fileno()
+
+ # Copy-pasted from Python 3.5 source code
+ def _decref_socketios(self):
+ if self._makefile_refs > 0:
+ self._makefile_refs -= 1
+ if self._closed:
+ self.close()
+
+ def recv(self, *args, **kwargs):
+ try:
+ data = self.connection.recv(*args, **kwargs)
+ except OpenSSL.SSL.SysCallError as e:
+ if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
+ return b''
+ else:
+ raise SocketError(str(e))
+ except OpenSSL.SSL.ZeroReturnError as e:
+ if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
+ return b''
+ else:
+ raise
+ except OpenSSL.SSL.WantReadError:
+ rd = util.wait_for_read(self.socket, self.socket.gettimeout())
+ if not rd:
+ raise timeout('The read operation timed out')
+ else:
+ return self.recv(*args, **kwargs)
+ else:
+ return data
+
+ def recv_into(self, *args, **kwargs):
+ try:
+ return self.connection.recv_into(*args, **kwargs)
+ except OpenSSL.SSL.SysCallError as e:
+ if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
+ return 0
+ else:
+ raise SocketError(str(e))
+ except OpenSSL.SSL.ZeroReturnError as e:
+ if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
+ return 0
+ else:
+ raise
+ except OpenSSL.SSL.WantReadError:
+ rd = util.wait_for_read(self.socket, self.socket.gettimeout())
+ if not rd:
+ raise timeout('The read operation timed out')
+ else:
+ return self.recv_into(*args, **kwargs)
+
+ def settimeout(self, timeout):
+ return self.socket.settimeout(timeout)
+
+ def _send_until_done(self, data):
+ while True:
+ try:
+ return self.connection.send(data)
+ except OpenSSL.SSL.WantWriteError:
+ wr = util.wait_for_write(self.socket, self.socket.gettimeout())
+ if not wr:
+ raise timeout()
+ continue
+ except OpenSSL.SSL.SysCallError as e:
+ raise SocketError(str(e))
+
+ def sendall(self, data):
+ total_sent = 0
+ while total_sent < len(data):
+ sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
+ total_sent += sent
+
+ def shutdown(self):
+ # FIXME rethrow compatible exceptions should we ever use this
+ self.connection.shutdown()
+
+ def close(self):
+ if self._makefile_refs < 1:
+ try:
+ self._closed = True
+ return self.connection.close()
+ except OpenSSL.SSL.Error:
+ return
+ else:
+ self._makefile_refs -= 1
+
+ def getpeercert(self, binary_form=False):
+ x509 = self.connection.get_peer_certificate()
+
+ if not x509:
+ return x509
+
+ if binary_form:
+ return OpenSSL.crypto.dump_certificate(
+ OpenSSL.crypto.FILETYPE_ASN1,
+ x509)
+
+ return {
+ 'subject': (
+ (('commonName', x509.get_subject().CN),),
+ ),
+ 'subjectAltName': get_subj_alt_name(x509)
+ }
+
+ def _reuse(self):
+ self._makefile_refs += 1
+
+ def _drop(self):
+ if self._makefile_refs < 1:
+ self.close()
+ else:
+ self._makefile_refs -= 1
+
+
+if _fileobject: # Platform-specific: Python 2
+ def makefile(self, mode, bufsize=-1):
+ self._makefile_refs += 1
+ return _fileobject(self, mode, bufsize, close=True)
+else: # Platform-specific: Python 3
+ makefile = backport_makefile
+
+WrappedSocket.makefile = makefile
+
+
+class PyOpenSSLContext(object):
+ """
+ I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
+ for translating the interface of the standard library ``SSLContext`` object
+ to calls into PyOpenSSL.
+ """
+ def __init__(self, protocol):
+ self.protocol = _openssl_versions[protocol]
+ self._ctx = OpenSSL.SSL.Context(self.protocol)
+ self._options = 0
+ self.check_hostname = False
+
+ @property
+ def options(self):
+ return self._options
+
+ @options.setter
+ def options(self, value):
+ self._options = value
+ self._ctx.set_options(value)
+
+ @property
+ def verify_mode(self):
+ return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
+
+ @verify_mode.setter
+ def verify_mode(self, value):
+ self._ctx.set_verify(
+ _stdlib_to_openssl_verify[value],
+ _verify_callback
+ )
+
+ def set_default_verify_paths(self):
+ self._ctx.set_default_verify_paths()
+
+ def set_ciphers(self, ciphers):
+ if isinstance(ciphers, six.text_type):
+ ciphers = ciphers.encode('utf-8')
+ self._ctx.set_cipher_list(ciphers)
+
+ def load_verify_locations(self, cafile=None, capath=None, cadata=None):
+ if cafile is not None:
+ cafile = cafile.encode('utf-8')
+ if capath is not None:
+ capath = capath.encode('utf-8')
+ self._ctx.load_verify_locations(cafile, capath)
+ if cadata is not None:
+ self._ctx.load_verify_locations(BytesIO(cadata))
+
+ def load_cert_chain(self, certfile, keyfile=None, password=None):
+ self._ctx.use_certificate_file(certfile)
+ if password is not None:
+ self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password)
+ self._ctx.use_privatekey_file(keyfile or certfile)
+
+ def wrap_socket(self, sock, server_side=False,
+ do_handshake_on_connect=True, suppress_ragged_eofs=True,
+ server_hostname=None):
+ cnx = OpenSSL.SSL.Connection(self._ctx, sock)
+
+ if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3
+ server_hostname = server_hostname.encode('utf-8')
+
+ if server_hostname is not None:
+ cnx.set_tlsext_host_name(server_hostname)
+
+ cnx.set_connect_state()
+
+ while True:
+ try:
+ cnx.do_handshake()
+ except OpenSSL.SSL.WantReadError:
+ rd = util.wait_for_read(sock, sock.gettimeout())
+ if not rd:
+ raise timeout('select timed out')
+ continue
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError('bad handshake: %r' % e)
+ break
+
+ return WrappedSocket(cnx, sock)
+
+
+def _verify_callback(cnx, x509, err_no, err_depth, return_code):
+ return err_no == 0
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/securetransport.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/securetransport.py
new file mode 100644
index 0000000..fcc3011
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/securetransport.py
@@ -0,0 +1,808 @@
+# SPDX-License-Identifier: MIT
+"""
+SecureTranport support for urllib3 via ctypes.
+
+This makes platform-native TLS available to urllib3 users on macOS without the
+use of a compiler. This is an important feature because the Python Package
+Index is moving to become a TLSv1.2-or-higher server, and the default OpenSSL
+that ships with macOS is not capable of doing TLSv1.2. The only way to resolve
+this is to give macOS users an alternative solution to the problem, and that
+solution is to use SecureTransport.
+
+We use ctypes here because this solution must not require a compiler. That's
+because pip is not allowed to require a compiler either.
+
+This is not intended to be a seriously long-term solution to this problem.
+The hope is that PEP 543 will eventually solve this issue for us, at which
+point we can retire this contrib module. But in the short term, we need to
+solve the impending tire fire that is Python on Mac without this kind of
+contrib module. So...here we are.
+
+To use this module, simply import and inject it::
+
+ import urllib3.contrib.securetransport
+ urllib3.contrib.securetransport.inject_into_urllib3()
+
+Happy TLSing!
+"""
+from __future__ import absolute_import
+
+import contextlib
+import ctypes
+import errno
+import os.path
+import shutil
+import socket
+import ssl
+import threading
+import weakref
+
+from .. import util
+from ._securetransport.bindings import (
+ Security, SecurityConst, CoreFoundation
+)
+from ._securetransport.low_level import (
+ _assert_no_error, _cert_array_from_pem, _temporary_keychain,
+ _load_client_cert_chain
+)
+
+try: # Platform-specific: Python 2
+ from socket import _fileobject
+except ImportError: # Platform-specific: Python 3
+ _fileobject = None
+ from ..packages.backports.makefile import backport_makefile
+
+try:
+ memoryview(b'')
+except NameError:
+ raise ImportError("SecureTransport only works on Pythons with memoryview")
+
+__all__ = ['inject_into_urllib3', 'extract_from_urllib3']
+
+# SNI always works
+HAS_SNI = True
+
+orig_util_HAS_SNI = util.HAS_SNI
+orig_util_SSLContext = util.ssl_.SSLContext
+
+# This dictionary is used by the read callback to obtain a handle to the
+# calling wrapped socket. This is a pretty silly approach, but for now it'll
+# do. I feel like I should be able to smuggle a handle to the wrapped socket
+# directly in the SSLConnectionRef, but for now this approach will work I
+# guess.
+#
+# We need to lock around this structure for inserts, but we don't do it for
+# reads/writes in the callbacks. The reasoning here goes as follows:
+#
+# 1. It is not possible to call into the callbacks before the dictionary is
+# populated, so once in the callback the id must be in the dictionary.
+# 2. The callbacks don't mutate the dictionary, they only read from it, and
+# so cannot conflict with any of the insertions.
+#
+# This is good: if we had to lock in the callbacks we'd drastically slow down
+# the performance of this code.
+_connection_refs = weakref.WeakValueDictionary()
+_connection_ref_lock = threading.Lock()
+
+# Limit writes to 16kB. This is OpenSSL's limit, but we'll cargo-cult it over
+# for no better reason than we need *a* limit, and this one is right there.
+SSL_WRITE_BLOCKSIZE = 16384
+
+# This is our equivalent of util.ssl_.DEFAULT_CIPHERS, but expanded out to
+# individual cipher suites. We need to do this becuase this is how
+# SecureTransport wants them.
+CIPHER_SUITES = [
+ SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+ SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+ SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+ SecurityConst.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384,
+ SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,
+ SecurityConst.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256,
+ SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,
+ SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384,
+ SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,
+ SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+ SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+ SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,
+ SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256,
+ SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
+ SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA,
+ SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
+ SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
+ SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+ SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+ SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,
+ SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256,
+ SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA,
+ SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA,
+ SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384,
+ SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256,
+ SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256,
+ SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256,
+ SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA,
+ SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA,
+]
+
+# Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of
+# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version.
+_protocol_to_min_max = {
+ ssl.PROTOCOL_SSLv23: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12),
+}
+
+if hasattr(ssl, "PROTOCOL_SSLv2"):
+ _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = (
+ SecurityConst.kSSLProtocol2, SecurityConst.kSSLProtocol2
+ )
+if hasattr(ssl, "PROTOCOL_SSLv3"):
+ _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = (
+ SecurityConst.kSSLProtocol3, SecurityConst.kSSLProtocol3
+ )
+if hasattr(ssl, "PROTOCOL_TLSv1"):
+ _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = (
+ SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol1
+ )
+if hasattr(ssl, "PROTOCOL_TLSv1_1"):
+ _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = (
+ SecurityConst.kTLSProtocol11, SecurityConst.kTLSProtocol11
+ )
+if hasattr(ssl, "PROTOCOL_TLSv1_2"):
+ _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = (
+ SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12
+ )
+if hasattr(ssl, "PROTOCOL_TLS"):
+ _protocol_to_min_max[ssl.PROTOCOL_TLS] = _protocol_to_min_max[ssl.PROTOCOL_SSLv23]
+
+
+def inject_into_urllib3():
+ """
+ Monkey-patch urllib3 with SecureTransport-backed SSL-support.
+ """
+ util.ssl_.SSLContext = SecureTransportContext
+ util.HAS_SNI = HAS_SNI
+ util.ssl_.HAS_SNI = HAS_SNI
+ util.IS_SECURETRANSPORT = True
+ util.ssl_.IS_SECURETRANSPORT = True
+
+
+def extract_from_urllib3():
+ """
+ Undo monkey-patching by :func:`inject_into_urllib3`.
+ """
+ util.ssl_.SSLContext = orig_util_SSLContext
+ util.HAS_SNI = orig_util_HAS_SNI
+ util.ssl_.HAS_SNI = orig_util_HAS_SNI
+ util.IS_SECURETRANSPORT = False
+ util.ssl_.IS_SECURETRANSPORT = False
+
+
+def _read_callback(connection_id, data_buffer, data_length_pointer):
+ """
+ SecureTransport read callback. This is called by ST to request that data
+ be returned from the socket.
+ """
+ wrapped_socket = None
+ try:
+ wrapped_socket = _connection_refs.get(connection_id)
+ if wrapped_socket is None:
+ return SecurityConst.errSSLInternal
+ base_socket = wrapped_socket.socket
+
+ requested_length = data_length_pointer[0]
+
+ timeout = wrapped_socket.gettimeout()
+ error = None
+ read_count = 0
+ buffer = (ctypes.c_char * requested_length).from_address(data_buffer)
+ buffer_view = memoryview(buffer)
+
+ try:
+ while read_count < requested_length:
+ if timeout is None or timeout >= 0:
+ readables = util.wait_for_read([base_socket], timeout)
+ if not readables:
+ raise socket.error(errno.EAGAIN, 'timed out')
+
+ # We need to tell ctypes that we have a buffer that can be
+ # written to. Upsettingly, we do that like this:
+ chunk_size = base_socket.recv_into(
+ buffer_view[read_count:requested_length]
+ )
+ read_count += chunk_size
+ if not chunk_size:
+ if not read_count:
+ return SecurityConst.errSSLClosedGraceful
+ break
+ except (socket.error) as e:
+ error = e.errno
+
+ if error is not None and error != errno.EAGAIN:
+ if error == errno.ECONNRESET:
+ return SecurityConst.errSSLClosedAbort
+ raise
+
+ data_length_pointer[0] = read_count
+
+ if read_count != requested_length:
+ return SecurityConst.errSSLWouldBlock
+
+ return 0
+ except Exception as e:
+ if wrapped_socket is not None:
+ wrapped_socket._exception = e
+ return SecurityConst.errSSLInternal
+
+
+def _write_callback(connection_id, data_buffer, data_length_pointer):
+ """
+ SecureTransport write callback. This is called by ST to request that data
+ actually be sent on the network.
+ """
+ wrapped_socket = None
+ try:
+ wrapped_socket = _connection_refs.get(connection_id)
+ if wrapped_socket is None:
+ return SecurityConst.errSSLInternal
+ base_socket = wrapped_socket.socket
+
+ bytes_to_write = data_length_pointer[0]
+ data = ctypes.string_at(data_buffer, bytes_to_write)
+
+ timeout = wrapped_socket.gettimeout()
+ error = None
+ sent = 0
+
+ try:
+ while sent < bytes_to_write:
+ if timeout is None or timeout >= 0:
+ writables = util.wait_for_write([base_socket], timeout)
+ if not writables:
+ raise socket.error(errno.EAGAIN, 'timed out')
+ chunk_sent = base_socket.send(data)
+ sent += chunk_sent
+
+ # This has some needless copying here, but I'm not sure there's
+ # much value in optimising this data path.
+ data = data[chunk_sent:]
+ except (socket.error) as e:
+ error = e.errno
+
+ if error is not None and error != errno.EAGAIN:
+ if error == errno.ECONNRESET:
+ return SecurityConst.errSSLClosedAbort
+ raise
+
+ data_length_pointer[0] = sent
+ if sent != bytes_to_write:
+ return SecurityConst.errSSLWouldBlock
+
+ return 0
+ except Exception as e:
+ if wrapped_socket is not None:
+ wrapped_socket._exception = e
+ return SecurityConst.errSSLInternal
+
+
+# We need to keep these two objects references alive: if they get GC'd while
+# in use then SecureTransport could attempt to call a function that is in freed
+# memory. That would be...uh...bad. Yeah, that's the word. Bad.
+_read_callback_pointer = Security.SSLReadFunc(_read_callback)
+_write_callback_pointer = Security.SSLWriteFunc(_write_callback)
+
+
+class WrappedSocket(object):
+ """
+ API-compatibility wrapper for Python's OpenSSL wrapped socket object.
+
+ Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage
+ collector of PyPy.
+ """
+ def __init__(self, socket):
+ self.socket = socket
+ self.context = None
+ self._makefile_refs = 0
+ self._closed = False
+ self._exception = None
+ self._keychain = None
+ self._keychain_dir = None
+ self._client_cert_chain = None
+
+ # We save off the previously-configured timeout and then set it to
+ # zero. This is done because we use select and friends to handle the
+ # timeouts, but if we leave the timeout set on the lower socket then
+ # Python will "kindly" call select on that socket again for us. Avoid
+ # that by forcing the timeout to zero.
+ self._timeout = self.socket.gettimeout()
+ self.socket.settimeout(0)
+
+ @contextlib.contextmanager
+ def _raise_on_error(self):
+ """
+ A context manager that can be used to wrap calls that do I/O from
+ SecureTransport. If any of the I/O callbacks hit an exception, this
+ context manager will correctly propagate the exception after the fact.
+ This avoids silently swallowing those exceptions.
+
+ It also correctly forces the socket closed.
+ """
+ self._exception = None
+
+ # We explicitly don't catch around this yield because in the unlikely
+ # event that an exception was hit in the block we don't want to swallow
+ # it.
+ yield
+ if self._exception is not None:
+ exception, self._exception = self._exception, None
+ self.close()
+ raise exception
+
+ def _set_ciphers(self):
+ """
+ Sets up the allowed ciphers. By default this matches the set in
+ util.ssl_.DEFAULT_CIPHERS, at least as supported by macOS. This is done
+ custom and doesn't allow changing at this time, mostly because parsing
+ OpenSSL cipher strings is going to be a freaking nightmare.
+ """
+ ciphers = (Security.SSLCipherSuite * len(CIPHER_SUITES))(*CIPHER_SUITES)
+ result = Security.SSLSetEnabledCiphers(
+ self.context, ciphers, len(CIPHER_SUITES)
+ )
+ _assert_no_error(result)
+
+ def _custom_validate(self, verify, trust_bundle):
+ """
+ Called when we have set custom validation. We do this in two cases:
+ first, when cert validation is entirely disabled; and second, when
+ using a custom trust DB.
+ """
+ # If we disabled cert validation, just say: cool.
+ if not verify:
+ return
+
+ # We want data in memory, so load it up.
+ if os.path.isfile(trust_bundle):
+ with open(trust_bundle, 'rb') as f:
+ trust_bundle = f.read()
+
+ cert_array = None
+ trust = Security.SecTrustRef()
+
+ try:
+ # Get a CFArray that contains the certs we want.
+ cert_array = _cert_array_from_pem(trust_bundle)
+
+ # Ok, now the hard part. We want to get the SecTrustRef that ST has
+ # created for this connection, shove our CAs into it, tell ST to
+ # ignore everything else it knows, and then ask if it can build a
+ # chain. This is a buuuunch of code.
+ result = Security.SSLCopyPeerTrust(
+ self.context, ctypes.byref(trust)
+ )
+ _assert_no_error(result)
+ if not trust:
+ raise ssl.SSLError("Failed to copy trust reference")
+
+ result = Security.SecTrustSetAnchorCertificates(trust, cert_array)
+ _assert_no_error(result)
+
+ result = Security.SecTrustSetAnchorCertificatesOnly(trust, True)
+ _assert_no_error(result)
+
+ trust_result = Security.SecTrustResultType()
+ result = Security.SecTrustEvaluate(
+ trust, ctypes.byref(trust_result)
+ )
+ _assert_no_error(result)
+ finally:
+ if trust:
+ CoreFoundation.CFRelease(trust)
+
+ if cert_array is None:
+ CoreFoundation.CFRelease(cert_array)
+
+ # Ok, now we can look at what the result was.
+ successes = (
+ SecurityConst.kSecTrustResultUnspecified,
+ SecurityConst.kSecTrustResultProceed
+ )
+ if trust_result.value not in successes:
+ raise ssl.SSLError(
+ "certificate verify failed, error code: %d" %
+ trust_result.value
+ )
+
+ def handshake(self,
+ server_hostname,
+ verify,
+ trust_bundle,
+ min_version,
+ max_version,
+ client_cert,
+ client_key,
+ client_key_passphrase):
+ """
+ Actually performs the TLS handshake. This is run automatically by
+ wrapped socket, and shouldn't be needed in user code.
+ """
+ # First, we do the initial bits of connection setup. We need to create
+ # a context, set its I/O funcs, and set the connection reference.
+ self.context = Security.SSLCreateContext(
+ None, SecurityConst.kSSLClientSide, SecurityConst.kSSLStreamType
+ )
+ result = Security.SSLSetIOFuncs(
+ self.context, _read_callback_pointer, _write_callback_pointer
+ )
+ _assert_no_error(result)
+
+ # Here we need to compute the handle to use. We do this by taking the
+ # id of self modulo 2**31 - 1. If this is already in the dictionary, we
+ # just keep incrementing by one until we find a free space.
+ with _connection_ref_lock:
+ handle = id(self) % 2147483647
+ while handle in _connection_refs:
+ handle = (handle + 1) % 2147483647
+ _connection_refs[handle] = self
+
+ result = Security.SSLSetConnection(self.context, handle)
+ _assert_no_error(result)
+
+ # If we have a server hostname, we should set that too.
+ if server_hostname:
+ if not isinstance(server_hostname, bytes):
+ server_hostname = server_hostname.encode('utf-8')
+
+ result = Security.SSLSetPeerDomainName(
+ self.context, server_hostname, len(server_hostname)
+ )
+ _assert_no_error(result)
+
+ # Setup the ciphers.
+ self._set_ciphers()
+
+ # Set the minimum and maximum TLS versions.
+ result = Security.SSLSetProtocolVersionMin(self.context, min_version)
+ _assert_no_error(result)
+ result = Security.SSLSetProtocolVersionMax(self.context, max_version)
+ _assert_no_error(result)
+
+ # If there's a trust DB, we need to use it. We do that by telling
+ # SecureTransport to break on server auth. We also do that if we don't
+ # want to validate the certs at all: we just won't actually do any
+ # authing in that case.
+ if not verify or trust_bundle is not None:
+ result = Security.SSLSetSessionOption(
+ self.context,
+ SecurityConst.kSSLSessionOptionBreakOnServerAuth,
+ True
+ )
+ _assert_no_error(result)
+
+ # If there's a client cert, we need to use it.
+ if client_cert:
+ self._keychain, self._keychain_dir = _temporary_keychain()
+ self._client_cert_chain = _load_client_cert_chain(
+ self._keychain, client_cert, client_key
+ )
+ result = Security.SSLSetCertificate(
+ self.context, self._client_cert_chain
+ )
+ _assert_no_error(result)
+
+ while True:
+ with self._raise_on_error():
+ result = Security.SSLHandshake(self.context)
+
+ if result == SecurityConst.errSSLWouldBlock:
+ raise socket.timeout("handshake timed out")
+ elif result == SecurityConst.errSSLServerAuthCompleted:
+ self._custom_validate(verify, trust_bundle)
+ continue
+ else:
+ _assert_no_error(result)
+ break
+
+ def fileno(self):
+ return self.socket.fileno()
+
+ # Copy-pasted from Python 3.5 source code
+ def _decref_socketios(self):
+ if self._makefile_refs > 0:
+ self._makefile_refs -= 1
+ if self._closed:
+ self.close()
+
+ def recv(self, bufsiz):
+ buffer = ctypes.create_string_buffer(bufsiz)
+ bytes_read = self.recv_into(buffer, bufsiz)
+ data = buffer[:bytes_read]
+ return data
+
+ def recv_into(self, buffer, nbytes=None):
+ # Read short on EOF.
+ if self._closed:
+ return 0
+
+ if nbytes is None:
+ nbytes = len(buffer)
+
+ buffer = (ctypes.c_char * nbytes).from_buffer(buffer)
+ processed_bytes = ctypes.c_size_t(0)
+
+ with self._raise_on_error():
+ result = Security.SSLRead(
+ self.context, buffer, nbytes, ctypes.byref(processed_bytes)
+ )
+
+ # There are some result codes that we want to treat as "not always
+ # errors". Specifically, those are errSSLWouldBlock,
+ # errSSLClosedGraceful, and errSSLClosedNoNotify.
+ if (result == SecurityConst.errSSLWouldBlock):
+ # If we didn't process any bytes, then this was just a time out.
+ # However, we can get errSSLWouldBlock in situations when we *did*
+ # read some data, and in those cases we should just read "short"
+ # and return.
+ if processed_bytes.value == 0:
+ # Timed out, no data read.
+ raise socket.timeout("recv timed out")
+ elif result in (SecurityConst.errSSLClosedGraceful, SecurityConst.errSSLClosedNoNotify):
+ # The remote peer has closed this connection. We should do so as
+ # well. Note that we don't actually return here because in
+ # principle this could actually be fired along with return data.
+ # It's unlikely though.
+ self.close()
+ else:
+ _assert_no_error(result)
+
+ # Ok, we read and probably succeeded. We should return whatever data
+ # was actually read.
+ return processed_bytes.value
+
+ def settimeout(self, timeout):
+ self._timeout = timeout
+
+ def gettimeout(self):
+ return self._timeout
+
+ def send(self, data):
+ processed_bytes = ctypes.c_size_t(0)
+
+ with self._raise_on_error():
+ result = Security.SSLWrite(
+ self.context, data, len(data), ctypes.byref(processed_bytes)
+ )
+
+ if result == SecurityConst.errSSLWouldBlock and processed_bytes.value == 0:
+ # Timed out
+ raise socket.timeout("send timed out")
+ else:
+ _assert_no_error(result)
+
+ # We sent, and probably succeeded. Tell them how much we sent.
+ return processed_bytes.value
+
+ def sendall(self, data):
+ total_sent = 0
+ while total_sent < len(data):
+ sent = self.send(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
+ total_sent += sent
+
+ def shutdown(self):
+ with self._raise_on_error():
+ Security.SSLClose(self.context)
+
+ def close(self):
+ # TODO: should I do clean shutdown here? Do I have to?
+ if self._makefile_refs < 1:
+ self._closed = True
+ if self.context:
+ CoreFoundation.CFRelease(self.context)
+ self.context = None
+ if self._client_cert_chain:
+ CoreFoundation.CFRelease(self._client_cert_chain)
+ self._client_cert_chain = None
+ if self._keychain:
+ Security.SecKeychainDelete(self._keychain)
+ CoreFoundation.CFRelease(self._keychain)
+ shutil.rmtree(self._keychain_dir)
+ self._keychain = self._keychain_dir = None
+ return self.socket.close()
+ else:
+ self._makefile_refs -= 1
+
+ def getpeercert(self, binary_form=False):
+ # Urgh, annoying.
+ #
+ # Here's how we do this:
+ #
+ # 1. Call SSLCopyPeerTrust to get hold of the trust object for this
+ # connection.
+ # 2. Call SecTrustGetCertificateAtIndex for index 0 to get the leaf.
+ # 3. To get the CN, call SecCertificateCopyCommonName and process that
+ # string so that it's of the appropriate type.
+ # 4. To get the SAN, we need to do something a bit more complex:
+ # a. Call SecCertificateCopyValues to get the data, requesting
+ # kSecOIDSubjectAltName.
+ # b. Mess about with this dictionary to try to get the SANs out.
+ #
+ # This is gross. Really gross. It's going to be a few hundred LoC extra
+ # just to repeat something that SecureTransport can *already do*. So my
+ # operating assumption at this time is that what we want to do is
+ # instead to just flag to urllib3 that it shouldn't do its own hostname
+ # validation when using SecureTransport.
+ if not binary_form:
+ raise ValueError(
+ "SecureTransport only supports dumping binary certs"
+ )
+ trust = Security.SecTrustRef()
+ certdata = None
+ der_bytes = None
+
+ try:
+ # Grab the trust store.
+ result = Security.SSLCopyPeerTrust(
+ self.context, ctypes.byref(trust)
+ )
+ _assert_no_error(result)
+ if not trust:
+ # Probably we haven't done the handshake yet. No biggie.
+ return None
+
+ cert_count = Security.SecTrustGetCertificateCount(trust)
+ if not cert_count:
+ # Also a case that might happen if we haven't handshaked.
+ # Handshook? Handshaken?
+ return None
+
+ leaf = Security.SecTrustGetCertificateAtIndex(trust, 0)
+ assert leaf
+
+ # Ok, now we want the DER bytes.
+ certdata = Security.SecCertificateCopyData(leaf)
+ assert certdata
+
+ data_length = CoreFoundation.CFDataGetLength(certdata)
+ data_buffer = CoreFoundation.CFDataGetBytePtr(certdata)
+ der_bytes = ctypes.string_at(data_buffer, data_length)
+ finally:
+ if certdata:
+ CoreFoundation.CFRelease(certdata)
+ if trust:
+ CoreFoundation.CFRelease(trust)
+
+ return der_bytes
+
+ def _reuse(self):
+ self._makefile_refs += 1
+
+ def _drop(self):
+ if self._makefile_refs < 1:
+ self.close()
+ else:
+ self._makefile_refs -= 1
+
+
+if _fileobject: # Platform-specific: Python 2
+ def makefile(self, mode, bufsize=-1):
+ self._makefile_refs += 1
+ return _fileobject(self, mode, bufsize, close=True)
+else: # Platform-specific: Python 3
+ def makefile(self, mode="r", buffering=None, *args, **kwargs):
+ # We disable buffering with SecureTransport because it conflicts with
+ # the buffering that ST does internally (see issue #1153 for more).
+ buffering = 0
+ return backport_makefile(self, mode, buffering, *args, **kwargs)
+
+WrappedSocket.makefile = makefile
+
+
+class SecureTransportContext(object):
+ """
+ I am a wrapper class for the SecureTransport library, to translate the
+ interface of the standard library ``SSLContext`` object to calls into
+ SecureTransport.
+ """
+ def __init__(self, protocol):
+ self._min_version, self._max_version = _protocol_to_min_max[protocol]
+ self._options = 0
+ self._verify = False
+ self._trust_bundle = None
+ self._client_cert = None
+ self._client_key = None
+ self._client_key_passphrase = None
+
+ @property
+ def check_hostname(self):
+ """
+ SecureTransport cannot have its hostname checking disabled. For more,
+ see the comment on getpeercert() in this file.
+ """
+ return True
+
+ @check_hostname.setter
+ def check_hostname(self, value):
+ """
+ SecureTransport cannot have its hostname checking disabled. For more,
+ see the comment on getpeercert() in this file.
+ """
+ pass
+
+ @property
+ def options(self):
+ # TODO: Well, crap.
+ #
+ # So this is the bit of the code that is the most likely to cause us
+ # trouble. Essentially we need to enumerate all of the SSL options that
+ # users might want to use and try to see if we can sensibly translate
+ # them, or whether we should just ignore them.
+ return self._options
+
+ @options.setter
+ def options(self, value):
+ # TODO: Update in line with above.
+ self._options = value
+
+ @property
+ def verify_mode(self):
+ return ssl.CERT_REQUIRED if self._verify else ssl.CERT_NONE
+
+ @verify_mode.setter
+ def verify_mode(self, value):
+ self._verify = True if value == ssl.CERT_REQUIRED else False
+
+ def set_default_verify_paths(self):
+ # So, this has to do something a bit weird. Specifically, what it does
+ # is nothing.
+ #
+ # This means that, if we had previously had load_verify_locations
+ # called, this does not undo that. We need to do that because it turns
+ # out that the rest of the urllib3 code will attempt to load the
+ # default verify paths if it hasn't been told about any paths, even if
+ # the context itself was sometime earlier. We resolve that by just
+ # ignoring it.
+ pass
+
+ def load_default_certs(self):
+ return self.set_default_verify_paths()
+
+ def set_ciphers(self, ciphers):
+ # For now, we just require the default cipher string.
+ if ciphers != util.ssl_.DEFAULT_CIPHERS:
+ raise ValueError(
+ "SecureTransport doesn't support custom cipher strings"
+ )
+
+ def load_verify_locations(self, cafile=None, capath=None, cadata=None):
+ # OK, we only really support cadata and cafile.
+ if capath is not None:
+ raise ValueError(
+ "SecureTransport does not support cert directories"
+ )
+
+ self._trust_bundle = cafile or cadata
+
+ def load_cert_chain(self, certfile, keyfile=None, password=None):
+ self._client_cert = certfile
+ self._client_key = keyfile
+ self._client_cert_passphrase = password
+
+ def wrap_socket(self, sock, server_side=False,
+ do_handshake_on_connect=True, suppress_ragged_eofs=True,
+ server_hostname=None):
+ # So, what do we do here? Firstly, we assert some properties. This is a
+ # stripped down shim, so there is some functionality we don't support.
+ # See PEP 543 for the real deal.
+ assert not server_side
+ assert do_handshake_on_connect
+ assert suppress_ragged_eofs
+
+ # Ok, we're good to go. Now we want to create the wrapped socket object
+ # and store it in the appropriate place.
+ wrapped_socket = WrappedSocket(sock)
+
+ # Now we can handshake
+ wrapped_socket.handshake(
+ server_hostname, self._verify, self._trust_bundle,
+ self._min_version, self._max_version, self._client_cert,
+ self._client_key, self._client_key_passphrase
+ )
+ return wrapped_socket
diff --git a/collectors/python.d.plugin/python_modules/urllib3/contrib/socks.py b/collectors/python.d.plugin/python_modules/urllib3/contrib/socks.py
new file mode 100644
index 0000000..1cb7928
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/contrib/socks.py
@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+"""
+This module contains provisional support for SOCKS proxies from within
+urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and
+SOCKS5. To enable its functionality, either install PySocks or install this
+module with the ``socks`` extra.
+
+The SOCKS implementation supports the full range of urllib3 features. It also
+supports the following SOCKS features:
+
+- SOCKS4
+- SOCKS4a
+- SOCKS5
+- Usernames and passwords for the SOCKS proxy
+
+Known Limitations:
+
+- Currently PySocks does not support contacting remote websites via literal
+ IPv6 addresses. Any such connection attempt will fail. You must use a domain
+ name.
+- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any
+ such connection attempt will fail.
+"""
+from __future__ import absolute_import
+
+try:
+ import socks
+except ImportError:
+ import warnings
+ from ..exceptions import DependencyWarning
+
+ warnings.warn((
+ 'SOCKS support in urllib3 requires the installation of optional '
+ 'dependencies: specifically, PySocks. For more information, see '
+ 'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies'
+ ),
+ DependencyWarning
+ )
+ raise
+
+from socket import error as SocketError, timeout as SocketTimeout
+
+from ..connection import (
+ HTTPConnection, HTTPSConnection
+)
+from ..connectionpool import (
+ HTTPConnectionPool, HTTPSConnectionPool
+)
+from ..exceptions import ConnectTimeoutError, NewConnectionError
+from ..poolmanager import PoolManager
+from ..util.url import parse_url
+
+try:
+ import ssl
+except ImportError:
+ ssl = None
+
+
+class SOCKSConnection(HTTPConnection):
+ """
+ A plain-text HTTP connection that connects via a SOCKS proxy.
+ """
+ def __init__(self, *args, **kwargs):
+ self._socks_options = kwargs.pop('_socks_options')
+ super(SOCKSConnection, self).__init__(*args, **kwargs)
+
+ def _new_conn(self):
+ """
+ Establish a new connection via the SOCKS proxy.
+ """
+ extra_kw = {}
+ if self.source_address:
+ extra_kw['source_address'] = self.source_address
+
+ if self.socket_options:
+ extra_kw['socket_options'] = self.socket_options
+
+ try:
+ conn = socks.create_connection(
+ (self.host, self.port),
+ proxy_type=self._socks_options['socks_version'],
+ proxy_addr=self._socks_options['proxy_host'],
+ proxy_port=self._socks_options['proxy_port'],
+ proxy_username=self._socks_options['username'],
+ proxy_password=self._socks_options['password'],
+ proxy_rdns=self._socks_options['rdns'],
+ timeout=self.timeout,
+ **extra_kw
+ )
+
+ except SocketTimeout as e:
+ raise ConnectTimeoutError(
+ self, "Connection to %s timed out. (connect timeout=%s)" %
+ (self.host, self.timeout))
+
+ except socks.ProxyError as e:
+ # This is fragile as hell, but it seems to be the only way to raise
+ # useful errors here.
+ if e.socket_err:
+ error = e.socket_err
+ if isinstance(error, SocketTimeout):
+ raise ConnectTimeoutError(
+ self,
+ "Connection to %s timed out. (connect timeout=%s)" %
+ (self.host, self.timeout)
+ )
+ else:
+ raise NewConnectionError(
+ self,
+ "Failed to establish a new connection: %s" % error
+ )
+ else:
+ raise NewConnectionError(
+ self,
+ "Failed to establish a new connection: %s" % e
+ )
+
+ except SocketError as e: # Defensive: PySocks should catch all these.
+ raise NewConnectionError(
+ self, "Failed to establish a new connection: %s" % e)
+
+ return conn
+
+
+# We don't need to duplicate the Verified/Unverified distinction from
+# urllib3/connection.py here because the HTTPSConnection will already have been
+# correctly set to either the Verified or Unverified form by that module. This
+# means the SOCKSHTTPSConnection will automatically be the correct type.
+class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection):
+ pass
+
+
+class SOCKSHTTPConnectionPool(HTTPConnectionPool):
+ ConnectionCls = SOCKSConnection
+
+
+class SOCKSHTTPSConnectionPool(HTTPSConnectionPool):
+ ConnectionCls = SOCKSHTTPSConnection
+
+
+class SOCKSProxyManager(PoolManager):
+ """
+ A version of the urllib3 ProxyManager that routes connections via the
+ defined SOCKS proxy.
+ """
+ pool_classes_by_scheme = {
+ 'http': SOCKSHTTPConnectionPool,
+ 'https': SOCKSHTTPSConnectionPool,
+ }
+
+ def __init__(self, proxy_url, username=None, password=None,
+ num_pools=10, headers=None, **connection_pool_kw):
+ parsed = parse_url(proxy_url)
+
+ if parsed.scheme == 'socks5':
+ socks_version = socks.PROXY_TYPE_SOCKS5
+ rdns = False
+ elif parsed.scheme == 'socks5h':
+ socks_version = socks.PROXY_TYPE_SOCKS5
+ rdns = True
+ elif parsed.scheme == 'socks4':
+ socks_version = socks.PROXY_TYPE_SOCKS4
+ rdns = False
+ elif parsed.scheme == 'socks4a':
+ socks_version = socks.PROXY_TYPE_SOCKS4
+ rdns = True
+ else:
+ raise ValueError(
+ "Unable to determine SOCKS version from %s" % proxy_url
+ )
+
+ self.proxy_url = proxy_url
+
+ socks_options = {
+ 'socks_version': socks_version,
+ 'proxy_host': parsed.host,
+ 'proxy_port': parsed.port,
+ 'username': username,
+ 'password': password,
+ 'rdns': rdns
+ }
+ connection_pool_kw['_socks_options'] = socks_options
+
+ super(SOCKSProxyManager, self).__init__(
+ num_pools, headers, **connection_pool_kw
+ )
+
+ self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme
diff --git a/collectors/python.d.plugin/python_modules/urllib3/exceptions.py b/collectors/python.d.plugin/python_modules/urllib3/exceptions.py
new file mode 100644
index 0000000..a71cabe
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/exceptions.py
@@ -0,0 +1,247 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+from .packages.six.moves.http_client import (
+ IncompleteRead as httplib_IncompleteRead
+)
+# Base Exceptions
+
+
+class HTTPError(Exception):
+ "Base exception used by this module."
+ pass
+
+
+class HTTPWarning(Warning):
+ "Base warning used by this module."
+ pass
+
+
+class PoolError(HTTPError):
+ "Base exception for errors caused within a pool."
+ def __init__(self, pool, message):
+ self.pool = pool
+ HTTPError.__init__(self, "%s: %s" % (pool, message))
+
+ def __reduce__(self):
+ # For pickling purposes.
+ return self.__class__, (None, None)
+
+
+class RequestError(PoolError):
+ "Base exception for PoolErrors that have associated URLs."
+ def __init__(self, pool, url, message):
+ self.url = url
+ PoolError.__init__(self, pool, message)
+
+ def __reduce__(self):
+ # For pickling purposes.
+ return self.__class__, (None, self.url, None)
+
+
+class SSLError(HTTPError):
+ "Raised when SSL certificate fails in an HTTPS connection."
+ pass
+
+
+class ProxyError(HTTPError):
+ "Raised when the connection to a proxy fails."
+ pass
+
+
+class DecodeError(HTTPError):
+ "Raised when automatic decoding based on Content-Type fails."
+ pass
+
+
+class ProtocolError(HTTPError):
+ "Raised when something unexpected happens mid-request/response."
+ pass
+
+
+#: Renamed to ProtocolError but aliased for backwards compatibility.
+ConnectionError = ProtocolError
+
+
+# Leaf Exceptions
+
+class MaxRetryError(RequestError):
+ """Raised when the maximum number of retries is exceeded.
+
+ :param pool: The connection pool
+ :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
+ :param string url: The requested Url
+ :param exceptions.Exception reason: The underlying error
+
+ """
+
+ def __init__(self, pool, url, reason=None):
+ self.reason = reason
+
+ message = "Max retries exceeded with url: %s (Caused by %r)" % (
+ url, reason)
+
+ RequestError.__init__(self, pool, url, message)
+
+
+class HostChangedError(RequestError):
+ "Raised when an existing pool gets a request for a foreign host."
+
+ def __init__(self, pool, url, retries=3):
+ message = "Tried to open a foreign host with url: %s" % url
+ RequestError.__init__(self, pool, url, message)
+ self.retries = retries
+
+
+class TimeoutStateError(HTTPError):
+ """ Raised when passing an invalid state to a timeout """
+ pass
+
+
+class TimeoutError(HTTPError):
+ """ Raised when a socket timeout error occurs.
+
+ Catching this error will catch both :exc:`ReadTimeoutErrors
+ <ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
+ """
+ pass
+
+
+class ReadTimeoutError(TimeoutError, RequestError):
+ "Raised when a socket timeout occurs while receiving data from a server"
+ pass
+
+
+# This timeout error does not have a URL attached and needs to inherit from the
+# base HTTPError
+class ConnectTimeoutError(TimeoutError):
+ "Raised when a socket timeout occurs while connecting to a server"
+ pass
+
+
+class NewConnectionError(ConnectTimeoutError, PoolError):
+ "Raised when we fail to establish a new connection. Usually ECONNREFUSED."
+ pass
+
+
+class EmptyPoolError(PoolError):
+ "Raised when a pool runs out of connections and no more are allowed."
+ pass
+
+
+class ClosedPoolError(PoolError):
+ "Raised when a request enters a pool after the pool has been closed."
+ pass
+
+
+class LocationValueError(ValueError, HTTPError):
+ "Raised when there is something wrong with a given URL input."
+ pass
+
+
+class LocationParseError(LocationValueError):
+ "Raised when get_host or similar fails to parse the URL input."
+
+ def __init__(self, location):
+ message = "Failed to parse: %s" % location
+ HTTPError.__init__(self, message)
+
+ self.location = location
+
+
+class ResponseError(HTTPError):
+ "Used as a container for an error reason supplied in a MaxRetryError."
+ GENERIC_ERROR = 'too many error responses'
+ SPECIFIC_ERROR = 'too many {status_code} error responses'
+
+
+class SecurityWarning(HTTPWarning):
+ "Warned when perfoming security reducing actions"
+ pass
+
+
+class SubjectAltNameWarning(SecurityWarning):
+ "Warned when connecting to a host with a certificate missing a SAN."
+ pass
+
+
+class InsecureRequestWarning(SecurityWarning):
+ "Warned when making an unverified HTTPS request."
+ pass
+
+
+class SystemTimeWarning(SecurityWarning):
+ "Warned when system time is suspected to be wrong"
+ pass
+
+
+class InsecurePlatformWarning(SecurityWarning):
+ "Warned when certain SSL configuration is not available on a platform."
+ pass
+
+
+class SNIMissingWarning(HTTPWarning):
+ "Warned when making a HTTPS request without SNI available."
+ pass
+
+
+class DependencyWarning(HTTPWarning):
+ """
+ Warned when an attempt is made to import a module with missing optional
+ dependencies.
+ """
+ pass
+
+
+class ResponseNotChunked(ProtocolError, ValueError):
+ "Response needs to be chunked in order to read it as chunks."
+ pass
+
+
+class BodyNotHttplibCompatible(HTTPError):
+ """
+ Body should be httplib.HTTPResponse like (have an fp attribute which
+ returns raw chunks) for read_chunked().
+ """
+ pass
+
+
+class IncompleteRead(HTTPError, httplib_IncompleteRead):
+ """
+ Response length doesn't match expected Content-Length
+
+ Subclass of http_client.IncompleteRead to allow int value
+ for `partial` to avoid creating large objects on streamed
+ reads.
+ """
+ def __init__(self, partial, expected):
+ super(IncompleteRead, self).__init__(partial, expected)
+
+ def __repr__(self):
+ return ('IncompleteRead(%i bytes read, '
+ '%i more expected)' % (self.partial, self.expected))
+
+
+class InvalidHeader(HTTPError):
+ "The header provided was somehow invalid."
+ pass
+
+
+class ProxySchemeUnknown(AssertionError, ValueError):
+ "ProxyManager does not support the supplied scheme"
+ # TODO(t-8ch): Stop inheriting from AssertionError in v2.0.
+
+ def __init__(self, scheme):
+ message = "Not supported proxy scheme %s" % scheme
+ super(ProxySchemeUnknown, self).__init__(message)
+
+
+class HeaderParsingError(HTTPError):
+ "Raised by assert_header_parsing, but we convert it to a log.warning statement."
+ def __init__(self, defects, unparsed_data):
+ message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data)
+ super(HeaderParsingError, self).__init__(message)
+
+
+class UnrewindableBodyError(HTTPError):
+ "urllib3 encountered an error when trying to rewind a body"
+ pass
diff --git a/collectors/python.d.plugin/python_modules/urllib3/fields.py b/collectors/python.d.plugin/python_modules/urllib3/fields.py
new file mode 100644
index 0000000..de7577b
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/fields.py
@@ -0,0 +1,179 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+import email.utils
+import mimetypes
+
+from .packages import six
+
+
+def guess_content_type(filename, default='application/octet-stream'):
+ """
+ Guess the "Content-Type" of a file.
+
+ :param filename:
+ The filename to guess the "Content-Type" of using :mod:`mimetypes`.
+ :param default:
+ If no "Content-Type" can be guessed, default to `default`.
+ """
+ if filename:
+ return mimetypes.guess_type(filename)[0] or default
+ return default
+
+
+def format_header_param(name, value):
+ """
+ Helper function to format and quote a single header parameter.
+
+ Particularly useful for header parameters which might contain
+ non-ASCII values, like file names. This follows RFC 2231, as
+ suggested by RFC 2388 Section 4.4.
+
+ :param name:
+ The name of the parameter, a string expected to be ASCII only.
+ :param value:
+ The value of the parameter, provided as a unicode string.
+ """
+ if not any(ch in value for ch in '"\\\r\n'):
+ result = '%s="%s"' % (name, value)
+ try:
+ result.encode('ascii')
+ except (UnicodeEncodeError, UnicodeDecodeError):
+ pass
+ else:
+ return result
+ if not six.PY3 and isinstance(value, six.text_type): # Python 2:
+ value = value.encode('utf-8')
+ value = email.utils.encode_rfc2231(value, 'utf-8')
+ value = '%s*=%s' % (name, value)
+ return value
+
+
+class RequestField(object):
+ """
+ A data container for request body parameters.
+
+ :param name:
+ The name of this request field.
+ :param data:
+ The data/value body.
+ :param filename:
+ An optional filename of the request field.
+ :param headers:
+ An optional dict-like object of headers to initially use for the field.
+ """
+ def __init__(self, name, data, filename=None, headers=None):
+ self._name = name
+ self._filename = filename
+ self.data = data
+ self.headers = {}
+ if headers:
+ self.headers = dict(headers)
+
+ @classmethod
+ def from_tuples(cls, fieldname, value):
+ """
+ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
+
+ Supports constructing :class:`~urllib3.fields.RequestField` from
+ parameter of key/value strings AND key/filetuple. A filetuple is a
+ (filename, data, MIME type) tuple where the MIME type is optional.
+ For example::
+
+ 'foo': 'bar',
+ 'fakefile': ('foofile.txt', 'contents of foofile'),
+ 'realfile': ('barfile.txt', open('realfile').read()),
+ 'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'),
+ 'nonamefile': 'contents of nonamefile field',
+
+ Field names and filenames must be unicode.
+ """
+ if isinstance(value, tuple):
+ if len(value) == 3:
+ filename, data, content_type = value
+ else:
+ filename, data = value
+ content_type = guess_content_type(filename)
+ else:
+ filename = None
+ content_type = None
+ data = value
+
+ request_param = cls(fieldname, data, filename=filename)
+ request_param.make_multipart(content_type=content_type)
+
+ return request_param
+
+ def _render_part(self, name, value):
+ """
+ Overridable helper function to format a single header parameter.
+
+ :param name:
+ The name of the parameter, a string expected to be ASCII only.
+ :param value:
+ The value of the parameter, provided as a unicode string.
+ """
+ return format_header_param(name, value)
+
+ def _render_parts(self, header_parts):
+ """
+ Helper function to format and quote a single header.
+
+ Useful for single headers that are composed of multiple items. E.g.,
+ 'Content-Disposition' fields.
+
+ :param header_parts:
+ A sequence of (k, v) typles or a :class:`dict` of (k, v) to format
+ as `k1="v1"; k2="v2"; ...`.
+ """
+ parts = []
+ iterable = header_parts
+ if isinstance(header_parts, dict):
+ iterable = header_parts.items()
+
+ for name, value in iterable:
+ if value is not None:
+ parts.append(self._render_part(name, value))
+
+ return '; '.join(parts)
+
+ def render_headers(self):
+ """
+ Renders the headers for this request field.
+ """
+ lines = []
+
+ sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location']
+ for sort_key in sort_keys:
+ if self.headers.get(sort_key, False):
+ lines.append('%s: %s' % (sort_key, self.headers[sort_key]))
+
+ for header_name, header_value in self.headers.items():
+ if header_name not in sort_keys:
+ if header_value:
+ lines.append('%s: %s' % (header_name, header_value))
+
+ lines.append('\r\n')
+ return '\r\n'.join(lines)
+
+ def make_multipart(self, content_disposition=None, content_type=None,
+ content_location=None):
+ """
+ Makes this request field into a multipart request field.
+
+ This method overrides "Content-Disposition", "Content-Type" and
+ "Content-Location" headers to the request parameter.
+
+ :param content_type:
+ The 'Content-Type' of the request body.
+ :param content_location:
+ The 'Content-Location' of the request body.
+
+ """
+ self.headers['Content-Disposition'] = content_disposition or 'form-data'
+ self.headers['Content-Disposition'] += '; '.join([
+ '', self._render_parts(
+ (('name', self._name), ('filename', self._filename))
+ )
+ ])
+ self.headers['Content-Type'] = content_type
+ self.headers['Content-Location'] = content_location
diff --git a/collectors/python.d.plugin/python_modules/urllib3/filepost.py b/collectors/python.d.plugin/python_modules/urllib3/filepost.py
new file mode 100644
index 0000000..3febc9c
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/filepost.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+import codecs
+
+from uuid import uuid4
+from io import BytesIO
+
+from .packages import six
+from .packages.six import b
+from .fields import RequestField
+
+writer = codecs.lookup('utf-8')[3]
+
+
+def choose_boundary():
+ """
+ Our embarrassingly-simple replacement for mimetools.choose_boundary.
+ """
+ return uuid4().hex
+
+
+def iter_field_objects(fields):
+ """
+ Iterate over fields.
+
+ Supports list of (k, v) tuples and dicts, and lists of
+ :class:`~urllib3.fields.RequestField`.
+
+ """
+ if isinstance(fields, dict):
+ i = six.iteritems(fields)
+ else:
+ i = iter(fields)
+
+ for field in i:
+ if isinstance(field, RequestField):
+ yield field
+ else:
+ yield RequestField.from_tuples(*field)
+
+
+def iter_fields(fields):
+ """
+ .. deprecated:: 1.6
+
+ Iterate over fields.
+
+ The addition of :class:`~urllib3.fields.RequestField` makes this function
+ obsolete. Instead, use :func:`iter_field_objects`, which returns
+ :class:`~urllib3.fields.RequestField` objects.
+
+ Supports list of (k, v) tuples and dicts.
+ """
+ if isinstance(fields, dict):
+ return ((k, v) for k, v in six.iteritems(fields))
+
+ return ((k, v) for k, v in fields)
+
+
+def encode_multipart_formdata(fields, boundary=None):
+ """
+ Encode a dictionary of ``fields`` using the multipart/form-data MIME format.
+
+ :param fields:
+ Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`).
+
+ :param boundary:
+ If not specified, then a random boundary will be generated using
+ :func:`mimetools.choose_boundary`.
+ """
+ body = BytesIO()
+ if boundary is None:
+ boundary = choose_boundary()
+
+ for field in iter_field_objects(fields):
+ body.write(b('--%s\r\n' % (boundary)))
+
+ writer(body).write(field.render_headers())
+ data = field.data
+
+ if isinstance(data, int):
+ data = str(data) # Backwards compatibility
+
+ if isinstance(data, six.text_type):
+ writer(body).write(data)
+ else:
+ body.write(data)
+
+ body.write(b'\r\n')
+
+ body.write(b('--%s--\r\n' % (boundary)))
+
+ content_type = str('multipart/form-data; boundary=%s' % boundary)
+
+ return body.getvalue(), content_type
diff --git a/collectors/python.d.plugin/python_modules/urllib3/packages/__init__.py b/collectors/python.d.plugin/python_modules/urllib3/packages/__init__.py
new file mode 100644
index 0000000..170e974
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/packages/__init__.py
@@ -0,0 +1,5 @@
+from __future__ import absolute_import
+
+from . import ssl_match_hostname
+
+__all__ = ('ssl_match_hostname', )
diff --git a/collectors/python.d.plugin/python_modules/urllib3/packages/backports/__init__.py b/collectors/python.d.plugin/python_modules/urllib3/packages/backports/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/packages/backports/__init__.py
diff --git a/collectors/python.d.plugin/python_modules/urllib3/packages/backports/makefile.py b/collectors/python.d.plugin/python_modules/urllib3/packages/backports/makefile.py
new file mode 100644
index 0000000..8ab122f
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/packages/backports/makefile.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+"""
+backports.makefile
+~~~~~~~~~~~~~~~~~~
+
+Backports the Python 3 ``socket.makefile`` method for use with anything that
+wants to create a "fake" socket object.
+"""
+import io
+
+from socket import SocketIO
+
+
+def backport_makefile(self, mode="r", buffering=None, encoding=None,
+ errors=None, newline=None):
+ """
+ Backport of ``socket.makefile`` from Python 3.5.
+ """
+ if not set(mode) <= set(["r", "w", "b"]):
+ raise ValueError(
+ "invalid mode %r (only r, w, b allowed)" % (mode,)
+ )
+ writing = "w" in mode
+ reading = "r" in mode or not writing
+ assert reading or writing
+ binary = "b" in mode
+ rawmode = ""
+ if reading:
+ rawmode += "r"
+ if writing:
+ rawmode += "w"
+ raw = SocketIO(self, rawmode)
+ self._makefile_refs += 1
+ if buffering is None:
+ buffering = -1
+ if buffering < 0:
+ buffering = io.DEFAULT_BUFFER_SIZE
+ if buffering == 0:
+ if not binary:
+ raise ValueError("unbuffered streams must be binary")
+ return raw
+ if reading and writing:
+ buffer = io.BufferedRWPair(raw, raw, buffering)
+ elif reading:
+ buffer = io.BufferedReader(raw, buffering)
+ else:
+ assert writing
+ buffer = io.BufferedWriter(raw, buffering)
+ if binary:
+ return buffer
+ text = io.TextIOWrapper(buffer, encoding, errors, newline)
+ text.mode = mode
+ return text
diff --git a/collectors/python.d.plugin/python_modules/urllib3/packages/ordered_dict.py b/collectors/python.d.plugin/python_modules/urllib3/packages/ordered_dict.py
new file mode 100644
index 0000000..9f7c0e6
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/packages/ordered_dict.py
@@ -0,0 +1,260 @@
+# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
+# Passes Python2.7's test suite and incorporates all the latest updates.
+# Copyright 2009 Raymond Hettinger, released under the MIT License.
+# http://code.activestate.com/recipes/576693/
+# SPDX-License-Identifier: MIT
+try:
+ from thread import get_ident as _get_ident
+except ImportError:
+ from dummy_thread import get_ident as _get_ident
+
+try:
+ from _abcoll import KeysView, ValuesView, ItemsView
+except ImportError:
+ pass
+
+
+class OrderedDict(dict):
+ 'Dictionary that remembers insertion order'
+ # An inherited dict maps keys to values.
+ # The inherited dict provides __getitem__, __len__, __contains__, and get.
+ # The remaining methods are order-aware.
+ # Big-O running times for all methods are the same as for regular dictionaries.
+
+ # The internal self.__map dictionary maps keys to links in a doubly linked list.
+ # The circular doubly linked list starts and ends with a sentinel element.
+ # The sentinel element never gets deleted (this simplifies the algorithm).
+ # Each link is stored as a list of length three: [PREV, NEXT, KEY].
+
+ def __init__(self, *args, **kwds):
+ '''Initialize an ordered dictionary. Signature is the same as for
+ regular dictionaries, but keyword arguments are not recommended
+ because their insertion order is arbitrary.
+
+ '''
+ if len(args) > 1:
+ raise TypeError('expected at most 1 arguments, got %d' % len(args))
+ try:
+ self.__root
+ except AttributeError:
+ self.__root = root = [] # sentinel node
+ root[:] = [root, root, None]
+ self.__map = {}
+ self.__update(*args, **kwds)
+
+ def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
+ 'od.__setitem__(i, y) <==> od[i]=y'
+ # Setting a new item creates a new link which goes at the end of the linked
+ # list, and the inherited dictionary is updated with the new key/value pair.
+ if key not in self:
+ root = self.__root
+ last = root[0]
+ last[1] = root[0] = self.__map[key] = [last, root, key]
+ dict_setitem(self, key, value)
+
+ def __delitem__(self, key, dict_delitem=dict.__delitem__):
+ 'od.__delitem__(y) <==> del od[y]'
+ # Deleting an existing item uses self.__map to find the link which is
+ # then removed by updating the links in the predecessor and successor nodes.
+ dict_delitem(self, key)
+ link_prev, link_next, key = self.__map.pop(key)
+ link_prev[1] = link_next
+ link_next[0] = link_prev
+
+ def __iter__(self):
+ 'od.__iter__() <==> iter(od)'
+ root = self.__root
+ curr = root[1]
+ while curr is not root:
+ yield curr[2]
+ curr = curr[1]
+
+ def __reversed__(self):
+ 'od.__reversed__() <==> reversed(od)'
+ root = self.__root
+ curr = root[0]
+ while curr is not root:
+ yield curr[2]
+ curr = curr[0]
+
+ def clear(self):
+ 'od.clear() -> None. Remove all items from od.'
+ try:
+ for node in self.__map.itervalues():
+ del node[:]
+ root = self.__root
+ root[:] = [root, root, None]
+ self.__map.clear()
+ except AttributeError:
+ pass
+ dict.clear(self)
+
+ def popitem(self, last=True):
+ '''od.popitem() -> (k, v), return and remove a (key, value) pair.
+ Pairs are returned in LIFO order if last is true or FIFO order if false.
+
+ '''
+ if not self:
+ raise KeyError('dictionary is empty')
+ root = self.__root
+ if last:
+ link = root[0]
+ link_prev = link[0]
+ link_prev[1] = root
+ root[0] = link_prev
+ else:
+ link = root[1]
+ link_next = link[1]
+ root[1] = link_next
+ link_next[0] = root
+ key = link[2]
+ del self.__map[key]
+ value = dict.pop(self, key)
+ return key, value
+
+ # -- the following methods do not depend on the internal structure --
+
+ def keys(self):
+ 'od.keys() -> list of keys in od'
+ return list(self)
+
+ def values(self):
+ 'od.values() -> list of values in od'
+ return [self[key] for key in self]
+
+ def items(self):
+ 'od.items() -> list of (key, value) pairs in od'
+ return [(key, self[key]) for key in self]
+
+ def iterkeys(self):
+ 'od.iterkeys() -> an iterator over the keys in od'
+ return iter(self)
+
+ def itervalues(self):
+ 'od.itervalues -> an iterator over the values in od'
+ for k in self:
+ yield self[k]
+
+ def iteritems(self):
+ 'od.iteritems -> an iterator over the (key, value) items in od'
+ for k in self:
+ yield (k, self[k])
+
+ def update(*args, **kwds):
+ '''od.update(E, **F) -> None. Update od from dict/iterable E and F.
+
+ If E is a dict instance, does: for k in E: od[k] = E[k]
+ If E has a .keys() method, does: for k in E.keys(): od[k] = E[k]
+ Or if E is an iterable of items, does: for k, v in E: od[k] = v
+ In either case, this is followed by: for k, v in F.items(): od[k] = v
+
+ '''
+ if len(args) > 2:
+ raise TypeError('update() takes at most 2 positional '
+ 'arguments (%d given)' % (len(args),))
+ elif not args:
+ raise TypeError('update() takes at least 1 argument (0 given)')
+ self = args[0]
+ # Make progressively weaker assumptions about "other"
+ other = ()
+ if len(args) == 2:
+ other = args[1]
+ if isinstance(other, dict):
+ for key in other:
+ self[key] = other[key]
+ elif hasattr(other, 'keys'):
+ for key in other.keys():
+ self[key] = other[key]
+ else:
+ for key, value in other:
+ self[key] = value
+ for key, value in kwds.items():
+ self[key] = value
+
+ __update = update # let subclasses override update without breaking __init__
+
+ __marker = object()
+
+ def pop(self, key, default=__marker):
+ '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
+ If key is not found, d is returned if given, otherwise KeyError is raised.
+
+ '''
+ if key in self:
+ result = self[key]
+ del self[key]
+ return result
+ if default is self.__marker:
+ raise KeyError(key)
+ return default
+
+ def setdefault(self, key, default=None):
+ 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
+ if key in self:
+ return self[key]
+ self[key] = default
+ return default
+
+ def __repr__(self, _repr_running={}):
+ 'od.__repr__() <==> repr(od)'
+ call_key = id(self), _get_ident()
+ if call_key in _repr_running:
+ return '...'
+ _repr_running[call_key] = 1
+ try:
+ if not self:
+ return '%s()' % (self.__class__.__name__,)
+ return '%s(%r)' % (self.__class__.__name__, self.items())
+ finally:
+ del _repr_running[call_key]
+
+ def __reduce__(self):
+ 'Return state information for pickling'
+ items = [[k, self[k]] for k in self]
+ inst_dict = vars(self).copy()
+ for k in vars(OrderedDict()):
+ inst_dict.pop(k, None)
+ if inst_dict:
+ return (self.__class__, (items,), inst_dict)
+ return self.__class__, (items,)
+
+ def copy(self):
+ 'od.copy() -> a shallow copy of od'
+ return self.__class__(self)
+
+ @classmethod
+ def fromkeys(cls, iterable, value=None):
+ '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
+ and values equal to v (which defaults to None).
+
+ '''
+ d = cls()
+ for key in iterable:
+ d[key] = value
+ return d
+
+ def __eq__(self, other):
+ '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
+ while comparison to a regular mapping is order-insensitive.
+
+ '''
+ if isinstance(other, OrderedDict):
+ return len(self)==len(other) and self.items() == other.items()
+ return dict.__eq__(self, other)
+
+ def __ne__(self, other):
+ return not self == other
+
+ # -- the following methods are only used in Python 2.7 --
+
+ def viewkeys(self):
+ "od.viewkeys() -> a set-like object providing a view on od's keys"
+ return KeysView(self)
+
+ def viewvalues(self):
+ "od.viewvalues() -> an object providing a view on od's values"
+ return ValuesView(self)
+
+ def viewitems(self):
+ "od.viewitems() -> a set-like object providing a view on od's items"
+ return ItemsView(self)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/packages/six.py b/collectors/python.d.plugin/python_modules/urllib3/packages/six.py
new file mode 100644
index 0000000..31df501
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/packages/six.py
@@ -0,0 +1,852 @@
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+# Copyright (c) 2010-2015 Benjamin Peterson
+#
+# SPDX-License-Identifier: MIT
+
+from __future__ import absolute_import
+
+import functools
+import itertools
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson <benjamin@python.org>"
+__version__ = "1.10.0"
+
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+PY34 = sys.version_info[0:2] >= (3, 4)
+
+if PY3:
+ string_types = str,
+ integer_types = int,
+ class_types = type,
+ text_type = str
+ binary_type = bytes
+
+ MAXSIZE = sys.maxsize
+else:
+ string_types = basestring,
+ integer_types = (int, long)
+ class_types = (type, types.ClassType)
+ text_type = unicode
+ binary_type = str
+
+ if sys.platform.startswith("java"):
+ # Jython always uses 32 bits.
+ MAXSIZE = int((1 << 31) - 1)
+ else:
+ # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+ class X(object):
+
+ def __len__(self):
+ return 1 << 31
+ try:
+ len(X())
+ except OverflowError:
+ # 32-bit
+ MAXSIZE = int((1 << 31) - 1)
+ else:
+ # 64-bit
+ MAXSIZE = int((1 << 63) - 1)
+ del X
+
+
+def _add_doc(func, doc):
+ """Add documentation to a function."""
+ func.__doc__ = doc
+
+
+def _import_module(name):
+ """Import module, returning the module after the last dot."""
+ __import__(name)
+ return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+ def __init__(self, name):
+ self.name = name
+
+ def __get__(self, obj, tp):
+ result = self._resolve()
+ setattr(obj, self.name, result) # Invokes __set__.
+ try:
+ # This is a bit ugly, but it avoids running this again by
+ # removing this descriptor.
+ delattr(obj.__class__, self.name)
+ except AttributeError:
+ pass
+ return result
+
+
+class MovedModule(_LazyDescr):
+
+ def __init__(self, name, old, new=None):
+ super(MovedModule, self).__init__(name)
+ if PY3:
+ if new is None:
+ new = name
+ self.mod = new
+ else:
+ self.mod = old
+
+ def _resolve(self):
+ return _import_module(self.mod)
+
+ def __getattr__(self, attr):
+ _module = self._resolve()
+ value = getattr(_module, attr)
+ setattr(self, attr, value)
+ return value
+
+
+class _LazyModule(types.ModuleType):
+
+ def __init__(self, name):
+ super(_LazyModule, self).__init__(name)
+ self.__doc__ = self.__class__.__doc__
+
+ def __dir__(self):
+ attrs = ["__doc__", "__name__"]
+ attrs += [attr.name for attr in self._moved_attributes]
+ return attrs
+
+ # Subclasses should override this
+ _moved_attributes = []
+
+
+class MovedAttribute(_LazyDescr):
+
+ def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+ super(MovedAttribute, self).__init__(name)
+ if PY3:
+ if new_mod is None:
+ new_mod = name
+ self.mod = new_mod
+ if new_attr is None:
+ if old_attr is None:
+ new_attr = name
+ else:
+ new_attr = old_attr
+ self.attr = new_attr
+ else:
+ self.mod = old_mod
+ if old_attr is None:
+ old_attr = name
+ self.attr = old_attr
+
+ def _resolve(self):
+ module = _import_module(self.mod)
+ return getattr(module, self.attr)
+
+
+class _SixMetaPathImporter(object):
+
+ """
+ A meta path importer to import six.moves and its submodules.
+
+ This class implements a PEP302 finder and loader. It should be compatible
+ with Python 2.5 and all existing versions of Python3
+ """
+
+ def __init__(self, six_module_name):
+ self.name = six_module_name
+ self.known_modules = {}
+
+ def _add_module(self, mod, *fullnames):
+ for fullname in fullnames:
+ self.known_modules[self.name + "." + fullname] = mod
+
+ def _get_module(self, fullname):
+ return self.known_modules[self.name + "." + fullname]
+
+ def find_module(self, fullname, path=None):
+ if fullname in self.known_modules:
+ return self
+ return None
+
+ def __get_module(self, fullname):
+ try:
+ return self.known_modules[fullname]
+ except KeyError:
+ raise ImportError("This loader does not know module " + fullname)
+
+ def load_module(self, fullname):
+ try:
+ # in case of a reload
+ return sys.modules[fullname]
+ except KeyError:
+ pass
+ mod = self.__get_module(fullname)
+ if isinstance(mod, MovedModule):
+ mod = mod._resolve()
+ else:
+ mod.__loader__ = self
+ sys.modules[fullname] = mod
+ return mod
+
+ def is_package(self, fullname):
+ """
+ Return true, if the named module is a package.
+
+ We need this method to get correct spec objects with
+ Python 3.4 (see PEP451)
+ """
+ return hasattr(self.__get_module(fullname), "__path__")
+
+ def get_code(self, fullname):
+ """Return None
+
+ Required, if is_package is implemented"""
+ self.__get_module(fullname) # eventually raises ImportError
+ return None
+ get_source = get_code # same as get_code
+
+_importer = _SixMetaPathImporter(__name__)
+
+
+class _MovedItems(_LazyModule):
+
+ """Lazy loading of moved objects"""
+ __path__ = [] # mark as package
+
+
+_moved_attributes = [
+ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+ MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+ MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+ MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+ MovedAttribute("intern", "__builtin__", "sys"),
+ MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+ MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
+ MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
+ MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
+ MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
+ MovedAttribute("reduce", "__builtin__", "functools"),
+ MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
+ MovedAttribute("StringIO", "StringIO", "io"),
+ MovedAttribute("UserDict", "UserDict", "collections"),
+ MovedAttribute("UserList", "UserList", "collections"),
+ MovedAttribute("UserString", "UserString", "collections"),
+ MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+ MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+ MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+ MovedModule("builtins", "__builtin__"),
+ MovedModule("configparser", "ConfigParser"),
+ MovedModule("copyreg", "copy_reg"),
+ MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
+ MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
+ MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+ MovedModule("http_cookies", "Cookie", "http.cookies"),
+ MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+ MovedModule("html_parser", "HTMLParser", "html.parser"),
+ MovedModule("http_client", "httplib", "http.client"),
+ MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+ MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
+ MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+ MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+ MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+ MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+ MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+ MovedModule("cPickle", "cPickle", "pickle"),
+ MovedModule("queue", "Queue"),
+ MovedModule("reprlib", "repr"),
+ MovedModule("socketserver", "SocketServer"),
+ MovedModule("_thread", "thread", "_thread"),
+ MovedModule("tkinter", "Tkinter"),
+ MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+ MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+ MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+ MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+ MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+ MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
+ MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+ MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+ MovedModule("tkinter_colorchooser", "tkColorChooser",
+ "tkinter.colorchooser"),
+ MovedModule("tkinter_commondialog", "tkCommonDialog",
+ "tkinter.commondialog"),
+ MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+ MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+ MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+ MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+ "tkinter.simpledialog"),
+ MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
+ MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
+ MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
+ MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+ MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
+ MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
+]
+# Add windows specific modules.
+if sys.platform == "win32":
+ _moved_attributes += [
+ MovedModule("winreg", "_winreg"),
+ ]
+
+for attr in _moved_attributes:
+ setattr(_MovedItems, attr.name, attr)
+ if isinstance(attr, MovedModule):
+ _importer._add_module(attr, "moves." + attr.name)
+del attr
+
+_MovedItems._moved_attributes = _moved_attributes
+
+moves = _MovedItems(__name__ + ".moves")
+_importer._add_module(moves, "moves")
+
+
+class Module_six_moves_urllib_parse(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_parse"""
+
+
+_urllib_parse_moved_attributes = [
+ MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
+ MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
+ MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
+ MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
+ MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
+ MovedAttribute("urljoin", "urlparse", "urllib.parse"),
+ MovedAttribute("urlparse", "urlparse", "urllib.parse"),
+ MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
+ MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
+ MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
+ MovedAttribute("quote", "urllib", "urllib.parse"),
+ MovedAttribute("quote_plus", "urllib", "urllib.parse"),
+ MovedAttribute("unquote", "urllib", "urllib.parse"),
+ MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+ MovedAttribute("urlencode", "urllib", "urllib.parse"),
+ MovedAttribute("splitquery", "urllib", "urllib.parse"),
+ MovedAttribute("splittag", "urllib", "urllib.parse"),
+ MovedAttribute("splituser", "urllib", "urllib.parse"),
+ MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
+ MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
+ MovedAttribute("uses_params", "urlparse", "urllib.parse"),
+ MovedAttribute("uses_query", "urlparse", "urllib.parse"),
+ MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
+]
+for attr in _urllib_parse_moved_attributes:
+ setattr(Module_six_moves_urllib_parse, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+ "moves.urllib_parse", "moves.urllib.parse")
+
+
+class Module_six_moves_urllib_error(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_error"""
+
+
+_urllib_error_moved_attributes = [
+ MovedAttribute("URLError", "urllib2", "urllib.error"),
+ MovedAttribute("HTTPError", "urllib2", "urllib.error"),
+ MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
+]
+for attr in _urllib_error_moved_attributes:
+ setattr(Module_six_moves_urllib_error, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+ "moves.urllib_error", "moves.urllib.error")
+
+
+class Module_six_moves_urllib_request(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_request"""
+
+
+_urllib_request_moved_attributes = [
+ MovedAttribute("urlopen", "urllib2", "urllib.request"),
+ MovedAttribute("install_opener", "urllib2", "urllib.request"),
+ MovedAttribute("build_opener", "urllib2", "urllib.request"),
+ MovedAttribute("pathname2url", "urllib", "urllib.request"),
+ MovedAttribute("url2pathname", "urllib", "urllib.request"),
+ MovedAttribute("getproxies", "urllib", "urllib.request"),
+ MovedAttribute("Request", "urllib2", "urllib.request"),
+ MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
+ MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
+ MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
+ MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
+ MovedAttribute("FileHandler", "urllib2", "urllib.request"),
+ MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
+ MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
+ MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
+ MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
+ MovedAttribute("urlretrieve", "urllib", "urllib.request"),
+ MovedAttribute("urlcleanup", "urllib", "urllib.request"),
+ MovedAttribute("URLopener", "urllib", "urllib.request"),
+ MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
+ MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+]
+for attr in _urllib_request_moved_attributes:
+ setattr(Module_six_moves_urllib_request, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+ "moves.urllib_request", "moves.urllib.request")
+
+
+class Module_six_moves_urllib_response(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_response"""
+
+
+_urllib_response_moved_attributes = [
+ MovedAttribute("addbase", "urllib", "urllib.response"),
+ MovedAttribute("addclosehook", "urllib", "urllib.response"),
+ MovedAttribute("addinfo", "urllib", "urllib.response"),
+ MovedAttribute("addinfourl", "urllib", "urllib.response"),
+]
+for attr in _urllib_response_moved_attributes:
+ setattr(Module_six_moves_urllib_response, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+ "moves.urllib_response", "moves.urllib.response")
+
+
+class Module_six_moves_urllib_robotparser(_LazyModule):
+
+ """Lazy loading of moved objects in six.moves.urllib_robotparser"""
+
+
+_urllib_robotparser_moved_attributes = [
+ MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+]
+for attr in _urllib_robotparser_moved_attributes:
+ setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+ "moves.urllib_robotparser", "moves.urllib.robotparser")
+
+
+class Module_six_moves_urllib(types.ModuleType):
+
+ """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+ __path__ = [] # mark as package
+ parse = _importer._get_module("moves.urllib_parse")
+ error = _importer._get_module("moves.urllib_error")
+ request = _importer._get_module("moves.urllib_request")
+ response = _importer._get_module("moves.urllib_response")
+ robotparser = _importer._get_module("moves.urllib_robotparser")
+
+ def __dir__(self):
+ return ['parse', 'error', 'request', 'response', 'robotparser']
+
+_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
+ "moves.urllib")
+
+
+def add_move(move):
+ """Add an item to six.moves."""
+ setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+ """Remove item from six.moves."""
+ try:
+ delattr(_MovedItems, name)
+ except AttributeError:
+ try:
+ del moves.__dict__[name]
+ except KeyError:
+ raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+ _meth_func = "__func__"
+ _meth_self = "__self__"
+
+ _func_closure = "__closure__"
+ _func_code = "__code__"
+ _func_defaults = "__defaults__"
+ _func_globals = "__globals__"
+else:
+ _meth_func = "im_func"
+ _meth_self = "im_self"
+
+ _func_closure = "func_closure"
+ _func_code = "func_code"
+ _func_defaults = "func_defaults"
+ _func_globals = "func_globals"
+
+
+try:
+ advance_iterator = next
+except NameError:
+ def advance_iterator(it):
+ return it.next()
+next = advance_iterator
+
+
+try:
+ callable = callable
+except NameError:
+ def callable(obj):
+ return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+ def get_unbound_function(unbound):
+ return unbound
+
+ create_bound_method = types.MethodType
+
+ def create_unbound_method(func, cls):
+ return func
+
+ Iterator = object
+else:
+ def get_unbound_function(unbound):
+ return unbound.im_func
+
+ def create_bound_method(func, obj):
+ return types.MethodType(func, obj, obj.__class__)
+
+ def create_unbound_method(func, cls):
+ return types.MethodType(func, None, cls)
+
+ class Iterator(object):
+
+ def next(self):
+ return type(self).__next__(self)
+
+ callable = callable
+_add_doc(get_unbound_function,
+ """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_closure = operator.attrgetter(_func_closure)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+get_function_globals = operator.attrgetter(_func_globals)
+
+
+if PY3:
+ def iterkeys(d, **kw):
+ return iter(d.keys(**kw))
+
+ def itervalues(d, **kw):
+ return iter(d.values(**kw))
+
+ def iteritems(d, **kw):
+ return iter(d.items(**kw))
+
+ def iterlists(d, **kw):
+ return iter(d.lists(**kw))
+
+ viewkeys = operator.methodcaller("keys")
+
+ viewvalues = operator.methodcaller("values")
+
+ viewitems = operator.methodcaller("items")
+else:
+ def iterkeys(d, **kw):
+ return d.iterkeys(**kw)
+
+ def itervalues(d, **kw):
+ return d.itervalues(**kw)
+
+ def iteritems(d, **kw):
+ return d.iteritems(**kw)
+
+ def iterlists(d, **kw):
+ return d.iterlists(**kw)
+
+ viewkeys = operator.methodcaller("viewkeys")
+
+ viewvalues = operator.methodcaller("viewvalues")
+
+ viewitems = operator.methodcaller("viewitems")
+
+_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
+_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
+_add_doc(iteritems,
+ "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(iterlists,
+ "Return an iterator over the (key, [values]) pairs of a dictionary.")
+
+
+if PY3:
+ def b(s):
+ return s.encode("latin-1")
+
+ def u(s):
+ return s
+ unichr = chr
+ import struct
+ int2byte = struct.Struct(">B").pack
+ del struct
+ byte2int = operator.itemgetter(0)
+ indexbytes = operator.getitem
+ iterbytes = iter
+ import io
+ StringIO = io.StringIO
+ BytesIO = io.BytesIO
+ _assertCountEqual = "assertCountEqual"
+ if sys.version_info[1] <= 1:
+ _assertRaisesRegex = "assertRaisesRegexp"
+ _assertRegex = "assertRegexpMatches"
+ else:
+ _assertRaisesRegex = "assertRaisesRegex"
+ _assertRegex = "assertRegex"
+else:
+ def b(s):
+ return s
+ # Workaround for standalone backslash
+
+ def u(s):
+ return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+ unichr = unichr
+ int2byte = chr
+
+ def byte2int(bs):
+ return ord(bs[0])
+
+ def indexbytes(buf, i):
+ return ord(buf[i])
+ iterbytes = functools.partial(itertools.imap, ord)
+ import StringIO
+ StringIO = BytesIO = StringIO.StringIO
+ _assertCountEqual = "assertItemsEqual"
+ _assertRaisesRegex = "assertRaisesRegexp"
+ _assertRegex = "assertRegexpMatches"
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+def assertCountEqual(self, *args, **kwargs):
+ return getattr(self, _assertCountEqual)(*args, **kwargs)
+
+
+def assertRaisesRegex(self, *args, **kwargs):
+ return getattr(self, _assertRaisesRegex)(*args, **kwargs)
+
+
+def assertRegex(self, *args, **kwargs):
+ return getattr(self, _assertRegex)(*args, **kwargs)
+
+
+if PY3:
+ exec_ = getattr(moves.builtins, "exec")
+
+ def reraise(tp, value, tb=None):
+ if value is None:
+ value = tp()
+ if value.__traceback__ is not tb:
+ raise value.with_traceback(tb)
+ raise value
+
+else:
+ def exec_(_code_, _globs_=None, _locs_=None):
+ """Execute code in a namespace."""
+ if _globs_ is None:
+ frame = sys._getframe(1)
+ _globs_ = frame.f_globals
+ if _locs_ is None:
+ _locs_ = frame.f_locals
+ del frame
+ elif _locs_ is None:
+ _locs_ = _globs_
+ exec("""exec _code_ in _globs_, _locs_""")
+
+ exec_("""def reraise(tp, value, tb=None):
+ raise tp, value, tb
+""")
+
+
+if sys.version_info[:2] == (3, 2):
+ exec_("""def raise_from(value, from_value):
+ if from_value is None:
+ raise value
+ raise value from from_value
+""")
+elif sys.version_info[:2] > (3, 2):
+ exec_("""def raise_from(value, from_value):
+ raise value from from_value
+""")
+else:
+ def raise_from(value, from_value):
+ raise value
+
+
+print_ = getattr(moves.builtins, "print", None)
+if print_ is None:
+ def print_(*args, **kwargs):
+ """The new-style print function for Python 2.4 and 2.5."""
+ fp = kwargs.pop("file", sys.stdout)
+ if fp is None:
+ return
+
+ def write(data):
+ if not isinstance(data, basestring):
+ data = str(data)
+ # If the file has an encoding, encode unicode with it.
+ if (isinstance(fp, file) and
+ isinstance(data, unicode) and
+ fp.encoding is not None):
+ errors = getattr(fp, "errors", None)
+ if errors is None:
+ errors = "strict"
+ data = data.encode(fp.encoding, errors)
+ fp.write(data)
+ want_unicode = False
+ sep = kwargs.pop("sep", None)
+ if sep is not None:
+ if isinstance(sep, unicode):
+ want_unicode = True
+ elif not isinstance(sep, str):
+ raise TypeError("sep must be None or a string")
+ end = kwargs.pop("end", None)
+ if end is not None:
+ if isinstance(end, unicode):
+ want_unicode = True
+ elif not isinstance(end, str):
+ raise TypeError("end must be None or a string")
+ if kwargs:
+ raise TypeError("invalid keyword arguments to print()")
+ if not want_unicode:
+ for arg in args:
+ if isinstance(arg, unicode):
+ want_unicode = True
+ break
+ if want_unicode:
+ newline = unicode("\n")
+ space = unicode(" ")
+ else:
+ newline = "\n"
+ space = " "
+ if sep is None:
+ sep = space
+ if end is None:
+ end = newline
+ for i, arg in enumerate(args):
+ if i:
+ write(sep)
+ write(arg)
+ write(end)
+if sys.version_info[:2] < (3, 3):
+ _print = print_
+
+ def print_(*args, **kwargs):
+ fp = kwargs.get("file", sys.stdout)
+ flush = kwargs.pop("flush", False)
+ _print(*args, **kwargs)
+ if flush and fp is not None:
+ fp.flush()
+
+_add_doc(reraise, """Reraise an exception.""")
+
+if sys.version_info[0:2] < (3, 4):
+ def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
+ updated=functools.WRAPPER_UPDATES):
+ def wrapper(f):
+ f = functools.wraps(wrapped, assigned, updated)(f)
+ f.__wrapped__ = wrapped
+ return f
+ return wrapper
+else:
+ wraps = functools.wraps
+
+
+def with_metaclass(meta, *bases):
+ """Create a base class with a metaclass."""
+ # This requires a bit of explanation: the basic idea is to make a dummy
+ # metaclass for one level of class instantiation that replaces itself with
+ # the actual metaclass.
+ class metaclass(meta):
+
+ def __new__(cls, name, this_bases, d):
+ return meta(name, bases, d)
+ return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+def add_metaclass(metaclass):
+ """Class decorator for creating a class with a metaclass."""
+ def wrapper(cls):
+ orig_vars = cls.__dict__.copy()
+ slots = orig_vars.get('__slots__')
+ if slots is not None:
+ if isinstance(slots, str):
+ slots = [slots]
+ for slots_var in slots:
+ orig_vars.pop(slots_var)
+ orig_vars.pop('__dict__', None)
+ orig_vars.pop('__weakref__', None)
+ return metaclass(cls.__name__, cls.__bases__, orig_vars)
+ return wrapper
+
+
+def python_2_unicode_compatible(klass):
+ """
+ A decorator that defines __unicode__ and __str__ methods under Python 2.
+ Under Python 3 it does nothing.
+
+ To support Python 2 and 3 with a single code base, define a __str__ method
+ returning text and apply this decorator to the class.
+ """
+ if PY2:
+ if '__str__' not in klass.__dict__:
+ raise ValueError("@python_2_unicode_compatible cannot be applied "
+ "to %s because it doesn't define __str__()." %
+ klass.__name__)
+ klass.__unicode__ = klass.__str__
+ klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+ return klass
+
+
+# Complete the moves implementation.
+# This code is at the end of this module to speed up module loading.
+# Turn this module into a package.
+__path__ = [] # required for PEP 302 and PEP 451
+__package__ = __name__ # see PEP 366 @ReservedAssignment
+if globals().get("__spec__") is not None:
+ __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable
+# Remove other six meta path importers, since they cause problems. This can
+# happen if six is removed from sys.modules and then reloaded. (Setuptools does
+# this for some reason.)
+if sys.meta_path:
+ for i, importer in enumerate(sys.meta_path):
+ # Here's some real nastiness: Another "instance" of the six module might
+ # be floating around. Therefore, we can't use isinstance() to check for
+ # the six meta path importer, since the other six instance will have
+ # inserted an importer with different class.
+ if (type(importer).__name__ == "_SixMetaPathImporter" and
+ importer.name == __name__):
+ del sys.meta_path[i]
+ break
+ del i, importer
+# Finally, add the importer to the meta path import hook.
+sys.meta_path.append(_importer)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/packages/ssl_match_hostname/__init__.py b/collectors/python.d.plugin/python_modules/urllib3/packages/ssl_match_hostname/__init__.py
new file mode 100644
index 0000000..2aeeeff
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/packages/ssl_match_hostname/__init__.py
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: MIT
+import sys
+
+try:
+ # Our match_hostname function is the same as 3.5's, so we only want to
+ # import the match_hostname function if it's at least that good.
+ if sys.version_info < (3, 5):
+ raise ImportError("Fallback to vendored code")
+
+ from ssl import CertificateError, match_hostname
+except ImportError:
+ try:
+ # Backport of the function from a pypi module
+ from backports.ssl_match_hostname import CertificateError, match_hostname
+ except ImportError:
+ # Our vendored copy
+ from ._implementation import CertificateError, match_hostname
+
+# Not needed, but documenting what we provide.
+__all__ = ('CertificateError', 'match_hostname')
diff --git a/collectors/python.d.plugin/python_modules/urllib3/packages/ssl_match_hostname/_implementation.py b/collectors/python.d.plugin/python_modules/urllib3/packages/ssl_match_hostname/_implementation.py
new file mode 100644
index 0000000..647e081
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/packages/ssl_match_hostname/_implementation.py
@@ -0,0 +1,156 @@
+"""The match_hostname() function from Python 3.3.3, essential when using SSL."""
+
+# SPDX-License-Identifier: Python-2.0
+
+import re
+import sys
+
+# ipaddress has been backported to 2.6+ in pypi. If it is installed on the
+# system, use it to handle IPAddress ServerAltnames (this was added in
+# python-3.5) otherwise only do DNS matching. This allows
+# backports.ssl_match_hostname to continue to be used all the way back to
+# python-2.4.
+try:
+ import ipaddress
+except ImportError:
+ ipaddress = None
+
+__version__ = '3.5.0.1'
+
+
+class CertificateError(ValueError):
+ pass
+
+
+def _dnsname_match(dn, hostname, max_wildcards=1):
+ """Matching according to RFC 6125, section 6.4.3
+
+ http://tools.ietf.org/html/rfc6125#section-6.4.3
+ """
+ pats = []
+ if not dn:
+ return False
+
+ # Ported from python3-syntax:
+ # leftmost, *remainder = dn.split(r'.')
+ parts = dn.split(r'.')
+ leftmost = parts[0]
+ remainder = parts[1:]
+
+ wildcards = leftmost.count('*')
+ if wildcards > max_wildcards:
+ # Issue #17980: avoid denials of service by refusing more
+ # than one wildcard per fragment. A survey of established
+ # policy among SSL implementations showed it to be a
+ # reasonable choice.
+ raise CertificateError(
+ "too many wildcards in certificate DNS name: " + repr(dn))
+
+ # speed up common case w/o wildcards
+ if not wildcards:
+ return dn.lower() == hostname.lower()
+
+ # RFC 6125, section 6.4.3, subitem 1.
+ # The client SHOULD NOT attempt to match a presented identifier in which
+ # the wildcard character comprises a label other than the left-most label.
+ if leftmost == '*':
+ # When '*' is a fragment by itself, it matches a non-empty dotless
+ # fragment.
+ pats.append('[^.]+')
+ elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
+ # RFC 6125, section 6.4.3, subitem 3.
+ # The client SHOULD NOT attempt to match a presented identifier
+ # where the wildcard character is embedded within an A-label or
+ # U-label of an internationalized domain name.
+ pats.append(re.escape(leftmost))
+ else:
+ # Otherwise, '*' matches any dotless string, e.g. www*
+ pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
+
+ # add the remaining fragments, ignore any wildcards
+ for frag in remainder:
+ pats.append(re.escape(frag))
+
+ pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
+ return pat.match(hostname)
+
+
+def _to_unicode(obj):
+ if isinstance(obj, str) and sys.version_info < (3,):
+ obj = unicode(obj, encoding='ascii', errors='strict')
+ return obj
+
+def _ipaddress_match(ipname, host_ip):
+ """Exact matching of IP addresses.
+
+ RFC 6125 explicitly doesn't define an algorithm for this
+ (section 1.7.2 - "Out of Scope").
+ """
+ # OpenSSL may add a trailing newline to a subjectAltName's IP address
+ # Divergence from upstream: ipaddress can't handle byte str
+ ip = ipaddress.ip_address(_to_unicode(ipname).rstrip())
+ return ip == host_ip
+
+
+def match_hostname(cert, hostname):
+ """Verify that *cert* (in decoded format as returned by
+ SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125
+ rules are followed, but IP addresses are not accepted for *hostname*.
+
+ CertificateError is raised on failure. On success, the function
+ returns nothing.
+ """
+ if not cert:
+ raise ValueError("empty or no certificate, match_hostname needs a "
+ "SSL socket or SSL context with either "
+ "CERT_OPTIONAL or CERT_REQUIRED")
+ try:
+ # Divergence from upstream: ipaddress can't handle byte str
+ host_ip = ipaddress.ip_address(_to_unicode(hostname))
+ except ValueError:
+ # Not an IP address (common case)
+ host_ip = None
+ except UnicodeError:
+ # Divergence from upstream: Have to deal with ipaddress not taking
+ # byte strings. addresses should be all ascii, so we consider it not
+ # an ipaddress in this case
+ host_ip = None
+ except AttributeError:
+ # Divergence from upstream: Make ipaddress library optional
+ if ipaddress is None:
+ host_ip = None
+ else:
+ raise
+ dnsnames = []
+ san = cert.get('subjectAltName', ())
+ for key, value in san:
+ if key == 'DNS':
+ if host_ip is None and _dnsname_match(value, hostname):
+ return
+ dnsnames.append(value)
+ elif key == 'IP Address':
+ if host_ip is not None and _ipaddress_match(value, host_ip):
+ return
+ dnsnames.append(value)
+ if not dnsnames:
+ # The subject is only checked when there is no dNSName entry
+ # in subjectAltName
+ for sub in cert.get('subject', ()):
+ for key, value in sub:
+ # XXX according to RFC 2818, the most specific Common Name
+ # must be used.
+ if key == 'commonName':
+ if _dnsname_match(value, hostname):
+ return
+ dnsnames.append(value)
+ if len(dnsnames) > 1:
+ raise CertificateError("hostname %r "
+ "doesn't match either of %s"
+ % (hostname, ', '.join(map(repr, dnsnames))))
+ elif len(dnsnames) == 1:
+ raise CertificateError("hostname %r "
+ "doesn't match %r"
+ % (hostname, dnsnames[0]))
+ else:
+ raise CertificateError("no appropriate commonName or "
+ "subjectAltName fields were found")
diff --git a/collectors/python.d.plugin/python_modules/urllib3/poolmanager.py b/collectors/python.d.plugin/python_modules/urllib3/poolmanager.py
new file mode 100644
index 0000000..adea9bc
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/poolmanager.py
@@ -0,0 +1,441 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+import collections
+import functools
+import logging
+
+from ._collections import RecentlyUsedContainer
+from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+from .connectionpool import port_by_scheme
+from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown
+from .packages.six.moves.urllib.parse import urljoin
+from .request import RequestMethods
+from .util.url import parse_url
+from .util.retry import Retry
+
+
+__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
+
+
+log = logging.getLogger(__name__)
+
+SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
+ 'ssl_version', 'ca_cert_dir', 'ssl_context')
+
+# All known keyword arguments that could be provided to the pool manager, its
+# pools, or the underlying connections. This is used to construct a pool key.
+_key_fields = (
+ 'key_scheme', # str
+ 'key_host', # str
+ 'key_port', # int
+ 'key_timeout', # int or float or Timeout
+ 'key_retries', # int or Retry
+ 'key_strict', # bool
+ 'key_block', # bool
+ 'key_source_address', # str
+ 'key_key_file', # str
+ 'key_cert_file', # str
+ 'key_cert_reqs', # str
+ 'key_ca_certs', # str
+ 'key_ssl_version', # str
+ 'key_ca_cert_dir', # str
+ 'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
+ 'key_maxsize', # int
+ 'key_headers', # dict
+ 'key__proxy', # parsed proxy url
+ 'key__proxy_headers', # dict
+ 'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples
+ 'key__socks_options', # dict
+ 'key_assert_hostname', # bool or string
+ 'key_assert_fingerprint', # str
+)
+
+#: The namedtuple class used to construct keys for the connection pool.
+#: All custom key schemes should include the fields in this key at a minimum.
+PoolKey = collections.namedtuple('PoolKey', _key_fields)
+
+
+def _default_key_normalizer(key_class, request_context):
+ """
+ Create a pool key out of a request context dictionary.
+
+ According to RFC 3986, both the scheme and host are case-insensitive.
+ Therefore, this function normalizes both before constructing the pool
+ key for an HTTPS request. If you wish to change this behaviour, provide
+ alternate callables to ``key_fn_by_scheme``.
+
+ :param key_class:
+ The class to use when constructing the key. This should be a namedtuple
+ with the ``scheme`` and ``host`` keys at a minimum.
+ :type key_class: namedtuple
+ :param request_context:
+ A dictionary-like object that contain the context for a request.
+ :type request_context: dict
+
+ :return: A namedtuple that can be used as a connection pool key.
+ :rtype: PoolKey
+ """
+ # Since we mutate the dictionary, make a copy first
+ context = request_context.copy()
+ context['scheme'] = context['scheme'].lower()
+ context['host'] = context['host'].lower()
+
+ # These are both dictionaries and need to be transformed into frozensets
+ for key in ('headers', '_proxy_headers', '_socks_options'):
+ if key in context and context[key] is not None:
+ context[key] = frozenset(context[key].items())
+
+ # The socket_options key may be a list and needs to be transformed into a
+ # tuple.
+ socket_opts = context.get('socket_options')
+ if socket_opts is not None:
+ context['socket_options'] = tuple(socket_opts)
+
+ # Map the kwargs to the names in the namedtuple - this is necessary since
+ # namedtuples can't have fields starting with '_'.
+ for key in list(context.keys()):
+ context['key_' + key] = context.pop(key)
+
+ # Default to ``None`` for keys missing from the context
+ for field in key_class._fields:
+ if field not in context:
+ context[field] = None
+
+ return key_class(**context)
+
+
+#: A dictionary that maps a scheme to a callable that creates a pool key.
+#: This can be used to alter the way pool keys are constructed, if desired.
+#: Each PoolManager makes a copy of this dictionary so they can be configured
+#: globally here, or individually on the instance.
+key_fn_by_scheme = {
+ 'http': functools.partial(_default_key_normalizer, PoolKey),
+ 'https': functools.partial(_default_key_normalizer, PoolKey),
+}
+
+pool_classes_by_scheme = {
+ 'http': HTTPConnectionPool,
+ 'https': HTTPSConnectionPool,
+}
+
+
+class PoolManager(RequestMethods):
+ """
+ Allows for arbitrary requests while transparently keeping track of
+ necessary connection pools for you.
+
+ :param num_pools:
+ Number of connection pools to cache before discarding the least
+ recently used pool.
+
+ :param headers:
+ Headers to include with all requests, unless other headers are given
+ explicitly.
+
+ :param \\**connection_pool_kw:
+ Additional parameters are used to create fresh
+ :class:`urllib3.connectionpool.ConnectionPool` instances.
+
+ Example::
+
+ >>> manager = PoolManager(num_pools=2)
+ >>> r = manager.request('GET', 'http://google.com/')
+ >>> r = manager.request('GET', 'http://google.com/mail')
+ >>> r = manager.request('GET', 'http://yahoo.com/')
+ >>> len(manager.pools)
+ 2
+
+ """
+
+ proxy = None
+
+ def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
+ RequestMethods.__init__(self, headers)
+ self.connection_pool_kw = connection_pool_kw
+ self.pools = RecentlyUsedContainer(num_pools,
+ dispose_func=lambda p: p.close())
+
+ # Locally set the pool classes and keys so other PoolManagers can
+ # override them.
+ self.pool_classes_by_scheme = pool_classes_by_scheme
+ self.key_fn_by_scheme = key_fn_by_scheme.copy()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.clear()
+ # Return False to re-raise any potential exceptions
+ return False
+
+ def _new_pool(self, scheme, host, port, request_context=None):
+ """
+ Create a new :class:`ConnectionPool` based on host, port, scheme, and
+ any additional pool keyword arguments.
+
+ If ``request_context`` is provided, it is provided as keyword arguments
+ to the pool class used. This method is used to actually create the
+ connection pools handed out by :meth:`connection_from_url` and
+ companion methods. It is intended to be overridden for customization.
+ """
+ pool_cls = self.pool_classes_by_scheme[scheme]
+ if request_context is None:
+ request_context = self.connection_pool_kw.copy()
+
+ # Although the context has everything necessary to create the pool,
+ # this function has historically only used the scheme, host, and port
+ # in the positional args. When an API change is acceptable these can
+ # be removed.
+ for key in ('scheme', 'host', 'port'):
+ request_context.pop(key, None)
+
+ if scheme == 'http':
+ for kw in SSL_KEYWORDS:
+ request_context.pop(kw, None)
+
+ return pool_cls(host, port, **request_context)
+
+ def clear(self):
+ """
+ Empty our store of pools and direct them all to close.
+
+ This will not affect in-flight connections, but they will not be
+ re-used after completion.
+ """
+ self.pools.clear()
+
+ def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None):
+ """
+ Get a :class:`ConnectionPool` based on the host, port, and scheme.
+
+ If ``port`` isn't given, it will be derived from the ``scheme`` using
+ ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
+ provided, it is merged with the instance's ``connection_pool_kw``
+ variable and used to create the new connection pool, if one is
+ needed.
+ """
+
+ if not host:
+ raise LocationValueError("No host specified.")
+
+ request_context = self._merge_pool_kwargs(pool_kwargs)
+ request_context['scheme'] = scheme or 'http'
+ if not port:
+ port = port_by_scheme.get(request_context['scheme'].lower(), 80)
+ request_context['port'] = port
+ request_context['host'] = host
+
+ return self.connection_from_context(request_context)
+
+ def connection_from_context(self, request_context):
+ """
+ Get a :class:`ConnectionPool` based on the request context.
+
+ ``request_context`` must at least contain the ``scheme`` key and its
+ value must be a key in ``key_fn_by_scheme`` instance variable.
+ """
+ scheme = request_context['scheme'].lower()
+ pool_key_constructor = self.key_fn_by_scheme[scheme]
+ pool_key = pool_key_constructor(request_context)
+
+ return self.connection_from_pool_key(pool_key, request_context=request_context)
+
+ def connection_from_pool_key(self, pool_key, request_context=None):
+ """
+ Get a :class:`ConnectionPool` based on the provided pool key.
+
+ ``pool_key`` should be a namedtuple that only contains immutable
+ objects. At a minimum it must have the ``scheme``, ``host``, and
+ ``port`` fields.
+ """
+ with self.pools.lock:
+ # If the scheme, host, or port doesn't match existing open
+ # connections, open a new ConnectionPool.
+ pool = self.pools.get(pool_key)
+ if pool:
+ return pool
+
+ # Make a fresh ConnectionPool of the desired type
+ scheme = request_context['scheme']
+ host = request_context['host']
+ port = request_context['port']
+ pool = self._new_pool(scheme, host, port, request_context=request_context)
+ self.pools[pool_key] = pool
+
+ return pool
+
+ def connection_from_url(self, url, pool_kwargs=None):
+ """
+ Similar to :func:`urllib3.connectionpool.connection_from_url`.
+
+ If ``pool_kwargs`` is not provided and a new pool needs to be
+ constructed, ``self.connection_pool_kw`` is used to initialize
+ the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
+ is provided, it is used instead. Note that if a new pool does not
+ need to be created for the request, the provided ``pool_kwargs`` are
+ not used.
+ """
+ u = parse_url(url)
+ return self.connection_from_host(u.host, port=u.port, scheme=u.scheme,
+ pool_kwargs=pool_kwargs)
+
+ def _merge_pool_kwargs(self, override):
+ """
+ Merge a dictionary of override values for self.connection_pool_kw.
+
+ This does not modify self.connection_pool_kw and returns a new dict.
+ Any keys in the override dictionary with a value of ``None`` are
+ removed from the merged dictionary.
+ """
+ base_pool_kwargs = self.connection_pool_kw.copy()
+ if override:
+ for key, value in override.items():
+ if value is None:
+ try:
+ del base_pool_kwargs[key]
+ except KeyError:
+ pass
+ else:
+ base_pool_kwargs[key] = value
+ return base_pool_kwargs
+
+ def urlopen(self, method, url, redirect=True, **kw):
+ """
+ Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
+ with custom cross-host redirect logic and only sends the request-uri
+ portion of the ``url``.
+
+ The given ``url`` parameter must be absolute, such that an appropriate
+ :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
+ """
+ u = parse_url(url)
+ conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
+
+ kw['assert_same_host'] = False
+ kw['redirect'] = False
+ if 'headers' not in kw:
+ kw['headers'] = self.headers
+
+ if self.proxy is not None and u.scheme == "http":
+ response = conn.urlopen(method, url, **kw)
+ else:
+ response = conn.urlopen(method, u.request_uri, **kw)
+
+ redirect_location = redirect and response.get_redirect_location()
+ if not redirect_location:
+ return response
+
+ # Support relative URLs for redirecting.
+ redirect_location = urljoin(url, redirect_location)
+
+ # RFC 7231, Section 6.4.4
+ if response.status == 303:
+ method = 'GET'
+
+ retries = kw.get('retries')
+ if not isinstance(retries, Retry):
+ retries = Retry.from_int(retries, redirect=redirect)
+
+ try:
+ retries = retries.increment(method, url, response=response, _pool=conn)
+ except MaxRetryError:
+ if retries.raise_on_redirect:
+ raise
+ return response
+
+ kw['retries'] = retries
+ kw['redirect'] = redirect
+
+ log.info("Redirecting %s -> %s", url, redirect_location)
+ return self.urlopen(method, redirect_location, **kw)
+
+
+class ProxyManager(PoolManager):
+ """
+ Behaves just like :class:`PoolManager`, but sends all requests through
+ the defined proxy, using the CONNECT method for HTTPS URLs.
+
+ :param proxy_url:
+ The URL of the proxy to be used.
+
+ :param proxy_headers:
+ A dictionary contaning headers that will be sent to the proxy. In case
+ of HTTP they are being sent with each request, while in the
+ HTTPS/CONNECT case they are sent only once. Could be used for proxy
+ authentication.
+
+ Example:
+ >>> proxy = urllib3.ProxyManager('http://localhost:3128/')
+ >>> r1 = proxy.request('GET', 'http://google.com/')
+ >>> r2 = proxy.request('GET', 'http://httpbin.org/')
+ >>> len(proxy.pools)
+ 1
+ >>> r3 = proxy.request('GET', 'https://httpbin.org/')
+ >>> r4 = proxy.request('GET', 'https://twitter.com/')
+ >>> len(proxy.pools)
+ 3
+
+ """
+
+ def __init__(self, proxy_url, num_pools=10, headers=None,
+ proxy_headers=None, **connection_pool_kw):
+
+ if isinstance(proxy_url, HTTPConnectionPool):
+ proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host,
+ proxy_url.port)
+ proxy = parse_url(proxy_url)
+ if not proxy.port:
+ port = port_by_scheme.get(proxy.scheme, 80)
+ proxy = proxy._replace(port=port)
+
+ if proxy.scheme not in ("http", "https"):
+ raise ProxySchemeUnknown(proxy.scheme)
+
+ self.proxy = proxy
+ self.proxy_headers = proxy_headers or {}
+
+ connection_pool_kw['_proxy'] = self.proxy
+ connection_pool_kw['_proxy_headers'] = self.proxy_headers
+
+ super(ProxyManager, self).__init__(
+ num_pools, headers, **connection_pool_kw)
+
+ def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None):
+ if scheme == "https":
+ return super(ProxyManager, self).connection_from_host(
+ host, port, scheme, pool_kwargs=pool_kwargs)
+
+ return super(ProxyManager, self).connection_from_host(
+ self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs)
+
+ def _set_proxy_headers(self, url, headers=None):
+ """
+ Sets headers needed by proxies: specifically, the Accept and Host
+ headers. Only sets headers not provided by the user.
+ """
+ headers_ = {'Accept': '*/*'}
+
+ netloc = parse_url(url).netloc
+ if netloc:
+ headers_['Host'] = netloc
+
+ if headers:
+ headers_.update(headers)
+ return headers_
+
+ def urlopen(self, method, url, redirect=True, **kw):
+ "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
+ u = parse_url(url)
+
+ if u.scheme == "http":
+ # For proxied HTTPS requests, httplib sets the necessary headers
+ # on the CONNECT to the proxy. For HTTP, we'll definitely
+ # need to set 'Host' at the very least.
+ headers = kw.get('headers', self.headers)
+ kw['headers'] = self._set_proxy_headers(url, headers)
+
+ return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
+
+
+def proxy_from_url(url, **kw):
+ return ProxyManager(proxy_url=url, **kw)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/request.py b/collectors/python.d.plugin/python_modules/urllib3/request.py
new file mode 100644
index 0000000..f783319
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/request.py
@@ -0,0 +1,149 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+
+from .filepost import encode_multipart_formdata
+from .packages.six.moves.urllib.parse import urlencode
+
+
+__all__ = ['RequestMethods']
+
+
+class RequestMethods(object):
+ """
+ Convenience mixin for classes who implement a :meth:`urlopen` method, such
+ as :class:`~urllib3.connectionpool.HTTPConnectionPool` and
+ :class:`~urllib3.poolmanager.PoolManager`.
+
+ Provides behavior for making common types of HTTP request methods and
+ decides which type of request field encoding to use.
+
+ Specifically,
+
+ :meth:`.request_encode_url` is for sending requests whose fields are
+ encoded in the URL (such as GET, HEAD, DELETE).
+
+ :meth:`.request_encode_body` is for sending requests whose fields are
+ encoded in the *body* of the request using multipart or www-form-urlencoded
+ (such as for POST, PUT, PATCH).
+
+ :meth:`.request` is for making any kind of request, it will look up the
+ appropriate encoding format and use one of the above two methods to make
+ the request.
+
+ Initializer parameters:
+
+ :param headers:
+ Headers to include with all requests, unless other headers are given
+ explicitly.
+ """
+
+ _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS'])
+
+ def __init__(self, headers=None):
+ self.headers = headers or {}
+
+ def urlopen(self, method, url, body=None, headers=None,
+ encode_multipart=True, multipart_boundary=None,
+ **kw): # Abstract
+ raise NotImplemented("Classes extending RequestMethods must implement "
+ "their own ``urlopen`` method.")
+
+ def request(self, method, url, fields=None, headers=None, **urlopen_kw):
+ """
+ Make a request using :meth:`urlopen` with the appropriate encoding of
+ ``fields`` based on the ``method`` used.
+
+ This is a convenience method that requires the least amount of manual
+ effort. It can be used in most situations, while still having the
+ option to drop down to more specific methods when necessary, such as
+ :meth:`request_encode_url`, :meth:`request_encode_body`,
+ or even the lowest level :meth:`urlopen`.
+ """
+ method = method.upper()
+
+ if method in self._encode_url_methods:
+ return self.request_encode_url(method, url, fields=fields,
+ headers=headers,
+ **urlopen_kw)
+ else:
+ return self.request_encode_body(method, url, fields=fields,
+ headers=headers,
+ **urlopen_kw)
+
+ def request_encode_url(self, method, url, fields=None, headers=None,
+ **urlopen_kw):
+ """
+ Make a request using :meth:`urlopen` with the ``fields`` encoded in
+ the url. This is useful for request methods like GET, HEAD, DELETE, etc.
+ """
+ if headers is None:
+ headers = self.headers
+
+ extra_kw = {'headers': headers}
+ extra_kw.update(urlopen_kw)
+
+ if fields:
+ url += '?' + urlencode(fields)
+
+ return self.urlopen(method, url, **extra_kw)
+
+ def request_encode_body(self, method, url, fields=None, headers=None,
+ encode_multipart=True, multipart_boundary=None,
+ **urlopen_kw):
+ """
+ Make a request using :meth:`urlopen` with the ``fields`` encoded in
+ the body. This is useful for request methods like POST, PUT, PATCH, etc.
+
+ When ``encode_multipart=True`` (default), then
+ :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode
+ the payload with the appropriate content type. Otherwise
+ :meth:`urllib.urlencode` is used with the
+ 'application/x-www-form-urlencoded' content type.
+
+ Multipart encoding must be used when posting files, and it's reasonably
+ safe to use it in other times too. However, it may break request
+ signing, such as with OAuth.
+
+ Supports an optional ``fields`` parameter of key/value strings AND
+ key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
+ the MIME type is optional. For example::
+
+ fields = {
+ 'foo': 'bar',
+ 'fakefile': ('foofile.txt', 'contents of foofile'),
+ 'realfile': ('barfile.txt', open('realfile').read()),
+ 'typedfile': ('bazfile.bin', open('bazfile').read(),
+ 'image/jpeg'),
+ 'nonamefile': 'contents of nonamefile field',
+ }
+
+ When uploading a file, providing a filename (the first parameter of the
+ tuple) is optional but recommended to best mimick behavior of browsers.
+
+ Note that if ``headers`` are supplied, the 'Content-Type' header will
+ be overwritten because it depends on the dynamic random boundary string
+ which is used to compose the body of the request. The random boundary
+ string can be explicitly set with the ``multipart_boundary`` parameter.
+ """
+ if headers is None:
+ headers = self.headers
+
+ extra_kw = {'headers': {}}
+
+ if fields:
+ if 'body' in urlopen_kw:
+ raise TypeError(
+ "request got values for both 'fields' and 'body', can only specify one.")
+
+ if encode_multipart:
+ body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
+ else:
+ body, content_type = urlencode(fields), 'application/x-www-form-urlencoded'
+
+ extra_kw['body'] = body
+ extra_kw['headers'] = {'Content-Type': content_type}
+
+ extra_kw['headers'].update(headers)
+ extra_kw.update(urlopen_kw)
+
+ return self.urlopen(method, url, **extra_kw)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/response.py b/collectors/python.d.plugin/python_modules/urllib3/response.py
new file mode 100644
index 0000000..cf14a30
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/response.py
@@ -0,0 +1,623 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+from contextlib import contextmanager
+import zlib
+import io
+import logging
+from socket import timeout as SocketTimeout
+from socket import error as SocketError
+
+from ._collections import HTTPHeaderDict
+from .exceptions import (
+ BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError,
+ ResponseNotChunked, IncompleteRead, InvalidHeader
+)
+from .packages.six import string_types as basestring, binary_type, PY3
+from .packages.six.moves import http_client as httplib
+from .connection import HTTPException, BaseSSLError
+from .util.response import is_fp_closed, is_response_to_head
+
+log = logging.getLogger(__name__)
+
+
+class DeflateDecoder(object):
+
+ def __init__(self):
+ self._first_try = True
+ self._data = binary_type()
+ self._obj = zlib.decompressobj()
+
+ def __getattr__(self, name):
+ return getattr(self._obj, name)
+
+ def decompress(self, data):
+ if not data:
+ return data
+
+ if not self._first_try:
+ return self._obj.decompress(data)
+
+ self._data += data
+ try:
+ decompressed = self._obj.decompress(data)
+ if decompressed:
+ self._first_try = False
+ self._data = None
+ return decompressed
+ except zlib.error:
+ self._first_try = False
+ self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
+ try:
+ return self.decompress(self._data)
+ finally:
+ self._data = None
+
+
+class GzipDecoder(object):
+
+ def __init__(self):
+ self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+
+ def __getattr__(self, name):
+ return getattr(self._obj, name)
+
+ def decompress(self, data):
+ if not data:
+ return data
+ return self._obj.decompress(data)
+
+
+def _get_decoder(mode):
+ if mode == 'gzip':
+ return GzipDecoder()
+
+ return DeflateDecoder()
+
+
+class HTTPResponse(io.IOBase):
+ """
+ HTTP Response container.
+
+ Backwards-compatible to httplib's HTTPResponse but the response ``body`` is
+ loaded and decoded on-demand when the ``data`` property is accessed. This
+ class is also compatible with the Python standard library's :mod:`io`
+ module, and can hence be treated as a readable object in the context of that
+ framework.
+
+ Extra parameters for behaviour not present in httplib.HTTPResponse:
+
+ :param preload_content:
+ If True, the response's body will be preloaded during construction.
+
+ :param decode_content:
+ If True, attempts to decode specific content-encoding's based on headers
+ (like 'gzip' and 'deflate') will be skipped and raw data will be used
+ instead.
+
+ :param original_response:
+ When this HTTPResponse wrapper is generated from an httplib.HTTPResponse
+ object, it's convenient to include the original for debug purposes. It's
+ otherwise unused.
+
+ :param retries:
+ The retries contains the last :class:`~urllib3.util.retry.Retry` that
+ was used during the request.
+
+ :param enforce_content_length:
+ Enforce content length checking. Body returned by server must match
+ value of Content-Length header, if present. Otherwise, raise error.
+ """
+
+ CONTENT_DECODERS = ['gzip', 'deflate']
+ REDIRECT_STATUSES = [301, 302, 303, 307, 308]
+
+ def __init__(self, body='', headers=None, status=0, version=0, reason=None,
+ strict=0, preload_content=True, decode_content=True,
+ original_response=None, pool=None, connection=None,
+ retries=None, enforce_content_length=False, request_method=None):
+
+ if isinstance(headers, HTTPHeaderDict):
+ self.headers = headers
+ else:
+ self.headers = HTTPHeaderDict(headers)
+ self.status = status
+ self.version = version
+ self.reason = reason
+ self.strict = strict
+ self.decode_content = decode_content
+ self.retries = retries
+ self.enforce_content_length = enforce_content_length
+
+ self._decoder = None
+ self._body = None
+ self._fp = None
+ self._original_response = original_response
+ self._fp_bytes_read = 0
+
+ if body and isinstance(body, (basestring, binary_type)):
+ self._body = body
+
+ self._pool = pool
+ self._connection = connection
+
+ if hasattr(body, 'read'):
+ self._fp = body
+
+ # Are we using the chunked-style of transfer encoding?
+ self.chunked = False
+ self.chunk_left = None
+ tr_enc = self.headers.get('transfer-encoding', '').lower()
+ # Don't incur the penalty of creating a list and then discarding it
+ encodings = (enc.strip() for enc in tr_enc.split(","))
+ if "chunked" in encodings:
+ self.chunked = True
+
+ # Determine length of response
+ self.length_remaining = self._init_length(request_method)
+
+ # If requested, preload the body.
+ if preload_content and not self._body:
+ self._body = self.read(decode_content=decode_content)
+
+ def get_redirect_location(self):
+ """
+ Should we redirect and where to?
+
+ :returns: Truthy redirect location string if we got a redirect status
+ code and valid location. ``None`` if redirect status and no
+ location. ``False`` if not a redirect status code.
+ """
+ if self.status in self.REDIRECT_STATUSES:
+ return self.headers.get('location')
+
+ return False
+
+ def release_conn(self):
+ if not self._pool or not self._connection:
+ return
+
+ self._pool._put_conn(self._connection)
+ self._connection = None
+
+ @property
+ def data(self):
+ # For backwords-compat with earlier urllib3 0.4 and earlier.
+ if self._body:
+ return self._body
+
+ if self._fp:
+ return self.read(cache_content=True)
+
+ @property
+ def connection(self):
+ return self._connection
+
+ def tell(self):
+ """
+ Obtain the number of bytes pulled over the wire so far. May differ from
+ the amount of content returned by :meth:``HTTPResponse.read`` if bytes
+ are encoded on the wire (e.g, compressed).
+ """
+ return self._fp_bytes_read
+
+ def _init_length(self, request_method):
+ """
+ Set initial length value for Response content if available.
+ """
+ length = self.headers.get('content-length')
+
+ if length is not None and self.chunked:
+ # This Response will fail with an IncompleteRead if it can't be
+ # received as chunked. This method falls back to attempt reading
+ # the response before raising an exception.
+ log.warning("Received response with both Content-Length and "
+ "Transfer-Encoding set. This is expressly forbidden "
+ "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
+ "attempting to process response as Transfer-Encoding: "
+ "chunked.")
+ return None
+
+ elif length is not None:
+ try:
+ # RFC 7230 section 3.3.2 specifies multiple content lengths can
+ # be sent in a single Content-Length header
+ # (e.g. Content-Length: 42, 42). This line ensures the values
+ # are all valid ints and that as long as the `set` length is 1,
+ # all values are the same. Otherwise, the header is invalid.
+ lengths = set([int(val) for val in length.split(',')])
+ if len(lengths) > 1:
+ raise InvalidHeader("Content-Length contained multiple "
+ "unmatching values (%s)" % length)
+ length = lengths.pop()
+ except ValueError:
+ length = None
+ else:
+ if length < 0:
+ length = None
+
+ # Convert status to int for comparison
+ # In some cases, httplib returns a status of "_UNKNOWN"
+ try:
+ status = int(self.status)
+ except ValueError:
+ status = 0
+
+ # Check for responses that shouldn't include a body
+ if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD':
+ length = 0
+
+ return length
+
+ def _init_decoder(self):
+ """
+ Set-up the _decoder attribute if necessary.
+ """
+ # Note: content-encoding value should be case-insensitive, per RFC 7230
+ # Section 3.2
+ content_encoding = self.headers.get('content-encoding', '').lower()
+ if self._decoder is None and content_encoding in self.CONTENT_DECODERS:
+ self._decoder = _get_decoder(content_encoding)
+
+ def _decode(self, data, decode_content, flush_decoder):
+ """
+ Decode the data passed in and potentially flush the decoder.
+ """
+ try:
+ if decode_content and self._decoder:
+ data = self._decoder.decompress(data)
+ except (IOError, zlib.error) as e:
+ content_encoding = self.headers.get('content-encoding', '').lower()
+ raise DecodeError(
+ "Received response with content-encoding: %s, but "
+ "failed to decode it." % content_encoding, e)
+
+ if flush_decoder and decode_content:
+ data += self._flush_decoder()
+
+ return data
+
+ def _flush_decoder(self):
+ """
+ Flushes the decoder. Should only be called if the decoder is actually
+ being used.
+ """
+ if self._decoder:
+ buf = self._decoder.decompress(b'')
+ return buf + self._decoder.flush()
+
+ return b''
+
+ @contextmanager
+ def _error_catcher(self):
+ """
+ Catch low-level python exceptions, instead re-raising urllib3
+ variants, so that low-level exceptions are not leaked in the
+ high-level api.
+
+ On exit, release the connection back to the pool.
+ """
+ clean_exit = False
+
+ try:
+ try:
+ yield
+
+ except SocketTimeout:
+ # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
+ # there is yet no clean way to get at it from this context.
+ raise ReadTimeoutError(self._pool, None, 'Read timed out.')
+
+ except BaseSSLError as e:
+ # FIXME: Is there a better way to differentiate between SSLErrors?
+ if 'read operation timed out' not in str(e): # Defensive:
+ # This shouldn't happen but just in case we're missing an edge
+ # case, let's avoid swallowing SSL errors.
+ raise
+
+ raise ReadTimeoutError(self._pool, None, 'Read timed out.')
+
+ except (HTTPException, SocketError) as e:
+ # This includes IncompleteRead.
+ raise ProtocolError('Connection broken: %r' % e, e)
+
+ # If no exception is thrown, we should avoid cleaning up
+ # unnecessarily.
+ clean_exit = True
+ finally:
+ # If we didn't terminate cleanly, we need to throw away our
+ # connection.
+ if not clean_exit:
+ # The response may not be closed but we're not going to use it
+ # anymore so close it now to ensure that the connection is
+ # released back to the pool.
+ if self._original_response:
+ self._original_response.close()
+
+ # Closing the response may not actually be sufficient to close
+ # everything, so if we have a hold of the connection close that
+ # too.
+ if self._connection:
+ self._connection.close()
+
+ # If we hold the original response but it's closed now, we should
+ # return the connection back to the pool.
+ if self._original_response and self._original_response.isclosed():
+ self.release_conn()
+
+ def read(self, amt=None, decode_content=None, cache_content=False):
+ """
+ Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
+ parameters: ``decode_content`` and ``cache_content``.
+
+ :param amt:
+ How much of the content to read. If specified, caching is skipped
+ because it doesn't make sense to cache partial content as the full
+ response.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+
+ :param cache_content:
+ If True, will save the returned data such that the same result is
+ returned despite of the state of the underlying file object. This
+ is useful if you want the ``.data`` property to continue working
+ after having ``.read()`` the file object. (Overridden if ``amt`` is
+ set.)
+ """
+ self._init_decoder()
+ if decode_content is None:
+ decode_content = self.decode_content
+
+ if self._fp is None:
+ return
+
+ flush_decoder = False
+ data = None
+
+ with self._error_catcher():
+ if amt is None:
+ # cStringIO doesn't like amt=None
+ data = self._fp.read()
+ flush_decoder = True
+ else:
+ cache_content = False
+ data = self._fp.read(amt)
+ if amt != 0 and not data: # Platform-specific: Buggy versions of Python.
+ # Close the connection when no data is returned
+ #
+ # This is redundant to what httplib/http.client _should_
+ # already do. However, versions of python released before
+ # December 15, 2012 (http://bugs.python.org/issue16298) do
+ # not properly close the connection in all cases. There is
+ # no harm in redundantly calling close.
+ self._fp.close()
+ flush_decoder = True
+ if self.enforce_content_length and self.length_remaining not in (0, None):
+ # This is an edge case that httplib failed to cover due
+ # to concerns of backward compatibility. We're
+ # addressing it here to make sure IncompleteRead is
+ # raised during streaming, so all calls with incorrect
+ # Content-Length are caught.
+ raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
+
+ if data:
+ self._fp_bytes_read += len(data)
+ if self.length_remaining is not None:
+ self.length_remaining -= len(data)
+
+ data = self._decode(data, decode_content, flush_decoder)
+
+ if cache_content:
+ self._body = data
+
+ return data
+
+ def stream(self, amt=2**16, decode_content=None):
+ """
+ A generator wrapper for the read() method. A call will block until
+ ``amt`` bytes have been read from the connection or until the
+ connection is closed.
+
+ :param amt:
+ How much of the content to read. The generator will return up to
+ much data per iteration, but may return less. This is particularly
+ likely when using compressed data. However, the empty string will
+ never be returned.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+ """
+ if self.chunked and self.supports_chunked_reads():
+ for line in self.read_chunked(amt, decode_content=decode_content):
+ yield line
+ else:
+ while not is_fp_closed(self._fp):
+ data = self.read(amt=amt, decode_content=decode_content)
+
+ if data:
+ yield data
+
+ @classmethod
+ def from_httplib(ResponseCls, r, **response_kw):
+ """
+ Given an :class:`httplib.HTTPResponse` instance ``r``, return a
+ corresponding :class:`urllib3.response.HTTPResponse` object.
+
+ Remaining parameters are passed to the HTTPResponse constructor, along
+ with ``original_response=r``.
+ """
+ headers = r.msg
+
+ if not isinstance(headers, HTTPHeaderDict):
+ if PY3: # Python 3
+ headers = HTTPHeaderDict(headers.items())
+ else: # Python 2
+ headers = HTTPHeaderDict.from_httplib(headers)
+
+ # HTTPResponse objects in Python 3 don't have a .strict attribute
+ strict = getattr(r, 'strict', 0)
+ resp = ResponseCls(body=r,
+ headers=headers,
+ status=r.status,
+ version=r.version,
+ reason=r.reason,
+ strict=strict,
+ original_response=r,
+ **response_kw)
+ return resp
+
+ # Backwards-compatibility methods for httplib.HTTPResponse
+ def getheaders(self):
+ return self.headers
+
+ def getheader(self, name, default=None):
+ return self.headers.get(name, default)
+
+ # Overrides from io.IOBase
+ def close(self):
+ if not self.closed:
+ self._fp.close()
+
+ if self._connection:
+ self._connection.close()
+
+ @property
+ def closed(self):
+ if self._fp is None:
+ return True
+ elif hasattr(self._fp, 'isclosed'):
+ return self._fp.isclosed()
+ elif hasattr(self._fp, 'closed'):
+ return self._fp.closed
+ else:
+ return True
+
+ def fileno(self):
+ if self._fp is None:
+ raise IOError("HTTPResponse has no file to get a fileno from")
+ elif hasattr(self._fp, "fileno"):
+ return self._fp.fileno()
+ else:
+ raise IOError("The file-like object this HTTPResponse is wrapped "
+ "around has no file descriptor")
+
+ def flush(self):
+ if self._fp is not None and hasattr(self._fp, 'flush'):
+ return self._fp.flush()
+
+ def readable(self):
+ # This method is required for `io` module compatibility.
+ return True
+
+ def readinto(self, b):
+ # This method is required for `io` module compatibility.
+ temp = self.read(len(b))
+ if len(temp) == 0:
+ return 0
+ else:
+ b[:len(temp)] = temp
+ return len(temp)
+
+ def supports_chunked_reads(self):
+ """
+ Checks if the underlying file-like object looks like a
+ httplib.HTTPResponse object. We do this by testing for the fp
+ attribute. If it is present we assume it returns raw chunks as
+ processed by read_chunked().
+ """
+ return hasattr(self._fp, 'fp')
+
+ def _update_chunk_length(self):
+ # First, we'll figure out length of a chunk and then
+ # we'll try to read it from socket.
+ if self.chunk_left is not None:
+ return
+ line = self._fp.fp.readline()
+ line = line.split(b';', 1)[0]
+ try:
+ self.chunk_left = int(line, 16)
+ except ValueError:
+ # Invalid chunked protocol response, abort.
+ self.close()
+ raise httplib.IncompleteRead(line)
+
+ def _handle_chunk(self, amt):
+ returned_chunk = None
+ if amt is None:
+ chunk = self._fp._safe_read(self.chunk_left)
+ returned_chunk = chunk
+ self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ elif amt < self.chunk_left:
+ value = self._fp._safe_read(amt)
+ self.chunk_left = self.chunk_left - amt
+ returned_chunk = value
+ elif amt == self.chunk_left:
+ value = self._fp._safe_read(amt)
+ self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ returned_chunk = value
+ else: # amt > self.chunk_left
+ returned_chunk = self._fp._safe_read(self.chunk_left)
+ self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ return returned_chunk
+
+ def read_chunked(self, amt=None, decode_content=None):
+ """
+ Similar to :meth:`HTTPResponse.read`, but with an additional
+ parameter: ``decode_content``.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+ """
+ self._init_decoder()
+ # FIXME: Rewrite this method and make it a class with a better structured logic.
+ if not self.chunked:
+ raise ResponseNotChunked(
+ "Response is not chunked. "
+ "Header 'transfer-encoding: chunked' is missing.")
+ if not self.supports_chunked_reads():
+ raise BodyNotHttplibCompatible(
+ "Body should be httplib.HTTPResponse like. "
+ "It should have have an fp attribute which returns raw chunks.")
+
+ # Don't bother reading the body of a HEAD request.
+ if self._original_response and is_response_to_head(self._original_response):
+ self._original_response.close()
+ return
+
+ with self._error_catcher():
+ while True:
+ self._update_chunk_length()
+ if self.chunk_left == 0:
+ break
+ chunk = self._handle_chunk(amt)
+ decoded = self._decode(chunk, decode_content=decode_content,
+ flush_decoder=False)
+ if decoded:
+ yield decoded
+
+ if decode_content:
+ # On CPython and PyPy, we should never need to flush the
+ # decoder. However, on Jython we *might* need to, so
+ # lets defensively do it anyway.
+ decoded = self._flush_decoder()
+ if decoded: # Platform-specific: Jython.
+ yield decoded
+
+ # Chunk content ends with \r\n: discard it.
+ while True:
+ line = self._fp.fp.readline()
+ if not line:
+ # Some sites may not end with '\r\n'.
+ break
+ if line == b'\r\n':
+ break
+
+ # We read everything; close the "file".
+ if self._original_response:
+ self._original_response.close()
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/__init__.py b/collectors/python.d.plugin/python_modules/urllib3/util/__init__.py
new file mode 100644
index 0000000..bba628d
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/__init__.py
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+# For backwards compatibility, provide imports that used to be here.
+from .connection import is_connection_dropped
+from .request import make_headers
+from .response import is_fp_closed
+from .ssl_ import (
+ SSLContext,
+ HAS_SNI,
+ IS_PYOPENSSL,
+ IS_SECURETRANSPORT,
+ assert_fingerprint,
+ resolve_cert_reqs,
+ resolve_ssl_version,
+ ssl_wrap_socket,
+)
+from .timeout import (
+ current_time,
+ Timeout,
+)
+
+from .retry import Retry
+from .url import (
+ get_host,
+ parse_url,
+ split_first,
+ Url,
+)
+from .wait import (
+ wait_for_read,
+ wait_for_write
+)
+
+__all__ = (
+ 'HAS_SNI',
+ 'IS_PYOPENSSL',
+ 'IS_SECURETRANSPORT',
+ 'SSLContext',
+ 'Retry',
+ 'Timeout',
+ 'Url',
+ 'assert_fingerprint',
+ 'current_time',
+ 'is_connection_dropped',
+ 'is_fp_closed',
+ 'get_host',
+ 'parse_url',
+ 'make_headers',
+ 'resolve_cert_reqs',
+ 'resolve_ssl_version',
+ 'split_first',
+ 'ssl_wrap_socket',
+ 'wait_for_read',
+ 'wait_for_write'
+)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/connection.py b/collectors/python.d.plugin/python_modules/urllib3/util/connection.py
new file mode 100644
index 0000000..3bd69e8
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/connection.py
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+import socket
+from .wait import wait_for_read
+from .selectors import HAS_SELECT, SelectorError
+
+
+def is_connection_dropped(conn): # Platform-specific
+ """
+ Returns True if the connection is dropped and should be closed.
+
+ :param conn:
+ :class:`httplib.HTTPConnection` object.
+
+ Note: For platforms like AppEngine, this will always return ``False`` to
+ let the platform handle connection recycling transparently for us.
+ """
+ sock = getattr(conn, 'sock', False)
+ if sock is False: # Platform-specific: AppEngine
+ return False
+ if sock is None: # Connection already closed (such as by httplib).
+ return True
+
+ if not HAS_SELECT:
+ return False
+
+ try:
+ return bool(wait_for_read(sock, timeout=0.0))
+ except SelectorError:
+ return True
+
+
+# This function is copied from socket.py in the Python 2.7 standard
+# library test suite. Added to its signature is only `socket_options`.
+# One additional modification is that we avoid binding to IPv6 servers
+# discovered in DNS if the system doesn't have IPv6 functionality.
+def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ source_address=None, socket_options=None):
+ """Connect to *address* and return the socket object.
+
+ Convenience function. Connect to *address* (a 2-tuple ``(host,
+ port)``) and return the socket object. Passing the optional
+ *timeout* parameter will set the timeout on the socket instance
+ before attempting to connect. If no *timeout* is supplied, the
+ global default timeout setting returned by :func:`getdefaulttimeout`
+ is used. If *source_address* is set it must be a tuple of (host, port)
+ for the socket to bind as a source address before making the connection.
+ An host of '' or port 0 tells the OS to use the default.
+ """
+
+ host, port = address
+ if host.startswith('['):
+ host = host.strip('[]')
+ err = None
+
+ # Using the value from allowed_gai_family() in the context of getaddrinfo lets
+ # us select whether to work with IPv4 DNS records, IPv6 records, or both.
+ # The original create_connection function always returns all records.
+ family = allowed_gai_family()
+
+ for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
+ af, socktype, proto, canonname, sa = res
+ sock = None
+ try:
+ sock = socket.socket(af, socktype, proto)
+
+ # If provided, set socket level options before connecting.
+ _set_socket_options(sock, socket_options)
+
+ if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+ sock.settimeout(timeout)
+ if source_address:
+ sock.bind(source_address)
+ sock.connect(sa)
+ return sock
+
+ except socket.error as e:
+ err = e
+ if sock is not None:
+ sock.close()
+ sock = None
+
+ if err is not None:
+ raise err
+
+ raise socket.error("getaddrinfo returns an empty list")
+
+
+def _set_socket_options(sock, options):
+ if options is None:
+ return
+
+ for opt in options:
+ sock.setsockopt(*opt)
+
+
+def allowed_gai_family():
+ """This function is designed to work in the context of
+ getaddrinfo, where family=socket.AF_UNSPEC is the default and
+ will perform a DNS search for both IPv6 and IPv4 records."""
+
+ family = socket.AF_INET
+ if HAS_IPV6:
+ family = socket.AF_UNSPEC
+ return family
+
+
+def _has_ipv6(host):
+ """ Returns True if the system can bind an IPv6 address. """
+ sock = None
+ has_ipv6 = False
+
+ if socket.has_ipv6:
+ # has_ipv6 returns true if cPython was compiled with IPv6 support.
+ # It does not tell us if the system has IPv6 support enabled. To
+ # determine that we must bind to an IPv6 address.
+ # https://github.com/shazow/urllib3/pull/611
+ # https://bugs.python.org/issue658327
+ try:
+ sock = socket.socket(socket.AF_INET6)
+ sock.bind((host, 0))
+ has_ipv6 = True
+ except Exception:
+ pass
+
+ if sock:
+ sock.close()
+ return has_ipv6
+
+
+HAS_IPV6 = _has_ipv6('::1')
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/request.py b/collectors/python.d.plugin/python_modules/urllib3/util/request.py
new file mode 100644
index 0000000..18f27b0
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/request.py
@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+from base64 import b64encode
+
+from ..packages.six import b, integer_types
+from ..exceptions import UnrewindableBodyError
+
+ACCEPT_ENCODING = 'gzip,deflate'
+_FAILEDTELL = object()
+
+
+def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
+ basic_auth=None, proxy_basic_auth=None, disable_cache=None):
+ """
+ Shortcuts for generating request headers.
+
+ :param keep_alive:
+ If ``True``, adds 'connection: keep-alive' header.
+
+ :param accept_encoding:
+ Can be a boolean, list, or string.
+ ``True`` translates to 'gzip,deflate'.
+ List will get joined by comma.
+ String will be used as provided.
+
+ :param user_agent:
+ String representing the user-agent you want, such as
+ "python-urllib3/0.6"
+
+ :param basic_auth:
+ Colon-separated username:password string for 'authorization: basic ...'
+ auth header.
+
+ :param proxy_basic_auth:
+ Colon-separated username:password string for 'proxy-authorization: basic ...'
+ auth header.
+
+ :param disable_cache:
+ If ``True``, adds 'cache-control: no-cache' header.
+
+ Example::
+
+ >>> make_headers(keep_alive=True, user_agent="Batman/1.0")
+ {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
+ >>> make_headers(accept_encoding=True)
+ {'accept-encoding': 'gzip,deflate'}
+ """
+ headers = {}
+ if accept_encoding:
+ if isinstance(accept_encoding, str):
+ pass
+ elif isinstance(accept_encoding, list):
+ accept_encoding = ','.join(accept_encoding)
+ else:
+ accept_encoding = ACCEPT_ENCODING
+ headers['accept-encoding'] = accept_encoding
+
+ if user_agent:
+ headers['user-agent'] = user_agent
+
+ if keep_alive:
+ headers['connection'] = 'keep-alive'
+
+ if basic_auth:
+ headers['authorization'] = 'Basic ' + \
+ b64encode(b(basic_auth)).decode('utf-8')
+
+ if proxy_basic_auth:
+ headers['proxy-authorization'] = 'Basic ' + \
+ b64encode(b(proxy_basic_auth)).decode('utf-8')
+
+ if disable_cache:
+ headers['cache-control'] = 'no-cache'
+
+ return headers
+
+
+def set_file_position(body, pos):
+ """
+ If a position is provided, move file to that point.
+ Otherwise, we'll attempt to record a position for future use.
+ """
+ if pos is not None:
+ rewind_body(body, pos)
+ elif getattr(body, 'tell', None) is not None:
+ try:
+ pos = body.tell()
+ except (IOError, OSError):
+ # This differentiates from None, allowing us to catch
+ # a failed `tell()` later when trying to rewind the body.
+ pos = _FAILEDTELL
+
+ return pos
+
+
+def rewind_body(body, body_pos):
+ """
+ Attempt to rewind body to a certain position.
+ Primarily used for request redirects and retries.
+
+ :param body:
+ File-like object that supports seek.
+
+ :param int pos:
+ Position to seek to in file.
+ """
+ body_seek = getattr(body, 'seek', None)
+ if body_seek is not None and isinstance(body_pos, integer_types):
+ try:
+ body_seek(body_pos)
+ except (IOError, OSError):
+ raise UnrewindableBodyError("An error occurred when rewinding request "
+ "body for redirect/retry.")
+ elif body_pos is _FAILEDTELL:
+ raise UnrewindableBodyError("Unable to record file position for rewinding "
+ "request body during a redirect/retry.")
+ else:
+ raise ValueError("body_pos must be of type integer, "
+ "instead it was %s." % type(body_pos))
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/response.py b/collectors/python.d.plugin/python_modules/urllib3/util/response.py
new file mode 100644
index 0000000..e4cda93
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/response.py
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+from ..packages.six.moves import http_client as httplib
+
+from ..exceptions import HeaderParsingError
+
+
+def is_fp_closed(obj):
+ """
+ Checks whether a given file-like object is closed.
+
+ :param obj:
+ The file-like object to check.
+ """
+
+ try:
+ # Check `isclosed()` first, in case Python3 doesn't set `closed`.
+ # GH Issue #928
+ return obj.isclosed()
+ except AttributeError:
+ pass
+
+ try:
+ # Check via the official file-like-object way.
+ return obj.closed
+ except AttributeError:
+ pass
+
+ try:
+ # Check if the object is a container for another file-like object that
+ # gets released on exhaustion (e.g. HTTPResponse).
+ return obj.fp is None
+ except AttributeError:
+ pass
+
+ raise ValueError("Unable to determine whether fp is closed.")
+
+
+def assert_header_parsing(headers):
+ """
+ Asserts whether all headers have been successfully parsed.
+ Extracts encountered errors from the result of parsing headers.
+
+ Only works on Python 3.
+
+ :param headers: Headers to verify.
+ :type headers: `httplib.HTTPMessage`.
+
+ :raises urllib3.exceptions.HeaderParsingError:
+ If parsing errors are found.
+ """
+
+ # This will fail silently if we pass in the wrong kind of parameter.
+ # To make debugging easier add an explicit check.
+ if not isinstance(headers, httplib.HTTPMessage):
+ raise TypeError('expected httplib.Message, got {0}.'.format(
+ type(headers)))
+
+ defects = getattr(headers, 'defects', None)
+ get_payload = getattr(headers, 'get_payload', None)
+
+ unparsed_data = None
+ if get_payload: # Platform-specific: Python 3.
+ unparsed_data = get_payload()
+
+ if defects or unparsed_data:
+ raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)
+
+
+def is_response_to_head(response):
+ """
+ Checks whether the request of a response has been a HEAD-request.
+ Handles the quirks of AppEngine.
+
+ :param conn:
+ :type conn: :class:`httplib.HTTPResponse`
+ """
+ # FIXME: Can we do this somehow without accessing private httplib _method?
+ method = response._method
+ if isinstance(method, int): # Platform-specific: Appengine
+ return method == 3
+ return method.upper() == 'HEAD'
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/retry.py b/collectors/python.d.plugin/python_modules/urllib3/util/retry.py
new file mode 100644
index 0000000..61e63af
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/retry.py
@@ -0,0 +1,402 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+import time
+import logging
+from collections import namedtuple
+from itertools import takewhile
+import email
+import re
+
+from ..exceptions import (
+ ConnectTimeoutError,
+ MaxRetryError,
+ ProtocolError,
+ ReadTimeoutError,
+ ResponseError,
+ InvalidHeader,
+)
+from ..packages import six
+
+
+log = logging.getLogger(__name__)
+
+# Data structure for representing the metadata of requests that result in a retry.
+RequestHistory = namedtuple('RequestHistory', ["method", "url", "error",
+ "status", "redirect_location"])
+
+
+class Retry(object):
+ """ Retry configuration.
+
+ Each retry attempt will create a new Retry object with updated values, so
+ they can be safely reused.
+
+ Retries can be defined as a default for a pool::
+
+ retries = Retry(connect=5, read=2, redirect=5)
+ http = PoolManager(retries=retries)
+ response = http.request('GET', 'http://example.com/')
+
+ Or per-request (which overrides the default for the pool)::
+
+ response = http.request('GET', 'http://example.com/', retries=Retry(10))
+
+ Retries can be disabled by passing ``False``::
+
+ response = http.request('GET', 'http://example.com/', retries=False)
+
+ Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
+ retries are disabled, in which case the causing exception will be raised.
+
+ :param int total:
+ Total number of retries to allow. Takes precedence over other counts.
+
+ Set to ``None`` to remove this constraint and fall back on other
+ counts. It's a good idea to set this to some sensibly-high value to
+ account for unexpected edge cases and avoid infinite retry loops.
+
+ Set to ``0`` to fail on the first retry.
+
+ Set to ``False`` to disable and imply ``raise_on_redirect=False``.
+
+ :param int connect:
+ How many connection-related errors to retry on.
+
+ These are errors raised before the request is sent to the remote server,
+ which we assume has not triggered the server to process the request.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ :param int read:
+ How many times to retry on read errors.
+
+ These errors are raised after the request was sent to the server, so the
+ request may have side-effects.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ :param int redirect:
+ How many redirects to perform. Limit this to avoid infinite redirect
+ loops.
+
+ A redirect is a HTTP response with a status code 301, 302, 303, 307 or
+ 308.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ Set to ``False`` to disable and imply ``raise_on_redirect=False``.
+
+ :param int status:
+ How many times to retry on bad status codes.
+
+ These are retries made on responses, where status code matches
+ ``status_forcelist``.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ :param iterable method_whitelist:
+ Set of uppercased HTTP method verbs that we should retry on.
+
+ By default, we only retry on methods which are considered to be
+ idempotent (multiple requests with the same parameters end with the
+ same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.
+
+ Set to a ``False`` value to retry on any verb.
+
+ :param iterable status_forcelist:
+ A set of integer HTTP status codes that we should force a retry on.
+ A retry is initiated if the request method is in ``method_whitelist``
+ and the response status code is in ``status_forcelist``.
+
+ By default, this is disabled with ``None``.
+
+ :param float backoff_factor:
+ A backoff factor to apply between attempts after the second try
+ (most errors are resolved immediately by a second try without a
+ delay). urllib3 will sleep for::
+
+ {backoff factor} * (2 ^ ({number of total retries} - 1))
+
+ seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
+ for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
+ than :attr:`Retry.BACKOFF_MAX`.
+
+ By default, backoff is disabled (set to 0).
+
+ :param bool raise_on_redirect: Whether, if the number of redirects is
+ exhausted, to raise a MaxRetryError, or to return a response with a
+ response code in the 3xx range.
+
+ :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
+ whether we should raise an exception, or return a response,
+ if status falls in ``status_forcelist`` range and retries have
+ been exhausted.
+
+ :param tuple history: The history of the request encountered during
+ each call to :meth:`~Retry.increment`. The list is in the order
+ the requests occurred. Each list item is of class :class:`RequestHistory`.
+
+ :param bool respect_retry_after_header:
+ Whether to respect Retry-After header on status codes defined as
+ :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
+
+ """
+
+ DEFAULT_METHOD_WHITELIST = frozenset([
+ 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
+
+ RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
+
+ #: Maximum backoff time.
+ BACKOFF_MAX = 120
+
+ def __init__(self, total=10, connect=None, read=None, redirect=None, status=None,
+ method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
+ backoff_factor=0, raise_on_redirect=True, raise_on_status=True,
+ history=None, respect_retry_after_header=True):
+
+ self.total = total
+ self.connect = connect
+ self.read = read
+ self.status = status
+
+ if redirect is False or total is False:
+ redirect = 0
+ raise_on_redirect = False
+
+ self.redirect = redirect
+ self.status_forcelist = status_forcelist or set()
+ self.method_whitelist = method_whitelist
+ self.backoff_factor = backoff_factor
+ self.raise_on_redirect = raise_on_redirect
+ self.raise_on_status = raise_on_status
+ self.history = history or tuple()
+ self.respect_retry_after_header = respect_retry_after_header
+
+ def new(self, **kw):
+ params = dict(
+ total=self.total,
+ connect=self.connect, read=self.read, redirect=self.redirect, status=self.status,
+ method_whitelist=self.method_whitelist,
+ status_forcelist=self.status_forcelist,
+ backoff_factor=self.backoff_factor,
+ raise_on_redirect=self.raise_on_redirect,
+ raise_on_status=self.raise_on_status,
+ history=self.history,
+ )
+ params.update(kw)
+ return type(self)(**params)
+
+ @classmethod
+ def from_int(cls, retries, redirect=True, default=None):
+ """ Backwards-compatibility for the old retries format."""
+ if retries is None:
+ retries = default if default is not None else cls.DEFAULT
+
+ if isinstance(retries, Retry):
+ return retries
+
+ redirect = bool(redirect) and None
+ new_retries = cls(retries, redirect=redirect)
+ log.debug("Converted retries value: %r -> %r", retries, new_retries)
+ return new_retries
+
+ def get_backoff_time(self):
+ """ Formula for computing the current backoff
+
+ :rtype: float
+ """
+ # We want to consider only the last consecutive errors sequence (Ignore redirects).
+ consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None,
+ reversed(self.history))))
+ if consecutive_errors_len <= 1:
+ return 0
+
+ backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
+ return min(self.BACKOFF_MAX, backoff_value)
+
+ def parse_retry_after(self, retry_after):
+ # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
+ if re.match(r"^\s*[0-9]+\s*$", retry_after):
+ seconds = int(retry_after)
+ else:
+ retry_date_tuple = email.utils.parsedate(retry_after)
+ if retry_date_tuple is None:
+ raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
+ retry_date = time.mktime(retry_date_tuple)
+ seconds = retry_date - time.time()
+
+ if seconds < 0:
+ seconds = 0
+
+ return seconds
+
+ def get_retry_after(self, response):
+ """ Get the value of Retry-After in seconds. """
+
+ retry_after = response.getheader("Retry-After")
+
+ if retry_after is None:
+ return None
+
+ return self.parse_retry_after(retry_after)
+
+ def sleep_for_retry(self, response=None):
+ retry_after = self.get_retry_after(response)
+ if retry_after:
+ time.sleep(retry_after)
+ return True
+
+ return False
+
+ def _sleep_backoff(self):
+ backoff = self.get_backoff_time()
+ if backoff <= 0:
+ return
+ time.sleep(backoff)
+
+ def sleep(self, response=None):
+ """ Sleep between retry attempts.
+
+ This method will respect a server's ``Retry-After`` response header
+ and sleep the duration of the time requested. If that is not present, it
+ will use an exponential backoff. By default, the backoff factor is 0 and
+ this method will return immediately.
+ """
+
+ if response:
+ slept = self.sleep_for_retry(response)
+ if slept:
+ return
+
+ self._sleep_backoff()
+
+ def _is_connection_error(self, err):
+ """ Errors when we're fairly sure that the server did not receive the
+ request, so it should be safe to retry.
+ """
+ return isinstance(err, ConnectTimeoutError)
+
+ def _is_read_error(self, err):
+ """ Errors that occur after the request has been started, so we should
+ assume that the server began processing it.
+ """
+ return isinstance(err, (ReadTimeoutError, ProtocolError))
+
+ def _is_method_retryable(self, method):
+ """ Checks if a given HTTP method should be retried upon, depending if
+ it is included on the method whitelist.
+ """
+ if self.method_whitelist and method.upper() not in self.method_whitelist:
+ return False
+
+ return True
+
+ def is_retry(self, method, status_code, has_retry_after=False):
+ """ Is this method/status code retryable? (Based on whitelists and control
+ variables such as the number of total retries to allow, whether to
+ respect the Retry-After header, whether this header is present, and
+ whether the returned status code is on the list of status codes to
+ be retried upon on the presence of the aforementioned header)
+ """
+ if not self._is_method_retryable(method):
+ return False
+
+ if self.status_forcelist and status_code in self.status_forcelist:
+ return True
+
+ return (self.total and self.respect_retry_after_header and
+ has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES))
+
+ def is_exhausted(self):
+ """ Are we out of retries? """
+ retry_counts = (self.total, self.connect, self.read, self.redirect, self.status)
+ retry_counts = list(filter(None, retry_counts))
+ if not retry_counts:
+ return False
+
+ return min(retry_counts) < 0
+
+ def increment(self, method=None, url=None, response=None, error=None,
+ _pool=None, _stacktrace=None):
+ """ Return a new Retry object with incremented retry counters.
+
+ :param response: A response object, or None, if the server did not
+ return a response.
+ :type response: :class:`~urllib3.response.HTTPResponse`
+ :param Exception error: An error encountered during the request, or
+ None if the response was received successfully.
+
+ :return: A new ``Retry`` object.
+ """
+ if self.total is False and error:
+ # Disabled, indicate to re-raise the error.
+ raise six.reraise(type(error), error, _stacktrace)
+
+ total = self.total
+ if total is not None:
+ total -= 1
+
+ connect = self.connect
+ read = self.read
+ redirect = self.redirect
+ status_count = self.status
+ cause = 'unknown'
+ status = None
+ redirect_location = None
+
+ if error and self._is_connection_error(error):
+ # Connect retry?
+ if connect is False:
+ raise six.reraise(type(error), error, _stacktrace)
+ elif connect is not None:
+ connect -= 1
+
+ elif error and self._is_read_error(error):
+ # Read retry?
+ if read is False or not self._is_method_retryable(method):
+ raise six.reraise(type(error), error, _stacktrace)
+ elif read is not None:
+ read -= 1
+
+ elif response and response.get_redirect_location():
+ # Redirect retry?
+ if redirect is not None:
+ redirect -= 1
+ cause = 'too many redirects'
+ redirect_location = response.get_redirect_location()
+ status = response.status
+
+ else:
+ # Incrementing because of a server error like a 500 in
+ # status_forcelist and a the given method is in the whitelist
+ cause = ResponseError.GENERIC_ERROR
+ if response and response.status:
+ if status_count is not None:
+ status_count -= 1
+ cause = ResponseError.SPECIFIC_ERROR.format(
+ status_code=response.status)
+ status = response.status
+
+ history = self.history + (RequestHistory(method, url, error, status, redirect_location),)
+
+ new_retry = self.new(
+ total=total,
+ connect=connect, read=read, redirect=redirect, status=status_count,
+ history=history)
+
+ if new_retry.is_exhausted():
+ raise MaxRetryError(_pool, url, error or ResponseError(cause))
+
+ log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
+
+ return new_retry
+
+ def __repr__(self):
+ return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
+ 'read={self.read}, redirect={self.redirect}, status={self.status})').format(
+ cls=type(self), self=self)
+
+
+# For backwards compatibility (equivalent to pre-v1.9):
+Retry.DEFAULT = Retry(3)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py b/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py
new file mode 100644
index 0000000..de5e498
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py
@@ -0,0 +1,588 @@
+# SPDX-License-Identifier: MIT
+# Backport of selectors.py from Python 3.5+ to support Python < 3.4
+# Also has the behavior specified in PEP 475 which is to retry syscalls
+# in the case of an EINTR error. This module is required because selectors34
+# does not follow this behavior and instead returns that no dile descriptor
+# events have occurred rather than retry the syscall. The decision to drop
+# support for select.devpoll is made to maintain 100% test coverage.
+
+import errno
+import math
+import select
+import socket
+import sys
+import time
+
+from collections import namedtuple
+
+try:
+ from collections import Mapping
+except ImportError:
+ from collections.abc import Mapping
+
+try:
+ monotonic = time.monotonic
+except (AttributeError, ImportError): # Python 3.3<
+ monotonic = time.time
+
+EVENT_READ = (1 << 0)
+EVENT_WRITE = (1 << 1)
+
+HAS_SELECT = True # Variable that shows whether the platform has a selector.
+_SYSCALL_SENTINEL = object() # Sentinel in case a system call returns None.
+_DEFAULT_SELECTOR = None
+
+
+class SelectorError(Exception):
+ def __init__(self, errcode):
+ super(SelectorError, self).__init__()
+ self.errno = errcode
+
+ def __repr__(self):
+ return "<SelectorError errno={0}>".format(self.errno)
+
+ def __str__(self):
+ return self.__repr__()
+
+
+def _fileobj_to_fd(fileobj):
+ """ Return a file descriptor from a file object. If
+ given an integer will simply return that integer back. """
+ if isinstance(fileobj, int):
+ fd = fileobj
+ else:
+ try:
+ fd = int(fileobj.fileno())
+ except (AttributeError, TypeError, ValueError):
+ raise ValueError("Invalid file object: {0!r}".format(fileobj))
+ if fd < 0:
+ raise ValueError("Invalid file descriptor: {0}".format(fd))
+ return fd
+
+
+# Determine which function to use to wrap system calls because Python 3.5+
+# already handles the case when system calls are interrupted.
+if sys.version_info >= (3, 5):
+ def _syscall_wrapper(func, _, *args, **kwargs):
+ """ This is the short-circuit version of the below logic
+ because in Python 3.5+ all system calls automatically restart
+ and recalculate their timeouts. """
+ try:
+ return func(*args, **kwargs)
+ except (OSError, IOError, select.error) as e:
+ errcode = None
+ if hasattr(e, "errno"):
+ errcode = e.errno
+ raise SelectorError(errcode)
+else:
+ def _syscall_wrapper(func, recalc_timeout, *args, **kwargs):
+ """ Wrapper function for syscalls that could fail due to EINTR.
+ All functions should be retried if there is time left in the timeout
+ in accordance with PEP 475. """
+ timeout = kwargs.get("timeout", None)
+ if timeout is None:
+ expires = None
+ recalc_timeout = False
+ else:
+ timeout = float(timeout)
+ if timeout < 0.0: # Timeout less than 0 treated as no timeout.
+ expires = None
+ else:
+ expires = monotonic() + timeout
+
+ args = list(args)
+ if recalc_timeout and "timeout" not in kwargs:
+ raise ValueError(
+ "Timeout must be in args or kwargs to be recalculated")
+
+ result = _SYSCALL_SENTINEL
+ while result is _SYSCALL_SENTINEL:
+ try:
+ result = func(*args, **kwargs)
+ # OSError is thrown by select.select
+ # IOError is thrown by select.epoll.poll
+ # select.error is thrown by select.poll.poll
+ # Aren't we thankful for Python 3.x rework for exceptions?
+ except (OSError, IOError, select.error) as e:
+ # select.error wasn't a subclass of OSError in the past.
+ errcode = None
+ if hasattr(e, "errno"):
+ errcode = e.errno
+ elif hasattr(e, "args"):
+ errcode = e.args[0]
+
+ # Also test for the Windows equivalent of EINTR.
+ is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and
+ errcode == errno.WSAEINTR))
+
+ if is_interrupt:
+ if expires is not None:
+ current_time = monotonic()
+ if current_time > expires:
+ raise OSError(errno=errno.ETIMEDOUT)
+ if recalc_timeout:
+ if "timeout" in kwargs:
+ kwargs["timeout"] = expires - current_time
+ continue
+ if errcode:
+ raise SelectorError(errcode)
+ else:
+ raise
+ return result
+
+
+SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data'])
+
+
+class _SelectorMapping(Mapping):
+ """ Mapping of file objects to selector keys """
+
+ def __init__(self, selector):
+ self._selector = selector
+
+ def __len__(self):
+ return len(self._selector._fd_to_key)
+
+ def __getitem__(self, fileobj):
+ try:
+ fd = self._selector._fileobj_lookup(fileobj)
+ return self._selector._fd_to_key[fd]
+ except KeyError:
+ raise KeyError("{0!r} is not registered.".format(fileobj))
+
+ def __iter__(self):
+ return iter(self._selector._fd_to_key)
+
+
+class BaseSelector(object):
+ """ Abstract Selector class
+
+ A selector supports registering file objects to be monitored
+ for specific I/O events.
+
+ A file object is a file descriptor or any object with a
+ `fileno()` method. An arbitrary object can be attached to the
+ file object which can be used for example to store context info,
+ a callback, etc.
+
+ A selector can use various implementations (select(), poll(), epoll(),
+ and kqueue()) depending on the platform. The 'DefaultSelector' class uses
+ the most efficient implementation for the current platform.
+ """
+ def __init__(self):
+ # Maps file descriptors to keys.
+ self._fd_to_key = {}
+
+ # Read-only mapping returned by get_map()
+ self._map = _SelectorMapping(self)
+
+ def _fileobj_lookup(self, fileobj):
+ """ Return a file descriptor from a file object.
+ This wraps _fileobj_to_fd() to do an exhaustive
+ search in case the object is invalid but we still
+ have it in our map. Used by unregister() so we can
+ unregister an object that was previously registered
+ even if it is closed. It is also used by _SelectorMapping
+ """
+ try:
+ return _fileobj_to_fd(fileobj)
+ except ValueError:
+
+ # Search through all our mapped keys.
+ for key in self._fd_to_key.values():
+ if key.fileobj is fileobj:
+ return key.fd
+
+ # Raise ValueError after all.
+ raise
+
+ def register(self, fileobj, events, data=None):
+ """ Register a file object for a set of events to monitor. """
+ if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
+ raise ValueError("Invalid events: {0!r}".format(events))
+
+ key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)
+
+ if key.fd in self._fd_to_key:
+ raise KeyError("{0!r} (FD {1}) is already registered"
+ .format(fileobj, key.fd))
+
+ self._fd_to_key[key.fd] = key
+ return key
+
+ def unregister(self, fileobj):
+ """ Unregister a file object from being monitored. """
+ try:
+ key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
+ except KeyError:
+ raise KeyError("{0!r} is not registered".format(fileobj))
+
+ # Getting the fileno of a closed socket on Windows errors with EBADF.
+ except socket.error as e: # Platform-specific: Windows.
+ if e.errno != errno.EBADF:
+ raise
+ else:
+ for key in self._fd_to_key.values():
+ if key.fileobj is fileobj:
+ self._fd_to_key.pop(key.fd)
+ break
+ else:
+ raise KeyError("{0!r} is not registered".format(fileobj))
+ return key
+
+ def modify(self, fileobj, events, data=None):
+ """ Change a registered file object monitored events and data. """
+ # NOTE: Some subclasses optimize this operation even further.
+ try:
+ key = self._fd_to_key[self._fileobj_lookup(fileobj)]
+ except KeyError:
+ raise KeyError("{0!r} is not registered".format(fileobj))
+
+ if events != key.events:
+ self.unregister(fileobj)
+ key = self.register(fileobj, events, data)
+
+ elif data != key.data:
+ # Use a shortcut to update the data.
+ key = key._replace(data=data)
+ self._fd_to_key[key.fd] = key
+
+ return key
+
+ def select(self, timeout=None):
+ """ Perform the actual selection until some monitored file objects
+ are ready or the timeout expires. """
+ raise NotImplementedError()
+
+ def close(self):
+ """ Close the selector. This must be called to ensure that all
+ underlying resources are freed. """
+ self._fd_to_key.clear()
+ self._map = None
+
+ def get_key(self, fileobj):
+ """ Return the key associated with a registered file object. """
+ mapping = self.get_map()
+ if mapping is None:
+ raise RuntimeError("Selector is closed")
+ try:
+ return mapping[fileobj]
+ except KeyError:
+ raise KeyError("{0!r} is not registered".format(fileobj))
+
+ def get_map(self):
+ """ Return a mapping of file objects to selector keys """
+ return self._map
+
+ def _key_from_fd(self, fd):
+ """ Return the key associated to a given file descriptor
+ Return None if it is not found. """
+ try:
+ return self._fd_to_key[fd]
+ except KeyError:
+ return None
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
+
+# Almost all platforms have select.select()
+if hasattr(select, "select"):
+ class SelectSelector(BaseSelector):
+ """ Select-based selector. """
+ def __init__(self):
+ super(SelectSelector, self).__init__()
+ self._readers = set()
+ self._writers = set()
+
+ def register(self, fileobj, events, data=None):
+ key = super(SelectSelector, self).register(fileobj, events, data)
+ if events & EVENT_READ:
+ self._readers.add(key.fd)
+ if events & EVENT_WRITE:
+ self._writers.add(key.fd)
+ return key
+
+ def unregister(self, fileobj):
+ key = super(SelectSelector, self).unregister(fileobj)
+ self._readers.discard(key.fd)
+ self._writers.discard(key.fd)
+ return key
+
+ def _select(self, r, w, timeout=None):
+ """ Wrapper for select.select because timeout is a positional arg """
+ return select.select(r, w, [], timeout)
+
+ def select(self, timeout=None):
+ # Selecting on empty lists on Windows errors out.
+ if not len(self._readers) and not len(self._writers):
+ return []
+
+ timeout = None if timeout is None else max(timeout, 0.0)
+ ready = []
+ r, w, _ = _syscall_wrapper(self._select, True, self._readers,
+ self._writers, timeout)
+ r = set(r)
+ w = set(w)
+ for fd in r | w:
+ events = 0
+ if fd in r:
+ events |= EVENT_READ
+ if fd in w:
+ events |= EVENT_WRITE
+
+ key = self._key_from_fd(fd)
+ if key:
+ ready.append((key, events & key.events))
+ return ready
+
+
+if hasattr(select, "poll"):
+ class PollSelector(BaseSelector):
+ """ Poll-based selector """
+ def __init__(self):
+ super(PollSelector, self).__init__()
+ self._poll = select.poll()
+
+ def register(self, fileobj, events, data=None):
+ key = super(PollSelector, self).register(fileobj, events, data)
+ event_mask = 0
+ if events & EVENT_READ:
+ event_mask |= select.POLLIN
+ if events & EVENT_WRITE:
+ event_mask |= select.POLLOUT
+ self._poll.register(key.fd, event_mask)
+ return key
+
+ def unregister(self, fileobj):
+ key = super(PollSelector, self).unregister(fileobj)
+ self._poll.unregister(key.fd)
+ return key
+
+ def _wrap_poll(self, timeout=None):
+ """ Wrapper function for select.poll.poll() so that
+ _syscall_wrapper can work with only seconds. """
+ if timeout is not None:
+ if timeout <= 0:
+ timeout = 0
+ else:
+ # select.poll.poll() has a resolution of 1 millisecond,
+ # round away from zero to wait *at least* timeout seconds.
+ timeout = math.ceil(timeout * 1e3)
+
+ result = self._poll.poll(timeout)
+ return result
+
+ def select(self, timeout=None):
+ ready = []
+ fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout)
+ for fd, event_mask in fd_events:
+ events = 0
+ if event_mask & ~select.POLLIN:
+ events |= EVENT_WRITE
+ if event_mask & ~select.POLLOUT:
+ events |= EVENT_READ
+
+ key = self._key_from_fd(fd)
+ if key:
+ ready.append((key, events & key.events))
+
+ return ready
+
+
+if hasattr(select, "epoll"):
+ class EpollSelector(BaseSelector):
+ """ Epoll-based selector """
+ def __init__(self):
+ super(EpollSelector, self).__init__()
+ self._epoll = select.epoll()
+
+ def fileno(self):
+ return self._epoll.fileno()
+
+ def register(self, fileobj, events, data=None):
+ key = super(EpollSelector, self).register(fileobj, events, data)
+ events_mask = 0
+ if events & EVENT_READ:
+ events_mask |= select.EPOLLIN
+ if events & EVENT_WRITE:
+ events_mask |= select.EPOLLOUT
+ _syscall_wrapper(self._epoll.register, False, key.fd, events_mask)
+ return key
+
+ def unregister(self, fileobj):
+ key = super(EpollSelector, self).unregister(fileobj)
+ try:
+ _syscall_wrapper(self._epoll.unregister, False, key.fd)
+ except SelectorError:
+ # This can occur when the fd was closed since registry.
+ pass
+ return key
+
+ def select(self, timeout=None):
+ if timeout is not None:
+ if timeout <= 0:
+ timeout = 0.0
+ else:
+ # select.epoll.poll() has a resolution of 1 millisecond
+ # but luckily takes seconds so we don't need a wrapper
+ # like PollSelector. Just for better rounding.
+ timeout = math.ceil(timeout * 1e3) * 1e-3
+ timeout = float(timeout)
+ else:
+ timeout = -1.0 # epoll.poll() must have a float.
+
+ # We always want at least 1 to ensure that select can be called
+ # with no file descriptors registered. Otherwise will fail.
+ max_events = max(len(self._fd_to_key), 1)
+
+ ready = []
+ fd_events = _syscall_wrapper(self._epoll.poll, True,
+ timeout=timeout,
+ maxevents=max_events)
+ for fd, event_mask in fd_events:
+ events = 0
+ if event_mask & ~select.EPOLLIN:
+ events |= EVENT_WRITE
+ if event_mask & ~select.EPOLLOUT:
+ events |= EVENT_READ
+
+ key = self._key_from_fd(fd)
+ if key:
+ ready.append((key, events & key.events))
+ return ready
+
+ def close(self):
+ self._epoll.close()
+ super(EpollSelector, self).close()
+
+
+if hasattr(select, "kqueue"):
+ class KqueueSelector(BaseSelector):
+ """ Kqueue / Kevent-based selector """
+ def __init__(self):
+ super(KqueueSelector, self).__init__()
+ self._kqueue = select.kqueue()
+
+ def fileno(self):
+ return self._kqueue.fileno()
+
+ def register(self, fileobj, events, data=None):
+ key = super(KqueueSelector, self).register(fileobj, events, data)
+ if events & EVENT_READ:
+ kevent = select.kevent(key.fd,
+ select.KQ_FILTER_READ,
+ select.KQ_EV_ADD)
+
+ _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+
+ if events & EVENT_WRITE:
+ kevent = select.kevent(key.fd,
+ select.KQ_FILTER_WRITE,
+ select.KQ_EV_ADD)
+
+ _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+
+ return key
+
+ def unregister(self, fileobj):
+ key = super(KqueueSelector, self).unregister(fileobj)
+ if key.events & EVENT_READ:
+ kevent = select.kevent(key.fd,
+ select.KQ_FILTER_READ,
+ select.KQ_EV_DELETE)
+ try:
+ _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+ except SelectorError:
+ pass
+ if key.events & EVENT_WRITE:
+ kevent = select.kevent(key.fd,
+ select.KQ_FILTER_WRITE,
+ select.KQ_EV_DELETE)
+ try:
+ _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+ except SelectorError:
+ pass
+
+ return key
+
+ def select(self, timeout=None):
+ if timeout is not None:
+ timeout = max(timeout, 0)
+
+ max_events = len(self._fd_to_key) * 2
+ ready_fds = {}
+
+ kevent_list = _syscall_wrapper(self._kqueue.control, True,
+ None, max_events, timeout)
+
+ for kevent in kevent_list:
+ fd = kevent.ident
+ event_mask = kevent.filter
+ events = 0
+ if event_mask == select.KQ_FILTER_READ:
+ events |= EVENT_READ
+ if event_mask == select.KQ_FILTER_WRITE:
+ events |= EVENT_WRITE
+
+ key = self._key_from_fd(fd)
+ if key:
+ if key.fd not in ready_fds:
+ ready_fds[key.fd] = (key, events & key.events)
+ else:
+ old_events = ready_fds[key.fd][1]
+ ready_fds[key.fd] = (key, (events | old_events) & key.events)
+
+ return list(ready_fds.values())
+
+ def close(self):
+ self._kqueue.close()
+ super(KqueueSelector, self).close()
+
+
+if not hasattr(select, 'select'): # Platform-specific: AppEngine
+ HAS_SELECT = False
+
+
+def _can_allocate(struct):
+ """ Checks that select structs can be allocated by the underlying
+ operating system, not just advertised by the select module. We don't
+ check select() because we'll be hopeful that most platforms that
+ don't have it available will not advertise it. (ie: GAE) """
+ try:
+ # select.poll() objects won't fail until used.
+ if struct == 'poll':
+ p = select.poll()
+ p.poll(0)
+
+ # All others will fail on allocation.
+ else:
+ getattr(select, struct)().close()
+ return True
+ except (OSError, AttributeError) as e:
+ return False
+
+
+# Choose the best implementation, roughly:
+# kqueue == epoll > poll > select. Devpoll not supported. (See above)
+# select() also can't accept a FD > FD_SETSIZE (usually around 1024)
+def DefaultSelector():
+ """ This function serves as a first call for DefaultSelector to
+ detect if the select module is being monkey-patched incorrectly
+ by eventlet, greenlet, and preserve proper behavior. """
+ global _DEFAULT_SELECTOR
+ if _DEFAULT_SELECTOR is None:
+ if _can_allocate('kqueue'):
+ _DEFAULT_SELECTOR = KqueueSelector
+ elif _can_allocate('epoll'):
+ _DEFAULT_SELECTOR = EpollSelector
+ elif _can_allocate('poll'):
+ _DEFAULT_SELECTOR = PollSelector
+ elif hasattr(select, 'select'):
+ _DEFAULT_SELECTOR = SelectSelector
+ else: # Platform-specific: AppEngine
+ raise ValueError('Platform does not have a selector')
+ return _DEFAULT_SELECTOR()
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/ssl_.py b/collectors/python.d.plugin/python_modules/urllib3/util/ssl_.py
new file mode 100644
index 0000000..ece3ec3
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/ssl_.py
@@ -0,0 +1,338 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+import errno
+import warnings
+import hmac
+
+from binascii import hexlify, unhexlify
+from hashlib import md5, sha1, sha256
+
+from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning
+
+
+SSLContext = None
+HAS_SNI = False
+IS_PYOPENSSL = False
+IS_SECURETRANSPORT = False
+
+# Maps the length of a digest to a possible hash function producing this digest
+HASHFUNC_MAP = {
+ 32: md5,
+ 40: sha1,
+ 64: sha256,
+}
+
+
+def _const_compare_digest_backport(a, b):
+ """
+ Compare two digests of equal length in constant time.
+
+ The digests must be of type str/bytes.
+ Returns True if the digests match, and False otherwise.
+ """
+ result = abs(len(a) - len(b))
+ for l, r in zip(bytearray(a), bytearray(b)):
+ result |= l ^ r
+ return result == 0
+
+
+_const_compare_digest = getattr(hmac, 'compare_digest',
+ _const_compare_digest_backport)
+
+
+try: # Test for SSL features
+ import ssl
+ from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
+ from ssl import HAS_SNI # Has SNI?
+except ImportError:
+ pass
+
+
+try:
+ from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
+except ImportError:
+ OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
+ OP_NO_COMPRESSION = 0x20000
+
+# A secure default.
+# Sources for more information on TLS ciphers:
+#
+# - https://wiki.mozilla.org/Security/Server_Side_TLS
+# - https://www.ssllabs.com/projects/best-practices/index.html
+# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
+#
+# The general intent is:
+# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
+# - prefer ECDHE over DHE for better performance,
+# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and
+# security,
+# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common,
+# - disable NULL authentication, MD5 MACs and DSS for security reasons.
+DEFAULT_CIPHERS = ':'.join([
+ 'ECDH+AESGCM',
+ 'ECDH+CHACHA20',
+ 'DH+AESGCM',
+ 'DH+CHACHA20',
+ 'ECDH+AES256',
+ 'DH+AES256',
+ 'ECDH+AES128',
+ 'DH+AES',
+ 'RSA+AESGCM',
+ 'RSA+AES',
+ '!aNULL',
+ '!eNULL',
+ '!MD5',
+])
+
+try:
+ from ssl import SSLContext # Modern SSL?
+except ImportError:
+ import sys
+
+ class SSLContext(object): # Platform-specific: Python 2 & 3.1
+ supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or
+ (3, 2) <= sys.version_info)
+
+ def __init__(self, protocol_version):
+ self.protocol = protocol_version
+ # Use default values from a real SSLContext
+ self.check_hostname = False
+ self.verify_mode = ssl.CERT_NONE
+ self.ca_certs = None
+ self.options = 0
+ self.certfile = None
+ self.keyfile = None
+ self.ciphers = None
+
+ def load_cert_chain(self, certfile, keyfile):
+ self.certfile = certfile
+ self.keyfile = keyfile
+
+ def load_verify_locations(self, cafile=None, capath=None):
+ self.ca_certs = cafile
+
+ if capath is not None:
+ raise SSLError("CA directories not supported in older Pythons")
+
+ def set_ciphers(self, cipher_suite):
+ if not self.supports_set_ciphers:
+ raise TypeError(
+ 'Your version of Python does not support setting '
+ 'a custom cipher suite. Please upgrade to Python '
+ '2.7, 3.2, or later if you need this functionality.'
+ )
+ self.ciphers = cipher_suite
+
+ def wrap_socket(self, socket, server_hostname=None, server_side=False):
+ warnings.warn(
+ 'A true SSLContext object is not available. This prevents '
+ 'urllib3 from configuring SSL appropriately and may cause '
+ 'certain SSL connections to fail. You can upgrade to a newer '
+ 'version of Python to solve this. For more information, see '
+ 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
+ '#ssl-warnings',
+ InsecurePlatformWarning
+ )
+ kwargs = {
+ 'keyfile': self.keyfile,
+ 'certfile': self.certfile,
+ 'ca_certs': self.ca_certs,
+ 'cert_reqs': self.verify_mode,
+ 'ssl_version': self.protocol,
+ 'server_side': server_side,
+ }
+ if self.supports_set_ciphers: # Platform-specific: Python 2.7+
+ return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
+ else: # Platform-specific: Python 2.6
+ return wrap_socket(socket, **kwargs)
+
+
+def assert_fingerprint(cert, fingerprint):
+ """
+ Checks if given fingerprint matches the supplied certificate.
+
+ :param cert:
+ Certificate as bytes object.
+ :param fingerprint:
+ Fingerprint as string of hexdigits, can be interspersed by colons.
+ """
+
+ fingerprint = fingerprint.replace(':', '').lower()
+ digest_length = len(fingerprint)
+ hashfunc = HASHFUNC_MAP.get(digest_length)
+ if not hashfunc:
+ raise SSLError(
+ 'Fingerprint of invalid length: {0}'.format(fingerprint))
+
+ # We need encode() here for py32; works on py2 and p33.
+ fingerprint_bytes = unhexlify(fingerprint.encode())
+
+ cert_digest = hashfunc(cert).digest()
+
+ if not _const_compare_digest(cert_digest, fingerprint_bytes):
+ raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
+ .format(fingerprint, hexlify(cert_digest)))
+
+
+def resolve_cert_reqs(candidate):
+ """
+ Resolves the argument to a numeric constant, which can be passed to
+ the wrap_socket function/method from the ssl module.
+ Defaults to :data:`ssl.CERT_NONE`.
+ If given a string it is assumed to be the name of the constant in the
+ :mod:`ssl` module or its abbrevation.
+ (So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
+ If it's neither `None` nor a string we assume it is already the numeric
+ constant which can directly be passed to wrap_socket.
+ """
+ if candidate is None:
+ return CERT_NONE
+
+ if isinstance(candidate, str):
+ res = getattr(ssl, candidate, None)
+ if res is None:
+ res = getattr(ssl, 'CERT_' + candidate)
+ return res
+
+ return candidate
+
+
+def resolve_ssl_version(candidate):
+ """
+ like resolve_cert_reqs
+ """
+ if candidate is None:
+ return PROTOCOL_SSLv23
+
+ if isinstance(candidate, str):
+ res = getattr(ssl, candidate, None)
+ if res is None:
+ res = getattr(ssl, 'PROTOCOL_' + candidate)
+ return res
+
+ return candidate
+
+
+def create_urllib3_context(ssl_version=None, cert_reqs=None,
+ options=None, ciphers=None):
+ """All arguments have the same meaning as ``ssl_wrap_socket``.
+
+ By default, this function does a lot of the same work that
+ ``ssl.create_default_context`` does on Python 3.4+. It:
+
+ - Disables SSLv2, SSLv3, and compression
+ - Sets a restricted set of server ciphers
+
+ If you wish to enable SSLv3, you can do::
+
+ from urllib3.util import ssl_
+ context = ssl_.create_urllib3_context()
+ context.options &= ~ssl_.OP_NO_SSLv3
+
+ You can do the same to enable compression (substituting ``COMPRESSION``
+ for ``SSLv3`` in the last line above).
+
+ :param ssl_version:
+ The desired protocol version to use. This will default to
+ PROTOCOL_SSLv23 which will negotiate the highest protocol that both
+ the server and your installation of OpenSSL support.
+ :param cert_reqs:
+ Whether to require the certificate verification. This defaults to
+ ``ssl.CERT_REQUIRED``.
+ :param options:
+ Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
+ ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``.
+ :param ciphers:
+ Which cipher suites to allow the server to select.
+ :returns:
+ Constructed SSLContext object with specified options
+ :rtype: SSLContext
+ """
+ context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)
+
+ # Setting the default here, as we may have no ssl module on import
+ cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
+
+ if options is None:
+ options = 0
+ # SSLv2 is easily broken and is considered harmful and dangerous
+ options |= OP_NO_SSLv2
+ # SSLv3 has several problems and is now dangerous
+ options |= OP_NO_SSLv3
+ # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
+ # (issue #309)
+ options |= OP_NO_COMPRESSION
+
+ context.options |= options
+
+ if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6
+ context.set_ciphers(ciphers or DEFAULT_CIPHERS)
+
+ context.verify_mode = cert_reqs
+ if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2
+ # We do our own verification, including fingerprints and alternative
+ # hostnames. So disable it here
+ context.check_hostname = False
+ return context
+
+
+def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
+ ca_certs=None, server_hostname=None,
+ ssl_version=None, ciphers=None, ssl_context=None,
+ ca_cert_dir=None):
+ """
+ All arguments except for server_hostname, ssl_context, and ca_cert_dir have
+ the same meaning as they do when using :func:`ssl.wrap_socket`.
+
+ :param server_hostname:
+ When SNI is supported, the expected hostname of the certificate
+ :param ssl_context:
+ A pre-made :class:`SSLContext` object. If none is provided, one will
+ be created using :func:`create_urllib3_context`.
+ :param ciphers:
+ A string of ciphers we wish the client to support. This is not
+ supported on Python 2.6 as the ssl module does not support it.
+ :param ca_cert_dir:
+ A directory containing CA certificates in multiple separate files, as
+ supported by OpenSSL's -CApath flag or the capath argument to
+ SSLContext.load_verify_locations().
+ """
+ context = ssl_context
+ if context is None:
+ # Note: This branch of code and all the variables in it are no longer
+ # used by urllib3 itself. We should consider deprecating and removing
+ # this code.
+ context = create_urllib3_context(ssl_version, cert_reqs,
+ ciphers=ciphers)
+
+ if ca_certs or ca_cert_dir:
+ try:
+ context.load_verify_locations(ca_certs, ca_cert_dir)
+ except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2
+ raise SSLError(e)
+ # Py33 raises FileNotFoundError which subclasses OSError
+ # These are not equivalent unless we check the errno attribute
+ except OSError as e: # Platform-specific: Python 3.3 and beyond
+ if e.errno == errno.ENOENT:
+ raise SSLError(e)
+ raise
+ elif getattr(context, 'load_default_certs', None) is not None:
+ # try to load OS default certs; works well on Windows (require Python3.4+)
+ context.load_default_certs()
+
+ if certfile:
+ context.load_cert_chain(certfile, keyfile)
+ if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI
+ return context.wrap_socket(sock, server_hostname=server_hostname)
+
+ warnings.warn(
+ 'An HTTPS request has been made, but the SNI (Subject Name '
+ 'Indication) extension to TLS is not available on this platform. '
+ 'This may cause the server to present an incorrect TLS '
+ 'certificate, which can cause validation failures. You can upgrade to '
+ 'a newer version of Python to solve this. For more information, see '
+ 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
+ '#ssl-warnings',
+ SNIMissingWarning
+ )
+ return context.wrap_socket(sock)
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/timeout.py b/collectors/python.d.plugin/python_modules/urllib3/util/timeout.py
new file mode 100644
index 0000000..4041cf9
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/timeout.py
@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+# The default socket timeout, used by httplib to indicate that no timeout was
+# specified by the user
+from socket import _GLOBAL_DEFAULT_TIMEOUT
+import time
+
+from ..exceptions import TimeoutStateError
+
+# A sentinel value to indicate that no timeout was specified by the user in
+# urllib3
+_Default = object()
+
+
+# Use time.monotonic if available.
+current_time = getattr(time, "monotonic", time.time)
+
+
+class Timeout(object):
+ """ Timeout configuration.
+
+ Timeouts can be defined as a default for a pool::
+
+ timeout = Timeout(connect=2.0, read=7.0)
+ http = PoolManager(timeout=timeout)
+ response = http.request('GET', 'http://example.com/')
+
+ Or per-request (which overrides the default for the pool)::
+
+ response = http.request('GET', 'http://example.com/', timeout=Timeout(10))
+
+ Timeouts can be disabled by setting all the parameters to ``None``::
+
+ no_timeout = Timeout(connect=None, read=None)
+ response = http.request('GET', 'http://example.com/, timeout=no_timeout)
+
+
+ :param total:
+ This combines the connect and read timeouts into one; the read timeout
+ will be set to the time leftover from the connect attempt. In the
+ event that both a connect timeout and a total are specified, or a read
+ timeout and a total are specified, the shorter timeout will be applied.
+
+ Defaults to None.
+
+ :type total: integer, float, or None
+
+ :param connect:
+ The maximum amount of time to wait for a connection attempt to a server
+ to succeed. Omitting the parameter will default the connect timeout to
+ the system default, probably `the global default timeout in socket.py
+ <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
+ None will set an infinite timeout for connection attempts.
+
+ :type connect: integer, float, or None
+
+ :param read:
+ The maximum amount of time to wait between consecutive
+ read operations for a response from the server. Omitting
+ the parameter will default the read timeout to the system
+ default, probably `the global default timeout in socket.py
+ <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
+ None will set an infinite timeout.
+
+ :type read: integer, float, or None
+
+ .. note::
+
+ Many factors can affect the total amount of time for urllib3 to return
+ an HTTP response.
+
+ For example, Python's DNS resolver does not obey the timeout specified
+ on the socket. Other factors that can affect total request time include
+ high CPU load, high swap, the program running at a low priority level,
+ or other behaviors.
+
+ In addition, the read and total timeouts only measure the time between
+ read operations on the socket connecting the client and the server,
+ not the total amount of time for the request to return a complete
+ response. For most requests, the timeout is raised because the server
+ has not sent the first byte in the specified time. This is not always
+ the case; if a server streams one byte every fifteen seconds, a timeout
+ of 20 seconds will not trigger, even though the request will take
+ several minutes to complete.
+
+ If your goal is to cut off any request after a set amount of wall clock
+ time, consider having a second "watcher" thread to cut off a slow
+ request.
+ """
+
+ #: A sentinel object representing the default timeout value
+ DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT
+
+ def __init__(self, total=None, connect=_Default, read=_Default):
+ self._connect = self._validate_timeout(connect, 'connect')
+ self._read = self._validate_timeout(read, 'read')
+ self.total = self._validate_timeout(total, 'total')
+ self._start_connect = None
+
+ def __str__(self):
+ return '%s(connect=%r, read=%r, total=%r)' % (
+ type(self).__name__, self._connect, self._read, self.total)
+
+ @classmethod
+ def _validate_timeout(cls, value, name):
+ """ Check that a timeout attribute is valid.
+
+ :param value: The timeout value to validate
+ :param name: The name of the timeout attribute to validate. This is
+ used to specify in error messages.
+ :return: The validated and casted version of the given value.
+ :raises ValueError: If it is a numeric value less than or equal to
+ zero, or the type is not an integer, float, or None.
+ """
+ if value is _Default:
+ return cls.DEFAULT_TIMEOUT
+
+ if value is None or value is cls.DEFAULT_TIMEOUT:
+ return value
+
+ if isinstance(value, bool):
+ raise ValueError("Timeout cannot be a boolean value. It must "
+ "be an int, float or None.")
+ try:
+ float(value)
+ except (TypeError, ValueError):
+ raise ValueError("Timeout value %s was %s, but it must be an "
+ "int, float or None." % (name, value))
+
+ try:
+ if value <= 0:
+ raise ValueError("Attempted to set %s timeout to %s, but the "
+ "timeout cannot be set to a value less "
+ "than or equal to 0." % (name, value))
+ except TypeError: # Python 3
+ raise ValueError("Timeout value %s was %s, but it must be an "
+ "int, float or None." % (name, value))
+
+ return value
+
+ @classmethod
+ def from_float(cls, timeout):
+ """ Create a new Timeout from a legacy timeout value.
+
+ The timeout value used by httplib.py sets the same timeout on the
+ connect(), and recv() socket requests. This creates a :class:`Timeout`
+ object that sets the individual timeouts to the ``timeout`` value
+ passed to this function.
+
+ :param timeout: The legacy timeout value.
+ :type timeout: integer, float, sentinel default object, or None
+ :return: Timeout object
+ :rtype: :class:`Timeout`
+ """
+ return Timeout(read=timeout, connect=timeout)
+
+ def clone(self):
+ """ Create a copy of the timeout object
+
+ Timeout properties are stored per-pool but each request needs a fresh
+ Timeout object to ensure each one has its own start/stop configured.
+
+ :return: a copy of the timeout object
+ :rtype: :class:`Timeout`
+ """
+ # We can't use copy.deepcopy because that will also create a new object
+ # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to
+ # detect the user default.
+ return Timeout(connect=self._connect, read=self._read,
+ total=self.total)
+
+ def start_connect(self):
+ """ Start the timeout clock, used during a connect() attempt
+
+ :raises urllib3.exceptions.TimeoutStateError: if you attempt
+ to start a timer that has been started already.
+ """
+ if self._start_connect is not None:
+ raise TimeoutStateError("Timeout timer has already been started.")
+ self._start_connect = current_time()
+ return self._start_connect
+
+ def get_connect_duration(self):
+ """ Gets the time elapsed since the call to :meth:`start_connect`.
+
+ :return: Elapsed time.
+ :rtype: float
+ :raises urllib3.exceptions.TimeoutStateError: if you attempt
+ to get duration for a timer that hasn't been started.
+ """
+ if self._start_connect is None:
+ raise TimeoutStateError("Can't get connect duration for timer "
+ "that has not started.")
+ return current_time() - self._start_connect
+
+ @property
+ def connect_timeout(self):
+ """ Get the value to use when setting a connection timeout.
+
+ This will be a positive float or integer, the value None
+ (never timeout), or the default system timeout.
+
+ :return: Connect timeout.
+ :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
+ """
+ if self.total is None:
+ return self._connect
+
+ if self._connect is None or self._connect is self.DEFAULT_TIMEOUT:
+ return self.total
+
+ return min(self._connect, self.total)
+
+ @property
+ def read_timeout(self):
+ """ Get the value for the read timeout.
+
+ This assumes some time has elapsed in the connection timeout and
+ computes the read timeout appropriately.
+
+ If self.total is set, the read timeout is dependent on the amount of
+ time taken by the connect timeout. If the connection time has not been
+ established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be
+ raised.
+
+ :return: Value to use for the read timeout.
+ :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
+ :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
+ has not yet been called on this object.
+ """
+ if (self.total is not None and
+ self.total is not self.DEFAULT_TIMEOUT and
+ self._read is not None and
+ self._read is not self.DEFAULT_TIMEOUT):
+ # In case the connect timeout has not yet been established.
+ if self._start_connect is None:
+ return self._read
+ return max(0, min(self.total - self.get_connect_duration(),
+ self._read))
+ elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT:
+ return max(0, self.total - self.get_connect_duration())
+ else:
+ return self._read
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/url.py b/collectors/python.d.plugin/python_modules/urllib3/util/url.py
new file mode 100644
index 0000000..99fd653
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/url.py
@@ -0,0 +1,231 @@
+# SPDX-License-Identifier: MIT
+from __future__ import absolute_import
+from collections import namedtuple
+
+from ..exceptions import LocationParseError
+
+
+url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment']
+
+# We only want to normalize urls with an HTTP(S) scheme.
+# urllib3 infers URLs without a scheme (None) to be http.
+NORMALIZABLE_SCHEMES = ('http', 'https', None)
+
+
+class Url(namedtuple('Url', url_attrs)):
+ """
+ Datastructure for representing an HTTP URL. Used as a return value for
+ :func:`parse_url`. Both the scheme and host are normalized as they are
+ both case-insensitive according to RFC 3986.
+ """
+ __slots__ = ()
+
+ def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None,
+ query=None, fragment=None):
+ if path and not path.startswith('/'):
+ path = '/' + path
+ if scheme:
+ scheme = scheme.lower()
+ if host and scheme in NORMALIZABLE_SCHEMES:
+ host = host.lower()
+ return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
+ query, fragment)
+
+ @property
+ def hostname(self):
+ """For backwards-compatibility with urlparse. We're nice like that."""
+ return self.host
+
+ @property
+ def request_uri(self):
+ """Absolute path including the query string."""
+ uri = self.path or '/'
+
+ if self.query is not None:
+ uri += '?' + self.query
+
+ return uri
+
+ @property
+ def netloc(self):
+ """Network location including host and port"""
+ if self.port:
+ return '%s:%d' % (self.host, self.port)
+ return self.host
+
+ @property
+ def url(self):
+ """
+ Convert self into a url
+
+ This function should more or less round-trip with :func:`.parse_url`. The
+ returned url may not be exactly the same as the url inputted to
+ :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls
+ with a blank port will have : removed).
+
+ Example: ::
+
+ >>> U = parse_url('http://google.com/mail/')
+ >>> U.url
+ 'http://google.com/mail/'
+ >>> Url('http', 'username:password', 'host.com', 80,
+ ... '/path', 'query', 'fragment').url
+ 'http://username:password@host.com:80/path?query#fragment'
+ """
+ scheme, auth, host, port, path, query, fragment = self
+ url = ''
+
+ # We use "is not None" we want things to happen with empty strings (or 0 port)
+ if scheme is not None:
+ url += scheme + '://'
+ if auth is not None:
+ url += auth + '@'
+ if host is not None:
+ url += host
+ if port is not None:
+ url += ':' + str(port)
+ if path is not None:
+ url += path
+ if query is not None:
+ url += '?' + query
+ if fragment is not None:
+ url += '#' + fragment
+
+ return url
+
+ def __str__(self):
+ return self.url
+
+
+def split_first(s, delims):
+ """
+ Given a string and an iterable of delimiters, split on the first found
+ delimiter. Return two split parts and the matched delimiter.
+
+ If not found, then the first part is the full input string.
+
+ Example::
+
+ >>> split_first('foo/bar?baz', '?/=')
+ ('foo', 'bar?baz', '/')
+ >>> split_first('foo/bar?baz', '123')
+ ('foo/bar?baz', '', None)
+
+ Scales linearly with number of delims. Not ideal for large number of delims.
+ """
+ min_idx = None
+ min_delim = None
+ for d in delims:
+ idx = s.find(d)
+ if idx < 0:
+ continue
+
+ if min_idx is None or idx < min_idx:
+ min_idx = idx
+ min_delim = d
+
+ if min_idx is None or min_idx < 0:
+ return s, '', None
+
+ return s[:min_idx], s[min_idx + 1:], min_delim
+
+
+def parse_url(url):
+ """
+ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
+ performed to parse incomplete urls. Fields not provided will be None.
+
+ Partly backwards-compatible with :mod:`urlparse`.
+
+ Example::
+
+ >>> parse_url('http://google.com/mail/')
+ Url(scheme='http', host='google.com', port=None, path='/mail/', ...)
+ >>> parse_url('google.com:80')
+ Url(scheme=None, host='google.com', port=80, path=None, ...)
+ >>> parse_url('/foo?bar')
+ Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
+ """
+
+ # While this code has overlap with stdlib's urlparse, it is much
+ # simplified for our needs and less annoying.
+ # Additionally, this implementations does silly things to be optimal
+ # on CPython.
+
+ if not url:
+ # Empty
+ return Url()
+
+ scheme = None
+ auth = None
+ host = None
+ port = None
+ path = None
+ fragment = None
+ query = None
+
+ # Scheme
+ if '://' in url:
+ scheme, url = url.split('://', 1)
+
+ # Find the earliest Authority Terminator
+ # (http://tools.ietf.org/html/rfc3986#section-3.2)
+ url, path_, delim = split_first(url, ['/', '?', '#'])
+
+ if delim:
+ # Reassemble the path
+ path = delim + path_
+
+ # Auth
+ if '@' in url:
+ # Last '@' denotes end of auth part
+ auth, url = url.rsplit('@', 1)
+
+ # IPv6
+ if url and url[0] == '[':
+ host, url = url.split(']', 1)
+ host += ']'
+
+ # Port
+ if ':' in url:
+ _host, port = url.split(':', 1)
+
+ if not host:
+ host = _host
+
+ if port:
+ # If given, ports must be integers. No whitespace, no plus or
+ # minus prefixes, no non-integer digits such as ^2 (superscript).
+ if not port.isdigit():
+ raise LocationParseError(url)
+ try:
+ port = int(port)
+ except ValueError:
+ raise LocationParseError(url)
+ else:
+ # Blank ports are cool, too. (rfc3986#section-3.2.3)
+ port = None
+
+ elif not host and url:
+ host = url
+
+ if not path:
+ return Url(scheme, auth, host, port, path, query, fragment)
+
+ # Fragment
+ if '#' in path:
+ path, fragment = path.split('#', 1)
+
+ # Query
+ if '?' in path:
+ path, query = path.split('?', 1)
+
+ return Url(scheme, auth, host, port, path, query, fragment)
+
+
+def get_host(url):
+ """
+ Deprecated. Use :func:`parse_url` instead.
+ """
+ p = parse_url(url)
+ return p.scheme or 'http', p.hostname, p.port
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/wait.py b/collectors/python.d.plugin/python_modules/urllib3/util/wait.py
new file mode 100644
index 0000000..21e7297
--- /dev/null
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/wait.py
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: MIT
+from .selectors import (
+ HAS_SELECT,
+ DefaultSelector,
+ EVENT_READ,
+ EVENT_WRITE
+)
+
+
+def _wait_for_io_events(socks, events, timeout=None):
+ """ Waits for IO events to be available from a list of sockets
+ or optionally a single socket if passed in. Returns a list of
+ sockets that can be interacted with immediately. """
+ if not HAS_SELECT:
+ raise ValueError('Platform does not have a selector')
+ if not isinstance(socks, list):
+ # Probably just a single socket.
+ if hasattr(socks, "fileno"):
+ socks = [socks]
+ # Otherwise it might be a non-list iterable.
+ else:
+ socks = list(socks)
+ with DefaultSelector() as selector:
+ for sock in socks:
+ selector.register(sock, events)
+ return [key[0].fileobj for key in
+ selector.select(timeout) if key[1] & events]
+
+
+def wait_for_read(socks, timeout=None):
+ """ Waits for reading to be available from a list of sockets
+ or optionally a single socket if passed in. Returns a list of
+ sockets that can be read from immediately. """
+ return _wait_for_io_events(socks, EVENT_READ, timeout)
+
+
+def wait_for_write(socks, timeout=None):
+ """ Waits for writing to be available from a list of sockets
+ or optionally a single socket if passed in. Returns a list of
+ sockets that can be written to immediately. """
+ return _wait_for_io_events(socks, EVENT_WRITE, timeout)
diff --git a/collectors/python.d.plugin/rabbitmq/Makefile.inc b/collectors/python.d.plugin/rabbitmq/Makefile.inc
new file mode 100644
index 0000000..7e67ef5
--- /dev/null
+++ b/collectors/python.d.plugin/rabbitmq/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += rabbitmq/rabbitmq.chart.py
+dist_pythonconfig_DATA += rabbitmq/rabbitmq.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += rabbitmq/README.md rabbitmq/Makefile.inc
+
diff --git a/collectors/python.d.plugin/rabbitmq/README.md b/collectors/python.d.plugin/rabbitmq/README.md
new file mode 100644
index 0000000..927adcc
--- /dev/null
+++ b/collectors/python.d.plugin/rabbitmq/README.md
@@ -0,0 +1,138 @@
+<!--
+title: "RabbitMQ monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/rabbitmq/README.md
+sidebar_label: "RabbitMQ"
+-->
+
+# RabbitMQ monitoring with Netdata
+
+Collects message broker global and per virtual host metrics.
+
+
+Following charts are drawn:
+
+1. **Queued Messages**
+
+ - ready
+ - unacknowledged
+
+2. **Message Rates**
+
+ - ack
+ - redelivered
+ - deliver
+ - publish
+
+3. **Global Counts**
+
+ - channels
+ - consumers
+ - connections
+ - queues
+ - exchanges
+
+4. **File Descriptors**
+
+ - used descriptors
+
+5. **Socket Descriptors**
+
+ - used descriptors
+
+6. **Erlang processes**
+
+ - used processes
+
+7. **Erlang run queue**
+
+ - Erlang run queue
+
+8. **Memory**
+
+ - free memory in megabytes
+
+9. **Disk Space**
+
+ - free disk space in gigabytes
+
+
+Per Vhost charts:
+
+1. **Vhost Messages**
+
+ - ack
+ - confirm
+ - deliver
+ - get
+ - get_no_ack
+ - publish
+ - redeliver
+ - return_unroutable
+
+2. Per Queue charts:
+
+ 1. **Queued Messages**
+
+ - messages
+ - paged_out
+ - persistent
+ - ready
+ - unacknowledged
+
+ 2. **Queue Messages stats**
+
+ - ack
+ - confirm
+ - deliver
+ - get
+ - get_no_ack
+ - publish
+ - redeliver
+ - return_unroutable
+
+## Configuration
+
+Edit the `python.d/rabbitmq.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/rabbitmq.conf
+```
+
+When no configuration file is found, module tries to connect to: `localhost:15672`.
+
+```yaml
+socket:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 15672
+ user : 'guest'
+ pass : 'guest'
+```
+
+---
+
+### Per-Queue Chart configuration
+
+RabbitMQ users with the "monitoring" tag cannot see all queue data. You'll need a user with read permissions.
+To create a dedicated user for netdata:
+
+```bash
+rabbitmqctl add_user netdata ChangeThisSuperSecretPassword
+rabbitmqctl set_permissions netdata "^$" "^$" ".*"
+```
+
+See [set_permissions](https://www.rabbitmq.com/rabbitmqctl.8.html#set_permissions) for details.
+
+Once the user is set up, add `collect_queues_metrics: yes` to your `rabbitmq.conf`:
+
+```yaml
+local:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 15672
+ user : 'netdata'
+ pass : 'ChangeThisSuperSecretPassword'
+ collect_queues_metrics : 'yes'
+```
diff --git a/collectors/python.d.plugin/rabbitmq/rabbitmq.chart.py b/collectors/python.d.plugin/rabbitmq/rabbitmq.chart.py
new file mode 100644
index 0000000..866b777
--- /dev/null
+++ b/collectors/python.d.plugin/rabbitmq/rabbitmq.chart.py
@@ -0,0 +1,443 @@
+# -*- coding: utf-8 -*-
+# Description: rabbitmq netdata python.d module
+# Author: ilyam8
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from json import loads
+
+from bases.FrameworkServices.UrlService import UrlService
+
+API_NODE = 'api/nodes'
+API_OVERVIEW = 'api/overview'
+API_QUEUES = 'api/queues'
+API_VHOSTS = 'api/vhosts'
+
+NODE_STATS = [
+ 'fd_used',
+ 'mem_used',
+ 'sockets_used',
+ 'proc_used',
+ 'disk_free',
+ 'run_queue'
+]
+
+OVERVIEW_STATS = [
+ 'object_totals.channels',
+ 'object_totals.consumers',
+ 'object_totals.connections',
+ 'object_totals.queues',
+ 'object_totals.exchanges',
+ 'queue_totals.messages_ready',
+ 'queue_totals.messages_unacknowledged',
+ 'message_stats.ack',
+ 'message_stats.redeliver',
+ 'message_stats.deliver',
+ 'message_stats.publish',
+ 'churn_rates.connection_created_details.rate',
+ 'churn_rates.connection_closed_details.rate',
+ 'churn_rates.channel_created_details.rate',
+ 'churn_rates.channel_closed_details.rate',
+ 'churn_rates.queue_created_details.rate',
+ 'churn_rates.queue_declared_details.rate',
+ 'churn_rates.queue_deleted_details.rate'
+]
+
+QUEUE_STATS = [
+ 'messages',
+ 'messages_paged_out',
+ 'messages_persistent',
+ 'messages_ready',
+ 'messages_unacknowledged',
+ 'message_stats.ack',
+ 'message_stats.confirm',
+ 'message_stats.deliver',
+ 'message_stats.get',
+ 'message_stats.get_no_ack',
+ 'message_stats.publish',
+ 'message_stats.redeliver',
+ 'message_stats.return_unroutable',
+]
+
+VHOST_MESSAGE_STATS = [
+ 'message_stats.ack',
+ 'message_stats.confirm',
+ 'message_stats.deliver',
+ 'message_stats.get',
+ 'message_stats.get_no_ack',
+ 'message_stats.publish',
+ 'message_stats.redeliver',
+ 'message_stats.return_unroutable',
+]
+
+ORDER = [
+ 'queued_messages',
+ 'connection_churn_rates',
+ 'channel_churn_rates',
+ 'queue_churn_rates',
+ 'message_rates',
+ 'global_counts',
+ 'file_descriptors',
+ 'socket_descriptors',
+ 'erlang_processes',
+ 'erlang_run_queue',
+ 'memory',
+ 'disk_space'
+]
+
+CHARTS = {
+ 'file_descriptors': {
+ 'options': [None, 'File Descriptors', 'descriptors', 'overview', 'rabbitmq.file_descriptors', 'line'],
+ 'lines': [
+ ['fd_used', 'used', 'absolute']
+ ]
+ },
+ 'memory': {
+ 'options': [None, 'Memory', 'MiB', 'overview', 'rabbitmq.memory', 'area'],
+ 'lines': [
+ ['mem_used', 'used', 'absolute', 1, 1 << 20]
+ ]
+ },
+ 'disk_space': {
+ 'options': [None, 'Disk Space', 'GiB', 'overview', 'rabbitmq.disk_space', 'area'],
+ 'lines': [
+ ['disk_free', 'free', 'absolute', 1, 1 << 30]
+ ]
+ },
+ 'socket_descriptors': {
+ 'options': [None, 'Socket Descriptors', 'descriptors', 'overview', 'rabbitmq.sockets', 'line'],
+ 'lines': [
+ ['sockets_used', 'used', 'absolute']
+ ]
+ },
+ 'erlang_processes': {
+ 'options': [None, 'Erlang Processes', 'processes', 'overview', 'rabbitmq.processes', 'line'],
+ 'lines': [
+ ['proc_used', 'used', 'absolute']
+ ]
+ },
+ 'erlang_run_queue': {
+ 'options': [None, 'Erlang Run Queue', 'processes', 'overview', 'rabbitmq.erlang_run_queue', 'line'],
+ 'lines': [
+ ['run_queue', 'length', 'absolute']
+ ]
+ },
+ 'global_counts': {
+ 'options': [None, 'Global Counts', 'counts', 'overview', 'rabbitmq.global_counts', 'line'],
+ 'lines': [
+ ['object_totals_channels', 'channels', 'absolute'],
+ ['object_totals_consumers', 'consumers', 'absolute'],
+ ['object_totals_connections', 'connections', 'absolute'],
+ ['object_totals_queues', 'queues', 'absolute'],
+ ['object_totals_exchanges', 'exchanges', 'absolute']
+ ]
+ },
+ 'connection_churn_rates': {
+ 'options': [None, 'Connection Churn Rates', 'operations/s', 'overview', 'rabbitmq.connection_churn_rates', 'line'],
+ 'lines': [
+ ['churn_rates_connection_created_details_rate', 'created', 'absolute'],
+ ['churn_rates_connection_closed_details_rate', 'closed', 'absolute']
+ ]
+ },
+ 'channel_churn_rates': {
+ 'options': [None, 'Channel Churn Rates', 'operations/s', 'overview', 'rabbitmq.channel_churn_rates', 'line'],
+ 'lines': [
+ ['churn_rates_channel_created_details_rate', 'created', 'absolute'],
+ ['churn_rates_channel_closed_details_rate', 'closed', 'absolute']
+ ]
+ },
+ 'queue_churn_rates': {
+ 'options': [None, 'Queue Churn Rates', 'operations/s', 'overview', 'rabbitmq.queue_churn_rates', 'line'],
+ 'lines': [
+ ['churn_rates_queue_created_details_rate', 'created', 'absolute'],
+ ['churn_rates_queue_declared_details_rate', 'declared', 'absolute'],
+ ['churn_rates_queue_deleted_details_rate', 'deleted', 'absolute']
+ ]
+ },
+ 'queued_messages': {
+ 'options': [None, 'Queued Messages', 'messages', 'overview', 'rabbitmq.queued_messages', 'stacked'],
+ 'lines': [
+ ['queue_totals_messages_ready', 'ready', 'absolute'],
+ ['queue_totals_messages_unacknowledged', 'unacknowledged', 'absolute']
+ ]
+ },
+ 'message_rates': {
+ 'options': [None, 'Message Rates', 'messages/s', 'overview', 'rabbitmq.message_rates', 'line'],
+ 'lines': [
+ ['message_stats_ack', 'ack', 'incremental'],
+ ['message_stats_redeliver', 'redeliver', 'incremental'],
+ ['message_stats_deliver', 'deliver', 'incremental'],
+ ['message_stats_publish', 'publish', 'incremental']
+ ]
+ }
+}
+
+
+def vhost_chart_template(name):
+ order = [
+ 'vhost_{0}_message_stats'.format(name),
+ ]
+ family = 'vhost {0}'.format(name)
+
+ charts = {
+ order[0]: {
+ 'options': [
+ None, 'Vhost "{0}" Messages'.format(name), 'messages/s', family, 'rabbitmq.vhost_messages', 'stacked'],
+ 'lines': [
+ ['vhost_{0}_message_stats_ack'.format(name), 'ack', 'incremental'],
+ ['vhost_{0}_message_stats_confirm'.format(name), 'confirm', 'incremental'],
+ ['vhost_{0}_message_stats_deliver'.format(name), 'deliver', 'incremental'],
+ ['vhost_{0}_message_stats_get'.format(name), 'get', 'incremental'],
+ ['vhost_{0}_message_stats_get_no_ack'.format(name), 'get_no_ack', 'incremental'],
+ ['vhost_{0}_message_stats_publish'.format(name), 'publish', 'incremental'],
+ ['vhost_{0}_message_stats_redeliver'.format(name), 'redeliver', 'incremental'],
+ ['vhost_{0}_message_stats_return_unroutable'.format(name), 'return_unroutable', 'incremental'],
+ ]
+ },
+ }
+
+ return order, charts
+
+def queue_chart_template(queue_id):
+ vhost, name = queue_id
+ order = [
+ 'vhost_{0}_queue_{1}_queued_message'.format(vhost, name),
+ 'vhost_{0}_queue_{1}_messages_stats'.format(vhost, name),
+ ]
+ family = 'vhost {0}'.format(vhost)
+
+ charts = {
+ order[0]: {
+ 'options': [
+ None, 'Queue "{0}" in "{1}" queued messages'.format(name, vhost), 'messages', family, 'rabbitmq.queue_messages', 'line'],
+ 'lines': [
+ ['vhost_{0}_queue_{1}_messages'.format(vhost, name), 'messages', 'absolute'],
+ ['vhost_{0}_queue_{1}_messages_paged_out'.format(vhost, name), 'paged_out', 'absolute'],
+ ['vhost_{0}_queue_{1}_messages_persistent'.format(vhost, name), 'persistent', 'absolute'],
+ ['vhost_{0}_queue_{1}_messages_ready'.format(vhost, name), 'ready', 'absolute'],
+ ['vhost_{0}_queue_{1}_messages_unacknowledged'.format(vhost, name), 'unack', 'absolute'],
+ ]
+ },
+ order[1]: {
+ 'options': [
+ None, 'Queue "{0}" in "{1}" messages stats'.format(name, vhost), 'messages/s', family, 'rabbitmq.queue_messages_stats', 'line'],
+ 'lines': [
+ ['vhost_{0}_queue_{1}_message_stats_ack'.format(vhost, name), 'ack', 'incremental'],
+ ['vhost_{0}_queue_{1}_message_stats_confirm'.format(vhost, name), 'confirm', 'incremental'],
+ ['vhost_{0}_queue_{1}_message_stats_deliver'.format(vhost, name), 'deliver', 'incremental'],
+ ['vhost_{0}_queue_{1}_message_stats_get'.format(vhost, name), 'get', 'incremental'],
+ ['vhost_{0}_queue_{1}_message_stats_get_no_ack'.format(vhost, name), 'get_no_ack', 'incremental'],
+ ['vhost_{0}_queue_{1}_message_stats_publish'.format(vhost, name), 'publish', 'incremental'],
+ ['vhost_{0}_queue_{1}_message_stats_redeliver'.format(vhost, name), 'redeliver', 'incremental'],
+ ['vhost_{0}_queue_{1}_message_stats_return_unroutable'.format(vhost, name), 'return_unroutable', 'incremental'],
+ ]
+ },
+ }
+
+ return order, charts
+
+
+class VhostStatsBuilder:
+ def __init__(self):
+ self.stats = None
+
+ def set(self, raw_stats):
+ self.stats = raw_stats
+
+ def name(self):
+ return self.stats['name']
+
+ def has_msg_stats(self):
+ return bool(self.stats.get('message_stats'))
+
+ def msg_stats(self):
+ name = self.name()
+ stats = fetch_data(raw_data=self.stats, metrics=VHOST_MESSAGE_STATS)
+ return dict(('vhost_{0}_{1}'.format(name, k), v) for k, v in stats.items())
+
+class QueueStatsBuilder:
+ def __init__(self):
+ self.stats = None
+
+ def set(self, raw_stats):
+ self.stats = raw_stats
+
+ def id(self):
+ return self.stats['vhost'], self.stats['name']
+
+ def queue_stats(self):
+ vhost, name = self.id()
+ stats = fetch_data(raw_data=self.stats, metrics=QUEUE_STATS)
+ return dict(('vhost_{0}_queue_{1}_{2}'.format(vhost, name, k), v) for k, v in stats.items())
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.url = '{0}://{1}:{2}'.format(
+ configuration.get('scheme', 'http'),
+ configuration.get('host', '127.0.0.1'),
+ configuration.get('port', 15672),
+ )
+ self.node_name = str()
+ self.vhost = VhostStatsBuilder()
+ self.collected_vhosts = set()
+ self.collect_queues_metrics = configuration.get('collect_queues_metrics', False)
+ self.debug("collect_queues_metrics is {0}".format("enabled" if self.collect_queues_metrics else "disabled"))
+ if self.collect_queues_metrics:
+ self.queue = QueueStatsBuilder()
+ self.collected_queues = set()
+
+ def _get_data(self):
+ data = dict()
+
+ stats = self.get_overview_stats()
+ if not stats:
+ return None
+
+ data.update(stats)
+
+ stats = self.get_nodes_stats()
+ if not stats:
+ return None
+
+ data.update(stats)
+
+ stats = self.get_vhosts_stats()
+ if stats:
+ data.update(stats)
+
+ if self.collect_queues_metrics:
+ stats = self.get_queues_stats()
+ if stats:
+ data.update(stats)
+
+ return data or None
+
+ def get_overview_stats(self):
+ url = '{0}/{1}'.format(self.url, API_OVERVIEW)
+ self.debug("doing http request to '{0}'".format(url))
+ raw = self._get_raw_data(url)
+ if not raw:
+ return None
+
+ data = loads(raw)
+ self.node_name = data['node']
+ self.debug("found node name: '{0}'".format(self.node_name))
+
+ stats = fetch_data(raw_data=data, metrics=OVERVIEW_STATS)
+ self.debug("number of metrics: {0}".format(len(stats)))
+ return stats
+
+ def get_nodes_stats(self):
+ if self.node_name == "":
+ self.error("trying to get node stats, but node name is not set")
+ return None
+
+ url = '{0}/{1}/{2}'.format(self.url, API_NODE, self.node_name)
+ self.debug("doing http request to '{0}'".format(url))
+ raw = self._get_raw_data(url)
+ if not raw:
+ return None
+
+ data = loads(raw)
+ stats = fetch_data(raw_data=data, metrics=NODE_STATS)
+ handle_disabled_disk_monitoring(stats)
+ self.debug("number of metrics: {0}".format(len(stats)))
+ return stats
+
+ def get_vhosts_stats(self):
+ url = '{0}/{1}'.format(self.url, API_VHOSTS)
+ self.debug("doing http request to '{0}'".format(url))
+ raw = self._get_raw_data(url)
+ if not raw:
+ return None
+
+ data = dict()
+ vhosts = loads(raw)
+ charts_initialized = len(self.charts) > 0
+
+ for vhost in vhosts:
+ self.vhost.set(vhost)
+ if not self.vhost.has_msg_stats():
+ continue
+
+ if charts_initialized and self.vhost.name() not in self.collected_vhosts:
+ self.collected_vhosts.add(self.vhost.name())
+ self.add_vhost_charts(self.vhost.name())
+
+ data.update(self.vhost.msg_stats())
+
+ self.debug("number of vhosts: {0}, metrics: {1}".format(len(vhosts), len(data)))
+ return data
+
+ def get_queues_stats(self):
+ url = '{0}/{1}'.format(self.url, API_QUEUES)
+ self.debug("doing http request to '{0}'".format(url))
+ raw = self._get_raw_data(url)
+ if not raw:
+ return None
+
+ data = dict()
+ queues = loads(raw)
+ charts_initialized = len(self.charts) > 0
+
+ for queue in queues:
+ self.queue.set(queue)
+ if self.queue.id()[0] not in self.collected_vhosts:
+ continue
+
+ if charts_initialized and self.queue.id() not in self.collected_queues:
+ self.collected_queues.add(self.queue.id())
+ self.add_queue_charts(self.queue.id())
+
+ data.update(self.queue.queue_stats())
+
+ self.debug("number of queues: {0}, metrics: {1}".format(len(queues), len(data)))
+ return data
+
+ def add_vhost_charts(self, vhost_name):
+ order, charts = vhost_chart_template(vhost_name)
+
+ for chart_name in order:
+ params = [chart_name] + charts[chart_name]['options']
+ dimensions = charts[chart_name]['lines']
+
+ new_chart = self.charts.add_chart(params)
+ for dimension in dimensions:
+ new_chart.add_dimension(dimension)
+
+ def add_queue_charts(self, queue_id):
+ order, charts = queue_chart_template(queue_id)
+
+ for chart_name in order:
+ params = [chart_name] + charts[chart_name]['options']
+ dimensions = charts[chart_name]['lines']
+
+ new_chart = self.charts.add_chart(params)
+ for dimension in dimensions:
+ new_chart.add_dimension(dimension)
+
+
+def fetch_data(raw_data, metrics):
+ data = dict()
+ for metric in metrics:
+ value = raw_data
+ metrics_list = metric.split('.')
+ try:
+ for m in metrics_list:
+ value = value[m]
+ except (KeyError, TypeError):
+ continue
+ data['_'.join(metrics_list)] = value
+
+ return data
+
+
+def handle_disabled_disk_monitoring(node_stats):
+ # https://github.com/netdata/netdata/issues/7218
+ # can be "disk_free": "disk_free_monitoring_disabled"
+ v = node_stats.get('disk_free')
+ if v and not isinstance(v, int):
+ del node_stats['disk_free']
diff --git a/collectors/python.d.plugin/rabbitmq/rabbitmq.conf b/collectors/python.d.plugin/rabbitmq/rabbitmq.conf
new file mode 100644
index 0000000..47d47a1
--- /dev/null
+++ b/collectors/python.d.plugin/rabbitmq/rabbitmq.conf
@@ -0,0 +1,86 @@
+# netdata python.d.plugin configuration for rabbitmq
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, rabbitmq plugin also supports the following:
+#
+# host: 'ipaddress' # Server ip address or hostname. Default: 127.0.0.1
+# port: 'port' # Rabbitmq port. Default: 15672
+# scheme: 'scheme' # http or https. Default: http
+#
+# if the URL is password protected, the following are supported:
+#
+# user: 'username'
+# pass: 'password'
+#
+# Rabbitmq plugin can also collect stats per vhost per queues, which is disabled
+# by default. Please note that enabling this can induced a serious overhead on
+# both netdata and rabbitmq if a look of queues are configured and used.
+#
+# collect_queues_metrics: 'yes/no'
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+#
+local:
+ host: '127.0.0.1'
+ user: 'guest'
+ pass: 'guest'
diff --git a/collectors/python.d.plugin/rethinkdbs/Makefile.inc b/collectors/python.d.plugin/rethinkdbs/Makefile.inc
new file mode 100644
index 0000000..dec6044
--- /dev/null
+++ b/collectors/python.d.plugin/rethinkdbs/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += rethinkdbs/rethinkdbs.chart.py
+dist_pythonconfig_DATA += rethinkdbs/rethinkdbs.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += rethinkdbs/README.md rethinkdbs/Makefile.inc
+
diff --git a/collectors/python.d.plugin/rethinkdbs/README.md b/collectors/python.d.plugin/rethinkdbs/README.md
new file mode 100644
index 0000000..d3fa355
--- /dev/null
+++ b/collectors/python.d.plugin/rethinkdbs/README.md
@@ -0,0 +1,53 @@
+<!--
+title: "RethinkDB monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/rethinkdbs/README.md
+sidebar_label: "RethinkDB"
+-->
+
+# RethinkDB monitoring with Netdata
+
+Collects database server and cluster statistics.
+
+Following charts are drawn:
+
+1. **Connected Servers**
+
+ - connected
+ - missing
+
+2. **Active Clients**
+
+ - active
+
+3. **Queries** per second
+
+ - queries
+
+4. **Documents** per second
+
+ - documents
+
+## Configuration
+
+Edit the `python.d/rethinkdbs.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/rethinkdbs.conf
+```
+
+```yaml
+localhost:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 28015
+ user : "user"
+ password : "pass"
+```
+
+When no configuration file is found, module tries to connect to `127.0.0.1:28015`.
+
+---
+
+
diff --git a/collectors/python.d.plugin/rethinkdbs/rethinkdbs.chart.py b/collectors/python.d.plugin/rethinkdbs/rethinkdbs.chart.py
new file mode 100644
index 0000000..e3fbc36
--- /dev/null
+++ b/collectors/python.d.plugin/rethinkdbs/rethinkdbs.chart.py
@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+# Description: rethinkdb netdata python.d module
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+try:
+ import rethinkdb as rdb
+
+ HAS_RETHINKDB = True
+except ImportError:
+ HAS_RETHINKDB = False
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+ORDER = [
+ 'cluster_connected_servers',
+ 'cluster_clients_active',
+ 'cluster_queries',
+ 'cluster_documents',
+]
+
+
+def cluster_charts():
+ return {
+ 'cluster_connected_servers': {
+ 'options': [None, 'Connected Servers', 'servers', 'cluster', 'rethinkdb.cluster_connected_servers',
+ 'stacked'],
+ 'lines': [
+ ['cluster_servers_connected', 'connected'],
+ ['cluster_servers_missing', 'missing'],
+ ]
+ },
+ 'cluster_clients_active': {
+ 'options': [None, 'Active Clients', 'clients', 'cluster', 'rethinkdb.cluster_clients_active',
+ 'line'],
+ 'lines': [
+ ['cluster_clients_active', 'active'],
+ ]
+ },
+ 'cluster_queries': {
+ 'options': [None, 'Queries', 'queries/s', 'cluster', 'rethinkdb.cluster_queries', 'line'],
+ 'lines': [
+ ['cluster_queries_per_sec', 'queries'],
+ ]
+ },
+ 'cluster_documents': {
+ 'options': [None, 'Documents', 'documents/s', 'cluster', 'rethinkdb.cluster_documents', 'line'],
+ 'lines': [
+ ['cluster_read_docs_per_sec', 'reads'],
+ ['cluster_written_docs_per_sec', 'writes'],
+ ]
+ },
+ }
+
+
+def server_charts(n):
+ o = [
+ '{0}_client_connections'.format(n),
+ '{0}_clients_active'.format(n),
+ '{0}_queries'.format(n),
+ '{0}_documents'.format(n),
+ ]
+ f = 'server {0}'.format(n)
+
+ c = {
+ o[0]: {
+ 'options': [None, 'Client Connections', 'connections', f, 'rethinkdb.client_connections', 'line'],
+ 'lines': [
+ ['{0}_client_connections'.format(n), 'connections'],
+ ]
+ },
+ o[1]: {
+ 'options': [None, 'Active Clients', 'clients', f, 'rethinkdb.clients_active', 'line'],
+ 'lines': [
+ ['{0}_clients_active'.format(n), 'active'],
+ ]
+ },
+ o[2]: {
+ 'options': [None, 'Queries', 'queries/s', f, 'rethinkdb.queries', 'line'],
+ 'lines': [
+ ['{0}_queries_total'.format(n), 'queries', 'incremental'],
+ ]
+ },
+ o[3]: {
+ 'options': [None, 'Documents', 'documents/s', f, 'rethinkdb.documents', 'line'],
+ 'lines': [
+ ['{0}_read_docs_total'.format(n), 'reads', 'incremental'],
+ ['{0}_written_docs_total'.format(n), 'writes', 'incremental'],
+ ]
+ },
+ }
+
+ return o, c
+
+
+class Cluster:
+ def __init__(self, raw):
+ self.raw = raw
+
+ def data(self):
+ qe = self.raw['query_engine']
+
+ return {
+ 'cluster_clients_active': qe['clients_active'],
+ 'cluster_queries_per_sec': qe['queries_per_sec'],
+ 'cluster_read_docs_per_sec': qe['read_docs_per_sec'],
+ 'cluster_written_docs_per_sec': qe['written_docs_per_sec'],
+ 'cluster_servers_connected': 0,
+ 'cluster_servers_missing': 0,
+ }
+
+
+class Server:
+ def __init__(self, raw):
+ self.name = raw['server']
+ self.raw = raw
+
+ def error(self):
+ return self.raw.get('error')
+
+ def data(self):
+ qe = self.raw['query_engine']
+
+ d = {
+ 'client_connections': qe['client_connections'],
+ 'clients_active': qe['clients_active'],
+ 'queries_total': qe['queries_total'],
+ 'read_docs_total': qe['read_docs_total'],
+ 'written_docs_total': qe['written_docs_total'],
+ }
+
+ return dict(('{0}_{1}'.format(self.name, k), d[k]) for k in d)
+
+
+# https://pypi.org/project/rethinkdb/2.4.0/
+# rdb.RethinkDB() can be used as rdb drop in replacement.
+# https://github.com/rethinkdb/rethinkdb-python#quickstart
+def get_rethinkdb():
+ if hasattr(rdb, 'RethinkDB'):
+ return rdb.RethinkDB()
+ return rdb
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = list(ORDER)
+ self.definitions = cluster_charts()
+ self.host = self.configuration.get('host', '127.0.0.1')
+ self.port = self.configuration.get('port', 28015)
+ self.user = self.configuration.get('user', 'admin')
+ self.password = self.configuration.get('password')
+ self.timeout = self.configuration.get('timeout', 2)
+ self.rdb = None
+ self.conn = None
+ self.alive = True
+
+ def check(self):
+ if not HAS_RETHINKDB:
+ self.error('"rethinkdb" module is needed to use rethinkdbs.py')
+ return False
+
+ self.debug("rethinkdb driver version {0}".format(rdb.__version__))
+ self.rdb = get_rethinkdb()
+
+ if not self.connect():
+ return None
+
+ stats = self.get_stats()
+
+ if not stats:
+ return None
+
+ for v in stats[1:]:
+ if get_id(v) == 'server':
+ o, c = server_charts(v['server'])
+ self.order.extend(o)
+ self.definitions.update(c)
+
+ return True
+
+ def get_data(self):
+ if not self.is_alive():
+ return None
+
+ stats = self.get_stats()
+
+ if not stats:
+ return None
+
+ data = dict()
+
+ # cluster
+ data.update(Cluster(stats[0]).data())
+
+ # servers
+ for v in stats[1:]:
+ if get_id(v) != 'server':
+ continue
+
+ s = Server(v)
+
+ if s.error():
+ data['cluster_servers_missing'] += 1
+ else:
+ data['cluster_servers_connected'] += 1
+ data.update(s.data())
+
+ return data
+
+ def get_stats(self):
+ try:
+ return list(self.rdb.db('rethinkdb').table('stats').run(self.conn).items)
+ except rdb.errors.ReqlError:
+ self.alive = False
+ return None
+
+ def connect(self):
+ try:
+ self.conn = self.rdb.connect(
+ host=self.host,
+ port=self.port,
+ user=self.user,
+ password=self.password,
+ timeout=self.timeout,
+ )
+ self.alive = True
+ return True
+ except rdb.errors.ReqlError as error:
+ self.error('Connection to {0}:{1} failed: {2}'.format(self.host, self.port, error))
+ return False
+
+ def reconnect(self):
+ # The connection is already closed after rdb.errors.ReqlError,
+ # so we do not need to call conn.close()
+ if self.connect():
+ return True
+ return False
+
+ def is_alive(self):
+ if not self.alive:
+ return self.reconnect()
+ return True
+
+
+def get_id(v):
+ return v['id'][0]
diff --git a/collectors/python.d.plugin/rethinkdbs/rethinkdbs.conf b/collectors/python.d.plugin/rethinkdbs/rethinkdbs.conf
new file mode 100644
index 0000000..d671acb
--- /dev/null
+++ b/collectors/python.d.plugin/rethinkdbs/rethinkdbs.conf
@@ -0,0 +1,76 @@
+# netdata python.d.plugin configuration for rethinkdb
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, rethinkdb also supports the following:
+#
+# host: IP or HOSTNAME # default is 'localhost'
+# port: PORT # default is 28015
+# user: USERNAME # default is 'admin'
+# password: PASSWORD # not set by default
+# timeout: TIMEOUT # default is 2
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+local:
+ name: 'local'
+ host: 'localhost'
diff --git a/collectors/python.d.plugin/retroshare/Makefile.inc b/collectors/python.d.plugin/retroshare/Makefile.inc
new file mode 100644
index 0000000..891193e
--- /dev/null
+++ b/collectors/python.d.plugin/retroshare/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += retroshare/retroshare.chart.py
+dist_pythonconfig_DATA += retroshare/retroshare.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += retroshare/README.md retroshare/Makefile.inc
+
diff --git a/collectors/python.d.plugin/retroshare/README.md b/collectors/python.d.plugin/retroshare/README.md
new file mode 100644
index 0000000..297df9f
--- /dev/null
+++ b/collectors/python.d.plugin/retroshare/README.md
@@ -0,0 +1,47 @@
+<!--
+title: "RetroShare monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/retroshare/README.md
+sidebar_label: "RetroShare"
+-->
+
+# RetroShare monitoring with Netdata
+
+Monitors application bandwidth, peers and DHT metrics.
+
+This module will monitor one or more `RetroShare` applications, depending on your configuration.
+
+## Charts
+
+This module produces the following charts:
+
+- Bandwidth in `kilobits/s`
+- Peers in `peers`
+- DHT in `peers`
+
+
+## Configuration
+
+Edit the `python.d/retroshare.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/retroshare.conf
+```
+
+Here is an example for 2 servers:
+
+```yaml
+localhost:
+ url : 'http://localhost:9090'
+ user : "user"
+ password : "pass"
+
+remote:
+ url : 'http://203.0.113.1:9090'
+ user : "user"
+ password : "pass"
+```
+---
+
+
diff --git a/collectors/python.d.plugin/retroshare/retroshare.chart.py b/collectors/python.d.plugin/retroshare/retroshare.chart.py
new file mode 100644
index 0000000..3f9593e
--- /dev/null
+++ b/collectors/python.d.plugin/retroshare/retroshare.chart.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Description: RetroShare netdata python.d module
+# Authors: sehraf
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import json
+
+from bases.FrameworkServices.UrlService import UrlService
+
+ORDER = [
+ 'bandwidth',
+ 'peers',
+ 'dht',
+]
+
+CHARTS = {
+ 'bandwidth': {
+ 'options': [None, 'RetroShare Bandwidth', 'kilobits/s', 'RetroShare', 'retroshare.bandwidth', 'area'],
+ 'lines': [
+ ['bandwidth_up_kb', 'Upload'],
+ ['bandwidth_down_kb', 'Download']
+ ]
+ },
+ 'peers': {
+ 'options': [None, 'RetroShare Peers', 'peers', 'RetroShare', 'retroshare.peers', 'line'],
+ 'lines': [
+ ['peers_all', 'All friends'],
+ ['peers_connected', 'Connected friends']
+ ]
+ },
+ 'dht': {
+ 'options': [None, 'Retroshare DHT', 'peers', 'RetroShare', 'retroshare.dht', 'line'],
+ 'lines': [
+ ['dht_size_all', 'DHT nodes estimated'],
+ ['dht_size_rs', 'RS nodes estimated']
+ ]
+ }
+}
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.baseurl = self.configuration.get('url', 'http://localhost:9090')
+
+ def _get_stats(self):
+ """
+ Format data received from http request
+ :return: dict
+ """
+ try:
+ raw = self._get_raw_data()
+ parsed = json.loads(raw)
+ if str(parsed['returncode']) != 'ok':
+ return None
+ except (TypeError, ValueError):
+ return None
+
+ return parsed['data'][0]
+
+ def _get_data(self):
+ """
+ Get data from API
+ :return: dict
+ """
+ self.url = self.baseurl + '/api/v2/stats'
+ data = self._get_stats()
+ if data is None:
+ return None
+
+ data['bandwidth_up_kb'] = data['bandwidth_up_kb'] * -1
+ if data['dht_active'] is False:
+ data['dht_size_all'] = None
+ data['dht_size_rs'] = None
+
+ return data
diff --git a/collectors/python.d.plugin/retroshare/retroshare.conf b/collectors/python.d.plugin/retroshare/retroshare.conf
new file mode 100644
index 0000000..3d0af53
--- /dev/null
+++ b/collectors/python.d.plugin/retroshare/retroshare.conf
@@ -0,0 +1,72 @@
+# netdata python.d.plugin configuration for RetroShare
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, RetroShare also supports the following:
+#
+# - url: 'url' # the URL to the WebUI
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name: 'local'
+ url: 'http://localhost:9090'
diff --git a/collectors/python.d.plugin/riakkv/Makefile.inc b/collectors/python.d.plugin/riakkv/Makefile.inc
new file mode 100644
index 0000000..87d29f8
--- /dev/null
+++ b/collectors/python.d.plugin/riakkv/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += riakkv/riakkv.chart.py
+dist_pythonconfig_DATA += riakkv/riakkv.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += riakkv/README.md riakkv/Makefile.inc
+
diff --git a/collectors/python.d.plugin/riakkv/README.md b/collectors/python.d.plugin/riakkv/README.md
new file mode 100644
index 0000000..fe62c67
--- /dev/null
+++ b/collectors/python.d.plugin/riakkv/README.md
@@ -0,0 +1,126 @@
+<!--
+title: "Riak KV monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/riakkv/README.md
+sidebar_label: "Riak KV"
+-->
+
+# Riak KV monitoring with Netdata
+
+Collects database stats from `/stats` endpoint.
+
+## Requirements
+
+- An accessible `/stats` endpoint. See [the Riak KV configuration reference documentation](https://docs.riak.com/riak/kv/2.2.3/configuring/reference/#client-interfaces)
+ for how to enable this.
+
+The following charts are included, which are mostly derived from the metrics
+listed
+[here](https://docs.riak.com/riak/kv/latest/using/reference/statistics-monitoring/index.html#riak-metrics-to-graph).
+
+1. **Throughput** in operations/s
+
+- **KV operations**
+ - gets
+ - puts
+
+- **Data type updates**
+ - counters
+ - sets
+ - maps
+
+- **Search queries**
+ - queries
+
+- **Search documents**
+ - indexed
+
+- **Strong consistency operations**
+ - gets
+ - puts
+
+2. **Latency** in milliseconds
+
+- **KV latency** of the past minute
+ - get (mean, median, 95th / 99th / 100th percentile)
+ - put (mean, median, 95th / 99th / 100th percentile)
+
+- **Data type latency** of the past minute
+ - counter_merge (mean, median, 95th / 99th / 100th percentile)
+ - set_merge (mean, median, 95th / 99th / 100th percentile)
+ - map_merge (mean, median, 95th / 99th / 100th percentile)
+
+- **Search latency** of the past minute
+ - query (median, min, max, 95th / 99th percentile)
+ - index (median, min, max, 95th / 99th percentile)
+
+- **Strong consistency latency** of the past minute
+ - get (mean, median, 95th / 99th / 100th percentile)
+ - put (mean, median, 95th / 99th / 100th percentile)
+
+3. **Erlang VM metrics**
+
+- **System counters**
+ - processes
+
+- **Memory allocation** in MB
+ - processes.allocated
+ - processes.used
+
+4. **General load / health metrics**
+
+- **Siblings encountered in KV operations** during the past minute
+ - get (mean, median, 95th / 99th / 100th percentile)
+
+- **Object size in KV operations** during the past minute in KB
+ - get (mean, median, 95th / 99th / 100th percentile)
+
+- **Message queue length** in unprocessed messages
+ - vnodeq_size (mean, median, 95th / 99th / 100th percentile)
+
+- **Index operations** encountered by Search
+ - errors
+
+- **Protocol buffer connections**
+ - active
+
+- **Repair operations coordinated by this node**
+ - read
+
+- **Active finite state machines by kind**
+ - get
+ - put
+ - secondary_index
+ - list_keys
+
+- **Rejected finite state machines**
+ - get
+ - put
+
+- **Number of writes to Search failed due to bad data format by reason**
+ - bad_entry
+ - extract_fail
+
+## Configuration
+
+Edit the `python.d/riakkv.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/riakkv.conf
+```
+
+The module needs to be passed the full URL to Riak's stats endpoint.
+For example:
+
+```yaml
+myriak:
+ url: http://myriak.example.com:8098/stats
+```
+
+With no explicit configuration given, the module will attempt to connect to
+`http://localhost:8098/stats`.
+
+The default update frequency for the plugin is set to 2 seconds as Riak
+internally updates the metrics every second. If we were to update the metrics
+every second, the resulting graph would contain odd jitter.
diff --git a/collectors/python.d.plugin/riakkv/riakkv.chart.py b/collectors/python.d.plugin/riakkv/riakkv.chart.py
new file mode 100644
index 0000000..c390c8b
--- /dev/null
+++ b/collectors/python.d.plugin/riakkv/riakkv.chart.py
@@ -0,0 +1,334 @@
+# -*- coding: utf-8 -*-
+# Description: riak netdata python.d module
+#
+# See also:
+# https://docs.riak.com/riak/kv/latest/using/reference/statistics-monitoring/index.html
+
+from json import loads
+
+from bases.FrameworkServices.UrlService import UrlService
+
+# Riak updates the metrics at the /stats endpoint every 1 second.
+# If we use `update_every = 1` here, that means we might get weird jitter in the graph,
+# so the default is set to 2 seconds to prevent it.
+update_every = 2
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = [
+ # Throughput metrics
+ # https://docs.riak.com/riak/kv/latest/using/reference/statistics-monitoring/index.html#throughput-metrics
+ # Collected in totals.
+ "kv.node_operations", # K/V node operations.
+ "dt.vnode_updates", # Data type vnode updates.
+ "search.queries", # Search queries on the node.
+ "search.documents", # Documents indexed by Search.
+ "consistent.operations", # Consistent node operations.
+
+ # Latency metrics
+ # https://docs.riak.com/riak/kv/latest/using/reference/statistics-monitoring/index.html#throughput-metrics
+ # Collected for the past minute in milliseconds,
+ # returned from riak in microseconds.
+ "kv.latency.get", # K/V GET FSM traversal latency.
+ "kv.latency.put", # K/V PUT FSM traversal latency.
+ "dt.latency.counter", # Update Counter Data type latency.
+ "dt.latency.set", # Update Set Data type latency.
+ "dt.latency.map", # Update Map Data type latency.
+ "search.latency.query", # Search query latency.
+ "search.latency.index", # Time it takes for search to index a new document.
+ "consistent.latency.get", # Strong consistent read latency.
+ "consistent.latency.put", # Strong consistent write latency.
+
+ # Erlang resource usage metrics
+ # https://docs.riak.com/riak/kv/latest/using/reference/statistics-monitoring/index.html#erlang-resource-usage-metrics
+ # Processes collected as a gauge,
+ # memory collected as Megabytes, returned as bytes from Riak.
+ "vm.processes", # Number of processes currently running in the Erlang VM.
+ "vm.memory.processes", # Total amount of memory allocated & used for Erlang processes.
+
+ # General Riak Load / Health metrics
+ # https://docs.riak.com/riak/kv/latest/using/reference/statistics-monitoring/index.html#general-riak-load-health-metrics
+ # The following are collected by Riak over the past minute:
+ "kv.siblings_encountered.get", # Siblings encountered during GET operations by this node.
+ "kv.objsize.get", # Object size encountered by this node.
+ "search.vnodeq_size", # Number of unprocessed messages in the vnode message queues (Search).
+ # The following are calculated in total, or as gauges:
+ "search.index_errors", # Errors of the search subsystem while indexing documents.
+ "core.pbc", # Number of currently active protocol buffer connections.
+ "core.repairs", # Total read repair operations coordinated by this node.
+ "core.fsm_active", # Active finite state machines by kind.
+ "core.fsm_rejected", # Rejected finite state machines by kind.
+
+ # General Riak Search Load / Health metrics
+ # https://docs.riak.com/riak/kv/latest/using/reference/statistics-monitoring/index.html#general-riak-search-load-health-metrics
+ # Reported as counters.
+ "search.errors", # Write and read errors of the Search subsystem.
+]
+
+CHARTS = {
+ # Throughput metrics
+ "kv.node_operations": {
+ "options": [None, "Reads & writes coordinated by this node", "operations/s", "throughput", "riak.kv.throughput",
+ "line"],
+ "lines": [
+ ["node_gets_total", "gets", "incremental"],
+ ["node_puts_total", "puts", "incremental"]
+ ]
+ },
+ "dt.vnode_updates": {
+ "options": [None, "Update operations coordinated by local vnodes by data type", "operations/s", "throughput",
+ "riak.dt.vnode_updates", "line"],
+ "lines": [
+ ["vnode_counter_update_total", "counters", "incremental"],
+ ["vnode_set_update_total", "sets", "incremental"],
+ ["vnode_map_update_total", "maps", "incremental"],
+ ]
+ },
+ "search.queries": {
+ "options": [None, "Search queries on the node", "queries/s", "throughput", "riak.search", "line"],
+ "lines": [
+ ["search_query_throughput_count", "queries", "incremental"]
+ ]
+ },
+ "search.documents": {
+ "options": [None, "Documents indexed by search", "documents/s", "throughput", "riak.search.documents", "line"],
+ "lines": [
+ ["search_index_throughput_count", "indexed", "incremental"]
+ ]
+ },
+ "consistent.operations": {
+ "options": [None, "Consistent node operations", "operations/s", "throughput", "riak.consistent.operations",
+ "line"],
+ "lines": [
+ ["consistent_gets_total", "gets", "incremental"],
+ ["consistent_puts_total", "puts", "incremental"],
+ ]
+ },
+
+ # Latency metrics
+ "kv.latency.get": {
+ "options": [None, "Time between reception of a client GET request and subsequent response to client", "ms",
+ "latency", "riak.kv.latency.get", "line"],
+ "lines": [
+ ["node_get_fsm_time_mean", "mean", "absolute", 1, 1000],
+ ["node_get_fsm_time_median", "median", "absolute", 1, 1000],
+ ["node_get_fsm_time_95", "95", "absolute", 1, 1000],
+ ["node_get_fsm_time_99", "99", "absolute", 1, 1000],
+ ["node_get_fsm_time_100", "100", "absolute", 1, 1000],
+ ]
+ },
+ "kv.latency.put": {
+ "options": [None, "Time between reception of a client PUT request and subsequent response to client", "ms",
+ "latency", "riak.kv.latency.put", "line"],
+ "lines": [
+ ["node_put_fsm_time_mean", "mean", "absolute", 1, 1000],
+ ["node_put_fsm_time_median", "median", "absolute", 1, 1000],
+ ["node_put_fsm_time_95", "95", "absolute", 1, 1000],
+ ["node_put_fsm_time_99", "99", "absolute", 1, 1000],
+ ["node_put_fsm_time_100", "100", "absolute", 1, 1000],
+ ]
+ },
+ "dt.latency.counter": {
+ "options": [None, "Time it takes to perform an Update Counter operation", "ms", "latency",
+ "riak.dt.latency.counter_merge", "line"],
+ "lines": [
+ ["object_counter_merge_time_mean", "mean", "absolute", 1, 1000],
+ ["object_counter_merge_time_median", "median", "absolute", 1, 1000],
+ ["object_counter_merge_time_95", "95", "absolute", 1, 1000],
+ ["object_counter_merge_time_99", "99", "absolute", 1, 1000],
+ ["object_counter_merge_time_100", "100", "absolute", 1, 1000],
+ ]
+ },
+ "dt.latency.set": {
+ "options": [None, "Time it takes to perform an Update Set operation", "ms", "latency",
+ "riak.dt.latency.set_merge", "line"],
+ "lines": [
+ ["object_set_merge_time_mean", "mean", "absolute", 1, 1000],
+ ["object_set_merge_time_median", "median", "absolute", 1, 1000],
+ ["object_set_merge_time_95", "95", "absolute", 1, 1000],
+ ["object_set_merge_time_99", "99", "absolute", 1, 1000],
+ ["object_set_merge_time_100", "100", "absolute", 1, 1000],
+ ]
+ },
+ "dt.latency.map": {
+ "options": [None, "Time it takes to perform an Update Map operation", "ms", "latency",
+ "riak.dt.latency.map_merge", "line"],
+ "lines": [
+ ["object_map_merge_time_mean", "mean", "absolute", 1, 1000],
+ ["object_map_merge_time_median", "median", "absolute", 1, 1000],
+ ["object_map_merge_time_95", "95", "absolute", 1, 1000],
+ ["object_map_merge_time_99", "99", "absolute", 1, 1000],
+ ["object_map_merge_time_100", "100", "absolute", 1, 1000],
+ ]
+ },
+ "search.latency.query": {
+ "options": [None, "Search query latency", "ms", "latency", "riak.search.latency.query", "line"],
+ "lines": [
+ ["search_query_latency_median", "median", "absolute", 1, 1000],
+ ["search_query_latency_min", "min", "absolute", 1, 1000],
+ ["search_query_latency_95", "95", "absolute", 1, 1000],
+ ["search_query_latency_99", "99", "absolute", 1, 1000],
+ ["search_query_latency_999", "999", "absolute", 1, 1000],
+ ["search_query_latency_max", "max", "absolute", 1, 1000],
+ ]
+ },
+ "search.latency.index": {
+ "options": [None, "Time it takes Search to index a new document", "ms", "latency", "riak.search.latency.index",
+ "line"],
+ "lines": [
+ ["search_index_latency_median", "median", "absolute", 1, 1000],
+ ["search_index_latency_min", "min", "absolute", 1, 1000],
+ ["search_index_latency_95", "95", "absolute", 1, 1000],
+ ["search_index_latency_99", "99", "absolute", 1, 1000],
+ ["search_index_latency_999", "999", "absolute", 1, 1000],
+ ["search_index_latency_max", "max", "absolute", 1, 1000],
+ ]
+ },
+
+ # Riak Strong Consistency metrics
+ "consistent.latency.get": {
+ "options": [None, "Strongly consistent read latency", "ms", "latency", "riak.consistent.latency.get", "line"],
+ "lines": [
+ ["consistent_get_time_mean", "mean", "absolute", 1, 1000],
+ ["consistent_get_time_median", "median", "absolute", 1, 1000],
+ ["consistent_get_time_95", "95", "absolute", 1, 1000],
+ ["consistent_get_time_99", "99", "absolute", 1, 1000],
+ ["consistent_get_time_100", "100", "absolute", 1, 1000],
+ ]
+ },
+ "consistent.latency.put": {
+ "options": [None, "Strongly consistent write latency", "ms", "latency", "riak.consistent.latency.put", "line"],
+ "lines": [
+ ["consistent_put_time_mean", "mean", "absolute", 1, 1000],
+ ["consistent_put_time_median", "median", "absolute", 1, 1000],
+ ["consistent_put_time_95", "95", "absolute", 1, 1000],
+ ["consistent_put_time_99", "99", "absolute", 1, 1000],
+ ["consistent_put_time_100", "100", "absolute", 1, 1000],
+ ]
+ },
+
+ # BEAM metrics
+ "vm.processes": {
+ "options": [None, "Total processes running in the Erlang VM", "total", "vm", "riak.vm", "line"],
+ "lines": [
+ ["sys_process_count", "processes", "absolute"],
+ ]
+ },
+ "vm.memory.processes": {
+ "options": [None, "Memory allocated & used by Erlang processes", "MB", "vm", "riak.vm.memory.processes",
+ "line"],
+ "lines": [
+ ["memory_processes", "allocated", "absolute", 1, 1024 * 1024],
+ ["memory_processes_used", "used", "absolute", 1, 1024 * 1024]
+ ]
+ },
+
+ # General Riak Load/Health metrics
+ "kv.siblings_encountered.get": {
+ "options": [None, "Number of siblings encountered during GET operations by this node during the past minute",
+ "siblings", "load", "riak.kv.siblings_encountered.get", "line"],
+ "lines": [
+ ["node_get_fsm_siblings_mean", "mean", "absolute"],
+ ["node_get_fsm_siblings_median", "median", "absolute"],
+ ["node_get_fsm_siblings_95", "95", "absolute"],
+ ["node_get_fsm_siblings_99", "99", "absolute"],
+ ["node_get_fsm_siblings_100", "100", "absolute"],
+ ]
+ },
+ "kv.objsize.get": {
+ "options": [None, "Object size encountered by this node during the past minute", "KB", "load",
+ "riak.kv.objsize.get", "line"],
+ "lines": [
+ ["node_get_fsm_objsize_mean", "mean", "absolute", 1, 1024],
+ ["node_get_fsm_objsize_median", "median", "absolute", 1, 1024],
+ ["node_get_fsm_objsize_95", "95", "absolute", 1, 1024],
+ ["node_get_fsm_objsize_99", "99", "absolute", 1, 1024],
+ ["node_get_fsm_objsize_100", "100", "absolute", 1, 1024],
+ ]
+ },
+ "search.vnodeq_size": {
+ "options": [None,
+ "Number of unprocessed messages in the vnode message queues of Search on this node in the past minute",
+ "messages", "load", "riak.search.vnodeq_size", "line"],
+ "lines": [
+ ["riak_search_vnodeq_mean", "mean", "absolute"],
+ ["riak_search_vnodeq_median", "median", "absolute"],
+ ["riak_search_vnodeq_95", "95", "absolute"],
+ ["riak_search_vnodeq_99", "99", "absolute"],
+ ["riak_search_vnodeq_100", "100", "absolute"],
+ ]
+ },
+ "search.index_errors": {
+ "options": [None, "Number of document index errors encountered by Search", "errors", "load",
+ "riak.search.index", "line"],
+ "lines": [
+ ["search_index_fail_count", "errors", "absolute"]
+ ]
+ },
+ "core.pbc": {
+ "options": [None, "Protocol buffer connections by status", "connections", "load",
+ "riak.core.protobuf_connections", "line"],
+ "lines": [
+ ["pbc_active", "active", "absolute"],
+ # ["pbc_connects", "established_pastmin", "absolute"]
+ ]
+ },
+ "core.repairs": {
+ "options": [None, "Number of repair operations this node has coordinated", "repairs", "load",
+ "riak.core.repairs", "line"],
+ "lines": [
+ ["read_repairs", "read", "absolute"]
+ ]
+ },
+ "core.fsm_active": {
+ "options": [None, "Active finite state machines by kind", "fsms", "load", "riak.core.fsm_active", "line"],
+ "lines": [
+ ["node_get_fsm_active", "get", "absolute"],
+ ["node_put_fsm_active", "put", "absolute"],
+ ["index_fsm_active", "secondary index", "absolute"],
+ ["list_fsm_active", "list keys", "absolute"]
+ ]
+ },
+ "core.fsm_rejected": {
+ # Writing "Sidejob's" here seems to cause some weird issues: it results in this chart being rendered in
+ # its own context and additionally, moves the entire Riak graph all the way up to the top of the Netdata
+ # dashboard for some reason.
+ "options": [None, "Finite state machines being rejected by Sidejobs overload protection", "fsms", "load",
+ "riak.core.fsm_rejected", "line"],
+ "lines": [
+ ["node_get_fsm_rejected", "get", "absolute"],
+ ["node_put_fsm_rejected", "put", "absolute"]
+ ]
+ },
+
+ # General Riak Search Load / Health metrics
+ "search.errors": {
+ "options": [None, "Number of writes to Search failed due to bad data format by reason", "writes", "load",
+ "riak.search.index", "line"],
+ "lines": [
+ ["search_index_bad_entry_count", "bad_entry", "absolute"],
+ ["search_index_extract_fail_count", "extract_fail", "absolute"],
+ ]
+ }
+}
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+
+ def _get_data(self):
+ """
+ Format data received from http request
+ :return: dict
+ """
+ raw = self._get_raw_data()
+ if not raw:
+ return None
+
+ try:
+ return loads(raw)
+ except (TypeError, ValueError) as err:
+ self.error(err)
+ return None
diff --git a/collectors/python.d.plugin/riakkv/riakkv.conf b/collectors/python.d.plugin/riakkv/riakkv.conf
new file mode 100644
index 0000000..be01c48
--- /dev/null
+++ b/collectors/python.d.plugin/riakkv/riakkv.conf
@@ -0,0 +1,68 @@
+# netdata python.d.plugin configuration for riak
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+local:
+ url : 'http://localhost:8098/stats'
diff --git a/collectors/python.d.plugin/samba/Makefile.inc b/collectors/python.d.plugin/samba/Makefile.inc
new file mode 100644
index 0000000..230a8ba
--- /dev/null
+++ b/collectors/python.d.plugin/samba/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += samba/samba.chart.py
+dist_pythonconfig_DATA += samba/samba.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += samba/README.md samba/Makefile.inc
+
diff --git a/collectors/python.d.plugin/samba/README.md b/collectors/python.d.plugin/samba/README.md
new file mode 100644
index 0000000..767df12
--- /dev/null
+++ b/collectors/python.d.plugin/samba/README.md
@@ -0,0 +1,121 @@
+<!--
+title: "Samba monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/samba/README.md
+sidebar_label: "Samba"
+-->
+
+# Samba monitoring with Netdata
+
+Monitors the performance metrics of Samba file sharing using `smbstatus` command-line tool.
+
+Executed commands:
+
+- `sudo -n smbstatus -P`
+
+## Requirements
+
+- `smbstatus` program
+- `sudo` program
+- `smbd` must be compiled with profiling enabled
+- `smbd` must be started either with the `-P 1` option or inside `smb.conf` using `smbd profiling level`
+
+The module uses `smbstatus`, which can only be executed by `root`. It uses
+`sudo` and assumes that it is configured such that the `netdata` user can execute `smbstatus` as root without a
+password.
+
+- Add to your `/etc/sudoers` file:
+
+`which smbstatus` shows the full path to the binary.
+
+```bash
+netdata ALL=(root) NOPASSWD: /path/to/smbstatus
+```
+
+- Reset Netdata's systemd
+ unit [CapabilityBoundingSet](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#Capabilities) (Linux
+ distributions with systemd)
+
+The default CapabilityBoundingSet doesn't allow using `sudo`, and is quite strict in general. Resetting is not optimal, but a next-best solution given the inability to execute `smbstatus` using `sudo`.
+
+
+As the `root` user, do the following:
+
+```cmd
+mkdir /etc/systemd/system/netdata.service.d
+echo -e '[Service]\nCapabilityBoundingSet=~' | tee /etc/systemd/system/netdata.service.d/unset-capability-bounding-set.conf
+systemctl daemon-reload
+systemctl restart netdata.service
+```
+
+## Charts
+
+1. **Syscall R/Ws** in kilobytes/s
+
+ - sendfile
+ - recvfile
+
+2. **Smb2 R/Ws** in kilobytes/s
+
+ - readout
+ - writein
+ - readin
+ - writeout
+
+3. **Smb2 Create/Close** in operations/s
+
+ - create
+ - close
+
+4. **Smb2 Info** in operations/s
+
+ - getinfo
+ - setinfo
+
+5. **Smb2 Find** in operations/s
+
+ - find
+
+6. **Smb2 Notify** in operations/s
+
+ - notify
+
+7. **Smb2 Lesser Ops** as counters
+
+ - tcon
+ - negprot
+ - tdis
+ - cancel
+ - logoff
+ - flush
+ - lock
+ - keepalive
+ - break
+ - sessetup
+
+## Enable the collector
+
+The `samba` collector is disabled by default. To enable it, use `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`, to edit the `python.d.conf`
+file.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d.conf
+```
+
+Change the value of the `samba` setting to `yes`. Save the file and restart the Netdata Agent with `sudo systemctl
+restart netdata`, or the [appropriate method](/docs/configure/start-stop-restart.md) for your system.
+
+## Configuration
+
+Edit the `python.d/samba.conf` configuration file using `edit-config` from the
+Netdata [config directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/samba.conf
+```
+
+---
+
+
diff --git a/collectors/python.d.plugin/samba/samba.chart.py b/collectors/python.d.plugin/samba/samba.chart.py
new file mode 100644
index 0000000..8eebcd6
--- /dev/null
+++ b/collectors/python.d.plugin/samba/samba.chart.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+# Description: samba netdata python.d module
+# Author: Christopher Cox <chris_cox@endlessnow.com>
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# The netdata user needs to be able to be able to sudo the smbstatus program
+# without password:
+# netdata ALL=(ALL) NOPASSWD: /usr/bin/smbstatus -P
+#
+# This makes calls to smbstatus -P
+#
+# This just looks at a couple of values out of syscall, and some from smb2.
+#
+# The Lesser Ops chart is merely a display of current counter values. They
+# didn't seem to change much to me. However, if you notice something changing
+# a lot there, bring one or more out into its own chart and make it incremental
+# (like find and notify... good examples).
+
+import re
+import os
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+from bases.collection import find_binary
+
+disabled_by_default = True
+
+update_every = 5
+
+ORDER = [
+ 'syscall_rw',
+ 'smb2_rw',
+ 'smb2_create_close',
+ 'smb2_info',
+ 'smb2_find',
+ 'smb2_notify',
+ 'smb2_sm_count'
+]
+
+CHARTS = {
+ 'syscall_rw': {
+ 'options': [None, 'R/Ws', 'KiB/s', 'syscall', 'syscall.rw', 'area'],
+ 'lines': [
+ ['syscall_sendfile_bytes', 'sendfile', 'incremental', 1, 1024],
+ ['syscall_recvfile_bytes', 'recvfile', 'incremental', -1, 1024]
+ ]
+ },
+ 'smb2_rw': {
+ 'options': [None, 'R/Ws', 'KiB/s', 'smb2', 'smb2.rw', 'area'],
+ 'lines': [
+ ['smb2_read_outbytes', 'readout', 'incremental', 1, 1024],
+ ['smb2_write_inbytes', 'writein', 'incremental', -1, 1024],
+ ['smb2_read_inbytes', 'readin', 'incremental', 1, 1024],
+ ['smb2_write_outbytes', 'writeout', 'incremental', -1, 1024]
+ ]
+ },
+ 'smb2_create_close': {
+ 'options': [None, 'Create/Close', 'operations/s', 'smb2', 'smb2.create_close', 'line'],
+ 'lines': [
+ ['smb2_create_count', 'create', 'incremental', 1, 1],
+ ['smb2_close_count', 'close', 'incremental', -1, 1]
+ ]
+ },
+ 'smb2_info': {
+ 'options': [None, 'Info', 'operations/s', 'smb2', 'smb2.get_set_info', 'line'],
+ 'lines': [
+ ['smb2_getinfo_count', 'getinfo', 'incremental', 1, 1],
+ ['smb2_setinfo_count', 'setinfo', 'incremental', -1, 1]
+ ]
+ },
+ 'smb2_find': {
+ 'options': [None, 'Find', 'operations/s', 'smb2', 'smb2.find', 'line'],
+ 'lines': [
+ ['smb2_find_count', 'find', 'incremental', 1, 1]
+ ]
+ },
+ 'smb2_notify': {
+ 'options': [None, 'Notify', 'operations/s', 'smb2', 'smb2.notify', 'line'],
+ 'lines': [
+ ['smb2_notify_count', 'notify', 'incremental', 1, 1]
+ ]
+ },
+ 'smb2_sm_count': {
+ 'options': [None, 'Lesser Ops', 'count', 'smb2', 'smb2.sm_counters', 'stacked'],
+ 'lines': [
+ ['smb2_tcon_count', 'tcon', 'absolute', 1, 1],
+ ['smb2_negprot_count', 'negprot', 'absolute', 1, 1],
+ ['smb2_tdis_count', 'tdis', 'absolute', 1, 1],
+ ['smb2_cancel_count', 'cancel', 'absolute', 1, 1],
+ ['smb2_logoff_count', 'logoff', 'absolute', 1, 1],
+ ['smb2_flush_count', 'flush', 'absolute', 1, 1],
+ ['smb2_lock_count', 'lock', 'absolute', 1, 1],
+ ['smb2_keepalive_count', 'keepalive', 'absolute', 1, 1],
+ ['smb2_break_count', 'break', 'absolute', 1, 1],
+ ['smb2_sessetup_count', 'sessetup', 'absolute', 1, 1]
+ ]
+ }
+}
+
+SUDO = 'sudo'
+SMBSTATUS = 'smbstatus'
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ ExecutableService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.rgx_smb2 = re.compile(r'(smb2_[^:]+|syscall_.*file_bytes):\s+(\d+)')
+
+ def check(self):
+ smbstatus_binary = find_binary(SMBSTATUS)
+ if not smbstatus_binary:
+ self.error("can't locate '{0}' binary".format(SMBSTATUS))
+ return False
+
+ if os.getuid() == 0:
+ self.command = ' '.join([smbstatus_binary, '-P'])
+ return ExecutableService.check(self)
+
+ sudo_binary = find_binary(SUDO)
+ if not sudo_binary:
+ self.error("can't locate '{0}' binary".format(SUDO))
+ return False
+ command = [sudo_binary, '-n', '-l', smbstatus_binary, '-P']
+ smbstatus = '{0} -P'.format(smbstatus_binary)
+ allowed = self._get_raw_data(command=command)
+ if not (allowed and allowed[0].strip() == smbstatus):
+ self.error("not allowed to run sudo for command '{0}'".format(smbstatus))
+ return False
+ self.command = ' '.join([sudo_binary, '-n', smbstatus_binary, '-P'])
+ return ExecutableService.check(self)
+
+ def _get_data(self):
+ """
+ Format data received from shell command
+ :return: dict
+ """
+ raw_data = self._get_raw_data()
+ if not raw_data:
+ return None
+
+ parsed = self.rgx_smb2.findall(' '.join(raw_data))
+
+ return dict(parsed) or None
diff --git a/collectors/python.d.plugin/samba/samba.conf b/collectors/python.d.plugin/samba/samba.conf
new file mode 100644
index 0000000..db15d4e
--- /dev/null
+++ b/collectors/python.d.plugin/samba/samba.conf
@@ -0,0 +1,60 @@
+# netdata python.d.plugin configuration for samba
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+update_every: 5
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds \ No newline at end of file
diff --git a/collectors/python.d.plugin/sensors/Makefile.inc b/collectors/python.d.plugin/sensors/Makefile.inc
new file mode 100644
index 0000000..5fb26e1
--- /dev/null
+++ b/collectors/python.d.plugin/sensors/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += sensors/sensors.chart.py
+dist_pythonconfig_DATA += sensors/sensors.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += sensors/README.md sensors/Makefile.inc
+
diff --git a/collectors/python.d.plugin/sensors/README.md b/collectors/python.d.plugin/sensors/README.md
new file mode 100644
index 0000000..e791195
--- /dev/null
+++ b/collectors/python.d.plugin/sensors/README.md
@@ -0,0 +1,33 @@
+<!--
+title: "Linux machine sensors monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/sensors/README.md
+sidebar_label: "Linux machine sensors"
+-->
+
+# Linux machine sensors monitoring with Netdata
+
+Reads system sensors information (temperature, voltage, electric current, power, etc.).
+
+Charts are created dynamically.
+
+## Configuration
+
+Edit the `python.d/sensors.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/sensors.conf
+```
+
+### possible issues
+
+There have been reports from users that on certain servers, ACPI ring buffer errors are printed by the kernel (`dmesg`) when ACPI sensors are being accessed.
+We are tracking such cases in issue [#827](https://github.com/netdata/netdata/issues/827).
+Please join this discussion for help.
+
+When `lm-sensors` doesn't work on your device (e.g. for RPi temperatures), use [the legacy bash collector](https://learn.netdata.cloud/docs/agent/collectors/charts.d.plugin/sensors)
+
+---
+
+
diff --git a/collectors/python.d.plugin/sensors/sensors.chart.py b/collectors/python.d.plugin/sensors/sensors.chart.py
new file mode 100644
index 0000000..701bf64
--- /dev/null
+++ b/collectors/python.d.plugin/sensors/sensors.chart.py
@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+# Description: sensors netdata python.d plugin
+# Author: Pawel Krupa (paulfantom)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from collections import defaultdict
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from third_party import lm_sensors as sensors
+
+ORDER = [
+ 'temperature',
+ 'fan',
+ 'voltage',
+ 'current',
+ 'power',
+ 'energy',
+ 'humidity',
+]
+
+# This is a prototype of chart definition which is used to dynamically create self.definitions
+CHARTS = {
+ 'temperature': {
+ 'options': [None, 'Temperature', 'Celsius', 'temperature', 'sensors.temperature', 'line'],
+ 'lines': [
+ [None, None, 'absolute', 1, 1000]
+ ]
+ },
+ 'voltage': {
+ 'options': [None, 'Voltage', 'Volts', 'voltage', 'sensors.voltage', 'line'],
+ 'lines': [
+ [None, None, 'absolute', 1, 1000]
+ ]
+ },
+ 'current': {
+ 'options': [None, 'Current', 'Ampere', 'current', 'sensors.current', 'line'],
+ 'lines': [
+ [None, None, 'absolute', 1, 1000]
+ ]
+ },
+ 'power': {
+ 'options': [None, 'Power', 'Watt', 'power', 'sensors.power', 'line'],
+ 'lines': [
+ [None, None, 'absolute', 1, 1000]
+ ]
+ },
+ 'fan': {
+ 'options': [None, 'Fans speed', 'Rotations/min', 'fans', 'sensors.fan', 'line'],
+ 'lines': [
+ [None, None, 'absolute', 1, 1000]
+ ]
+ },
+ 'energy': {
+ 'options': [None, 'Energy', 'Joule', 'energy', 'sensors.energy', 'line'],
+ 'lines': [
+ [None, None, 'incremental', 1, 1000]
+ ]
+ },
+ 'humidity': {
+ 'options': [None, 'Humidity', 'Percent', 'humidity', 'sensors.humidity', 'line'],
+ 'lines': [
+ [None, None, 'absolute', 1, 1000]
+ ]
+ }
+}
+
+LIMITS = {
+ 'temperature': [-127, 1000],
+ 'voltage': [-127, 127],
+ 'current': [-127, 127],
+ 'fan': [0, 65535]
+}
+
+TYPE_MAP = {
+ 0: 'voltage',
+ 1: 'fan',
+ 2: 'temperature',
+ 3: 'power',
+ 4: 'energy',
+ 5: 'current',
+ 6: 'humidity',
+ # 7: 'max_main',
+ # 16: 'vid',
+ # 17: 'intrusion',
+ # 18: 'max_other',
+ # 24: 'beep_enable'
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = list()
+ self.definitions = dict()
+ self.chips = configuration.get('chips')
+ self.priority = 60000
+
+ def get_data(self):
+ seen, data = dict(), dict()
+ try:
+ for chip in sensors.ChipIterator():
+ chip_name = sensors.chip_snprintf_name(chip)
+ seen[chip_name] = defaultdict(list)
+
+ for feat in sensors.FeatureIterator(chip):
+ if feat.type not in TYPE_MAP:
+ continue
+
+ feat_type = TYPE_MAP[feat.type]
+ feat_name = str(feat.name.decode())
+ feat_label = sensors.get_label(chip, feat)
+ feat_limits = LIMITS.get(feat_type)
+ sub_feat = next(sensors.SubFeatureIterator(chip, feat)) # current value
+
+ if not sub_feat:
+ continue
+
+ try:
+ v = sensors.get_value(chip, sub_feat.number)
+ except sensors.SensorsError:
+ continue
+
+ if v is None:
+ continue
+
+ seen[chip_name][feat_type].append((feat_name, feat_label))
+
+ if feat_limits and (v < feat_limits[0] or v > feat_limits[1]):
+ continue
+
+ data[chip_name + '_' + feat_name] = int(v * 1000)
+
+ except sensors.SensorsError as error:
+ self.error(error)
+ return None
+
+ self.update_sensors_charts(seen)
+
+ return data or None
+
+ def update_sensors_charts(self, seen):
+ for chip_name, feat in seen.items():
+ if self.chips and not any([chip_name.startswith(ex) for ex in self.chips]):
+ continue
+
+ for feat_type, sub_feat in feat.items():
+ if feat_type not in ORDER or feat_type not in CHARTS:
+ continue
+
+ chart_id = '{}_{}'.format(chip_name, feat_type)
+ if chart_id in self.charts:
+ continue
+
+ params = [chart_id] + list(CHARTS[feat_type]['options'])
+ new_chart = self.charts.add_chart(params)
+ new_chart.params['priority'] = self.get_chart_priority(feat_type)
+
+ for name, label in sub_feat:
+ lines = list(CHARTS[feat_type]['lines'][0])
+ lines[0] = chip_name + '_' + name
+ lines[1] = label
+ new_chart.add_dimension(lines)
+
+ def check(self):
+ try:
+ sensors.init()
+ except sensors.SensorsError as error:
+ self.error(error)
+ return False
+
+ self.priority = self.charts.priority
+
+ return bool(self.get_data() and self.charts)
+
+ def get_chart_priority(self, feat_type):
+ for i, v in enumerate(ORDER):
+ if v == feat_type:
+ return self.priority + i
+ return self.priority
diff --git a/collectors/python.d.plugin/sensors/sensors.conf b/collectors/python.d.plugin/sensors/sensors.conf
new file mode 100644
index 0000000..d3369ba
--- /dev/null
+++ b/collectors/python.d.plugin/sensors/sensors.conf
@@ -0,0 +1,61 @@
+# netdata python.d.plugin configuration for sensors
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# Limit the number of sensors types.
+# Comment the ones you want to disable.
+# Also, re-arranging this list controls the order of the charts at the
+# netdata dashboard.
+
+types:
+ - temperature
+ - fan
+ - voltage
+ - current
+ - power
+ - energy
+ - humidity
+
+# ----------------------------------------------------------------------
+# Limit the number of sensors chips.
+# Uncomment the first line (chips:) and add chip names below it.
+# The chip names that start with like that will be matched.
+# You can find the chip names using the sensors command.
+
+#chips:
+# - i8k
+# - coretemp
+#
+# chip names can be found using the sensors shell command
+# the prefix is matched (anything that starts like that)
+#
+#----------------------------------------------------------------------
+
diff --git a/collectors/python.d.plugin/smartd_log/Makefile.inc b/collectors/python.d.plugin/smartd_log/Makefile.inc
new file mode 100644
index 0000000..dc1d0f3
--- /dev/null
+++ b/collectors/python.d.plugin/smartd_log/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += smartd_log/smartd_log.chart.py
+dist_pythonconfig_DATA += smartd_log/smartd_log.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += smartd_log/README.md smartd_log/Makefile.inc
+
diff --git a/collectors/python.d.plugin/smartd_log/README.md b/collectors/python.d.plugin/smartd_log/README.md
new file mode 100644
index 0000000..eef34ce
--- /dev/null
+++ b/collectors/python.d.plugin/smartd_log/README.md
@@ -0,0 +1,125 @@
+<!--
+title: "Storage devices monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/smartd_log/README.md
+sidebar_label: "S.M.A.R.T. attributes"
+-->
+
+# Storage devices monitoring with Netdata
+
+Monitors `smartd` log files to collect HDD/SSD S.M.A.R.T attributes.
+
+## Requirements
+
+- `smartmontools`
+
+It produces following charts for SCSI devices:
+
+1. **Read Error Corrected**
+
+2. **Read Error Uncorrected**
+
+3. **Write Error Corrected**
+
+4. **Write Error Uncorrected**
+
+5. **Verify Error Corrected**
+
+6. **Verify Error Uncorrected**
+
+7. **Temperature**
+
+For ATA devices:
+
+1. **Read Error Rate**
+
+2. **Seek Error Rate**
+
+3. **Soft Read Error Rate**
+
+4. **Write Error Rate**
+
+5. **SATA Interface Downshift**
+
+6. **UDMA CRC Error Count**
+
+7. **Throughput Performance**
+
+8. **Seek Time Performance**
+
+9. **Start/Stop Count**
+
+10. **Power-On Hours Count**
+
+11. **Power Cycle Count**
+
+12. **Unexpected Power Loss**
+
+13. **Spin-Up Time**
+
+14. **Spin-up Retries**
+
+15. **Calibration Retries**
+
+16. **Temperature**
+
+17. **Reallocated Sectors Count**
+
+18. **Reserved Block Count**
+
+19. **Program Fail Count**
+
+20. **Erase Fail Count**
+
+21. **Wear Leveller Worst Case Erase Count**
+
+22. **Unused Reserved NAND Blocks**
+
+23. **Reallocation Event Count**
+
+24. **Current Pending Sector Count**
+
+25. **Offline Uncorrectable Sector Count**
+
+26. **Percent Lifetime Used**
+
+## prerequisite
+
+`smartd` must be running with `-A` option to write smartd attribute information to files.
+
+For this you need to set `smartd_opts` (or `SMARTD_ARGS`, check _smartd.service_ content) in `/etc/default/smartmontools`:
+
+```
+# dump smartd attrs info every 600 seconds
+smartd_opts="-A /var/log/smartd/ -i 600"
+```
+
+You may need to create the smartd directory before smartd will write to it:
+
+```sh
+mkdir -p /var/log/smartd
+```
+
+Otherwise, all the smartd `.csv` files may get written to `/var/lib/smartmontools` (default location). See also <https://linux.die.net/man/8/smartd> for more info on the `-A --attributelog=PREFIX` command.
+
+`smartd` appends logs at every run. It's strongly recommended to use `logrotate` for smartd files.
+
+## Configuration
+
+Edit the `python.d/smartd_log.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/smartd_log.conf
+```
+
+```yaml
+local:
+ log_path : '/var/log/smartd/'
+```
+
+If no configuration is given, module will attempt to read log files in `/var/log/smartd/` directory.
+
+---
+
+
diff --git a/collectors/python.d.plugin/smartd_log/smartd_log.chart.py b/collectors/python.d.plugin/smartd_log/smartd_log.chart.py
new file mode 100644
index 0000000..dc4e95d
--- /dev/null
+++ b/collectors/python.d.plugin/smartd_log/smartd_log.chart.py
@@ -0,0 +1,772 @@
+# -*- coding: utf-8 -*-
+# Description: smart netdata python.d module
+# Author: ilyam8, vorph1
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+import re
+from copy import deepcopy
+from time import time
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from bases.collection import read_last_line
+
+INCREMENTAL = 'incremental'
+ABSOLUTE = 'absolute'
+
+ATA = 'ata'
+SCSI = 'scsi'
+CSV = '.csv'
+
+DEF_RESCAN_INTERVAL = 60
+DEF_AGE = 30
+DEF_PATH = '/var/log/smartd'
+
+ATTR1 = '1'
+ATTR2 = '2'
+ATTR3 = '3'
+ATTR4 = '4'
+ATTR5 = '5'
+ATTR7 = '7'
+ATTR8 = '8'
+ATTR9 = '9'
+ATTR10 = '10'
+ATTR11 = '11'
+ATTR12 = '12'
+ATTR13 = '13'
+ATTR170 = '170'
+ATTR171 = '171'
+ATTR172 = '172'
+ATTR173 = '173'
+ATTR174 = '174'
+ATTR180 = '180'
+ATTR183 = '183'
+ATTR190 = '190'
+ATTR194 = '194'
+ATTR196 = '196'
+ATTR197 = '197'
+ATTR198 = '198'
+ATTR199 = '199'
+ATTR202 = '202'
+ATTR206 = '206'
+ATTR233 = '233'
+ATTR249 = '249'
+ATTR_READ_ERR_COR = 'read-total-err-corrected'
+ATTR_READ_ERR_UNC = 'read-total-unc-errors'
+ATTR_WRITE_ERR_COR = 'write-total-err-corrected'
+ATTR_WRITE_ERR_UNC = 'write-total-unc-errors'
+ATTR_VERIFY_ERR_COR = 'verify-total-err-corrected'
+ATTR_VERIFY_ERR_UNC = 'verify-total-unc-errors'
+ATTR_TEMPERATURE = 'temperature'
+
+RE_ATA = re.compile(
+ '(\d+);' # attribute
+ '(\d+);' # normalized value
+ '(\d+)', # raw value
+ re.X
+)
+
+RE_SCSI = re.compile(
+ '([a-z-]+);' # attribute
+ '([0-9.]+)', # raw value
+ re.X
+)
+
+ORDER = [
+ # errors
+ 'read_error_rate',
+ 'seek_error_rate',
+ 'soft_read_error_rate',
+ 'write_error_rate',
+ 'read_total_err_corrected',
+ 'read_total_unc_errors',
+ 'write_total_err_corrected',
+ 'write_total_unc_errors',
+ 'verify_total_err_corrected',
+ 'verify_total_unc_errors',
+ # external failure
+ 'sata_interface_downshift',
+ 'udma_crc_error_count',
+ # performance
+ 'throughput_performance',
+ 'seek_time_performance',
+ # power
+ 'start_stop_count',
+ 'power_on_hours_count',
+ 'power_cycle_count',
+ 'unexpected_power_loss',
+ # spin
+ 'spin_up_time',
+ 'spin_up_retries',
+ 'calibration_retries',
+ # temperature
+ 'airflow_temperature_celsius',
+ 'temperature_celsius',
+ # wear
+ 'reallocated_sectors_count',
+ 'reserved_block_count',
+ 'program_fail_count',
+ 'erase_fail_count',
+ 'wear_leveller_worst_case_erase_count',
+ 'unused_reserved_nand_blocks',
+ 'reallocation_event_count',
+ 'current_pending_sector_count',
+ 'offline_uncorrectable_sector_count',
+ 'percent_lifetime_used',
+ 'media_wearout_indicator',
+]
+
+CHARTS = {
+ 'read_error_rate': {
+ 'options': [None, 'Read Error Rate', 'value', 'errors', 'smartd_log.read_error_rate', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR1],
+ 'algo': ABSOLUTE,
+ },
+ 'seek_error_rate': {
+ 'options': [None, 'Seek Error Rate', 'value', 'errors', 'smartd_log.seek_error_rate', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR7],
+ 'algo': ABSOLUTE,
+ },
+ 'soft_read_error_rate': {
+ 'options': [None, 'Soft Read Error Rate', 'errors', 'errors', 'smartd_log.soft_read_error_rate', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR13],
+ 'algo': INCREMENTAL,
+ },
+ 'write_error_rate': {
+ 'options': [None, 'Write Error Rate', 'value', 'errors', 'smartd_log.write_error_rate', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR206],
+ 'algo': ABSOLUTE,
+ },
+ 'read_total_err_corrected': {
+ 'options': [None, 'Read Error Corrected', 'errors', 'errors', 'smartd_log.read_total_err_corrected', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR_READ_ERR_COR],
+ 'algo': INCREMENTAL,
+ },
+ 'read_total_unc_errors': {
+ 'options': [None, 'Read Error Uncorrected', 'errors', 'errors', 'smartd_log.read_total_unc_errors', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR_READ_ERR_UNC],
+ 'algo': INCREMENTAL,
+ },
+ 'write_total_err_corrected': {
+ 'options': [None, 'Write Error Corrected', 'errors', 'errors', 'smartd_log.write_total_err_corrected', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR_WRITE_ERR_COR],
+ 'algo': INCREMENTAL,
+ },
+ 'write_total_unc_errors': {
+ 'options': [None, 'Write Error Uncorrected', 'errors', 'errors', 'smartd_log.write_total_unc_errors', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR_WRITE_ERR_UNC],
+ 'algo': INCREMENTAL,
+ },
+ 'verify_total_err_corrected': {
+ 'options': [None, 'Verify Error Corrected', 'errors', 'errors', 'smartd_log.verify_total_err_corrected',
+ 'line'],
+ 'lines': [],
+ 'attrs': [ATTR_VERIFY_ERR_COR],
+ 'algo': INCREMENTAL,
+ },
+ 'verify_total_unc_errors': {
+ 'options': [None, 'Verify Error Uncorrected', 'errors', 'errors', 'smartd_log.verify_total_unc_errors', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR_VERIFY_ERR_UNC],
+ 'algo': INCREMENTAL,
+ },
+ 'sata_interface_downshift': {
+ 'options': [None, 'SATA Interface Downshift', 'events', 'external failure',
+ 'smartd_log.sata_interface_downshift', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR183],
+ 'algo': INCREMENTAL,
+ },
+ 'udma_crc_error_count': {
+ 'options': [None, 'UDMA CRC Error Count', 'errors', 'external failure', 'smartd_log.udma_crc_error_count',
+ 'line'],
+ 'lines': [],
+ 'attrs': [ATTR199],
+ 'algo': INCREMENTAL,
+ },
+ 'throughput_performance': {
+ 'options': [None, 'Throughput Performance', 'value', 'performance', 'smartd_log.throughput_performance',
+ 'line'],
+ 'lines': [],
+ 'attrs': [ATTR2],
+ 'algo': ABSOLUTE,
+ },
+ 'seek_time_performance': {
+ 'options': [None, 'Seek Time Performance', 'value', 'performance', 'smartd_log.seek_time_performance', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR8],
+ 'algo': ABSOLUTE,
+ },
+ 'start_stop_count': {
+ 'options': [None, 'Start/Stop Count', 'events', 'power', 'smartd_log.start_stop_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR4],
+ 'algo': ABSOLUTE,
+ },
+ 'power_on_hours_count': {
+ 'options': [None, 'Power-On Hours Count', 'hours', 'power', 'smartd_log.power_on_hours_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR9],
+ 'algo': ABSOLUTE,
+ },
+ 'power_cycle_count': {
+ 'options': [None, 'Power Cycle Count', 'events', 'power', 'smartd_log.power_cycle_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR12],
+ 'algo': ABSOLUTE,
+ },
+ 'unexpected_power_loss': {
+ 'options': [None, 'Unexpected Power Loss', 'events', 'power', 'smartd_log.unexpected_power_loss', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR174],
+ 'algo': ABSOLUTE,
+ },
+ 'spin_up_time': {
+ 'options': [None, 'Spin-Up Time', 'ms', 'spin', 'smartd_log.spin_up_time', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR3],
+ 'algo': ABSOLUTE,
+ },
+ 'spin_up_retries': {
+ 'options': [None, 'Spin-up Retries', 'retries', 'spin', 'smartd_log.spin_up_retries', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR10],
+ 'algo': INCREMENTAL,
+ },
+ 'calibration_retries': {
+ 'options': [None, 'Calibration Retries', 'retries', 'spin', 'smartd_log.calibration_retries', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR11],
+ 'algo': INCREMENTAL,
+ },
+ 'airflow_temperature_celsius': {
+ 'options': [None, 'Airflow Temperature Celsius', 'celsius', 'temperature',
+ 'smartd_log.airflow_temperature_celsius', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR190],
+ 'algo': ABSOLUTE,
+ },
+ 'temperature_celsius': {
+ 'options': [None, 'Temperature', 'celsius', 'temperature', 'smartd_log.temperature_celsius', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR194, ATTR_TEMPERATURE],
+ 'algo': ABSOLUTE,
+ },
+ 'reallocated_sectors_count': {
+ 'options': [None, 'Reallocated Sectors Count', 'sectors', 'wear', 'smartd_log.reallocated_sectors_count',
+ 'line'],
+ 'lines': [],
+ 'attrs': [ATTR5],
+ 'algo': ABSOLUTE,
+ },
+ 'reserved_block_count': {
+ 'options': [None, 'Reserved Block Count', 'percentage', 'wear', 'smartd_log.reserved_block_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR170],
+ 'algo': ABSOLUTE,
+ },
+ 'program_fail_count': {
+ 'options': [None, 'Program Fail Count', 'errors', 'wear', 'smartd_log.program_fail_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR171],
+ 'algo': INCREMENTAL,
+ },
+ 'erase_fail_count': {
+ 'options': [None, 'Erase Fail Count', 'failures', 'wear', 'smartd_log.erase_fail_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR172],
+ 'algo': INCREMENTAL,
+ },
+ 'wear_leveller_worst_case_erase_count': {
+ 'options': [None, 'Wear Leveller Worst Case Erase Count', 'erases', 'wear',
+ 'smartd_log.wear_leveller_worst_case_erase_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR173],
+ 'algo': ABSOLUTE,
+ },
+ 'unused_reserved_nand_blocks': {
+ 'options': [None, 'Unused Reserved NAND Blocks', 'blocks', 'wear', 'smartd_log.unused_reserved_nand_blocks',
+ 'line'],
+ 'lines': [],
+ 'attrs': [ATTR180],
+ 'algo': ABSOLUTE,
+ },
+ 'reallocation_event_count': {
+ 'options': [None, 'Reallocation Event Count', 'events', 'wear', 'smartd_log.reallocation_event_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR196],
+ 'algo': INCREMENTAL,
+ },
+ 'current_pending_sector_count': {
+ 'options': [None, 'Current Pending Sector Count', 'sectors', 'wear', 'smartd_log.current_pending_sector_count',
+ 'line'],
+ 'lines': [],
+ 'attrs': [ATTR197],
+ 'algo': ABSOLUTE,
+ },
+ 'offline_uncorrectable_sector_count': {
+ 'options': [None, 'Offline Uncorrectable Sector Count', 'sectors', 'wear',
+ 'smartd_log.offline_uncorrectable_sector_count', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR198],
+ 'algo': ABSOLUTE,
+
+ },
+ 'percent_lifetime_used': {
+ 'options': [None, 'Percent Lifetime Used', 'percentage', 'wear', 'smartd_log.percent_lifetime_used', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR202],
+ 'algo': ABSOLUTE,
+ },
+ 'media_wearout_indicator': {
+ 'options': [None, 'Media Wearout Indicator', 'percentage', 'wear', 'smartd_log.media_wearout_indicator', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR233],
+ 'algo': ABSOLUTE,
+ },
+ 'nand_writes_1gib': {
+ 'options': [None, 'NAND Writes', 'GiB', 'wear', 'smartd_log.nand_writes_1gib', 'line'],
+ 'lines': [],
+ 'attrs': [ATTR249],
+ 'algo': ABSOLUTE,
+ },
+}
+
+# NOTE: 'parse_temp' decodes ATA 194 raw value. Not heavily tested. Written by @Ferroin
+# C code:
+# https://github.com/smartmontools/smartmontools/blob/master/smartmontools/atacmds.cpp#L2051
+#
+# Calling 'parse_temp' on the raw value will return a 4-tuple, containing
+# * temperature
+# * minimum
+# * maximum
+# * over-temperature count
+# substituting None for values it can't decode.
+#
+# Example:
+# >>> parse_temp(42952491042)
+# >>> (34, 10, 43, None)
+#
+#
+# def check_temp_word(i):
+# if i <= 0x7F:
+# return 0x11
+# elif i <= 0xFF:
+# return 0x01
+# elif 0xFF80 <= i:
+# return 0x10
+# return 0x00
+#
+#
+# def check_temp_range(t, b0, b1):
+# if b0 > b1:
+# t0, t1 = b1, b0
+# else:
+# t0, t1 = b0, b1
+#
+# if all([
+# -60 <= t0,
+# t0 <= t,
+# t <= t1,
+# t1 <= 120,
+# not (t0 == -1 and t1 <= 0)
+# ]):
+# return t0, t1
+# return None, None
+#
+#
+# def parse_temp(raw):
+# byte = list()
+# word = list()
+# for i in range(0, 6):
+# byte.append(0xFF & (raw >> (i * 8)))
+# for i in range(0, 3):
+# word.append(0xFFFF & (raw >> (i * 16)))
+#
+# ctwd = check_temp_word(word[0])
+#
+# if not word[2]:
+# if ctwd and not word[1]:
+# # byte[0] is temp, no other data
+# return byte[0], None, None, None
+#
+# if ctwd and all(check_temp_range(byte[0], byte[2], byte[3])):
+# # byte[0] is temp, byte[2] is max or min, byte[3] is min or max
+# trange = check_temp_range(byte[0], byte[2], byte[3])
+# return byte[0], trange[0], trange[1], None
+#
+# if ctwd and all(check_temp_range(byte[0], byte[1], byte[2])):
+# # byte[0] is temp, byte[1] is max or min, byte[2] is min or max
+# trange = check_temp_range(byte[0], byte[1], byte[2])
+# return byte[0], trange[0], trange[1], None
+#
+# return None, None, None, None
+#
+# if ctwd:
+# if all(
+# [
+# ctwd & check_temp_word(word[1]) & check_temp_word(word[2]) != 0x00,
+# all(check_temp_range(byte[0], byte[2], byte[4])),
+# ]
+# ):
+# # byte[0] is temp, byte[2] is max or min, byte[4] is min or max
+# trange = check_temp_range(byte[0], byte[2], byte[4])
+# return byte[0], trange[0], trange[1], None
+# else:
+# trange = check_temp_range(byte[0], byte[2], byte[3])
+# if word[2] < 0x7FFF and all(trange) and trange[1] >= 40:
+# # byte[0] is temp, byte[2] is max or min, byte[3] is min or max, word[2] is overtemp count
+# return byte[0], trange[0], trange[1], word[2]
+# # no data
+# return None, None, None, None
+
+
+CHARTED_ATTRS = dict((attr, k) for k, v in CHARTS.items() for attr in v['attrs'])
+
+
+class BaseAtaSmartAttribute:
+ def __init__(self, name, normalized_value, raw_value):
+ self.name = name
+ self.normalized_value = normalized_value
+ self.raw_value = raw_value
+
+ def value(self):
+ raise NotImplementedError
+
+
+class AtaRaw(BaseAtaSmartAttribute):
+ def value(self):
+ return self.raw_value
+
+
+class AtaNormalized(BaseAtaSmartAttribute):
+ def value(self):
+ return self.normalized_value
+
+
+class Ata3(BaseAtaSmartAttribute):
+ def value(self):
+ value = int(self.raw_value)
+ # https://github.com/netdata/netdata/issues/5919
+ #
+ # 3;151;38684000679;
+ # 423 (Average 447)
+ # 38684000679 & 0xFFF -> 423
+ # (38684000679 & 0xFFF0000) >> 16 -> 447
+ if value > 1e6:
+ return value & 0xFFF
+ return value
+
+
+class Ata9(BaseAtaSmartAttribute):
+ def value(self):
+ value = int(self.raw_value)
+ if value > 1e6:
+ return value & 0xFFFF
+ return value
+
+
+class Ata190(BaseAtaSmartAttribute):
+ def value(self):
+ return 100 - int(self.normalized_value)
+
+
+class Ata194(BaseAtaSmartAttribute):
+ # https://github.com/netdata/netdata/issues/3041
+ # https://github.com/netdata/netdata/issues/5919
+ #
+ # The low byte is the current temperature, the third lowest is the maximum, and the fifth lowest is the minimum
+ def value(self):
+ value = int(self.raw_value)
+ if value > 1e6:
+ return value & 0xFF
+ return min(int(self.normalized_value), int(self.raw_value))
+
+
+class BaseSCSISmartAttribute:
+ def __init__(self, name, raw_value):
+ self.name = name
+ self.raw_value = raw_value
+
+ def value(self):
+ raise NotImplementedError
+
+
+class SCSIRaw(BaseSCSISmartAttribute):
+ def value(self):
+ return self.raw_value
+
+
+def ata_attribute_factory(value):
+ name = value[0]
+
+ if name == ATTR3:
+ return Ata3(*value)
+ elif name == ATTR9:
+ return Ata9(*value)
+ elif name == ATTR190:
+ return Ata190(*value)
+ elif name == ATTR194:
+ return Ata194(*value)
+ elif name in [
+ ATTR1,
+ ATTR7,
+ ATTR202,
+ ATTR206,
+ ATTR233,
+ ]:
+ return AtaNormalized(*value)
+
+ return AtaRaw(*value)
+
+
+def scsi_attribute_factory(value):
+ return SCSIRaw(*value)
+
+
+def attribute_factory(value):
+ name = value[0]
+ if name.isdigit():
+ return ata_attribute_factory(value)
+ return scsi_attribute_factory(value)
+
+
+def handle_error(*errors):
+ def on_method(method):
+ def on_call(*args):
+ try:
+ return method(*args)
+ except errors:
+ return None
+
+ return on_call
+
+ return on_method
+
+
+class DiskLogFile:
+ def __init__(self, full_path):
+ self.path = full_path
+ self.size = os.path.getsize(full_path)
+
+ @handle_error(OSError)
+ def is_changed(self):
+ return self.size != os.path.getsize(self.path)
+
+ @handle_error(OSError)
+ def is_active(self, current_time, limit):
+ return (current_time - os.path.getmtime(self.path)) / 60 < limit
+
+ @handle_error(OSError)
+ def read(self):
+ self.size = os.path.getsize(self.path)
+ return read_last_line(self.path)
+
+
+class BaseDisk:
+ def __init__(self, name, log_file):
+ self.raw_name = name
+ self.name = re.sub(r'_+', '_', name)
+ self.log_file = log_file
+ self.attrs = list()
+ self.alive = True
+ self.charted = False
+
+ def __eq__(self, other):
+ if isinstance(other, BaseDisk):
+ return self.raw_name == other.raw_name
+ return self.raw_name == other
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ return hash(repr(self))
+
+ def parser(self, data):
+ raise NotImplementedError
+
+ @handle_error(TypeError)
+ def populate_attrs(self):
+ self.attrs = list()
+ line = self.log_file.read()
+ for value in self.parser(line):
+ self.attrs.append(attribute_factory(value))
+
+ return len(self.attrs)
+
+ def data(self):
+ data = dict()
+ for attr in self.attrs:
+ data['{0}_{1}'.format(self.name, attr.name)] = attr.value()
+ return data
+
+
+class ATADisk(BaseDisk):
+ def parser(self, data):
+ return RE_ATA.findall(data)
+
+
+class SCSIDisk(BaseDisk):
+ def parser(self, data):
+ return RE_SCSI.findall(data)
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = deepcopy(CHARTS)
+ self.log_path = configuration.get('log_path', DEF_PATH)
+ self.age = configuration.get('age', DEF_AGE)
+ self.exclude = configuration.get('exclude_disks', str()).split()
+ self.disks = list()
+ self.runs = 0
+ self.do_force_rescan = False
+
+ def check(self):
+ return self.scan() > 0
+
+ def get_data(self):
+ self.runs += 1
+
+ if self.do_force_rescan or self.runs % DEF_RESCAN_INTERVAL == 0:
+ self.cleanup()
+ self.scan()
+ self.do_force_rescan = False
+
+ data = dict()
+
+ for disk in self.disks:
+ if not disk.alive:
+ continue
+
+ if not disk.charted:
+ self.add_disk_to_charts(disk)
+
+ changed = disk.log_file.is_changed()
+
+ if changed is None:
+ disk.alive = False
+ self.do_force_rescan = True
+ continue
+
+ if changed and disk.populate_attrs() is None:
+ disk.alive = False
+ self.do_force_rescan = True
+ continue
+
+ data.update(disk.data())
+
+ return data
+
+ def cleanup(self):
+ current_time = time()
+ for disk in self.disks[:]:
+ if any(
+ [
+ not disk.alive,
+ not disk.log_file.is_active(current_time, self.age),
+ ]
+ ):
+ self.disks.remove(disk.raw_name)
+ self.remove_disk_from_charts(disk)
+
+ def scan(self):
+ self.debug('scanning {0}'.format(self.log_path))
+ current_time = time()
+
+ for full_name in os.listdir(self.log_path):
+ disk = self.create_disk_from_file(full_name, current_time)
+ if not disk:
+ continue
+ self.disks.append(disk)
+
+ return len(self.disks)
+
+ def create_disk_from_file(self, full_name, current_time):
+ if not full_name.endswith(CSV):
+ self.debug('skipping {0}: not a csv file'.format(full_name))
+ return None
+
+ name = os.path.basename(full_name).split('.')[-3]
+ path = os.path.join(self.log_path, full_name)
+
+ if name in self.disks:
+ self.debug('skipping {0}: already in disks'.format(full_name))
+ return None
+
+ if [p for p in self.exclude if p in name]:
+ self.debug('skipping {0}: filtered by `exclude` option'.format(full_name))
+ return None
+
+ if not os.access(path, os.R_OK):
+ self.debug('skipping {0}: not readable'.format(full_name))
+ return None
+
+ if os.path.getsize(path) == 0:
+ self.debug('skipping {0}: zero size'.format(full_name))
+ return None
+
+ if (current_time - os.path.getmtime(path)) / 60 > self.age:
+ self.debug('skipping {0}: haven\'t been updated for last {1} minutes'.format(full_name, self.age))
+ return None
+
+ if ATA in full_name:
+ disk = ATADisk(name, DiskLogFile(path))
+ elif SCSI in full_name:
+ disk = SCSIDisk(name, DiskLogFile(path))
+ else:
+ self.debug('skipping {0}: unknown type'.format(full_name))
+ return None
+
+ disk.populate_attrs()
+ if not disk.attrs:
+ self.error('skipping {0}: parsing failed'.format(full_name))
+ return None
+
+ self.debug('added {0}'.format(full_name))
+ return disk
+
+ def add_disk_to_charts(self, disk):
+ if len(self.charts) == 0 or disk.charted:
+ return
+ disk.charted = True
+
+ for attr in disk.attrs:
+ chart_id = CHARTED_ATTRS.get(attr.name)
+
+ if not chart_id or chart_id not in self.charts:
+ continue
+
+ chart = self.charts[chart_id]
+ dim = [
+ '{0}_{1}'.format(disk.name, attr.name),
+ disk.name,
+ CHARTS[chart_id]['algo'],
+ ]
+
+ if dim[0] in self.charts[chart_id].dimensions:
+ chart.hide_dimension(dim[0], reverse=True)
+ else:
+ chart.add_dimension(dim)
+
+ def remove_disk_from_charts(self, disk):
+ if len(self.charts) == 0 or not disk.charted:
+ return
+
+ for attr in disk.attrs:
+ chart_id = CHARTED_ATTRS.get(attr.name)
+
+ if not chart_id or chart_id not in self.charts:
+ continue
+
+ self.charts[chart_id].del_dimension('{0}_{1}'.format(disk.name, attr.name))
diff --git a/collectors/python.d.plugin/smartd_log/smartd_log.conf b/collectors/python.d.plugin/smartd_log/smartd_log.conf
new file mode 100644
index 0000000..6c01d95
--- /dev/null
+++ b/collectors/python.d.plugin/smartd_log/smartd_log.conf
@@ -0,0 +1,75 @@
+# netdata python.d.plugin configuration for smartd log
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, smartd_log also supports the following:
+#
+# log_path: '/path/to/smartd_logs' # path to smartd log files. Default is /var/log/smartd
+# exclude_disks: 'PATTERN1 PATTERN2' # space separated patterns. If the pattern is in the drive name, the module will not collect data for it.
+#
+# ----------------------------------------------------------------------
+
+custom:
+ name: smartd_log
+ log_path: '/var/log/smartd/'
+
+debian:
+ name: smartd_log
+ log_path: '/var/lib/smartmontools/'
diff --git a/collectors/python.d.plugin/spigotmc/Makefile.inc b/collectors/python.d.plugin/spigotmc/Makefile.inc
new file mode 100644
index 0000000..f9fa8b6
--- /dev/null
+++ b/collectors/python.d.plugin/spigotmc/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += spigotmc/spigotmc.chart.py
+dist_pythonconfig_DATA += spigotmc/spigotmc.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += spigotmc/README.md spigotmc/Makefile.inc
+
diff --git a/collectors/python.d.plugin/spigotmc/README.md b/collectors/python.d.plugin/spigotmc/README.md
new file mode 100644
index 0000000..0648318
--- /dev/null
+++ b/collectors/python.d.plugin/spigotmc/README.md
@@ -0,0 +1,38 @@
+<!--
+title: "SpigotMC monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/spigotmc/README.md
+sidebar_label: "SpigotMC"
+-->
+
+# SpigotMC monitoring with Netdata
+
+Performs basic monitoring for Spigot Minecraft servers.
+
+It provides two charts, one tracking server-side ticks-per-second in
+1, 5 and 15 minute averages, and one tracking the number of currently
+active users.
+
+This is not compatible with Spigot plugins which change the format of
+the data returned by the `tps` or `list` console commands.
+
+## Configuration
+
+Edit the `python.d/spigotmc.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/spigotmc.conf
+```
+
+```yaml
+host: localhost
+port: 25575
+password: pass
+```
+
+By default, a connection to port 25575 on the local system is attempted with an empty password.
+
+---
+
+
diff --git a/collectors/python.d.plugin/spigotmc/spigotmc.chart.py b/collectors/python.d.plugin/spigotmc/spigotmc.chart.py
new file mode 100644
index 0000000..81370fb
--- /dev/null
+++ b/collectors/python.d.plugin/spigotmc/spigotmc.chart.py
@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+# Description: spigotmc netdata python.d module
+# Author: Austin S. Hemmelgarn (Ferroin)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import platform
+import re
+import socket
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from third_party import mcrcon
+
+# Update only every 5 seconds because collection takes in excess of
+# 100ms sometimes, and most people won't care about second-by-second data.
+update_every = 5
+
+PRECISION = 100
+
+COMMAND_TPS = 'tps'
+COMMAND_LIST = 'list'
+COMMAND_ONLINE = 'online'
+
+ORDER = [
+ 'tps',
+ 'mem',
+ 'users',
+]
+
+CHARTS = {
+ 'tps': {
+ 'options': [None, 'Spigot Ticks Per Second', 'ticks', 'spigotmc', 'spigotmc.tps', 'line'],
+ 'lines': [
+ ['tps1', '1 Minute Average', 'absolute', 1, PRECISION],
+ ['tps5', '5 Minute Average', 'absolute', 1, PRECISION],
+ ['tps15', '15 Minute Average', 'absolute', 1, PRECISION]
+ ]
+ },
+ 'users': {
+ 'options': [None, 'Minecraft Users', 'users', 'spigotmc', 'spigotmc.users', 'area'],
+ 'lines': [
+ ['users', 'Users', 'absolute', 1, 1]
+ ]
+ },
+ 'mem': {
+ 'options': [None, 'Minecraft Memory Usage', 'MiB', 'spigotmc', 'spigotmc.mem', 'line'],
+ 'lines': [
+ ['mem_used', 'used', 'absolute', 1, 1],
+ ['mem_alloc', 'allocated', 'absolute', 1, 1],
+ ['mem_max', 'max', 'absolute', 1, 1]
+ ]
+ }
+}
+
+_TPS_REGEX = re.compile(
+ # Examples:
+ # §6TPS from last 1m, 5m, 15m: §a*20.0, §a*20.0, §a*20.0
+ # §6Current Memory Usage: §a936/65536 mb (Max: 65536 mb)
+ r'^.*: .*?' # Message lead-in
+ r'(\d{1,2}.\d+), .*?' # 1-minute TPS value
+ r'(\d{1,2}.\d+), .*?' # 5-minute TPS value
+ r'(\d{1,2}\.\d+).*?' # 15-minute TPS value
+ r'(\s.*?(\d+)\/(\d+).*?: (\d+).*)?', # Current Memory Usage / Total Memory (Max Memory)
+ re.MULTILINE
+)
+_LIST_REGEX = re.compile(
+ # Examples:
+ # There are 4 of a max 50 players online: player1, player2, player3, player4
+ # §6There are §c4§6 out of maximum §c50§6 players online.
+ # §6There are §c3§6/§c1§6 out of maximum §c50§6 players online.
+ # §6当前有 §c4§6 个玩家在线,最大在线人数为 §c50§6 个玩家.
+ # §c4§6 人のプレイヤーが接続中です。最大接続可能人数\:§c 50
+ r'[^§](\d+)(?:.*?(?=/).*?[^§](\d+))?', # Current user count.
+ re.X
+)
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.host = self.configuration.get('host', 'localhost')
+ self.port = self.configuration.get('port', 25575)
+ self.password = self.configuration.get('password', '')
+ self.console = mcrcon.MCRcon()
+ self.alive = True
+
+ def check(self):
+ if platform.system() != 'Linux':
+ self.error('Only supported on Linux.')
+ return False
+ try:
+ self.connect()
+ except (mcrcon.MCRconException, socket.error) as err:
+ self.error('Error connecting.')
+ self.error(repr(err))
+ return False
+
+ return self._get_data()
+
+ def connect(self):
+ self.console.connect(self.host, self.port, self.password)
+
+ def reconnect(self):
+ self.error('try reconnect.')
+ try:
+ try:
+ self.console.disconnect()
+ except mcrcon.MCRconException:
+ pass
+ self.console.connect(self.host, self.port, self.password)
+ self.alive = True
+ except (mcrcon.MCRconException, socket.error) as err:
+ self.error('Error connecting.')
+ self.error(repr(err))
+ return False
+ return True
+
+ def is_alive(self):
+ if any(
+ [
+ not self.alive,
+ self.console.socket.getsockopt(socket.IPPROTO_TCP, socket.TCP_INFO, 0) != 1
+ ]
+ ):
+ return self.reconnect()
+ return True
+
+ def _get_data(self):
+ if not self.is_alive():
+ return None
+
+ data = {}
+
+ try:
+ raw = self.console.command(COMMAND_TPS)
+ match = _TPS_REGEX.match(raw)
+ if match:
+ data['tps1'] = int(float(match.group(1)) * PRECISION)
+ data['tps5'] = int(float(match.group(2)) * PRECISION)
+ data['tps15'] = int(float(match.group(3)) * PRECISION)
+ if match.group(4):
+ data['mem_used'] = int(match.group(5))
+ data['mem_alloc'] = int(match.group(6))
+ data['mem_max'] = int(match.group(7))
+ else:
+ self.error('Unable to process TPS values.')
+ if not raw:
+ self.error(
+ "'{0}' command returned no value, make sure you set correct password".format(COMMAND_TPS))
+ except mcrcon.MCRconException:
+ self.error('Unable to fetch TPS values.')
+ except socket.error:
+ self.error('Connection is dead.')
+ self.alive = False
+ return None
+
+ try:
+ raw = self.console.command(COMMAND_LIST)
+ match = _LIST_REGEX.search(raw)
+ if not match:
+ raw = self.console.command(COMMAND_ONLINE)
+ match = _LIST_REGEX.search(raw)
+ if match:
+ users = int(match.group(1))
+ hidden_users = match.group(2)
+ if hidden_users:
+ hidden_users = int(hidden_users)
+ else:
+ hidden_users = 0
+ data['users'] = users + hidden_users
+ else:
+ if not raw:
+ self.error("'{0}' and '{1}' commands returned no value, make sure you set correct password".format(
+ COMMAND_LIST, COMMAND_ONLINE))
+ self.error('Unable to process user counts.')
+ except mcrcon.MCRconException:
+ self.error('Unable to fetch user counts.')
+ except socket.error:
+ self.error('Connection is dead.')
+ self.alive = False
+ return None
+
+ return data
diff --git a/collectors/python.d.plugin/spigotmc/spigotmc.conf b/collectors/python.d.plugin/spigotmc/spigotmc.conf
new file mode 100644
index 0000000..f0064ea
--- /dev/null
+++ b/collectors/python.d.plugin/spigotmc/spigotmc.conf
@@ -0,0 +1,66 @@
+# netdata python.d.plugin configuration for spigotmc
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# In addition to the above, spigotmc supports the following:
+#
+# host: localhost # The host to connect to. Defaults to the local system.
+# port: 25575 # The port the remote console is listening on.
+# password: '' # The remote console password. Most be set correctly.
diff --git a/collectors/python.d.plugin/springboot/Makefile.inc b/collectors/python.d.plugin/springboot/Makefile.inc
new file mode 100644
index 0000000..06775f9
--- /dev/null
+++ b/collectors/python.d.plugin/springboot/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += springboot/springboot.chart.py
+dist_pythonconfig_DATA += springboot/springboot.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += springboot/README.md springboot/Makefile.inc
+
diff --git a/collectors/python.d.plugin/springboot/README.md b/collectors/python.d.plugin/springboot/README.md
new file mode 100644
index 0000000..cdbc9a9
--- /dev/null
+++ b/collectors/python.d.plugin/springboot/README.md
@@ -0,0 +1,145 @@
+<!--
+title: "Java Spring Boot 2 application monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/springboot/README.md
+sidebar_label: "Java Spring Boot 2 applications"
+-->
+
+# Java Spring Boot 2 application monitoring with Netdata
+
+Monitors one or more Java Spring-boot applications depending on configuration.
+Netdata can be used to monitor running Java [Spring Boot](https://spring.io/) applications that expose their metrics with the use of the **Spring Boot Actuator** included in Spring Boot library.
+
+## Configuration
+
+The Spring Boot Actuator exposes these metrics over HTTP and is very easy to use:
+
+- add `org.springframework.boot:spring-boot-starter-actuator` to your application dependencies
+- set `endpoints.metrics.sensitive=false` in your `application.properties`
+
+You can create custom Metrics by add and inject a PublicMetrics in your application.
+This is a example to add custom metrics:
+
+```java
+package com.example;
+
+import org.springframework.boot.actuate.endpoint.PublicMetrics;
+import org.springframework.boot.actuate.metrics.Metric;
+import org.springframework.stereotype.Service;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryPoolMXBean;
+import java.util.ArrayList;
+import java.util.Collection;
+
+@Service
+public class HeapPoolMetrics implements PublicMetrics {
+
+ private static final String PREFIX = "mempool.";
+ private static final String KEY_EDEN = PREFIX + "eden";
+ private static final String KEY_SURVIVOR = PREFIX + "survivor";
+ private static final String KEY_TENURED = PREFIX + "tenured";
+
+ @Override
+ public Collection<Metric<?>> metrics() {
+ Collection<Metric<?>> result = new ArrayList<>(4);
+ for (MemoryPoolMXBean mem : ManagementFactory.getMemoryPoolMXBeans()) {
+ String poolName = mem.getName();
+ String name = null;
+ if (poolName.indexOf("Eden Space") != -1) {
+ name = KEY_EDEN;
+ } else if (poolName.indexOf("Survivor Space") != -1) {
+ name = KEY_SURVIVOR;
+ } else if (poolName.indexOf("Tenured Gen") != -1 || poolName.indexOf("Old Gen") != -1) {
+ name = KEY_TENURED;
+ }
+
+ if (name != null) {
+ result.add(newMemoryMetric(name, mem.getUsage().getMax()));
+ result.add(newMemoryMetric(name + ".init", mem.getUsage().getInit()));
+ result.add(newMemoryMetric(name + ".committed", mem.getUsage().getCommitted()));
+ result.add(newMemoryMetric(name + ".used", mem.getUsage().getUsed()));
+ }
+ }
+ return result;
+ }
+
+ private Metric<Long> newMemoryMetric(String name, long bytes) {
+ return new Metric<>(name, bytes / 1024);
+ }
+}
+```
+
+Please refer [Spring Boot Actuator: Production-ready Features](https://docs.spring.io/spring-boot/docs/current/reference/html/production-ready-features.html#production-ready) and [81. Actuator - Part IX. ‘How-to’ guides](https://docs.spring.io/spring-boot/docs/current/reference/html/howto.html#howto-actuator) for more information.
+
+## Charts
+
+1. **Response Codes** in requests/s
+
+ - 1xx
+ - 2xx
+ - 3xx
+ - 4xx
+ - 5xx
+ - others
+
+2. **Threads**
+
+ - daemon
+ - total
+
+3. **GC Time** in milliseconds and **GC Operations** in operations/s
+
+ - Copy
+ - MarkSweep
+ - ...
+
+4. **Heap Memory Usage** in KB
+
+ - used
+ - committed
+
+## Usage
+
+Edit the `python.d/springboot.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/springboot.conf
+```
+
+This module defines some common charts, and you can add custom charts by change the configurations.
+
+The configuration format is like:
+
+```yaml
+<id>:
+ name: '<name>'
+ url: '<metrics endpoint>' # ex. http://localhost:8080/metrics
+ user: '<username>' # optional
+ pass: '<password>' # optional
+ defaults:
+ [<chart-id>]: true|false
+ extras:
+ - id: '<chart-id>'
+ options:
+ title: '***'
+ units: '***'
+ family: '***'
+ context: 'springboot.***'
+ charttype: 'stacked' | 'area' | 'line'
+ lines:
+ - { dimension: 'myapp_ok', name: 'ok', algorithm: 'absolute', multiplier: 1, divisor: 1} # it shows "myapp.ok" metrics
+ - { dimension: 'myapp_ng', name: 'ng', algorithm: 'absolute', multiplier: 1, divisor: 1} # it shows "myapp.ng" metrics
+```
+
+By default, it creates `response_code`, `threads`, `gc_time`, `gc_ope` abd `heap` charts.
+You can disable the default charts by set `defaults.<chart-id>: false`.
+
+The dimension name of extras charts should replace `.` to `_`.
+
+Please check
+[springboot.conf](https://raw.githubusercontent.com/netdata/netdata/master/collectors/python.d.plugin/springboot/springboot.conf)
+for more examples.
+
+
diff --git a/collectors/python.d.plugin/springboot/springboot.chart.py b/collectors/python.d.plugin/springboot/springboot.chart.py
new file mode 100644
index 0000000..dbe11d6
--- /dev/null
+++ b/collectors/python.d.plugin/springboot/springboot.chart.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# Description: tomcat netdata python.d module
+# Author: Wing924
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import json
+
+from bases.FrameworkServices.UrlService import UrlService
+
+DEFAULT_ORDER = [
+ 'response_code',
+ 'threads',
+ 'gc_time',
+ 'gc_ope',
+ 'heap',
+]
+
+DEFAULT_CHARTS = {
+ 'response_code': {
+ 'options': [None, "Response Codes", "requests/s", "response", "springboot.response_code", "stacked"],
+ 'lines': [
+ ["resp_other", 'Other', 'incremental'],
+ ["resp_1xx", '1xx', 'incremental'],
+ ["resp_2xx", '2xx', 'incremental'],
+ ["resp_3xx", '3xx', 'incremental'],
+ ["resp_4xx", '4xx', 'incremental'],
+ ["resp_5xx", '5xx', 'incremental'],
+ ]
+ },
+ 'threads': {
+ 'options': [None, "Threads", "current threads", "threads", "springboot.threads", "area"],
+ 'lines': [
+ ["threads_daemon", 'daemon', 'absolute'],
+ ["threads", 'total', 'absolute'],
+ ]
+ },
+ 'gc_time': {
+ 'options': [None, "GC Time", "milliseconds", "garbage collection", "springboot.gc_time", "stacked"],
+ 'lines': [
+ ["gc_copy_time", 'Copy', 'incremental'],
+ ["gc_marksweepcompact_time", 'MarkSweepCompact', 'incremental'],
+ ["gc_parnew_time", 'ParNew', 'incremental'],
+ ["gc_concurrentmarksweep_time", 'ConcurrentMarkSweep', 'incremental'],
+ ["gc_ps_scavenge_time", 'PS Scavenge', 'incremental'],
+ ["gc_ps_marksweep_time", 'PS MarkSweep', 'incremental'],
+ ["gc_g1_young_generation_time", 'G1 Young Generation', 'incremental'],
+ ["gc_g1_old_generation_time", 'G1 Old Generation', 'incremental'],
+ ]
+ },
+ 'gc_ope': {
+ 'options': [None, "GC Operations", "operations/s", "garbage collection", "springboot.gc_ope", "stacked"],
+ 'lines': [
+ ["gc_copy_count", 'Copy', 'incremental'],
+ ["gc_marksweepcompact_count", 'MarkSweepCompact', 'incremental'],
+ ["gc_parnew_count", 'ParNew', 'incremental'],
+ ["gc_concurrentmarksweep_count", 'ConcurrentMarkSweep', 'incremental'],
+ ["gc_ps_scavenge_count", 'PS Scavenge', 'incremental'],
+ ["gc_ps_marksweep_count", 'PS MarkSweep', 'incremental'],
+ ["gc_g1_young_generation_count", 'G1 Young Generation', 'incremental'],
+ ["gc_g1_old_generation_count", 'G1 Old Generation', 'incremental'],
+ ]
+ },
+ 'heap': {
+ 'options': [None, "Heap Memory Usage", "KiB", "heap memory", "springboot.heap", "area"],
+ 'lines': [
+ ["heap_committed", 'committed', "absolute"],
+ ["heap_used", 'used', "absolute"],
+ ]
+ }
+}
+
+
+class ExtraChartError(ValueError):
+ pass
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.url = self.configuration.get('url', "http://localhost:8080/metrics")
+ self._setup_charts()
+
+ def _get_data(self):
+ """
+ Format data received from http request
+ :return: dict
+ """
+ raw_data = self._get_raw_data()
+ if not raw_data:
+ return None
+
+ try:
+ data = json.loads(raw_data)
+ except ValueError:
+ self.debug('%s is not a valid JSON page' % self.url)
+ return None
+
+ result = {
+ 'resp_1xx': 0,
+ 'resp_2xx': 0,
+ 'resp_3xx': 0,
+ 'resp_4xx': 0,
+ 'resp_5xx': 0,
+ 'resp_other': 0,
+ }
+
+ for key, value in data.iteritems():
+ if 'counter.status.' in key:
+ status_type = key[15:16] + 'xx'
+ if status_type[0] not in '12345':
+ status_type = 'other'
+ result['resp_' + status_type] += value
+ else:
+ result[key.replace('.', '_')] = value
+
+ return result or None
+
+ def _setup_charts(self):
+ self.order = []
+ self.definitions = {}
+ defaults = self.configuration.get('defaults', {})
+
+ for chart in DEFAULT_ORDER:
+ if defaults.get(chart, True):
+ self.order.append(chart)
+ self.definitions[chart] = DEFAULT_CHARTS[chart]
+
+ for extra in self.configuration.get('extras', []):
+ self._add_extra_chart(extra)
+ self.order.append(extra['id'])
+
+ def _add_extra_chart(self, chart):
+ chart_id = chart.get('id', None) or self.die('id is not defined in extra chart')
+ options = chart.get('options', None) or self.die('option is not defined in extra chart: %s' % chart_id)
+ lines = chart.get('lines', None) or self.die('lines is not defined in extra chart: %s' % chart_id)
+
+ title = options.get('title', None) or self.die('title is missing: %s' % chart_id)
+ units = options.get('units', None) or self.die('units is missing: %s' % chart_id)
+ family = options.get('family', title)
+ context = options.get('context', 'springboot.' + title)
+ charttype = options.get('charttype', 'line')
+
+ result = {
+ 'options': [None, title, units, family, context, charttype],
+ 'lines': [],
+ }
+
+ for line in lines:
+ dimension = line.get('dimension', None) or self.die('dimension is missing: %s' % chart_id)
+ name = line.get('name', dimension)
+ algorithm = line.get('algorithm', 'absolute')
+ multiplier = line.get('multiplier', 1)
+ divisor = line.get('divisor', 1)
+ result['lines'].append([dimension, name, algorithm, multiplier, divisor])
+
+ self.definitions[chart_id] = result
+
+ @staticmethod
+ def die(error_message):
+ raise ExtraChartError(error_message)
diff --git a/collectors/python.d.plugin/springboot/springboot.conf b/collectors/python.d.plugin/springboot/springboot.conf
new file mode 100644
index 0000000..0cb369c
--- /dev/null
+++ b/collectors/python.d.plugin/springboot/springboot.conf
@@ -0,0 +1,118 @@
+# netdata python.d.plugin configuration for springboot
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, this plugin also supports the following:
+#
+# url: 'http://127.0.0.1/metrics' # the URL of the spring boot actuator metrics
+#
+# if the URL is password protected, the following are supported:
+#
+# user: 'username'
+# pass: 'password'
+#
+# defaults:
+# [chart_id]: true | false # enables/disables default charts, defaults true.
+# extras: {} # defines extra charts to monitor, please see the example below
+# - id: [chart_id]
+# options: {}
+# lines: []
+#
+# If all defaults is disabled and no extra charts are defined, this module will disable itself, as it has no data to
+# collect.
+#
+# Configuration example
+# ---------------------
+# example:
+# name: 'example'
+# url: 'http://localhost:8080/metrics'
+# defaults:
+# response_code: true
+# threads: true
+# gc_time: true
+# gc_ope: true
+# heap: false
+# extras:
+# - id: 'heap'
+# options: { title: 'Heap Memory Usage', units: 'KB', family: 'heap memory', context: 'springboot.heap', charttype: 'stacked' }
+# lines:
+# - { dimension: 'mem_free', name: 'free'}
+# - { dimension: 'mempool_eden_used', name: 'eden', algorithm: 'absolute', multiplier: 1, divisor: 1}
+# - { dimension: 'mempool_survivor_used', name: 'survivor', algorithm: 'absolute', multiplier: 1, divisor: 1}
+# - { dimension: 'mempool_tenured_used', name: 'tenured', algorithm: 'absolute', multiplier: 1, divisor: 1}
+# - id: 'heap_eden'
+# options: { title: 'Eden Memory Usage', units: 'KB', family: 'heap memory', context: 'springboot.heap_eden', charttype: 'area' }
+# lines:
+# - { dimension: 'mempool_eden_used', name: 'used'}
+# - { dimension: 'mempool_eden_committed', name: 'committed'}
+# - id: 'heap_survivor'
+# options: { title: 'Survivor Memory Usage', units: 'KB', family: 'heap memory', context: 'springboot.heap_survivor', charttype: 'area' }
+# lines:
+# - { dimension: 'mempool_survivor_used', name: 'used'}
+# - { dimension: 'mempool_survivor_committed', name: 'committed'}
+# - id: 'heap_tenured'
+# options: { title: 'Tenured Memory Usage', units: 'KB', family: 'heap memory', context: 'springboot.heap_tenured', charttype: 'area' }
+# lines:
+# - { dimension: 'mempool_tenured_used', name: 'used'}
+# - { dimension: 'mempool_tenured_committed', name: 'committed'}
+
+
+local:
+ name: 'local'
+ url: 'http://localhost:8080/metrics'
+
+local_ip:
+ name: 'local'
+ url: 'http://127.0.0.1:8080/metrics'
diff --git a/collectors/python.d.plugin/squid/Makefile.inc b/collectors/python.d.plugin/squid/Makefile.inc
new file mode 100644
index 0000000..76ecff8
--- /dev/null
+++ b/collectors/python.d.plugin/squid/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += squid/squid.chart.py
+dist_pythonconfig_DATA += squid/squid.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += squid/README.md squid/Makefile.inc
+
diff --git a/collectors/python.d.plugin/squid/README.md b/collectors/python.d.plugin/squid/README.md
new file mode 100644
index 0000000..c29b69a
--- /dev/null
+++ b/collectors/python.d.plugin/squid/README.md
@@ -0,0 +1,58 @@
+<!--
+title: "Squid monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/squid/README.md
+sidebar_label: "Squid"
+-->
+
+# Squid monitoring with Netdata
+
+Monitors one or more squid instances depending on configuration.
+
+It produces following charts:
+
+1. **Client Bandwidth** in kilobits/s
+
+ - in
+ - out
+ - hits
+
+2. **Client Requests** in requests/s
+
+ - requests
+ - hits
+ - errors
+
+3. **Server Bandwidth** in kilobits/s
+
+ - in
+ - out
+
+4. **Server Requests** in requests/s
+
+ - requests
+ - errors
+
+## Configuration
+
+Edit the `python.d/squid.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/squid.conf
+```
+
+```yaml
+priority : 50000
+
+local:
+ request : 'cache_object://localhost:3128/counters'
+ host : 'localhost'
+ port : 3128
+```
+
+Without any configuration module will try to autodetect where squid presents its `counters` data
+
+---
+
+
diff --git a/collectors/python.d.plugin/squid/squid.chart.py b/collectors/python.d.plugin/squid/squid.chart.py
new file mode 100644
index 0000000..bcae2d8
--- /dev/null
+++ b/collectors/python.d.plugin/squid/squid.chart.py
@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# Description: squid netdata python.d module
+# Author: Pawel Krupa (paulfantom)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from bases.FrameworkServices.SocketService import SocketService
+
+ORDER = [
+ 'clients_net',
+ 'clients_requests',
+ 'servers_net',
+ 'servers_requests',
+]
+
+CHARTS = {
+ 'clients_net': {
+ 'options': [None, 'Squid Client Bandwidth', 'kilobits/s', 'clients', 'squid.clients_net', 'area'],
+ 'lines': [
+ ['client_http_kbytes_in', 'in', 'incremental', 8, 1],
+ ['client_http_kbytes_out', 'out', 'incremental', -8, 1],
+ ['client_http_hit_kbytes_out', 'hits', 'incremental', -8, 1]
+ ]
+ },
+ 'clients_requests': {
+ 'options': [None, 'Squid Client Requests', 'requests/s', 'clients', 'squid.clients_requests', 'line'],
+ 'lines': [
+ ['client_http_requests', 'requests', 'incremental'],
+ ['client_http_hits', 'hits', 'incremental'],
+ ['client_http_errors', 'errors', 'incremental', -1, 1]
+ ]
+ },
+ 'servers_net': {
+ 'options': [None, 'Squid Server Bandwidth', 'kilobits/s', 'servers', 'squid.servers_net', 'area'],
+ 'lines': [
+ ['server_all_kbytes_in', 'in', 'incremental', 8, 1],
+ ['server_all_kbytes_out', 'out', 'incremental', -8, 1]
+ ]
+ },
+ 'servers_requests': {
+ 'options': [None, 'Squid Server Requests', 'requests/s', 'servers', 'squid.servers_requests', 'line'],
+ 'lines': [
+ ['server_all_requests', 'requests', 'incremental'],
+ ['server_all_errors', 'errors', 'incremental', -1, 1]
+ ]
+ }
+}
+
+
+class Service(SocketService):
+ def __init__(self, configuration=None, name=None):
+ SocketService.__init__(self, configuration=configuration, name=name)
+ self._keep_alive = True
+ self.request = ''
+ self.host = 'localhost'
+ self.port = 3128
+ self.order = ORDER
+ self.definitions = CHARTS
+
+ def _get_data(self):
+ """
+ Get data via http request
+ :return: dict
+ """
+ response = self._get_raw_data()
+
+ data = dict()
+ try:
+ raw = ''
+ for tmp in response.split('\r\n'):
+ if tmp.startswith('sample_time'):
+ raw = tmp
+ break
+
+ if raw.startswith('<'):
+ self.error('invalid data received')
+ return None
+
+ for row in raw.split('\n'):
+ if row.startswith(('client', 'server.all')):
+ tmp = row.split('=')
+ data[tmp[0].replace('.', '_').strip(' ')] = int(tmp[1])
+
+ except (ValueError, AttributeError, TypeError):
+ self.error('invalid data received')
+ return None
+
+ if not data:
+ self.error('no data received')
+ return None
+ return data
+
+ def _check_raw_data(self, data):
+ header = data[:1024].lower()
+
+ if 'connection: keep-alive' in header:
+ self._keep_alive = True
+ else:
+ self._keep_alive = False
+
+ if data[-7:] == '\r\n0\r\n\r\n' and 'transfer-encoding: chunked' in header: # HTTP/1.1 response
+ self.debug('received full response from squid')
+ return True
+
+ self.debug('waiting more data from squid')
+ return False
+
+ def check(self):
+ """
+ Parse essential configuration, autodetect squid configuration (if needed), and check if data is available
+ :return: boolean
+ """
+ self._parse_config()
+ # format request
+ req = self.request.decode()
+ if not req.startswith('GET'):
+ req = 'GET ' + req
+ if not req.endswith(' HTTP/1.1\r\n\r\n'):
+ req += ' HTTP/1.1\r\n\r\n'
+ self.request = req.encode()
+ if self._get_data() is not None:
+ return True
+ else:
+ return False
diff --git a/collectors/python.d.plugin/squid/squid.conf b/collectors/python.d.plugin/squid/squid.conf
new file mode 100644
index 0000000..b90a52c
--- /dev/null
+++ b/collectors/python.d.plugin/squid/squid.conf
@@ -0,0 +1,167 @@
+# netdata python.d.plugin configuration for squid
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, squid also supports the following:
+#
+# host : 'IP or HOSTNAME' # the host to connect to
+# port : PORT # the port to connect to
+# request: 'URL' # the URL to request from squid
+#
+
+# ----------------------------------------------------------------------
+# SQUID CONFIGURATION
+#
+# See:
+# http://wiki.squid-cache.org/Features/CacheManager
+#
+# In short, add to your squid configuration these:
+#
+# http_access allow localhost manager
+# http_access deny manager
+#
+# To remotely monitor a squid:
+#
+# acl managerAdmin src 192.0.2.1
+# http_access allow localhost manager
+# http_access allow managerAdmin manager
+# http_access deny manager
+#
+
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+tcp3128old:
+ name : 'local'
+ host : 'localhost'
+ port : 3128
+ request : 'cache_object://localhost:3128/counters'
+
+tcp8080old:
+ name : 'local'
+ host : 'localhost'
+ port : 8080
+ request : 'cache_object://localhost:3128/counters'
+
+tcp3128new:
+ name : 'local'
+ host : 'localhost'
+ port : 3128
+ request : '/squid-internal-mgr/counters'
+
+tcp8080new:
+ name : 'local'
+ host : 'localhost'
+ port : 8080
+ request : '/squid-internal-mgr/counters'
+
+# IPv4
+
+tcp3128oldipv4:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 3128
+ request : 'cache_object://127.0.0.1:3128/counters'
+
+tcp8080oldipv4:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 8080
+ request : 'cache_object://127.0.0.1:3128/counters'
+
+tcp3128newipv4:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 3128
+ request : '/squid-internal-mgr/counters'
+
+tcp8080newipv4:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 8080
+ request : '/squid-internal-mgr/counters'
+
+# IPv6
+
+tcp3128oldipv6:
+ name : 'local'
+ host : '::1'
+ port : 3128
+ request : 'cache_object://[::1]:3128/counters'
+
+tcp8080oldipv6:
+ name : 'local'
+ host : '::1'
+ port : 8080
+ request : 'cache_object://[::1]:3128/counters'
+
+tcp3128newipv6:
+ name : 'local'
+ host : '::1'
+ port : 3128
+ request : '/squid-internal-mgr/counters'
+
+tcp8080newipv6:
+ name : 'local'
+ host : '::1'
+ port : 8080
+ request : '/squid-internal-mgr/counters'
+
diff --git a/collectors/python.d.plugin/tomcat/Makefile.inc b/collectors/python.d.plugin/tomcat/Makefile.inc
new file mode 100644
index 0000000..940a783
--- /dev/null
+++ b/collectors/python.d.plugin/tomcat/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += tomcat/tomcat.chart.py
+dist_pythonconfig_DATA += tomcat/tomcat.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += tomcat/README.md tomcat/Makefile.inc
+
diff --git a/collectors/python.d.plugin/tomcat/README.md b/collectors/python.d.plugin/tomcat/README.md
new file mode 100644
index 0000000..b7525b8
--- /dev/null
+++ b/collectors/python.d.plugin/tomcat/README.md
@@ -0,0 +1,53 @@
+<!--
+title: "Apache Tomcat monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/tomcat/README.md
+sidebar_label: "Tomcat"
+-->
+
+# Apache Tomcat monitoring with Netdata
+
+Presents memory utilization of tomcat containers.
+
+Charts:
+
+1. **Requests** per second
+
+ - accesses
+
+2. **Volume** in KB/s
+
+ - volume
+
+3. **Threads**
+
+ - current
+ - busy
+
+4. **JVM Free Memory** in MB
+
+ - jvm
+
+## Configuration
+
+Edit the `python.d/tomcat.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/tomcat.conf
+```
+
+```yaml
+localhost:
+ name : 'local'
+ url : 'http://127.0.0.1:8080/manager/status?XML=true'
+ user : 'tomcat_username'
+ pass : 'secret_tomcat_password'
+```
+
+Without configuration, module attempts to connect to `http://localhost:8080/manager/status?XML=true`, without any credentials.
+So it will probably fail.
+
+---
+
+
diff --git a/collectors/python.d.plugin/tomcat/tomcat.chart.py b/collectors/python.d.plugin/tomcat/tomcat.chart.py
new file mode 100644
index 0000000..90315f8
--- /dev/null
+++ b/collectors/python.d.plugin/tomcat/tomcat.chart.py
@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+# Description: tomcat netdata python.d module
+# Author: Pawel Krupa (paulfantom)
+# Author: Wei He (Wing924)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import re
+import xml.etree.ElementTree as ET
+
+from bases.FrameworkServices.UrlService import UrlService
+
+MiB = 1 << 20
+
+# Regex fix for Tomcat single quote XML attributes
+# affecting Tomcat < 8.5.24 & 9.0.2 running with Java > 9
+# cf. https://bz.apache.org/bugzilla/show_bug.cgi?id=61603
+single_quote_regex = re.compile(r"='([^']+)'([^']+)''")
+
+ORDER = [
+ 'accesses',
+ 'bandwidth',
+ 'processing_time',
+ 'threads',
+ 'jvm',
+ 'jvm_eden',
+ 'jvm_survivor',
+ 'jvm_tenured',
+]
+
+CHARTS = {
+ 'accesses': {
+ 'options': [None, 'Requests', 'requests/s', 'statistics', 'tomcat.accesses', 'area'],
+ 'lines': [
+ ['requestCount', 'accesses', 'incremental'],
+ ['errorCount', 'errors', 'incremental'],
+ ]
+ },
+ 'bandwidth': {
+ 'options': [None, 'Bandwidth', 'KiB/s', 'statistics', 'tomcat.bandwidth', 'area'],
+ 'lines': [
+ ['bytesSent', 'sent', 'incremental', 1, 1024],
+ ['bytesReceived', 'received', 'incremental', 1, 1024],
+ ]
+ },
+ 'processing_time': {
+ 'options': [None, 'processing time', 'seconds', 'statistics', 'tomcat.processing_time', 'area'],
+ 'lines': [
+ ['processingTime', 'processing time', 'incremental', 1, 1000]
+ ]
+ },
+ 'threads': {
+ 'options': [None, 'Threads', 'current threads', 'statistics', 'tomcat.threads', 'area'],
+ 'lines': [
+ ['currentThreadCount', 'current', 'absolute'],
+ ['currentThreadsBusy', 'busy', 'absolute']
+ ]
+ },
+ 'jvm': {
+ 'options': [None, 'JVM Memory Pool Usage', 'MiB', 'memory', 'tomcat.jvm', 'stacked'],
+ 'lines': [
+ ['free', 'free', 'absolute', 1, MiB],
+ ['eden_used', 'eden', 'absolute', 1, MiB],
+ ['survivor_used', 'survivor', 'absolute', 1, MiB],
+ ['tenured_used', 'tenured', 'absolute', 1, MiB],
+ ['code_cache_used', 'code cache', 'absolute', 1, MiB],
+ ['compressed_used', 'compressed', 'absolute', 1, MiB],
+ ['metaspace_used', 'metaspace', 'absolute', 1, MiB],
+ ]
+ },
+ 'jvm_eden': {
+ 'options': [None, 'Eden Memory Usage', 'MiB', 'memory', 'tomcat.jvm_eden', 'area'],
+ 'lines': [
+ ['eden_used', 'used', 'absolute', 1, MiB],
+ ['eden_committed', 'committed', 'absolute', 1, MiB],
+ ['eden_max', 'max', 'absolute', 1, MiB]
+ ]
+ },
+ 'jvm_survivor': {
+ 'options': [None, 'Survivor Memory Usage', 'MiB', 'memory', 'tomcat.jvm_survivor', 'area'],
+ 'lines': [
+ ['survivor_used', 'used', 'absolute', 1, MiB],
+ ['survivor_committed', 'committed', 'absolute', 1, MiB],
+ ['survivor_max', 'max', 'absolute', 1, MiB],
+ ]
+ },
+ 'jvm_tenured': {
+ 'options': [None, 'Tenured Memory Usage', 'MiB', 'memory', 'tomcat.jvm_tenured', 'area'],
+ 'lines': [
+ ['tenured_used', 'used', 'absolute', 1, MiB],
+ ['tenured_committed', 'committed', 'absolute', 1, MiB],
+ ['tenured_max', 'max', 'absolute', 1, MiB]
+ ]
+ }
+}
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.url = self.configuration.get('url', 'http://127.0.0.1:8080/manager/status?XML=true')
+ self.connector_name = self.configuration.get('connector_name', None)
+ self.parse = self.xml_parse
+
+ def xml_parse(self, data):
+ try:
+ return ET.fromstring(data)
+ except ET.ParseError:
+ self.debug('%s is not a valid XML page. Please add "?XML=true" to tomcat status page.' % self.url)
+ return None
+
+ def xml_single_quote_fix_parse(self, data):
+ data = single_quote_regex.sub(r"='\g<1>\g<2>'", data)
+ return self.xml_parse(data)
+
+ def check(self):
+ self._manager = self._build_manager()
+
+ raw_data = self._get_raw_data()
+ if not raw_data:
+ return False
+
+ if single_quote_regex.search(raw_data):
+ self.warning('Tomcat status page is returning invalid single quote XML, please consider upgrading '
+ 'your Tomcat installation. See https://bz.apache.org/bugzilla/show_bug.cgi?id=61603')
+ self.parse = self.xml_single_quote_fix_parse
+
+ return self.parse(raw_data) is not None
+
+ def _get_data(self):
+ """
+ Format data received from http request
+ :return: dict
+ """
+ data = None
+ raw_data = self._get_raw_data()
+ if raw_data:
+ xml = self.parse(raw_data)
+ if xml is None:
+ return None
+
+ data = {}
+
+ jvm = xml.find('jvm')
+
+ connector = None
+ if self.connector_name:
+ for conn in xml.findall('connector'):
+ if self.connector_name in conn.get('name'):
+ connector = conn
+ break
+ else:
+ connector = xml.find('connector')
+
+ memory = jvm.find('memory')
+ data['free'] = memory.get('free')
+ data['total'] = memory.get('total')
+
+ for pool in jvm.findall('memorypool'):
+ name = pool.get('name')
+ if 'Eden Space' in name:
+ data['eden_used'] = pool.get('usageUsed')
+ data['eden_committed'] = pool.get('usageCommitted')
+ data['eden_max'] = pool.get('usageMax')
+ elif 'Survivor Space' in name:
+ data['survivor_used'] = pool.get('usageUsed')
+ data['survivor_committed'] = pool.get('usageCommitted')
+ data['survivor_max'] = pool.get('usageMax')
+ elif 'Tenured Gen' in name or 'Old Gen' in name:
+ data['tenured_used'] = pool.get('usageUsed')
+ data['tenured_committed'] = pool.get('usageCommitted')
+ data['tenured_max'] = pool.get('usageMax')
+ elif name == 'Code Cache':
+ data['code_cache_used'] = pool.get('usageUsed')
+ data['code_cache_committed'] = pool.get('usageCommitted')
+ data['code_cache_max'] = pool.get('usageMax')
+ elif name == 'Compressed':
+ data['compressed_used'] = pool.get('usageUsed')
+ data['compressed_committed'] = pool.get('usageCommitted')
+ data['compressed_max'] = pool.get('usageMax')
+ elif name == 'Metaspace':
+ data['metaspace_used'] = pool.get('usageUsed')
+ data['metaspace_committed'] = pool.get('usageCommitted')
+ data['metaspace_max'] = pool.get('usageMax')
+
+ if connector is not None:
+ thread_info = connector.find('threadInfo')
+ data['currentThreadsBusy'] = thread_info.get('currentThreadsBusy')
+ data['currentThreadCount'] = thread_info.get('currentThreadCount')
+
+ request_info = connector.find('requestInfo')
+ data['processingTime'] = request_info.get('processingTime')
+ data['requestCount'] = request_info.get('requestCount')
+ data['errorCount'] = request_info.get('errorCount')
+ data['bytesReceived'] = request_info.get('bytesReceived')
+ data['bytesSent'] = request_info.get('bytesSent')
+
+ return data or None
diff --git a/collectors/python.d.plugin/tomcat/tomcat.conf b/collectors/python.d.plugin/tomcat/tomcat.conf
new file mode 100644
index 0000000..009591b
--- /dev/null
+++ b/collectors/python.d.plugin/tomcat/tomcat.conf
@@ -0,0 +1,89 @@
+# netdata python.d.plugin configuration for tomcat
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, tomcat also supports the following:
+#
+# url: 'URL' # the URL to fetch nginx's status stats
+#
+# if the URL is password protected, the following are supported:
+#
+# user: 'username'
+# pass: 'password'
+#
+# if you have multiple connectors, the following are supported:
+#
+# connector_name: 'ajp-bio-8009' # default is null, which use first connector in status XML
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+ name : 'local'
+ url : 'http://localhost:8080/manager/status?XML=true'
+
+localipv4:
+ name : 'local'
+ url : 'http://127.0.0.1:8080/manager/status?XML=true'
+
+localipv6:
+ name : 'local'
+ url : 'http://[::1]:8080/manager/status?XML=true'
diff --git a/collectors/python.d.plugin/tor/Makefile.inc b/collectors/python.d.plugin/tor/Makefile.inc
new file mode 100644
index 0000000..5a45f9b
--- /dev/null
+++ b/collectors/python.d.plugin/tor/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += tor/tor.chart.py
+dist_pythonconfig_DATA += tor/tor.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += tor/README.md tor/Makefile.inc
+
diff --git a/collectors/python.d.plugin/tor/README.md b/collectors/python.d.plugin/tor/README.md
new file mode 100644
index 0000000..b57d77c
--- /dev/null
+++ b/collectors/python.d.plugin/tor/README.md
@@ -0,0 +1,66 @@
+<!--
+title: "Tor monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/tor/README.md
+sidebar_label: "Tor"
+-->
+
+# Tor monitoring with Netdata
+
+Connects to the Tor control port to collect traffic statistics.
+
+## Requirements
+
+- `tor` program
+- `stem` python package
+
+It produces only one chart:
+
+1. **Traffic**
+
+ - read
+ - write
+
+## Configuration
+
+Edit the `python.d/tor.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/tor.conf
+```
+
+Needs only `control_port`.
+
+Here is an example for local server:
+
+```yaml
+update_every : 1
+priority : 60000
+
+local_tcp:
+ name: 'local'
+ control_port: 9051
+ password: <password> # if required
+
+local_socket:
+ name: 'local'
+ control_port: '/var/run/tor/control'
+ password: <password> # if required
+```
+
+### prerequisite
+
+Add to `/etc/tor/torrc`:
+
+```
+ControlPort 9051
+```
+
+For more options please read the manual.
+
+Without configuration, module attempts to connect to `127.0.0.1:9051`.
+
+---
+
+
diff --git a/collectors/python.d.plugin/tor/tor.chart.py b/collectors/python.d.plugin/tor/tor.chart.py
new file mode 100644
index 0000000..8dc021a
--- /dev/null
+++ b/collectors/python.d.plugin/tor/tor.chart.py
@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+# Description: adaptec_raid netdata python.d module
+# Author: Federico Ceratto <federico.ceratto@gmail.com>
+# Author: Ilya Mashchenko (ilyam8)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+try:
+ import stem
+ import stem.connection
+ import stem.control
+
+ STEM_AVAILABLE = True
+except ImportError:
+ STEM_AVAILABLE = False
+
+DEF_PORT = 'default'
+
+ORDER = [
+ 'traffic',
+]
+
+CHARTS = {
+ 'traffic': {
+ 'options': [None, 'Tor Traffic', 'KiB/s', 'traffic', 'tor.traffic', 'area'],
+ 'lines': [
+ ['read', 'read', 'incremental', 1, 1024],
+ ['write', 'write', 'incremental', 1, -1024],
+ ]
+ }
+}
+
+
+class Service(SimpleService):
+ """Provide netdata service for Tor"""
+
+ def __init__(self, configuration=None, name=None):
+ super(Service, self).__init__(configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.port = self.configuration.get('control_port', DEF_PORT)
+ self.password = self.configuration.get('password')
+ self.use_socket = isinstance(self.port, str) and self.port != DEF_PORT and not self.port.isdigit()
+ self.conn = None
+ self.alive = False
+
+ def check(self):
+ if not STEM_AVAILABLE:
+ self.error('the stem library is missing')
+ return False
+
+ return self.connect()
+
+ def get_data(self):
+ if not self.alive and not self.reconnect():
+ return None
+
+ data = dict()
+
+ try:
+ data['read'] = self.conn.get_info('traffic/read')
+ data['write'] = self.conn.get_info('traffic/written')
+ except stem.ControllerError as error:
+ self.debug(error)
+ self.alive = False
+
+ return data or None
+
+ def authenticate(self):
+ try:
+ self.conn.authenticate(password=self.password)
+ except stem.connection.AuthenticationFailure as error:
+ self.error('authentication error: {0}'.format(error))
+ return False
+ return True
+
+ def connect_via_port(self):
+ try:
+ self.conn = stem.control.Controller.from_port(port=self.port)
+ except (stem.SocketError, ValueError) as error:
+ self.error(error)
+
+ def connect_via_socket(self):
+ try:
+ self.conn = stem.control.Controller.from_socket_file(path=self.port)
+ except (stem.SocketError, ValueError) as error:
+ self.error(error)
+
+ def connect(self):
+ if self.conn:
+ self.conn.close()
+ self.conn = None
+
+ if self.use_socket:
+ self.connect_via_socket()
+ else:
+ self.connect_via_port()
+
+ if self.conn and self.authenticate():
+ self.alive = True
+
+ return self.alive
+
+ def reconnect(self):
+ return self.connect()
diff --git a/collectors/python.d.plugin/tor/tor.conf b/collectors/python.d.plugin/tor/tor.conf
new file mode 100644
index 0000000..bf09b21
--- /dev/null
+++ b/collectors/python.d.plugin/tor/tor.conf
@@ -0,0 +1,79 @@
+# netdata python.d.plugin configuration for tor
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, tor plugin also supports the following:
+#
+# control_port: 'port' # tor control port
+# password: 'password' # tor control password
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+#
+# local_tcp:
+# name: 'local'
+# control_port: 9051
+# password: <password>
+#
+# local_socket:
+# name: 'local'
+# control_port: '/var/run/tor/control'
+# password: <password>
diff --git a/collectors/python.d.plugin/traefik/Makefile.inc b/collectors/python.d.plugin/traefik/Makefile.inc
new file mode 100644
index 0000000..926d56d
--- /dev/null
+++ b/collectors/python.d.plugin/traefik/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += traefik/traefik.chart.py
+dist_pythonconfig_DATA += traefik/traefik.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += traefik/README.md traefik/Makefile.inc
+
diff --git a/collectors/python.d.plugin/traefik/README.md b/collectors/python.d.plugin/traefik/README.md
new file mode 100644
index 0000000..251cdf2
--- /dev/null
+++ b/collectors/python.d.plugin/traefik/README.md
@@ -0,0 +1,74 @@
+<!--
+title: "Traefik monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/traefik/README.md
+sidebar_label: "Traefik"
+-->
+
+# Traefik monitoring with Netdata
+
+Uses the `health` API to provide statistics.
+
+It produces:
+
+1. **Responses** by statuses
+
+ - success (1xx, 2xx, 304)
+ - error (5xx)
+ - redirect (3xx except 304)
+ - bad (4xx)
+ - other (all other responses)
+
+2. **Responses** by codes
+
+ - 2xx (successful)
+ - 5xx (internal server errors)
+ - 3xx (redirect)
+ - 4xx (bad)
+ - 1xx (informational)
+ - other (non-standart responses)
+
+3. **Detailed Response Codes** requests/s (number of responses for each response code family individually)
+
+4. **Requests**/s
+
+ - request statistics
+
+5. **Total response time**
+
+ - sum of all response time
+
+6. **Average response time**
+
+7. **Average response time per iteration**
+
+8. **Uptime**
+
+ - Traefik server uptime
+
+## Configuration
+
+Edit the `python.d/traefik.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/traefik.conf
+```
+
+Needs only `url` to server's `health`
+
+Here is an example for local server:
+
+```yaml
+update_every : 1
+priority : 60000
+
+local:
+ url : 'http://localhost:8080/health'
+```
+
+Without configuration, module attempts to connect to `http://localhost:8080/health`.
+
+---
+
+
diff --git a/collectors/python.d.plugin/traefik/traefik.chart.py b/collectors/python.d.plugin/traefik/traefik.chart.py
new file mode 100644
index 0000000..5a49846
--- /dev/null
+++ b/collectors/python.d.plugin/traefik/traefik.chart.py
@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+# Description: traefik netdata python.d module
+# Author: Alexandre Menezes (@ale_menezes)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from collections import defaultdict
+from json import loads
+
+from bases.FrameworkServices.UrlService import UrlService
+
+ORDER = [
+ 'response_statuses',
+ 'response_codes',
+ 'detailed_response_codes',
+ 'requests',
+ 'total_response_time',
+ 'average_response_time',
+ 'average_response_time_per_iteration',
+ 'uptime'
+]
+
+CHARTS = {
+ 'response_statuses': {
+ 'options': [None, 'Response statuses', 'requests/s', 'responses', 'traefik.response_statuses', 'stacked'],
+ 'lines': [
+ ['successful_requests', 'success', 'incremental'],
+ ['server_errors', 'error', 'incremental'],
+ ['redirects', 'redirect', 'incremental'],
+ ['bad_requests', 'bad', 'incremental'],
+ ['other_requests', 'other', 'incremental']
+ ]
+ },
+ 'response_codes': {
+ 'options': [None, 'Responses by codes', 'requests/s', 'responses', 'traefik.response_codes', 'stacked'],
+ 'lines': [
+ ['2xx', None, 'incremental'],
+ ['5xx', None, 'incremental'],
+ ['3xx', None, 'incremental'],
+ ['4xx', None, 'incremental'],
+ ['1xx', None, 'incremental'],
+ ['other', None, 'incremental']
+ ]
+ },
+ 'detailed_response_codes': {
+ 'options': [None, 'Detailed response codes', 'requests/s', 'responses', 'traefik.detailed_response_codes',
+ 'stacked'],
+ 'lines': []
+ },
+ 'requests': {
+ 'options': [None, 'Requests', 'requests/s', 'requests', 'traefik.requests', 'line'],
+ 'lines': [
+ ['total_count', 'requests', 'incremental']
+ ]
+ },
+ 'total_response_time': {
+ 'options': [None, 'Total response time', 'seconds', 'timings', 'traefik.total_response_time', 'line'],
+ 'lines': [
+ ['total_response_time_sec', 'response', 'absolute', 1, 10000]
+ ]
+ },
+ 'average_response_time': {
+ 'options': [None, 'Average response time', 'milliseconds', 'timings', 'traefik.average_response_time', 'line'],
+ 'lines': [
+ ['average_response_time_sec', 'response', 'absolute', 1, 1000]
+ ]
+ },
+ 'average_response_time_per_iteration': {
+ 'options': [None, 'Average response time per iteration', 'milliseconds', 'timings',
+ 'traefik.average_response_time_per_iteration', 'line'],
+ 'lines': [
+ ['average_response_time_per_iteration_sec', 'response', 'incremental', 1, 10000]
+ ]
+ },
+ 'uptime': {
+ 'options': [None, 'Uptime', 'seconds', 'uptime', 'traefik.uptime', 'line'],
+ 'lines': [
+ ['uptime_sec', 'uptime', 'absolute']
+ ]
+ }
+}
+
+HEALTH_STATS = [
+ 'uptime_sec',
+ 'average_response_time_sec',
+ 'total_response_time_sec',
+ 'total_count',
+ 'total_status_code_count'
+]
+
+
+class Service(UrlService):
+ def __init__(self, configuration=None, name=None):
+ UrlService.__init__(self, configuration=configuration, name=name)
+ self.url = self.configuration.get('url', 'http://localhost:8080/health')
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.last_total_response_time = 0
+ self.last_total_count = 0
+ self.data = {
+ 'successful_requests': 0,
+ 'redirects': 0,
+ 'bad_requests': 0,
+ 'server_errors': 0,
+ 'other_requests': 0,
+ '1xx': 0,
+ '2xx': 0,
+ '3xx': 0,
+ '4xx': 0,
+ '5xx': 0,
+ 'other': 0,
+ 'average_response_time_per_iteration_sec': 0,
+ }
+
+ def _get_data(self):
+ data = self._get_raw_data()
+
+ if not data:
+ return None
+
+ data = loads(data)
+
+ self.get_data_per_code_status(raw_data=data)
+
+ self.get_data_per_code_family(raw_data=data)
+
+ self.get_data_per_code(raw_data=data)
+
+ self.data.update(fetch_data_(raw_data=data, metrics=HEALTH_STATS))
+
+ self.data['average_response_time_sec'] *= 1000000
+ self.data['total_response_time_sec'] *= 10000
+ if data['total_count'] != self.last_total_count:
+ self.data['average_response_time_per_iteration_sec'] = \
+ (data['total_response_time_sec'] - self.last_total_response_time) * \
+ 1000000 / (data['total_count'] - self.last_total_count)
+ else:
+ self.data['average_response_time_per_iteration_sec'] = 0
+ self.last_total_response_time = data['total_response_time_sec']
+ self.last_total_count = data['total_count']
+
+ return self.data or None
+
+ def get_data_per_code_status(self, raw_data):
+ data = defaultdict(int)
+ for code, value in raw_data['total_status_code_count'].items():
+ code_prefix = code[0]
+ if code_prefix == '1' or code_prefix == '2' or code == '304':
+ data['successful_requests'] += value
+ elif code_prefix == '3':
+ data['redirects'] += value
+ elif code_prefix == '4':
+ data['bad_requests'] += value
+ elif code_prefix == '5':
+ data['server_errors'] += value
+ else:
+ data['other_requests'] += value
+ self.data.update(data)
+
+ def get_data_per_code_family(self, raw_data):
+ data = defaultdict(int)
+ for code, value in raw_data['total_status_code_count'].items():
+ code_prefix = code[0]
+ if code_prefix == '1':
+ data['1xx'] += value
+ elif code_prefix == '2':
+ data['2xx'] += value
+ elif code_prefix == '3':
+ data['3xx'] += value
+ elif code_prefix == '4':
+ data['4xx'] += value
+ elif code_prefix == '5':
+ data['5xx'] += value
+ else:
+ data['other'] += value
+ self.data.update(data)
+
+ def get_data_per_code(self, raw_data):
+ for code, value in raw_data['total_status_code_count'].items():
+ if self.charts:
+ if code not in self.data:
+ self.charts['detailed_response_codes'].add_dimension([code, code, 'incremental'])
+ self.data[code] = value
+
+
+def fetch_data_(raw_data, metrics):
+ data = dict()
+
+ for metric in metrics:
+ value = raw_data
+ metrics_list = metric.split('.')
+ try:
+ for m in metrics_list:
+ value = value[m]
+ except KeyError:
+ continue
+ data['_'.join(metrics_list)] = value
+
+ return data
diff --git a/collectors/python.d.plugin/traefik/traefik.conf b/collectors/python.d.plugin/traefik/traefik.conf
new file mode 100644
index 0000000..e3f182d
--- /dev/null
+++ b/collectors/python.d.plugin/traefik/traefik.conf
@@ -0,0 +1,77 @@
+# netdata python.d.plugin configuration for traefik health data API
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, traefik plugin also supports the following:
+#
+# url: '<scheme>://<host>:<port>/<health_page_api>'
+# # http://localhost:8080/health
+#
+# if the URL is password protected, the following are supported:
+#
+# user: 'username'
+# pass: 'password'
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+#
+local:
+ url: 'http://localhost:8080/health'
diff --git a/collectors/python.d.plugin/uwsgi/Makefile.inc b/collectors/python.d.plugin/uwsgi/Makefile.inc
new file mode 100644
index 0000000..75d96de
--- /dev/null
+++ b/collectors/python.d.plugin/uwsgi/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += uwsgi/uwsgi.chart.py
+dist_pythonconfig_DATA += uwsgi/uwsgi.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += uwsgi/README.md uwsgi/Makefile.inc
+
diff --git a/collectors/python.d.plugin/uwsgi/README.md b/collectors/python.d.plugin/uwsgi/README.md
new file mode 100644
index 0000000..58db1a4
--- /dev/null
+++ b/collectors/python.d.plugin/uwsgi/README.md
@@ -0,0 +1,52 @@
+<!--
+title: "uWSGI monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/uwsgi/README.md
+sidebar_label: "uWSGI"
+-->
+
+# uWSGI monitoring with Netdata
+
+Monitors performance metrics exposed by [`Stats Server`](https://uwsgi-docs.readthedocs.io/en/latest/StatsServer.html).
+
+
+Following charts are drawn:
+
+1. **Requests**
+
+ - requests per second
+ - transmitted data
+ - average request time
+
+2. **Memory**
+
+ - rss
+ - vsz
+
+3. **Exceptions**
+4. **Harakiris**
+5. **Respawns**
+
+## Configuration
+
+Edit the `python.d/uwsgi.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/uwsgi.conf
+```
+
+```yaml
+socket:
+ name : 'local'
+ socket : '/tmp/stats.socket'
+
+localhost:
+ name : 'local'
+ host : 'localhost'
+ port : 1717
+```
+
+When no configuration file is found, module tries to connect to TCP/IP socket: `localhost:1717`.
+
+
diff --git a/collectors/python.d.plugin/uwsgi/uwsgi.chart.py b/collectors/python.d.plugin/uwsgi/uwsgi.chart.py
new file mode 100644
index 0000000..e4d9000
--- /dev/null
+++ b/collectors/python.d.plugin/uwsgi/uwsgi.chart.py
@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+# Description: uwsgi netdata python.d module
+# Author: Robbert Segeren (robbert-ef)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import json
+from copy import deepcopy
+
+from bases.FrameworkServices.SocketService import SocketService
+
+ORDER = [
+ 'requests',
+ 'tx',
+ 'avg_rt',
+ 'memory_rss',
+ 'memory_vsz',
+ 'exceptions',
+ 'harakiri',
+ 'respawn',
+]
+
+DYNAMIC_CHARTS = [
+ 'requests',
+ 'tx',
+ 'avg_rt',
+ 'memory_rss',
+ 'memory_vsz',
+]
+
+# NOTE: lines are created dynamically in `check()` method
+CHARTS = {
+ 'requests': {
+ 'options': [None, 'Requests', 'requests/s', 'requests', 'uwsgi.requests', 'stacked'],
+ 'lines': [
+ ['requests', 'requests', 'incremental']
+ ]
+ },
+ 'tx': {
+ 'options': [None, 'Transmitted data', 'KiB/s', 'requests', 'uwsgi.tx', 'stacked'],
+ 'lines': [
+ ['tx', 'tx', 'incremental']
+ ]
+ },
+ 'avg_rt': {
+ 'options': [None, 'Average request time', 'milliseconds', 'requests', 'uwsgi.avg_rt', 'line'],
+ 'lines': [
+ ['avg_rt', 'avg_rt', 'absolute']
+ ]
+ },
+ 'memory_rss': {
+ 'options': [None, 'RSS (Resident Set Size)', 'MiB', 'memory', 'uwsgi.memory_rss', 'stacked'],
+ 'lines': [
+ ['memory_rss', 'memory_rss', 'absolute', 1, 1 << 20]
+ ]
+ },
+ 'memory_vsz': {
+ 'options': [None, 'VSZ (Virtual Memory Size)', 'MiB', 'memory', 'uwsgi.memory_vsz', 'stacked'],
+ 'lines': [
+ ['memory_vsz', 'memory_vsz', 'absolute', 1, 1 << 20]
+ ]
+ },
+ 'exceptions': {
+ 'options': [None, 'Exceptions', 'exceptions', 'exceptions', 'uwsgi.exceptions', 'line'],
+ 'lines': [
+ ['exceptions', 'exceptions', 'incremental']
+ ]
+ },
+ 'harakiri': {
+ 'options': [None, 'Harakiris', 'harakiris', 'harakiris', 'uwsgi.harakiris', 'line'],
+ 'lines': [
+ ['harakiri_count', 'harakiris', 'incremental']
+ ]
+ },
+ 'respawn': {
+ 'options': [None, 'Respawns', 'respawns', 'respawns', 'uwsgi.respawns', 'line'],
+ 'lines': [
+ ['respawn_count', 'respawns', 'incremental']
+ ]
+ },
+}
+
+
+class Service(SocketService):
+ def __init__(self, configuration=None, name=None):
+ super(Service, self).__init__(configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = deepcopy(CHARTS)
+ self.url = self.configuration.get('host', 'localhost')
+ self.port = self.configuration.get('port', 1717)
+ # Clear dynamic dimensions, these are added during `_get_data()` to allow adding workers at run-time
+ for chart in DYNAMIC_CHARTS:
+ self.definitions[chart]['lines'] = []
+ self.last_result = {}
+ self.workers = []
+
+ def read_data(self):
+ """
+ Read data from socket and parse as JSON.
+ :return: (dict) stats
+ """
+ raw_data = self._get_raw_data()
+ if not raw_data:
+ return None
+ try:
+ return json.loads(raw_data)
+ except ValueError as err:
+ self.error(err)
+ return None
+
+ def check(self):
+ """
+ Parse configuration and check if we can read data.
+ :return: boolean
+ """
+ self._parse_config()
+ return bool(self.read_data())
+
+ def add_worker_dimensions(self, key):
+ """
+ Helper to add dimensions for a worker.
+ :param key: (int or str) worker identifier
+ :return:
+ """
+ for chart in DYNAMIC_CHARTS:
+ for line in CHARTS[chart]['lines']:
+ dimension_id = '{}_{}'.format(line[0], key)
+ dimension_name = str(key)
+
+ dimension = [dimension_id, dimension_name] + line[2:]
+ self.charts[chart].add_dimension(dimension)
+
+ @staticmethod
+ def _check_raw_data(data):
+ # The server will close the connection when it's done sending
+ # data, so just keep looping until that happens.
+ return False
+
+ def _get_data(self):
+ """
+ Read data from socket
+ :return: dict
+ """
+ stats = self.read_data()
+ if not stats:
+ return None
+
+ result = {
+ 'exceptions': 0,
+ 'harakiri_count': 0,
+ 'respawn_count': 0,
+ }
+
+ for worker in stats['workers']:
+ key = worker['pid']
+
+ # Add dimensions for new workers
+ if key not in self.workers:
+ self.add_worker_dimensions(key)
+ self.workers.append(key)
+
+ result['requests_{}'.format(key)] = worker['requests']
+ result['tx_{}'.format(key)] = worker['tx']
+ result['avg_rt_{}'.format(key)] = worker['avg_rt']
+
+ # avg_rt is not reset by uwsgi, so reset here
+ if self.last_result.get('requests_{}'.format(key)) == worker['requests']:
+ result['avg_rt_{}'.format(key)] = 0
+
+ result['memory_rss_{}'.format(key)] = worker['rss']
+ result['memory_vsz_{}'.format(key)] = worker['vsz']
+
+ result['exceptions'] += worker['exceptions']
+ result['harakiri_count'] += worker['harakiri_count']
+ result['respawn_count'] += worker['respawn_count']
+
+ self.last_result = result
+ return result
diff --git a/collectors/python.d.plugin/uwsgi/uwsgi.conf b/collectors/python.d.plugin/uwsgi/uwsgi.conf
new file mode 100644
index 0000000..7d09e73
--- /dev/null
+++ b/collectors/python.d.plugin/uwsgi/uwsgi.conf
@@ -0,0 +1,92 @@
+# netdata python.d.plugin configuration for uwsgi
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, uwsgi also supports the following:
+#
+# socket: 'path/to/uwsgistats.sock'
+#
+# or
+# host: 'IP or HOSTNAME' # the host to connect to
+# port: PORT # the port to connect to
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+#
+
+socket:
+ name : 'local'
+ socket : '/tmp/stats.socket'
+
+localhost:
+ name : 'local'
+ host : 'localhost'
+ port : 1717
+
+localipv4:
+ name : 'local'
+ host : '127.0.0.1'
+ port : 1717
+
+localipv6:
+ name : 'local'
+ host : '::1'
+ port : 1717
diff --git a/collectors/python.d.plugin/varnish/Makefile.inc b/collectors/python.d.plugin/varnish/Makefile.inc
new file mode 100644
index 0000000..2469b05
--- /dev/null
+++ b/collectors/python.d.plugin/varnish/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += varnish/varnish.chart.py
+dist_pythonconfig_DATA += varnish/varnish.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += varnish/README.md varnish/Makefile.inc
+
diff --git a/collectors/python.d.plugin/varnish/README.md b/collectors/python.d.plugin/varnish/README.md
new file mode 100644
index 0000000..018905f
--- /dev/null
+++ b/collectors/python.d.plugin/varnish/README.md
@@ -0,0 +1,65 @@
+<!--
+title: "Varnish Cache monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/varnish/README.md
+sidebar_label: "Varnish Cache"
+-->
+
+# Varnish Cache monitoring with Netdata
+
+Provides HTTP accelerator global, Backends (VBE) and Storages (SMF, SMA, MSE) statistics using `varnishstat` tool.
+
+Note that both, Varnish-Cache (free and open source) and Varnish-Plus (Commercial/Enterprise version), are supported.
+
+## Requirements
+
+- `netdata` user must be a member of the `varnish` group
+
+## Charts
+
+This module produces the following charts:
+
+- Connections Statistics in `connections/s`
+- Client Requests in `requests/s`
+- All History Hit Rate Ratio in `percent`
+- Current Poll Hit Rate Ratio in `percent`
+- Expired Objects in `expired/s`
+- Least Recently Used Nuked Objects in `nuked/s`
+- Number Of Threads In All Pools in `pools`
+- Threads Statistics in `threads/s`
+- Current Queue Length in `requests`
+- Backend Connections Statistics in `connections/s`
+- Requests To The Backend in `requests/s`
+- ESI Statistics in `problems/s`
+- Memory Usage in `MiB`
+- Uptime in `seconds`
+
+For every backend (VBE):
+
+- Backend Response Statistics in `kilobits/s`
+
+For every storage (SMF, SMA, or MSE):
+
+- Storage Usage in `KiB`
+- Storage Allocated Objects
+
+## Configuration
+
+Edit the `python.d/varnish.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/varnish.conf
+```
+
+Only one parameter is supported:
+
+```yaml
+instance_name: 'name'
+```
+
+The name of the `varnishd` instance to get logs from. If not specified, the host name is used.
+
+---
+
+
diff --git a/collectors/python.d.plugin/varnish/varnish.chart.py b/collectors/python.d.plugin/varnish/varnish.chart.py
new file mode 100644
index 0000000..506ad02
--- /dev/null
+++ b/collectors/python.d.plugin/varnish/varnish.chart.py
@@ -0,0 +1,385 @@
+# -*- coding: utf-8 -*-
+# Description: varnish netdata python.d module
+# Author: ilyam8
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import re
+
+from bases.FrameworkServices.ExecutableService import ExecutableService
+from bases.collection import find_binary
+
+ORDER = [
+ 'session_connections',
+ 'client_requests',
+ 'all_time_hit_rate',
+ 'current_poll_hit_rate',
+ 'cached_objects_expired',
+ 'cached_objects_nuked',
+ 'threads_total',
+ 'threads_statistics',
+ 'threads_queue_len',
+ 'backend_connections',
+ 'backend_requests',
+ 'esi_statistics',
+ 'memory_usage',
+ 'uptime'
+]
+
+CHARTS = {
+ 'session_connections': {
+ 'options': [None, 'Connections Statistics', 'connections/s',
+ 'client metrics', 'varnish.session_connection', 'line'],
+ 'lines': [
+ ['sess_conn', 'accepted', 'incremental'],
+ ['sess_dropped', 'dropped', 'incremental']
+ ]
+ },
+ 'client_requests': {
+ 'options': [None, 'Client Requests', 'requests/s',
+ 'client metrics', 'varnish.client_requests', 'line'],
+ 'lines': [
+ ['client_req', 'received', 'incremental']
+ ]
+ },
+ 'all_time_hit_rate': {
+ 'options': [None, 'All History Hit Rate Ratio', 'percentage', 'cache performance',
+ 'varnish.all_time_hit_rate', 'stacked'],
+ 'lines': [
+ ['cache_hit', 'hit', 'percentage-of-absolute-row'],
+ ['cache_miss', 'miss', 'percentage-of-absolute-row'],
+ ['cache_hitpass', 'hitpass', 'percentage-of-absolute-row']]
+ },
+ 'current_poll_hit_rate': {
+ 'options': [None, 'Current Poll Hit Rate Ratio', 'percentage', 'cache performance',
+ 'varnish.current_poll_hit_rate', 'stacked'],
+ 'lines': [
+ ['cache_hit', 'hit', 'percentage-of-incremental-row'],
+ ['cache_miss', 'miss', 'percentage-of-incremental-row'],
+ ['cache_hitpass', 'hitpass', 'percentage-of-incremental-row']
+ ]
+ },
+ 'cached_objects_expired': {
+ 'options': [None, 'Expired Objects', 'expired/s', 'cache performance',
+ 'varnish.cached_objects_expired', 'line'],
+ 'lines': [
+ ['n_expired', 'objects', 'incremental']
+ ]
+ },
+ 'cached_objects_nuked': {
+ 'options': [None, 'Least Recently Used Nuked Objects', 'nuked/s', 'cache performance',
+ 'varnish.cached_objects_nuked', 'line'],
+ 'lines': [
+ ['n_lru_nuked', 'objects', 'incremental']
+ ]
+ },
+ 'threads_total': {
+ 'options': [None, 'Number Of Threads In All Pools', 'number', 'thread related metrics',
+ 'varnish.threads_total', 'line'],
+ 'lines': [
+ ['threads', None, 'absolute']
+ ]
+ },
+ 'threads_statistics': {
+ 'options': [None, 'Threads Statistics', 'threads/s', 'thread related metrics',
+ 'varnish.threads_statistics', 'line'],
+ 'lines': [
+ ['threads_created', 'created', 'incremental'],
+ ['threads_failed', 'failed', 'incremental'],
+ ['threads_limited', 'limited', 'incremental']
+ ]
+ },
+ 'threads_queue_len': {
+ 'options': [None, 'Current Queue Length', 'requests', 'thread related metrics',
+ 'varnish.threads_queue_len', 'line'],
+ 'lines': [
+ ['thread_queue_len', 'in queue']
+ ]
+ },
+ 'backend_connections': {
+ 'options': [None, 'Backend Connections Statistics', 'connections/s', 'backend metrics',
+ 'varnish.backend_connections', 'line'],
+ 'lines': [
+ ['backend_conn', 'successful', 'incremental'],
+ ['backend_unhealthy', 'unhealthy', 'incremental'],
+ ['backend_reuse', 'reused', 'incremental'],
+ ['backend_toolate', 'closed', 'incremental'],
+ ['backend_recycle', 'recycled', 'incremental'],
+ ['backend_fail', 'failed', 'incremental']
+ ]
+ },
+ 'backend_requests': {
+ 'options': [None, 'Requests To The Backend', 'requests/s', 'backend metrics',
+ 'varnish.backend_requests', 'line'],
+ 'lines': [
+ ['backend_req', 'sent', 'incremental']
+ ]
+ },
+ 'esi_statistics': {
+ 'options': [None, 'ESI Statistics', 'problems/s', 'esi related metrics', 'varnish.esi_statistics', 'line'],
+ 'lines': [
+ ['esi_errors', 'errors', 'incremental'],
+ ['esi_warnings', 'warnings', 'incremental']
+ ]
+ },
+ 'memory_usage': {
+ 'options': [None, 'Memory Usage', 'MiB', 'memory usage', 'varnish.memory_usage', 'stacked'],
+ 'lines': [
+ ['memory_free', 'free', 'absolute', 1, 1 << 20],
+ ['memory_allocated', 'allocated', 'absolute', 1, 1 << 20]]
+ },
+ 'uptime': {
+ 'lines': [
+ ['uptime', None, 'absolute']
+ ],
+ 'options': [None, 'Uptime', 'seconds', 'uptime', 'varnish.uptime', 'line']
+ }
+}
+
+
+def backend_charts_template(name):
+ order = [
+ '{0}_response_statistics'.format(name),
+ ]
+
+ charts = {
+ order[0]: {
+ 'options': [None, 'Backend "{0}"'.format(name), 'kilobits/s', 'backend response statistics',
+ 'varnish.backend', 'area'],
+ 'lines': [
+ ['{0}_beresp_hdrbytes'.format(name), 'header', 'incremental', 8, 1000],
+ ['{0}_beresp_bodybytes'.format(name), 'body', 'incremental', -8, 1000]
+ ]
+ },
+ }
+
+ return order, charts
+
+
+def storage_charts_template(name):
+ order = [
+ 'storage_{0}_usage'.format(name),
+ 'storage_{0}_alloc_objs'.format(name)
+ ]
+
+ charts = {
+ order[0]: {
+ 'options': [None, 'Storage "{0}" Usage'.format(name), 'KiB', 'storage usage', 'varnish.storage_usage', 'stacked'],
+ 'lines': [
+ ['{0}.g_space'.format(name), 'free', 'absolute', 1, 1 << 10],
+ ['{0}.g_bytes'.format(name), 'allocated', 'absolute', 1, 1 << 10]
+ ]
+ },
+ order[1]: {
+ 'options': [None, 'Storage "{0}" Allocated Objects'.format(name), 'objects', 'storage usage', 'varnish.storage_alloc_objs', 'line'],
+ 'lines': [
+ ['{0}.g_alloc'.format(name), 'allocated', 'absolute']
+ ]
+ }
+ }
+
+ return order, charts
+
+
+VARNISHSTAT = 'varnishstat'
+
+re_version = re.compile(r'varnish-(?:plus-)?(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
+
+
+class VarnishVersion:
+ def __init__(self, major, minor, patch):
+ self.major = major
+ self.minor = minor
+ self.patch = patch
+
+ def __str__(self):
+ return '{0}.{1}.{2}'.format(self.major, self.minor, self.patch)
+
+
+class Parser:
+ _backend_new = re.compile(r'VBE.([\d\w_.]+)\(.*?\).(beresp[\w_]+)\s+(\d+)')
+ _backend_old = re.compile(r'VBE\.[\d\w-]+\.([\w\d_-]+).(beresp[\w_]+)\s+(\d+)')
+ _default = re.compile(r'([A-Z]+\.)?([\d\w_.]+)\s+(\d+)')
+
+ def __init__(self):
+ self.re_default = None
+ self.re_backend = None
+
+ def init(self, data):
+ data = ''.join(data)
+ parsed_main = Parser._default.findall(data)
+ if parsed_main:
+ self.re_default = Parser._default
+
+ parsed_backend = Parser._backend_new.findall(data)
+ if parsed_backend:
+ self.re_backend = Parser._backend_new
+ else:
+ parsed_backend = Parser._backend_old.findall(data)
+ if parsed_backend:
+ self.re_backend = Parser._backend_old
+
+ def server_stats(self, data):
+ return self.re_default.findall(''.join(data))
+
+ def backend_stats(self, data):
+ return self.re_backend.findall(''.join(data))
+
+
+class Service(ExecutableService):
+ def __init__(self, configuration=None, name=None):
+ ExecutableService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.instance_name = configuration.get('instance_name')
+ self.parser = Parser()
+ self.command = None
+ self.collected_vbe = set()
+ self.collected_storages = set()
+
+ def create_command(self):
+ varnishstat = find_binary(VARNISHSTAT)
+
+ if not varnishstat:
+ self.error("can't locate '{0}' binary or binary is not executable by user netdata".format(VARNISHSTAT))
+ return False
+
+ command = [varnishstat, '-V']
+ reply = self._get_raw_data(stderr=True, command=command)
+ if not reply:
+ self.error(
+ "no output from '{0}'. Is varnish running? Not enough privileges?".format(' '.join(self.command)))
+ return False
+
+ ver = parse_varnish_version(reply)
+ if not ver:
+ self.error("failed to parse reply from '{0}', used regex :'{1}', reply : {2}".format(
+ ' '.join(command), re_version.pattern, reply))
+ return False
+
+ if self.instance_name:
+ self.command = [varnishstat, '-1', '-n', self.instance_name]
+ else:
+ self.command = [varnishstat, '-1']
+
+ if ver.major > 4:
+ self.command.extend(['-t', '1'])
+
+ self.info("varnish version: {0}, will use command: '{1}'".format(ver, ' '.join(self.command)))
+
+ return True
+
+ def check(self):
+ if not self.create_command():
+ return False
+
+ # STDOUT is not empty
+ reply = self._get_raw_data()
+ if not reply:
+ self.error("no output from '{0}'. Is it running? Not enough privileges?".format(' '.join(self.command)))
+ return False
+
+ self.parser.init(reply)
+
+ # Output is parsable
+ if not self.parser.re_default:
+ self.error('cant parse the output...')
+ return False
+
+ return True
+
+ def get_data(self):
+ """
+ Format data received from shell command
+ :return: dict
+ """
+ raw = self._get_raw_data()
+ if not raw:
+ return None
+
+ data = dict()
+ server_stats = self.parser.server_stats(raw)
+ if not server_stats:
+ return None
+
+ stats = dict((param, value) for _, param, value in server_stats)
+ data.update(stats)
+
+ self.get_vbe_backends(data, raw)
+ self.get_storages(server_stats)
+
+ # varnish 5 uses default.g_bytes and default.g_space
+ data['memory_allocated'] = data.get('s0.g_bytes') or data.get('default.g_bytes')
+ data['memory_free'] = data.get('s0.g_space') or data.get('default.g_space')
+
+ return data
+
+ def get_vbe_backends(self, data, raw):
+ if not self.parser.re_backend:
+ return
+ stats = self.parser.backend_stats(raw)
+ if not stats:
+ return
+
+ for (name, param, value) in stats:
+ data['_'.join([name, param])] = value
+ if name in self.collected_vbe:
+ continue
+ self.collected_vbe.add(name)
+ self.add_backend_charts(name)
+
+ def get_storages(self, server_stats):
+ # Storage types:
+ # - SMF: File Storage
+ # - SMA: Malloc Storage
+ # - MSE: Massive Storage Engine (Varnish-Plus only)
+ #
+ # Stats example:
+ # [('SMF.', 'ssdStorage.c_req', '47686'),
+ # ('SMF.', 'ssdStorage.c_fail', '0'),
+ # ('SMF.', 'ssdStorage.c_bytes', '668102656'),
+ # ('SMF.', 'ssdStorage.c_freed', '140980224'),
+ # ('SMF.', 'ssdStorage.g_alloc', '39753'),
+ # ('SMF.', 'ssdStorage.g_bytes', '527122432'),
+ # ('SMF.', 'ssdStorage.g_space', '53159968768'),
+ # ('SMF.', 'ssdStorage.g_smf', '40130'),
+ # ('SMF.', 'ssdStorage.g_smf_frag', '311'),
+ # ('SMF.', 'ssdStorage.g_smf_large', '66')]
+ storages = [name for typ, name, _ in server_stats if typ.startswith(('SMF', 'SMA', 'MSE')) and name.endswith('g_space')]
+ if not storages:
+ return
+ for storage in storages:
+ storage = storage.split('.')[0]
+ if storage in self.collected_storages:
+ continue
+ self.collected_storages.add(storage)
+ self.add_storage_charts(storage)
+
+ def add_backend_charts(self, backend_name):
+ self.add_charts(backend_name, backend_charts_template)
+
+ def add_storage_charts(self, storage_name):
+ self.add_charts(storage_name, storage_charts_template)
+
+ def add_charts(self, name, charts_template):
+ order, charts = charts_template(name)
+
+ for chart_name in order:
+ params = [chart_name] + charts[chart_name]['options']
+ dimensions = charts[chart_name]['lines']
+
+ new_chart = self.charts.add_chart(params)
+ for dimension in dimensions:
+ new_chart.add_dimension(dimension)
+
+
+def parse_varnish_version(lines):
+ m = re_version.search(lines[0])
+ if not m:
+ return None
+
+ m = m.groupdict()
+ return VarnishVersion(
+ int(m['major']),
+ int(m['minor']),
+ int(m['patch']),
+ )
diff --git a/collectors/python.d.plugin/varnish/varnish.conf b/collectors/python.d.plugin/varnish/varnish.conf
new file mode 100644
index 0000000..54bfe4d
--- /dev/null
+++ b/collectors/python.d.plugin/varnish/varnish.conf
@@ -0,0 +1,66 @@
+# netdata python.d.plugin configuration for varnish
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, varnish also supports the following:
+#
+# instance_name: 'name' # the name of the varnishd instance to get logs from. If not specified, the host name is used.
+#
+# ----------------------------------------------------------------------
diff --git a/collectors/python.d.plugin/w1sensor/Makefile.inc b/collectors/python.d.plugin/w1sensor/Makefile.inc
new file mode 100644
index 0000000..bddf146
--- /dev/null
+++ b/collectors/python.d.plugin/w1sensor/Makefile.inc
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += w1sensor/w1sensor.chart.py
+dist_pythonconfig_DATA += w1sensor/w1sensor.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += w1sensor/README.md w1sensor/Makefile.inc
+
diff --git a/collectors/python.d.plugin/w1sensor/README.md b/collectors/python.d.plugin/w1sensor/README.md
new file mode 100644
index 0000000..b6d2b2d
--- /dev/null
+++ b/collectors/python.d.plugin/w1sensor/README.md
@@ -0,0 +1,28 @@
+<!--
+title: "1-Wire Sensors monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/w1sensor/README.md
+sidebar_label: "1-Wire sensors"
+-->
+
+# 1-Wire Sensors monitoring with Netdata
+
+Monitors sensor temperature.
+
+On Linux these are supported by the wire, w1_gpio, and w1_therm modules.
+Currently temperature sensors are supported and automatically detected.
+
+Charts are created dynamically based on the number of detected sensors.
+
+## Configuration
+
+Edit the `python.d/w1sensor.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/w1sensor.conf
+```
+
+---
+
+
diff --git a/collectors/python.d.plugin/w1sensor/w1sensor.chart.py b/collectors/python.d.plugin/w1sensor/w1sensor.chart.py
new file mode 100644
index 0000000..c4f847b
--- /dev/null
+++ b/collectors/python.d.plugin/w1sensor/w1sensor.chart.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# Description: 1-wire temperature monitor netdata python.d module
+# Author: Diomidis Spinellis <http://www.spinellis.gr>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+import re
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+# default module values (can be overridden per job in `config`)
+update_every = 5
+
+# Location where 1-Wire devices can be found
+W1_DIR = '/sys/bus/w1/devices/'
+
+# Lines matching the following regular expression contain a temperature value
+RE_TEMP = re.compile(r' t=(\d+)')
+
+ORDER = [
+ 'temp',
+]
+
+CHARTS = {
+ 'temp': {
+ 'options': [None, '1-Wire Temperature Sensor', 'Celsius', 'Temperature', 'w1sensor.temp', 'line'],
+ 'lines': []
+ }
+}
+
+# Known and supported family members
+# Based on linux/drivers/w1/w1_family.h and w1/slaves/w1_therm.c
+THERM_FAMILY = {
+ '10': 'W1_THERM_DS18S20',
+ '22': 'W1_THERM_DS1822',
+ '28': 'W1_THERM_DS18B20',
+ '3b': 'W1_THERM_DS1825',
+ '42': 'W1_THERM_DS28EA00',
+}
+
+
+class Service(SimpleService):
+ """Provide netdata service for 1-Wire sensors"""
+
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.probes = []
+
+ def check(self):
+ """Auto-detect available 1-Wire sensors, setting line definitions
+ and probes to be monitored."""
+ try:
+ file_names = os.listdir(W1_DIR)
+ except OSError as err:
+ self.error(err)
+ return False
+
+ lines = []
+ for file_name in file_names:
+ if file_name[2] != '-':
+ continue
+ if not file_name[0:2] in THERM_FAMILY:
+ continue
+
+ self.probes.append(file_name)
+ identifier = file_name[3:]
+ name = identifier
+ config_name = self.configuration.get('name_' + identifier)
+ if config_name:
+ name = config_name
+ lines.append(['w1sensor_temp_' + identifier, name, 'absolute',
+ 1, 10])
+ self.definitions['temp']['lines'] = lines
+ return len(self.probes) > 0
+
+ def get_data(self):
+ """Return data read from sensors."""
+ data = dict()
+
+ for file_name in self.probes:
+ file_path = W1_DIR + file_name + '/w1_slave'
+ identifier = file_name[3:]
+ try:
+ with open(file_path, 'r') as device_file:
+ for line in device_file:
+ matched = RE_TEMP.search(line)
+ if matched:
+ # Round to one decimal digit to filter-out noise
+ value = round(int(matched.group(1)) / 1000., 1)
+ value = int(value * 10)
+ data['w1sensor_temp_' + identifier] = value
+ except (OSError, IOError) as err:
+ self.error(err)
+ continue
+ return data or None
diff --git a/collectors/python.d.plugin/w1sensor/w1sensor.conf b/collectors/python.d.plugin/w1sensor/w1sensor.conf
new file mode 100644
index 0000000..1727100
--- /dev/null
+++ b/collectors/python.d.plugin/w1sensor/w1sensor.conf
@@ -0,0 +1,72 @@
+# netdata python.d.plugin configuration for w1sensor
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 5
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 5 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, example also supports the following:
+#
+# name_<1-Wire id>: '<human readable name>'
+# This allows associating a human readable name with a sensor's 1-Wire
+# identifier. Example:
+# name_00000022276e: 'Machine room'
+# name_00000022298f: 'Rack 12'
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
diff --git a/collectors/python.d.plugin/zscores/Makefile.inc b/collectors/python.d.plugin/zscores/Makefile.inc
new file mode 100644
index 0000000..d8b1824
--- /dev/null
+++ b/collectors/python.d.plugin/zscores/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_python_DATA += zscores/zscores.chart.py
+dist_pythonconfig_DATA += zscores/zscores.conf
+
+# do not install these files, but include them in the distribution
+dist_noinst_DATA += zscores/README.md zscores/Makefile.inc
diff --git a/collectors/python.d.plugin/zscores/README.md b/collectors/python.d.plugin/zscores/README.md
new file mode 100644
index 0000000..4f84a6c
--- /dev/null
+++ b/collectors/python.d.plugin/zscores/README.md
@@ -0,0 +1,144 @@
+<!--
+title: "zscores"
+description: "Use statistical anomaly detection to narrow your focus and shorten root cause analysis."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/zscores/README.md
+-->
+
+# Z-Scores - basic anomaly detection for your key metrics and charts
+
+Smoothed, rolling [Z-Scores](https://en.wikipedia.org/wiki/Standard_score) for selected metrics or charts.
+
+This collector uses the [Netdata rest api](https://learn.netdata.cloud/docs/agent/web/api) to get the `mean` and `stddev`
+for each dimension on specified charts over a time range (defined by `train_secs` and `offset_secs`). For each dimension
+it will calculate a Z-Score as `z = (x - mean) / stddev` (clipped at `z_clip`). Scores are then smoothed over
+time (`z_smooth_n`) and, if `mode: 'per_chart'`, aggregated across dimensions to a smoothed, rolling chart level Z-Score
+at each time step.
+
+## Charts
+
+Two charts are produced:
+
+- **Z-Score** (`zscores.z`): This chart shows the calculated Z-Score per chart (or dimension if `mode='per_dim'`).
+- **Z-Score >3** (`zscores.3stddev`): This chart shows a `1` if the absolute value of the Z-Score is greater than 3 or
+ a `0` otherwise.
+
+Below is an example of the charts produced by this collector and a typical example of how they would look when things
+are 'normal' on the system. Most of the zscores tend to bounce randomly around a range typically between 0 to +3 (or -3
+to +3 if `z_abs: 'false'`), a few charts might stay steady at a more constant higher value depending on your
+configuration and the typical workload on your system (typically those charts that do not change that much have a
+smaller range of values on which to calculate a zscore and so tend to have a higher typical zscore).
+
+So really its a combination of the zscores values themselves plus, perhaps more importantly, how they change when
+something strange occurs on your system which can be most useful.
+
+![zscores-collector-normal](https://user-images.githubusercontent.com/2178292/108776300-21d44d00-755a-11eb-92a4-ecb8f7d2f175.png)
+
+For example, if we go onto the system and run a command
+like [`stress-ng --all 2`](https://wiki.ubuntu.com/Kernel/Reference/stress-ng) to create some stress, we see many charts
+begin to have zscores that jump outside the typical range. When the absolute zscore for a chart is greater than 3 you
+will see a corresponding line appear on the `zscores.3stddev` chart to make it a bit clearer what charts might be worth
+looking at first (for more background information on why 3 stddev
+see [here](https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule#:~:text=In%20the%20empirical%20sciences%20the,99.7%25%20probability%20as%20near%20certainty.))
+.
+
+In the example below we basically took a sledge hammer to our system so its not surprising that lots of charts light up
+after we run the stress command. In a more realistic setting you might just see a handful of charts with strange zscores
+and that could be a good indication of where to look first.
+
+![zscores-collector-abnormal](https://user-images.githubusercontent.com/2178292/108776316-28fb5b00-755a-11eb-80de-ec5d38089ecc.png)
+
+Then as the issue passes the zscores should settle back down into their normal range again as they are calculated in a
+rolling and smoothed way (as defined by your `zscores.conf` file).
+
+![zscores-collector-normal-again](https://user-images.githubusercontent.com/2178292/108776439-4fb99180-755a-11eb-8bb7-b4df144cb44c.png)
+
+## Requirements
+
+This collector will only work with Python 3 and requires the below packages be installed.
+
+```bash
+# become netdata user
+sudo su -s /bin/bash netdata
+# install required packages
+pip3 install numpy pandas requests netdata-pandas==0.0.38
+```
+
+## Configuration
+
+Install the underlying Python requirements, Enable the collector and restart Netdata.
+
+```bash
+cd /etc/netdata/
+sudo ./edit-config python.d.conf
+# Set `zscores: no` to `zscores: yes`
+sudo systemctl restart netdata
+```
+
+The configuration for the zscores collector defines how it will behave on your system and might take some
+experimentation with over time to set it optimally. Out of the box, the config comes with
+some [sane defaults](https://www.netdata.cloud/blog/redefining-monitoring-netdata/) to get you started.
+
+If you are unsure about any of the below configuration options then it's best to just ignore all this and leave
+the `zscores.conf` files alone to begin with. Then you can return to it later if you would like to tune things a bit
+more once the collector is running for a while.
+
+Edit the `python.d/zscores.conf` configuration file using `edit-config` from the your
+agent's [config directory](https://learn.netdata.cloud/guides/step-by-step/step-04#find-your-netdataconf-file), which is
+usually at `/etc/netdata`.
+
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/zscores.conf
+```
+
+The default configuration should look something like this. Here you can see each parameter (with sane defaults) and some
+information about each one and what it does.
+
+```bash
+# what host to pull data from
+host: '127.0.0.1:19999'
+# What charts to pull data for - A regex like 'system\..*|' or 'system\..*|apps.cpu|apps.mem' etc.
+charts_regex: 'system\..*'
+# length of time to base calculations off for mean and stddev
+train_secs: 14400 # use last 4 hours to work out the mean and stddev for the zscore
+# offset preceding latest data to ignore when calculating mean and stddev
+offset_secs: 300 # ignore last 5 minutes of data when calculating the mean and stddev
+# recalculate the mean and stddev every n steps of the collector
+train_every_n: 900 # recalculate mean and stddev every 15 minutes
+# smooth the z score by averaging it over last n values
+z_smooth_n: 15 # take a rolling average of the last 15 zscore values to reduce sensitivity to temporary 'spikes'
+# cap absolute value of zscore (before smoothing) for better stability
+z_clip: 10 # cap each zscore at 10 so as to avoid really large individual zscores swamping any rolling average
+# set z_abs: 'true' to make all zscores be absolute values only.
+z_abs: 'true'
+# burn in period in which to initially calculate mean and stddev on every step
+burn_in: 2 # on startup of the collector continually update the mean and stddev in case any gaps or initial calculations fail to return
+# mode can be to get a zscore 'per_dim' or 'per_chart'
+mode: 'per_chart' # 'per_chart' means individual dimension level smoothed zscores will be aggregated to one zscore per chart per time step
+# per_chart_agg is how you aggregate from dimension to chart when mode='per_chart'
+per_chart_agg: 'mean' # 'absmax' will take the max absolute value across all dimensions but will maintain the sign. 'mean' will just average.
+```
+
+## Notes
+
+- Python 3 is required as the [`netdata-pandas`](https://github.com/netdata/netdata-pandas) package uses python async
+ libraries ([asks](https://pypi.org/project/asks/) and [trio](https://pypi.org/project/trio/)) to make asynchronous
+ calls to the netdata rest api to get the required data for each chart when calculating the mean and stddev.
+- It may take a few hours or so for the collector to 'settle' into it's typical behaviour in terms of the scores you
+ will see in the normal running of your system.
+- The zscore you see for each chart when using `mode: 'per_chart'` as actually an aggregated zscore across all the
+ dimensions on the underlying chart.
+- If you set `mode: 'per_dim'` then you will see a zscore for each dimension on each chart as opposed to one per chart.
+- As this collector does some calculations itself in python you may want to try it out first on a test or development
+ system to get a sense of its performance characteristics. Most of the work in calculating the mean and stddev will be
+ pushed down to the underlying Netdata C libraries via the rest api. But some data wrangling and calculations are then
+ done using [Pandas](https://pandas.pydata.org/) and [Numpy](https://numpy.org/) within the collector itself.
+- On a development n1-standard-2 (2 vCPUs, 7.5 GB memory) vm running Ubuntu 18.04 LTS and not doing any work some of the
+ typical performance characteristics we saw from running this collector were:
+ - A runtime (`netdata.runtime_zscores`) of ~50ms when doing scoring and ~500ms when recalculating the mean and
+ stddev.
+ - Typically 3%-3.5% cpu usage from scoring, jumping to ~35% for one second when recalculating the mean and stddev.
+ - About ~50mb of ram (`apps.mem`) being continually used by the `python.d.plugin`.
+- If you activate this collector on a fresh node, it might take a little while to build up enough data to calculate a
+ proper zscore. So until you actually have `train_secs` of available data the mean and stddev calculated will be subject
+ to more noise.
diff --git a/collectors/python.d.plugin/zscores/zscores.chart.py b/collectors/python.d.plugin/zscores/zscores.chart.py
new file mode 100644
index 0000000..1099b93
--- /dev/null
+++ b/collectors/python.d.plugin/zscores/zscores.chart.py
@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+# Description: zscores netdata python.d module
+# Author: andrewm4894
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import datetime
+import re
+
+import requests
+import numpy as np
+import pandas as pd
+
+from bases.FrameworkServices.SimpleService import SimpleService
+from netdata_pandas.data import get_data, get_allmetrics
+
+priority = 60000
+update_every = 5
+disabled_by_default = True
+
+ORDER = [
+ 'z',
+ '3stddev'
+]
+
+CHARTS = {
+ 'z': {
+ 'options': ['z', 'Z Score', 'z', 'Z Score', 'zscores.z', 'line'],
+ 'lines': []
+ },
+ '3stddev': {
+ 'options': ['3stddev', 'Z Score >3', 'count', '3 Stddev', 'zscores.3stddev', 'stacked'],
+ 'lines': []
+ },
+}
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.host = self.configuration.get('host', '127.0.0.1:19999')
+ self.charts_regex = re.compile(self.configuration.get('charts_regex', 'system.*'))
+ self.charts_to_exclude = self.configuration.get('charts_to_exclude', '').split(',')
+ self.charts_in_scope = [
+ c for c in
+ list(filter(self.charts_regex.match,
+ requests.get(f'http://{self.host}/api/v1/charts').json()['charts'].keys()))
+ if c not in self.charts_to_exclude
+ ]
+ self.train_secs = self.configuration.get('train_secs', 14400)
+ self.offset_secs = self.configuration.get('offset_secs', 300)
+ self.train_every_n = self.configuration.get('train_every_n', 900)
+ self.z_smooth_n = self.configuration.get('z_smooth_n', 15)
+ self.z_clip = self.configuration.get('z_clip', 10)
+ self.z_abs = bool(self.configuration.get('z_abs', True))
+ self.burn_in = self.configuration.get('burn_in', 2)
+ self.mode = self.configuration.get('mode', 'per_chart')
+ self.per_chart_agg = self.configuration.get('per_chart_agg', 'mean')
+ self.order = ORDER
+ self.definitions = CHARTS
+ self.collected_dims = {'z': set(), '3stddev': set()}
+ self.df_mean = pd.DataFrame()
+ self.df_std = pd.DataFrame()
+ self.df_z_history = pd.DataFrame()
+
+ def check(self):
+ _ = get_allmetrics(self.host, self.charts_in_scope, wide=True, col_sep='.')
+ return True
+
+ def validate_charts(self, chart, data, algorithm='absolute', multiplier=1, divisor=1):
+ """If dimension not in chart then add it.
+ """
+ for dim in data:
+ if dim not in self.collected_dims[chart]:
+ self.collected_dims[chart].add(dim)
+ self.charts[chart].add_dimension([dim, dim, algorithm, multiplier, divisor])
+
+ for dim in list(self.collected_dims[chart]):
+ if dim not in data:
+ self.collected_dims[chart].remove(dim)
+ self.charts[chart].del_dimension(dim, hide=False)
+
+ def train_model(self):
+ """Calculate the mean and stddev for all relevant metrics and store them for use in calulcating zscore at each timestep.
+ """
+ before = int(datetime.now().timestamp()) - self.offset_secs
+ after = before - self.train_secs
+
+ self.df_mean = get_data(
+ self.host, self.charts_in_scope, after, before, points=10, group='average', col_sep='.'
+ ).mean().to_frame().rename(columns={0: "mean"})
+
+ self.df_std = get_data(
+ self.host, self.charts_in_scope, after, before, points=10, group='stddev', col_sep='.'
+ ).mean().to_frame().rename(columns={0: "std"})
+
+ def create_data(self, df_allmetrics):
+ """Use x, mean, stddev to generate z scores and 3stddev flags via some pandas manipulation.
+ Returning two dictionaries of dimensions and measures, one for each chart.
+
+ :param df_allmetrics <pd.DataFrame>: pandas dataframe with latest data from api/v1/allmetrics.
+ :return: (<dict>,<dict>) tuple of dictionaries, one for zscores and the other for a flag if abs(z)>3.
+ """
+ # calculate clipped z score for each available metric
+ df_z = pd.concat([self.df_mean, self.df_std, df_allmetrics], axis=1, join='inner')
+ df_z['z'] = ((df_z['value'] - df_z['mean']) / df_z['std']).clip(-self.z_clip, self.z_clip).fillna(0) * 100
+ if self.z_abs:
+ df_z['z'] = df_z['z'].abs()
+
+ # append last z_smooth_n rows of zscores to history table in wide format
+ self.df_z_history = self.df_z_history.append(
+ df_z[['z']].reset_index().pivot_table(values='z', columns='index'), sort=True
+ ).tail(self.z_smooth_n)
+
+ # get average zscore for last z_smooth_n for each metric
+ df_z_smooth = self.df_z_history.melt(value_name='z').groupby('index')['z'].mean().to_frame()
+ df_z_smooth['3stddev'] = np.where(abs(df_z_smooth['z']) > 300, 1, 0)
+ data_z = df_z_smooth['z'].add_suffix('_z').to_dict()
+
+ # aggregate to chart level if specified
+ if self.mode == 'per_chart':
+ df_z_smooth['chart'] = ['.'.join(x[0:2]) + '_z' for x in df_z_smooth.index.str.split('.').to_list()]
+ if self.per_chart_agg == 'absmax':
+ data_z = \
+ list(df_z_smooth.groupby('chart').agg({'z': lambda x: max(x, key=abs)})['z'].to_dict().values())[0]
+ else:
+ data_z = list(df_z_smooth.groupby('chart').agg({'z': [self.per_chart_agg]})['z'].to_dict().values())[0]
+
+ data_3stddev = {}
+ for k in data_z:
+ data_3stddev[k.replace('_z', '')] = 1 if abs(data_z[k]) > 300 else 0
+
+ return data_z, data_3stddev
+
+ def get_data(self):
+
+ if self.runs_counter <= self.burn_in or self.runs_counter % self.train_every_n == 0:
+ self.train_model()
+
+ data_z, data_3stddev = self.create_data(
+ get_allmetrics(self.host, self.charts_in_scope, wide=True, col_sep='.').transpose())
+ data = {**data_z, **data_3stddev}
+
+ self.validate_charts('z', data_z, divisor=100)
+ self.validate_charts('3stddev', data_3stddev)
+
+ return data
diff --git a/collectors/python.d.plugin/zscores/zscores.conf b/collectors/python.d.plugin/zscores/zscores.conf
new file mode 100644
index 0000000..07d62eb
--- /dev/null
+++ b/collectors/python.d.plugin/zscores/zscores.conf
@@ -0,0 +1,108 @@
+# netdata python.d.plugin configuration for example
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+update_every: 5
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# penalty indicates whether to apply penalty to update_every in case of failures.
+# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
+# penalty: yes
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# penalty: yes # the JOB's penalty
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
+#
+# Additionally to the above, example also supports the following:
+#
+# - none
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+local:
+ name: 'local'
+
+ # what host to pull data from
+ host: '127.0.0.1:19999'
+
+ # what charts to pull data for - A regex like 'system\..*|' or 'system\..*|apps.cpu|apps.mem' etc.
+ charts_regex: 'system\..*'
+
+ # Charts to exclude, useful if you would like to exclude some specific charts.
+ # Note: should be a ',' separated string like 'chart.name,chart.name'.
+ charts_to_exclude: 'system.uptime'
+
+ # length of time to base calculations off for mean and stddev
+ train_secs: 14400 # use last 4 hours to work out the mean and stddev for the zscore
+
+ # offset preceding latest data to ignore when calculating mean and stddev
+ offset_secs: 300 # ignore last 5 minutes of data when calculating the mean and stddev
+
+ # recalculate the mean and stddev every n steps of the collector
+ train_every_n: 900 # recalculate mean and stddev every 15 minutes
+
+ # smooth the z score by averaging it over last n values
+ z_smooth_n: 15 # take a rolling average of the last 15 zscore values to reduce sensitivity to temporary 'spikes'
+
+ # cap absolute value of zscore (before smoothing) for better stability
+ z_clip: 10 # cap each zscore at 10 so as to avoid really large individual zscores swamping any rolling average
+
+ # set z_abs: 'true' to make all zscores be absolute values only.
+ z_abs: 'true'
+
+ # burn in period in which to initially calculate mean and stddev on every step
+ burn_in: 2 # on startup of the collector continually update the mean and stddev in case any gaps or initial calculations fail to return
+
+ # mode can be to get a zscore 'per_dim' or 'per_chart'
+ mode: 'per_chart' # 'per_chart' means individual dimension level smoothed zscores will be aggregated to one zscore per chart per time step
+
+ # per_chart_agg is how you aggregate from dimension to chart when mode='per_chart'
+ per_chart_agg: 'mean' # 'absmax' will take the max absolute value across all dimensions but will maintain the sign. 'mean' will just average.
diff --git a/collectors/slabinfo.plugin/Makefile.am b/collectors/slabinfo.plugin/Makefile.am
new file mode 100644
index 0000000..07796ea
--- /dev/null
+++ b/collectors/slabinfo.plugin/Makefile.am
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+CLEANFILES = \
+ slabinfo.plugin \
+ $(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/slabinfo.plugin/README.md b/collectors/slabinfo.plugin/README.md
new file mode 100644
index 0000000..2f49046
--- /dev/null
+++ b/collectors/slabinfo.plugin/README.md
@@ -0,0 +1,29 @@
+<!--
+title: "slabinfo.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/slabinfo.plugin/README.md
+-->
+
+# slabinfo.plugin
+
+SLAB is a cache mechanism used by the Kernel to avoid fragmentation.
+
+Each internal structure (process, file descriptor, inode...) is stored within a SLAB.
+
+## configuring Netdata for slabinfo
+
+The plugin is disabled by default because it collects and displays a huge amount of metrics.
+To enable it set `slabinfo = yes` in the `plugins` section of the `netdata.conf` configuration file.
+
+There is currently no configuration needed for the plugin itself.
+
+As `/proc/slabinfo` is only readable by root, this plugin is setuid root.
+
+## For what use
+
+This slabinfo details allows to have clues on actions done on your system.
+In the following screenshot, you can clearly see a `find` done on a ext4 filesystem (the number of `ext4_inode_cache` & `dentry` are rising fast), and a few seconds later, an admin issued a `echo 3 > /proc/sys/vm/drop_cached` as their count dropped.
+
+![netdata_slabinfo](https://user-images.githubusercontent.com/9157986/64433811-7f06e500-d0bf-11e9-8e1e-087497e61033.png)
+
+
+
diff --git a/collectors/slabinfo.plugin/slabinfo.c b/collectors/slabinfo.plugin/slabinfo.c
new file mode 100644
index 0000000..2e47ee2
--- /dev/null
+++ b/collectors/slabinfo.plugin/slabinfo.c
@@ -0,0 +1,393 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "daemon/common.h"
+#include "libnetdata/required_dummies.h"
+
+#define PLUGIN_SLABINFO_NAME "slabinfo.plugin"
+#define PLUGIN_SLABINFO_PROCFILE "/proc/slabinfo"
+
+#define CHART_TYPE "mem"
+#define CHART_FAMILY "slab"
+#define CHART_PRIO 3000
+
+// #define slabdebug(...) if (debug) { fprintf(stderr, __VA_ARGS__); }
+#define slabdebug(args...) if (debug) { \
+ fprintf(stderr, "slabinfo.plugin DEBUG (%04d@%-10.10s:%-15.15s)::", __LINE__, __FILE__, __FUNCTION__); \
+ fprintf(stderr, ##args); \
+ fprintf(stderr, "\n"); }
+
+int running = 1;
+int debug = 0;
+size_t lines_discovered = 0;
+int redraw_chart = 0;
+
+// ----------------------------------------------------------------------------
+
+// Slabinfo format :
+// format 2.1 Was provided by 57ed3eda977a215f054102b460ab0eb5d8d112e6 (2.6.24-rc6) as:
+// seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
+// seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
+// seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
+//
+// With max values:
+// seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
+// cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size, sinfo.objects_per_slab, (1 << sinfo.cache_order));
+// seq_printf(m, " : tunables %4u %4u %4u",
+// sinfo.limit, sinfo.batchcount, sinfo.shared);
+// seq_printf(m, " : slabdata %6lu %6lu %6lu",
+// sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
+//
+// If CONFIG_DEBUG_SLAB is set, it will also add columns from slabinfo_show_stats (for SLAB only):
+// seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu %4lu",
+// allocs, high, grown, reaped, errors, max_freeable, node_allocs, node_frees, overflows);
+// seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
+// allochit, allocmiss, freehit, freemiss);
+//
+// Implementation choices:
+// - Iterates through a linked list of kmem_cache.
+// - Name is a char* from struct kmem_cache (mm/slab.h).
+// - max name size found is 24:
+// grep -roP 'kmem_cache_create\(".+"'| awk '{split($0,a,"\""); print a[2],length(a[2]); }' | sort -k2 -n
+// - Using uint64 everywhere, as types fits and allows to use standard helpers
+
+struct slabinfo {
+ // procfile fields
+ const char *name;
+ uint64_t active_objs;
+ uint64_t num_objs;
+ uint64_t obj_size;
+ uint64_t obj_per_slab;
+ uint64_t pages_per_slab;
+ uint64_t tune_limit;
+ uint64_t tune_batchcnt;
+ uint64_t tune_shared_factor;
+ uint64_t data_active_slabs;
+ uint64_t data_num_slabs;
+ uint64_t data_shared_avail;
+
+ // Calculated fields
+ uint64_t mem_usage;
+ uint64_t mem_waste;
+ uint8_t obj_filling;
+
+ uint32_t hash;
+ struct slabinfo *next;
+} *slabinfo_root = NULL, *slabinfo_next = NULL, *slabinfo_last_used = NULL;
+
+// The code is very inspired from "proc_net_dev.c" and "perf_plugin.c"
+
+// Get the existing object, or create a new one
+static struct slabinfo *get_slabstruct(const char *name) {
+ struct slabinfo *s;
+
+ slabdebug("--> Requested slabstruct %s", name);
+
+ uint32_t hash = simple_hash(name);
+
+ // Search it, from the next to the end
+ for (s = slabinfo_next; s; s = s->next) {
+ if ((hash = s->hash) && !strcmp(name, s->name)) {
+ slabdebug("<-- Found existing slabstruct after %s", slabinfo_last_used->name);
+ // Prepare the next run
+ slabinfo_next = s->next;
+ slabinfo_last_used = s;
+ return s;
+ }
+ }
+
+ // Search it from the beginning to the last position we used
+ for (s = slabinfo_root; s != slabinfo_last_used; s = s->next) {
+ if (hash == s->hash && !strcmp(name, s->name)) {
+ slabdebug("<-- Found existing slabstruct after root %s", slabinfo_root->name);
+ slabinfo_next = s->next;
+ slabinfo_last_used = s;
+ return s;
+ }
+ }
+
+ // Create a new one
+ s = callocz(1, sizeof(struct slabinfo));
+ s->name = strdupz(name);
+ s->hash = hash;
+
+ // Add it to the current position
+ if (slabinfo_root) {
+ slabdebug("<-- Creating new slabstruct after %s", slabinfo_last_used->name);
+ s->next = slabinfo_last_used->next;
+ slabinfo_last_used->next = s;
+ slabinfo_last_used = s;
+ }
+ else {
+ slabdebug("<-- Creating new slabstruct as root");
+ slabinfo_root = slabinfo_last_used = s;
+ }
+
+ return s;
+}
+
+
+// Read a full pass of slabinfo to update the structs
+struct slabinfo *read_file_slabinfo() {
+
+ slabdebug("-> Reading procfile %s", PLUGIN_SLABINFO_PROCFILE);
+
+ static procfile *ff = NULL;
+ static long slab_pagesize = 0;
+
+ if (unlikely(!slab_pagesize)) {
+ slab_pagesize = sysconf(_SC_PAGESIZE);
+ slabdebug(" Discovered pagesize: %ld", slab_pagesize);
+ }
+
+ if(unlikely(!ff)) {
+ ff = procfile_reopen(ff, PLUGIN_SLABINFO_PROCFILE, " ,:" , PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff)) {
+ error("<- Cannot open file '%s", PLUGIN_SLABINFO_PROCFILE);
+ exit(1);
+ }
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff)) {
+ error("<- Cannot read file '%s'", PLUGIN_SLABINFO_PROCFILE);
+ exit(0);
+ }
+
+
+ // Iterate on all lines to populate / update the slabinfo struct
+ size_t lines = procfile_lines(ff), l;
+ if (unlikely(lines != lines_discovered)) {
+ lines_discovered = lines;
+ redraw_chart = 1;
+ }
+
+ slabdebug(" Read %lu lines from procfile", (unsigned long)lines);
+ for(l = 2; l < lines; l++) {
+ if (unlikely(procfile_linewords(ff, l) < 14)) {
+ slabdebug(" Line %zu has only %zu words, skipping", l, procfile_linewords(ff,l));
+ continue;
+ }
+
+ char *name = procfile_lineword(ff, l, 0);
+ struct slabinfo *s = get_slabstruct(name);
+
+ s->active_objs = str2uint64_t(procfile_lineword(ff, l, 1));
+ s->num_objs = str2uint64_t(procfile_lineword(ff, l, 2));
+ s->obj_size = str2uint64_t(procfile_lineword(ff, l, 3));
+ s->obj_per_slab = str2uint64_t(procfile_lineword(ff, l, 4));
+ s->pages_per_slab = str2uint64_t(procfile_lineword(ff, l, 5));
+
+ s->tune_limit = str2uint64_t(procfile_lineword(ff, l, 7));
+ s->tune_batchcnt = str2uint64_t(procfile_lineword(ff, l, 8));
+ s->tune_shared_factor = str2uint64_t(procfile_lineword(ff, l, 9));
+
+ s->data_active_slabs = str2uint64_t(procfile_lineword(ff, l, 11));
+ s->data_num_slabs = str2uint64_t(procfile_lineword(ff, l, 12));
+ s->data_shared_avail = str2uint64_t(procfile_lineword(ff, l, 13));
+
+ uint32_t memperslab = s->pages_per_slab * slab_pagesize;
+ // Internal fragmentation: loss per slab, due to objects not being a multiple of pagesize
+ //uint32_t lossperslab = memperslab - s->obj_per_slab * s->obj_size;
+
+ // Total usage = slabs * pages per slab * page size
+ s->mem_usage = (uint64_t)(s->data_num_slabs * memperslab);
+
+ // Wasted memory (filling): slabs allocated but not filled: sum total slab - sum total objects
+ s->mem_waste = s->mem_usage - (uint64_t)(s->active_objs * s->obj_size);
+ //if (s->data_num_slabs > 1)
+ // s->mem_waste += s->data_num_slabs * lossperslab;
+
+
+ // Slab filling efficiency
+ if (s->num_objs > 0)
+ s->obj_filling = 100 * s->active_objs / s->num_objs;
+ else
+ s->obj_filling = 0;
+
+ slabdebug(" Updated slab %s: %"PRIu64" %"PRIu64" %"PRIu64" %"PRIu64" %"PRIu64" / %"PRIu64" %"PRIu64" %"PRIu64" / %"PRIu64" %"PRIu64" %"PRIu64" / %"PRIu64" %"PRIu64" %hhu",
+ name, s->active_objs, s->num_objs, s->obj_size, s->obj_per_slab, s->pages_per_slab,
+ s->tune_limit, s->tune_batchcnt, s->tune_shared_factor,
+ s->data_active_slabs, s->data_num_slabs, s->data_shared_avail,
+ s->mem_usage, s->mem_waste, s->obj_filling);
+ }
+
+ return slabinfo_root;
+}
+
+
+
+unsigned int do_slab_stats(int update_every) {
+
+ static unsigned int loops = 0;
+ struct slabinfo *sactive = NULL, *s = NULL;
+
+ // Main processing loop
+ while (running) {
+
+ sactive = read_file_slabinfo();
+
+ // Init Charts
+ if (unlikely(redraw_chart)) {
+ redraw_chart = 0;
+ // Memory Usage
+ printf("CHART %s.%s '' 'Memory Usage' 'B' '%s' '' line %d %d %s\n"
+ , CHART_TYPE
+ , "slabmemory"
+ , CHART_FAMILY
+ , CHART_PRIO
+ , update_every
+ , PLUGIN_SLABINFO_NAME
+ );
+ for (s = sactive; s; s = s->next) {
+ printf("DIMENSION %s '' absolute 1 1\n", s->name);
+ }
+
+ // Slab active usage (filling)
+ printf("CHART %s.%s '' 'Object Filling' '%%' '%s' '' line %d %d %s\n"
+ , CHART_TYPE
+ , "slabfilling"
+ , CHART_FAMILY
+ , CHART_PRIO + 1
+ , update_every
+ , PLUGIN_SLABINFO_NAME
+ );
+ for (s = sactive; s; s = s->next) {
+ printf("DIMENSION %s '' absolute 1 1\n", s->name);
+ }
+
+ // Memory waste
+ printf("CHART %s.%s '' 'Memory waste' 'B' '%s' '' line %d %d %s\n"
+ , CHART_TYPE
+ , "slabwaste"
+ , CHART_FAMILY
+ , CHART_PRIO + 2
+ , update_every
+ , PLUGIN_SLABINFO_NAME
+ );
+ for (s = sactive; s; s = s->next) {
+ printf("DIMENSION %s '' absolute 1 1\n", s->name);
+ }
+ }
+
+
+ //
+ // Memory usage
+ //
+ printf("BEGIN %s.%s\n"
+ , CHART_TYPE
+ , "slabmemory"
+ );
+ for (s = sactive; s; s = s->next) {
+ printf("SET %s = %"PRIu64"\n"
+ , s->name
+ , s->mem_usage
+ );
+ }
+ printf("END\n");
+
+ //
+ // Slab active usage
+ //
+ printf("BEGIN %s.%s\n"
+ , CHART_TYPE
+ , "slabfilling"
+ );
+ for (s = sactive; s; s = s->next) {
+ printf("SET %s = %u\n"
+ , s->name
+ , s->obj_filling
+ );
+ }
+ printf("END\n");
+
+ //
+ // Memory waste
+ //
+ printf("BEGIN %s.%s\n"
+ , CHART_TYPE
+ , "slabwaste"
+ );
+ for (s = sactive; s; s = s->next) {
+ printf("SET %s = %"PRIu64"\n"
+ , s->name
+ , s->mem_waste
+ );
+ }
+ printf("END\n");
+
+
+ loops++;
+
+ sleep(update_every);
+ }
+
+ return loops;
+}
+
+
+
+
+// ----------------------------------------------------------------------------
+// main
+
+void usage(void) {
+ fprintf(stderr, "%s\n", program_name);
+ exit(1);
+}
+
+int main(int argc, char **argv) {
+ clocks_init();
+
+ program_name = argv[0];
+ program_version = "0.1";
+ error_log_syslog = 0;
+
+ int update_every = 1, i, n, freq = 0;
+
+ for (i = 1; i < argc; i++) {
+ // Frequency parsing
+ if(isdigit(*argv[i]) && !freq) {
+ n = (int) str2l(argv[i]);
+ if (n > 0) {
+ if (n >= UPDATE_EVERY_MAX) {
+ error("Invalid interval value: %s", argv[i]);
+ exit(1);
+ }
+ freq = n;
+ }
+ }
+ else if (strcmp("debug", argv[i]) == 0) {
+ debug = 1;
+ continue;
+ }
+ else {
+ fprintf(stderr,
+ "netdata slabinfo.plugin %s\n"
+ "This program is a data collector plugin for netdata.\n"
+ "\n"
+ "Available command line options:\n"
+ "\n"
+ " COLLECTION_FREQUENCY data collection frequency in seconds\n"
+ " minimum: %d\n"
+ "\n"
+ " debug enable verbose output\n"
+ " default: disabled\n"
+ "\n",
+ program_version,
+ update_every
+ );
+ exit(1);
+ }
+ }
+
+ if(freq >= update_every)
+ update_every = freq;
+ else if(freq)
+ error("update frequency %d seconds is too small for slabinfo. Using %d.", freq, update_every);
+
+
+ // Call the main function. Time drift to be added
+ do_slab_stats(update_every);
+
+ return 0;
+}
diff --git a/collectors/statsd.plugin/Makefile.am b/collectors/statsd.plugin/Makefile.am
new file mode 100644
index 0000000..c8144c1
--- /dev/null
+++ b/collectors/statsd.plugin/Makefile.am
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
+
+statsdconfigdir=$(libconfigdir)/statsd.d
+dist_statsdconfig_DATA = \
+ example.conf \
+ k6.conf \
+ asterisk.conf \
+ $(NULL)
+
+userstatsdconfigdir=$(configdir)/statsd.d
+dist_userstatsdconfig_DATA = \
+ $(NULL)
+
+# Explicitly install directories to avoid permission issues due to umask
+install-exec-local:
+ $(INSTALL) -d $(DESTDIR)$(userstatsdconfigdir)
diff --git a/collectors/statsd.plugin/README.md b/collectors/statsd.plugin/README.md
new file mode 100644
index 0000000..b46ca28
--- /dev/null
+++ b/collectors/statsd.plugin/README.md
@@ -0,0 +1,699 @@
+<!--
+title: "statsd.plugin"
+description: "The Netdata Agent is a fully-featured StatsD server that collects metrics from any custom application and visualizes them in real-time."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/statsd.plugin/README.md
+-->
+
+StatsD is a system to collect data from any application. Applications send metrics to it, usually via non-blocking UDP communication, and StatsD servers collect these metrics, perform a few simple calculations on them and push them to backend time-series databases.
+
+If you want to learn more about the StatsD protocol, we have written a [blog post](https://www.netdata.cloud/blog/introduction-to-statsd/) about it!
+
+
+Netdata is a fully featured statsd server. It can collect statsd formatted metrics, visualize them on its dashboards and store them in it's database for long-term retention.
+
+Netdata statsd is inside Netdata (an internal plugin, running inside the Netdata daemon), it is configured via `netdata.conf` and by-default listens on standard statsd port 8125. Netdata supports both TCP and UDP packets at the same time.
+
+Since statsd is embedded in Netdata, it means you now have a statsd server embedded on all your servers.
+
+Netdata statsd is fast. It can collect several millions of metrics per second on modern hardware, using just 1 CPU core. The implementation uses two threads: one thread collects metrics, another thread updates the charts from the collected data.
+
+## Available StatsD synthetic application charts
+
+Netdata ships with a few synthetic chart definitions to automatically present application metrics into a more uniform way. These synthetic charts are configuration files (you can create your own) that re-arrange statsd metrics into a more meaningful way.
+
+On synthetic charts, we can have alarms as with any metric and chart.
+
+- [K6 load testing tool](https://k6.io)
+ - **Description:** k6 is a developer-centric, free and open-source load testing tool built for making performance testing a productive and enjoyable experience.
+ - [Documentation](/collectors/statsd.plugin/k6.md)
+ - [Configuration](https://github.com/netdata/netdata/blob/master/collectors/statsd.plugin/k6.conf)
+- [Asterisk](https://www.asterisk.org/)
+ - **Description:** Asterisk is an Open Source PBX and telephony toolkit.
+ - [Documentation](/collectors/statsd.plugin/asterisk.md)
+ - [Configuration](https://github.com/netdata/netdata/blob/master/collectors/statsd.plugin/asterisk.conf)
+
+## Metrics supported by Netdata
+
+Netdata fully supports the StatsD protocol and also extends it to support more advanced Netdata specific use cases. All StatsD client libraries can be used with Netdata too.
+
+- **Gauges**
+
+ The application sends `name:value|g`, where `value` is any **decimal/fractional** number, StatsD reports the latest value collected and the number of times it was updated (events).
+
+ The application may increment or decrement a previous value, by setting the first character of the value to `+` or `-` (so, the only way to set a gauge to an absolute negative value, is to first set it to zero).
+
+ [Sampling rate](#sampling-rates) is supported.
+ [Tags](#tags) are supported for changing chart units, family and dimension name.
+
+ When a gauge is not collected and the setting is not to show gaps on the charts (the default), the last value will be shown, until a data collection event changes it.
+
+- **Counters** and **Meters**
+
+ The application sends `name:value|c`, `name:value|C` or `name:value|m`, where `value` is a positive or negative **integer** number of events occurred, StatsD reports the **rate** and the number of times it was updated (events).
+
+ `:value` can be omitted and StatsD will assume it is `1`. `|c`, `|C` and `|m` can be omitted and StatsD will assume it is `|m`. So, the application may send just `name` and StatsD will parse it as `name:1|m`.
+
+ - Counters use `|c` (etsy/StatsD compatible) or `|C` (brubeck compatible)
+ - Meters use `|m`
+
+ [Sampling rate](#sampling-rates) is supported.
+ [Tags](#tags) are supported for changing chart units, family and dimension name.
+
+ When a counter or meter is not collected, StatsD **defaults** to showing a zero value, until a data collection event changes the value.
+
+- **Timers** and **Histograms**
+
+ The application sends `name:value|ms` or `name:value|h`, where `value` is any **decimal/fractional** number, StatsD reports **min**, **max**, **average**, **95th percentile**, **median** and **standard deviation** and the total number of times it was updated (events). Internally it also calculates the **sum**, which is available for synthetic charts.
+
+ - Timers use `|ms`
+ - Histograms use `|h`
+
+ The only difference between the two, is the `units` of the charts, as timers report *milliseconds*.
+
+ [Sampling rate](#sampling-rates) is supported.
+ [Tags](#tags) are supported for changing chart units and family.
+
+ When a counter or meter is not collected, StatsD **defaults** to showing a zero value, until a data collection event changes the value.
+
+- **Sets**
+
+ The application sends `name:value|s`, where `value` is anything (**number or text**, leading and trailing spaces are removed), StatsD reports the number of unique values sent and the number of times it was updated (events).
+
+ Sampling rate is **not** supported for Sets. `value` is always considered text (so `01` and `1` are considered different).
+
+ [Tags](#tags) are supported for changing chart units and family.
+
+ When a set is not collected, Netdata **defaults** to showing a zero value, until a data collection event changes the value.
+
+- **Dictionaries**
+
+ The application sends `name:value|d`, where `value` is anything (**number or text**, leading and trailing spaces are removed), StatsD reports the number of events sent for each `value` and the total times `name` was updated (events).
+
+ Sampling rate is **not** supported for Dictionaries. `value` is always considered text (so `01` and `1` are considered different).
+
+ [Tags](#tags) are supported for changing chart units and family.
+
+ When a set is not collected, Netdata **defaults** to showing a zero value, until a data collection event changes the value.
+
+#### Sampling Rates
+
+The application may append `|@sampling_rate`, where `sampling_rate` is a number from `0.0` to `1.0` in order for StatD to extrapolate the value and predict the total for the entire period. If the application reports to StatsD a value for 1/10th of the time, it can append `|@0.1` to the metrics it sends to statsd.
+
+#### Tags
+
+The application may append `|#tag1:value1,tag2:value2,tag3:value3` etc, where `tagX` and `valueX` are strings. `:valueX` can be omitted.
+
+Currently, Netdata uses only 2 tags:
+
+ * `units=string` which sets the units of the chart that is automatically generated
+ * `family=string` which sets the family of the chart that is automatically generated (the family is the submenu of the dashboard)
+ * `name=string` which sets the name of the dimension of the chart that is automatically generated (only for counters, meters, gauges)
+
+Other tags are parsed, but currently are ignored.
+
+Charts are not updated to change units or dimension names once they are created. So, either send the tags on every event, or use the special `zinit` value to initiaze the charts at the beginning. `zinit` is a special value that can be used on any chart, to have netdata initialize the charts, without actually setting any values to them. So, instead of sending `my.metric:VALUE|c|#units=bytes,name=size` every time, the application can send at the beginning `my.metric:zinit|c|#units=bytes,name=size` and then `my.metric:VALUE|c`.
+
+#### Overlapping metrics
+
+Netdata's StatsD server maintains different indexes for each of the metric types supported. This means the same metric `name` may exist under different types concurrently.
+
+#### How to name your metrics
+
+A good practice is to name your metrics like `application.operation.metric`, where:
+
+- `application` is the application name - Netdata will automatically create a dashboard section based on the first keyword of the metrics, so you can have all your applications in different sections.
+- `operation` is the operation your application is executing, like `dbquery`, `request`, `response`, etc.
+- `metric` is anything you want to name your metric as. Netdata will automatically append the metric type (meter, counter, gauge, set, dictionary, timer, histogram) to the generated chart.
+
+Using [Tags](#tags) you can also change the submenus of the dashboard, the units of the charts and for meters, counters and gauges, the name of dimension. So, you can have a usable default view without using [Synthetic StatsD charts](#synthetic-statsd-charts)
+
+#### Multiple metrics per packet
+
+Netdata accepts multiple metrics per packet if each is terminated with a newline (`\n`) at the end.
+
+#### TCP packets
+
+Netdata listens for both TCP and UDP packets. For TCP, is it important to always append `\n` on each metric, as Netdata will use the newline character to detect if a metric is split into multiple TCP packets.
+
+
+#### UDP packets
+
+When sending multiple metrics over a single UDP message, it is important not to exceed the network MTU, which is usually 1500 bytes.
+
+Netdata will accept UDP packets up to 9000 bytes, but the underlying network will not exceed MTU.
+
+> You can read more about the network maximum transmission unit(MTU) in this cloudflare [article](https://www.cloudflare.com/en-gb/learning/network-layer/what-is-mtu/).
+
+## Configuration
+
+You can find the configuration at `/etc/netdata/netdata.conf`:
+
+```
+[statsd]
+ # enabled = yes
+ # decimal detail = 1000
+ # update every (flushInterval) = 1
+ # udp messages to process at once = 10
+ # create private charts for metrics matching = *
+ # max private charts allowed = 200
+ # max private charts hard limit = 1000
+ # private charts memory mode = save
+ # private charts history = 3996
+ # histograms and timers percentile (percentThreshold) = 95.00000
+ # add dimension for number of events received = no
+ # gaps on gauges (deleteGauges) = no
+ # gaps on counters (deleteCounters) = no
+ # gaps on meters (deleteMeters) = no
+ # gaps on sets (deleteSets) = no
+ # gaps on histograms (deleteHistograms) = no
+ # gaps on timers (deleteTimers) = no
+ # listen backlog = 4096
+ # default port = 8125
+ # bind to = udp:localhost:8125 tcp:localhost:8125
+```
+
+### StatsD main config options
+
+- `enabled = yes|no`
+
+ controls if StatsD will be enabled for this Netdata. The default is enabled.
+
+- `default port = 8125`
+
+ controls the default port StatsD will use if no port is defined in the following setting.
+
+- `bind to = udp:localhost tcp:localhost`
+
+ is a space separated list of IPs and ports to listen to. The format is `PROTOCOL:IP:PORT` - if `PORT` is omitted, the `default port` will be used. If `IP` is IPv6, it needs to be enclosed in `[]`. `IP` can also be `*` (to listen on all IPs) or even a hostname.
+
+- `update every (flushInterval) = 1` seconds, controls the frequency StatsD will push the collected metrics to Netdata charts.
+
+- `decimal detail = 1000` controls the number of fractional digits in gauges and histograms. Netdata collects metrics using signed 64-bit integers and their fractional detail is controlled using multipliers and divisors. This setting is used to multiply all collected values to convert them to integers and is also set as the divisors, so that the final data will be a floating point number with this fractional detail (1000 = X.0 - X.999, 10000 = X.0 - X.9999, etc).
+
+The rest of the settings are discussed below.
+
+## StatsD charts
+
+Netdata can visualize StatsD collected metrics in 2 ways:
+
+1. Each metric gets its own **private chart**. This is the default and does not require any configuration. You can adjust the default parameters.
+
+2. **Synthetic charts** can be created, combining multiple metrics, independently of their metric types. For this type of charts, special configuration is required, to define the chart title, type, units, its dimensions, etc.
+
+### Private metric charts
+
+Private charts are controlled with `create private charts for metrics matching = *`. This setting accepts a space-separated list of [simple patterns](/libnetdata/simple_pattern/README.md). Netdata will create private charts for all metrics **by default**.
+
+For example, to render charts for all `myapp.*` metrics, except `myapp.*.badmetric`, use:
+
+```
+create private charts for metrics matching = !myapp.*.badmetric myapp.*
+```
+
+You can specify Netdata StatsD to have a different `memory mode` than the rest of the Netdata Agent. You can read more about `memory mode` in the [documentation](/database/README.md).
+
+The default behavior is to use the same settings as the rest of the Netdata Agent. If you wish to change them, edit the following settings:
+- `private charts memory mode`
+- `private charts history`
+
+### Optimize private metric charts visualization and storage
+
+If you have thousands of metrics, each with its own private chart, you may notice that your web browser becomes slow when you view the Netdata dashboard (this is a web browser issue we need to address at the Netdata UI). So, Netdata has a protection to stop creating charts when `max private charts allowed = 200` (soft limit) is reached.
+
+The metrics above this soft limit are still processed by Netdata, can be used in synthetic charts and will be available to be sent to backend time-series databases, up to `max private charts hard limit = 1000`. So, between 200 and 1000 charts, Netdata will still generate charts, but they will automatically be created with `memory mode = none` (Netdata will not maintain a database for them). These metrics will be sent to backend time series databases, if the backend configuration is set to `as collected`.
+
+Metrics above the hard limit are still collected, but they can only be used in synthetic charts (once a metric is added to chart, it will be sent to backend servers too).
+
+Example private charts (automatically generated without any configuration):
+
+#### Counters
+
+- Scope: **count the events of something** (e.g. number of file downloads)
+- Format: `name:INTEGER|c` or `name:INTEGER|C` or `name|c`
+- StatsD increments the counter by the `INTEGER` number supplied (positive, or negative).
+
+![image](https://cloud.githubusercontent.com/assets/2662304/26131553/4a26d19c-3aa3-11e7-94e8-c53b5ed6ebc3.png)
+
+#### Gauges
+
+- Scope: **report the value of something** (e.g. cache memory used by the application server)
+- Format: `name:FLOAT|g`
+- StatsD remembers the last value supplied, and can increment or decrement the latest value if `FLOAT` begins with `+` or `-`.
+
+![image](https://cloud.githubusercontent.com/assets/2662304/26131575/5d54e6f0-3aa3-11e7-9099-bc4440cd4592.png)
+
+#### histograms
+
+- Scope: **statistics on a size of events** (e.g. statistics on the sizes of files downloaded)
+- Format: `name:FLOAT|h`
+- StatsD maintains a list of all the values supplied and provides statistics on them.
+
+![image](https://cloud.githubusercontent.com/assets/2662304/26131587/704de72a-3aa3-11e7-9ea9-0d2bb778c150.png)
+
+The same chart with `sum` unselected, to show the detail of the dimensions supported:
+![image](https://cloud.githubusercontent.com/assets/2662304/26131598/8076443a-3aa3-11e7-9ffa-ea535aee9c9f.png)
+
+#### Meters
+
+This is identical to `counter`.
+
+- Scope: **count the events of something** (e.g. number of file downloads)
+- Format: `name:INTEGER|m` or `name|m` or just `name`
+- StatsD increments the counter by the `INTEGER` number supplied (positive, or negative).
+
+![image](https://cloud.githubusercontent.com/assets/2662304/26131605/8fdf5a06-3aa3-11e7-963f-7ecf207d1dbc.png)
+
+#### Sets
+
+- Scope: **count the unique occurrences of something** (e.g. unique filenames downloaded, or unique users that downloaded files)
+- Format: `name:TEXT|s`
+- StatsD maintains a unique index of all values supplied, and reports the unique entries in it.
+
+![image](https://cloud.githubusercontent.com/assets/2662304/26131612/9eaa7b1a-3aa3-11e7-903b-d881e9a35be2.png)
+
+#### Timers
+
+- Scope: **statistics on the duration of events** (e.g. statistics for the duration of file downloads)
+- Format: `name:FLOAT|ms`
+- StatsD maintains a list of all the values supplied and provides statistics on them.
+
+![image](https://cloud.githubusercontent.com/assets/2662304/26131629/bc34f2d2-3aa3-11e7-8a07-f2fc94ba4352.png)
+
+### Synthetic StatsD charts
+
+Use synthetic charts to create dedicated sections on the dashboard to render your StatsD charts.
+
+Synthetic charts are organized in
+
+- **application** aka section in Netdata Dashboard.
+- **charts for each application** aka family in Netdata Dashboard.
+- **StatsD metrics for each chart** /aka charts and context Netdata Dashboard.
+
+> You can read more about how the Netdata Agent organizes information in the relevant [documentation](/web/README.md)
+
+For each application you need to create a `.conf` file in `/etc/netdata/statsd.d`.
+
+For example, if you want to monitor the application `myapp` using StatsD and Netdata, create the file `/etc/netdata/statsd.d/myapp.conf`, with this content:
+```
+[app]
+ name = myapp
+ metrics = myapp.*
+ private charts = no
+ gaps when not collected = no
+ history = 60
+# memory mode = ram
+
+[dictionary]
+ m1 = metric1
+ m2 = metric2
+
+# replace 'mychart' with the chart id
+# the chart will be named: myapp.mychart
+[mychart]
+ name = mychart
+ title = my chart title
+ family = my family
+ context = chart.context
+ units = tests/s
+ priority = 91000
+ type = area
+ dimension = myapp.metric1 m1
+ dimension = myapp.metric2 m2
+```
+
+Using the above configuration `myapp` should get its own section on the dashboard, having one chart with 2 dimensions.
+
+`[app]` starts a new application definition. The supported settings in this section are:
+
+- `name` defines the name of the app.
+- `metrics` is a Netdata [simple pattern](/libnetdata/simple_pattern/README.md). This pattern should match all the possible StatsD metrics that will be participating in the application `myapp`.
+- `private charts = yes|no`, enables or disables private charts for the metrics matched.
+- `gaps when not collected = yes|no`, enables or disables gaps on the charts of the application in case that no metrics are collected.
+- `memory mode` sets the memory mode for all charts of the application. The default is the global default for Netdata (not the global default for StatsD private charts). We suggest not to use this (we have commented it out in the example) and let your app use the global default for Netdata, which is our dbengine.
+
+- `history` sets the size of the round robin database for this application. The default is the global default for Netdata (not the global default for StatsD private charts). This is only relevant if you use `memory mode = save`. Read more on our [metrics storage(]/docs/store/change-metrics-storage.md) doc.
+
+`[dictionary]` defines name-value associations. These are used to renaming metrics, when added to synthetic charts. Metric names are also defined at each `dimension` line. However, using the dictionary dimension names can be declared globally, for each app and is the only way to rename dimensions when using patterns. Of course the dictionary can be empty or missing.
+
+Then, add any number of charts. Each chart should start with `[id]`. The chart will be called `app_name.id`. `family` controls the submenu on the dashboard. `context` controls the alarm templates. `priority` controls the ordering of the charts on the dashboard. The rest of the settings are informational.
+
+Add any number of metrics to a chart, using `dimension` lines. These lines accept 5 space separated parameters:
+
+1. the metric name, as it is collected (it has to be matched by the `metrics =` pattern of the app)
+2. the dimension name, as it should be shown on the chart
+3. an optional selector (type) of the value to shown (see below)
+4. an optional multiplier
+5. an optional divider
+6. optional flags, space separated and enclosed in quotes. All the external plugins `DIMENSION` flags can be used. Currently the only usable flag is `hidden`, to add the dimension, but not show it on the dashboard. This is usually needed to have the values available for percentage calculation, or use them in alarms.
+
+So, the format is this:
+
+```
+dimension = [pattern] METRIC NAME TYPE MULTIPLIER DIVIDER OPTIONS
+```
+
+`pattern` is a keyword. When set, `METRIC` is expected to be a Netdata [simple pattern](/libnetdata/simple_pattern/README.md) that will be used to match all the StatsD metrics to be added to the chart. So, `pattern` automatically matches any number of StatsD metrics, all of which will be added as separate chart dimensions.
+
+`TYPE`, `MULTIPLIER`, `DIVIDER` and `OPTIONS` are optional.
+
+`TYPE` can be:
+
+- `events` to show the number of events received by StatsD for this metric
+- `last` to show the last value, as calculated at the flush interval of the metric (the default)
+
+Then for histograms and timers the following types are also supported:
+
+- `min`, show the minimum value
+- `max`, show the maximum value
+- `sum`, show the sum of all values
+- `average` (same as `last`)
+- `percentile`, show the 95th percentile (or any other percentile, as configured at StatsD global config)
+- `median`, show the median of all values (i.e. sort all values and get the middle value)
+- `stddev`, show the standard deviation of the values
+
+#### Example synthetic charts
+
+StatsD metrics: `foo` and `bar`.
+
+Contents of file `/etc/netdata/stats.d/foobar.conf`:
+
+```
+[app]
+ name = foobarapp
+ metrics = foo bar
+ private charts = yes
+
+[foobar_chart1]
+ title = Hey, foo and bar together
+ family = foobar_family
+ context = foobarapp.foobars
+ units = foobars
+ type = area
+ dimension = foo 'foo me' last 1 1
+ dimension = bar 'bar me' last 1 1
+```
+
+Metrics sent to statsd: `foo:10|g` and `bar:20|g`.
+
+Private charts:
+
+![screenshot from 2017-08-03 23-28-19](https://user-images.githubusercontent.com/2662304/28942295-7c3a73a8-78a3-11e7-88e5-a9a006bb7465.png)
+
+Synthetic chart:
+
+![screenshot from 2017-08-03 23-29-14](https://user-images.githubusercontent.com/2662304/28942317-958a2c68-78a3-11e7-853f-32850141dd36.png)
+
+#### Renaming StatsD synthetic charts' metrics
+
+You can define a dictionary to rename metrics sent by StatsD clients. This enables you to send response `"200"` and Netdata visualize it as `succesful connection`
+
+The `[dictionary]` section accepts any number of `name = value` pairs.
+
+Netdata uses this dictionary as follows:
+
+1. When a `dimension` has a non-empty `NAME`, that name is looked up at the dictionary.
+
+2. If the above lookup gives nothing, or the `dimension` has an empty `NAME`, the original StatsD metric name is looked up at the dictionary.
+
+3. If any of the above succeeds, Netdata uses the `value` of the dictionary, to set the name of the dimension. The dimensions will have as ID the original StatsD metric name, and as name, the dictionary value.
+
+Use the dictionary in 2 ways:
+
+1. set `dimension = myapp.metric1 ''` and have at the dictionary `myapp.metric1 = metric1 name`
+2. set `dimension = myapp.metric1 'm1'` and have at the dictionary `m1 = metric1 name`
+
+In both cases, the dimension will be added with ID `myapp.metric1` and will be named `metric1 name`. So, in alarms use either of the 2 as `${myapp.metric1}` or `${metric1 name}`.
+
+> keep in mind that if you add multiple times the same StatsD metric to a chart, Netdata will append `TYPE` to the dimension ID, so `myapp.metric1` will be added as `myapp.metric1_last` or `myapp.metric1_events`, etc. If you add multiple times the same metric with the same `TYPE` to a chart, Netdata will also append an incremental counter to the dimension ID, i.e. `myapp.metric1_last1`, `myapp.metric1_last2`, etc.
+
+#### Dimension patterns
+
+Netdata allows adding multiple dimensions to a chart, by matching the StatsD metrics with a Netdata simple pattern.
+
+Assume we have an API that provides StatsD metrics for each response code per method it supports, like these:
+
+```
+myapp.api.get.200
+myapp.api.get.400
+myapp.api.get.500
+myapp.api.del.200
+myapp.api.del.400
+myapp.api.del.500
+myapp.api.post.200
+myapp.api.post.400
+myapp.api.post.500
+myapp.api.all.200
+myapp.api.all.400
+myapp.api.all.500
+```
+
+In order to add all the response codes of `myapp.api.get` to a chart, we simply make the following configuration:
+
+```
+[api_get_responses]
+ ...
+ dimension = pattern 'myapp.api.get.* '' last 1 1
+```
+
+The above will add dimension named `200`, `400` and `500`. Netdata extracts the wildcard part of the metric name - so the dimensions will be named with whatever the `*` matched.
+
+You can rename the dimensions with this:
+
+```
+[dictionary]
+ get.200 = 200 ok
+ get.400 = 400 bad request
+ get.500 = 500 cannot connect to db
+
+[api_get_responses]
+ ...
+ dimension = pattern 'myapp.api.get.* 'get.' last 1 1
+```
+
+Note that we added a `NAME` to the dimension line with `get.`. This is prefixed to the wildcarded part of the metric name, to compose the key for looking up the dictionary. So `500` became `get.500` which was looked up to the dictionary to find value `500 cannot connect to db`. This way we can have different dimension names, for each of the API methods (i.e. `get.500 = 500 cannot connect to db` while `post.500 = 500 cannot write to disk`).
+
+To add all 200s across all API methods to a chart, you can do this:
+
+```
+[ok_by_method]
+ ...
+ dimension = pattern 'myapp.api.*.200 '' last 1 1
+```
+
+The above will add `get`, `post`, `del` and `all` to the chart.
+
+If `all` is not wanted (a `stacked` chart does not need the `all` dimension, since the sum of the dimensions provides the total), the line should be:
+
+```
+[ok_by_method]
+ ...
+ dimension = pattern '!myapp.api.all.* myapp.api.*.200 '' last 1 1
+```
+
+With the above, all methods except `all` will be added to the chart.
+
+To automatically rename the methods, you can use this:
+
+```
+[dictionary]
+ method.get = GET
+ method.post = ADD
+ method.del = DELETE
+
+[ok_by_method]
+ ...
+ dimension = pattern '!myapp.api.all.* myapp.api.*.200 'method.' last 1 1
+```
+
+Using the above, the dimensions will be added as `GET`, `ADD` and `DELETE`.
+
+## StatsD examples
+
+### Python
+
+It's really easy to instrument your python application with StatsD, for example using [jsocol/pystatsd](https://github.com/jsocol/pystatsd).
+
+```python
+import statsd
+c = statsd.StatsClient('localhost', 8125)
+c.incr('foo') # Increment the 'foo' counter.
+for i in range(100000000):
+ c.incr('bar')
+ c.incr('foo')
+ if i % 3:
+ c.decr('bar')
+ c.timing('stats.timed', 320) # Record a 320ms 'stats.timed'.
+```
+
+You can find detailed documentation in their [documentation page](https://statsd.readthedocs.io/en/v3.3/).
+
+### Javascript and Node.js
+
+Using the client library by [sivy/node-statsd](https://github.com/sivy/node-statsd), you can easily embed StatsD into your Node.js project.
+
+```javascript
+ var StatsD = require('node-statsd'),
+ client = new StatsD();
+
+ // Timing: sends a timing command with the specified milliseconds
+ client.timing('response_time', 42);
+
+ // Increment: Increments a stat by a value (default is 1)
+ client.increment('my_counter');
+
+ // Decrement: Decrements a stat by a value (default is -1)
+ client.decrement('my_counter');
+
+ // Using the callback
+ client.set(['foo', 'bar'], 42, function(error, bytes){
+ //this only gets called once after all messages have been sent
+ if(error){
+ console.error('Oh noes! There was an error:', error);
+ } else {
+ console.log('Successfully sent', bytes, 'bytes');
+ }
+ });
+
+ // Sampling, tags and callback are optional and could be used in any combination
+ client.histogram('my_histogram', 42, 0.25); // 25% Sample Rate
+ client.histogram('my_histogram', 42, ['tag']); // User-defined tag
+ client.histogram('my_histogram', 42, next); // Callback
+ client.histogram('my_histogram', 42, 0.25, ['tag']);
+ client.histogram('my_histogram', 42, 0.25, next);
+ client.histogram('my_histogram', 42, ['tag'], next);
+ client.histogram('my_histogram', 42, 0.25, ['tag'], next);
+```
+### Other languages
+
+You can also use StatsD with:
+- Golang, thanks to [alexcesaro/statsd](https://github.com/alexcesaro/statsd)
+- Ruby, thanks to [reinh/statsd](https://github.com/reinh/statsd)
+- Java, thanks to [DataDog/java-dogstatsd-client](https://github.com/DataDog/java-dogstatsd-client)
+
+
+### Shell
+
+Getting the proper support for a programming language is not always easy, but the Unix shell is available on most Unix systems. You can use shell and `nc` to instrument your systems and send metric data to Netdata's StatsD implementation.
+
+Using the method you can send metrics from any script. You can generate events like: backup.started, backup.ended, backup.time, or even tail logs and convert them to metrics.
+
+> **IMPORTANT**:
+>
+> To send StatsD messages you need from the `netcat` package, the `nc` command.
+> There are multiple versions of this package. Please try to experiment with the `nc` command you have available on your right system, to find the right parameters.
+>
+> In the examples below, we assume the `openbsd-netcat` is installed.
+
+If you plan to send short StatsD events at sporadic occasions, use UDP. The messages should not be too long (remember, most networks support up to 1500 bytes MTU, which is also the limit for StatsD messages over UDP). The good thing is that using UDP will not block your script, even if the StatsD server is not there (UDP messages are "fire-and-forget").
+
+
+For UDP use this:
+
+```sh
+echo "APPLICATION.METRIC:VALUE|TYPE" | nc -u -w 0 localhost 8125
+```
+
+`-u` turns on UDP, `-w 0` tells `nc` not to wait for a response from StatsD (idle time to close the connection).
+
+where:
+
+- `APPLICATION` is any name for your application
+- `METRIC` is the name for the specific metric
+- `VALUE` is the value for that metric (**meters**, **counters**, **gauges**, **timers** and **histograms** accept integer/decimal/fractional numbers, **sets** and **dictionaries** accept strings)
+- `TYPE` is one of `m`, `c`, `g`, `ms`, `h`, `s`, `d` to define the metric type.
+
+For tailing a log and converting it to metrics, do something like this:
+
+```sh
+tail -f some.log | awk 'awk commands to parse the log and format statsd metrics' | nc -N -w 120 localhost 8125
+```
+
+`-N` tells `nc` to close the socket once it receives EOF on its input. `-w 120` tells `nc` to stop if the connection is idle for 120 seconds. The timeout is needed to stop the `nc` command if you restart Netdata while `nc` is connected to it. Without it, `nc` will sit idle forever.
+
+When you embed the above commands to a script, you may notice that all the metrics are sent to StatsD with a delay. They are buffered in the pipes `|`. You can turn them to real-time by prepending each command with `stdbuf -i0 -oL -eL command to be run`, like this:
+
+```sh
+stdbuf -i0 -oL -eL tail -f some.log |\
+ stdbuf -i0 -oL -eL awk 'awk commands to parse the log and format statsd metrics' |\
+ stdbuf -i0 -oL -eL nc -N -w 120 localhost 8125
+```
+
+If you use `mawk` you also need to run awk with `-W interactive`.
+
+Examples:
+
+To set `myapp.used_memory` as gauge to value `123456`, use:
+
+```sh
+echo "myapp.used_memory:123456|g|#units:bytes" | nc -u -w 0 localhost 8125
+```
+
+To increment `myapp.files_sent` by `10`, as a counter, use:
+
+```sh
+echo "myapp.files_sent:10|c|#units:files" | nc -u -w 0 localhost 8125
+```
+
+You can send multiple metrics like this:
+
+```sh
+# send multiple metrics via UDP
+printf "myapp.used_memory:123456|g|#units:bytes\nmyapp.files_sent:10|c|#units:files\n" | nc -u -w 0 localhost 8125
+```
+
+Remember, for UDP communication each packet should not exceed the MTU. So, if you plan to push too many metrics at once, prefer TCP communication:
+
+```sh
+# send multiple metrics via TCP
+cat /tmp/statsd.metrics.txt | nc -N -w 120 localhost 8125
+```
+
+You can also use this little function to take care of all the details:
+
+```sh
+#!/usr/bin/env bash
+
+# we assume nc is from the openbsd-netcat package
+
+STATSD_HOST="localhost"
+STATSD_PORT="8125"
+statsd() {
+ local options="-u -w 0" all="${*}"
+
+ # replace all spaces with newlines
+ all="${all// /\\n}"
+
+ # if the string length of all parameters given is above 1000, use TCP
+ [ "${#all}" -gt 1000 ] && options="-N -w 0"
+
+ # send the metrics to statsd
+ printf "${all}\n" | nc ${options} ${STATSD_HOST} ${STATSD_PORT} || return 1
+
+ return 0
+}
+
+if [ ! -z "${*}" ]
+then
+ statsd "${@}"
+fi
+```
+
+You can use it like this:
+
+```sh
+# first, source it in your script
+source statsd.sh
+
+# then, at any point:
+statsd "myapp.used_memory:123456|g|#units:bytes" "myapp.files_sent:10|c|#units:files" ...
+```
+
+or even at a terminal prompt, like this:
+
+```sh
+./statsd.sh "myapp.used_memory:123456|g|#units:bytes" "myapp.files_sent:10|c|#units:files" ...
+```
+
+The function is smart enough to call `nc` just once and pass all the metrics to it. It will also automatically switch to TCP if the metrics to send are above 1000 bytes.
+
+If you have gotten thus far, make sure to check out our [community forums](https://community.netdata.cloud) to share your experience using Netdata with StatsD.
diff --git a/collectors/statsd.plugin/asterisk.conf b/collectors/statsd.plugin/asterisk.conf
new file mode 100644
index 0000000..160b80f
--- /dev/null
+++ b/collectors/statsd.plugin/asterisk.conf
@@ -0,0 +1,208 @@
+[app]
+ name = asterisk
+ metrics = asterisk.*
+ private charts = yes
+ gaps when not collected = no
+
+[dictionary]
+ # https://www.voip-info.org/asterisk-variable-hangupcause/
+ q931.1 = unallocated 1
+ q931.2 = no route transit net 2
+ q931.3 = no route destination 3
+ q931.6 = channel unacceptable 6
+ q931.7 = call awarded delivered 7
+ q931.16 = normal 16
+ q931.17 = busy 17
+ q931.18 = no response 18
+ q931.19 = no answer 19
+ q931.21 = rejected call 21
+ q931.22 = number changed 22
+ q931.27 = dst out of order 27
+ q931.28 = invalid number 28
+ q931.29 = rejected facility 29
+ q931.30 = response to status 30
+ q931.31 = normal unspecified 31
+ q931.34 = congestion circuit 34
+ q931.38 = net out of order 38
+ q931.41 = normal tmp fail 41
+ q931.42 = congestion switch 42
+ q931.43 = access info discarded 43
+ q931.44 = requested chan unavail 44
+ q931.45 = pre empted 45
+ q931.47 = resource unavailable, unspecified 47
+ q931.50 = facility not subscribed 50
+ q931.52 = outgoing call barred 52
+ q931.54 = incoming call barred 54
+ q931.57 = bearer capability not auth 57
+ q931.58 = bearer capability not avail 58
+ q931.65 = bearer capability not implemented 65
+ q931.66 = chan not implemented 66
+ q931.69 = facility not implemented 67
+ q931.81 = invalid call reference 81
+ q931.88 = incompatible destination 88
+ q931.95 = invalid msg specified 95
+ q931.96 = mandatory ie missing 96
+ q931.97 = message type non exist 97
+ q931.98 = wrong message 98
+ q931.99 = ie non exist 99
+ q931.100 = invalid ie contents 100
+ q931.101 = wrong call state 101
+ q931.102 = recovery on timer expire 102
+ q931.103 = mandatory ie length error 103
+ q931.111 = protocol error 111
+ q931.127 = interworking 127
+
+
+[channels]
+ name = channels
+ title = Active Channels
+ family = channels
+ context = asterisk.channels
+ units = channels
+ priority = 91000
+ type = stacked
+ dimension = pattern asterisk.channels.count 'channels' last 1 1
+ # FIXME: netdata needs to prevent this from going negative
+
+[endpoints]
+ name = endpoints
+ title = Active Endpoints
+ family = endpoints
+ context = asterisk.endpoints
+ units = endpoints
+ priority = 91005
+ type = stacked
+ dimension = pattern asterisk.endpoints.count 'endpoints' last 1 1
+
+[endpoints_by_status]
+ name = endpoints_by_status
+ title = Active Endpoints by Status
+ family = endpoints
+ context = asterisk.endpoints_by_status
+ units = endpoints
+ priority = 91006
+ type = stacked
+ dimension = pattern asterisk.endpoints.state.* '' last 1 1
+
+[sip_channels_by_endpoint]
+ name = sip_channels_by_endpoint
+ title = Active SIP channels by endpoint
+ family = channels
+ context = asterisk.sip_channels_by_endpoint
+ units = channels
+ priority = 91110
+ type = stacked
+ dimension = pattern asterisk.endpoints.SIP.*.channels '' last 1 1
+
+[pjsip_channels_by_endpoint]
+ name = pjsip_channels_by_endpoint
+ title = Active PJSIP channels by endpoint
+ family = channels
+ context = asterisk.pjsip_channels_by_endpoint
+ units = channels
+ priority = 91111
+ type = stacked
+ dimension = pattern asterisk.endpoints.PJSIP.*.channels '' last 1 1
+
+[dialstatuses]
+ name = dialstatuses
+ title = Distribution of Dial Statuses
+ family = dial_statuses
+ context = asterisk.dialstatus
+ units = calls
+ priority = 91150
+ type = stacked
+ dimension = pattern 'asterisk.dialstatus.*' '' last 1 1
+
+[calltime]
+ name = calltime
+ title = Asterisk Channels Call Duration
+ family = calltime
+ context = asterisk.calltime
+ units = milliseconds
+ priority = 91160
+ type = stacked
+ dimension = asterisk.channels.calltime 'calltime' average 1 1
+ dimension = asterisk.channels.calltime 'sum' sum 1 1 hidden
+ dimension = asterisk.channels.calltime 'count' events 1 1 hidden
+
+[hangupcause]
+ name = hangupcause
+ title = Distribution of Hangup Causes
+ family = hangup_causes
+ context = asterisk.hangupcause
+ units = calls
+ priority = 91200
+ type = stacked
+ dimension = pattern 'asterisk.hangupcause.*' 'q931.' last 1 1
+
+[hangupcause_answer]
+ name = hangupcause_answer
+ title = Distribution of Hangup Causes for ANSWERed calls
+ family = hangup_causes
+ context = asterisk.hangupcause_answer
+ units = calls
+ priority = 91210
+ type = stacked
+ dimension = pattern 'asterisk.dialhangupcause.ANSWER.*' 'q931.' last 1 1
+
+[hangupcause_busy]
+ name = hangupcause_busy
+ title = Distribution of Hangup Causes for BUSY calls
+ family = hangup_causes
+ context = asterisk.hangupcause_busy
+ units = calls
+ priority = 91215
+ type = stacked
+ dimension = pattern 'asterisk.dialhangupcause.BUSY.*' 'q931.' last 1 1
+
+[hangupcause_cancel]
+ name = hangupcause_cancel
+ title = Distribution of Hangup Causes for CANCELled calls
+ family = hangup_causes
+ context = asterisk.hangupcause_cancel
+ units = calls
+ priority = 91220
+ type = stacked
+ dimension = pattern 'asterisk.dialhangupcause.CANCEL.*' 'q931.' last 1 1
+
+[hangupcause_chanunavail]
+ name = hangupcause_chanunavail
+ title = Distribution of Hangup Causes for CHANUNVAILed calls
+ family = hangup_causes
+ context = asterisk.hangupcause_chanunavail
+ units = calls
+ priority = 91230
+ type = stacked
+ dimension = pattern 'asterisk.dialhangupcause.CHANUNAVAIL.*' 'q931.' last 1 1
+
+[hangupcause_congestion]
+ name = hangupcause_congestion
+ title = Distribution of Hangup Causes for CONGESTIONed calls
+ family = hangup_causes
+ context = asterisk.hangupcause_congestion
+ units = calls
+ priority = 91240
+ type = stacked
+ dimension = pattern 'asterisk.dialhangupcause.CONGESTION.*' 'q931.' last 1 1
+
+[events]
+ name = events
+ title = Asterisk Dialplan Events
+ family = events
+ context = asterisk.events
+ units = events/s
+ priority = 91400
+ type = stacked
+ dimension = pattern 'asterisk.stasis.message.ast_channel_*_type' '' last 1 1
+
+[qualify]
+ name = qualify
+ title = Asterisk PJSIP Peers Qualify
+ family = qualify
+ context = asterisk.qualify
+ units = milliseconds
+ priority = 91500
+ type = stacked
+ dimension = pattern 'asterisk.PJSIP.contacts.*.rtt' '' max 1 1
+ # FIXME: netdata needs to set update every = 15 on this
diff --git a/collectors/statsd.plugin/asterisk.md b/collectors/statsd.plugin/asterisk.md
new file mode 100644
index 0000000..94da94e
--- /dev/null
+++ b/collectors/statsd.plugin/asterisk.md
@@ -0,0 +1,61 @@
+<!--
+title: "Asterisk monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/statsd.plugin/asterisk.md
+
+sidebar_label: "Asterisk"
+-->
+
+# Asterisk monitoring with Netdata
+
+Monitors [Asterisk](https://www.asterisk.org/) dialplan application's statistics.
+
+## Requirements
+
+- Asterisk [integrated with StatsD](https://www.asterisk.org/integrating-asterisk-with-statsd/).
+
+## Configuration
+
+Netdata ships
+with [asterisk.conf](https://github.com/netdata/netdata/blob/master/collectors/statsd.plugin/asterisk.conf) with
+preconfigured charts.
+
+To receive Asterisk metrics in Netdata, uncomment the following lines in the `/etc/asterisk/statsd.conf` file:
+
+```ini
+[general]
+enabled = yes ; When set to yes, statsd support is enabled
+server = 127.0.0.1 ; server[:port] of statsd server to use.
+ ; If not specified, the port is 8125
+prefix = asterisk ; Prefix to prepend to all metrics
+```
+
+> See [statsd.conf.sample](https://github.com/asterisk/asterisk/blob/master/configs/samples/statsd.conf.sample) for all available options.
+
+## Charts and metrics
+
+<details><summary>Click to see screenshots of the charts.</summary>
+
+![image](https://user-images.githubusercontent.com/2662304/158055351-fcc7a7fb-9b95-4656-bdc6-2e5f5a909215.png)
+![image](https://user-images.githubusercontent.com/2662304/158055367-cfd25cd5-d71a-4bab-8cd1-bfcc47bc7312.png)
+
+</details>
+
+Mapping Asterisk StatsD metrics and Netdata charts.
+
+| Chart | Metrics |
+|------------------------------------------------------|--------------------------------------------|
+| Active Channels | asterisk.channels.count |
+| Active Endpoints | asterisk.endpoints.count |
+| Active Endpoints by Status | asterisk.endpoints.state.* |
+| Active SIP channels by endpoint | asterisk.endpoints.SIP.*.channels |
+| Active PJSIP channels by endpoint | asterisk.endpoints.PJSIP.*.channels |
+| Distribution of Dial Statuses | asterisk.dialstatus.* |
+| Asterisk Channels Call Duration | asterisk.channels.calltime |
+| Distribution of Hangup Causes | asterisk.hangupcause.* |
+| Distribution of Hangup Causes for ANSWERed calls | asterisk.dialhangupcause.ANSWER.* |
+| Distribution of Hangup Causes for BUSY calls | asterisk.dialhangupcause.BUSY.* |
+| Distribution of Hangup Causes for CANCELled calls | asterisk.dialhangupcause.CANCEL.* |
+| Distribution of Hangup Causes for CHANUNVAILed calls | asterisk.dialhangupcause.CHANUNAVAIL.* |
+| Distribution of Hangup Causes for CONGESTIONed calls | asterisk.dialhangupcause.CONGESTION.* |
+| Asterisk Dialplan Events | asterisk.stasis.message.ast_channel_*_type |
+| Asterisk PJSIP Peers Qualify | asterisk.PJSIP.contacts.*.rtt |
diff --git a/collectors/statsd.plugin/example.conf b/collectors/statsd.plugin/example.conf
new file mode 100644
index 0000000..2c7de6c
--- /dev/null
+++ b/collectors/statsd.plugin/example.conf
@@ -0,0 +1,64 @@
+# statsd synthetic charts configuration
+
+# You can add many .conf files in /etc/netdata/statsd.d/,
+# one for each of your apps.
+
+# start a new app - you can add many apps in the same file
+[app]
+ # give a name for this app
+ # this controls the main menu on the dashboard
+ # and will be the prefix for all charts of the app
+ name = myexampleapp
+
+ # match all the metrics of the app
+ metrics = myexampleapp.*
+
+ # shall private charts of these metrics be created?
+ private charts = no
+
+ # shall gaps be shown when metrics are not collected?
+ gaps when not collected = no
+
+ # the memory mode for the charts of this app: none|map|save
+ # the default is to use the global memory mode
+ #memory mode = ram
+
+ # the history size for the charts of this app, in seconds
+ # the default is to use the global history
+ #history = 3600
+
+# create a chart
+# this is its id - the chart will be named myexampleapp.myexamplechart
+[myexamplechart]
+ # a name for the chart, similar to the id (2 names for each chart)
+ name = myexamplechart
+
+ # the chart title
+ title = my chart title
+
+ # the submenu of the dashboard
+ family = my family
+
+ # the context for alarm templates
+ context = chart.context
+
+ # the units of the chart
+ units = tests/s
+
+ # the sorting priority of the chart on the dashboard
+ priority = 91000
+
+ # the type of chart to create: line | area | stacked
+ type = area
+
+ # one or more dimensions for the chart
+ # type = events | last | min | max | sum | average | percentile | median | stddev
+ # events = the number of events for this metric
+ # last = the last value collected
+ # all the others are only valid for histograms and timers
+ dimension = myexampleapp.metric1 avg average 1 1
+ dimension = myexampleapp.metric1 lower min 1 1
+ dimension = myexampleapp.metric1 upper max 1 1
+ dimension = myexampleapp.metric2 other last 1 1
+
+# You can add as many charts as needed
diff --git a/collectors/statsd.plugin/k6.conf b/collectors/statsd.plugin/k6.conf
new file mode 100644
index 0000000..3bef00c
--- /dev/null
+++ b/collectors/statsd.plugin/k6.conf
@@ -0,0 +1,110 @@
+[app]
+ name = k6
+ metrics = k6*
+ private charts = no
+ gaps when not collected = yes
+
+[dictionary]
+ http_reqs = HTTP Requests
+ http_reqs_failed = Failed HTTP Requests
+ vus = Virtual active users
+ vus_max = max Virtual active users
+ iteration_duration = iteration duration
+ iteration_duration_max = max iteration duration
+ iteration_duration_min = min iteration duration
+ iteration_duration_avg = avg iteration duration
+ dropped_iterations = Dropped iterations
+ http_req_blocked = Blocked HTTP requests
+ http_req_connecting = Connecting HTTP requests
+ http_req_sending = Sending HTTP requests
+ http_req_receiving = Receiving HTTP requests
+ http_req_waiting = Waiting HTTP requests
+ http_req_duration_median = Median HTTP req duration
+ http_req_duration_average = Avg HTTP req duration
+ http_req_duration = HTTP req duration
+ http_req_duration_max = max HTTP req duration
+ http_req_duration_min = min HTTP req duration
+ http_req_duration_p95 = 95 percentile of HTTP req duration
+ data_received = Received data
+ data_sent = Sent data
+
+
+[http_reqs]
+ name = http_reqs
+ title = HTTP Requests rate
+ family = http requests
+ context = k6.http_requests
+ dimension = k6.http_reqs http_reqs last 1 1 sum
+ type = line
+ units = requests/s
+
+[http_reqs]
+ name = http_reqs_failed
+ title = Failed HTTP Requests rate
+ family = http requests
+ context = k6.http_requests
+ dimension = k6.http_reqs_failed http_reqs_failed last 1 1 sum
+ type = line
+ units = requests/s
+
+[vus]
+ name = vus
+ title = Virtual Active Users
+ family = k6_metrics
+ dimension = k6.vus vus last 1 1
+ dimension = k6.vus_max vus_max last 1 1
+ type = line
+ units = vus
+
+[iteration_duration]
+ name = iteration_duration_2
+ title = Iteration duration
+ family = k6_metrics
+ dimension = k6.iteration_duration iteration_duration last 1 1
+ dimension = k6.iteration_duration iteration_duration_max max 1 1
+ dimension = k6.iteration_duration iteration_duration_min min 1 1
+ dimension = k6.iteration_duration iteration_duration_avg average 1 1
+ type = line
+ units = s
+
+[dropped_iterations]
+ name = dropped_iterations
+ title = Dropped Iterations
+ family = k6_metrics
+ dimension = k6.dropped_iterations dropped_iterations last 1 1
+ units = iterations
+ type = line
+
+[data]
+ name = data
+ title = K6 Data
+ family = k6_metrics
+ dimension = k6.data_received data_received last 1 1
+ dimension = k6.data_sent data_sent last -1 1
+ units = kb/s
+ type = area
+
+[http_req_duration_types]
+ name = http_req_duration_types
+ title = HTTP Requests total duration
+ family = http requests
+ dimension = k6.http_req_sending http_req_sending last 1 1
+ dimension = k6.http_req_waiting http_req_waiting last 1 1
+ dimension = k6.http_req_receiving http_req_receiving last 1 1
+ dimension = k6.http_req_blocked http_req_blocked last 1 1
+ dimension = k6.http_req_connecting http_req_connecting last 1 1
+ units = ms
+ type = stacked
+
+[http_req_duration]
+ name = http_req_duration
+ title = HTTP duration metrics
+ family = http requests
+ dimension = k6.http_req_duration http_req_duration_median median 1 1
+ dimension = k6.http_req_duration http_req_duration_max max 1 1
+ dimension = k6.http_req_duration http_req_duration_average average 1 1
+ dimension = k6.http_req_duration http_req_duration_min min 1 1
+ dimension = k6.http_req_duration http_req_duration_p95 percentile 1 1
+ dimension = k6.http_req_duration http_req_duration last 1 1
+ units = ms
+ type = line
diff --git a/collectors/statsd.plugin/k6.md b/collectors/statsd.plugin/k6.md
new file mode 100644
index 0000000..4f8c701
--- /dev/null
+++ b/collectors/statsd.plugin/k6.md
@@ -0,0 +1,76 @@
+<!--
+title: "K6 load test monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/statsd.plugin/k6.md
+
+sidebar_label: "K6 Load Testing"
+-->
+
+# K6 Load Testing monitoring with Netdata
+
+Monitors the impact of load testing experiments performed with [K6](https://k6.io/).
+
+You can read more about the metrics that K6 sends in the [K6 documentation](https://k6.io/docs/using-k6/metrics/).
+
+## Requirements
+
+- When running the k6 experiment, specify a [StatsD output](https://k6.io/docs/results-visualization/statsd/).
+ - Tip: K6 currently supports tags only with [datadog output](https://k6.io/docs/results-visualization/datadog/), which is in essence StatsD. Netdata can be used with both.
+
+## Metrics
+
+![image](https://user-images.githubusercontent.com/13405632/117691411-8a7baf00-b1c4-11eb-9d87-8e9e7214871f.png)
+
+
+### HTTP Requests
+
+Number of HTTP requests that K6 generates, per second.
+
+### Failed HTTP Requests
+
+Number of failed HTTP requests that K6 generates, per second.
+
+### Virtual Active Users
+Current number of active virtual users.
+
+### Iteration Duration
+
+The time it took K6 to complete one full iteration of the main function.
+
+### Dropped Iterations
+
+The number of iterations that could not be started either due to lack of Virtual Users or lack of time.
+
+### Data
+
+The amount of data received and sent.
+
+### TTP Requests total duration
+
+The total duration it took for a round-trip of an HTTP request. It includes:
+- Blocked HTTP requests: time spent blocked before initiating the request
+- Connecting HTTP requests: time spent establishing TCP connection to the remote host
+- Sending HTTP requests: time spent sending data to the remote host
+- Receiving HTTP requests: time spent receiving data from the remote host
+- Waiting HTTP requests: time spent waiting for response from the remote host
+
+### HTTP duration metrics
+
+Different metrics on the HTTP request as defined by K6. The HTTP request duration is defined by K6 as: `HTTP sending request` + `HTTP receiving request` + `HTTP waiting request`.
+
+Metrics:
+- Median
+- Average
+- Max
+- Min
+- 95th percentile
+- absolute (the value as it is, without any computation)
+
+## Configuration
+
+The collector is preconfigured and defined in `statsd.plugin/k6.conf`.
+
+Due to being a StatsD collector, you only need to define the configuration file and then send data to Netdata using the StatsD protocol.
+
+If Netdata is running on the same machine as K6, no further configuration is required. Otherwise, you will have to [point K6](https://k6.io/docs/results-visualization/statsd/) to your node and make sure that the K6 process can reach Netdata.
+
+The default namespace that is used in the configuration is `k6`. If you change it in K6, you will have to change it as well in the configuration file `k6.conf`.
diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c
new file mode 100644
index 0000000..67d7ed2
--- /dev/null
+++ b/collectors/statsd.plugin/statsd.c
@@ -0,0 +1,2844 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "daemon/common.h"
+
+#define STATSD_CHART_PREFIX "statsd"
+
+#define PLUGIN_STATSD_NAME "statsd.plugin"
+
+#define STATSD_LISTEN_PORT 8125
+#define STATSD_LISTEN_BACKLOG 4096
+
+#define WORKER_JOB_TYPE_TCP_CONNECTED 0
+#define WORKER_JOB_TYPE_TCP_DISCONNECTED 1
+#define WORKER_JOB_TYPE_RCV_DATA 2
+#define WORKER_JOB_TYPE_SND_DATA 3
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 4
+#error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least 4
+#endif
+
+// --------------------------------------------------------------------------------------
+
+// DO NOT ENABLE MULTITHREADING - IT IS NOT WELL TESTED
+// #define STATSD_MULTITHREADED 1
+
+#define STATSD_DICTIONARY_OPTIONS (DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_ADD_IN_FRONT)
+#define STATSD_DECIMAL_DETAIL 1000 // floating point values get multiplied by this, with the same divisor
+
+// --------------------------------------------------------------------------------------------------------------------
+// data specific to each metric type
+
+typedef struct statsd_metric_gauge {
+ NETDATA_DOUBLE value;
+} STATSD_METRIC_GAUGE;
+
+typedef struct statsd_metric_counter { // counter and meter
+ long long value;
+} STATSD_METRIC_COUNTER;
+
+typedef struct statsd_histogram_extensions {
+ netdata_mutex_t mutex;
+
+ // average is stored in metric->last
+ collected_number last_min;
+ collected_number last_max;
+ collected_number last_percentile;
+ collected_number last_median;
+ collected_number last_stddev;
+ collected_number last_sum;
+
+ int zeroed;
+
+ RRDDIM *rd_min;
+ RRDDIM *rd_max;
+ RRDDIM *rd_percentile;
+ RRDDIM *rd_median;
+ RRDDIM *rd_stddev;
+ //RRDDIM *rd_sum;
+
+ size_t size;
+ size_t used;
+ NETDATA_DOUBLE *values; // dynamic array of values collected
+} STATSD_METRIC_HISTOGRAM_EXTENSIONS;
+
+typedef struct statsd_metric_histogram { // histogram and timer
+ STATSD_METRIC_HISTOGRAM_EXTENSIONS *ext;
+} STATSD_METRIC_HISTOGRAM;
+
+typedef struct statsd_metric_set {
+ DICTIONARY *dict;
+ size_t unique;
+} STATSD_METRIC_SET;
+
+typedef struct statsd_metric_dictionary_item {
+ size_t count;
+ RRDDIM *rd;
+} STATSD_METRIC_DICTIONARY_ITEM;
+
+typedef struct statsd_metric_dictionary {
+ DICTIONARY *dict;
+ size_t unique;
+} STATSD_METRIC_DICTIONARY;
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// this is a metric - for all types of metrics
+
+typedef enum statsd_metric_options {
+ STATSD_METRIC_OPTION_NONE = 0x00000000, // no options set
+ STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED = 0x00000001, // do not update the chart dimension, when this metric is not collected
+ STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED = 0x00000002, // render a private chart for this metric
+ STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED = 0x00000004, // the metric has been checked if it should get private chart or not
+ STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT = 0x00000008, // show the count of events for this private chart
+ STATSD_METRIC_OPTION_CHECKED_IN_APPS = 0x00000010, // set when this metric has been checked against apps
+ STATSD_METRIC_OPTION_USED_IN_APPS = 0x00000020, // set when this metric is used in apps
+ STATSD_METRIC_OPTION_CHECKED = 0x00000040, // set when the charting thread checks this metric for use in charts (its usefulness)
+ STATSD_METRIC_OPTION_USEFUL = 0x00000080, // set when the charting thread finds the metric useful (i.e. used in a chart)
+ STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED = 0x00000100, // set when the collection is full for this metric
+ STATSD_METRIC_OPTION_UPDATED_CHART_METADATA = 0x00000200, // set when the private chart metadata have been updated via tags
+} STATS_METRIC_OPTIONS;
+
+typedef enum statsd_metric_type {
+ STATSD_METRIC_TYPE_GAUGE,
+ STATSD_METRIC_TYPE_COUNTER,
+ STATSD_METRIC_TYPE_METER,
+ STATSD_METRIC_TYPE_TIMER,
+ STATSD_METRIC_TYPE_HISTOGRAM,
+ STATSD_METRIC_TYPE_SET,
+ STATSD_METRIC_TYPE_DICTIONARY
+} STATSD_METRIC_TYPE;
+
+
+typedef struct statsd_metric {
+ const char *name; // the name of the metric - linked to dictionary name
+ uint32_t hash; // hash of the name
+
+ STATSD_METRIC_TYPE type;
+
+ // metadata about data collection
+ collected_number events; // the number of times this metric has been collected (never resets)
+ size_t count; // the number of times this metric has been collected since the last flush
+
+ // the actual collected data
+ union {
+ STATSD_METRIC_GAUGE gauge;
+ STATSD_METRIC_COUNTER counter;
+ STATSD_METRIC_HISTOGRAM histogram;
+ STATSD_METRIC_SET set;
+ STATSD_METRIC_DICTIONARY dictionary;
+ };
+
+ char *units;
+ char *dimname;
+ char *family;
+
+ // chart related members
+ STATS_METRIC_OPTIONS options; // STATSD_METRIC_OPTION_* (bitfield)
+ char reset; // set to 1 by the charting thread to instruct the collector thread(s) to reset this metric
+ collected_number last; // the last value sent to netdata
+ RRDSET *st; // the private chart of this metric
+ RRDDIM *rd_value; // the dimension of this metric value
+ RRDDIM *rd_count; // the dimension for the number of events received
+
+ // linking, used for walking through all metrics
+ struct statsd_metric *next_useful;
+} STATSD_METRIC;
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// each type of metric has its own index
+
+typedef struct statsd_index {
+ char *name; // the name of the index of metrics
+ size_t events; // the number of events processed for this index
+ size_t metrics; // the number of metrics in this index
+ size_t useful; // the number of useful metrics in this index
+
+ STATSD_METRIC_TYPE type; // the type of index
+ DICTIONARY *dict;
+
+ STATSD_METRIC *first_useful; // the linked list of useful metrics (new metrics are added in front)
+
+ STATS_METRIC_OPTIONS default_options; // default options for all metrics in this index
+} STATSD_INDEX;
+
+// --------------------------------------------------------------------------------------------------------------------
+// synthetic charts
+
+typedef enum statsd_app_chart_dimension_value_type {
+ STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_LAST,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_SUM,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_MIN,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_MAX,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV
+} STATSD_APP_CHART_DIM_VALUE_TYPE;
+
+typedef struct statsd_app_chart_dimension {
+ const char *name; // the name of this dimension
+ const char *metric; // the source metric name of this dimension
+ uint32_t metric_hash; // hash for fast string comparisons
+
+ SIMPLE_PATTERN *metric_pattern; // set when the 'metric' is a simple pattern
+
+ collected_number multiplier; // the multiplier of the dimension
+ collected_number divisor; // the divisor of the dimension
+ RRDDIM_FLAGS flags; // the RRDDIM flags for this dimension
+ RRDDIM_OPTIONS options; // the RRDDIM options for this dimension
+
+ STATSD_APP_CHART_DIM_VALUE_TYPE value_type; // which value to use of the source metric
+
+ RRDDIM *rd; // a pointer to the RRDDIM that has been created for this dimension
+ collected_number *value_ptr; // a pointer to the source metric value
+ RRD_ALGORITHM algorithm; // the algorithm of this dimension
+
+ struct statsd_app_chart_dimension *next; // the next dimension for this chart
+} STATSD_APP_CHART_DIM;
+
+typedef struct statsd_app_chart {
+ const char *id;
+ const char *name;
+ const char *title;
+ const char *family;
+ const char *context;
+ const char *units;
+ const char *module;
+ long priority;
+ RRDSET_TYPE chart_type;
+ STATSD_APP_CHART_DIM *dimensions;
+ size_t dimensions_count;
+ size_t dimensions_linked_count;
+
+ RRDSET *st;
+ struct statsd_app_chart *next;
+} STATSD_APP_CHART;
+
+typedef struct statsd_app {
+ const char *name;
+ SIMPLE_PATTERN *metrics;
+ STATS_METRIC_OPTIONS default_options;
+ RRD_MEMORY_MODE rrd_memory_mode;
+ DICTIONARY *dict;
+ long rrd_history_entries;
+
+ const char *source;
+ STATSD_APP_CHART *charts;
+ struct statsd_app *next;
+} STATSD_APP;
+
+// --------------------------------------------------------------------------------------------------------------------
+// global statsd data
+
+struct collection_thread_status {
+ int status;
+ size_t max_sockets;
+
+ netdata_thread_t thread;
+};
+
+static struct statsd {
+ STATSD_INDEX gauges;
+ STATSD_INDEX counters;
+ STATSD_INDEX timers;
+ STATSD_INDEX histograms;
+ STATSD_INDEX meters;
+ STATSD_INDEX sets;
+ STATSD_INDEX dictionaries;
+
+ size_t unknown_types;
+ size_t socket_errors;
+ size_t tcp_socket_connects;
+ size_t tcp_socket_disconnects;
+ size_t tcp_socket_connected;
+ size_t tcp_socket_reads;
+ size_t tcp_packets_received;
+ size_t tcp_bytes_read;
+ size_t udp_socket_reads;
+ size_t udp_packets_received;
+ size_t udp_bytes_read;
+
+ int enabled;
+ int update_every;
+ SIMPLE_PATTERN *charts_for;
+
+ size_t tcp_idle_timeout;
+ collected_number decimal_detail;
+ size_t private_charts;
+ size_t max_private_charts_hard;
+ long private_charts_rrd_history_entries;
+ unsigned int private_charts_hidden:1;
+
+ STATSD_APP *apps;
+ size_t recvmmsg_size;
+ size_t histogram_increase_step;
+ double histogram_percentile;
+ char *histogram_percentile_str;
+ size_t dictionary_max_unique;
+
+ int threads;
+ struct collection_thread_status *collection_threads_status;
+
+ LISTEN_SOCKETS sockets;
+} statsd = {
+ .enabled = 1,
+ .max_private_charts_hard = 1000,
+ .private_charts_hidden = 0,
+ .recvmmsg_size = 10,
+ .decimal_detail = STATSD_DECIMAL_DETAIL,
+
+ .gauges = {
+ .name = "gauge",
+ .events = 0,
+ .metrics = 0,
+ .dict = NULL,
+ .type = STATSD_METRIC_TYPE_GAUGE,
+ .default_options = STATSD_METRIC_OPTION_NONE
+ },
+ .counters = {
+ .name = "counter",
+ .events = 0,
+ .metrics = 0,
+ .dict = NULL,
+ .type = STATSD_METRIC_TYPE_COUNTER,
+ .default_options = STATSD_METRIC_OPTION_NONE
+ },
+ .timers = {
+ .name = "timer",
+ .events = 0,
+ .metrics = 0,
+ .dict = NULL,
+ .type = STATSD_METRIC_TYPE_TIMER,
+ .default_options = STATSD_METRIC_OPTION_NONE
+ },
+ .histograms = {
+ .name = "histogram",
+ .events = 0,
+ .metrics = 0,
+ .dict = NULL,
+ .type = STATSD_METRIC_TYPE_HISTOGRAM,
+ .default_options = STATSD_METRIC_OPTION_NONE
+ },
+ .meters = {
+ .name = "meter",
+ .events = 0,
+ .metrics = 0,
+ .dict = NULL,
+ .type = STATSD_METRIC_TYPE_METER,
+ .default_options = STATSD_METRIC_OPTION_NONE
+ },
+ .sets = {
+ .name = "set",
+ .events = 0,
+ .metrics = 0,
+ .dict = NULL,
+ .type = STATSD_METRIC_TYPE_SET,
+ .default_options = STATSD_METRIC_OPTION_NONE
+ },
+ .dictionaries = {
+ .name = "dictionary",
+ .events = 0,
+ .metrics = 0,
+ .dict = NULL,
+ .type = STATSD_METRIC_TYPE_DICTIONARY,
+ .default_options = STATSD_METRIC_OPTION_NONE
+ },
+
+ .tcp_idle_timeout = 600,
+
+ .apps = NULL,
+ .histogram_percentile = 95.0,
+ .histogram_increase_step = 10,
+ .dictionary_max_unique = 200,
+ .threads = 0,
+ .collection_threads_status = NULL,
+ .sockets = {
+ .config = &netdata_config,
+ .config_section = CONFIG_SECTION_STATSD,
+ .default_bind_to = "udp:localhost tcp:localhost",
+ .default_port = STATSD_LISTEN_PORT,
+ .backlog = STATSD_LISTEN_BACKLOG
+ },
+};
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd index management - add/find metrics
+
+static void dictionary_metric_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
+ STATSD_INDEX *index = (STATSD_INDEX *)data;
+ STATSD_METRIC *m = (STATSD_METRIC *)value;
+ const char *name = dictionary_acquired_item_name(item);
+
+ debug(D_STATSD, "Creating new %s metric '%s'", index->name, name);
+
+ m->name = name;
+ m->hash = simple_hash(name);
+ m->type = index->type;
+ m->options = index->default_options;
+
+ if (m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
+ m->histogram.ext = callocz(1,sizeof(STATSD_METRIC_HISTOGRAM_EXTENSIONS));
+ netdata_mutex_init(&m->histogram.ext->mutex);
+ }
+
+ __atomic_fetch_add(&index->metrics, 1, __ATOMIC_RELAXED);
+}
+
+static void dictionary_metric_delete_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
+ (void)data; // STATSD_INDEX *index = (STATSD_INDEX *)data;
+ (void)item;
+ STATSD_METRIC *m = (STATSD_METRIC *)value;
+
+ if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
+ freez(m->histogram.ext);
+ m->histogram.ext = NULL;
+ }
+
+ freez(m->units);
+ freez(m->family);
+ freez(m->dimname);
+}
+
+static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, const char *name) {
+ debug(D_STATSD, "searching for metric '%s' under '%s'", name, index->name);
+
+#ifdef STATSD_MULTITHREADED
+ // avoid the write lock of dictionary_set() for existing metrics
+ STATSD_METRIC *m = dictionary_get(index->dict, name);
+ if(!m) m = dictionary_set(index->dict, name, NULL, sizeof(STATSD_METRIC));
+#else
+ // no locks here, go faster
+ // this will call the dictionary_metric_insert_callback() if an item
+ // is inserted, otherwise it will return the existing one.
+ // We used the flag DICT_OPTION_DONT_OVERWRITE_VALUE to support this.
+ STATSD_METRIC *m = dictionary_set(index->dict, name, NULL, sizeof(STATSD_METRIC));
+#endif
+
+ index->events++;
+ return m;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd parsing numbers
+
+static inline NETDATA_DOUBLE statsd_parse_float(const char *v, NETDATA_DOUBLE def) {
+ NETDATA_DOUBLE value;
+
+ if(likely(v && *v)) {
+ char *e = NULL;
+ value = str2ndd(v, &e);
+ if(unlikely(e && *e))
+ error("STATSD: excess data '%s' after value '%s'", e, v);
+ }
+ else
+ value = def;
+
+ return value;
+}
+
+static inline NETDATA_DOUBLE statsd_parse_sampling_rate(const char *v) {
+ NETDATA_DOUBLE sampling_rate = statsd_parse_float(v, 1.0);
+ if(unlikely(isless(sampling_rate, 0.001))) sampling_rate = 0.001;
+ if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
+ return sampling_rate;
+}
+
+static inline long long statsd_parse_int(const char *v, long long def) {
+ long long value;
+
+ if(likely(v && *v)) {
+ char *e = NULL;
+ value = str2ll(v, &e);
+ if(unlikely(e && *e))
+ error("STATSD: excess data '%s' after value '%s'", e, v);
+ }
+ else
+ value = def;
+
+ return value;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd processors per metric type
+
+static inline void statsd_reset_metric(STATSD_METRIC *m) {
+ m->reset = 0;
+ m->count = 0;
+}
+
+static inline int value_is_zinit(const char *value) {
+ return (value && *value == 'z' && *++value == 'i' && *++value == 'n' && *++value == 'i' && *++value == 't' && *++value == '\0');
+}
+
+#define is_metric_checked(m) ((m)->options & STATSD_METRIC_OPTION_CHECKED)
+#define is_metric_useful_for_collection(m) (!is_metric_checked(m) || ((m)->options & STATSD_METRIC_OPTION_USEFUL))
+
+static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, const char *sampling) {
+ if(!is_metric_useful_for_collection(m)) return;
+
+ if(unlikely(!value || !*value)) {
+ error("STATSD: metric '%s' of type gauge, with empty value is ignored.", m->name);
+ return;
+ }
+
+ if(unlikely(m->reset)) {
+ // no need to reset anything specific for gauges
+ statsd_reset_metric(m);
+ }
+
+ if(unlikely(value_is_zinit(value))) {
+ // magic loading of metric, without affecting anything
+ }
+ else {
+ if (unlikely(*value == '+' || *value == '-'))
+ m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_sampling_rate(sampling);
+ else
+ m->gauge.value = statsd_parse_float(value, 1.0);
+
+ m->events++;
+ m->count++;
+ }
+}
+
+static inline void statsd_process_counter_or_meter(STATSD_METRIC *m, const char *value, const char *sampling) {
+ if(!is_metric_useful_for_collection(m)) return;
+
+ // we accept empty values for counters
+
+ if(unlikely(m->reset)) statsd_reset_metric(m);
+
+ if(unlikely(value_is_zinit(value))) {
+ // magic loading of metric, without affecting anything
+ }
+ else {
+ m->counter.value += llrintndd((NETDATA_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling));
+
+ m->events++;
+ m->count++;
+ }
+}
+
+#define statsd_process_counter(m, value, sampling) statsd_process_counter_or_meter(m, value, sampling)
+#define statsd_process_meter(m, value, sampling) statsd_process_counter_or_meter(m, value, sampling)
+
+static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const char *value, const char *sampling, const char *type) {
+ if(!is_metric_useful_for_collection(m)) return;
+
+ if(unlikely(!value || !*value)) {
+ error("STATSD: metric of type %s, with empty value is ignored.", type);
+ return;
+ }
+
+ if(unlikely(m->reset)) {
+ m->histogram.ext->used = 0;
+ statsd_reset_metric(m);
+ }
+
+ if(unlikely(value_is_zinit(value))) {
+ // magic loading of metric, without affecting anything
+ }
+ else {
+ NETDATA_DOUBLE v = statsd_parse_float(value, 1.0);
+ NETDATA_DOUBLE sampling_rate = statsd_parse_sampling_rate(sampling);
+ if(unlikely(isless(sampling_rate, 0.01))) sampling_rate = 0.01;
+ if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
+
+ long long samples = llrintndd(1.0 / sampling_rate);
+ while(samples-- > 0) {
+
+ if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) {
+ netdata_mutex_lock(&m->histogram.ext->mutex);
+ m->histogram.ext->size += statsd.histogram_increase_step;
+ m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(NETDATA_DOUBLE) * m->histogram.ext->size);
+ netdata_mutex_unlock(&m->histogram.ext->mutex);
+ }
+
+ m->histogram.ext->values[m->histogram.ext->used++] = v;
+ }
+
+ m->events++;
+ m->count++;
+ }
+}
+
+#define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer")
+#define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram")
+
+static void dictionary_metric_set_value_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
+ (void)item;
+ (void)value;
+ STATSD_METRIC *m = (STATSD_METRIC *)data;
+ m->set.unique++;
+}
+
+static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
+ if(!is_metric_useful_for_collection(m)) return;
+
+ if(unlikely(!value || !*value)) {
+ error("STATSD: metric of type set, with empty value is ignored.");
+ return;
+ }
+
+ if(unlikely(m->reset)) {
+ if(likely(m->set.dict)) {
+ dictionary_destroy(m->set.dict);
+ m->set.dict = NULL;
+ }
+ statsd_reset_metric(m);
+ }
+
+ if (unlikely(!m->set.dict)) {
+ m->set.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+ dictionary_register_insert_callback(m->set.dict, dictionary_metric_set_value_insert_callback, m);
+ m->set.unique = 0;
+ }
+
+ if(unlikely(value_is_zinit(value))) {
+ // magic loading of metric, without affecting anything
+ }
+ else {
+#ifdef STATSD_MULTITHREADED
+ // avoid the write lock to check if something is already there
+ if(!dictionary_get(m->set.dict, value))
+ dictionary_set(m->set.dict, value, NULL, 0);
+#else
+ dictionary_set(m->set.dict, value, NULL, 0);
+#endif
+ m->events++;
+ m->count++;
+ }
+}
+
+static void dictionary_metric_dict_value_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
+ (void)item;
+ (void)value;
+ STATSD_METRIC *m = (STATSD_METRIC *)data;
+ m->dictionary.unique++;
+}
+
+static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value) {
+ if(!is_metric_useful_for_collection(m)) return;
+
+ if(unlikely(!value || !*value)) {
+ error("STATSD: metric of type set, with empty value is ignored.");
+ return;
+ }
+
+ if(unlikely(m->reset))
+ statsd_reset_metric(m);
+
+ if (unlikely(!m->dictionary.dict)) {
+ m->dictionary.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+ dictionary_register_insert_callback(m->dictionary.dict, dictionary_metric_dict_value_insert_callback, m);
+ m->dictionary.unique = 0;
+ }
+
+ if(unlikely(value_is_zinit(value))) {
+ // magic loading of metric, without affecting anything
+ }
+ else {
+ STATSD_METRIC_DICTIONARY_ITEM *t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_get(m->dictionary.dict, value);
+
+ if (unlikely(!t)) {
+ if(!t && m->dictionary.unique >= statsd.dictionary_max_unique)
+ value = "other";
+
+ t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_set(m->dictionary.dict, value, NULL, sizeof(STATSD_METRIC_DICTIONARY_ITEM));
+ }
+
+ t->count++;
+ m->events++;
+ m->count++;
+ }
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd parsing
+
+static inline const char *statsd_parse_skip_up_to(const char *s, char d1, char d2, char d3) {
+ char c;
+
+ for(c = *s; c && c != d1 && c != d2 && c != d3 && c != '\r' && c != '\n'; c = *++s) ;
+
+ return s;
+}
+
+const char *statsd_parse_skip_spaces(const char *s) {
+ char c;
+
+ for(c = *s; c && ( c == ' ' || c == '\t' || c == '\r' || c == '\n' ); c = *++s) ;
+
+ return s;
+}
+
+static inline const char *statsd_parse_field_trim(const char *start, char *end) {
+ if(unlikely(!start || !*start)) {
+ start = end;
+ return start;
+ }
+
+ while(start <= end && (*start == ' ' || *start == '\t'))
+ start++;
+
+ *end = '\0';
+ end--;
+ while(end >= start && (*end == ' ' || *end == '\t'))
+ *end-- = '\0';
+
+ return start;
+}
+
+static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling, const char *tags) {
+ debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', sampling '%s', tags '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)", tags?tags:"(null)");
+
+ if(unlikely(!name || !*name)) return;
+ if(unlikely(!type || !*type)) type = "m";
+
+ STATSD_METRIC *m = NULL;
+
+ char t0 = type[0], t1 = type[1];
+ if(unlikely(t0 == 'g' && t1 == '\0')) {
+ statsd_process_gauge(
+ m = statsd_find_or_add_metric(&statsd.gauges, name),
+ value, sampling);
+ }
+ else if(unlikely((t0 == 'c' || t0 == 'C') && t1 == '\0')) {
+ // etsy/statsd uses 'c'
+ // brubeck uses 'C'
+ statsd_process_counter(
+ m = statsd_find_or_add_metric(&statsd.counters, name),
+ value, sampling);
+ }
+ else if(unlikely(t0 == 'm' && t1 == '\0')) {
+ statsd_process_meter(
+ m = statsd_find_or_add_metric(&statsd.meters, name),
+ value, sampling);
+ }
+ else if(unlikely(t0 == 'h' && t1 == '\0')) {
+ statsd_process_histogram(
+ m = statsd_find_or_add_metric(&statsd.histograms, name),
+ value, sampling);
+ }
+ else if(unlikely(t0 == 's' && t1 == '\0')) {
+ statsd_process_set(
+ m = statsd_find_or_add_metric(&statsd.sets, name),
+ value);
+ }
+ else if(unlikely(t0 == 'd' && t1 == '\0')) {
+ statsd_process_dictionary(
+ m = statsd_find_or_add_metric(&statsd.dictionaries, name),
+ value);
+ }
+ else if(unlikely(t0 == 'm' && t1 == 's' && type[2] == '\0')) {
+ statsd_process_timer(
+ m = statsd_find_or_add_metric(&statsd.timers, name),
+ value, sampling);
+ }
+ else {
+ statsd.unknown_types++;
+ error("STATSD: metric '%s' with value '%s' is sent with unknown metric type '%s'", name, value?value:"", type);
+ }
+
+ if(m && tags && *tags) {
+ const char *s = tags;
+ while(*s) {
+ const char *tagkey = NULL, *tagvalue = NULL;
+ char *tagkey_end = NULL, *tagvalue_end = NULL;
+
+ s = tagkey_end = (char *)statsd_parse_skip_up_to(tagkey = s, ':', '=', ',');
+ if(tagkey == tagkey_end) {
+ if (*s) {
+ s++;
+ s = statsd_parse_skip_spaces(s);
+ }
+ continue;
+ }
+
+ if(likely(*s == ':' || *s == '='))
+ s = tagvalue_end = (char *) statsd_parse_skip_up_to(tagvalue = ++s, ',', '\0', '\0');
+
+ if(*s == ',') s++;
+
+ statsd_parse_field_trim(tagkey, tagkey_end);
+ statsd_parse_field_trim(tagvalue, tagvalue_end);
+
+ if(tagkey && *tagkey && tagvalue && *tagvalue) {
+ if (strcmp(tagkey, "units") == 0 && (!m->units || strcmp(m->units, tagvalue) != 0)) {
+ m->units = strdupz(tagvalue);
+ m->options |= STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;
+ }
+
+ if (strcmp(tagkey, "name") == 0 && (!m->dimname || strcmp(m->dimname, tagvalue) != 0)) {
+ m->dimname = strdupz(tagvalue);
+ m->options |= STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;
+ }
+
+ if (strcmp(tagkey, "family") == 0 && (!m->family || strcmp(m->family, tagvalue) != 0)) {
+ m->family = strdupz(tagvalue);
+ m->options |= STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;
+ }
+ }
+ }
+ }
+}
+
+static inline size_t statsd_process(char *buffer, size_t size, int require_newlines) {
+ buffer[size] = '\0';
+ debug(D_STATSD, "RECEIVED: %zu bytes: '%s'", size, buffer);
+
+ const char *s = buffer;
+ while(*s) {
+ const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL, *tags = NULL;
+ char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL, *tags_end = NULL;
+
+ s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '=', '|');
+ if(name == name_end) {
+ if (*s) {
+ s++;
+ s = statsd_parse_skip_spaces(s);
+ }
+ continue;
+ }
+
+ if(likely(*s == ':' || *s == '='))
+ s = value_end = (char *) statsd_parse_skip_up_to(value = ++s, '|', '@', '#');
+
+ if(likely(*s == '|'))
+ s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@', '#');
+
+ while(*s == '|' || *s == '@' || *s == '#') {
+ // parse all the fields that may be appended
+
+ if ((*s == '|' && s[1] == '@') || *s == '@') {
+ s = sampling_end = (char *)statsd_parse_skip_up_to(sampling = ++s, '|', '@', '#');
+ if (*sampling == '@') sampling++;
+ }
+ else if ((*s == '|' && s[1] == '#') || *s == '#') {
+ s = tags_end = (char *)statsd_parse_skip_up_to(tags = ++s, '|', '@', '#');
+ if (*tags == '#') tags++;
+ }
+ else {
+ // unknown field, skip it
+ s = (char *)statsd_parse_skip_up_to(++s, '|', '@', '#');
+ }
+ }
+
+ // skip everything until the end of the line
+ while(*s && *s != '\n') s++;
+
+ if(unlikely(require_newlines && *s != '\n' && s > buffer)) {
+ // move the remaining data to the beginning
+ size -= (name - buffer);
+ memmove(buffer, name, size);
+ return size;
+ }
+ else
+ s = statsd_parse_skip_spaces(s);
+
+ statsd_process_metric(
+ statsd_parse_field_trim(name, name_end)
+ , statsd_parse_field_trim(value, value_end)
+ , statsd_parse_field_trim(type, type_end)
+ , statsd_parse_field_trim(sampling, sampling_end)
+ , statsd_parse_field_trim(tags, tags_end)
+ );
+ }
+
+ return 0;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd pollfd interface
+
+#define STATSD_TCP_BUFFER_SIZE 65536 // minimize tcp reads
+#define STATSD_UDP_BUFFER_SIZE 9000 // this should be up to MTU
+
+typedef enum {
+ STATSD_SOCKET_DATA_TYPE_TCP,
+ STATSD_SOCKET_DATA_TYPE_UDP
+} STATSD_SOCKET_DATA_TYPE;
+
+struct statsd_tcp {
+ STATSD_SOCKET_DATA_TYPE type;
+ size_t size;
+ size_t len;
+ char buffer[];
+};
+
+#ifdef HAVE_RECVMMSG
+struct statsd_udp {
+ int *running;
+ STATSD_SOCKET_DATA_TYPE type;
+ size_t size;
+ struct iovec *iovecs;
+ struct mmsghdr *msgs;
+};
+#else
+struct statsd_udp {
+ int *running;
+ STATSD_SOCKET_DATA_TYPE type;
+ char buffer[STATSD_UDP_BUFFER_SIZE];
+};
+#endif
+
+// new TCP client connected
+static void *statsd_add_callback(POLLINFO *pi, short int *events, void *data) {
+ (void)pi;
+ (void)data;
+
+ worker_is_busy(WORKER_JOB_TYPE_TCP_CONNECTED);
+ *events = POLLIN;
+
+ struct statsd_tcp *t = (struct statsd_tcp *)callocz(sizeof(struct statsd_tcp) + STATSD_TCP_BUFFER_SIZE, 1);
+ t->type = STATSD_SOCKET_DATA_TYPE_TCP;
+ t->size = STATSD_TCP_BUFFER_SIZE - 1;
+ statsd.tcp_socket_connects++;
+ statsd.tcp_socket_connected++;
+
+ worker_is_idle();
+ return t;
+}
+
+// TCP client disconnected
+static void statsd_del_callback(POLLINFO *pi) {
+ worker_is_busy(WORKER_JOB_TYPE_TCP_DISCONNECTED);
+
+ struct statsd_tcp *t = pi->data;
+
+ if(likely(t)) {
+ if(t->type == STATSD_SOCKET_DATA_TYPE_TCP) {
+ if(t->len != 0) {
+ statsd.socket_errors++;
+ error("STATSD: client is probably sending unterminated metrics. Closed socket left with '%s'. Trying to process it.", t->buffer);
+ statsd_process(t->buffer, t->len, 0);
+ }
+ statsd.tcp_socket_disconnects++;
+ statsd.tcp_socket_connected--;
+ }
+ else
+ error("STATSD: internal error: received socket data type is %d, but expected %d", (int)t->type, (int)STATSD_SOCKET_DATA_TYPE_TCP);
+
+ freez(t);
+ }
+
+ worker_is_idle();
+}
+
+// Receive data
+static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
+ int retval = -1;
+ worker_is_busy(WORKER_JOB_TYPE_RCV_DATA);
+
+ *events = POLLIN;
+
+ int fd = pi->fd;
+
+ switch(pi->socktype) {
+ case SOCK_STREAM: {
+ struct statsd_tcp *d = (struct statsd_tcp *)pi->data;
+ if(unlikely(!d)) {
+ error("STATSD: internal error: expected TCP data pointer is NULL");
+ statsd.socket_errors++;
+ retval = -1;
+ goto cleanup;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_TCP)) {
+ error("STATSD: internal error: socket data type should be %d, but it is %d", (int)STATSD_SOCKET_DATA_TYPE_TCP, (int)d->type);
+ statsd.socket_errors++;
+ retval = -1;
+ goto cleanup;
+ }
+#endif
+
+ int ret = 0;
+ ssize_t rc;
+ do {
+ rc = recv(fd, &d->buffer[d->len], d->size - d->len, MSG_DONTWAIT);
+ if (rc < 0) {
+ // read failed
+ if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
+ error("STATSD: recv() on TCP socket %d failed.", fd);
+ statsd.socket_errors++;
+ ret = -1;
+ }
+ }
+ else if (!rc) {
+ // connection closed
+ debug(D_STATSD, "STATSD: client disconnected.");
+ ret = -1;
+ }
+ else {
+ // data received
+ d->len += rc;
+ statsd.tcp_socket_reads++;
+ statsd.tcp_bytes_read += rc;
+ }
+
+ if(likely(d->len > 0)) {
+ statsd.tcp_packets_received++;
+ d->len = statsd_process(d->buffer, d->len, 1);
+ }
+
+ if(unlikely(ret == -1)) {
+ retval = -1;
+ goto cleanup;
+ }
+
+ } while (rc != -1);
+ break;
+ }
+
+ case SOCK_DGRAM: {
+ struct statsd_udp *d = (struct statsd_udp *)pi->data;
+ if(unlikely(!d)) {
+ error("STATSD: internal error: expected UDP data pointer is NULL");
+ statsd.socket_errors++;
+ retval = -1;
+ goto cleanup;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_UDP)) {
+ error("STATSD: internal error: socket data should be %d, but it is %d", (int)d->type, (int)STATSD_SOCKET_DATA_TYPE_UDP);
+ statsd.socket_errors++;
+ retval = -1;
+ goto cleanup;
+ }
+#endif
+
+#ifdef HAVE_RECVMMSG
+ ssize_t rc;
+ do {
+ rc = recvmmsg(fd, d->msgs, (unsigned int)d->size, MSG_DONTWAIT, NULL);
+ if (rc < 0) {
+ // read failed
+ if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
+ error("STATSD: recvmmsg() on UDP socket %d failed.", fd);
+ statsd.socket_errors++;
+ retval = -1;
+ goto cleanup;
+ }
+ } else if (rc) {
+ // data received
+ statsd.udp_socket_reads++;
+ statsd.udp_packets_received += rc;
+
+ size_t i;
+ for (i = 0; i < (size_t)rc; ++i) {
+ size_t len = (size_t)d->msgs[i].msg_len;
+ statsd.udp_bytes_read += len;
+ statsd_process(d->msgs[i].msg_hdr.msg_iov->iov_base, len, 0);
+ }
+ }
+ } while (rc != -1);
+
+#else // !HAVE_RECVMMSG
+ ssize_t rc;
+ do {
+ rc = recv(fd, d->buffer, STATSD_UDP_BUFFER_SIZE - 1, MSG_DONTWAIT);
+ if (rc < 0) {
+ // read failed
+ if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
+ error("STATSD: recv() on UDP socket %d failed.", fd);
+ statsd.socket_errors++;
+ retval = -1;
+ goto cleanup;
+ }
+ } else if (rc) {
+ // data received
+ statsd.udp_socket_reads++;
+ statsd.udp_packets_received++;
+ statsd.udp_bytes_read += rc;
+ statsd_process(d->buffer, (size_t) rc, 0);
+ }
+ } while (rc != -1);
+#endif
+
+ break;
+ }
+
+ default: {
+ error("STATSD: internal error: unknown socktype %d on socket %d", pi->socktype, fd);
+ statsd.socket_errors++;
+ retval = -1;
+ goto cleanup;
+ }
+ }
+
+ retval = 0;
+cleanup:
+ worker_is_idle();
+ return retval;
+}
+
+static int statsd_snd_callback(POLLINFO *pi, short int *events) {
+ (void)pi;
+ (void)events;
+
+ worker_is_busy(WORKER_JOB_TYPE_SND_DATA);
+ error("STATSD: snd_callback() called, but we never requested to send data to statsd clients.");
+ worker_is_idle();
+
+ return -1;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd child thread to collect metrics from network
+
+void statsd_collector_thread_cleanup(void *data) {
+ struct statsd_udp *d = data;
+ *d->running = 0;
+
+ info("cleaning up...");
+
+#ifdef HAVE_RECVMMSG
+ size_t i;
+ for (i = 0; i < d->size; i++)
+ freez(d->iovecs[i].iov_base);
+
+ freez(d->iovecs);
+ freez(d->msgs);
+#endif
+
+ freez(d);
+ worker_unregister();
+}
+
+void *statsd_collector_thread(void *ptr) {
+ struct collection_thread_status *status = ptr;
+ status->status = 1;
+
+ worker_register("STATSD");
+ worker_register_job_name(WORKER_JOB_TYPE_TCP_CONNECTED, "tcp connect");
+ worker_register_job_name(WORKER_JOB_TYPE_TCP_DISCONNECTED, "tcp disconnect");
+ worker_register_job_name(WORKER_JOB_TYPE_RCV_DATA, "receive");
+ worker_register_job_name(WORKER_JOB_TYPE_SND_DATA, "send");
+
+ info("STATSD collector thread started with taskid %d", gettid());
+
+ struct statsd_udp *d = callocz(sizeof(struct statsd_udp), 1);
+ d->running = &status->status;
+
+ netdata_thread_cleanup_push(statsd_collector_thread_cleanup, d);
+
+#ifdef HAVE_RECVMMSG
+ d->type = STATSD_SOCKET_DATA_TYPE_UDP;
+ d->size = statsd.recvmmsg_size;
+ d->iovecs = callocz(sizeof(struct iovec), d->size);
+ d->msgs = callocz(sizeof(struct mmsghdr), d->size);
+
+ size_t i;
+ for (i = 0; i < d->size; i++) {
+ d->iovecs[i].iov_base = mallocz(STATSD_UDP_BUFFER_SIZE);
+ d->iovecs[i].iov_len = STATSD_UDP_BUFFER_SIZE - 1;
+ d->msgs[i].msg_hdr.msg_iov = &d->iovecs[i];
+ d->msgs[i].msg_hdr.msg_iovlen = 1;
+ }
+#endif
+
+ poll_events(&statsd.sockets
+ , statsd_add_callback
+ , statsd_del_callback
+ , statsd_rcv_callback
+ , statsd_snd_callback
+ , NULL
+ , NULL // No access control pattern
+ , 0 // No dns lookups for access control pattern
+ , (void *)d
+ , 0 // tcp request timeout, 0 = disabled
+ , statsd.tcp_idle_timeout // tcp idle timeout, 0 = disabled
+ , statsd.update_every * 1000
+ , ptr // timer_data
+ , status->max_sockets
+ );
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd applications configuration files parsing
+
+#define STATSD_CONF_LINE_MAX 8192
+
+static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *filename) {
+ if(!type || !*type) type = "last";
+
+ if(!strcmp(type, "events")) return STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS;
+ else if(!strcmp(type, "last")) return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
+ else if(!strcmp(type, "min")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MIN;
+ else if(!strcmp(type, "max")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MAX;
+ else if(!strcmp(type, "sum")) return STATSD_APP_CHART_DIM_VALUE_TYPE_SUM;
+ else if(!strcmp(type, "average")) return STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE;
+ else if(!strcmp(type, "median")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN;
+ else if(!strcmp(type, "stddev")) return STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV;
+ else if(!strcmp(type, "percentile")) return STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE;
+
+ error("STATSD: invalid type '%s' at line %zu of file '%s'. Using 'last'.", type, line, filename);
+ return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
+}
+
+static const char *valuetype2string(STATSD_APP_CHART_DIM_VALUE_TYPE type) {
+ switch(type) {
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS: return "events";
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST: return "last";
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN: return "min";
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX: return "max";
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM: return "sum";
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE: return "average";
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN: return "median";
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV: return "stddev";
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE: return "percentile";
+ }
+
+ return "unknown";
+}
+
+static STATSD_APP_CHART_DIM *add_dimension_to_app_chart(
+ STATSD_APP *app __maybe_unused
+ , STATSD_APP_CHART *chart
+ , const char *metric_name
+ , const char *dim_name
+ , collected_number multiplier
+ , collected_number divisor
+ , RRDDIM_FLAGS flags
+ , RRDDIM_OPTIONS options
+ , STATSD_APP_CHART_DIM_VALUE_TYPE value_type
+) {
+ STATSD_APP_CHART_DIM *dim = callocz(sizeof(STATSD_APP_CHART_DIM), 1);
+
+ dim->metric = strdupz(metric_name);
+ dim->metric_hash = simple_hash(dim->metric);
+
+ dim->name = strdupz((dim_name)?dim_name:"");
+ dim->multiplier = multiplier;
+ dim->divisor = divisor;
+ dim->value_type = value_type;
+ dim->flags = flags;
+ dim->options = options;
+
+ if(!dim->multiplier)
+ dim->multiplier = 1;
+
+ if(!dim->divisor)
+ dim->divisor = 1;
+
+ // append it to the list of dimension
+ STATSD_APP_CHART_DIM *tdim;
+ for(tdim = chart->dimensions; tdim && tdim->next ; tdim = tdim->next) ;
+ if(!tdim) {
+ dim->next = chart->dimensions;
+ chart->dimensions = dim;
+ }
+ else {
+ dim->next = tdim->next;
+ tdim->next = dim;
+ }
+ chart->dimensions_count++;
+
+ debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT,
+ dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor);
+
+ return dim;
+}
+
+static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHART *chart, DICTIONARY *dict) {
+ debug(D_STATSD, "STATSD configuration reading file '%s'", filename);
+
+ char *buffer = mallocz(STATSD_CONF_LINE_MAX + 1);
+
+ FILE *fp = fopen(filename, "r");
+ if(!fp) {
+ error("STATSD: cannot open file '%s'.", filename);
+ freez(buffer);
+ return -1;
+ }
+
+ size_t line = 0;
+ char *s;
+ while(fgets(buffer, STATSD_CONF_LINE_MAX, fp) != NULL) {
+ buffer[STATSD_CONF_LINE_MAX] = '\0';
+ line++;
+
+ s = trim(buffer);
+ if (!s || *s == '#') {
+ debug(D_STATSD, "STATSD: ignoring line %zu of file '%s', it is empty.", line, filename);
+ continue;
+ }
+
+ debug(D_STATSD, "STATSD: processing line %zu of file '%s': %s", line, filename, buffer);
+
+ if(*s == 'i' && strncmp(s, "include", 7) == 0) {
+ s = trim(&s[7]);
+ if(s && *s) {
+ char *tmp;
+ if(*s == '/')
+ tmp = strdupz(s);
+ else {
+ // the file to be included is relative to current file
+ // find the directory name from the file we already read
+ char *filename2 = strdupz(filename); // copy filename, since dirname() will change it
+ char *dir = dirname(filename2); // find the directory part of the filename
+ tmp = strdupz_path_subpath(dir, s); // compose the new filename to read;
+ freez(filename2); // free the filename we copied
+ }
+ statsd_readfile(tmp, app, chart, dict);
+ freez(tmp);
+ }
+ else
+ error("STATSD: ignoring line %zu of file '%s', include filename is empty", line, filename);
+
+ continue;
+ }
+
+ int len = (int) strlen(s);
+ if (*s == '[' && s[len - 1] == ']') {
+ // new section
+ s[len - 1] = '\0';
+ s++;
+
+ if (!strcmp(s, "app")) {
+ // a new app
+ app = callocz(sizeof(STATSD_APP), 1);
+ app->name = strdupz("unnamed");
+ app->rrd_memory_mode = localhost->rrd_memory_mode;
+ app->rrd_history_entries = localhost->rrd_history_entries;
+
+ app->next = statsd.apps;
+ statsd.apps = app;
+ chart = NULL;
+ dict = NULL;
+
+ {
+ char lineandfile[FILENAME_MAX + 1];
+ snprintfz(lineandfile, FILENAME_MAX, "%zu@%s", line, filename);
+ app->source = strdupz(lineandfile);
+ }
+ }
+ else if(app) {
+ if(!strcmp(s, "dictionary")) {
+ if(!app->dict)
+ app->dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
+
+ dict = app->dict;
+ }
+ else {
+ dict = NULL;
+
+ // a new chart
+ chart = callocz(sizeof(STATSD_APP_CHART), 1);
+ netdata_fix_chart_id(s);
+ chart->id = strdupz(s);
+ chart->name = strdupz(s);
+ chart->title = strdupz("Statsd chart");
+ chart->context = strdupz(s);
+ chart->family = strdupz("overview");
+ chart->units = strdupz("value");
+ chart->priority = NETDATA_CHART_PRIO_STATSD_PRIVATE;
+ chart->chart_type = RRDSET_TYPE_LINE;
+
+ chart->next = app->charts;
+ app->charts = chart;
+
+ if (!strncmp(
+ filename,
+ netdata_configured_stock_config_dir,
+ strlen(netdata_configured_stock_config_dir))) {
+ char tmpfilename[FILENAME_MAX + 1];
+ strncpyz(tmpfilename, filename, FILENAME_MAX);
+ chart->module = strdupz(basename(tmpfilename));
+ } else {
+ chart->module = strdupz("synthetic_chart");
+ }
+ }
+ }
+ else
+ error("STATSD: ignoring line %zu ('%s') of file '%s', [app] is not defined.", line, s, filename);
+
+ continue;
+ }
+
+ if(!app) {
+ error("STATSD: ignoring line %zu ('%s') of file '%s', it is outside all sections.", line, s, filename);
+ continue;
+ }
+
+ char *name = s;
+ char *value = strchr(s, '=');
+ if(!value) {
+ error("STATSD: ignoring line %zu ('%s') of file '%s', there is no = in it.", line, s, filename);
+ continue;
+ }
+ *value = '\0';
+ value++;
+
+ name = trim(name);
+ value = trim(value);
+
+ if(!name || *name == '#') {
+ error("STATSD: ignoring line %zu of file '%s', name is empty.", line, filename);
+ continue;
+ }
+ if(!value) {
+ debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s', value is empty.", line, filename);
+ continue;
+ }
+
+ if(unlikely(dict)) {
+ // parse [dictionary] members
+
+ dictionary_set(dict, name, value, strlen(value) + 1);
+ }
+ else if(!chart) {
+ // parse [app] members
+
+ if(!strcmp(name, "name")) {
+ freez((void *)app->name);
+ netdata_fix_chart_name(value);
+ app->name = strdupz(value);
+ }
+ else if (!strcmp(name, "metrics")) {
+ simple_pattern_free(app->metrics);
+ app->metrics = simple_pattern_create(value, NULL, SIMPLE_PATTERN_EXACT);
+ }
+ else if (!strcmp(name, "private charts")) {
+ if (!strcmp(value, "yes") || !strcmp(value, "on"))
+ app->default_options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ else
+ app->default_options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ }
+ else if (!strcmp(name, "gaps when not collected")) {
+ if (!strcmp(value, "yes") || !strcmp(value, "on"))
+ app->default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+ }
+ else if (!strcmp(name, "memory mode")) {
+ // this is not supported anymore
+ // with the implementation of storage engines, all charts have the same storage engine always
+ // app->rrd_memory_mode = rrd_memory_mode_id(value);
+ ;
+ }
+ else if (!strcmp(name, "history")) {
+ app->rrd_history_entries = atol(value);
+ if (app->rrd_history_entries < 5)
+ app->rrd_history_entries = 5;
+ }
+ else {
+ error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [app] section.", line, name, filename);
+ continue;
+ }
+ }
+ else {
+ // parse [chart] members
+
+ if(!strcmp(name, "name")) {
+ freez((void *)chart->name);
+ netdata_fix_chart_id(value);
+ chart->name = strdupz(value);
+ }
+ else if(!strcmp(name, "title")) {
+ freez((void *)chart->title);
+ chart->title = strdupz(value);
+ }
+ else if (!strcmp(name, "family")) {
+ freez((void *)chart->family);
+ chart->family = strdupz(value);
+ }
+ else if (!strcmp(name, "context")) {
+ freez((void *)chart->context);
+ netdata_fix_chart_id(value);
+ chart->context = strdupz(value);
+ }
+ else if (!strcmp(name, "units")) {
+ freez((void *)chart->units);
+ chart->units = strdupz(value);
+ }
+ else if (!strcmp(name, "priority")) {
+ chart->priority = atol(value);
+ }
+ else if (!strcmp(name, "type")) {
+ chart->chart_type = rrdset_type_id(value);
+ }
+ else if (!strcmp(name, "dimension")) {
+ // metric [name [type [multiplier [divisor]]]]
+ char *words[10] = { NULL };
+ size_t num_words = pluginsd_split_words(value, words, 10, NULL, NULL, 0);
+
+ int pattern = 0;
+ size_t i = 0;
+ char *metric_name = get_word(words, num_words, i++);
+
+ if(strcmp(metric_name, "pattern") == 0) {
+ metric_name = get_word(words, num_words, i++);
+ pattern = 1;
+ }
+
+ char *dim_name = get_word(words, num_words, i++);
+ char *type = get_word(words, num_words, i++);
+ char *multiplier = get_word(words, num_words, i++);
+ char *divisor = get_word(words, num_words, i++);
+ char *opts = get_word(words, num_words, i++);
+
+ RRDDIM_FLAGS flags = RRDDIM_FLAG_NONE;
+ RRDDIM_OPTIONS options = RRDDIM_OPTION_NONE;
+ if(opts && *opts) {
+ if(strstr(opts, "hidden") != NULL) options |= RRDDIM_OPTION_HIDDEN;
+ if(strstr(opts, "noreset") != NULL) options |= RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS;
+ if(strstr(opts, "nooverflow") != NULL) options |= RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS;
+ }
+
+ if(!pattern) {
+ if(app->dict) {
+ if(dim_name && *dim_name) {
+ char *n = dictionary_get(app->dict, dim_name);
+ if(n) dim_name = n;
+ }
+ else {
+ dim_name = dictionary_get(app->dict, metric_name);
+ }
+ }
+
+ if(!dim_name || !*dim_name)
+ dim_name = metric_name;
+ }
+
+ STATSD_APP_CHART_DIM *dim = add_dimension_to_app_chart(
+ app
+ , chart
+ , metric_name
+ , dim_name
+ , (multiplier && *multiplier)?str2l(multiplier):1
+ , (divisor && *divisor)?str2l(divisor):1
+ , flags
+ ,
+ options, string2valuetype(type, line, filename)
+ );
+
+ if(pattern)
+ dim->metric_pattern = simple_pattern_create(dim->metric, NULL, SIMPLE_PATTERN_EXACT);
+ }
+ else {
+ error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [%s] section.", line, name, filename, chart->id);
+ continue;
+ }
+ }
+ }
+
+ freez(buffer);
+ fclose(fp);
+ return 0;
+}
+
+static int statsd_file_callback(const char *filename, void *data) {
+ (void)data;
+ return statsd_readfile(filename, NULL, NULL, NULL);
+}
+
+static inline void statsd_readdir(const char *user_path, const char *stock_path, const char *subpath) {
+ recursive_config_double_dir_load(user_path, stock_path, subpath, statsd_file_callback, NULL, 0);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// send metrics to netdata - in private charts - called from the main thread
+
+// extract chart type and chart id from metric name
+static inline void statsd_get_metric_type_and_id(STATSD_METRIC *m, char *type, char *id, char *context, const char *metrictype, size_t len) {
+
+ // The full chart type.id looks like this:
+ // ${STATSD_CHART_PREFIX} + "_" + ${METRIC_NAME} + "_" + ${METRIC_TYPE}
+ //
+ // where:
+ // STATSD_CHART_PREFIX = "statsd" as defined above
+ // METRIC_NAME = whatever the user gave to statsd
+ // METRIC_TYPE = "gauge", "counter", "meter", "timer", "histogram", "set", "dictionary"
+
+ // for chart type, we want:
+ // ${STATSD_CHART_PREFIX} + "_" + the first word of ${METRIC_NAME}
+
+ // find the first word of ${METRIC_NAME}
+ char firstword[len + 1], *s = "";
+ strncpyz(firstword, m->name, len);
+ for (s = firstword; *s ; s++) {
+ if (unlikely(*s == '.' || *s == '_')) {
+ *s = '\0';
+ s++;
+ break;
+ }
+ }
+ // firstword has the first word of ${METRIC_NAME}
+ // s has the remaining, if any
+
+ // create the chart type:
+ snprintfz(type, len, STATSD_CHART_PREFIX "_%s", firstword);
+
+ // for chart id, we want:
+ // the remaining of the words of ${METRIC_NAME} + "_" + ${METRIC_TYPE}
+ // or the ${METRIC_NAME} has no remaining words, the ${METRIC_TYPE} alone
+ if(*s)
+ snprintfz(id, len, "%s_%s", s, metrictype);
+ else
+ snprintfz(id, len, "%s", metrictype);
+
+ // for the context, we want the full of both the above, separated with a dot (type.id):
+ snprintfz(context, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
+
+ // make sure they don't have illegal characters
+ netdata_fix_chart_id(type);
+ netdata_fix_chart_id(id);
+ netdata_fix_chart_id(context);
+}
+
+static inline RRDSET *statsd_private_rrdset_create(
+ STATSD_METRIC *m __maybe_unused
+ , const char *type
+ , const char *id
+ , const char *name
+ , const char *family
+ , const char *context
+ , const char *title
+ , const char *units
+ , long priority
+ , int update_every
+ , RRDSET_TYPE chart_type
+) {
+ if(!m->st)
+ statsd.private_charts++;
+
+ RRDSET *st = rrdset_create_custom(
+ localhost // host
+ , type // type
+ , id // id
+ , name // name
+ , family // family
+ , context // context
+ , title // title
+ , units // units
+ , PLUGIN_STATSD_NAME // plugin
+ , "private_chart" // module
+ , priority // priority
+ , update_every // update every
+ , chart_type // chart type
+ , default_rrd_memory_mode // memory mode
+ , default_rrd_history_entries // history
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
+
+ if(statsd.private_charts_hidden)
+ rrdset_flag_set(st, RRDSET_FLAG_HIDDEN);
+
+ // rrdset_flag_set(st, RRDSET_FLAG_DEBUG);
+ return st;
+}
+
+static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {
+ debug(D_STATSD, "updating private chart for gauge metric '%s'", m->name);
+
+ if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {
+ m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;
+
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, context, "gauge", RRD_ID_LENGTH_MAX);
+
+ char title[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for gauge %s", m->name);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , m->family?m->family:"gauges" // family (submenu)
+ , context // context
+ , title // title
+ , m->units?m->units:"value" // units
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
+ , statsd.update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ m->rd_value = rrddim_add(m->st, "gauge", m->dimname?m->dimname:NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(m->st, m->rd_value, m->last);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const char *dim, const char *family) {
+ debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name);
+
+ if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {
+ m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;
+
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, context, dim, RRD_ID_LENGTH_MAX);
+
+ char title[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for %s %s", dim, m->name);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , m->family?m->family:family // family (submenu)
+ , context // context
+ , title // title
+ , m->units?m->units:"events/s" // units
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
+ , statsd.update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ m->rd_value = rrddim_add(m->st, dim, m->dimname?m->dimname:NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(m->st, m->rd_value, m->last);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+static inline void statsd_private_chart_set(STATSD_METRIC *m) {
+ debug(D_STATSD, "updating private chart for set metric '%s'", m->name);
+
+ if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {
+ m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;
+
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, context, "set", RRD_ID_LENGTH_MAX);
+
+ char title[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for set %s", m->name);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , m->family?m->family:"sets" // family (submenu)
+ , context // context
+ , title // title
+ , m->units?m->units:"entries" // units
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
+ , statsd.update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ m->rd_value = rrddim_add(m->st, "set", m->dimname?m->dimname:"unique", 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(m->st, m->rd_value, m->last);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+static inline void statsd_private_chart_dictionary(STATSD_METRIC *m) {
+ debug(D_STATSD, "updating private chart for dictionary metric '%s'", m->name);
+
+ if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {
+ m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;
+
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, context, "dictionary", RRD_ID_LENGTH_MAX);
+
+ char title[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for dictionary %s", m->name);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , m->family?m->family:"dictionaries" // family (submenu)
+ , context // context
+ , title // title
+ , m->units?m->units:"events/s" // units
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
+ , statsd.update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ STATSD_METRIC_DICTIONARY_ITEM *t;
+ dfe_start_read(m->dictionary.dict, t) {
+ if (!t->rd) t->rd = rrddim_add(m->st, t_dfe.name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_set_by_pointer(m->st, t->rd, (collected_number)t->count);
+ }
+ dfe_done(t);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {
+ debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name);
+
+ if(unlikely(!m->st || m->options & STATSD_METRIC_OPTION_UPDATED_CHART_METADATA)) {
+ m->options &= ~STATSD_METRIC_OPTION_UPDATED_CHART_METADATA;
+
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, context, dim, RRD_ID_LENGTH_MAX);
+
+ char title[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for %s %s", dim, m->name);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , m->family?m->family:family // family (submenu)
+ , context // context
+ , title // title
+ , m->units?m->units:units // units
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
+ , statsd.update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ m->histogram.ext->rd_min = rrddim_add(m->st, "min", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_max = rrddim_add(m->st, "max", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_value = rrddim_add(m->st, "average", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_percentile = rrddim_add(m->st, statsd.histogram_percentile_str, NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_median = rrddim_add(m->st, "median", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_stddev = rrddim_add(m->st, "stddev", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+ //m->histogram.ext->rd_sum = rrddim_add(m->st, "sum", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_min, m->histogram.ext->last_min);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_max, m->histogram.ext->last_max);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_percentile, m->histogram.ext->last_percentile);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_median, m->histogram.ext->last_median);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_stddev, m->histogram.ext->last_stddev);
+ //rrddim_set_by_pointer(m->st, m->histogram.ext->rd_sum, m->histogram.ext->last_sum);
+ rrddim_set_by_pointer(m->st, m->rd_value, m->last);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd flush metrics
+
+static inline void statsd_flush_gauge(STATSD_METRIC *m) {
+ debug(D_STATSD, "flushing gauge metric '%s'", m->name);
+
+ int updated = 0;
+ if(unlikely(!m->reset && m->count)) {
+ m->last = (collected_number) (m->gauge.value * statsd.decimal_detail);
+
+ m->reset = 1;
+ updated = 1;
+ }
+
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
+ statsd_private_chart_gauge(m);
+}
+
+static inline void statsd_flush_counter_or_meter(STATSD_METRIC *m, const char *dim, const char *family) {
+ debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
+
+ int updated = 0;
+ if(unlikely(!m->reset && m->count)) {
+ m->last = m->counter.value;
+
+ m->reset = 1;
+ updated = 1;
+ }
+
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
+ statsd_private_chart_counter_or_meter(m, dim, family);
+}
+
+static inline void statsd_flush_counter(STATSD_METRIC *m) {
+ statsd_flush_counter_or_meter(m, "counter", "counters");
+}
+
+static inline void statsd_flush_meter(STATSD_METRIC *m) {
+ statsd_flush_counter_or_meter(m, "meter", "meters");
+}
+
+static inline void statsd_flush_set(STATSD_METRIC *m) {
+ debug(D_STATSD, "flushing set metric '%s'", m->name);
+
+ int updated = 0;
+ if(unlikely(!m->reset && m->count)) {
+ m->last = (collected_number)m->set.unique;
+
+ m->reset = 1;
+ updated = 1;
+ }
+ else {
+ m->last = 0;
+ }
+
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
+ statsd_private_chart_set(m);
+}
+
+static inline void statsd_flush_dictionary(STATSD_METRIC *m) {
+ debug(D_STATSD, "flushing dictionary metric '%s'", m->name);
+
+ int updated = 0;
+ if(unlikely(!m->reset && m->count)) {
+ m->last = (collected_number)m->dictionary.unique;
+
+ m->reset = 1;
+ updated = 1;
+ }
+ else {
+ m->last = 0;
+ }
+
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
+ statsd_private_chart_dictionary(m);
+
+ if(m->dictionary.unique >= statsd.dictionary_max_unique) {
+ if(!(m->options & STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED)) {
+ m->options |= STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED;
+ info(
+ "STATSD dictionary '%s' reach max of %zu items - try increasing 'dictionaries max unique dimensions' in netdata.conf",
+ m->name,
+ m->dictionary.unique);
+ }
+ }
+}
+
+static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {
+ debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
+
+ int updated = 0;
+ if(unlikely(!m->reset && m->count && m->histogram.ext->used > 0)) {
+ netdata_mutex_lock(&m->histogram.ext->mutex);
+
+ size_t len = m->histogram.ext->used;
+ NETDATA_DOUBLE *series = m->histogram.ext->values;
+ sort_series(series, len);
+
+ m->histogram.ext->last_min = (collected_number)roundndd(series[0] * statsd.decimal_detail);
+ m->histogram.ext->last_max = (collected_number)roundndd(series[len - 1] * statsd.decimal_detail);
+ m->last = (collected_number)roundndd(average(series, len) * statsd.decimal_detail);
+ m->histogram.ext->last_median = (collected_number)roundndd(median_on_sorted_series(series, len) * statsd.decimal_detail);
+ m->histogram.ext->last_stddev = (collected_number)roundndd(standard_deviation(series, len) * statsd.decimal_detail);
+ m->histogram.ext->last_sum = (collected_number)roundndd(sum(series, len) * statsd.decimal_detail);
+
+ size_t pct_len = (size_t)floor((double)len * statsd.histogram_percentile / 100.0);
+ if(pct_len < 1)
+ m->histogram.ext->last_percentile = (collected_number)(series[0] * statsd.decimal_detail);
+ else
+ m->histogram.ext->last_percentile = (collected_number)roundndd(series[pct_len - 1] * statsd.decimal_detail);
+
+ netdata_mutex_unlock(&m->histogram.ext->mutex);
+
+ debug(D_STATSD, "STATSD %s metric %s: min " COLLECTED_NUMBER_FORMAT ", max " COLLECTED_NUMBER_FORMAT ", last " COLLECTED_NUMBER_FORMAT ", pcent " COLLECTED_NUMBER_FORMAT ", median " COLLECTED_NUMBER_FORMAT ", stddev " COLLECTED_NUMBER_FORMAT ", sum " COLLECTED_NUMBER_FORMAT,
+ dim, m->name, m->histogram.ext->last_min, m->histogram.ext->last_max, m->last, m->histogram.ext->last_percentile, m->histogram.ext->last_median, m->histogram.ext->last_stddev, m->histogram.ext->last_sum);
+
+ m->histogram.ext->zeroed = 0;
+ m->reset = 1;
+ updated = 1;
+ }
+ else if(unlikely(!m->histogram.ext->zeroed)) {
+ // reset the metrics
+ // if we collected anything, they will be updated below
+ // this ensures that we report zeros if nothing is collected
+
+ m->histogram.ext->last_min = 0;
+ m->histogram.ext->last_max = 0;
+ m->last = 0;
+ m->histogram.ext->last_median = 0;
+ m->histogram.ext->last_stddev = 0;
+ m->histogram.ext->last_sum = 0;
+ m->histogram.ext->last_percentile = 0;
+
+ m->histogram.ext->zeroed = 1;
+ }
+
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
+ statsd_private_chart_timer_or_histogram(m, dim, family, units);
+}
+
+static inline void statsd_flush_timer(STATSD_METRIC *m) {
+ statsd_flush_timer_or_histogram(m, "timer", "timers", "milliseconds");
+}
+
+static inline void statsd_flush_histogram(STATSD_METRIC *m) {
+ statsd_flush_timer_or_histogram(m, "histogram", "histograms", "value");
+}
+
+static inline RRD_ALGORITHM statsd_algorithm_for_metric(STATSD_METRIC *m) {
+ switch(m->type) {
+ default:
+ case STATSD_METRIC_TYPE_GAUGE:
+ case STATSD_METRIC_TYPE_SET:
+ case STATSD_METRIC_TYPE_TIMER:
+ case STATSD_METRIC_TYPE_HISTOGRAM:
+ return RRD_ALGORITHM_ABSOLUTE;
+
+ case STATSD_METRIC_TYPE_METER:
+ case STATSD_METRIC_TYPE_COUNTER:
+ case STATSD_METRIC_TYPE_DICTIONARY:
+ return RRD_ALGORITHM_INCREMENTAL;
+ }
+}
+
+static inline void link_metric_to_app_dimension(STATSD_APP *app, STATSD_METRIC *m, STATSD_APP_CHART *chart, STATSD_APP_CHART_DIM *dim) {
+ if(dim->value_type == STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS) {
+ dim->value_ptr = &m->events;
+ dim->algorithm = RRD_ALGORITHM_INCREMENTAL;
+ }
+ else if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
+ dim->algorithm = RRD_ALGORITHM_ABSOLUTE;
+ dim->divisor *= statsd.decimal_detail;
+
+ switch(dim->value_type) {
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS:
+ // will never match - added to avoid warning
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST:
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE:
+ dim->value_ptr = &m->last;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM:
+ dim->value_ptr = &m->histogram.ext->last_sum;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN:
+ dim->value_ptr = &m->histogram.ext->last_min;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX:
+ dim->value_ptr = &m->histogram.ext->last_max;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN:
+ dim->value_ptr = &m->histogram.ext->last_median;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE:
+ dim->value_ptr = &m->histogram.ext->last_percentile;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV:
+ dim->value_ptr = &m->histogram.ext->last_stddev;
+ break;
+ }
+ }
+ else {
+ if (dim->value_type != STATSD_APP_CHART_DIM_VALUE_TYPE_LAST)
+ error("STATSD: unsupported value type for dimension '%s' of chart '%s' of app '%s' on metric '%s'", dim->name, chart->id, app->name, m->name);
+
+ dim->value_ptr = &m->last;
+ dim->algorithm = statsd_algorithm_for_metric(m);
+
+ if(m->type == STATSD_METRIC_TYPE_GAUGE)
+ dim->divisor *= statsd.decimal_detail;
+ }
+
+ if(unlikely(chart->st && dim->rd)) {
+ rrddim_set_algorithm(chart->st, dim->rd, dim->algorithm);
+ rrddim_set_multiplier(chart->st, dim->rd, dim->multiplier);
+ rrddim_set_divisor(chart->st, dim->rd, dim->divisor);
+ }
+
+ chart->dimensions_linked_count++;
+ m->options |= STATSD_METRIC_OPTION_USED_IN_APPS;
+ debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm));
+}
+
+static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC *m) {
+ (void)index;
+
+ STATSD_APP *app;
+ for(app = statsd.apps; app ;app = app->next) {
+ if(unlikely(simple_pattern_matches(app->metrics, m->name))) {
+ debug(D_STATSD, "metric '%s' matches app '%s'", m->name, app->name);
+
+ // the metric should get the options from the app
+
+ if(app->default_options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED)
+ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ else
+ m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+
+ if(app->default_options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)
+ m->options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+ else
+ m->options &= ~STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED;
+
+ // check if there is a chart in this app, willing to get this metric
+ STATSD_APP_CHART *chart;
+ for(chart = app->charts; chart; chart = chart->next) {
+
+ STATSD_APP_CHART_DIM *dim;
+ for(dim = chart->dimensions; dim ; dim = dim->next) {
+ if(unlikely(dim->metric_pattern)) {
+ size_t dim_name_len = strlen(dim->name);
+ size_t wildcarded_len = dim_name_len + strlen(m->name) + 1;
+ char wildcarded[wildcarded_len];
+
+ strcpy(wildcarded, dim->name);
+ char *ws = &wildcarded[dim_name_len];
+
+ if(simple_pattern_matches_extract(dim->metric_pattern, m->name, ws, wildcarded_len - dim_name_len)) {
+
+ char *final_name = NULL;
+
+ if(app->dict) {
+ if(likely(*wildcarded)) {
+ // use the name of the wildcarded string
+ final_name = dictionary_get(app->dict, wildcarded);
+ }
+
+ if(unlikely(!final_name)) {
+ // use the name of the metric
+ final_name = dictionary_get(app->dict, m->name);
+ }
+ }
+
+ if(unlikely(!final_name))
+ final_name = wildcarded;
+
+ add_dimension_to_app_chart(
+ app
+ , chart
+ , m->name
+ , final_name
+ , dim->multiplier
+ , dim->divisor
+ , dim->flags
+ , dim->options
+ , dim->value_type
+ );
+
+ // the new dimension is appended to the list
+ // so, it will be matched and linked later too
+ }
+ }
+ else if(!dim->value_ptr && dim->metric_hash == m->hash && !strcmp(dim->metric, m->name)) {
+ // we have a match - this metric should be linked to this dimension
+ link_metric_to_app_dimension(app, m, chart, dim);
+ }
+ }
+
+ }
+ }
+ }
+}
+
+static inline RRDDIM *statsd_add_dim_to_app_chart(STATSD_APP *app, STATSD_APP_CHART *chart, STATSD_APP_CHART_DIM *dim) {
+ (void)app;
+
+ // allow the same statsd metric to be added multiple times to the same chart
+
+ STATSD_APP_CHART_DIM *tdim;
+ size_t count_same_metric = 0, count_same_metric_value_type = 0;
+ size_t pos_same_metric_value_type = 0;
+
+ for (tdim = chart->dimensions; tdim && tdim->next; tdim = tdim->next) {
+ if (dim->metric_hash == tdim->metric_hash && !strcmp(dim->metric, tdim->metric)) {
+ count_same_metric++;
+
+ if(dim->value_type == tdim->value_type) {
+ count_same_metric_value_type++;
+ if (tdim == dim)
+ pos_same_metric_value_type = count_same_metric_value_type;
+ }
+ }
+ }
+
+ if(count_same_metric > 1) {
+ // the same metric is found multiple times
+
+ size_t len = strlen(dim->metric) + 100;
+ char metric[ len + 1 ];
+
+ if(count_same_metric_value_type > 1) {
+ // the same metric, with the same value type, is added multiple times
+ snprintfz(metric, len, "%s_%s%zu", dim->metric, valuetype2string(dim->value_type), pos_same_metric_value_type);
+ }
+ else {
+ // the same metric, with different value type is added
+ snprintfz(metric, len, "%s_%s", dim->metric, valuetype2string(dim->value_type));
+ }
+
+ dim->rd = rrddim_add(chart->st, metric, dim->name, dim->multiplier, dim->divisor, dim->algorithm);
+ if(dim->flags != RRDDIM_FLAG_NONE) dim->rd->flags |= dim->flags;
+ if(dim->options != RRDDIM_OPTION_NONE) dim->rd->options |= dim->options;
+ return dim->rd;
+ }
+
+ dim->rd = rrddim_add(chart->st, dim->metric, dim->name, dim->multiplier, dim->divisor, dim->algorithm);
+ if(dim->flags != RRDDIM_FLAG_NONE) dim->rd->flags |= dim->flags;
+ if(dim->options != RRDDIM_OPTION_NONE) dim->rd->options |= dim->options;
+ return dim->rd;
+}
+
+static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *chart) {
+ debug(D_STATSD, "updating chart '%s' for app '%s'", chart->id, app->name);
+
+ if(!chart->st) {
+ chart->st = rrdset_create_custom(
+ localhost // host
+ , app->name // type
+ , chart->id // id
+ , chart->name // name
+ , chart->family // family
+ , chart->context // context
+ , chart->title // title
+ , chart->units // units
+ , PLUGIN_STATSD_NAME // plugin
+ , chart->module // module
+ , chart->priority // priority
+ , statsd.update_every // update every
+ , chart->chart_type // chart type
+ , app->rrd_memory_mode // memory mode
+ , app->rrd_history_entries // history
+ );
+
+ rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);
+ // rrdset_flag_set(chart->st, RRDSET_FLAG_DEBUG);
+ }
+
+ STATSD_APP_CHART_DIM *dim;
+ for(dim = chart->dimensions; dim ;dim = dim->next) {
+ if(likely(!dim->metric_pattern)) {
+ if (unlikely(!dim->rd))
+ statsd_add_dim_to_app_chart(app, chart, dim);
+
+ if (unlikely(dim->value_ptr)) {
+ debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, rrddim_id(dim->rd), chart->id, rrdset_id(chart->st), app->name, *dim->value_ptr);
+ rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr);
+ }
+ }
+ }
+
+ rrdset_done(chart->st);
+ debug(D_STATSD, "completed update of chart '%s' for app '%s'", chart->id, app->name);
+}
+
+static inline void statsd_update_all_app_charts(void) {
+ // debug(D_STATSD, "updating app charts");
+
+ STATSD_APP *app;
+ for(app = statsd.apps; app ;app = app->next) {
+ // debug(D_STATSD, "updating charts for app '%s'", app->name);
+
+ STATSD_APP_CHART *chart;
+ for(chart = app->charts; chart ;chart = chart->next) {
+ if(unlikely(chart->dimensions_linked_count)) {
+ statsd_update_app_chart(app, chart);
+ }
+ }
+ }
+
+ // debug(D_STATSD, "completed update of app charts");
+}
+
+const char *statsd_metric_type_string(STATSD_METRIC_TYPE type) {
+ switch(type) {
+ case STATSD_METRIC_TYPE_COUNTER: return "counter";
+ case STATSD_METRIC_TYPE_GAUGE: return "gauge";
+ case STATSD_METRIC_TYPE_HISTOGRAM: return "histogram";
+ case STATSD_METRIC_TYPE_METER: return "meter";
+ case STATSD_METRIC_TYPE_SET: return "set";
+ case STATSD_METRIC_TYPE_DICTIONARY: return "dictionary";
+ case STATSD_METRIC_TYPE_TIMER: return "timer";
+ default: return "unknown";
+ }
+}
+
+static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_metric)(STATSD_METRIC *)) {
+ STATSD_METRIC *m;
+
+ // find the useful metrics (incremental = each time we are called, we check the new metrics only)
+ dfe_start_read(index->dict, m) {
+ // since we add new metrics at the beginning
+ // check for useful charts, until the point we last checked
+ if(unlikely(is_metric_checked(m))) break;
+
+ if(unlikely(!(m->options & STATSD_METRIC_OPTION_CHECKED_IN_APPS))) {
+ log_access("NEW STATSD METRIC '%s': '%s'", statsd_metric_type_string(m->type), m->name);
+ check_if_metric_is_for_app(index, m);
+ m->options |= STATSD_METRIC_OPTION_CHECKED_IN_APPS;
+ }
+
+ if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) {
+ if(unlikely(statsd.private_charts >= statsd.max_private_charts_hard)) {
+ debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name);
+ info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts_hard);
+ m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ }
+ else {
+ if (simple_pattern_matches(statsd.charts_for, m->name)) {
+ debug(D_STATSD, "STATSD: metric '%s' will be charted.", m->name);
+ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ } else {
+ debug(D_STATSD, "STATSD: metric '%s' will not be charted.", m->name);
+ m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ }
+ }
+
+ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED;
+ }
+
+ // mark it as checked
+ m->options |= STATSD_METRIC_OPTION_CHECKED;
+
+ // check if it is used in charts
+ if((m->options & (STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED|STATSD_METRIC_OPTION_USED_IN_APPS)) && !(m->options & STATSD_METRIC_OPTION_USEFUL)) {
+ m->options |= STATSD_METRIC_OPTION_USEFUL;
+ index->useful++;
+ m->next_useful = index->first_useful;
+ index->first_useful = m;
+ }
+ }
+ dfe_done(m);
+
+ // flush all the useful metrics
+ for(m = index->first_useful; m ; m = m->next_useful) {
+ flush_metric(m);
+ }
+}
+
+
+// --------------------------------------------------------------------------------------
+// statsd main thread
+
+static int statsd_listen_sockets_setup(void) {
+ return listen_sockets_setup(&statsd.sockets);
+}
+
+static void statsd_main_cleanup(void *data) {
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)data;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+ info("cleaning up...");
+
+ if (statsd.collection_threads_status) {
+ int i;
+ for (i = 0; i < statsd.threads; i++) {
+ if(statsd.collection_threads_status[i].status) {
+ info("STATSD: stopping data collection thread %d...", i + 1);
+ netdata_thread_cancel(statsd.collection_threads_status[i].thread);
+ }
+ else {
+ info("STATSD: data collection thread %d found stopped.", i + 1);
+ }
+ }
+ }
+
+ info("STATSD: closing sockets...");
+ listen_sockets_close(&statsd.sockets);
+
+ // destroy the dictionaries
+ dictionary_destroy(statsd.gauges.dict);
+ dictionary_destroy(statsd.meters.dict);
+ dictionary_destroy(statsd.counters.dict);
+ dictionary_destroy(statsd.histograms.dict);
+ dictionary_destroy(statsd.dictionaries.dict);
+ dictionary_destroy(statsd.sets.dict);
+ dictionary_destroy(statsd.timers.dict);
+
+ info("STATSD: cleanup completed.");
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+
+ worker_unregister();
+}
+
+#define WORKER_STATSD_FLUSH_GAUGES 0
+#define WORKER_STATSD_FLUSH_COUNTERS 1
+#define WORKER_STATSD_FLUSH_METERS 2
+#define WORKER_STATSD_FLUSH_TIMERS 3
+#define WORKER_STATSD_FLUSH_HISTOGRAMS 4
+#define WORKER_STATSD_FLUSH_SETS 5
+#define WORKER_STATSD_FLUSH_DICTIONARIES 6
+#define WORKER_STATSD_FLUSH_STATS 7
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 8
+#endif
+
+void *statsd_main(void *ptr) {
+ worker_register("STATSDFLUSH");
+ worker_register_job_name(WORKER_STATSD_FLUSH_GAUGES, "gauges");
+ worker_register_job_name(WORKER_STATSD_FLUSH_COUNTERS, "counters");
+ worker_register_job_name(WORKER_STATSD_FLUSH_METERS, "meters");
+ worker_register_job_name(WORKER_STATSD_FLUSH_TIMERS, "timers");
+ worker_register_job_name(WORKER_STATSD_FLUSH_HISTOGRAMS, "histograms");
+ worker_register_job_name(WORKER_STATSD_FLUSH_SETS, "sets");
+ worker_register_job_name(WORKER_STATSD_FLUSH_DICTIONARIES, "dictionaries");
+ worker_register_job_name(WORKER_STATSD_FLUSH_STATS, "statistics");
+
+ netdata_thread_cleanup_push(statsd_main_cleanup, ptr);
+
+ statsd.gauges.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+ statsd.meters.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+ statsd.counters.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+ statsd.histograms.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+ statsd.dictionaries.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+ statsd.sets.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+ statsd.timers.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+
+ dictionary_register_insert_callback(statsd.gauges.dict, dictionary_metric_insert_callback, &statsd.gauges);
+ dictionary_register_insert_callback(statsd.meters.dict, dictionary_metric_insert_callback, &statsd.meters);
+ dictionary_register_insert_callback(statsd.counters.dict, dictionary_metric_insert_callback, &statsd.counters);
+ dictionary_register_insert_callback(statsd.histograms.dict, dictionary_metric_insert_callback, &statsd.histograms);
+ dictionary_register_insert_callback(statsd.dictionaries.dict, dictionary_metric_insert_callback, &statsd.dictionaries);
+ dictionary_register_insert_callback(statsd.sets.dict, dictionary_metric_insert_callback, &statsd.sets);
+ dictionary_register_insert_callback(statsd.timers.dict, dictionary_metric_insert_callback, &statsd.timers);
+
+ dictionary_register_delete_callback(statsd.gauges.dict, dictionary_metric_delete_callback, &statsd.gauges);
+ dictionary_register_delete_callback(statsd.meters.dict, dictionary_metric_delete_callback, &statsd.meters);
+ dictionary_register_delete_callback(statsd.counters.dict, dictionary_metric_delete_callback, &statsd.counters);
+ dictionary_register_delete_callback(statsd.histograms.dict, dictionary_metric_delete_callback, &statsd.histograms);
+ dictionary_register_delete_callback(statsd.dictionaries.dict, dictionary_metric_delete_callback, &statsd.dictionaries);
+ dictionary_register_delete_callback(statsd.sets.dict, dictionary_metric_delete_callback, &statsd.sets);
+ dictionary_register_delete_callback(statsd.timers.dict, dictionary_metric_delete_callback, &statsd.timers);
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // statsd configuration
+
+ statsd.enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, "statsd", statsd.enabled);
+
+ statsd.update_every = default_rrd_update_every;
+ statsd.update_every = (int)config_get_number(CONFIG_SECTION_STATSD, "update every (flushInterval)", statsd.update_every);
+ if(statsd.update_every < default_rrd_update_every) {
+ error("STATSD: minimum flush interval %d given, but the minimum is the update every of netdata. Using %d", statsd.update_every, default_rrd_update_every);
+ statsd.update_every = default_rrd_update_every;
+ }
+
+#ifdef HAVE_RECVMMSG
+ statsd.recvmmsg_size = (size_t)config_get_number(CONFIG_SECTION_STATSD, "udp messages to process at once", (long long)statsd.recvmmsg_size);
+#endif
+
+ statsd.charts_for = simple_pattern_create(config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
+ statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts_hard);
+ statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
+ statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
+ statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout);
+ statsd.private_charts_hidden = (unsigned int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden);
+
+ statsd.histogram_percentile = (double)config_get_float(CONFIG_SECTION_STATSD, "histograms and timers percentile (percentThreshold)", statsd.histogram_percentile);
+ if(isless(statsd.histogram_percentile, 0) || isgreater(statsd.histogram_percentile, 100)) {
+ error("STATSD: invalid histograms and timers percentile %0.5f given", statsd.histogram_percentile);
+ statsd.histogram_percentile = 95.0;
+ }
+ {
+ char buffer[314 + 1];
+ snprintfz(buffer, 314, "%0.1f%%", statsd.histogram_percentile);
+ statsd.histogram_percentile_str = strdupz(buffer);
+ }
+
+ statsd.dictionary_max_unique = config_get_number(CONFIG_SECTION_STATSD, "dictionaries max unique dimensions", statsd.dictionary_max_unique);
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "add dimension for number of events received", 0)) {
+ statsd.gauges.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.counters.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.meters.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.sets.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.histograms.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.timers.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.dictionaries.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ }
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on gauges (deleteGauges)", 0))
+ statsd.gauges.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on counters (deleteCounters)", 0))
+ statsd.counters.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on meters (deleteMeters)", 0))
+ statsd.meters.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on sets (deleteSets)", 0))
+ statsd.sets.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on histograms (deleteHistograms)", 0))
+ statsd.histograms.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on timers (deleteTimers)", 0))
+ statsd.timers.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on dictionaries (deleteDictionaries)", 0))
+ statsd.dictionaries.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ size_t max_sockets = (size_t)config_get_number(CONFIG_SECTION_STATSD, "statsd server max TCP sockets", (long long int)(rlimit_nofile.rlim_cur / 4));
+
+#ifdef STATSD_MULTITHREADED
+ statsd.threads = (int)config_get_number(CONFIG_SECTION_STATSD, "threads", processors);
+ if(statsd.threads < 1) {
+ error("STATSD: Invalid number of threads %d, using %d", statsd.threads, processors);
+ statsd.threads = processors;
+ config_set_number(CONFIG_SECTION_STATSD, "collector threads", statsd.threads);
+ }
+#else
+ statsd.threads = 1;
+#endif
+
+ // read custom application definitions
+ statsd_readdir(netdata_configured_user_config_dir, netdata_configured_stock_config_dir, "statsd.d");
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // statsd setup
+
+ if(!statsd.enabled) goto cleanup;
+
+ statsd_listen_sockets_setup();
+ if(!statsd.sockets.opened) {
+ error("STATSD: No statsd sockets to listen to. statsd will be disabled.");
+ goto cleanup;
+ }
+
+ statsd.collection_threads_status = callocz((size_t)statsd.threads, sizeof(struct collection_thread_status));
+
+ int i;
+ for(i = 0; i < statsd.threads ;i++) {
+ statsd.collection_threads_status[i].max_sockets = max_sockets / statsd.threads;
+ char tag[NETDATA_THREAD_TAG_MAX + 1];
+ snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STATSD_COLLECTOR[%d]", i + 1);
+ netdata_thread_create(&statsd.collection_threads_status[i].thread, tag, NETDATA_THREAD_OPTION_DEFAULT, statsd_collector_thread, &statsd.collection_threads_status[i]);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // statsd monitoring charts
+
+ RRDSET *st_metrics = NULL;
+ RRDDIM *rd_metrics_gauge = NULL;
+ RRDDIM *rd_metrics_counter = NULL;
+ RRDDIM *rd_metrics_timer = NULL;
+ RRDDIM *rd_metrics_meter = NULL;
+ RRDDIM *rd_metrics_histogram = NULL;
+ RRDDIM *rd_metrics_set = NULL;
+ RRDDIM *rd_metrics_dictionary = NULL;
+ RRDSET *st_useful_metrics = NULL;
+ RRDDIM *rd_useful_metrics_gauge = NULL;
+ RRDDIM *rd_useful_metrics_counter = NULL;
+ RRDDIM *rd_useful_metrics_timer = NULL;
+ RRDDIM *rd_useful_metrics_meter = NULL;
+ RRDDIM *rd_useful_metrics_histogram = NULL;
+ RRDDIM *rd_useful_metrics_set = NULL;
+ RRDDIM *rd_useful_metrics_dictionary = NULL;
+ RRDSET *st_events = NULL;
+ RRDDIM *rd_events_gauge = NULL;
+ RRDDIM *rd_events_counter = NULL;
+ RRDDIM *rd_events_timer = NULL;
+ RRDDIM *rd_events_meter = NULL;
+ RRDDIM *rd_events_histogram = NULL;
+ RRDDIM *rd_events_set = NULL;
+ RRDDIM *rd_events_dictionary = NULL;
+ RRDDIM *rd_events_unknown = NULL;
+ RRDDIM *rd_events_errors = NULL;
+ RRDSET *st_reads = NULL;
+ RRDDIM *rd_reads_tcp = NULL;
+ RRDDIM *rd_reads_udp = NULL;
+ RRDSET *st_bytes = NULL;
+ RRDDIM *rd_bytes_tcp = NULL;
+ RRDDIM *rd_bytes_udp = NULL;
+ RRDSET *st_packets = NULL;
+ RRDDIM *rd_packets_tcp = NULL;
+ RRDDIM *rd_packets_udp = NULL;
+ RRDSET *st_tcp_connects = NULL;
+ RRDDIM *rd_tcp_connects = NULL;
+ RRDDIM *rd_tcp_disconnects = NULL;
+ RRDSET *st_tcp_connected = NULL;
+ RRDDIM *rd_tcp_connected = NULL;
+ RRDSET *st_pcharts = NULL;
+ RRDDIM *rd_pcharts = NULL;
+
+ if(global_statistics_enabled) {
+ st_metrics = rrdset_create_localhost(
+ "netdata",
+ "statsd_metrics",
+ NULL,
+ "statsd",
+ NULL,
+ "Metrics in the netdata statsd database",
+ "metrics",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132010,
+ statsd.update_every,
+ RRDSET_TYPE_STACKED);
+ rd_metrics_gauge = rrddim_add(st_metrics, "gauges", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_metrics_counter = rrddim_add(st_metrics, "counters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_metrics_timer = rrddim_add(st_metrics, "timers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_metrics_meter = rrddim_add(st_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_metrics_histogram = rrddim_add(st_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_metrics_set = rrddim_add(st_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_metrics_dictionary = rrddim_add(st_metrics, "dictionaries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ st_useful_metrics = rrdset_create_localhost(
+ "netdata",
+ "statsd_useful_metrics",
+ NULL,
+ "statsd",
+ NULL,
+ "Useful metrics in the netdata statsd database",
+ "metrics",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132010,
+ statsd.update_every,
+ RRDSET_TYPE_STACKED);
+ rd_useful_metrics_gauge = rrddim_add(st_useful_metrics, "gauges", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_useful_metrics_counter = rrddim_add(st_useful_metrics, "counters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_useful_metrics_timer = rrddim_add(st_useful_metrics, "timers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_useful_metrics_meter = rrddim_add(st_useful_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_useful_metrics_histogram = rrddim_add(st_useful_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_useful_metrics_set = rrddim_add(st_useful_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_useful_metrics_dictionary = rrddim_add(st_useful_metrics, "dictionaries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ st_events = rrdset_create_localhost(
+ "netdata",
+ "statsd_events",
+ NULL,
+ "statsd",
+ NULL,
+ "Events processed by the netdata statsd server",
+ "events/s",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132011,
+ statsd.update_every,
+ RRDSET_TYPE_STACKED);
+ rd_events_gauge = rrddim_add(st_events, "gauges", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_events_counter = rrddim_add(st_events, "counters", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_events_timer = rrddim_add(st_events, "timers", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_events_meter = rrddim_add(st_events, "meters", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_events_histogram = rrddim_add(st_events, "histograms", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_events_set = rrddim_add(st_events, "sets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_events_dictionary = rrddim_add(st_events, "dictionaries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_events_unknown = rrddim_add(st_events, "unknown", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_events_errors = rrddim_add(st_events, "errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ st_reads = rrdset_create_localhost(
+ "netdata",
+ "statsd_reads",
+ NULL,
+ "statsd",
+ NULL,
+ "Read operations made by the netdata statsd server",
+ "reads/s",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132012,
+ statsd.update_every,
+ RRDSET_TYPE_STACKED);
+ rd_reads_tcp = rrddim_add(st_reads, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_reads_udp = rrddim_add(st_reads, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ st_bytes = rrdset_create_localhost(
+ "netdata",
+ "statsd_bytes",
+ NULL,
+ "statsd",
+ NULL,
+ "Bytes read by the netdata statsd server",
+ "kilobits/s",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132013,
+ statsd.update_every,
+ RRDSET_TYPE_STACKED);
+ rd_bytes_tcp = rrddim_add(st_bytes, "tcp", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ rd_bytes_udp = rrddim_add(st_bytes, "udp", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+
+ st_packets = rrdset_create_localhost(
+ "netdata",
+ "statsd_packets",
+ NULL,
+ "statsd",
+ NULL,
+ "Network packets processed by the netdata statsd server",
+ "packets/s",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132014,
+ statsd.update_every,
+ RRDSET_TYPE_STACKED);
+ rd_packets_tcp = rrddim_add(st_packets, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_packets_udp = rrddim_add(st_packets, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ st_tcp_connects = rrdset_create_localhost(
+ "netdata",
+ "tcp_connects",
+ NULL,
+ "statsd",
+ NULL,
+ "statsd server TCP connects and disconnects",
+ "events",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132015,
+ statsd.update_every,
+ RRDSET_TYPE_LINE);
+ rd_tcp_connects = rrddim_add(st_tcp_connects, "connects", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_tcp_disconnects = rrddim_add(st_tcp_connects, "disconnects", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ st_tcp_connected = rrdset_create_localhost(
+ "netdata",
+ "tcp_connected",
+ NULL,
+ "statsd",
+ NULL,
+ "statsd server TCP connected sockets",
+ "sockets",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132016,
+ statsd.update_every,
+ RRDSET_TYPE_LINE);
+ rd_tcp_connected = rrddim_add(st_tcp_connected, "connected", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ st_pcharts = rrdset_create_localhost(
+ "netdata",
+ "private_charts",
+ NULL,
+ "statsd",
+ NULL,
+ "Private metric charts created by the netdata statsd server",
+ "charts",
+ PLUGIN_STATSD_NAME,
+ "stats",
+ 132020,
+ statsd.update_every,
+ RRDSET_TYPE_AREA);
+ rd_pcharts = rrddim_add(st_pcharts, "charts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // statsd thread to turn metrics into charts
+
+ usec_t step = statsd.update_every * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ while(!netdata_exit) {
+ worker_is_idle();
+ heartbeat_next(&hb, step);
+
+ worker_is_busy(WORKER_STATSD_FLUSH_GAUGES);
+ statsd_flush_index_metrics(&statsd.gauges, statsd_flush_gauge);
+
+ worker_is_busy(WORKER_STATSD_FLUSH_COUNTERS);
+ statsd_flush_index_metrics(&statsd.counters, statsd_flush_counter);
+
+ worker_is_busy(WORKER_STATSD_FLUSH_METERS);
+ statsd_flush_index_metrics(&statsd.meters, statsd_flush_meter);
+
+ worker_is_busy(WORKER_STATSD_FLUSH_TIMERS);
+ statsd_flush_index_metrics(&statsd.timers, statsd_flush_timer);
+
+ worker_is_busy(WORKER_STATSD_FLUSH_HISTOGRAMS);
+ statsd_flush_index_metrics(&statsd.histograms, statsd_flush_histogram);
+
+ worker_is_busy(WORKER_STATSD_FLUSH_SETS);
+ statsd_flush_index_metrics(&statsd.sets, statsd_flush_set);
+
+ worker_is_busy(WORKER_STATSD_FLUSH_DICTIONARIES);
+ statsd_flush_index_metrics(&statsd.dictionaries,statsd_flush_dictionary);
+
+ worker_is_busy(WORKER_STATSD_FLUSH_STATS);
+ statsd_update_all_app_charts();
+
+ if(unlikely(netdata_exit))
+ break;
+
+ if(global_statistics_enabled) {
+ rrddim_set_by_pointer(st_metrics, rd_metrics_gauge, (collected_number)statsd.gauges.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_counter, (collected_number)statsd.counters.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_timer, (collected_number)statsd.timers.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_meter, (collected_number)statsd.meters.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_histogram, (collected_number)statsd.histograms.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_set, (collected_number)statsd.sets.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_dictionary, (collected_number)statsd.dictionaries.metrics);
+ rrdset_done(st_metrics);
+
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_gauge, (collected_number)statsd.gauges.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_counter, (collected_number)statsd.counters.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_timer, (collected_number)statsd.timers.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_meter, (collected_number)statsd.meters.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_histogram, (collected_number)statsd.histograms.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_set, (collected_number)statsd.sets.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_dictionary, (collected_number)statsd.dictionaries.useful);
+ rrdset_done(st_useful_metrics);
+
+ rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events);
+ rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events);
+ rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events);
+ rrddim_set_by_pointer(st_events, rd_events_meter, (collected_number)statsd.meters.events);
+ rrddim_set_by_pointer(st_events, rd_events_histogram, (collected_number)statsd.histograms.events);
+ rrddim_set_by_pointer(st_events, rd_events_set, (collected_number)statsd.sets.events);
+ rrddim_set_by_pointer(st_events, rd_events_dictionary, (collected_number)statsd.dictionaries.events);
+ rrddim_set_by_pointer(st_events, rd_events_unknown, (collected_number)statsd.unknown_types);
+ rrddim_set_by_pointer(st_events, rd_events_errors, (collected_number)statsd.socket_errors);
+ rrdset_done(st_events);
+
+ rrddim_set_by_pointer(st_reads, rd_reads_tcp, (collected_number)statsd.tcp_socket_reads);
+ rrddim_set_by_pointer(st_reads, rd_reads_udp, (collected_number)statsd.udp_socket_reads);
+ rrdset_done(st_reads);
+
+ rrddim_set_by_pointer(st_bytes, rd_bytes_tcp, (collected_number)statsd.tcp_bytes_read);
+ rrddim_set_by_pointer(st_bytes, rd_bytes_udp, (collected_number)statsd.udp_bytes_read);
+ rrdset_done(st_bytes);
+
+ rrddim_set_by_pointer(st_packets, rd_packets_tcp, (collected_number)statsd.tcp_packets_received);
+ rrddim_set_by_pointer(st_packets, rd_packets_udp, (collected_number)statsd.udp_packets_received);
+ rrdset_done(st_packets);
+
+ rrddim_set_by_pointer(st_tcp_connects, rd_tcp_connects, (collected_number)statsd.tcp_socket_connects);
+ rrddim_set_by_pointer(st_tcp_connects, rd_tcp_disconnects, (collected_number)statsd.tcp_socket_disconnects);
+ rrdset_done(st_tcp_connects);
+
+ rrddim_set_by_pointer(st_tcp_connected, rd_tcp_connected, (collected_number)statsd.tcp_socket_connected);
+ rrdset_done(st_tcp_connected);
+
+ rrddim_set_by_pointer(st_pcharts, rd_pcharts, (collected_number)statsd.private_charts);
+ rrdset_done(st_pcharts);
+ }
+ }
+
+cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/tc.plugin/Makefile.am b/collectors/tc.plugin/Makefile.am
new file mode 100644
index 0000000..9bc6cf2
--- /dev/null
+++ b/collectors/tc.plugin/Makefile.am
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+CLEANFILES = \
+ tc-qos-helper.sh \
+ $(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+dist_plugins_SCRIPTS = \
+ tc-qos-helper.sh \
+ $(NULL)
+
+dist_noinst_DATA = \
+ tc-qos-helper.sh.in \
+ README.md \
+ $(NULL)
diff --git a/collectors/tc.plugin/README.md b/collectors/tc.plugin/README.md
new file mode 100644
index 0000000..32c20f4
--- /dev/null
+++ b/collectors/tc.plugin/README.md
@@ -0,0 +1,205 @@
+<!--
+title: "tc.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/tc.plugin/README.md
+-->
+
+# tc.plugin
+
+Live demo - **[see it in action here](https://registry.my-netdata.io/#menu_tc)** !
+
+![qos](https://cloud.githubusercontent.com/assets/2662304/14439411/b7f36254-0033-11e6-93f0-c739bb6a1c3a.gif)
+
+Netdata monitors `tc` QoS classes for all interfaces.
+
+If you also use [FireQOS](http://firehol.org/tutorial/fireqos-new-user/) it will collect interface and class names.
+
+There is a [shell helper](https://raw.githubusercontent.com/netdata/netdata/master/collectors/tc.plugin/tc-qos-helper.sh.in) for this (all parsing is done by the plugin in `C` code - this shell script is just a configuration for the command to run to get `tc` output).
+
+The source of the tc plugin is [here](https://raw.githubusercontent.com/netdata/netdata/master/collectors/tc.plugin/plugin_tc.c). It is somewhat complex, because a state machine was needed to keep track of all the `tc` classes, including the pseudo classes tc dynamically creates.
+
+## Motivation
+
+One category of metrics missing in Linux monitoring, is bandwidth consumption for each open socket (inbound and outbound traffic). So, you cannot tell how much bandwidth your web server, your database server, your backup, your ssh sessions, etc are using.
+
+To solve this problem, the most *adventurous* Linux monitoring tools install kernel modules to capture all traffic, analyze it and provide reports per application. A lot of work, CPU intensive and with a great degree of risk (due to the kernel modules involved which might affect the stability of the whole system). Not to mention that such solutions are probably better suited for a core linux router in your network.
+
+Others use NFACCT, the netfilter accounting module which is already part of the Linux firewall. However, this would require configuring a firewall on every system you want to measure bandwidth (just FYI, I do install a firewall on every server - and I strongly advise you to do so too - but configuring accounting on all servers seems overkill when you don't really need it for billing purposes).
+
+**There is however a much simpler approach**.
+
+## QoS
+
+One of the features the Linux kernel has, but it is rarely used, is its ability to **apply QoS on traffic**. Even most interesting is that it can apply QoS to **both inbound and outbound traffic**.
+
+QoS is about 2 features:
+
+1. **Classify traffic**
+
+ Classification is the process of organizing traffic in groups, called **classes**. Classification can evaluate every aspect of network packets, like source and destination ports, source and destination IPs, netfilter marks, etc.
+
+ When you classify traffic, you just assign a label to it. Of course classes have some properties themselves (like queuing mechanisms), but let's say it is that simple: **a label**. For example **I call `web server` traffic, the traffic from my server's tcp/80, tcp/443 and to my server's tcp/80, tcp/443, while I call `web surfing` all other tcp/80 and tcp/443 traffic**. You can use any combinations you like. There is no limit.
+
+2. **Apply traffic shaping rules to these classes**
+
+ Traffic shaping is used to control how network interface bandwidth should be shared among the classes. Normally, you need to do this, when there is not enough bandwidth to satisfy all the demand, or when you want to control the supply of bandwidth to certain services. Of course classification is sufficient for monitoring traffic, but traffic shaping is also quite important, as we will explain in the next section.
+
+## Why you want QoS
+
+1. **Monitoring the bandwidth used by services**
+
+ Netdata provides wonderful real-time charts, like this one (wait to see the orange `rsync` part):
+
+ ![qos3](https://cloud.githubusercontent.com/assets/2662304/14474189/713ede84-0104-11e6-8c9c-8dca5c2abd63.gif)
+
+2. **Ensure sensitive administrative tasks will not starve for bandwidth**
+
+ Have you tried to ssh to a server when the network is congested? If you have, you already know it does not work very well. QoS can guarantee that services like ssh, dns, ntp, etc will always have a small supply of bandwidth. So, no matter what happens, you will be able to ssh to your server and DNS will always work.
+
+3. **Ensure administrative tasks will not monopolize all the bandwidth**
+
+ Services like backups, file copies, database dumps, etc can easily monopolize all the available bandwidth. It is common for example a nightly backup, or a huge file transfer to negatively influence the end-user experience. QoS can fix that.
+
+4. **Ensure each end-user connection will get a fair cut of the available bandwidth.**
+
+ Several QoS queuing disciplines in Linux do this automatically, without any configuration from you. The result is that new sockets are favored over older ones, so that users will get a snappier experience, while others are transferring large amounts of traffic.
+
+5. **Protect the servers from DDoS attacks.**
+
+ When your system is under a DDoS attack, it will get a lot more bandwidth compared to the one it can handle and probably your applications will crash. Setting a limit on the inbound traffic using QoS, will protect your servers (throttle the requests) and depending on the size of the attack may allow your legitimate users to access the server, while the attack is taking place.
+
+ Using QoS together with a [SYNPROXY](/collectors/proc.plugin/README.md) will provide a great degree of protection against most DDoS attacks. Actually when I wrote that article, a few folks tried to DDoS the Netdata demo site to see in real-time the SYNPROXY operation. They did not do it right, but anyway a great deal of requests reached the Netdata server. What saved Netdata was QoS. The Netdata demo server has QoS installed, so the requests were throttled and the server did not even reach the point of resource starvation. Read about it [here](/collectors/proc.plugin/README.md).
+
+On top of all these, QoS is extremely light. You will configure it once, and this is it. It will not bother you again and it will not use any noticeable CPU resources, especially on application and database servers.
+
+```
+- ensure administrative tasks (like ssh, dns, etc) will always have a small but guaranteed bandwidth. So, no matter what happens, I will be able to ssh to my server and DNS will work.
+
+- ensure other administrative tasks will not monopolize all the available bandwidth. So, my nightly backup will not hurt my users, a developer that is copying files over the net will not get all the available bandwidth, etc.
+
+- ensure each end-user connection will get a fair cut of the available bandwidth.
+```
+
+Once **traffic classification** is applied, we can use **[netdata](https://github.com/netdata/netdata)** to visualize the bandwidth consumption per class in real-time (no configuration is needed for Netdata - it will figure it out).
+
+QoS, is extremely light. You will configure it once, and this is it. It will not bother you again and it will not use any noticeable CPU resources, especially on application and database servers.
+
+This is QoS from a home linux router. Check these features:
+
+1. It is real-time (per second updates)
+2. QoS really works in Linux - check that the `background` traffic is squeezed when `surfing` needs it.
+
+![test2](https://cloud.githubusercontent.com/assets/2662304/14093004/68966020-f553-11e5-98fe-ffee2086fafd.gif)
+
+---
+
+## QoS in Linux?
+
+Of course, `tc` is probably **the most undocumented, complicated and unfriendly** command in Linux.
+
+For example, do you know that for matching a simple port range in `tc`, e.g. all the high ports, from 1025 to 65535 inclusive, you have to match these:
+
+```
+1025/0xffff
+1026/0xfffe
+1028/0xfffc
+1032/0xfff8
+1040/0xfff0
+1056/0xffe0
+1088/0xffc0
+1152/0xff80
+1280/0xff00
+1536/0xfe00
+2048/0xf800
+4096/0xf000
+8192/0xe000
+16384/0xc000
+32768/0x8000
+```
+
+To do it the hard way, you can go through the [tc configuration steps](#qos-configuration-with-tc). An easier way is to use **[FireQOS](https://firehol.org/tutorial/fireqos-new-user/)**, a tool that simplifies QoS management in Linux.
+
+## Qos Configuration with FireHOL
+
+The **[FireHOL](https://firehol.org/)** package already distributes **[FireQOS](https://firehol.org/tutorial/fireqos-new-user/)**. Check the **[FireQOS tutorial](https://firehol.org/tutorial/fireqos-new-user/)** to learn how to write your own QoS configuration.
+
+With **[FireQOS](https://firehol.org/tutorial/fireqos-new-user/)**, it is **really simple for everyone to use QoS in Linux**. Just install the package `firehol`. It should already be available for your distribution. If not, check the **[FireHOL Installation Guide](https://firehol.org/installing/)**. After that, you will have the `fireqos` command which uses a configuration like the following `/etc/firehol/fireqos.conf`, used at the Netdata demo site:
+
+```sh
+ # configure the Netdata ports
+ server_netdata_ports="tcp/19999"
+
+ interface eth0 world bidirectional ethernet balanced rate 50Mbit
+ class arp
+ match arp
+
+ class icmp
+ match icmp
+
+ class dns commit 1Mbit
+ server dns
+ client dns
+
+ class ntp
+ server ntp
+ client ntp
+
+ class ssh commit 2Mbit
+ server ssh
+ client ssh
+
+ class rsync commit 2Mbit max 10Mbit
+ server rsync
+ client rsync
+
+ class web_server commit 40Mbit
+ server http
+ server netdata
+
+ class client
+ client surfing
+
+ class nms commit 1Mbit
+ match input src 10.2.3.5
+```
+
+Nothing more is needed. You just run `fireqos start` to apply this configuration, restart Netdata and you have real-time visualization of the bandwidth consumption of your applications. FireQOS is not a daemon. It will just convert the configuration to `tc` commands. It will run them and it will exit.
+
+**IMPORTANT**: If you copy this configuration to apply it to your system, please adapt the speeds - experiment in non-production environments to learn the tool, before applying it on your servers.
+
+And this is what you are going to get:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/14436322/c91d90a4-0024-11e6-9fb1-57cdef1580df.png)
+
+## QoS Configuration with tc
+
+First, setup the tc rules in rc.local using commands to assign different QoS markings to different classids. You can see one such example in [github issue #4563](https://github.com/netdata/netdata/issues/4563#issuecomment-455711973).
+
+Then, map the classids to names by creating `/etc/iproute2/tc_cls`. For example:
+
+```
+2:1 Standard
+2:8 LowPriorityData
+2:10 HighThroughputData
+2:16 OAM
+2:18 LowLatencyData
+2:24 BroadcastVideo
+2:26 MultimediaStreaming
+2:32 RealTimeInteractive
+2:34 MultimediaConferencing
+2:40 Signalling
+2:46 Telephony
+2:48 NetworkControl
+```
+
+Add the following configuration option in `/etc/netdata.conf`:
+
+```\[plugin:tc]
+ enable show all classes and qdiscs for all interfaces = yes
+```
+
+Finally, create `/etc/netdata/tc-qos-helper.conf` with this content:
+`tc_show="class"`
+
+Please note, that by default Netdata will enable monitoring metrics only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). Set `yes` for a chart instead of `auto` to enable it permanently. You can also set the `enable zero metrics` option to `yes` in the `[global]` section which enables charts with zero metrics for all internal Netdata plugins.
+
+
diff --git a/collectors/tc.plugin/plugin_tc.c b/collectors/tc.plugin/plugin_tc.c
new file mode 100644
index 0000000..a2e72ee
--- /dev/null
+++ b/collectors/tc.plugin/plugin_tc.c
@@ -0,0 +1,1182 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "daemon/common.h"
+
+#define RRD_TYPE_TC "tc"
+#define PLUGIN_TC_NAME "tc.plugin"
+
+// ----------------------------------------------------------------------------
+// /sbin/tc processor
+// this requires the script plugins.d/tc-qos-helper.sh
+
+#define TC_LINE_MAX 1024
+
+struct tc_class {
+ STRING *id;
+ STRING *name;
+ STRING *leafid;
+ STRING *parentid;
+
+ bool hasparent;
+ bool isleaf;
+ bool isqdisc;
+ bool render;
+ bool name_updated;
+ bool updated;
+
+ int unupdated; // the number of times, this has been found un-updated
+
+ unsigned long long bytes;
+ unsigned long long packets;
+ unsigned long long dropped;
+ unsigned long long tokens;
+ unsigned long long ctokens;
+
+ //unsigned long long overlimits;
+ //unsigned long long requeues;
+ //unsigned long long lended;
+ //unsigned long long borrowed;
+ //unsigned long long giants;
+
+ RRDDIM *rd_bytes;
+ RRDDIM *rd_packets;
+ RRDDIM *rd_dropped;
+ RRDDIM *rd_tokens;
+ RRDDIM *rd_ctokens;
+};
+
+struct tc_device {
+ STRING *id;
+ STRING *name;
+ STRING *family;
+
+ bool name_updated;
+ bool family_updated;
+
+ char enabled;
+ char enabled_bytes;
+ char enabled_packets;
+ char enabled_dropped;
+ char enabled_tokens;
+ char enabled_ctokens;
+ char enabled_all_classes_qdiscs;
+
+ RRDSET *st_bytes;
+ RRDSET *st_packets;
+ RRDSET *st_dropped;
+ RRDSET *st_tokens;
+ RRDSET *st_ctokens;
+
+ DICTIONARY *classes;
+};
+
+
+// ----------------------------------------------------------------------------
+// tc_class index
+
+static void tc_class_free_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+ // struct tc_device *d = data;
+ struct tc_class *c = value;
+
+ string_freez(c->id);
+ string_freez(c->name);
+ string_freez(c->leafid);
+ string_freez(c->parentid);
+}
+
+static bool tc_class_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
+ struct tc_device *d = data; (void)d;
+ struct tc_class *c = old_value; (void)c;
+ struct tc_class *new_c = new_value; (void)new_c;
+
+ error("TC: class '%s' is already in device '%s'. Ignoring duplicate.", dictionary_acquired_item_name(item), string2str(d->id));
+
+ tc_class_free_callback(item, new_value, data);
+
+ return true;
+}
+
+static void tc_class_index_init(struct tc_device *d) {
+ if(!d->classes) {
+ d->classes = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED);
+
+ dictionary_register_delete_callback(d->classes, tc_class_free_callback, d);
+ dictionary_register_conflict_callback(d->classes, tc_class_conflict_callback, d);
+ }
+}
+
+static void tc_class_index_destroy(struct tc_device *d) {
+ dictionary_destroy(d->classes);
+ d->classes = NULL;
+}
+
+static struct tc_class *tc_class_index_add(struct tc_device *d, struct tc_class *c) {
+ return dictionary_set(d->classes, string2str(c->id), c, sizeof(*c));
+}
+
+static void tc_class_index_del(struct tc_device *d, struct tc_class *c) {
+ dictionary_del(d->classes, string2str(c->id));
+}
+
+static inline struct tc_class *tc_class_index_find(struct tc_device *d, const char *id) {
+ return dictionary_get(d->classes, id);
+}
+
+// ----------------------------------------------------------------------------
+// tc_device index
+
+static DICTIONARY *tc_device_root_index = NULL;
+
+static void tc_device_add_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+ struct tc_device *d = value;
+ tc_class_index_init(d);
+}
+
+static void tc_device_free_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+ struct tc_device *d = value;
+
+ tc_class_index_destroy(d);
+
+ string_freez(d->id);
+ string_freez(d->name);
+ string_freez(d->family);
+}
+
+static void tc_device_index_init() {
+ if(!tc_device_root_index) {
+ tc_device_root_index = dictionary_create(
+ DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED | DICT_OPTION_ADD_IN_FRONT);
+
+ dictionary_register_insert_callback(tc_device_root_index, tc_device_add_callback, NULL);
+ dictionary_register_delete_callback(tc_device_root_index, tc_device_free_callback, NULL);
+ }
+}
+
+static void tc_device_index_destroy() {
+ dictionary_destroy(tc_device_root_index);
+ tc_device_root_index = NULL;
+}
+
+static struct tc_device *tc_device_index_add(struct tc_device *d) {
+ return dictionary_set(tc_device_root_index, string2str(d->id), d, sizeof(*d));
+}
+
+//static struct tc_device *tc_device_index_del(struct tc_device *d) {
+// dictionary_del(tc_device_root_index, string2str(d->id));
+// return d;
+//}
+
+static inline struct tc_device *tc_device_index_find(const char *id) {
+ return dictionary_get(tc_device_root_index, id);
+}
+
+// ----------------------------------------------------------------------------
+
+static inline void tc_class_free(struct tc_device *n, struct tc_class *c) {
+ debug(D_TC_LOOP, "Removing from device '%s' class '%s', parentid '%s', leafid '%s', unused=%d",
+ string2str(n->id), string2str(c->id), string2str(c->parentid), string2str(c->leafid),
+ c->unupdated);
+
+ tc_class_index_del(n, c);
+}
+
+static inline void tc_device_classes_cleanup(struct tc_device *d) {
+ static int cleanup_every = 999;
+
+ if(unlikely(cleanup_every > 0)) {
+ cleanup_every = (int) config_get_number("plugin:tc", "cleanup unused classes every", 120);
+ if(cleanup_every < 0) cleanup_every = -cleanup_every;
+ }
+
+ d->name_updated = false;
+ d->family_updated = false;
+
+ struct tc_class *c;
+ dfe_start_write(d->classes, c) {
+ if(unlikely(cleanup_every && c->unupdated >= cleanup_every))
+ tc_class_free(d, c);
+
+ else {
+ c->updated = false;
+ c->name_updated = false;
+ }
+ }
+ dfe_done(c);
+}
+
+static inline void tc_device_commit(struct tc_device *d) {
+ static int enable_new_interfaces = -1, enable_bytes = -1, enable_packets = -1, enable_dropped = -1, enable_tokens = -1, enable_ctokens = -1, enabled_all_classes_qdiscs = -1;
+
+ if(unlikely(enable_new_interfaces == -1)) {
+ enable_new_interfaces = config_get_boolean_ondemand("plugin:tc", "enable new interfaces detected at runtime", CONFIG_BOOLEAN_YES);
+ enable_bytes = config_get_boolean_ondemand("plugin:tc", "enable traffic charts for all interfaces", CONFIG_BOOLEAN_AUTO);
+ enable_packets = config_get_boolean_ondemand("plugin:tc", "enable packets charts for all interfaces", CONFIG_BOOLEAN_AUTO);
+ enable_dropped = config_get_boolean_ondemand("plugin:tc", "enable dropped charts for all interfaces", CONFIG_BOOLEAN_AUTO);
+ enable_tokens = config_get_boolean_ondemand("plugin:tc", "enable tokens charts for all interfaces", CONFIG_BOOLEAN_NO);
+ enable_ctokens = config_get_boolean_ondemand("plugin:tc", "enable ctokens charts for all interfaces", CONFIG_BOOLEAN_NO);
+ enabled_all_classes_qdiscs = config_get_boolean_ondemand("plugin:tc", "enable show all classes and qdiscs for all interfaces", CONFIG_BOOLEAN_NO);
+ }
+
+ if(unlikely(d->enabled == (char)-1)) {
+ char var_name[CONFIG_MAX_NAME + 1];
+ snprintfz(var_name, CONFIG_MAX_NAME, "qos for %s", string2str(d->id));
+
+ d->enabled = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_new_interfaces);
+
+ snprintfz(var_name, CONFIG_MAX_NAME, "traffic chart for %s", string2str(d->id));
+ d->enabled_bytes = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_bytes);
+
+ snprintfz(var_name, CONFIG_MAX_NAME, "packets chart for %s", string2str(d->id));
+ d->enabled_packets = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_packets);
+
+ snprintfz(var_name, CONFIG_MAX_NAME, "dropped packets chart for %s", string2str(d->id));
+ d->enabled_dropped = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_dropped);
+
+ snprintfz(var_name, CONFIG_MAX_NAME, "tokens chart for %s", string2str(d->id));
+ d->enabled_tokens = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_tokens);
+
+ snprintfz(var_name, CONFIG_MAX_NAME, "ctokens chart for %s", string2str(d->id));
+ d->enabled_ctokens = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_ctokens);
+
+ snprintfz(var_name, CONFIG_MAX_NAME, "show all classes for %s", string2str(d->id));
+ d->enabled_all_classes_qdiscs = (char)config_get_boolean_ondemand("plugin:tc", var_name, enabled_all_classes_qdiscs);
+ }
+
+ // we only need to add leaf classes
+ struct tc_class *c, *x /*, *root = NULL */;
+ unsigned long long bytes_sum = 0, packets_sum = 0, dropped_sum = 0, tokens_sum = 0, ctokens_sum = 0;
+ int active_nodes = 0, updated_classes = 0, updated_qdiscs = 0;
+
+ // prepare all classes
+ // we set reasonable defaults for the rest of the code below
+
+ dfe_start_read(d->classes, c) {
+ c->render = false; // do not render this class
+ c->isleaf = true; // this is a leaf class
+ c->hasparent = false; // without a parent
+
+ if(unlikely(!c->updated))
+ c->unupdated++; // increase its unupdated counter
+ else {
+ c->unupdated = 0; // reset its unupdated counter
+
+ // count how many of each kind
+ if(c->isqdisc)
+ updated_qdiscs++;
+ else
+ updated_classes++;
+ }
+ }
+ dfe_done(c);
+
+ if(unlikely(!d->enabled || (!updated_classes && !updated_qdiscs))) {
+ debug(D_TC_LOOP, "TC: Ignoring TC device '%s'. It is not enabled/updated.", string2str(d->name?d->name:d->id));
+ tc_device_classes_cleanup(d);
+ return;
+ }
+
+ if(unlikely(updated_classes && updated_qdiscs)) {
+ error("TC: device '%s' has active both classes (%d) and qdiscs (%d). Will render only qdiscs.", string2str(d->id), updated_classes, updated_qdiscs);
+
+ // set all classes to !updated
+ dfe_start_read(d->classes, c) {
+ if (unlikely(!c->isqdisc && c->updated))
+ c->updated = false;
+ }
+ dfe_done(c);
+ updated_classes = 0;
+ }
+
+ // mark the classes as leafs and parents
+ //
+ // TC is hierarchical:
+ // - classes can have other classes in them
+ // - the same is true for qdiscs (i.e. qdiscs have classes, that have other qdiscs)
+ //
+ // we need to present a chart with leaf nodes only, so that the sum
+ // of all dimensions of the chart, will be the total utilization
+ // of the interface.
+ //
+ // here we try to find the ones we need to report
+ // by default all nodes are marked with: isleaf = 1 (see above)
+ //
+ // so, here we remove the isleaf flag from nodes in the middle
+ // and we add the hasparent flag to leaf nodes we found their parent
+ if(likely(!d->enabled_all_classes_qdiscs)) {
+ dfe_start_read(d->classes, c) {
+ if(unlikely(!c->updated))
+ continue;
+
+ //debug(D_TC_LOOP, "TC: In device '%s', %s '%s' has leafid: '%s' and parentid '%s'.",
+ // d->id,
+ // c->isqdisc?"qdisc":"class",
+ // c->id,
+ // c->leafid?c->leafid:"NULL",
+ // c->parentid?c->parentid:"NULL");
+
+ // find if c is leaf or not
+ dfe_start_read(d->classes, x) {
+ if(unlikely(!x->updated || c == x || !x->parentid))
+ continue;
+
+ // classes have both parentid and leafid
+ // qdiscs have only parentid
+ // the following works for both (it is an OR)
+
+ if((x->parentid && c->id == x->parentid) ||
+ (c->leafid && x->parentid && c->leafid == x->parentid)) {
+ // debug(D_TC_LOOP, "TC: In device '%s', %s '%s' (leafid: '%s') has as leaf %s '%s' (parentid: '%s').", d->name?d->name:d->id, c->isqdisc?"qdisc":"class", c->name?c->name:c->id, c->leafid?c->leafid:c->id, x->isqdisc?"qdisc":"class", x->name?x->name:x->id, x->parentid?x->parentid:x->id);
+ c->isleaf = false;
+ x->hasparent = true;
+ }
+ }
+ dfe_done(x);
+ }
+ dfe_done(c);
+ }
+
+ dfe_start_read(d->classes, c) {
+ if(unlikely(!c->updated))
+ continue;
+
+ // debug(D_TC_LOOP, "TC: device '%s', %s '%s' isleaf=%d, hasparent=%d", d->id, (c->isqdisc)?"qdisc":"class", c->id, c->isleaf, c->hasparent);
+
+ if(unlikely((c->isleaf && c->hasparent) || d->enabled_all_classes_qdiscs)) {
+ c->render = true;
+ active_nodes++;
+ bytes_sum += c->bytes;
+ packets_sum += c->packets;
+ dropped_sum += c->dropped;
+ tokens_sum += c->tokens;
+ ctokens_sum += c->ctokens;
+ }
+
+ //if(unlikely(!c->hasparent)) {
+ // if(root) error("TC: multiple root class/qdisc for device '%s' (old: '%s', new: '%s')", d->id, root->id, c->id);
+ // root = c;
+ // debug(D_TC_LOOP, "TC: found root class/qdisc '%s'", root->id);
+ //}
+ }
+ dfe_done(c);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ // dump all the list to see what we know
+
+ if(unlikely(debug_flags & D_TC_LOOP)) {
+ dfe_start_read(d->classes, c) {
+ if(c->render) debug(D_TC_LOOP, "TC: final nodes dump for '%s': class %s, OK", string2str(d->name), string2str(c->id));
+ else debug(D_TC_LOOP, "TC: final nodes dump for '%s': class '%s', IGNORE (updated: %d, isleaf: %d, hasparent: %d, parent: '%s')",
+ string2str(d->name?d->name:d->id), string2str(c->id), c->updated, c->isleaf, c->hasparent, string2str(c->parentid));
+ }
+ dfe_done(c);
+ }
+#endif
+
+ if(unlikely(!active_nodes)) {
+ debug(D_TC_LOOP, "TC: Ignoring TC device '%s'. No useful classes/qdiscs.", string2str(d->name?d->name:d->id));
+ tc_device_classes_cleanup(d);
+ return;
+ }
+
+ debug(D_TC_LOOP, "TC: evaluating TC device '%s'. enabled = %d/%d (bytes: %d/%d, packets: %d/%d, dropped: %d/%d, tokens: %d/%d, ctokens: %d/%d, all_classes_qdiscs: %d/%d), classes: (bytes = %llu, packets = %llu, dropped = %llu, tokens = %llu, ctokens = %llu).",
+ string2str(d->name?d->name:d->id),
+ d->enabled, enable_new_interfaces,
+ d->enabled_bytes, enable_bytes,
+ d->enabled_packets, enable_packets,
+ d->enabled_dropped, enable_dropped,
+ d->enabled_tokens, enable_tokens,
+ d->enabled_ctokens, enable_ctokens,
+ d->enabled_all_classes_qdiscs, enabled_all_classes_qdiscs,
+ bytes_sum,
+ packets_sum,
+ dropped_sum,
+ tokens_sum,
+ ctokens_sum
+ );
+
+ // --------------------------------------------------------------------
+ // bytes
+
+ if(d->enabled_bytes == CONFIG_BOOLEAN_YES || (d->enabled_bytes == CONFIG_BOOLEAN_AUTO &&
+ (bytes_sum || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->enabled_bytes = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_bytes)) {
+ d->st_bytes = rrdset_create_localhost(
+ RRD_TYPE_TC,
+ string2str(d->id),
+ string2str(d->name ? d->name : d->id),
+ string2str(d->family ? d->family : d->id),
+ RRD_TYPE_TC ".qos",
+ "Class Usage",
+ "kilobits/s",
+ PLUGIN_TC_NAME,
+ NULL,
+ NETDATA_CHART_PRIO_TC_QOS,
+ localhost->rrd_update_every,
+ d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
+
+ rrdlabels_add(d->st_bytes->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name?d->name:d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family?d->family:d->id), RRDLABEL_SRC_AUTO);
+ }
+ else {
+ if(unlikely(d->name_updated))
+ rrdset_reset_name(d->st_bytes, string2str(d->name));
+
+ if(d->name && d->name_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name), RRDLABEL_SRC_AUTO);
+
+ if(d->family && d->family_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family), RRDLABEL_SRC_AUTO);
+
+ // TODO
+ // update the family
+ }
+
+ dfe_start_read(d->classes, c) {
+ if(unlikely(!c->render)) continue;
+
+ if(unlikely(!c->rd_bytes))
+ c->rd_bytes = rrddim_add(d->st_bytes, string2str(c->id), string2str(c->name?c->name:c->id), 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
+ else if(unlikely(c->name_updated))
+ rrddim_reset_name(d->st_bytes, c->rd_bytes, string2str(c->name));
+
+ rrddim_set_by_pointer(d->st_bytes, c->rd_bytes, c->bytes);
+ }
+ dfe_done(c);
+
+ rrdset_done(d->st_bytes);
+ }
+
+ // --------------------------------------------------------------------
+ // packets
+
+ if(d->enabled_packets == CONFIG_BOOLEAN_YES || (d->enabled_packets == CONFIG_BOOLEAN_AUTO &&
+ (packets_sum ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->enabled_packets = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_packets)) {
+ char id[RRD_ID_LENGTH_MAX + 1];
+ char name[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(id, RRD_ID_LENGTH_MAX, "%s_packets", string2str(d->id));
+ snprintfz(name, RRD_ID_LENGTH_MAX, "%s_packets", string2str(d->name ? d->name : d->id));
+
+ d->st_packets = rrdset_create_localhost(
+ RRD_TYPE_TC,
+ id,
+ name,
+ string2str(d->family ? d->family : d->id),
+ RRD_TYPE_TC ".qos_packets",
+ "Class Packets",
+ "packets/s",
+ PLUGIN_TC_NAME,
+ NULL,
+ NETDATA_CHART_PRIO_TC_QOS_PACKETS,
+ localhost->rrd_update_every,
+ d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
+
+ rrdlabels_add(d->st_bytes->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name?d->name:d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family?d->family:d->id), RRDLABEL_SRC_AUTO);
+ }
+ else {
+ if(unlikely(d->name_updated)) {
+ char name[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(name, RRD_ID_LENGTH_MAX, "%s_packets", string2str(d->name?d->name:d->id));
+ rrdset_reset_name(d->st_packets, name);
+ }
+
+ if(d->name && d->name_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name), RRDLABEL_SRC_AUTO);
+
+ if(d->family && d->family_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family), RRDLABEL_SRC_AUTO);
+
+ // TODO
+ // update the family
+ }
+
+ dfe_start_read(d->classes, c) {
+ if(unlikely(!c->render)) continue;
+
+ if(unlikely(!c->rd_packets))
+ c->rd_packets = rrddim_add(d->st_packets, string2str(c->id), string2str(c->name?c->name:c->id), 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ else if(unlikely(c->name_updated))
+ rrddim_reset_name(d->st_packets, c->rd_packets, string2str(c->name));
+
+ rrddim_set_by_pointer(d->st_packets, c->rd_packets, c->packets);
+ }
+ dfe_done(c);
+
+ rrdset_done(d->st_packets);
+ }
+
+ // --------------------------------------------------------------------
+ // dropped
+
+ if(d->enabled_dropped == CONFIG_BOOLEAN_YES || (d->enabled_dropped == CONFIG_BOOLEAN_AUTO &&
+ (dropped_sum ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->enabled_dropped = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_dropped)) {
+ char id[RRD_ID_LENGTH_MAX + 1];
+ char name[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(id, RRD_ID_LENGTH_MAX, "%s_dropped", string2str(d->id));
+ snprintfz(name, RRD_ID_LENGTH_MAX, "%s_dropped", string2str(d->name ? d->name : d->id));
+
+ d->st_dropped = rrdset_create_localhost(
+ RRD_TYPE_TC,
+ id,
+ name,
+ string2str(d->family ? d->family : d->id),
+ RRD_TYPE_TC ".qos_dropped",
+ "Class Dropped Packets",
+ "packets/s",
+ PLUGIN_TC_NAME,
+ NULL,
+ NETDATA_CHART_PRIO_TC_QOS_DROPPED,
+ localhost->rrd_update_every,
+ d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED);
+
+ rrdlabels_add(d->st_bytes->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name?d->name:d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family?d->family:d->id), RRDLABEL_SRC_AUTO);
+ }
+ else {
+ if(unlikely(d->name_updated)) {
+ char name[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(name, RRD_ID_LENGTH_MAX, "%s_dropped", string2str(d->name?d->name:d->id));
+ rrdset_reset_name(d->st_dropped, name);
+ }
+
+ if(d->name && d->name_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name), RRDLABEL_SRC_AUTO);
+
+ if(d->family && d->family_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family), RRDLABEL_SRC_AUTO);
+
+ // TODO
+ // update the family
+ }
+
+ dfe_start_read(d->classes, c) {
+ if(unlikely(!c->render)) continue;
+
+ if(unlikely(!c->rd_dropped))
+ c->rd_dropped = rrddim_add(d->st_dropped, string2str(c->id), string2str(c->name?c->name:c->id), 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ else if(unlikely(c->name_updated))
+ rrddim_reset_name(d->st_dropped, c->rd_dropped, string2str(c->name));
+
+ rrddim_set_by_pointer(d->st_dropped, c->rd_dropped, c->dropped);
+ }
+ dfe_done(c);
+
+ rrdset_done(d->st_dropped);
+ }
+
+ // --------------------------------------------------------------------
+ // tokens
+
+ if(d->enabled_tokens == CONFIG_BOOLEAN_YES || (d->enabled_tokens == CONFIG_BOOLEAN_AUTO &&
+ (tokens_sum ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->enabled_tokens = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_tokens)) {
+ char id[RRD_ID_LENGTH_MAX + 1];
+ char name[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(id, RRD_ID_LENGTH_MAX, "%s_tokens", string2str(d->id));
+ snprintfz(name, RRD_ID_LENGTH_MAX, "%s_tokens", string2str(d->name ? d->name : d->id));
+
+ d->st_tokens = rrdset_create_localhost(
+ RRD_TYPE_TC,
+ id,
+ name,
+ string2str(d->family ? d->family : d->id),
+ RRD_TYPE_TC ".qos_tokens",
+ "Class Tokens",
+ "tokens",
+ PLUGIN_TC_NAME,
+ NULL,
+ NETDATA_CHART_PRIO_TC_QOS_TOKENS,
+ localhost->rrd_update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdlabels_add(d->st_bytes->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name?d->name:d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family?d->family:d->id), RRDLABEL_SRC_AUTO);
+ }
+ else {
+ if(unlikely(d->name_updated)) {
+ char name[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(name, RRD_ID_LENGTH_MAX, "%s_tokens", string2str(d->name?d->name:d->id));
+ rrdset_reset_name(d->st_tokens, name);
+ }
+
+ if(d->name && d->name_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name), RRDLABEL_SRC_AUTO);
+
+ if(d->family && d->family_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family), RRDLABEL_SRC_AUTO);
+
+ // TODO
+ // update the family
+ }
+
+ dfe_start_read(d->classes, c) {
+ if(unlikely(!c->render)) continue;
+
+ if(unlikely(!c->rd_tokens)) {
+ c->rd_tokens = rrddim_add(d->st_tokens, string2str(c->id), string2str(c->name?c->name:c->id), 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+ else if(unlikely(c->name_updated))
+ rrddim_reset_name(d->st_tokens, c->rd_tokens, string2str(c->name));
+
+ rrddim_set_by_pointer(d->st_tokens, c->rd_tokens, c->tokens);
+ }
+ dfe_done(c);
+
+ rrdset_done(d->st_tokens);
+ }
+
+ // --------------------------------------------------------------------
+ // ctokens
+
+ if(d->enabled_ctokens == CONFIG_BOOLEAN_YES || (d->enabled_ctokens == CONFIG_BOOLEAN_AUTO &&
+ (ctokens_sum ||
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
+ d->enabled_ctokens = CONFIG_BOOLEAN_YES;
+
+ if(unlikely(!d->st_ctokens)) {
+ char id[RRD_ID_LENGTH_MAX + 1];
+ char name[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(id, RRD_ID_LENGTH_MAX, "%s_ctokens", string2str(d->id));
+ snprintfz(name, RRD_ID_LENGTH_MAX, "%s_ctokens", string2str(d->name ? d->name : d->id));
+
+ d->st_ctokens = rrdset_create_localhost(
+ RRD_TYPE_TC,
+ id,
+ name,
+ string2str(d->family ? d->family : d->id),
+ RRD_TYPE_TC ".qos_ctokens",
+ "Class cTokens",
+ "ctokens",
+ PLUGIN_TC_NAME,
+ NULL,
+ NETDATA_CHART_PRIO_TC_QOS_CTOKENS,
+ localhost->rrd_update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdlabels_add(d->st_bytes->rrdlabels, "device", string2str(d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name?d->name:d->id), RRDLABEL_SRC_AUTO);
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family?d->family:d->id), RRDLABEL_SRC_AUTO);
+ }
+ else {
+ debug(D_TC_LOOP, "TC: Updating _ctokens chart for device '%s'", string2str(d->name?d->name:d->id));
+
+ if(unlikely(d->name_updated)) {
+ char name[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(name, RRD_ID_LENGTH_MAX, "%s_ctokens", string2str(d->name?d->name:d->id));
+ rrdset_reset_name(d->st_ctokens, name);
+ }
+
+ if(d->name && d->name_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name), RRDLABEL_SRC_AUTO);
+
+ if(d->family && d->family_updated)
+ rrdlabels_add(d->st_bytes->rrdlabels, "family", string2str(d->family), RRDLABEL_SRC_AUTO);
+
+ // TODO
+ // update the family
+ }
+
+ dfe_start_read(d->classes, c) {
+ if(unlikely(!c->render)) continue;
+
+ if(unlikely(!c->rd_ctokens))
+ c->rd_ctokens = rrddim_add(d->st_ctokens, string2str(c->id), string2str(c->name?c->name:c->id), 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ else if(unlikely(c->name_updated))
+ rrddim_reset_name(d->st_ctokens, c->rd_ctokens, string2str(c->name));
+
+ rrddim_set_by_pointer(d->st_ctokens, c->rd_ctokens, c->ctokens);
+ }
+ dfe_done(c);
+
+ rrdset_done(d->st_ctokens);
+ }
+
+ tc_device_classes_cleanup(d);
+}
+
+static inline void tc_device_set_class_name(struct tc_device *d, char *id, char *name) {
+ if(unlikely(!name || !*name)) return;
+
+ struct tc_class *c = tc_class_index_find(d, id);
+ if(likely(c)) {
+ if(likely(c->name)) {
+ if(!strcmp(string2str(c->name), name)) return;
+ string_freez(c->name);
+ c->name = NULL;
+ }
+
+ if(likely(name && *name && strcmp(string2str(c->id), name) != 0)) {
+ debug(D_TC_LOOP, "TC: Setting device '%s', class '%s' name to '%s'", string2str(d->id), id, name);
+ c->name = string_strdupz(name);
+ c->name_updated = true;
+ }
+ }
+}
+
+static inline void tc_device_set_device_name(struct tc_device *d, char *name) {
+ if(unlikely(!name || !*name)) return;
+
+ if(d->name) {
+ if(!strcmp(string2str(d->name), name)) return;
+ string_freez(d->name);
+ d->name = NULL;
+ }
+
+ if(likely(name && *name && strcmp(string2str(d->id), name) != 0)) {
+ debug(D_TC_LOOP, "TC: Setting device '%s' name to '%s'", string2str(d->id), name);
+ d->name = string_strdupz(name);
+ d->name_updated = true;
+ }
+}
+
+static inline void tc_device_set_device_family(struct tc_device *d, char *family) {
+ string_freez(d->family);
+ d->family = NULL;
+
+ if(likely(family && *family && strcmp(string2str(d->id), family) != 0)) {
+ debug(D_TC_LOOP, "TC: Setting device '%s' family to '%s'", string2str(d->id), family);
+ d->family = string_strdupz(family);
+ d->family_updated = true;
+ }
+ // no need for null termination - it is already null
+}
+
+static inline struct tc_device *tc_device_create(char *id) {
+ struct tc_device *d = tc_device_index_find(id);
+
+ if(!d) {
+ debug(D_TC_LOOP, "TC: Creating device '%s'", id);
+
+ struct tc_device tmp = {
+ .id = string_strdupz(id),
+ .enabled = (char)-1,
+ };
+ d = tc_device_index_add(&tmp);
+ }
+
+ return(d);
+}
+
+static inline struct tc_class *tc_class_add(struct tc_device *n, char *id, bool qdisc, char *parentid, char *leafid) {
+ struct tc_class *c = tc_class_index_find(n, id);
+
+ if(!c) {
+ debug(D_TC_LOOP, "TC: Creating in device '%s', class id '%s', parentid '%s', leafid '%s'",
+ string2str(n->id), id, parentid?parentid:"", leafid?leafid:"");
+
+ struct tc_class tmp = {
+ .id = string_strdupz(id),
+ .isqdisc = qdisc,
+ .parentid = string_strdupz(parentid),
+ .leafid = string_strdupz(leafid),
+ };
+
+ tc_class_index_add(n, &tmp);
+ }
+ return(c);
+}
+
+//static inline void tc_device_free(struct tc_device *d) {
+// tc_device_index_del(d);
+//}
+
+static inline int tc_space(char c) {
+ switch(c) {
+ case ' ':
+ case '\t':
+ case '\r':
+ case '\n':
+ return 1;
+
+ default:
+ return 0;
+ }
+}
+
+static inline void tc_split_words(char *str, char **words, int max_words) {
+ char *s = str;
+ int i = 0;
+
+ // skip all white space
+ while(tc_space(*s)) s++;
+
+ // store the first word
+ words[i++] = s;
+
+ // while we have something
+ while(*s) {
+ // if it is a space
+ if(unlikely(tc_space(*s))) {
+
+ // terminate the word
+ *s++ = '\0';
+
+ // skip all white space
+ while(tc_space(*s)) s++;
+
+ // if we reached the end, stop
+ if(!*s) break;
+
+ // store the next word
+ if(i < max_words) words[i++] = s;
+ else break;
+ }
+ else s++;
+ }
+
+ // terminate the words
+ while(i < max_words) words[i++] = NULL;
+}
+
+static pid_t tc_child_pid = 0;
+
+static void tc_main_cleanup(void *ptr) {
+ worker_unregister();
+
+ tc_device_index_destroy();
+
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ info("cleaning up...");
+
+ if(tc_child_pid) {
+ info("TC: killing with SIGTERM tc-qos-helper process %d", tc_child_pid);
+ if(killpid(tc_child_pid) != -1) {
+ siginfo_t info;
+
+ info("TC: waiting for tc plugin child process pid %d to exit...", tc_child_pid);
+ waitid(P_PID, (id_t) tc_child_pid, &info, WEXITED);
+ }
+
+ tc_child_pid = 0;
+ }
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+#define WORKER_TC_CLASS 0
+#define WORKER_TC_BEGIN 1
+#define WORKER_TC_END 2
+#define WORKER_TC_SENT 3
+#define WORKER_TC_LENDED 4
+#define WORKER_TC_TOKENS 5
+#define WORKER_TC_SETDEVICENAME 6
+#define WORKER_TC_SETDEVICEGROUP 7
+#define WORKER_TC_SETCLASSNAME 8
+#define WORKER_TC_WORKTIME 9
+#define WORKER_TC_PLUGIN_TIME 10
+#define WORKER_TC_DEVICES 11
+#define WORKER_TC_CLASSES 12
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 13
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 10
+#endif
+
+void *tc_main(void *ptr) {
+ worker_register("TC");
+ worker_register_job_name(WORKER_TC_CLASS, "class");
+ worker_register_job_name(WORKER_TC_BEGIN, "begin");
+ worker_register_job_name(WORKER_TC_END, "end");
+ worker_register_job_name(WORKER_TC_SENT, "sent");
+ worker_register_job_name(WORKER_TC_LENDED, "lended");
+ worker_register_job_name(WORKER_TC_TOKENS, "tokens");
+ worker_register_job_name(WORKER_TC_SETDEVICENAME, "devicename");
+ worker_register_job_name(WORKER_TC_SETDEVICEGROUP, "devicegroup");
+ worker_register_job_name(WORKER_TC_SETCLASSNAME, "classname");
+ worker_register_job_name(WORKER_TC_WORKTIME, "worktime");
+
+ worker_register_job_custom_metric(WORKER_TC_PLUGIN_TIME, "tc script execution time", "milliseconds/run", WORKER_METRIC_ABSOLUTE);
+ worker_register_job_custom_metric(WORKER_TC_DEVICES, "number of devices", "devices", WORKER_METRIC_ABSOLUTE);
+ worker_register_job_custom_metric(WORKER_TC_CLASSES, "number of classes", "classes", WORKER_METRIC_ABSOLUTE);
+
+ tc_device_index_init();
+ netdata_thread_cleanup_push(tc_main_cleanup, ptr);
+
+ char command[FILENAME_MAX + 1];
+ char *words[PLUGINSD_MAX_WORDS] = { NULL };
+
+ uint32_t BEGIN_HASH = simple_hash("BEGIN");
+ uint32_t END_HASH = simple_hash("END");
+ uint32_t QDISC_HASH = simple_hash("qdisc");
+ uint32_t CLASS_HASH = simple_hash("class");
+ uint32_t SENT_HASH = simple_hash("Sent");
+ uint32_t LENDED_HASH = simple_hash("lended:");
+ uint32_t TOKENS_HASH = simple_hash("tokens:");
+ uint32_t SETDEVICENAME_HASH = simple_hash("SETDEVICENAME");
+ uint32_t SETDEVICEGROUP_HASH = simple_hash("SETDEVICEGROUP");
+ uint32_t SETCLASSNAME_HASH = simple_hash("SETCLASSNAME");
+ uint32_t WORKTIME_HASH = simple_hash("WORKTIME");
+ uint32_t first_hash;
+
+ snprintfz(command, TC_LINE_MAX, "%s/tc-qos-helper.sh", netdata_configured_primary_plugins_dir);
+ char *tc_script = config_get("plugin:tc", "script to run to get tc values", command);
+
+ while(!netdata_exit) {
+ FILE *fp_child_input, *fp_child_output;
+ struct tc_device *device = NULL;
+ struct tc_class *class = NULL;
+
+ snprintfz(command, TC_LINE_MAX, "exec %s %d", tc_script, localhost->rrd_update_every);
+ debug(D_TC_LOOP, "executing '%s'", command);
+
+ fp_child_output = netdata_popen(command, (pid_t *)&tc_child_pid, &fp_child_input);
+ if(unlikely(!fp_child_output)) {
+ error("TC: Cannot popen(\"%s\", \"r\").", command);
+ goto cleanup;
+ }
+
+ char buffer[TC_LINE_MAX+1] = "";
+ while(fgets(buffer, TC_LINE_MAX, fp_child_output) != NULL) {
+ if(unlikely(netdata_exit)) break;
+
+ buffer[TC_LINE_MAX] = '\0';
+ // debug(D_TC_LOOP, "TC: read '%s'", buffer);
+
+ tc_split_words(buffer, words, PLUGINSD_MAX_WORDS);
+
+ if(unlikely(!words[0] || !*words[0])) {
+ // debug(D_TC_LOOP, "empty line");
+ worker_is_idle();
+ continue;
+ }
+ // else debug(D_TC_LOOP, "First word is '%s'", words[0]);
+
+ first_hash = simple_hash(words[0]);
+
+ if(unlikely(device && ((first_hash == CLASS_HASH && strcmp(words[0], "class") == 0) || (first_hash == QDISC_HASH && strcmp(words[0], "qdisc") == 0)))) {
+ worker_is_busy(WORKER_TC_CLASS);
+
+ // debug(D_TC_LOOP, "CLASS line on class id='%s', parent='%s', parentid='%s', leaf='%s', leafid='%s'", words[2], words[3], words[4], words[5], words[6]);
+
+ char *type = words[1]; // the class/qdisc type: htb, fq_codel, etc
+ char *id = words[2]; // the class/qdisc major:minor
+ char *parent = words[3]; // the word 'parent' or 'root'
+ char *parentid = words[4]; // parentid
+ char *leaf = words[5]; // the word 'leaf'
+ char *leafid = words[6]; // leafid
+
+ int parent_is_root = 0;
+ int parent_is_parent = 0;
+ if(likely(parent)) {
+ parent_is_parent = !strcmp(parent, "parent");
+
+ if(!parent_is_parent)
+ parent_is_root = !strcmp(parent, "root");
+ }
+
+ if(likely(type && id && (parent_is_root || parent_is_parent))) {
+ bool qdisc = false;
+
+ if(first_hash == QDISC_HASH) {
+ qdisc = true;
+
+ if(!strcmp(type, "ingress")) {
+ // we don't want to get the ingress qdisc
+ // there should be an IFB interface for this
+
+ class = NULL;
+ worker_is_idle();
+ continue;
+ }
+
+ if(parent_is_parent && parentid) {
+ // eliminate the minor number from parentid
+ // why: parentid is the id of the parent class
+ // but major: is also the id of the parent qdisc
+
+ char *s = parentid;
+ while(*s && *s != ':') s++;
+ if(*s == ':') s[1] = '\0';
+ }
+ }
+
+ if(parent_is_root) {
+ parentid = NULL;
+ leafid = NULL;
+ }
+ else if(!leaf || strcmp(leaf, "leaf") != 0)
+ leafid = NULL;
+
+ char leafbuf[20 + 1] = "";
+ if(leafid && leafid[strlen(leafid) - 1] == ':') {
+ strncpyz(leafbuf, leafid, 20 - 1);
+ strcat(leafbuf, "1");
+ leafid = leafbuf;
+ }
+
+ class = tc_class_add(device, id, qdisc, parentid, leafid);
+ }
+ else {
+ // clear the last class
+ class = NULL;
+ }
+ }
+ else if(unlikely(first_hash == END_HASH && strcmp(words[0], "END") == 0)) {
+ worker_is_busy(WORKER_TC_END);
+
+ // debug(D_TC_LOOP, "END line");
+
+ if(likely(device)) {
+ netdata_thread_disable_cancelability();
+ tc_device_commit(device);
+ // tc_device_free(device);
+ netdata_thread_enable_cancelability();
+ }
+
+ device = NULL;
+ class = NULL;
+ }
+ else if(unlikely(first_hash == BEGIN_HASH && strcmp(words[0], "BEGIN") == 0)) {
+ worker_is_busy(WORKER_TC_BEGIN);
+
+ // debug(D_TC_LOOP, "BEGIN line on device '%s'", words[1]);
+
+ if(likely(words[1] && *words[1])) {
+ device = tc_device_create(words[1]);
+ }
+ else {
+ // tc_device_free(device);
+ device = NULL;
+ }
+
+ class = NULL;
+ }
+ else if(unlikely(device && class && first_hash == SENT_HASH && strcmp(words[0], "Sent") == 0)) {
+ worker_is_busy(WORKER_TC_SENT);
+
+ // debug(D_TC_LOOP, "SENT line '%s'", words[1]);
+ if(likely(words[1] && *words[1])) {
+ class->bytes = str2ull(words[1]);
+ class->updated = true;
+ }
+ else {
+ class->updated = false;
+ }
+
+ if(likely(words[3] && *words[3]))
+ class->packets = str2ull(words[3]);
+
+ if(likely(words[6] && *words[6]))
+ class->dropped = str2ull(words[6]);
+
+ //if(likely(words[8] && *words[8]))
+ // class->overlimits = str2ull(words[8]);
+
+ //if(likely(words[10] && *words[10]))
+ // class->requeues = str2ull(words[8]);
+ }
+ else if(unlikely(device && class && class->updated && first_hash == LENDED_HASH && strcmp(words[0], "lended:") == 0)) {
+ worker_is_busy(WORKER_TC_LENDED);
+
+ // debug(D_TC_LOOP, "LENDED line '%s'", words[1]);
+ //if(likely(words[1] && *words[1]))
+ // class->lended = str2ull(words[1]);
+
+ //if(likely(words[3] && *words[3]))
+ // class->borrowed = str2ull(words[3]);
+
+ //if(likely(words[5] && *words[5]))
+ // class->giants = str2ull(words[5]);
+ }
+ else if(unlikely(device && class && class->updated && first_hash == TOKENS_HASH && strcmp(words[0], "tokens:") == 0)) {
+ worker_is_busy(WORKER_TC_TOKENS);
+
+ // debug(D_TC_LOOP, "TOKENS line '%s'", words[1]);
+ if(likely(words[1] && *words[1]))
+ class->tokens = str2ull(words[1]);
+
+ if(likely(words[3] && *words[3]))
+ class->ctokens = str2ull(words[3]);
+ }
+ else if(unlikely(device && first_hash == SETDEVICENAME_HASH && strcmp(words[0], "SETDEVICENAME") == 0)) {
+ worker_is_busy(WORKER_TC_SETDEVICENAME);
+
+ // debug(D_TC_LOOP, "SETDEVICENAME line '%s'", words[1]);
+ if(likely(words[1] && *words[1]))
+ tc_device_set_device_name(device, words[1]);
+ }
+ else if(unlikely(device && first_hash == SETDEVICEGROUP_HASH && strcmp(words[0], "SETDEVICEGROUP") == 0)) {
+ worker_is_busy(WORKER_TC_SETDEVICEGROUP);
+
+ // debug(D_TC_LOOP, "SETDEVICEGROUP line '%s'", words[1]);
+ if(likely(words[1] && *words[1]))
+ tc_device_set_device_family(device, words[1]);
+ }
+ else if(unlikely(device && first_hash == SETCLASSNAME_HASH && strcmp(words[0], "SETCLASSNAME") == 0)) {
+ worker_is_busy(WORKER_TC_SETCLASSNAME);
+
+ // debug(D_TC_LOOP, "SETCLASSNAME line '%s' '%s'", words[1], words[2]);
+ char *id = words[1];
+ char *path = words[2];
+ if(likely(id && *id && path && *path))
+ tc_device_set_class_name(device, id, path);
+ }
+ else if(unlikely(first_hash == WORKTIME_HASH && strcmp(words[0], "WORKTIME") == 0)) {
+ worker_is_busy(WORKER_TC_WORKTIME);
+ worker_set_metric(WORKER_TC_PLUGIN_TIME, str2ll(words[1], NULL));
+
+ size_t number_of_devices = dictionary_entries(tc_device_root_index);
+ size_t number_of_classes = 0;
+
+ struct tc_device *d;
+ dfe_start_read(tc_device_root_index, d) {
+ number_of_classes += dictionary_entries(d->classes);
+ }
+ dfe_done(d);
+
+ worker_set_metric(WORKER_TC_DEVICES, number_of_devices);
+ worker_set_metric(WORKER_TC_CLASSES, number_of_classes);
+ }
+ //else {
+ // debug(D_TC_LOOP, "IGNORED line");
+ //}
+
+ worker_is_idle();
+ }
+
+ // fgets() failed or loop broke
+ int code = netdata_pclose(fp_child_input, fp_child_output, (pid_t)tc_child_pid);
+ tc_child_pid = 0;
+
+ if(unlikely(device)) {
+ // tc_device_free(device);
+ device = NULL;
+ class = NULL;
+ }
+
+ if(unlikely(netdata_exit))
+ goto cleanup;
+
+ if(code == 1 || code == 127) {
+ // 1 = DISABLE
+ // 127 = cannot even run it
+ error("TC: tc-qos-helper.sh exited with code %d. Disabling it.", code);
+ goto cleanup;
+ }
+
+ sleep((unsigned int) localhost->rrd_update_every);
+ }
+
+cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement
+ worker_unregister();
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/tc.plugin/tc-qos-helper.sh.in b/collectors/tc.plugin/tc-qos-helper.sh.in
new file mode 100755
index 0000000..97d4d01
--- /dev/null
+++ b/collectors/tc.plugin/tc-qos-helper.sh.in
@@ -0,0 +1,297 @@
+#!/usr/bin/env bash
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This script is a helper to allow netdata collect tc data.
+# tc output parsing has been implemented in C, inside netdata
+# This script allows setting names to dimensions.
+
+export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
+export LC_ALL=C
+
+# -----------------------------------------------------------------------------
+# logging functions
+
+PROGRAM_NAME="$(basename "$0")"
+PROGRAM_NAME="${PROGRAM_NAME/.plugin/}"
+
+logdate() {
+ date "+%Y-%m-%d %H:%M:%S"
+}
+
+log() {
+ local status="${1}"
+ shift
+
+ echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
+
+}
+
+warning() {
+ log WARNING "${@}"
+}
+
+error() {
+ log ERROR "${@}"
+}
+
+info() {
+ log INFO "${@}"
+}
+
+fatal() {
+ log FATAL "${@}"
+ exit 1
+}
+
+debug=0
+debug() {
+ [ $debug -eq 1 ] && log DEBUG "${@}"
+}
+
+# -----------------------------------------------------------------------------
+# find /var/run/fireqos
+
+# the default
+fireqos_run_dir="/var/run/fireqos"
+
+function realdir() {
+ local r
+ local t
+ r="$1"
+ t="$(readlink "$r")"
+
+ while [ "$t" ]; do
+ r=$(cd "$(dirname "$r")" && cd "$(dirname "$t")" && pwd -P)/$(basename "$t")
+ t=$(readlink "$r")
+ done
+
+ dirname "$r"
+}
+
+if [ ! -d "${fireqos_run_dir}" ]; then
+
+ # the fireqos executable - we will use it to find its config
+ fireqos="$(command -v fireqos 2>/dev/null)"
+
+ if [ -n "${fireqos}" ]; then
+
+ fireqos_exec_dir="$(realdir "${fireqos}")"
+
+ if [ -n "${fireqos_exec_dir}" ] && [ "${fireqos_exec_dir}" != "." ] && [ -f "${fireqos_exec_dir}/install.config" ]; then
+ LOCALSTATEDIR=
+ #shellcheck source=/dev/null
+ source "${fireqos_exec_dir}/install.config"
+
+ if [ -d "${LOCALSTATEDIR}/run/fireqos" ]; then
+ fireqos_run_dir="${LOCALSTATEDIR}/run/fireqos"
+ else
+ warning "FireQOS is installed as '${fireqos}', its installation config at '${fireqos_exec_dir}/install.config' specifies local state data at '${LOCALSTATEDIR}/run/fireqos', but this directory is not found or is not readable (check the permissions of its parents)."
+ fi
+ else
+ warning "Although FireQOS is installed on this system as '${fireqos}', I cannot find/read its installation configuration at '${fireqos_exec_dir}/install.config'."
+ fi
+ else
+ warning "FireQOS is not installed on this system. Use FireQOS to apply traffic QoS and expose the class names to netdata. Check https://github.com/netdata/netdata/tree/master/collectors/tc.plugin#tcplugin"
+ fi
+fi
+
+# -----------------------------------------------------------------------------
+
+[ -z "${NETDATA_PLUGINS_DIR}" ] && NETDATA_PLUGINS_DIR="$(dirname "${0}")"
+[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
+[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
+
+plugins_dir="${NETDATA_PLUGINS_DIR}"
+tc="$(command -v tc 2>/dev/null)"
+
+# -----------------------------------------------------------------------------
+# user configuration
+
+# time in seconds to refresh QoS class/qdisc names
+qos_get_class_names_every=120
+
+# time in seconds to exit - netdata will restart the script
+qos_exit_every=3600
+
+# what to use? classes or qdiscs?
+tc_show="qdisc" # can also be "class"
+
+# -----------------------------------------------------------------------------
+# check if we have a valid number for interval
+
+t=${1}
+update_every=$((t))
+[ $((update_every)) -lt 1 ] && update_every=${NETDATA_UPDATE_EVERY}
+[ $((update_every)) -lt 1 ] && update_every=1
+
+# -----------------------------------------------------------------------------
+# allow the user to override our defaults
+
+for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/tc-qos-helper.conf" "${NETDATA_USER_CONFIG_DIR}/tc-qos-helper.conf"; do
+ if [ -f "${CONFIG}" ]; then
+ info "Loading config file '${CONFIG}'..."
+ #shellcheck source=/dev/null
+ source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
+ else
+ warning "Cannot find file '${CONFIG}'."
+ fi
+done
+
+case "${tc_show}" in
+qdisc | class) ;;
+
+*)
+ error "tc_show variable can be either 'qdisc' or 'class' but is set to '${tc_show}'. Assuming it is 'qdisc'."
+ tc_show="qdisc"
+ ;;
+esac
+
+# -----------------------------------------------------------------------------
+# default sleep function
+
+LOOPSLEEPMS_LASTWORK=0
+loopsleepms() {
+ sleep "$1"
+}
+
+# if found and included, this file overwrites loopsleepms()
+# with a high resolution timer function for precise looping.
+#shellcheck source=/dev/null
+. "${plugins_dir}/loopsleepms.sh.inc"
+
+# -----------------------------------------------------------------------------
+# final checks we can run
+
+if [ -z "${tc}" ] || [ ! -x "${tc}" ]; then
+ fatal "cannot find command 'tc' in this system."
+fi
+
+tc_devices=
+fix_names=
+
+# -----------------------------------------------------------------------------
+
+setclassname() {
+ if [ "${tc_show}" = "qdisc" ]; then
+ echo "SETCLASSNAME $4 $2"
+ else
+ echo "SETCLASSNAME $3 $2"
+ fi
+}
+
+show_tc_cls() {
+ [ "${tc_show}" = "qdisc" ] && return 1
+
+ local x="${1}"
+
+ if [ -f /etc/iproute2/tc_cls ]; then
+ local classid name rest
+ while read -r classid name rest; do
+ if [ -z "${classid}" ] ||
+ [ -z "${name}" ] ||
+ [ "${classid}" = "#" ] ||
+ [ "${name}" = "#" ] ||
+ [ "${classid:0:1}" = "#" ] ||
+ [ "${name:0:1}" = "#" ]; then
+ continue
+ fi
+ setclassname "" "${name}" "${classid}"
+ done </etc/iproute2/tc_cls
+ return 0
+ fi
+ return 1
+}
+
+show_fireqos_names() {
+ local x="${1}" name n interface_dev interface_classes_monitor
+
+ if [ -f "${fireqos_run_dir}/ifaces/${x}" ]; then
+ name="$(<"${fireqos_run_dir}/ifaces/${x}")"
+ echo "SETDEVICENAME ${name}" || exit
+
+ #shellcheck source=/dev/null
+ source "${fireqos_run_dir}/${name}.conf"
+ for n in ${interface_classes_monitor}; do
+ # shellcheck disable=SC2086
+ setclassname ${n//|/ }
+ done
+ [ -n "${interface_dev}" ] && echo "SETDEVICEGROUP ${interface_dev}" || exit
+
+ return 0
+ fi
+
+ return 1
+}
+
+show_tc() {
+ local x="${1}"
+
+ echo "BEGIN ${x}" || exit
+
+ # netdata can parse the output of tc
+ ${tc} -s ${tc_show} show dev "${x}"
+
+ # check FireQOS names for classes
+ if [ -n "${fix_names}" ]; then
+ show_fireqos_names "${x}" || show_tc_cls "${x}"
+ fi
+
+ echo "END ${x}" || exit
+}
+
+find_tc_devices() {
+ local count=0 devs dev rest l
+
+ # find all the devices in the system
+ # without forking
+ while IFS=":| " read -r dev rest; do
+ count=$((count + 1))
+ [ ${count} -le 2 ] && continue
+ devs="${devs} ${dev}"
+ done </proc/net/dev
+
+ # from all the devices find the ones
+ # that have QoS defined
+ # unfortunately, one fork per device cannot be avoided
+ tc_devices=
+ for dev in ${devs}; do
+ l="$(${tc} class show dev "${dev}" 2>/dev/null)"
+ [ -n "${l}" ] && tc_devices="${tc_devices} ${dev}"
+ done
+}
+
+# update devices and class names
+# once every 2 minutes
+names_every=$((qos_get_class_names_every / update_every))
+
+# exit this script every hour
+# it will be restarted automatically
+exit_after=$((qos_exit_every / update_every))
+
+c=0
+gc=0
+while true; do
+ fix_names=
+ c=$((c + 1))
+ gc=$((gc + 1))
+
+ if [ ${c} -le 1 ] || [ ${c} -ge ${names_every} ]; then
+ c=1
+ fix_names="YES"
+ find_tc_devices
+ fi
+
+ for d in ${tc_devices}; do
+ show_tc "${d}"
+ done
+
+ echo "WORKTIME ${LOOPSLEEPMS_LASTWORK}" || exit
+
+ loopsleepms ${update_every}
+
+ [ ${gc} -gt ${exit_after} ] && exit 0
+done
diff --git a/collectors/timex.plugin/Makefile.am b/collectors/timex.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/timex.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/timex.plugin/README.md b/collectors/timex.plugin/README.md
new file mode 100644
index 0000000..18665f8
--- /dev/null
+++ b/collectors/timex.plugin/README.md
@@ -0,0 +1,31 @@
+<!--
+title: "timex.plugin"
+description: "Monitor the system clock synchronization state."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/timex.plugin/README.md
+-->
+
+# timex.plugin
+
+This plugin monitors the system kernel clock synchronization state.
+
+This plugin creates the following charts:
+
+- System clock synchronization state according to the system kernel
+- System clock status which gives the value of the `time_status` variable in the kernel
+- Computed time offset between local system and reference clock
+
+This is obtained from the information provided by the [ntp_adjtime()](https://man7.org/linux/man-pages/man2/adjtimex.2.html) system call.
+An unsynchronized clock may indicate a hardware clock error, or an issue with UTC synchronization.
+
+## Configuration
+
+Edit the `netdata.conf` configuration file using [`edit-config`](/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) from the [Netdata config directory](/docs/configure/nodes.md#the-netdata-config-directory), which is typically at `/etc/netdata`.
+
+Scroll down to the `[plugin:timex]` section to find the available options:
+
+```ini
+[plugin:timex]
+ # update every = 1
+ # clock synchronization state = yes
+ # time offset = yes
+```
diff --git a/collectors/timex.plugin/plugin_timex.c b/collectors/timex.plugin/plugin_timex.c
new file mode 100644
index 0000000..46cfc57
--- /dev/null
+++ b/collectors/timex.plugin/plugin_timex.c
@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "daemon/common.h"
+#include "libnetdata/os.h"
+
+#define PLUGIN_TIMEX_NAME "timex.plugin"
+
+#define CONFIG_SECTION_TIMEX "plugin:timex"
+
+struct status_codes {
+ char *name;
+ int code;
+ RRDDIM *rd;
+} sta_codes[] = {
+ // {"pll", STA_PLL, NULL},
+ // {"ppsfreq", STA_PPSFREQ, NULL},
+ // {"ppstime", STA_PPSTIME, NULL},
+ // {"fll", STA_FLL, NULL},
+ // {"ins", STA_INS, NULL},
+ // {"del", STA_DEL, NULL},
+ {"unsync", STA_UNSYNC, NULL},
+ // {"freqhold", STA_FREQHOLD, NULL},
+ // {"ppssignal", STA_PPSSIGNAL, NULL},
+ // {"ppsjitter", STA_PPSJITTER, NULL},
+ // {"ppswander", STA_PPSWANDER, NULL},
+ // {"ppserror", STA_PPSERROR, NULL},
+ {"clockerr", STA_CLOCKERR, NULL},
+ // {"nano", STA_NANO, NULL},
+ // {"clk", STA_CLK, NULL},
+ {NULL, 0, NULL},
+};
+
+static void timex_main_cleanup(void *ptr)
+{
+ worker_unregister();
+
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ info("cleaning up...");
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+void *timex_main(void *ptr)
+{
+ worker_register("TIMEX");
+ worker_register_job_name(0, "clock check");
+
+ netdata_thread_cleanup_push(timex_main_cleanup, ptr);
+
+ int update_every = (int)config_get_number(CONFIG_SECTION_TIMEX, "update every", 10);
+ if (update_every < localhost->rrd_update_every)
+ update_every = localhost->rrd_update_every;
+
+ int do_sync = config_get_boolean(CONFIG_SECTION_TIMEX, "clock synchronization state", CONFIG_BOOLEAN_YES);
+ int do_offset = config_get_boolean(CONFIG_SECTION_TIMEX, "time offset", CONFIG_BOOLEAN_YES);
+
+ if (unlikely(do_sync == CONFIG_BOOLEAN_NO && do_offset == CONFIG_BOOLEAN_NO)) {
+ info("No charts to show");
+ goto exit;
+ }
+
+ usec_t step = update_every * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ while (!netdata_exit) {
+ worker_is_idle();
+ heartbeat_next(&hb, step);
+ worker_is_busy(0);
+
+ struct timex timex_buf = {};
+ int sync_state = 0;
+ static int prev_sync_state = 0;
+
+ sync_state = ADJUST_TIMEX(&timex_buf);
+
+ int non_seq_failure = (sync_state == -1 && prev_sync_state != -1);
+ prev_sync_state = sync_state;
+
+ if (non_seq_failure) {
+ error("Cannot get clock synchronization state");
+ continue;
+ }
+
+ collected_number divisor = USEC_PER_MS;
+ if (timex_buf.status & STA_NANO)
+ divisor = NSEC_PER_MSEC;
+
+ // ----------------------------------------------------------------
+
+ if (do_sync) {
+ static RRDSET *st_sync_state = NULL;
+ static RRDDIM *rd_sync_state;
+
+ if (unlikely(!st_sync_state)) {
+ st_sync_state = rrdset_create_localhost(
+ "system",
+ "clock_sync_state",
+ NULL,
+ "clock synchronization",
+ NULL,
+ "System Clock Synchronization State",
+ "state",
+ PLUGIN_TIMEX_NAME,
+ NULL,
+ NETDATA_CHART_PRIO_CLOCK_SYNC_STATE,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rd_sync_state = rrddim_add(st_sync_state, "state", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_sync_state, rd_sync_state, sync_state != TIME_ERROR ? 1 : 0);
+ rrdset_done(st_sync_state);
+
+ static RRDSET *st_clock_status = NULL;
+
+ if (unlikely(!st_clock_status)) {
+ st_clock_status = rrdset_create_localhost(
+ "system",
+ "clock_status",
+ NULL,
+ "clock synchronization",
+ NULL,
+ "System Clock Status",
+ "status",
+ PLUGIN_TIMEX_NAME,
+ NULL,
+ NETDATA_CHART_PRIO_CLOCK_STATUS,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ for (int i = 0; sta_codes[i].name != NULL; i++) {
+ sta_codes[i].rd =
+ rrddim_add(st_clock_status, sta_codes[i].name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+ }
+
+ for (int i = 0; sta_codes[i].name != NULL; i++)
+ rrddim_set_by_pointer(st_clock_status, sta_codes[i].rd, timex_buf.status & sta_codes[i].code ? 1 : 0);
+
+ rrdset_done(st_clock_status);
+ }
+
+ if (do_offset) {
+ static RRDSET *st_offset = NULL;
+ static RRDDIM *rd_offset;
+
+ if (unlikely(!st_offset)) {
+ st_offset = rrdset_create_localhost(
+ "system",
+ "clock_sync_offset",
+ NULL,
+ "clock synchronization",
+ NULL,
+ "Computed Time Offset Between Local System and Reference Clock",
+ "milliseconds",
+ PLUGIN_TIMEX_NAME,
+ NULL,
+ NETDATA_CHART_PRIO_CLOCK_SYNC_OFFSET,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rd_offset = rrddim_add(st_offset, "offset", NULL, 1, divisor, RRD_ALGORITHM_ABSOLUTE);
+ }
+
+ rrddim_set_by_pointer(st_offset, rd_offset, timex_buf.offset);
+ rrdset_done(st_offset);
+ }
+ }
+
+exit:
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}
diff --git a/collectors/xenstat.plugin/Makefile.am b/collectors/xenstat.plugin/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/collectors/xenstat.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/xenstat.plugin/README.md b/collectors/xenstat.plugin/README.md
new file mode 100644
index 0000000..8cbe086
--- /dev/null
+++ b/collectors/xenstat.plugin/README.md
@@ -0,0 +1,53 @@
+<!--
+title: "xenstat.plugin"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/xenstat.plugin/README.md
+-->
+
+# xenstat.plugin
+
+`xenstat.plugin` collects XenServer and XCP-ng statistics.
+
+## Prerequisites
+
+1. install `xen-dom0-libs-devel` and `yajl-devel` using the package manager of your system.
+ Note: On Cent-OS systems you will need `centos-release-xen` repository and the required package for xen is `xen-devel`
+
+2. re-install Netdata from source. The installer will detect that the required libraries are now available and will also build xenstat.plugin.
+
+Keep in mind that `libxenstat` requires root access, so the plugin is setuid to root.
+
+## Charts
+
+The plugin provides XenServer and XCP-ng host and domains statistics:
+
+Host:
+
+1. Number of domains.
+
+Domain:
+
+1. CPU.
+2. Memory.
+3. Networks.
+4. VBDs.
+
+## Configuration
+
+If you need to disable xenstat for Netdata, edit /etc/netdata/netdata.conf and set:
+
+```
+[plugins]
+ xenstat = no
+```
+
+## Debugging
+
+You can run the plugin by hand:
+
+```
+sudo /usr/libexec/netdata/plugins.d/xenstat.plugin 1 debug
+```
+
+You will get verbose output on what the plugin does.
+
+
diff --git a/collectors/xenstat.plugin/xenstat_plugin.c b/collectors/xenstat.plugin/xenstat_plugin.c
new file mode 100644
index 0000000..ea98b9b
--- /dev/null
+++ b/collectors/xenstat.plugin/xenstat_plugin.c
@@ -0,0 +1,1071 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata/libnetdata.h"
+#include "libnetdata/required_dummies.h"
+
+#include <xenstat.h>
+#include <libxl.h>
+
+#define PLUGIN_XENSTAT_NAME "xenstat.plugin"
+
+#define NETDATA_CHART_PRIO_XENSTAT_NODE_CPUS 30001
+#define NETDATA_CHART_PRIO_XENSTAT_NODE_CPU_FREQ 30002
+#define NETDATA_CHART_PRIO_XENSTAT_NODE_MEM 30003
+#define NETDATA_CHART_PRIO_XENSTAT_NODE_TMEM 30004
+#define NETDATA_CHART_PRIO_XENSTAT_NODE_DOMAINS 30005
+
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_STATES 30101
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_CPU 30102
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_VCPU 30103
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_MEM 30104
+
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_TMEM_PAGES 30104
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_TMEM_OPERATIONS 30105
+
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_VBD_OO_REQ 30200
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_VBD_REQUESTS 30300
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_VBD_SECTORS 30400
+
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_NET_BYTES 30500
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_NET_PACKETS 30600
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_NET_ERRORS 30700
+#define NETDATA_CHART_PRIO_XENSTAT_DOMAIN_NET_DROPS 30800
+
+#define TYPE_LENGTH_MAX 200
+
+#define CHART_IS_OBSOLETE 1
+#define CHART_IS_NOT_OBSOLETE 0
+
+// Variables
+static int debug = 0;
+static int netdata_update_every = 1;
+
+struct vcpu_metrics {
+ unsigned int id;
+
+ unsigned int online;
+ unsigned long long ns;
+
+ int chart_generated;
+ int updated;
+
+ struct vcpu_metrics *next;
+};
+
+struct vbd_metrics {
+ unsigned int id;
+
+ unsigned int error;
+ unsigned long long oo_reqs;
+ unsigned long long rd_reqs;
+ unsigned long long wr_reqs;
+ unsigned long long rd_sects;
+ unsigned long long wr_sects;
+
+ int oo_req_chart_generated;
+ int requests_chart_generated;
+ int sectors_chart_generated;
+ int updated;
+
+ struct vbd_metrics *next;
+};
+
+struct network_metrics {
+ unsigned int id;
+
+ unsigned long long rbytes;
+ unsigned long long rpackets;
+ unsigned long long rerrs;
+ unsigned long long rdrops;
+
+ unsigned long long tbytes;
+ unsigned long long tpackets;
+ unsigned long long terrs;
+ unsigned long long tdrops;
+
+ int bytes_chart_generated;
+ int packets_chart_generated;
+ int errors_chart_generated;
+ int drops_chart_generated;
+ int updated;
+
+ struct network_metrics *next;
+};
+
+struct domain_metrics {
+ char *uuid;
+ uint32_t hash;
+
+ unsigned int id;
+ char *name;
+
+ // states
+ unsigned int running;
+ unsigned int blocked;
+ unsigned int paused;
+ unsigned int shutdown;
+ unsigned int crashed;
+ unsigned int dying;
+ unsigned int cur_vcpus;
+
+ unsigned long long cpu_ns;
+ unsigned long long cur_mem;
+ unsigned long long max_mem;
+
+ struct vcpu_metrics *vcpu_root;
+ struct vbd_metrics *vbd_root;
+ struct network_metrics *network_root;
+
+ int states_chart_generated;
+ int cpu_chart_generated;
+ int vcpu_chart_generated;
+ int num_vcpus_changed;
+ int mem_chart_generated;
+ int updated;
+
+ struct domain_metrics *next;
+};
+
+struct node_metrics{
+ unsigned long long tot_mem;
+ unsigned long long free_mem;
+ int num_domains;
+ unsigned int num_cpus;
+ unsigned long long node_cpu_hz;
+
+ struct domain_metrics *domain_root;
+};
+
+static struct node_metrics node_metrics = {
+ .domain_root = NULL
+};
+
+static inline struct domain_metrics *domain_metrics_get(const char *uuid, uint32_t hash) {
+ struct domain_metrics *d = NULL, *last = NULL;
+ for(d = node_metrics.domain_root; d ; last = d, d = d->next) {
+ if(unlikely(d->hash == hash && !strcmp(d->uuid, uuid)))
+ return d;
+ }
+
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: allocating memory for domain with uuid %s\n", uuid);
+
+ d = callocz(1, sizeof(struct domain_metrics));
+ d->uuid = strdupz(uuid);
+ d->hash = hash;
+
+ if(unlikely(!last)) {
+ d->next = node_metrics.domain_root;
+ node_metrics.domain_root = d;
+ }
+ else {
+ d->next = last->next;
+ last->next = d;
+ }
+
+ return d;
+}
+
+static struct domain_metrics *domain_metrics_free(struct domain_metrics *d) {
+ struct domain_metrics *cur = NULL, *last = NULL;
+ struct vcpu_metrics *vcpu, *vcpu_f;
+ struct vbd_metrics *vbd, *vbd_f;
+ struct network_metrics *network, *network_f;
+
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: freeing memory for domain '%s' id %u, uuid %s\n", d->name, d->id, d->uuid);
+
+ for(cur = node_metrics.domain_root; cur ; last = cur, cur = cur->next) {
+ if(unlikely(cur->hash == d->hash && !strcmp(cur->uuid, d->uuid))) break;
+ }
+
+ if(unlikely(!cur)) {
+ error("XENSTAT: failed to free domain metrics.");
+ return NULL;
+ }
+
+ if(likely(last))
+ last->next = cur->next;
+ else
+ node_metrics.domain_root = NULL;
+
+ freez(cur->uuid);
+ freez(cur->name);
+
+ vcpu = cur->vcpu_root;
+ while(vcpu) {
+ vcpu_f = vcpu;
+ vcpu = vcpu->next;
+ freez(vcpu_f);
+ }
+
+ vbd = cur->vbd_root;
+ while(vbd) {
+ vbd_f = vbd;
+ vbd = vbd->next;
+ freez(vbd_f);
+ }
+
+ network = cur->network_root;
+ while(network) {
+ network_f = network;
+ network = network->next;
+ freez(network_f);
+ }
+
+ freez(cur);
+
+ return last ? last : NULL;
+}
+
+static int vcpu_metrics_collect(struct domain_metrics *d, xenstat_domain *domain) {
+ unsigned int num_vcpus = 0;
+ xenstat_vcpu *vcpu = NULL;
+ struct vcpu_metrics *vcpu_m = NULL, *last_vcpu_m = NULL;
+
+ num_vcpus = xenstat_domain_num_vcpus(domain);
+
+ for(vcpu_m = d->vcpu_root; vcpu_m ; vcpu_m = vcpu_m->next)
+ vcpu_m->updated = 0;
+
+ vcpu_m = d->vcpu_root;
+
+ unsigned int i, num_online_vcpus=0;
+ for(i = 0; i < num_vcpus; i++) {
+ if(unlikely(!vcpu_m)) {
+ vcpu_m = callocz(1, sizeof(struct vcpu_metrics));
+
+ if(unlikely(i == 0)) d->vcpu_root = vcpu_m;
+ else last_vcpu_m->next = vcpu_m;
+ }
+
+ vcpu_m->id = i;
+
+ vcpu = xenstat_domain_vcpu(domain, i);
+
+ if(unlikely(!vcpu)) {
+ error("XENSTAT: cannot get VCPU statistics.");
+ return 1;
+ }
+
+ vcpu_m->online = xenstat_vcpu_online(vcpu);
+ if(likely(vcpu_m->online)) { num_online_vcpus++; }
+ vcpu_m->ns = xenstat_vcpu_ns(vcpu);
+
+ vcpu_m->updated = 1;
+
+ last_vcpu_m = vcpu_m;
+ vcpu_m = vcpu_m->next;
+ }
+
+ if(unlikely(num_online_vcpus != d->cur_vcpus)) {
+ d->num_vcpus_changed = 1;
+ d->cur_vcpus = num_online_vcpus;
+ }
+
+ return 0;
+}
+
+static int vbd_metrics_collect(struct domain_metrics *d, xenstat_domain *domain) {
+ unsigned int num_vbds = xenstat_domain_num_vbds(domain);
+ xenstat_vbd *vbd = NULL;
+ struct vbd_metrics *vbd_m = NULL, *last_vbd_m = NULL;
+
+ for(vbd_m = d->vbd_root; vbd_m ; vbd_m = vbd_m->next)
+ vbd_m->updated = 0;
+
+ vbd_m = d->vbd_root;
+
+ unsigned int i;
+ for(i = 0; i < num_vbds; i++) {
+ if(unlikely(!vbd_m)) {
+ vbd_m = callocz(1, sizeof(struct vbd_metrics));
+
+ if(unlikely(i == 0)) d->vbd_root = vbd_m;
+ else last_vbd_m->next = vbd_m;
+ }
+
+ vbd_m->id = i;
+
+ vbd = xenstat_domain_vbd(domain, i);
+
+ if(unlikely(!vbd)) {
+ error("XENSTAT: cannot get VBD statistics.");
+ return 1;
+ }
+
+#ifdef HAVE_XENSTAT_VBD_ERROR
+ vbd_m->error = xenstat_vbd_error(vbd);
+#else
+ vbd_m->error = 0;
+#endif
+ vbd_m->oo_reqs = xenstat_vbd_oo_reqs(vbd);
+ vbd_m->rd_reqs = xenstat_vbd_rd_reqs(vbd);
+ vbd_m->wr_reqs = xenstat_vbd_wr_reqs(vbd);
+ vbd_m->rd_sects = xenstat_vbd_rd_sects(vbd);
+ vbd_m->wr_sects = xenstat_vbd_wr_sects(vbd);
+
+ vbd_m->updated = 1;
+
+ last_vbd_m = vbd_m;
+ vbd_m = vbd_m->next;
+ }
+
+ return 0;
+}
+
+static int network_metrics_collect(struct domain_metrics *d, xenstat_domain *domain) {
+ unsigned int num_networks = xenstat_domain_num_networks(domain);
+ xenstat_network *network = NULL;
+ struct network_metrics *network_m = NULL, *last_network_m = NULL;
+
+ for(network_m = d->network_root; network_m ; network_m = network_m->next)
+ network_m->updated = 0;
+
+ network_m = d->network_root;
+
+ unsigned int i;
+ for(i = 0; i < num_networks; i++) {
+ if(unlikely(!network_m)) {
+ network_m = callocz(1, sizeof(struct network_metrics));
+
+ if(unlikely(i == 0)) d->network_root = network_m;
+ else last_network_m->next = network_m;
+ }
+
+ network_m->id = i;
+
+ network = xenstat_domain_network(domain, i);
+
+ if(unlikely(!network)) {
+ error("XENSTAT: cannot get network statistics.");
+ return 1;
+ }
+
+ network_m->rbytes = xenstat_network_rbytes(network);
+ network_m->rpackets = xenstat_network_rpackets(network);
+ network_m->rerrs = xenstat_network_rerrs(network);
+ network_m->rdrops = xenstat_network_rdrop(network);
+
+ network_m->tbytes = xenstat_network_tbytes(network);
+ network_m->tpackets = xenstat_network_tpackets(network);
+ network_m->terrs = xenstat_network_terrs(network);
+ network_m->tdrops = xenstat_network_tdrop(network);
+
+ network_m->updated = 1;
+
+ last_network_m = network_m;
+ network_m = network_m->next;
+ }
+
+ return 0;
+}
+
+static int xenstat_collect(xenstat_handle *xhandle, libxl_ctx *ctx, libxl_dominfo *info) {
+
+ // mark all old metrics as not-updated
+ struct domain_metrics *d;
+ for(d = node_metrics.domain_root; d ; d = d->next)
+ d->updated = 0;
+
+ xenstat_node *node = xenstat_get_node(xhandle, XENSTAT_ALL);
+ if (unlikely(!node)) {
+ error("XENSTAT: failed to retrieve statistics from libxenstat.");
+ return 1;
+ }
+
+ node_metrics.tot_mem = xenstat_node_tot_mem(node);
+ node_metrics.free_mem = xenstat_node_free_mem(node);
+ node_metrics.num_domains = xenstat_node_num_domains(node);
+ node_metrics.num_cpus = xenstat_node_num_cpus(node);
+ node_metrics.node_cpu_hz = xenstat_node_cpu_hz(node);
+
+ int i;
+ for(i = 0; i < node_metrics.num_domains; i++) {
+ xenstat_domain *domain = NULL;
+ char uuid[LIBXL_UUID_FMTLEN + 1];
+
+ domain = xenstat_node_domain_by_index(node, i);
+
+ // get domain UUID
+ unsigned int id = xenstat_domain_id(domain);
+ if(unlikely(libxl_domain_info(ctx, info, id))) {
+ error("XENSTAT: cannot get domain info.");
+ }
+ else {
+ snprintfz(uuid, LIBXL_UUID_FMTLEN, LIBXL_UUID_FMT "\n", LIBXL_UUID_BYTES(info->uuid));
+ }
+
+ uint32_t hash = simple_hash(uuid);
+ d = domain_metrics_get(uuid, hash);
+
+ d->id = id;
+ if(unlikely(!d->name)) {
+ d->name = strdupz(xenstat_domain_name(domain));
+ netdata_fix_chart_id(d->name);
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: domain id %u, uuid %s has name '%s'\n", d->id, d->uuid, d->name);
+ }
+
+ d->running = xenstat_domain_running(domain);
+ d->blocked = xenstat_domain_blocked(domain);
+ d->paused = xenstat_domain_paused(domain);
+ d->shutdown = xenstat_domain_shutdown(domain);
+ d->crashed = xenstat_domain_crashed(domain);
+ d->dying = xenstat_domain_dying(domain);
+
+ d->cpu_ns = xenstat_domain_cpu_ns(domain);
+ d->cur_mem = xenstat_domain_cur_mem(domain);
+ d->max_mem = xenstat_domain_max_mem(domain);
+
+ if(unlikely(vcpu_metrics_collect(d, domain) || vbd_metrics_collect(d, domain) || network_metrics_collect(d, domain))) {
+ xenstat_free_node(node);
+ return 1;
+ }
+
+ d->updated = 1;
+ }
+
+ xenstat_free_node(node);
+
+ return 0;
+}
+
+static void xenstat_send_node_metrics() {
+ static int mem_chart_generated = 0, domains_chart_generated = 0, cpus_chart_generated = 0, cpu_freq_chart_generated = 0;
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!mem_chart_generated)) {
+ printf("CHART xenstat.mem '' 'Memory Usage' 'MiB' 'memory' '' stacked %d %d '' %s\n"
+ , NETDATA_CHART_PRIO_XENSTAT_NODE_MEM
+ , netdata_update_every
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 %d\n", "free", netdata_update_every * 1024 * 1024);
+ printf("DIMENSION %s '' absolute 1 %d\n", "used", netdata_update_every * 1024 * 1024);
+ mem_chart_generated = 1;
+ }
+
+ printf(
+ "BEGIN xenstat.mem\n"
+ "SET free = %lld\n"
+ "SET used = %lld\n"
+ "END\n"
+ , (collected_number) node_metrics.free_mem
+ , (collected_number) (node_metrics.tot_mem - node_metrics.free_mem)
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!domains_chart_generated)) {
+ printf("CHART xenstat.domains '' 'Number of Domains' 'domains' 'domains' '' line %d %d '' %s\n"
+ , NETDATA_CHART_PRIO_XENSTAT_NODE_DOMAINS
+ , netdata_update_every
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 %d\n", "domains", netdata_update_every);
+ domains_chart_generated = 1;
+ }
+
+ printf(
+ "BEGIN xenstat.domains\n"
+ "SET domains = %lld\n"
+ "END\n"
+ , (collected_number) node_metrics.num_domains
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!cpus_chart_generated)) {
+ printf("CHART xenstat.cpus '' 'Number of CPUs' 'cpus' 'cpu' '' line %d %d '' %s\n"
+ , NETDATA_CHART_PRIO_XENSTAT_NODE_CPUS
+ , netdata_update_every
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 %d\n", "cpus", netdata_update_every);
+ cpus_chart_generated = 1;
+ }
+
+ printf(
+ "BEGIN xenstat.cpus\n"
+ "SET cpus = %lld\n"
+ "END\n"
+ , (collected_number) node_metrics.num_cpus
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!cpu_freq_chart_generated)) {
+ printf("CHART xenstat.cpu_freq '' 'CPU Frequency' 'MHz' 'cpu' '' line %d %d '' %s\n"
+ , NETDATA_CHART_PRIO_XENSTAT_NODE_CPU_FREQ
+ , netdata_update_every
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION %s '' absolute 1 %d\n", "frequency", netdata_update_every * 1024 * 1024);
+ cpu_freq_chart_generated = 1;
+ }
+
+ printf(
+ "BEGIN xenstat.cpu_freq\n"
+ "SET frequency = %lld\n"
+ "END\n"
+ , (collected_number) node_metrics.node_cpu_hz
+ );
+}
+
+static void print_domain_states_chart_definition(char *type, int obsolete_flag) {
+ printf("CHART %s.states '' 'Domain States' 'boolean' 'states' 'xendomain.states' line %d %d %s %s\n"
+ , type
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_STATES
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION running '' absolute 1 %d\n", netdata_update_every);
+ printf("DIMENSION blocked '' absolute 1 %d\n", netdata_update_every);
+ printf("DIMENSION paused '' absolute 1 %d\n", netdata_update_every);
+ printf("DIMENSION shutdown '' absolute 1 %d\n", netdata_update_every);
+ printf("DIMENSION crashed '' absolute 1 %d\n", netdata_update_every);
+ printf("DIMENSION dying '' absolute 1 %d\n", netdata_update_every);
+}
+
+static void print_domain_cpu_chart_definition(char *type, int obsolete_flag) {
+ printf("CHART %s.cpu '' 'CPU Usage (100%% = 1 core)' 'percentage' 'cpu' 'xendomain.cpu' line %d %d %s %s\n"
+ , type
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_CPU
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION used '' incremental 100 %d\n", netdata_update_every * 1000000000);
+}
+
+static void print_domain_mem_chart_definition(char *type, int obsolete_flag) {
+ printf("CHART %s.mem '' 'Memory Reservation' 'MiB' 'memory' 'xendomain.mem' line %d %d %s %s\n"
+ , type
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_MEM
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION maximum '' absolute 1 %d\n", netdata_update_every * 1024 * 1024);
+ printf("DIMENSION current '' absolute 1 %d\n", netdata_update_every * 1024 * 1024);
+}
+
+static void print_domain_vcpu_chart_definition(char *type, struct domain_metrics *d, int obsolete_flag) {
+ struct vcpu_metrics *vcpu_m;
+
+ printf("CHART %s.vcpu '' 'CPU Usage per VCPU' 'percentage' 'cpu' 'xendomain.vcpu' line %d %d %s %s\n"
+ , type
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_VCPU
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+
+ for(vcpu_m = d->vcpu_root; vcpu_m; vcpu_m = vcpu_m->next) {
+ if(likely(vcpu_m->updated && vcpu_m->online)) {
+ printf("DIMENSION vcpu%u '' incremental 100 %d\n", vcpu_m->id, netdata_update_every * 1000000000);
+ }
+ }
+}
+
+static void print_domain_vbd_oo_chart_definition(char *type, unsigned int vbd, int obsolete_flag) {
+ printf("CHART %s.oo_req_vbd%u '' 'VBD%u \"Out Of\" Requests' 'requests/s' 'vbd' 'xendomain.oo_req_vbd' line %u %d %s %s\n"
+ , type
+ , vbd
+ , vbd
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_VBD_OO_REQ + vbd
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION requests '' incremental 1 %d\n", netdata_update_every);
+}
+
+static void print_domain_vbd_requests_chart_definition(char *type, unsigned int vbd, int obsolete_flag) {
+ printf("CHART %s.requests_vbd%u '' 'VBD%u Requests' 'requests/s' 'vbd' 'xendomain.requests_vbd' line %u %d %s %s\n"
+ , type
+ , vbd
+ , vbd
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_VBD_REQUESTS + vbd
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION read '' incremental 1 %d\n", netdata_update_every);
+ printf("DIMENSION write '' incremental -1 %d\n", netdata_update_every);
+}
+
+static void print_domain_vbd_sectors_chart_definition(char *type, unsigned int vbd, int obsolete_flag) {
+ printf("CHART %s.sectors_vbd%u '' 'VBD%u Read/Written Sectors' 'sectors/s' 'vbd' 'xendomain.sectors_vbd' line %u %d %s %s\n"
+ , type
+ , vbd
+ , vbd
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_VBD_SECTORS + vbd
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION read '' incremental 1 %d\n", netdata_update_every);
+ printf("DIMENSION write '' incremental -1 %d\n", netdata_update_every);
+}
+
+static void print_domain_network_bytes_chart_definition(char *type, unsigned int network, int obsolete_flag) {
+ printf("CHART %s.bytes_network%u '' 'Network%u Received/Sent Bytes' 'kilobits/s' 'network' 'xendomain.bytes_network' line %u %d %s %s\n"
+ , type
+ , network
+ , network
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_NET_BYTES + network
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION received '' incremental 8 %d\n", netdata_update_every * 1000);
+ printf("DIMENSION sent '' incremental -8 %d\n", netdata_update_every * 1000);
+}
+
+static void print_domain_network_packets_chart_definition(char *type, unsigned int network, int obsolete_flag) {
+ printf("CHART %s.packets_network%u '' 'Network%u Received/Sent Packets' 'packets/s' 'network' 'xendomain.packets_network' line %u %d %s %s\n"
+ , type
+ , network
+ , network
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_NET_PACKETS + network
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION received '' incremental 1 %d\n", netdata_update_every);
+ printf("DIMENSION sent '' incremental -1 %d\n", netdata_update_every);
+}
+
+static void print_domain_network_errors_chart_definition(char *type, unsigned int network, int obsolete_flag) {
+ printf("CHART %s.errors_network%u '' 'Network%u Receive/Transmit Errors' 'errors/s' 'network' 'xendomain.errors_network' line %u %d %s %s\n"
+ , type
+ , network
+ , network
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_NET_PACKETS + network
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION received '' incremental 1 %d\n", netdata_update_every);
+ printf("DIMENSION sent '' incremental -1 %d\n", netdata_update_every);
+}
+
+static void print_domain_network_drops_chart_definition(char *type, unsigned int network, int obsolete_flag) {
+ printf("CHART %s.drops_network%u '' 'Network%u Receive/Transmit Drops' 'drops/s' 'network' 'xendomain.drops_network' line %u %d %s %s\n"
+ , type
+ , network
+ , network
+ , NETDATA_CHART_PRIO_XENSTAT_DOMAIN_NET_PACKETS + network
+ , netdata_update_every
+ , obsolete_flag ? "obsolete": "''"
+ , PLUGIN_XENSTAT_NAME
+ );
+ printf("DIMENSION received '' incremental 1 %d\n", netdata_update_every);
+ printf("DIMENSION sent '' incremental -1 %d\n", netdata_update_every);
+}
+
+static void xenstat_send_domain_metrics() {
+
+ if(unlikely(!node_metrics.domain_root)) return;
+ struct domain_metrics *d;
+
+ for(d = node_metrics.domain_root; d; d = d->next) {
+ char type[TYPE_LENGTH_MAX + 1];
+ snprintfz(type, TYPE_LENGTH_MAX, "xendomain_%s_%s", d->name, d->uuid);
+
+ if(likely(d->updated)) {
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!d->states_chart_generated)) {
+ print_domain_states_chart_definition(type, CHART_IS_NOT_OBSOLETE);
+ d->states_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.states\n"
+ "SET running = %lld\n"
+ "SET blocked = %lld\n"
+ "SET paused = %lld\n"
+ "SET shutdown = %lld\n"
+ "SET crashed = %lld\n"
+ "SET dying = %lld\n"
+ "END\n"
+ , type
+ , (collected_number)d->running
+ , (collected_number)d->blocked
+ , (collected_number)d->paused
+ , (collected_number)d->shutdown
+ , (collected_number)d->crashed
+ , (collected_number)d->dying
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!d->cpu_chart_generated)) {
+ print_domain_cpu_chart_definition(type, CHART_IS_NOT_OBSOLETE);
+ d->cpu_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.cpu\n"
+ "SET used = %lld\n"
+ "END\n"
+ , type
+ , (collected_number)d->cpu_ns
+ );
+
+ // ----------------------------------------------------------------
+
+ struct vcpu_metrics *vcpu_m;
+
+ if(unlikely(!d->vcpu_chart_generated || d->num_vcpus_changed)) {
+ print_domain_vcpu_chart_definition(type, d, CHART_IS_NOT_OBSOLETE);
+ d->num_vcpus_changed = 0;
+ d->vcpu_chart_generated = 1;
+ }
+
+ printf("BEGIN %s.vcpu\n", type);
+ for(vcpu_m = d->vcpu_root; vcpu_m; vcpu_m = vcpu_m->next) {
+ if(likely(vcpu_m->updated && vcpu_m->online)) {
+ printf(
+ "SET vcpu%u = %lld\n"
+ , vcpu_m->id
+ , (collected_number)vcpu_m->ns
+ );
+ }
+ }
+ printf("END\n");
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!d->mem_chart_generated)) {
+ print_domain_mem_chart_definition(type, CHART_IS_NOT_OBSOLETE);
+ d->mem_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.mem\n"
+ "SET maximum = %lld\n"
+ "SET current = %lld\n"
+ "END\n"
+ , type
+ , (collected_number)d->max_mem
+ , (collected_number)d->cur_mem
+ );
+
+ // ----------------------------------------------------------------
+
+ struct vbd_metrics *vbd_m;
+ for(vbd_m = d->vbd_root; vbd_m; vbd_m = vbd_m->next) {
+ if(likely(vbd_m->updated && !vbd_m->error)) {
+ if(unlikely(!vbd_m->oo_req_chart_generated)) {
+ print_domain_vbd_oo_chart_definition(type, vbd_m->id, CHART_IS_NOT_OBSOLETE);
+ vbd_m->oo_req_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.oo_req_vbd%u\n"
+ "SET requests = %lld\n"
+ "END\n"
+ , type
+ , vbd_m->id
+ , (collected_number)vbd_m->oo_reqs
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!vbd_m->requests_chart_generated)) {
+ print_domain_vbd_requests_chart_definition(type, vbd_m->id, CHART_IS_NOT_OBSOLETE);
+ vbd_m->requests_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.requests_vbd%u\n"
+ "SET read = %lld\n"
+ "SET write = %lld\n"
+ "END\n"
+ , type
+ , vbd_m->id
+ , (collected_number)vbd_m->rd_reqs
+ , (collected_number)vbd_m->wr_reqs
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!vbd_m->sectors_chart_generated)) {
+ print_domain_vbd_sectors_chart_definition(type, vbd_m->id, CHART_IS_NOT_OBSOLETE);
+ vbd_m->sectors_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.sectors_vbd%u\n"
+ "SET read = %lld\n"
+ "SET write = %lld\n"
+ "END\n"
+ , type
+ , vbd_m->id
+ , (collected_number)vbd_m->rd_sects
+ , (collected_number)vbd_m->wr_sects
+ );
+ }
+ else {
+ if(unlikely(vbd_m->oo_req_chart_generated
+ || vbd_m->requests_chart_generated
+ || vbd_m->sectors_chart_generated)) {
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: mark charts as obsolete for vbd %u, domain '%s', id %u, uuid %s\n", vbd_m->id, d->name, d->id, d->uuid);
+ print_domain_vbd_oo_chart_definition(type, vbd_m->id, CHART_IS_OBSOLETE);
+ print_domain_vbd_requests_chart_definition(type, vbd_m->id, CHART_IS_OBSOLETE);
+ print_domain_vbd_sectors_chart_definition(type, vbd_m->id, CHART_IS_OBSOLETE);
+ vbd_m->oo_req_chart_generated = 0;
+ vbd_m->requests_chart_generated = 0;
+ vbd_m->sectors_chart_generated = 0;
+ }
+ }
+ }
+
+ // ----------------------------------------------------------------
+
+ struct network_metrics *network_m;
+ for(network_m = d->network_root; network_m; network_m = network_m->next) {
+ if(likely(network_m->updated)) {
+ if(unlikely(!network_m->bytes_chart_generated)) {
+ print_domain_network_bytes_chart_definition(type, network_m->id, CHART_IS_NOT_OBSOLETE);
+ network_m->bytes_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.bytes_network%u\n"
+ "SET received = %lld\n"
+ "SET sent = %lld\n"
+ "END\n"
+ , type
+ , network_m->id
+ , (collected_number)network_m->rbytes
+ , (collected_number)network_m->tbytes
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!network_m->packets_chart_generated)) {
+ print_domain_network_packets_chart_definition(type, network_m->id, CHART_IS_NOT_OBSOLETE);
+ network_m->packets_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.packets_network%u\n"
+ "SET received = %lld\n"
+ "SET sent = %lld\n"
+ "END\n"
+ , type
+ , network_m->id
+ , (collected_number)network_m->rpackets
+ , (collected_number)network_m->tpackets
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!network_m->errors_chart_generated)) {
+ print_domain_network_errors_chart_definition(type, network_m->id, CHART_IS_NOT_OBSOLETE);
+ network_m->errors_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.errors_network%u\n"
+ "SET received = %lld\n"
+ "SET sent = %lld\n"
+ "END\n"
+ , type
+ , network_m->id
+ , (collected_number)network_m->rerrs
+ , (collected_number)network_m->terrs
+ );
+
+ // ----------------------------------------------------------------
+
+ if(unlikely(!network_m->drops_chart_generated)) {
+ print_domain_network_drops_chart_definition(type, network_m->id, CHART_IS_NOT_OBSOLETE);
+ network_m->drops_chart_generated = 1;
+ }
+ printf(
+ "BEGIN %s.drops_network%u\n"
+ "SET received = %lld\n"
+ "SET sent = %lld\n"
+ "END\n"
+ , type
+ , network_m->id
+ , (collected_number)network_m->rdrops
+ , (collected_number)network_m->tdrops
+ );
+ }
+ else {
+ if(unlikely(network_m->bytes_chart_generated
+ || network_m->packets_chart_generated
+ || network_m->errors_chart_generated
+ || network_m->drops_chart_generated))
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: mark charts as obsolete for network %u, domain '%s', id %u, uuid %s\n", network_m->id, d->name, d->id, d->uuid);
+ print_domain_network_bytes_chart_definition(type, network_m->id, CHART_IS_OBSOLETE);
+ print_domain_network_packets_chart_definition(type, network_m->id, CHART_IS_OBSOLETE);
+ print_domain_network_errors_chart_definition(type, network_m->id, CHART_IS_OBSOLETE);
+ print_domain_network_drops_chart_definition(type, network_m->id, CHART_IS_OBSOLETE);
+ network_m->bytes_chart_generated = 0;
+ network_m->packets_chart_generated = 0;
+ network_m->errors_chart_generated = 0;
+ network_m->drops_chart_generated = 0;
+ }
+ }
+ }
+ else{
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: mark charts as obsolete for domain '%s', id %u, uuid %s\n", d->name, d->id, d->uuid);
+ print_domain_states_chart_definition(type, CHART_IS_OBSOLETE);
+ print_domain_cpu_chart_definition(type, CHART_IS_OBSOLETE);
+ print_domain_vcpu_chart_definition(type, d, CHART_IS_OBSOLETE);
+ print_domain_mem_chart_definition(type, CHART_IS_OBSOLETE);
+
+ d = domain_metrics_free(d);
+ }
+ }
+}
+
+int main(int argc, char **argv) {
+ clocks_init();
+
+ // ------------------------------------------------------------------------
+ // initialization of netdata plugin
+
+ program_name = "xenstat.plugin";
+
+ // disable syslog
+ error_log_syslog = 0;
+
+ // set errors flood protection to 100 logs per hour
+ error_log_errors_per_period = 100;
+ error_log_throttle_period = 3600;
+
+ // ------------------------------------------------------------------------
+ // parse command line parameters
+
+ int i, freq = 0;
+ for(i = 1; i < argc ; i++) {
+ if(isdigit(*argv[i]) && !freq) {
+ int n = str2i(argv[i]);
+ if(n > 0 && n < 86400) {
+ freq = n;
+ continue;
+ }
+ }
+ else if(strcmp("version", argv[i]) == 0 || strcmp("-version", argv[i]) == 0 || strcmp("--version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) {
+ printf("xenstat.plugin %s\n", VERSION);
+ exit(0);
+ }
+ else if(strcmp("debug", argv[i]) == 0) {
+ debug = 1;
+ continue;
+ }
+ else if(strcmp("-h", argv[i]) == 0 || strcmp("--help", argv[i]) == 0) {
+ fprintf(stderr,
+ "\n"
+ " netdata xenstat.plugin %s\n"
+ " Copyright (C) 2019 Netdata Inc.\n"
+ " Released under GNU General Public License v3 or later.\n"
+ " All rights reserved.\n"
+ "\n"
+ " This program is a data collector plugin for netdata.\n"
+ "\n"
+ " Available command line options:\n"
+ "\n"
+ " COLLECTION_FREQUENCY data collection frequency in seconds\n"
+ " minimum: %d\n"
+ "\n"
+ " debug enable verbose output\n"
+ " default: disabled\n"
+ "\n"
+ " -v\n"
+ " -V\n"
+ " --version print version and exit\n"
+ "\n"
+ " -h\n"
+ " --help print this message and exit\n"
+ "\n"
+ " For more information:\n"
+ " https://github.com/netdata/netdata/tree/master/collectors/xenstat.plugin\n"
+ "\n"
+ , VERSION
+ , netdata_update_every
+ );
+ exit(1);
+ }
+
+ error("xenstat.plugin: ignoring parameter '%s'", argv[i]);
+ }
+
+ errno = 0;
+
+ if(freq >= netdata_update_every)
+ netdata_update_every = freq;
+ else if(freq)
+ error("update frequency %d seconds is too small for XENSTAT. Using %d.", freq, netdata_update_every);
+
+ // ------------------------------------------------------------------------
+ // initialize xen API handles
+ xenstat_handle *xhandle = NULL;
+ libxl_ctx *ctx = NULL;
+ libxl_dominfo info;
+
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: calling xenstat_init()\n");
+ xhandle = xenstat_init();
+ if (xhandle == NULL) {
+ error("XENSTAT: failed to initialize xenstat library.");
+ return 1;
+ }
+
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: calling libxl_ctx_alloc()\n");
+ if (libxl_ctx_alloc(&ctx, LIBXL_VERSION, 0, NULL)) {
+ error("XENSTAT: failed to initialize xl context.");
+ xenstat_uninit(xhandle);
+ return 1;
+ }
+ libxl_dominfo_init(&info);
+
+ // ------------------------------------------------------------------------
+ // the main loop
+
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: starting data collection\n");
+
+ time_t started_t = now_monotonic_sec();
+
+ size_t iteration;
+ usec_t step = netdata_update_every * USEC_PER_SEC;
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ for(iteration = 0; 1; iteration++) {
+ usec_t dt = heartbeat_next(&hb, step);
+
+ if(unlikely(netdata_exit)) break;
+
+ if(unlikely(debug && iteration))
+ fprintf(stderr, "xenstat.plugin: iteration %zu, dt %llu usec\n"
+ , iteration
+ , dt
+ );
+
+ if(likely(xhandle)) {
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: calling xenstat_collect()\n");
+ int ret = xenstat_collect(xhandle, ctx, &info);
+
+ if(likely(!ret)) {
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: calling xenstat_send_node_metrics()\n");
+ xenstat_send_node_metrics();
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: calling xenstat_send_domain_metrics()\n");
+ xenstat_send_domain_metrics();
+ }
+ else {
+ if(unlikely(debug)) fprintf(stderr, "xenstat.plugin: can't collect data\n");
+ }
+ }
+
+ fflush(stdout);
+
+ // restart check (14400 seconds)
+ if(unlikely(now_monotonic_sec() - started_t > 14400)) break;
+ }
+
+ libxl_ctx_free(ctx);
+ xenstat_uninit(xhandle);
+ info("XENSTAT process exiting");
+
+ return 0;
+}