diff options
Diffstat (limited to '')
| -rw-r--r-- | daemon/README.md | 218 | ||||
| -rw-r--r-- | daemon/main.c | 34 | ||||
| -rw-r--r-- | daemon/unit_test.c | 312 | ||||
| -rw-r--r-- | daemon/unit_test.h | 3 |
4 files changed, 436 insertions, 131 deletions
diff --git a/daemon/README.md b/daemon/README.md index 2facbdda6..0d4b0cdbb 100644 --- a/daemon/README.md +++ b/daemon/README.md @@ -4,13 +4,13 @@ - You can start Netdata by executing it with `/usr/sbin/netdata` (the installer will also start it). -- You can stop Netdata by killing it with `killall netdata`. - You can stop and start Netdata at any point. Netdata saves on exit its round robbin - database to `/var/cache/netdata` so that it will continue from where it stopped the last time. +- You can stop Netdata by killing it with `killall netdata`. You can stop and start Netdata at any point. Netdata + saves on exit its round robbin database to `/var/cache/netdata` so that it will continue from where it stopped the + last time. Access to the web site, for all graphs, is by default on port `19999`, so go to: -``` +```sh http://127.0.0.1:19999/ ``` @@ -18,7 +18,8 @@ You can get the running config file at any time, by accessing `http://127.0.0.1: ### Starting Netdata at boot -In the `system` directory you can find scripts and configurations for the various distros. +In the `system` directory you can find scripts and configurations for the +various distros. #### systemd @@ -45,7 +46,8 @@ systemctl start netdata #### init.d -In the system directory you can find `netdata-lsb`. Copy it to the proper place according to your distribution documentation. For Ubuntu, this can be done via running the following commands as root. +In the system directory you can find `netdata-lsb`. Copy it to the proper place according to your distribution +documentation. For Ubuntu, this can be done via running the following commands as root. ```sh # copy the Netdata startup file to /etc/init.d @@ -60,11 +62,13 @@ update-rc.d netdata defaults #### openrc (gentoo) -In the `system` directory you can find `netdata-openrc`. Copy it to the proper place according to your distribution documentation. +In the `system` directory you can find `netdata-openrc`. Copy it to the proper +place according to your distribution documentation. #### CentOS / Red Hat Enterprise Linux -For older versions of RHEL/CentOS that don't have systemd, an init script is included in the system directory. This can be installed by running the following commands as root. +For older versions of RHEL/CentOS that don't have systemd, an init script is included in the system directory. This can +be installed by running the following commands as root. ```sh # copy the Netdata startup file to /etc/init.d @@ -77,7 +81,8 @@ chmod +x /etc/init.d/netdata chkconfig --add netdata ``` -_There have been some recent work on the init script, see PR <https://github.com/netdata/netdata/pull/403>_ +_There have been some recent work on the init script, see PR +<https://github.com/netdata/netdata/pull/403>_ #### other systems @@ -99,7 +104,7 @@ The program will print the supported command line parameters. The command line options of the Netdata 1.10.0 version are the following: -``` +```sh ^ |.-. .-. .-. .-. . netdata | '-' '-' '-' '-' real-time performance monitoring, done right! @@ -188,34 +193,34 @@ Netdata uses 3 log files: 2. `access.log` 3. `debug.log` -Any of them can be disabled by setting it to `/dev/null` or `none` in `netdata.conf`. -By default `error.log` and `access.log` are enabled. `debug.log` is only enabled if -debugging/tracing is also enabled (Netdata needs to be compiled with debugging enabled). +Any of them can be disabled by setting it to `/dev/null` or `none` in `netdata.conf`. By default `error.log` and +`access.log` are enabled. `debug.log` is only enabled if debugging/tracing is also enabled (Netdata needs to be compiled +with debugging enabled). Log files are stored in `/var/log/netdata/` by default. -#### error.log +### error.log -The `error.log` is the `stderr` of the `netdata` daemon and all external plugins run by netdata. +The `error.log` is the `stderr` of the `netdata` daemon and all external plugins +run by `netdata`. So if any process, in the Netdata process tree, writes anything to its standard error, it will appear in `error.log`. -For most Netdata programs (including standard external plugins shipped by netdata), the -following lines may appear: +For most Netdata programs (including standard external plugins shipped by netdata), the following lines may appear: -| tag|description| +| tag | description | |:-:|:----------| -| `INFO`|Something important the user should know.| -| `ERROR`|Something that might disable a part of netdata.<br/>The log line includes `errno` (if it is not zero).| -| `FATAL`|Something prevented a program from running.<br/>The log line includes `errno` (if it is not zero) and the program exited.| +| `INFO` | Something important the user should know. | +| `ERROR` | Something that might disable a part of netdata.<br/>The log line includes `errno` (if it is not zero). | +| `FATAL` | Something prevented a program from running.<br/>The log line includes `errno` (if it is not zero) and the program exited. | -So, when auto-detection of data collection fail, `ERROR` lines are logged and the relevant modules -are disabled, but the program continues to run. +So, when auto-detection of data collection fail, `ERROR` lines are logged and the relevant modules are disabled, but the +program continues to run. When a Netdata program cannot run at all, a `FATAL` line is logged. -#### access.log +### access.log The `access.log` logs web requests. The format is: @@ -231,21 +236,22 @@ where: - `PERCENT_COMPRESSION` is the percentage of traffic saved due to compression. - `PREP_TIME` is the time in milliseconds needed to prepared the response. - `SENT_TIME` is the time in milliseconds needed to sent the response to the client. -- `TOTAL_TIME` is the total time the request was inside Netdata (from the first byte of the request to the last byte of the response). +- `TOTAL_TIME` is the total time the request was inside Netdata (from the first byte of the request to the last byte + of the response). - `ACTION` can be `filecopy`, `options` (used in CORS), `data` (API call). -#### debug.log +### debug.log See [debugging](#debugging). ## OOM Score -Netdata runs with `OOMScore = 1000`. This means Netdata will be the first to be killed when your -server runs out of memory. +Netdata runs with `OOMScore = 1000`. This means Netdata will be the first to be killed when your server runs out of +memory. You can set Netdata OOMScore in `netdata.conf`, like this: -``` +```conf [global] OOM score = 1000 ``` @@ -257,15 +263,13 @@ Netdata logs its OOM score when it starts: 2017-10-15 03:47:31: netdata INFO : Adjusted my Out-Of-Memory (OOM) score from 0 to 1000. ``` -#### OOM score and systemd +### OOM score and systemd -Netdata will not be able to lower its OOM Score below zero, when it is started as the `netdata` -user (systemd case). +Netdata will not be able to lower its OOM Score below zero, when it is started as the `netdata` user (systemd case). -To allow Netdata control its OOM Score in such cases, you will need to edit -`netdata.service` and set: +To allow Netdata control its OOM Score in such cases, you will need to edit `netdata.service` and set: -``` +```sh [Service] # The minimum Netdata Out-Of-Memory (OOM) score. # Netdata (via [global].OOM score in netdata.conf) can only increase the value set here. @@ -276,12 +280,11 @@ OOMScoreAdjust=-1000 Run `systemctl daemon-reload` to reload these changes. -The above, sets and OOMScore for Netdata to `-1000`, so that Netdata can increase it via -`netdata.conf`. +The above, sets and OOMScore for Netdata to `-1000`, so that Netdata can increase it via `netdata.conf`. If you want to control it entirely via systemd, you can set in `netdata.conf`: -``` +```conf [global] OOM score = keep ``` @@ -290,25 +293,26 @@ Using the above, whatever OOM Score you have set at `netdata.service` will be ma ## Netdata process scheduling policy -By default Netdata runs with the `idle` process scheduling policy, so that it uses CPU resources, only when there is idle CPU to spare. On very busy servers (or weak servers), this can lead to gaps on the charts. +By default Netdata runs with the `idle` process scheduling policy, so that it uses CPU resources, only when there is +idle CPU to spare. On very busy servers (or weak servers), this can lead to gaps on the charts. You can set Netdata scheduling policy in `netdata.conf`, like this: -``` +```conf [global] process scheduling policy = idle ``` You can use the following: -| policy|description| +| policy | description | |:----:|:----------| -| `idle`|use CPU only when there is spare - this is lower than nice 19 - it is the default for Netdata and it is so low that Netdata will run in "slow motion" under extreme system load, resulting in short (1-2 seconds) gaps at the charts.| -| `other`<br/>or<br/>`nice`|this is the default policy for all processes under Linux. It provides dynamic priorities based on the `nice` level of each process. Check below for setting this `nice` level for netdata.| -| `batch`|This policy is similar to `other` in that it schedules the thread according to its dynamic priority (based on the `nice` value). The difference is that this policy will cause the scheduler to always assume that the thread is CPU-intensive. Consequently, the scheduler will apply a small scheduling penalty with respect to wake-up behavior, so that this thread is mildly disfavored in scheduling decisions.| -| `fifo`|`fifo` can be used only with static priorities higher than 0, which means that when a `fifo` threads becomes runnable, it will always immediately preempt any currently running `other`, `batch`, or `idle` thread. `fifo` is a simple scheduling algorithm without time slicing.| -| `rr`|a simple enhancement of `fifo`. Everything described above for `fifo` also applies to `rr`, except that each thread is allowed to run only for a maximum time quantum.| -| `keep`<br/>or<br/>`none`|do not set scheduling policy, priority or nice level - i.e. keep running with whatever it is set already (e.g. by systemd).| +| `idle` | use CPU only when there is spare - this is lower than nice 19 - it is the default for Netdata and it is so low that Netdata will run in "slow motion" under extreme system load, resulting in short (1-2 seconds) gaps at the charts. | +| `other`<br/>or<br/>`nice` | this is the default policy for all processes under Linux. It provides dynamic priorities based on the `nice` level of each process. Check below for setting this `nice` level for netdata. | +| `batch` | This policy is similar to `other` in that it schedules the thread according to its dynamic priority (based on the `nice` value). The difference is that this policy will cause the scheduler to always assume that the thread is CPU-intensive. Consequently, the scheduler will apply a small scheduling penalty with respect to wake-up behavior, so that this thread is mildly disfavored in scheduling decisions. | +| `fifo` | `fifo` can be used only with static priorities higher than 0, which means that when a `fifo` threads becomes runnable, it will always immediately preempt any currently running `other`, `batch`, or `idle` thread. `fifo` is a simple scheduling algorithm without time slicing. | +| `rr` | a simple enhancement of `fifo`. Everything described above for `fifo` also applies to `rr`, except that each thread is allowed to run only for a maximum time quantum. | +| `keep`<br/>or<br/>`none` | do not set scheduling policy, priority or nice level - i.e. keep running with whatever it is set already (e.g. by systemd). | For more information see `man sched`. @@ -316,29 +320,31 @@ For more information see `man sched`. Once the policy is set to one of `rr` or `fifo`, the following will appear: -``` +```conf [global] process scheduling priority = 0 ``` -These priorities are usually from 0 to 99. Higher numbers make the process more important. +These priorities are usually from 0 to 99. Higher numbers make the process more +important. ### nice level for policies `other` or `batch` When the policy is set to `other`, `nice`, or `batch`, the following will appear: -``` +```conf [global] process nice level = 19 ``` ## scheduling settings and systemd -Netdata will not be able to set its scheduling policy and priority to more important values when it is started as the `netdata` user (systemd case). +Netdata will not be able to set its scheduling policy and priority to more important values when it is started as the +`netdata` user (systemd case). You can set these settings at `/etc/systemd/system/netdata.service`: -``` +```sh [Service] # By default Netdata switches to scheduling policy idle, which makes it use CPU, only # when there is spare available. @@ -357,20 +363,23 @@ You can set these settings at `/etc/systemd/system/netdata.service`: Run `systemctl daemon-reload` to reload these changes. -Now, tell Netdata to keep these settings, as set by systemd, by editing `netdata.conf` and setting: +Now, tell Netdata to keep these settings, as set by systemd, by editing +`netdata.conf` and setting: -``` +```conf [global] process scheduling policy = keep ``` -Using the above, whatever scheduling settings you have set at `netdata.service` will be maintained by netdata. +Using the above, whatever scheduling settings you have set at `netdata.service` +will be maintained by netdata. -#### Example 1: Netdata with nice -1 on non-systemd systems +### Example 1: Netdata with nice -1 on non-systemd systems -On a system that is not based on systemd, to make Netdata run with nice level -1 (a little bit higher to the default for all programs), edit `netdata.conf` and set: +On a system that is not based on systemd, to make Netdata run with nice level -1 (a little bit higher to the default for +all programs), edit `netdata.conf` and set: -``` +```conf [global] process scheduling policy = other process nice level = -1 @@ -384,16 +393,17 @@ sudo service netdata restart #### Example 2: Netdata with nice -1 on systemd systems -On a system that is based on systemd, to make Netdata run with nice level -1 (a little bit higher to the default for all programs), edit `netdata.conf` and set: +On a system that is based on systemd, to make Netdata run with nice level -1 (a little bit higher to the default for all +programs), edit `netdata.conf` and set: -``` +```conf [global] process scheduling policy = keep ``` edit /etc/systemd/system/netdata.service and set: -``` +```sh [Service] CPUSchedulingPolicy=other Nice=-1 @@ -408,45 +418,53 @@ sudo systemctl restart netdata ## Virtual memory -You may notice that netdata's virtual memory size, as reported by `ps` or `/proc/pid/status` (or even netdata's applications virtual memory chart) is unrealistically high. +You may notice that netdata's virtual memory size, as reported by `ps` or `/proc/pid/status` (or even netdata's +applications virtual memory chart) is unrealistically high. -For example, it may be reported to be 150+MB, even if the resident memory size is just 25MB. Similar values may be reported for Netdata plugins too. +For example, it may be reported to be 150+MB, even if the resident memory size is just 25MB. Similar values may be +reported for Netdata plugins too. -Check this for example: A Netdata installation with default settings on Ubuntu 16.04LTS. The top chart is **real memory used**, while the bottom one is **virtual memory**: +Check this for example: A Netdata installation with default settings on Ubuntu +16.04LTS. The top chart is **real memory used**, while the bottom one is +**virtual memory**:  -**Why does this happen?** +### Why does this happen? -The system memory allocator allocates virtual memory arenas, per thread running. -On Linux systems this defaults to 16MB per thread on 64 bit machines. So, if you get the -difference between real and virtual memory and divide it by 16MB you will roughly get the -number of threads running. +The system memory allocator allocates virtual memory arenas, per thread running. On Linux systems this defaults to 16MB +per thread on 64 bit machines. So, if you get the difference between real and virtual memory and divide it by 16MB you +will roughly get the number of threads running. -The system does this for speed. Having a separate memory arena for each thread, allows the -threads to run in parallel in multi-core systems, without any locks between them. +The system does this for speed. Having a separate memory arena for each thread, allows the threads to run in parallel in +multi-core systems, without any locks between them. -This behaviour is system specific. For example, the chart above when running Netdata on Alpine Linux (that uses **musl** instead of **glibc**) is this: +This behaviour is system specific. For example, the chart above when running +Netdata on Alpine Linux (that uses **musl** instead of **glibc**) is this:  -**Can we do anything to lower it?** +### Can we do anything to lower it? -Since Netdata already uses minimal memory allocations while it runs (i.e. it adapts its memory on start, so that while repeatedly collects data it does not do memory allocations), it already instructs the system memory allocator to minimize the memory arenas for each thread. We have also added [2 configuration options](https://github.com/netdata/netdata/blob/5645b1ee35248d94e6931b64a8688f7f0d865ec6/src/main.c#L410-L418) -to allow you tweak these settings: `glibc malloc arena max for plugins` and `glibc malloc arena max for netdata`. +Since Netdata already uses minimal memory allocations while it runs (i.e. it adapts its memory on start, so that while +repeatedly collects data it does not do memory allocations), it already instructs the system memory allocator to +minimize the memory arenas for each thread. We have also added [2 configuration +options](https://github.com/netdata/netdata/blob/5645b1ee35248d94e6931b64a8688f7f0d865ec6/src/main.c#L410-L418) to allow +you tweak these settings: `glibc malloc arena max for plugins` and `glibc malloc arena max for netdata`. -However, even if we instructed the memory allocator to use just one arena, it seems it allocates an arena per thread. +However, even if we instructed the memory allocator to use just one arena, it +seems it allocates an arena per thread. -Netdata also supports `jemalloc` and `tcmalloc`, however both behave exactly the same to the glibc memory allocator in this aspect. +Netdata also supports `jemalloc` and `tcmalloc`, however both behave exactly the +same to the glibc memory allocator in this aspect. -**Is this a problem?** +### Is this a problem? No, it is not. -Linux reserves real memory (physical RAM) in pages (on x86 machines pages are 4KB each). -So even if the system memory allocator is allocating huge amounts of virtual memory, -only the 4KB pages that are actually used are reserving physical RAM. The **real memory** chart -on Netdata application section, shows the amount of physical memory these pages occupy(it +Linux reserves real memory (physical RAM) in pages (on x86 machines pages are 4KB each). So even if the system memory +allocator is allocating huge amounts of virtual memory, only the 4KB pages that are actually used are reserving physical +RAM. The **real memory** chart on Netdata application section, shows the amount of physical memory these pages occupy(it accounts the whole pages, even if parts of them are actually used). ## Debugging @@ -455,13 +473,19 @@ When you compile Netdata with debugging: 1. compiler optimizations for your CPU are disabled (Netdata will run somewhat slower) -2. a lot of code is added all over netdata, to log debug messages to `/var/log/netdata/debug.log`. However, nothing is printed by default. Netdata allows you to select which sections of Netdata you want to trace. Tracing is activated via the config option `debug flags`. It accepts a hex number, to enable or disable specific sections. You can find the options supported at [log.h](../libnetdata/log/log.h). They are the `D_*` defines. The value `0xffffffffffffffff` will enable all possible debug flags. +2. a lot of code is added all over netdata, to log debug messages to `/var/log/netdata/debug.log`. However, nothing is + printed by default. Netdata allows you to select which sections of Netdata you want to trace. Tracing is activated + via the config option `debug flags`. It accepts a hex number, to enable or disable specific sections. You can find + the options supported at [log.h](../libnetdata/log/log.h). They are the `D_*` defines. The value + `0xffffffffffffffff` will enable all possible debug flags. -Once Netdata is compiled with debugging and tracing is enabled for a few sections, the file `/var/log/netdata/debug.log` will contain the messages. +Once Netdata is compiled with debugging and tracing is enabled for a few sections, the file `/var/log/netdata/debug.log` +will contain the messages. -> Do not forget to disable tracing (`debug flags = 0`) when you are done tracing. The file `debug.log` can grow too fast. +> Do not forget to disable tracing (`debug flags = 0`) when you are done tracing. The file `debug.log` can grow too +> fast. -#### compiling Netdata with debugging +### compiling Netdata with debugging To compile Netdata with debugging, use this: @@ -473,13 +497,17 @@ cd /usr/src/netdata.git CFLAGS="-O1 -ggdb -DNETDATA_INTERNAL_CHECKS=1" ./netdata-installer.sh ``` -The above will compile and install Netdata with debugging info embedded. You can now use `debug flags` to set the section(s) you need to trace. +The above will compile and install Netdata with debugging info embedded. You can now use `debug flags` to set the +section(s) you need to trace. -#### debugging crashes +### debugging crashes -We have made the most to make Netdata crash free. If however, Netdata crashes on your system, it would be very helpful to provide stack traces of the crash. Without them, is will be almost impossible to find the issue (the code base is quite large to find such an issue by just objerving it). +We have made the most to make Netdata crash free. If however, Netdata crashes on your system, it would be very helpful +to provide stack traces of the crash. Without them, is will be almost impossible to find the issue (the code base is +quite large to find such an issue by just objerving it). -To provide stack traces, **you need to have Netdata compiled with debugging**. There is no need to enable any tracing (`debug flags`). +To provide stack traces, **you need to have Netdata compiled with debugging**. There is no need to enable any tracing +(`debug flags`). Then you need to be in one of the following 2 cases: @@ -487,9 +515,10 @@ Then you need to be in one of the following 2 cases: 2. you can reproduce the crash -If you are not on these cases, you need to find a way to be (i.e. if your system does not produce core dumps, check your distro documentation to enable them). +If you are not on these cases, you need to find a way to be (i.e. if your system does not produce core dumps, check your +distro documentation to enable them). -#### Netdata crashes and you have a core dump +### Netdata crashes and you have a core dump > you need to have Netdata compiled with debugging info for this to work (check above) @@ -499,7 +528,7 @@ Run the following command and post the output on a github issue. gdb $(which netdata) /path/to/core/dump ``` -#### you can reproduce a Netdata crash on your system +### you can reproduce a Netdata crash on your system > you need to have Netdata compiled with debugging info for this to work (check above) @@ -509,6 +538,7 @@ Install the package `valgrind` and run: valgrind $(which netdata) -D ``` -Netdata will start and it will be a lot slower. Now reproduce the crash and `valgrind` will dump on your console the stack trace. Open a new github issue and post the output. +Netdata will start and it will be a lot slower. Now reproduce the crash and `valgrind` will dump on your console the +stack trace. Open a new github issue and post the output. [](<>) diff --git a/daemon/main.c b/daemon/main.c index bd0970fdd..4189ac7bd 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -306,7 +306,13 @@ int help(int exitcode) { " -W stacksize=N Set the stacksize (in bytes).\n\n" " -W debug_flags=N Set runtime tracing to debug.log.\n\n" " -W unittest Run internal unittests and exit.\n\n" +#ifdef ENABLE_DBENGINE " -W createdataset=N Create a DB engine dataset of N seconds and exit.\n\n" + " -W stresstest=A,B,C,D,E Run a DB engine stress test for A seconds,\n" + " with B writers and C readers, with a ramp up\n" + " time of D seconds for writers, a page cache\n" + " size of E MiB, and exit.\n\n" +#endif " -W set section option value\n" " set netdata.conf option from the command line.\n\n" " -W simple-pattern pattern string\n" @@ -719,7 +725,7 @@ int get_system_info(struct rrdhost_system_info *system_info) { } char n[51], v[101]; snprintfz(n, 50,"%s",name); - snprintfz(v, 101,"%s",value); + snprintfz(v, 100,"%s",value); if(unlikely(rrdhost_set_system_info_variable(system_info, n, v))) { info("Unexpected environment variable %s=%s", n, v); } @@ -887,6 +893,7 @@ int main(int argc, char **argv) { char* stacksize_string = "stacksize="; char* debug_flags_string = "debug_flags="; char* createdataset_string = "createdataset="; + char* stresstest_string = "stresstest="; if(strcmp(optarg, "unittest") == 0) { if(unit_test_buffer()) return 1; @@ -905,14 +912,33 @@ int main(int argc, char **argv) { fprintf(stderr, "\n\nALL TESTS PASSED\n\n"); return 0; } +#ifdef ENABLE_DBENGINE else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) { optarg += strlen(createdataset_string); -#ifdef ENABLE_DBENGINE - unsigned history_seconds = (unsigned )strtoull(optarg, NULL, 0); + unsigned history_seconds = strtoul(optarg, NULL, 0); generate_dbengine_dataset(history_seconds); -#endif return 0; } + else if(strncmp(optarg, stresstest_string, strlen(stresstest_string)) == 0) { + char *endptr; + unsigned test_duration_sec = 0, dset_charts = 0, query_threads = 0, ramp_up_seconds = 0, + page_cache_mb = 0; + + optarg += strlen(stresstest_string); + test_duration_sec = (unsigned)strtoul(optarg, &endptr, 0); + if (',' == *endptr) + dset_charts = (unsigned)strtoul(endptr + 1, &endptr, 0); + if (',' == *endptr) + query_threads = (unsigned)strtoul(endptr + 1, &endptr, 0); + if (',' == *endptr) + ramp_up_seconds = (unsigned)strtoul(endptr + 1, &endptr, 0); + if (',' == *endptr) + page_cache_mb = (unsigned)strtoul(endptr + 1, &endptr, 0); + dbengine_stress_test(test_duration_sec, dset_charts, query_threads, ramp_up_seconds, + page_cache_mb); + return 0; + } +#endif else if(strcmp(optarg, "simple-pattern") == 0) { if(optind + 2 > argc) { fprintf(stderr, "%s", "\nUSAGE: -W simple-pattern 'pattern' 'string'\n\n" diff --git a/daemon/unit_test.c b/daemon/unit_test.c index 36ccd9f6b..31718eeea 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -1688,7 +1688,8 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS st[i]->usec_since_last_update = USEC_PER_SEC * update_every; for (j = 0; j < DIMS; ++j) { - next = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c; + next = ((collected_number)i * DIMS) * REGION_POINTS[current_region] + + j * REGION_POINTS[current_region] + c; rrddim_set_by_pointer_fake_time(rd[i][j], next, time_now); } rrdset_done(st[i]); @@ -1719,13 +1720,14 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI for (j = 0; j < DIMS; ++j) { rd[i][j]->state->query_ops.init(rd[i][j], &handle, time_now, time_now + QUERY_BATCH * update_every); for (k = 0; k < QUERY_BATCH; ++k) { - last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c + k; + last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] + + j * REGION_POINTS[current_region] + c + k; expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); n = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved); value = unpack_storage_number(n); - same = (calculated_number_round(value * 10000000.0) == calculated_number_round(expected * 10000000.0)) ? 1 : 0; + same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; if(!same) { fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", @@ -1780,7 +1782,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c; expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); - same = (calculated_number_round(value * 10000000.0) == calculated_number_round(expected * 10000000.0)) ? 1 : 0; + same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; if(!same) { fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " CALCULATED_NUMBER_FORMAT ", RRDR found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", @@ -1902,7 +1904,7 @@ int test_dbengine(void) collected_number last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c - point_offset; calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); - uint8_t same = (calculated_number_round(value * 10000000.0) == calculated_number_round(expected * 10000000.0)) ? 1 : 0; + uint8_t same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; if(!same) { fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " CALCULATED_NUMBER_FORMAT ", RRDR found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", @@ -1932,20 +1934,27 @@ struct dbengine_chart_thread { uv_thread_t thread; RRDHOST *host; char *chartname; /* Will be prefixed by type, e.g. "example_local1.", "example_local2." etc */ - int dset_charts; /* number of charts */ - int dset_dims; /* dimensions per chart */ - int chart_i; /* current chart offset */ + unsigned dset_charts; /* number of charts */ + unsigned dset_dims; /* dimensions per chart */ + unsigned chart_i; /* current chart offset */ time_t time_present; /* current virtual time of the benchmark */ + volatile time_t time_max; /* latest timestamp of stored values */ unsigned history_seconds; /* how far back in the past to go */ + + volatile long done; /* initialize to 0, set to 1 to stop thread */ + struct completion charts_initialized; + unsigned long errors, stored_metrics_nr; /* statistics */ + + RRDSET *st; + RRDDIM *rd[]; /* dset_dims elements */ }; -collected_number generate_dbengine_chart_value(struct dbengine_chart_thread *thread_info, int dim_i, - time_t time_current) +collected_number generate_dbengine_chart_value(int chart_i, int dim_i, time_t time_current) { collected_number value; - value = ((collected_number)time_current) * thread_info->chart_i; - value += ((collected_number)time_current) * dim_i; + value = ((collected_number)time_current) * (chart_i + 1); + value += ((collected_number)time_current) * (dim_i + 1); value %= 1024LLU; return value; @@ -1956,44 +1965,47 @@ static void generate_dbengine_chart(void *arg) struct dbengine_chart_thread *thread_info = (struct dbengine_chart_thread *)arg; RRDHOST *host = thread_info->host; char *chartname = thread_info->chartname; - const int DSET_DIMS = thread_info->dset_dims; + const unsigned DSET_DIMS = thread_info->dset_dims; unsigned history_seconds = thread_info->history_seconds; time_t time_present = thread_info->time_present; - int j, update_every = 1; + unsigned j, update_every = 1; RRDSET *st; RRDDIM *rd[DSET_DIMS]; char name[RRD_ID_LENGTH_MAX + 1]; time_t time_current; // create the chart - snprintfz(name, RRD_ID_LENGTH_MAX, "example_local%d", thread_info->chart_i + 1); - st = rrdset_create(host, name, chartname, chartname, "example", NULL, chartname, chartname, chartname, NULL, 1, - update_every, RRDSET_TYPE_LINE); + snprintfz(name, RRD_ID_LENGTH_MAX, "example_local%u", thread_info->chart_i + 1); + thread_info->st = st = rrdset_create(host, name, chartname, chartname, "example", NULL, chartname, chartname, + chartname, NULL, 1, update_every, RRDSET_TYPE_LINE); for (j = 0 ; j < DSET_DIMS ; ++j) { - snprintfz(name, RRD_ID_LENGTH_MAX, "%s%d", chartname, j); + snprintfz(name, RRD_ID_LENGTH_MAX, "%s%u", chartname, j + 1); - rd[j] = rrddim_add(st, name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + thread_info->rd[j] = rd[j] = rrddim_add(st, name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } + complete(&thread_info->charts_initialized); // feed it with the test data time_current = time_present - history_seconds; for (j = 0 ; j < DSET_DIMS ; ++j) { rd[j]->last_collected_time.tv_sec = - st->last_collected_time.tv_sec = st->last_updated.tv_sec = time_current; + st->last_collected_time.tv_sec = st->last_updated.tv_sec = time_current - update_every; rd[j]->last_collected_time.tv_usec = st->last_collected_time.tv_usec = st->last_updated.tv_usec = 0; } - for( ; time_current < time_present ; ++time_current) { - st->usec_since_last_update = USEC_PER_SEC; + for( ; !thread_info->done && time_current < time_present ; time_current += update_every) { + st->usec_since_last_update = USEC_PER_SEC * update_every; for (j = 0; j < DSET_DIMS; ++j) { collected_number value; - value = generate_dbengine_chart_value(thread_info, j, time_current); + value = generate_dbengine_chart_value(thread_info->chart_i, j, time_current); rrddim_set_by_pointer_fake_time(rd[j], value, time_current); + ++thread_info->stored_metrics_nr; } rrdset_done(st); + thread_info->time_max = time_current; } } @@ -2003,7 +2015,7 @@ void generate_dbengine_dataset(unsigned history_seconds) const int DSET_DIMS = 128; const uint64_t EXPECTED_COMPRESSION_RATIO = 20; RRDHOST *host = NULL; - struct dbengine_chart_thread thread_info[DSET_CHARTS]; + struct dbengine_chart_thread **thread_info; int i; time_t time_present; @@ -2021,25 +2033,259 @@ void generate_dbengine_dataset(unsigned history_seconds) if (NULL == host) return; + thread_info = mallocz(sizeof(*thread_info) * DSET_CHARTS); + for (i = 0 ; i < DSET_CHARTS ; ++i) { + thread_info[i] = mallocz(sizeof(*thread_info[i]) + sizeof(RRDDIM *) * DSET_DIMS); + } fprintf(stderr, "\nRunning DB-engine workload generator\n"); time_present = now_realtime_sec(); for (i = 0 ; i < DSET_CHARTS ; ++i) { - thread_info[i].host = host; - thread_info[i].chartname = "random"; - thread_info[i].dset_charts = DSET_CHARTS; - thread_info[i].chart_i = i; - thread_info[i].dset_dims = DSET_DIMS; - thread_info[i].history_seconds = history_seconds; - thread_info[i].time_present = time_present; - assert(0 == uv_thread_create(&thread_info[i].thread, generate_dbengine_chart, &thread_info[i])); + thread_info[i]->host = host; + thread_info[i]->chartname = "random"; + thread_info[i]->dset_charts = DSET_CHARTS; + thread_info[i]->chart_i = i; + thread_info[i]->dset_dims = DSET_DIMS; + thread_info[i]->history_seconds = history_seconds; + thread_info[i]->time_present = time_present; + thread_info[i]->time_max = 0; + thread_info[i]->done = 0; + init_completion(&thread_info[i]->charts_initialized); + assert(0 == uv_thread_create(&thread_info[i]->thread, generate_dbengine_chart, thread_info[i])); + wait_for_completion(&thread_info[i]->charts_initialized); + destroy_completion(&thread_info[i]->charts_initialized); } for (i = 0 ; i < DSET_CHARTS ; ++i) { - assert(0 == uv_thread_join(&thread_info[i].thread)); + assert(0 == uv_thread_join(&thread_info[i]->thread)); } + for (i = 0 ; i < DSET_CHARTS ; ++i) { + freez(thread_info[i]); + } + freez(thread_info); rrd_wrlock(); rrdhost_free(host); rrd_unlock(); } + +struct dbengine_query_thread { + uv_thread_t thread; + RRDHOST *host; + char *chartname; /* Will be prefixed by type, e.g. "example_local1.", "example_local2." etc */ + unsigned dset_charts; /* number of charts */ + unsigned dset_dims; /* dimensions per chart */ + time_t time_present; /* current virtual time of the benchmark */ + unsigned history_seconds; /* how far back in the past to go */ + volatile long done; /* initialize to 0, set to 1 to stop thread */ + unsigned long errors, queries_nr, queried_metrics_nr; /* statistics */ + + struct dbengine_chart_thread *chart_threads[]; /* dset_charts elements */ +}; + +static void query_dbengine_chart(void *arg) +{ + struct dbengine_query_thread *thread_info = (struct dbengine_query_thread *)arg; + const int DSET_CHARTS = thread_info->dset_charts; + const int DSET_DIMS = thread_info->dset_dims; + time_t time_after, time_before, time_min, time_max, duration; + int i, j, update_every = 1; + RRDSET *st; + RRDDIM *rd; + uint8_t same; + time_t time_now, time_retrieved; + collected_number generatedv; + calculated_number value, expected; + storage_number n; + struct rrddim_query_handle handle; + + do { + // pick a chart and dimension + i = random() % DSET_CHARTS; + st = thread_info->chart_threads[i]->st; + j = random() % DSET_DIMS; + rd = thread_info->chart_threads[i]->rd[j]; + + time_min = thread_info->time_present - thread_info->history_seconds + 1; + time_max = thread_info->chart_threads[i]->time_max; + if (!time_max) { + time_before = time_after = time_min; + } else { + time_after = time_min + random() % (MAX(time_max - time_min, 1)); + duration = random() % 3600; + time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */ + } + + rd->state->query_ops.init(rd, &handle, time_after, time_before); + ++thread_info->queries_nr; + for (time_now = time_after ; time_now <= time_before ; time_now += update_every) { + generatedv = generate_dbengine_chart_value(i, j, time_now); + expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_EXISTS)); + + if (unlikely(rd->state->query_ops.is_finished(&handle))) { + fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " + CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n", + st->name, rd->name, (unsigned long) time_now, expected); + ++thread_info->errors; + break; + } + n = rd->state->query_ops.next_metric(&handle, &time_retrieved); + if (SN_EMPTY_SLOT == n) { + fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " + CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n", + st->name, rd->name, (unsigned long) time_now, expected); + ++thread_info->errors; + break; + } + ++thread_info->queried_metrics_nr; + value = unpack_storage_number(n); + + same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; + if (!same) { + fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " + CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", + st->name, rd->name, (unsigned long) time_now, expected, value); + ++thread_info->errors; + } + if (time_retrieved != time_now) { + fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n", + st->name, rd->name, (unsigned long) time_now, (unsigned long) time_retrieved); + ++thread_info->errors; + } + } + rd->state->query_ops.finalize(&handle); + } while(!thread_info->done); +} + +void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsigned QUERY_THREADS, + unsigned RAMP_UP_SECONDS, unsigned PAGE_CACHE_MB) +{ + const unsigned DSET_DIMS = 128; + const uint64_t EXPECTED_COMPRESSION_RATIO = 20; + const unsigned HISTORY_SECONDS = 3600 * 24 * 365; /* 1 year of history */ + RRDHOST *host = NULL; + struct dbengine_chart_thread **chart_threads; + struct dbengine_query_thread **query_threads; + unsigned i, j; + time_t time_start, time_end; + + if (!TEST_DURATION_SEC) + TEST_DURATION_SEC = 10; + if (!DSET_CHARTS) + DSET_CHARTS = 1; + if (!QUERY_THREADS) + QUERY_THREADS = 1; + if (PAGE_CACHE_MB < RRDENG_MIN_PAGE_CACHE_SIZE_MB) + PAGE_CACHE_MB = RRDENG_MIN_PAGE_CACHE_SIZE_MB; + + default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + default_rrdeng_page_cache_mb = PAGE_CACHE_MB; + // Worst case for uncompressible data + default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * HISTORY_SECONDS) / + (1024 * 1024); + default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100; + + error_log_limit_unlimited(); + debug(D_RRDHOST, "Initializing localhost with hostname 'dbengine-stress-test'"); + + host = dbengine_rrdhost_find_or_create("dbengine-stress-test"); + if (NULL == host) + return; + + chart_threads = mallocz(sizeof(*chart_threads) * DSET_CHARTS); + for (i = 0 ; i < DSET_CHARTS ; ++i) { + chart_threads[i] = mallocz(sizeof(*chart_threads[i]) + sizeof(RRDDIM *) * DSET_DIMS); + } + query_threads = mallocz(sizeof(*query_threads) * QUERY_THREADS); + for (i = 0 ; i < QUERY_THREADS ; ++i) { + query_threads[i] = mallocz(sizeof(*query_threads[i]) + sizeof(struct dbengine_chart_thread *) * DSET_CHARTS); + } + fprintf(stderr, "\nRunning DB-engine stress test, %u seconds writers ramp-up time,\n" + "%u seconds of concurrent readers and writers, %u writer threads, %u reader threads,\n" + "%u MiB of page cache.\n", + RAMP_UP_SECONDS, TEST_DURATION_SEC, DSET_CHARTS, QUERY_THREADS, PAGE_CACHE_MB); + + time_start = now_realtime_sec(); + for (i = 0 ; i < DSET_CHARTS ; ++i) { + chart_threads[i]->host = host; + chart_threads[i]->chartname = "random"; + chart_threads[i]->dset_charts = DSET_CHARTS; + chart_threads[i]->chart_i = i; + chart_threads[i]->dset_dims = DSET_DIMS; + chart_threads[i]->history_seconds = HISTORY_SECONDS; + chart_threads[i]->time_present = time_start; + chart_threads[i]->time_max = 0; + chart_threads[i]->done = 0; + chart_threads[i]->errors = chart_threads[i]->stored_metrics_nr = 0; + init_completion(&chart_threads[i]->charts_initialized); + assert(0 == uv_thread_create(&chart_threads[i]->thread, generate_dbengine_chart, chart_threads[i])); + } + /* barrier so that subsequent queries can access valid chart data */ + for (i = 0 ; i < DSET_CHARTS ; ++i) { + wait_for_completion(&chart_threads[i]->charts_initialized); + destroy_completion(&chart_threads[i]->charts_initialized); + } + sleep(RAMP_UP_SECONDS); + /* at this point data have already began being written to the database */ + for (i = 0 ; i < QUERY_THREADS ; ++i) { + query_threads[i]->host = host; + query_threads[i]->chartname = "random"; + query_threads[i]->dset_charts = DSET_CHARTS; + query_threads[i]->dset_dims = DSET_DIMS; + query_threads[i]->history_seconds = HISTORY_SECONDS; + query_threads[i]->time_present = time_start; + query_threads[i]->done = 0; + query_threads[i]->errors = query_threads[i]->queries_nr = query_threads[i]->queried_metrics_nr = 0; + for (j = 0 ; j < DSET_CHARTS ; ++j) { + query_threads[i]->chart_threads[j] = chart_threads[j]; + } + assert(0 == uv_thread_create(&query_threads[i]->thread, query_dbengine_chart, query_threads[i])); + } + sleep(TEST_DURATION_SEC); + /* stop workload */ + for (i = 0 ; i < DSET_CHARTS ; ++i) { + chart_threads[i]->done = 1; + } + for (i = 0 ; i < QUERY_THREADS ; ++i) { + query_threads[i]->done = 1; + } + for (i = 0 ; i < DSET_CHARTS ; ++i) { + assert(0 == uv_thread_join(&chart_threads[i]->thread)); + } + for (i = 0 ; i < QUERY_THREADS ; ++i) { + assert(0 == uv_thread_join(&query_threads[i]->thread)); + } + time_end = now_realtime_sec(); + fprintf(stderr, "\nDB-engine stress test finished in %ld seconds.\n", time_end - time_start); + unsigned long stored_metrics_nr = 0; + for (i = 0 ; i < DSET_CHARTS ; ++i) { + stored_metrics_nr += chart_threads[i]->stored_metrics_nr; + } + unsigned long queries_nr = 0, queried_metrics_nr = 0; + for (i = 0 ; i < QUERY_THREADS ; ++i) { + queries_nr += query_threads[i]->queries_nr; + queried_metrics_nr += query_threads[i]->queried_metrics_nr; + } + fprintf(stderr, "%u metrics were stored (dataset size of %lu MiB) in %u charts by 1 writer thread per chart.\n", + DSET_CHARTS * DSET_DIMS, stored_metrics_nr * sizeof(storage_number) / (1024 * 1024), DSET_CHARTS); + fprintf(stderr, "Metrics were being generated per 1 emulated second and time was accelerated.\n"); + fprintf(stderr, "%lu metric data points were queried by %u reader threads.\n", queried_metrics_nr, QUERY_THREADS); + fprintf(stderr, "Query starting time is randomly chosen from the beginning of the time-series up to the time of\n" + "the latest data point, and ending time from 1 second up to 1 hour after the starting time.\n"); + fprintf(stderr, "Performance is %lu written data points/sec and %lu read data points/sec.\n", + stored_metrics_nr / (time_end - time_start), queried_metrics_nr / (time_end - time_start)); + + for (i = 0 ; i < DSET_CHARTS ; ++i) { + freez(chart_threads[i]); + } + freez(chart_threads); + for (i = 0 ; i < QUERY_THREADS ; ++i) { + freez(query_threads[i]); + } + freez(query_threads); + rrdeng_exit(host->rrdeng_ctx); + rrd_wrlock(); + rrdhost_delete_charts(host); + rrd_unlock(); +} + #endif diff --git a/daemon/unit_test.h b/daemon/unit_test.h index fd3e80170..230a70085 100644 --- a/daemon/unit_test.h +++ b/daemon/unit_test.h @@ -11,6 +11,9 @@ extern int unit_test_buffer(void); #ifdef ENABLE_DBENGINE extern int test_dbengine(void); extern void generate_dbengine_dataset(unsigned history_seconds); +extern void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsigned QUERY_THREADS, + unsigned RAMP_UP_SECONDS, unsigned PAGE_CACHE_MB); + #endif #endif /* NETDATA_UNIT_TEST_H */ |