summaryrefslogtreecommitdiffstats
path: root/src/log_format_fwd.hh
blob: dbcc35684507c34373113b78d5eebb97cea11716 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
/**
 * Copyright (c) 2020, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file log_format_fwd.hh
 */

#ifndef lnav_log_format_fwd_hh
#define lnav_log_format_fwd_hh

#include <utility>

#include <sys/types.h>

#include "ArenaAlloc/arenaalloc.h"
#include "base/file_range.hh"
#include "base/map_util.hh"
#include "base/string_attr_type.hh"
#include "byte_array.hh"
#include "log_level.hh"
#include "pcrepp/pcre2pp.hh"
#include "robin_hood/robin_hood.h"
#include "yajlpp/yajlpp.hh"

class log_format;

struct log_level_stats {
    uint32_t lls_error_count{0};
    uint32_t lls_warning_count{0};
    uint32_t lls_total_count{0};

    log_level_stats& operator|=(const log_level_stats& rhs);
    void update_msg_count(log_level_t lvl);
};

struct log_op_description {
    nonstd::optional<intern_string_t> lod_id;
    lnav::map::small<size_t, std::string> lod_elements;

    log_op_description& operator|=(const log_op_description& rhs);
};

struct opid_sub_time_range {
    string_fragment ostr_subid;
    time_range ostr_range;
    bool ostr_open{true};
    log_level_stats ostr_level_stats;
    std::string ostr_description;

    bool operator<(const opid_sub_time_range& rhs) const
    {
        return this->ostr_range < rhs.ostr_range;
    }
};

struct opid_time_range {
    time_range otr_range;
    log_level_stats otr_level_stats;
    log_op_description otr_description;
    std::vector<opid_sub_time_range> otr_sub_ops;

    void close_sub_ops(const string_fragment& subid);

    opid_time_range& operator|=(const opid_time_range& rhs);
};

using log_opid_map = robin_hood::unordered_map<string_fragment,
                                               opid_time_range,
                                               frag_hasher,
                                               std::equal_to<string_fragment>>;

using sub_opid_map = robin_hood::unordered_map<string_fragment,
                                               string_fragment,
                                               frag_hasher,
                                               std::equal_to<string_fragment>>;

struct log_opid_state {
    log_opid_map los_opid_ranges;
    sub_opid_map los_sub_in_use;

    opid_sub_time_range* sub_op_in_use(ArenaAlloc::Alloc<char>& alloc,
                                       log_opid_map::iterator& op_iter,
                                       const string_fragment& subid,
                                       const timeval& tv,
                                       log_level_t level);
};

struct scan_batch_context {
    ArenaAlloc::Alloc<char>& sbc_allocator;
    log_opid_state sbc_opids;
    std::string sbc_cached_level_strings[4];
    log_level_t sbc_cached_level_values[4];
    size_t sbc_cached_level_count{0};
};

/**
 * Metadata for a single line in a log file.
 */
class logline {
public:
    static string_attr_type<void> L_PREFIX;
    static string_attr_type<void> L_TIMESTAMP;
    static string_attr_type<std::shared_ptr<logfile>> L_FILE;
    static string_attr_type<bookmark_metadata*> L_PARTITION;
    static string_attr_type<void> L_MODULE;
    static string_attr_type<void> L_OPID;
    static string_attr_type<bookmark_metadata*> L_META;

    /**
     * Construct a logline object with the given values.
     *
     * @param off The offset of the line in the file.
     * @param t The timestamp for the line.
     * @param millis The millisecond timestamp for the line.
     * @param l The logging level.
     */
    logline(file_off_t off,
            time_t t,
            uint16_t millis,
            log_level_t lev,
            uint8_t mod = 0,
            uint8_t opid = 0)
        : ll_offset(off), ll_has_ansi(false), ll_time(t), ll_millis(millis),
          ll_opid(opid), ll_sub_offset(0), ll_valid_utf(1), ll_level(lev),
          ll_module_id(mod), ll_meta_mark(0), ll_expr_mark(0)
    {
        memset(this->ll_schema, 0, sizeof(this->ll_schema));
    }

    logline(file_off_t off,
            const struct timeval& tv,
            log_level_t lev,
            uint8_t mod = 0,
            uint8_t opid = 0)
        : ll_offset(off), ll_has_ansi(false), ll_opid(opid), ll_sub_offset(0),
          ll_valid_utf(1), ll_level(lev), ll_module_id(mod), ll_meta_mark(0),
          ll_expr_mark(0)
    {
        this->set_time(tv);
        memset(this->ll_schema, 0, sizeof(this->ll_schema));
    }

    /** @return The offset of the line in the file. */
    file_off_t get_offset() const { return this->ll_offset; }

    uint16_t get_sub_offset() const { return this->ll_sub_offset; }

    void set_sub_offset(uint16_t suboff) { this->ll_sub_offset = suboff; }

    /** @return The timestamp for the line. */
    time_t get_time() const { return this->ll_time; }

    void to_exttm(struct exttm& tm_out) const
    {
        tm_out.et_tm = *gmtime(&this->ll_time);
        tm_out.et_nsec = this->ll_millis * 1000 * 1000;
    }

    void set_time(time_t t) { this->ll_time = t; }

    /** @return The millisecond timestamp for the line. */
    uint16_t get_millis() const { return this->ll_millis; }

    void set_millis(uint16_t m) { this->ll_millis = m; }

    uint64_t get_time_in_millis() const
    {
        return (this->ll_time * 1000ULL + (uint64_t) this->ll_millis);
    }

    struct timeval get_timeval() const
    {
        struct timeval retval = {this->ll_time, this->ll_millis * 1000};

        return retval;
    }

    void set_time(const struct timeval& tv)
    {
        this->ll_time = tv.tv_sec;
        this->ll_millis = tv.tv_usec / 1000;
    }

    void set_ignore(bool val)
    {
        if (val) {
            this->ll_level |= LEVEL_IGNORE;
        } else {
            this->ll_level &= ~LEVEL_IGNORE;
        }
    }

    bool is_ignored() const { return this->ll_level & LEVEL_IGNORE; }

    void set_mark(bool val)
    {
        if (val) {
            this->ll_level |= LEVEL_MARK;
        } else {
            this->ll_level &= ~LEVEL_MARK;
        }
    }

    bool is_marked() const { return this->ll_level & LEVEL_MARK; }

    void set_meta_mark(bool val) { this->ll_meta_mark = val; }

    bool is_meta_marked() const { return this->ll_meta_mark; }

    void set_expr_mark(bool val) { this->ll_expr_mark = val; }

    bool is_expr_marked() const { return this->ll_expr_mark; }

    void set_time_skew(bool val)
    {
        if (val) {
            this->ll_level |= LEVEL_TIME_SKEW;
        } else {
            this->ll_level &= ~LEVEL_TIME_SKEW;
        }
    }

    bool is_time_skewed() const { return this->ll_level & LEVEL_TIME_SKEW; }

    void set_valid_utf(bool v) { this->ll_valid_utf = v; }

    bool is_valid_utf() const { return this->ll_valid_utf; }

    void set_has_ansi(bool v) { this->ll_has_ansi = v; }

    bool has_ansi() const { return this->ll_has_ansi; }

    /** @param l The logging level. */
    void set_level(log_level_t l) { this->ll_level = l; };

    /** @return The logging level. */
    log_level_t get_level_and_flags() const
    {
        return (log_level_t) this->ll_level;
    }

    log_level_t get_msg_level() const
    {
        return (log_level_t) (this->ll_level & ~LEVEL__FLAGS);
    }

    const char* get_level_name() const
    {
        return level_names[this->ll_level & ~LEVEL__FLAGS];
    }

    bool is_message() const
    {
        return (this->ll_level & (LEVEL_IGNORE | LEVEL_CONTINUED)) == 0;
    }

    bool is_continued() const { return this->ll_level & LEVEL_CONTINUED; }

    uint8_t get_module_id() const { return this->ll_module_id; }

    void set_opid(uint8_t opid) { this->ll_opid = opid; }

    uint8_t get_opid() const { return this->ll_opid; }

    bool match_opid_hash(unsigned long hash) const
    {
        struct {
            unsigned int value : 6;
        } reduced = {
            (unsigned int) hash,
        };

        return this->ll_opid == reduced.value;
    }

    /**
     * @return  True if there is a schema value set for this log line.
     */
    bool has_schema() const
    {
        return (this->ll_schema[0] != 0 || this->ll_schema[1] != 0);
    }

    /**
     * Set the "schema" for this log line.  The schema ID is used to match log
     * lines that have a similar format when generating the logline table.  The
     * schema is set lazily so that startup is faster.
     *
     * @param ba The SHA-1 hash of the constant parts of this log line.
     */
    void set_schema(const byte_array<2, uint64_t>& ba)
    {
        memcpy(this->ll_schema, ba.in(), sizeof(this->ll_schema));
    }

    char get_schema() const { return this->ll_schema[0]; }

    /**
     * Perform a partial match of the given schema against this log line.
     * Storing the full schema is not practical, so we just keep the first four
     * bytes.
     *
     * @param  ba The SHA-1 hash of the constant parts of a log line.
     * @return    True if the first four bytes of the given schema match the
     *   schema stored in this log line.
     */
    bool match_schema(const byte_array<2, uint64_t>& ba) const
    {
        return memcmp(this->ll_schema, ba.in(), sizeof(this->ll_schema)) == 0;
    }

    /**
     * Compare loglines based on their timestamp.
     */
    bool operator<(const logline& rhs) const
    {
        return (this->ll_time < rhs.ll_time)
            || (this->ll_time == rhs.ll_time && this->ll_millis < rhs.ll_millis)
            || (this->ll_time == rhs.ll_time && this->ll_millis == rhs.ll_millis
                && this->ll_offset < rhs.ll_offset)
            || (this->ll_time == rhs.ll_time && this->ll_millis == rhs.ll_millis
                && this->ll_offset == rhs.ll_offset
                && this->ll_sub_offset < rhs.ll_sub_offset);
    }

    bool operator<(const time_t& rhs) const { return this->ll_time < rhs; }

    bool operator<(const struct timeval& rhs) const
    {
        return ((this->ll_time < rhs.tv_sec)
                || ((this->ll_time == rhs.tv_sec)
                    && (this->ll_millis < (rhs.tv_usec / 1000))));
    }

    bool operator<=(const struct timeval& rhs) const
    {
        return ((this->ll_time < rhs.tv_sec)
                || ((this->ll_time == rhs.tv_sec)
                    && (this->ll_millis <= (rhs.tv_usec / 1000))));
    }

private:
    file_off_t ll_offset : 63;
    uint8_t ll_has_ansi : 1;
    time_t ll_time;
    unsigned int ll_millis : 10;
    unsigned int ll_opid : 6;
    unsigned int ll_sub_offset : 15;
    unsigned int ll_valid_utf : 1;
    uint8_t ll_level;
    uint8_t ll_module_id : 6;
    uint8_t ll_meta_mark : 1;
    uint8_t ll_expr_mark : 1;
    char ll_schema[2];
};

struct format_tag_def {
    explicit format_tag_def(std::string name) : ftd_name(std::move(name)) {}

    struct path_restriction {
        std::string p_glob;

        bool matches(const char* fn) const;
    };

    std::string ftd_name;
    std::string ftd_description;
    std::vector<path_restriction> ftd_paths;
    factory_container<lnav::pcre2pp::code, int>::with_default_args<PCRE2_DOTALL>
        ftd_pattern;
    log_level_t ftd_level{LEVEL_UNKNOWN};
};

struct format_partition_def {
    explicit format_partition_def(std::string name) : fpd_name(std::move(name))
    {
    }

    struct path_restriction {
        std::string p_glob;

        bool matches(const char* fn) const;
    };

    std::string fpd_name;
    std::string fpd_description;
    std::vector<path_restriction> fpd_paths;
    factory_container<lnav::pcre2pp::code, int>::with_default_args<PCRE2_DOTALL>
        fpd_pattern;
    log_level_t fpd_level{LEVEL_UNKNOWN};
};

#endif