[partitions] revive partition functionality

NEWS.md

@@ -1,5 +1,16 @@
 ## lnav v0.12.1
 
+Features:
+* Log partitions can automatically be created by defining a log
+  message pattern in a log format.  Under a format definition,
+  add an entry into the "partitions" object in a format definition.
+  The "pattern" property specifies the regular expression to match
+  against a line in a file that matches the format.  If a match is
+  found, the partition name will be set to the value(s) captured
+  by the regex.  To restrict matches to certain files, you can add
+  a "paths" array whose object elements contain a "glob" property
+  that will be matched against file names.
+
 Interface changes:
 * Changed the breadcrumb bar styling to space things out
   more and make the divisions between items clearer.
@@ -14,6 +25,14 @@ Interface changes:
   one is active).
 * The focused line should be preserved more reliably in
   the LOG/TEXT views.
+* In the LOG view, the current partition name (as set
+  with the `:partition-name` command) is shown as the
+  first breadcrumb in the breadcrumb bar.  And, when
+  that breadcrumb is selected, you can select another
+  partition to jump to.
+* The `{` / `}` hotkeys, `:next-section`, and `:prev-section`
+  commands now work in the LOG view and take you to the
+  next/previous partition.
 
 ## lnav v0.12.0
 

docs/schemas/format-v1.schema.json

@@ -518,6 +518,74 @@
                     },
                     "additionalProperties": false
                 },
+                "partitions": {
+                    "description": "The partitions to automatically apply to log messages",
+                    "title": "/<format_name>/partitions",
+                    "type": "object",
+                    "patternProperties": {
+                        "([\\w:;\\._\\-]+)": {
+                            "description": "The type of partition to apply",
+                            "title": "/<format_name>/partitions/<partition_type>",
+                            "type": "object",
+                            "properties": {
+                                "paths": {
+                                    "description": "Restrict partitioning to the given paths",
+                                    "title": "/<format_name>/partitions/<partition_type>/paths",
+                                    "type": "array",
+                                    "items": {
+                                        "type": "object",
+                                        "properties": {
+                                            "glob": {
+                                                "title": "/<format_name>/partitions/<partition_type>/paths/glob",
+                                                "description": "The glob to match against file paths",
+                                                "type": "string",
+                                                "examples": [
+                                                    "*/system.log*"
+                                                ]
+                                            }
+                                        },
+                                        "additionalProperties": false
+                                    }
+                                },
+                                "pattern": {
+                                    "title": "/<format_name>/partitions/<partition_type>/pattern",
+                                    "description": "The regular expression to match against the body of the log message",
+                                    "type": "string",
+                                    "examples": [
+                                        "\\w+ is down"
+                                    ]
+                                },
+                                "description": {
+                                    "title": "/<format_name>/partitions/<partition_type>/description",
+                                    "description": "A description of this partition",
+                                    "type": "string"
+                                },
+                                "level": {
+                                    "title": "/<format_name>/partitions/<partition_type>/level",
+                                    "description": "Constrain hits to log messages with this level",
+                                    "type": "string",
+                                    "enum": [
+                                        "trace",
+                                        "debug5",
+                                        "debug4",
+                                        "debug3",
+                                        "debug2",
+                                        "debug",
+                                        "info",
+                                        "stats",
+                                        "notice",
+                                        "warning",
+                                        "error",
+                                        "critical",
+                                        "fatal"
+                                    ]
+                                }
+                            },
+                            "additionalProperties": false
+                        }
+                    },
+                    "additionalProperties": false
+                },
                 "action": {
                     "title": "/<format_name>/action",
                     "type": "object",

docs/source/formats.rst

@@ -367,6 +367,19 @@ object with the following fields:
 
     :glob: A glob pattern to check against the log files read by lnav.
 
+:partitions: This object contains a description of partitions that should
+  automatically be created in the log view.
+
+  :pattern: The regular expression evaluated over a line in the log file as
+    it is read in.  If there is a match, the log message the line is a part
+    of will be used as the start of the partition.  The name of the
+    partition will be taken from any captures in the regex.
+  :paths: This array contains objects that define restrictions on the file
+    paths in which partitions will be created.  The objects in this array
+    can contain:
+
+    :glob: A glob pattern to check against the log files read by lnav.
+
 .. _format_sample:
 
 :sample: A list of objects that contain sample log messages.  All formats

docs/source/usage.rst

@@ -265,10 +265,41 @@ columns will be included in the table.
 Taking Notes
 ------------
 
-A few of the columns in the log tables can be updated on a row-by-row basis to
-allow you to take notes.  The majority of the columns in a log table are
-read-only since they are backed by the log files themselves.  However, the
-following columns can be changed by an :code:`UPDATE` statement:
+As you are looking through logs, you might find that you want to leave some
+notes of your findings.  **lnav** can help here by saving information in
+the session without needing to modify the actual log files.  Thus, when
+you re-open the files in lnav, the notes will be restored.  The following
+types of information can be saved:
+
+:tags: Log messages can be tagged with the :ref:`:tag<tag>` command as a
+  simple way to leave a descriptive mark.  The tags attached to a
+  message will be shown underneath the message.  You can press
+  :kbd:`u` and :kbd:`Shift` + :kbd:`u` to jump to the next/previous
+  marked line.  A regular search will also match tags.
+
+:comments: Free-form text can be attached to a log message with the
+  :ref:`:comment<comment>` command.  The comment will be shown
+  underneath the message. If the text contains markdown syntax,
+  it will be rendered to the best of the terminal's ability.
+  You can press :kbd:`u` and :kbd:`Shift` + :kbd:`u` to jump to the
+  next/previous marked line.  A regular search will also match the
+  comment text.
+
+:partitions: The log view can be partitioned to provide some context
+  about where you are in a collection of logs.  For example, in logs
+  for a test run, partitions could be created with the name for each
+  test.  The current partition is shown in the breadcrumb bar and
+  prefixed by the "⊑" symbol.  You can select the partition breadcrumb
+  to jump to another partition.  Pressing :kbd:`{` and :kbd:`}` will
+  jump to the next/previous partition.
+
+Accessing notes through the SQLite interface
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The note taking functionality in lnav can also be accessed through the
+log tables exposed through SQLite.  The majority of the columns in a log
+table are read-only since they are backed by the log files themselves.
+However, the following columns can be changed by an :code:`UPDATE` statement:
 
 * **log_part** - The "partition" the log message belongs to.  This column can
   also be changed by the :ref:`:partition-name<partition_name>` command.

src/bookmarks.cc

@@ -64,10 +64,19 @@ bookmark_metadata::remove_tag(const std::string& tag)
 }
 
 bool
-bookmark_metadata::empty() const
+bookmark_metadata::empty(bookmark_metadata::categories props) const
 {
-    return this->bm_name.empty() && this->bm_comment.empty()
-        && this->bm_tags.empty() && this->bm_annotations.la_pairs.empty();
+    switch (props) {
+        case categories::any:
+            return this->bm_name.empty() && this->bm_comment.empty()
+                && this->bm_tags.empty()
+                && this->bm_annotations.la_pairs.empty();
+        case categories::partition:
+            return this->bm_name.empty();
+        case categories::notes:
+            return this->bm_comment.empty() && this->bm_tags.empty()
+                && this->bm_annotations.la_pairs.empty();
+    }
 }
 
 void

src/bookmarks.hh

@@ -47,6 +47,33 @@ struct logmsg_annotations {
 struct bookmark_metadata {
     static std::unordered_set<std::string> KNOWN_TAGS;
 
+    enum class categories : int {
+        any = 0,
+        partition = 0x01,
+        notes = 0x02,
+    };
+
+    bool has(categories props) const
+    {
+        if (props == categories::any) {
+            return true;
+        }
+
+        if (props == categories::partition && !this->bm_name.empty()) {
+            return true;
+        }
+
+        if (props == categories::notes
+            && (!this->bm_comment.empty()
+                || !this->bm_annotations.la_pairs.empty()
+                || !this->bm_tags.empty()))
+        {
+            return true;
+        }
+
+        return false;
+    }
+
     std::string bm_name;
     std::string bm_comment;
     logmsg_annotations bm_annotations;
@@ -56,7 +83,7 @@ struct bookmark_metadata {
 
     bool remove_tag(const std::string& tag);
 
-    bool empty() const;
+    bool empty(categories props) const;
 
     void clear();
 };

src/breadcrumb_curses.cc

@@ -67,8 +67,7 @@ breadcrumb_curses::do_update()
     attr_line_t crumbs_line;
     for (const auto& crumb : crumbs) {
         auto accum_width = crumbs_line.column_width();
-        auto elem_width = utf8_string_length(crumb.c_display_value.get_string())
-                              .template unwrap();
+        auto elem_width = crumb.c_display_value.column_width();
         auto is_selected = this->bc_selected_crumb
             && (crumb_index == this->bc_selected_crumb.value());
 

src/formats/nextflow_log.json

@@ -1,6 +1,7 @@
 {
     "$schema": "https://lnav.org/schemas/format-v1.schema.json",
     "nextflow_log": {
+        "title": "Nextflow log format",
         "description": "Format file for nextflow.io logs",
         "url": [
             "https://nextflow.io/docs/latest/cli.html#execution-logs"

src/formats/vmw_log.json

@@ -241,6 +241,17 @@
                 "pattern": "^Expected equality of these values:"
             }
         },
+        "partitions": {
+            "test-partition": {
+                "description": "Partition for gtest sections",
+                "paths": [
+                    {
+                        "glob": "*/test.log"
+                    }
+                ],
+                "pattern": "^\\[ RUN      \\] ([^\\n]+)"
+            }
+        },
         "sample": [
             {
                 "line": "2021-05-24T20:31:05.671Z - last log rotation time, 2021-05-24T09:30:02.683Z - time the service was last started, Section for VMware ESX, pid=1000080910, version=7.0.3, build=0, option=DEBUG"

src/help.md

@@ -219,7 +219,7 @@ that you can always use `q` to pop the top view off of the stack.
 | o/O                  | Move forward/backward to the log message with a matching 'operation ID' (opid) field.                                                                                                                                                                                                                                                                           |
 | u/U                  | Move forward/backward through any user bookmarks you have added using the 'm' key. This hotkey will also jump to the start of any log partitions that have been created with the 'partition-name' command.                                                                                                                                                      |
 | s/S                  | Move to the next/previous "slow down" in the log message rate. A slow down is detected by measuring how quickly the message rate has changed over the previous several messages. For example, if one message is logged every second for five seconds and then the last message arrives five seconds later, the last message will be highlighted as a slow down. |
-| {/}                  | Move to the previous/next location in history. Whenever you jump to a new location in the view, the location will be added to the history. The history is not updated when using only the arrow keys.                                                                                                                                                           |
+| {/}                  | Move to the previous/next section in the view.  In the LOG view, this moves through partitions.  In other views, it moves through sections of documents.                                                                                                                                                                                                        |
 
 ### Chronological Navigation
 

src/keymaps/default-keymap.json

@@ -11,8 +11,8 @@
         "keymap_def_pop_view": "Press ${ansi_bold}q${ansi_norm} to return to the previous view",
         "keymap_def_zoom": "Press ${ansi_bold}z${ansi_norm}/${ansi_bold}Z${ansi_norm} to zoom in/out",
         "keymap_def_clear": "Press ${ansi_bold}C${ansi_norm} to clear marked messages",
-        "keymap_def_prev_section": "Press ${ansi_bold}{${ansi_norm} to move to the previous section in history",
-        "keymap_def_next_section": "Press ${ansi_bold}}${ansi_norm} to move to the next section in history",
+        "keymap_def_prev_section": "Press ${ansi_bold}{${ansi_norm} to move to the previous section in the view",
+        "keymap_def_next_section": "Press ${ansi_bold}}${ansi_norm} to move to the next section in the view",
         "keymap_def_next_mark": "Press ${ansi_bold}c${ansi_norm} to copy marked lines to the clipboard; press ${ansi_bold}C${ansi_norm} to clear marked lines",
         "keymap_def_time_offset": "Press ${ansi_bold}s${ansi_norm}/${ansi_bold}S${ansi_norm} to move forward/backward through slow downs"
     },

src/lnav_commands.cc

@@ -3703,10 +3703,12 @@ com_clear_comment(exec_context& ec,
             bookmark_metadata& line_meta = *(line_meta_opt.value());
 
             line_meta.bm_comment.clear();
-            if (line_meta.empty()) {
-                lss.erase_bookmark_metadata(tc->get_selection());
+            if (line_meta.empty(bookmark_metadata::categories::notes)) {
                 tc->set_user_mark(
                     &textview_curses::BM_META, tc->get_selection(), false);
+                if (line_meta.empty(bookmark_metadata::categories::any)) {
+                    lss.erase_bookmark_metadata(tc->get_selection());
+                }
             }
 
             lss.set_line_meta_changed();
@@ -3786,7 +3788,7 @@ com_untag(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
 
         auto line_meta_opt = lss.find_bookmark_metadata(tc->get_selection());
         if (line_meta_opt) {
-            bookmark_metadata& line_meta = *(line_meta_opt.value());
+            auto& line_meta = *(line_meta_opt.value());
 
             for (size_t lpc = 1; lpc < args.size(); lpc++) {
                 std::string tag = args[lpc];
@@ -3796,7 +3798,7 @@ com_untag(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
                 }
                 line_meta.remove_tag(tag);
             }
-            if (line_meta.empty()) {
+            if (line_meta.empty(bookmark_metadata::categories::notes)) {
                 tc->set_user_mark(
                     &textview_curses::BM_META, tc->get_selection(), false);
             }
@@ -3868,12 +3870,14 @@ com_delete_tags(exec_context& ec,
                 line_meta->remove_tag(tag);
             }
 
-            if (line_meta->empty()) {
-                lss.erase_bookmark_metadata(*iter);
-                size_t off = distance(vbm.begin(), iter);
-
+            if (line_meta->empty(bookmark_metadata::categories::notes)) {
+                size_t off = std::distance(vbm.begin(), iter);
                 tc->set_user_mark(&textview_curses::BM_META, *iter, false);
-                iter = next(vbm.begin(), off);
+                if (line_meta->empty(bookmark_metadata::categories::any)) {
+                    lss.erase_bookmark_metadata(*iter);
+                }
+
+                iter = std::next(vbm.begin(), off);
             } else {
                 ++iter;
             }
@@ -3906,7 +3910,7 @@ com_partition_name(exec_context& ec,
             args[1] = trim(remaining_args(cmdline, args));
 
             tc.set_user_mark(
-                &textview_curses::BM_META, tc.get_selection(), true);
+                &textview_curses::BM_PARTITION, tc.get_selection(), true);
 
             auto& line_meta = lss.get_bookmark_metadata(tc.get_selection());
 
@@ -3932,7 +3936,7 @@ com_clear_partition(exec_context& ec,
     } else if (args.size() == 1) {
         textview_curses& tc = lnav_data.ld_views[LNV_LOG];
         logfile_sub_source& lss = lnav_data.ld_log_source;
-        auto& bv = tc.get_bookmarks()[&textview_curses::BM_META];
+        auto& bv = tc.get_bookmarks()[&textview_curses::BM_PARTITION];
         nonstd::optional<vis_line_t> part_start;
 
         if (binary_search(bv.begin(), bv.end(), tc.get_selection())) {
@@ -3948,10 +3952,12 @@ com_clear_partition(exec_context& ec,
             auto& line_meta = lss.get_bookmark_metadata(part_start.value());
 
             line_meta.bm_name.clear();
-            if (line_meta.empty()) {
-                lss.erase_bookmark_metadata(part_start.value());
+            if (line_meta.empty(bookmark_metadata::categories::partition)) {
                 tc.set_user_mark(
-                    &textview_curses::BM_META, part_start.value(), false);
+                    &textview_curses::BM_PARTITION, part_start.value(), false);
+                if (line_meta.empty(bookmark_metadata::categories::any)) {
+                    lss.erase_bookmark_metadata(part_start.value());
+                }
             }
 
             retval = "info: cleared partition name";

src/log_format.cc

@@ -4240,5 +4240,11 @@ format_tag_def::path_restriction::matches(const char* fn) const
     return fnmatch(this->p_glob.c_str(), fn, 0) == 0;
 }
 
+bool
+format_partition_def::path_restriction::matches(const char* fn) const
+{
+    return fnmatch(this->p_glob.c_str(), fn, 0) == 0;
+}
+
 /* XXX */
 #include "log_format_impls.cc"