From 8fe89ed6f9550031298b60b196b5630733875b8c Mon Sep 17 00:00:00 2001 From: "Soren I. Bjornstad" Date: Tue, 9 Jul 2019 17:03:09 -0500 Subject: [PATCH 1/7] allow arbitrary date expressions, not just YYYYMMDD Pass into the 'date' utility to parse them. NOTE: date -d is a GNU extension. Before releasing this, I need to include fallback information, as well as documentation. --- dr | 4 ++-- tests/test_dr | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dr b/dr index f100ee6..ab1d56b 100755 --- a/dr +++ b/dr @@ -603,7 +603,7 @@ dreamfind() { [ -n "$3" ] || die "oops" operator=$2 dateExpr=$3 - [[ "$dateExpr" =~ [012][0-9][0-9][0-9]-[01][0-9]-[0123][0-9] ]] || die "Invalid date (use YYYY-MM-DD)." + myDate=$(date '+%Y-%m-%d' -d "$dateExpr" 2>/dev/null) || die "Invalid date expression '$dateExpr'. Try YYYY-MM-DD, or a phrase like 'today', 'last Monday', or 'June 7'." case $operator in 'gt'|'>') awkop='>' ;; 'lt'|'<') awkop='<' ;; @@ -614,7 +614,7 @@ dreamfind() { *) die "Invalid operator!" ;; esac - newargs=$(awk "/Date: / { if (\$2 $awkop \"$dateExpr\") { print FILENAME } }" $DREAMGLOB) + newargs=$(awk "/Date: / { if (\$2 $awkop \"$myDate\") { print FILENAME } }" $DREAMGLOB) shift 2 ;; diff --git a/tests/test_dr b/tests/test_dr index 4e5d4d7..148945d 100755 --- a/tests/test_dr +++ b/tests/test_dr @@ -243,6 +243,10 @@ fn_check() { fn_test "4, 5" date '=' 2011-05-06 fn_test "4, 5" date '==' 2011-05-06 + # date expressions + fn_test "4, 5" date eq "May 6, 2011" + fn_test NOMATCH date ge "last Thursday" + # boundary conditions fn_test NOMATCH date gt 2011-05-08 fn_test "8, 9" date ge 2011-05-08 From 86732c6c9799d3a8ff3ec09aad84ca47ec8810e9 Mon Sep 17 00:00:00 2001 From: "Soren I. Bjornstad" Date: Tue, 21 Apr 2020 18:45:17 -0500 Subject: [PATCH 2/7] add -t option to find for total/proportion calc --- dr | 16 ++++++++++++++-- tests/test_dr | 10 ++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/dr b/dr index ab1d56b..0407512 100755 --- a/dr +++ b/dr @@ -50,7 +50,7 @@ cat [-f] :: print all content of matching dreams on stdout; if -f, fold lines to a maximum length of 80 characters dump-headers :: print headers of matching dreams edit :: pass matching dreams as args to \$EDITOR -find :: show numbers of matching dreams +find [-t] :: show numbers of matching dreams; with -t add total dreams filename-display :: print filenames of matching dreams get-header
:: print values of
for matching dreams (no line is printed for matching dreams that have no such
) @@ -1032,11 +1032,23 @@ case "$action" in ;; "find"|"f") + if [ "$1" = "-t" ]; then + dreamfind last + totnum=$(defileify $args) + shift + fi + dreamfind "$@" # remove leading zeroes and '.dre' for display, add separator commas results="$(defileify $args)" numMatches=$(trim "$(wc -w <<<"$results")") - echo -e "$numMatches $(numerize $numMatches "match" "matches"): [$results]" + if [ -n "$totnum" ]; then + proportion=$((numMatches * 10000 / totnum)) + tot=" of $totnum total dreams ($(printf "%d.%.2d" $((proportion/100)) $((proportion%100)))%)" + else + tot="" + fi + echo -e "$numMatches $(numerize $numMatches "match" "matches")$tot: [$results]" ;; "filename-display"|"fd") diff --git a/tests/test_dr b/tests/test_dr index 148945d..fb4fbad 100755 --- a/tests/test_dr +++ b/tests/test_dr @@ -304,6 +304,16 @@ fn_check() { # again, no good way to test -s (randomize order) } +@test "dr find -t - test getting totals from find" { + run dr find -t 1 2 + [ "$status" == 0 ] + [ "$output" == "2 matches of 9 total dreams (22.22%): [1, 2]" ] + + run dr find -t -rl 1 8 7 + [ "$status" == 0 ] + [ "$output" == "1 match of 9 total dreams (11.11%): [7]" ] +} + @test "dr find format - test the format of output from find" { # This is implicitly tested already, but better to make it explicit. run dr find 1 From cb013aab7d337c7fdedbad68a6f6ffdee2885c1c Mon Sep 17 00:00:00 2001 From: "Soren I. Bjornstad" Date: Tue, 29 Nov 2022 20:23:32 -0600 Subject: [PATCH 3/7] fix help typo --- dr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dr b/dr index 0407512..ca3a928 100755 --- a/dr +++ b/dr @@ -213,7 +213,7 @@ $(basename "$0") header-replace [-f]
[] In dreams matching , replace instances of in header
with .
and are EREs; is -specifically an hregex (see 'help dr search' for more information). +specifically an hregex (see 'dr help search' for more information). will not match across commas, but it may match and change several tags separately. For example, for the header 'Tags: bar, baz', we could find 'ba' From 792c896137f09617618ea3640f5d48965e098838 Mon Sep 17 00:00:00 2001 From: "Soren I. Bjornstad" Date: Fri, 2 Dec 2022 18:28:08 -0600 Subject: [PATCH 4/7] fix -f option to 'dr cat' not working (#29) --- dr | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dr b/dr index ca3a928..75aedef 100755 --- a/dr +++ b/dr @@ -1093,6 +1093,12 @@ case "$action" in ;; "cat"|"c") + # fold if requested; must be first or -f will be gobbled by dreamfind's getopts + if [ "$1" = "-f" ]; then + foldCmd="| fold -s" + shift + fi + dreamfind "$@" # paste files together with double newlines From 6d1678978e128c69b7200d46c433b47f2ca36a2b Mon Sep 17 00:00:00 2001 From: "Soren I. Bjornstad" Date: Fri, 2 Dec 2022 22:19:27 -0600 Subject: [PATCH 5/7] implement tabulation --- .gitignore | 2 + dr | 204 +++++++++++++++++++++++++++++++++++++++++++++++--- tests/test_dr | 36 ++++++++- 3 files changed, 231 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index e8ce08c..6112015 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ scripts/bin/* drwc/drwc tests/tap.out tests/results.xml +.mypy_cache/* +.metadata.json \ No newline at end of file diff --git a/dr b/dr index 75aedef..972f070 100755 --- a/dr +++ b/dr @@ -2,7 +2,7 @@ # shellcheck disable=SC2086,SC1117,SC2119 # %%% dr - Dreamdir utility program -# Copyright (c) 2015-2019 Soren Bjornstad; see LICENSE for details. +# Copyright (c) 2015-2022 Soren Bjornstad; see LICENSE for details. ##### NOTES ON SHELLCHECK DIRECTIVES ##### # (For some stupid reason there can't be other comments between the shebang and @@ -59,6 +59,9 @@ header-values in matching dreams (include frequency if -f specified) list-headers [-f] :: list headers used in at least one matching dream (include frequency if -f specified) +tabulate [-rtw] + :: show a table of matching dreams with columns for each +
; type '$(basename "$0") help tabulate' for info word-count [-o ] :: call 'drwc' to show word count of the specified dreams, not including the headers; '-o' passes through @@ -266,6 +269,7 @@ Create a new dream file using the next unused ID number and open it in \$EDITOR. If the dream file is unchanged when you exit your editor, it will be deleted. + CONFIGURATION The template by default contains Id, Date, and Tags headers, with Id and Date autofilled. You can customize the template by creating a .dream_template file in the root of your dreamdir. This file will be copied to create a new dream. @@ -277,6 +281,63 @@ You can include the following variables in your template: USAGEMSG } +usagemsg_tabulate() { + cat < [] + +Print a table of matching dreams, with one row for each dream and one column for +each of the supplied. + + OPTIONS + -r :: Raw mode: separate columns with hard tab characters, rather than + automatically sizing columns and filling with spaces. (This is + useful if you want to pipe the output to another program or copy it + into a spreadsheet.) + -t :: Truncate excessively long cells so that the table is no wider than your + terminal. + -w :: Wrap excessively long cells onto multiple lines so that the table is no + wider than your terminal. + +Only one of these options makes sense at a time; if more than one is specified, +the one highest in the list above wins. + +The options -t and -w are only supported if the 'column' utility is installed on +your system and knows how to perform the relevant formatting tasks (this is +not true on macOS's version, for instance). + + $(if hash column 2>/dev/null && column --version >/dev/null 2>&1; then + echo -e "** This system $(tput setaf 2)SUPPORTS$(tput sgr0) the -t and -w options. **"; + else + echo "** This system $(tput setaf 1)DOES NOT SUPPORT$(tput sgr0) the -t and -w options. **"; + fi) + + CONFIGURATION +By default, the "Id" and "Date" columns will not be truncated or wrapped when +-t or -w is used. You can customize this behavior by creating a +.unwrappable_headers file in your dreamdir containing a list of header names +that are not allowed to wrap, one per line. + + EXAMPLES +dr tabulate Id,Date,Title l 100 + :: Print a table of the ID numbers, dates, and titles of the last 100 + dreams recorded. + +dr tabulate -w Id,Title,People,Places t Tags travel + :: Print a table of the ID numbers, titles, people and places of dreams + tagged with 'travel', wrapping lines as needed. + +dr tabulate -r Id,Date,Title,People,Places | + awk -F $'\t' 'patsplit(\$4, arr, /,/) == 1 { print \$0 }' | + column -ts $'\t' + :: Print a table of the ID numbers, dates, titles, people and places of + dreams which tag precisely two people (i.e., the People header contains + exactly one comma). Note the use of '-r', followed by manually + reformatting the table, so 'awk' can tell where the columns begin and + end. +USAGEMSG +} + ##### UTILITY FUNCTIONS ##### # die() # Print arguments to stderr and exit the shell with the last exit code @@ -459,7 +520,7 @@ ENDSCRIPT # $ echo "Filename list: $(getrange 4-8)" # 00004.dre 00005.dre 00006.dre 00007.dre 00008.dre getrange() { - [ -n "$1" ] || "Invalid arguments given to getrange()" + [ -n "$1" ] || die "Invalid arguments given to getrange()" local startat; local endat startat=${1%@*} endat=${1#*@} @@ -937,8 +998,12 @@ def get_headers(): for line in f: if not line.strip(): break - header, value = (i.strip() for i in line.split(':\t')) - dream[header] = value + try: + header, value = (i.strip() for i in line.split(':\t')) + except ValueError: + print(f"Invalid header in {dreamfile} (skipping): {line}") + else: + dream[header] = value dreams[dreamfile[:-4]] = dream return dreams @@ -1004,6 +1069,28 @@ tacw() { fi } +columnw() { + if hash column 2>/dev/null; then + column "$@" + else + # https://unix.stackexchange.com/questions/602522/posix-equivalent-to-column-t + awk -F $'\t' '{ + if (max_column < NF) max_column = NF; + for (i = 1; i <= NF; i++) { + if (width[i] < length($i)) width[i] = length($i); + data[NR, i] = $i; + } + } + END { + for (i = 1; i < max_column; i++) format[i] = sprintf("%%-%ds ", width[i]); + format[max_column] = "%s\n"; + for (k = 1; k <= NR; k++) { + for (i = 1; i <= max_column; i++) printf format[i], data[k, i]; + } + }' + fi +} + # sed -i requires an empty argument on MacOS sed, # but cannot take one on GNU sed! This lets us choose the right one. # https://stackoverflow.com/a/38595160 @@ -1108,12 +1195,6 @@ case "$action" in # add color if running in an interactive terminal [ -t 1 ] && highlightCmd='| colorify' - # fold if requested - if [ "$1" = "-f" ]; then - foldCmd="| fold -s" - shift - fi - # evaluate pipeline with appropriate parts eval "$mainCmd $highlightCmd $foldCmd" ;; @@ -1182,6 +1263,107 @@ case "$action" in "regenerate-tags"|"rt") regenerate_tags ;; +"tabulate"|"t") + # Read options. + tabulate_raw=0 + tabulate_truncate=0 + tabulate_wrap=0 + while getopts ":rtw" opt; do + case $opt in + r) tabulate_raw=1 ;; + t) tabulate_truncate=1 ;; + w) tabulate_wrap=1 ;; + *) + die "Invalid option -$OPTARG (-rtw are valid; see 'dr help tabulate')" + ;; + esac + done + shift $((OPTIND-1)) + OPTIND=1 + + # Sanity check. + if [ $tabulate_truncate -eq 1 ] || [ $tabulate_wrap -eq 1 ]; then + if ! hash column 2>/dev/null; then + die "The truncate (-t) and wrap (-w) options to 'dr tabulate' are not supported on this system because the 'column' command is not installed." + elif ! column --version >/dev/null 2>&1; then + # macOS has a really cruddy version of 'column' + die "The truncate (-t) and wrap (-w) options to 'dr tabulate' are not supported because this system's version of 'column' lacks the required options." + fi + fi + [ -n "$1" ] || die "Usage: $(basename "$0") tabulate " + + # Build arrays of selected headers and matching dreams. + declare -A dreams + ids=('Id') + headers=() + + IFS=',' read -ra headers <<<"$1" + shift + for h in "${headers[@]}"; do + # Note the use of a delimiter _ in keys, because bash doesn't support + # multidimensional arrays (associative or otherwise). + dreams[Id_$h]="${h^^}" + done + + dreamfind "$@" + for d in $args; do + while read -r line; do + case "$line" in + "Id: "*) + curId="${line#Id: }" + ids+=("$curId") + dreams[${curId}_Id]=$curId + ;; + *": "*) + dreams[${curId}_${line%%:*}]="${line#*: }" + ;; + *) + # end of headers + break ;; + esac + done <"$d" + done + + # If we are potentially truncating or wrapping columns, decide which columns. + if [ $tabulate_truncate -eq 1 ] || [ $tabulate_wrap -eq 1 ]; then + all_headers=$(tr ' ' ',' <<<"${headers[@]}") + + # Read headers which we should not be allowed to wrap or truncate from + # the .unwrappable_headers file, or use defaults if not present. + if [ -f ".unwrappable_headers" ]; then + unwraps=$(<.unwrappable_headers) + else + unwraps=$'Id\nDate' + fi + for h in "${headers[@]}"; do + if ! grep -Fxq "$h" <<<"$unwraps"; then + wraps+=("$h") + fi + done + wrappable_headers=$(tr ' ' ',' <<<"${wraps[@]}") + fi + + # Select output mode. + if [ $tabulate_raw -eq 1 ]; then + column_command="cat" + elif [ $tabulate_truncate -eq 1 ]; then + column_command="columnw -ts $'\t' -N $all_headers -dT $wrappable_headers" + elif [ $tabulate_wrap -eq 1 ]; then + column_command="columnw -ts $'\t' -N $all_headers -dW $wrappable_headers" + else + column_command="columnw -ts $'\t'" + fi + + # Print rows. + for d in "${ids[@]}"; do + for h in "${headers[@]}"; do + echo -en "${dreams[${d}_$h]}\t" + done + echo "" + done | eval "$column_command" + + ;; + "act"|"a") [ -n "$1" ] || die "Usage: [chain of piped dr commands] | dr act " input=$(cat <&0) @@ -1211,6 +1393,8 @@ case "$action" in usagemsg_header_replace ;; "new"|"n") usagemsg_new ;; + "tabulate"|"t") + usagemsg_tabulate ;; *) usagemsg ;; esac diff --git a/tests/test_dr b/tests/test_dr index fb4fbad..767c8a5 100755 --- a/tests/test_dr +++ b/tests/test_dr @@ -224,7 +224,7 @@ fn_check() { fn_test "7, 6, 7" tagged Tags 'rel.*ship' tagged People OC } -@test "dr find date" { +@test "dr find date - test of date range-based searches" { # the basics fn_test "1, 2, 3" date lt 2011-05-06 fn_test "1, 2, 3, 4, 5" date le 2011-05-06 @@ -488,3 +488,37 @@ fn_check() { run dr wc -o -p 1 last [ "$output" == "$expected" ] } + +@test "dr tabulate - build a table with chosen columns for specified dreams" { + expected=$(cat <<-EXPECTED + ID DATE PEOPLE PLACES TAGS + 00001 2011-05-01 Albert Einstein Sunnyside Mythbusters + 00002 2011-05-01 keyboard, discounts + 00003 2011-05-01 device + 00004 2011-05-06 Q Sunnyside sex + EXPECTED + ) + run dr tabulate Id,Date,People,Places,Tags 1-4 + + # ignore trailing whitespace...wow that was annoying to figure out, it was identical in bcompare when I copied it out of the terminal! + [ "$(sed 's/^[ \t]*//;s/[ \t]*$//' <<<"$output")" = "$(sed 's/^[ \t]*//;s/[ \t]*$//' <<<"$expected")" ] +} + +@test "dr tabulate - build table from hard tabs" { + expected=$(cat <<-EXPECTED + ID DATE TAGS + 00001 2011-05-01 Mythbusters + EXPECTED + ) + + run dr tabulate -r Id,Date,Tags 1-2 +} + + +# expected=$(cat <<-EXPECTED +# Id Date People Places Tags +# 00001 2011-05-01 Albert Einstein School Mythbusters +# 00002 2011-05-01 keyboard, discounts +# 00003 2011-05-01 device +# 00004 2011-05-06 Q Sunnyside sex +# EXPECTED From d5c8726a3e4274225ff5e6e51d03ce659d5331d8 Mon Sep 17 00:00:00 2001 From: "Soren I. Bjornstad" Date: Sat, 20 Jan 2024 14:23:00 -0600 Subject: [PATCH 6/7] document limitation of requiring 'date -d' --- dr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dr b/dr index 972f070..b0b3a14 100755 --- a/dr +++ b/dr @@ -23,7 +23,7 @@ ##### CONSTANTS ##### # application version -declare -r MYVERSION="2.1.0" +declare -r MYVERSION="2.2.0" # matching pattern for dream files declare -r DREAMGLOB='[0-9][0-9][0-9][0-9][0-9].dre' @@ -120,6 +120,7 @@ results are concatenated. * d[ate] :: select dreams by date; is 'gt', 'ge', 'lt', 'le', 'eq', or 'ne' or the usual symbolic equivalents (>, >=, <, <=, =, !=), and is in YYYY-MM-DD format + [requires GNU-compatible 'date -d' option on your system] * g[rep] :: select all dreams matching ERE anywhere in the file (whether in headers, notes, or text) * t[agged] [-f]
:: select dreams with
matching @@ -664,7 +665,7 @@ dreamfind() { [ -n "$3" ] || die "oops" operator=$2 dateExpr=$3 - myDate=$(date '+%Y-%m-%d' -d "$dateExpr" 2>/dev/null) || die "Invalid date expression '$dateExpr'. Try YYYY-MM-DD, or a phrase like 'today', 'last Monday', or 'June 7'." + myDate=$(date '+%Y-%m-%d' -d "$dateExpr" 2>/dev/null) || die "Invalid date expression '$dateExpr', or no GNU 'date' on this system. If you have GNU 'date', try a YYYY-MM-DD format, or a phrase like 'today', 'last Monday', or 'June 7'." case $operator in 'gt'|'>') awkop='>' ;; 'lt'|'<') awkop='<' ;; From 77b7d69fd52eec2f847c02de36c33e2cae368c91 Mon Sep 17 00:00:00 2001 From: "Soren I. Bjornstad" Date: Sat, 20 Jan 2024 14:32:51 -0600 Subject: [PATCH 7/7] update changelog --- CHANGES.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 06d4079..1f51c62 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,22 @@ +Changes in 2.2.0 +---------------- + +New features: + +* `date` search expression can now find dreams based on arbitrary expressions + like “yesterday” or “three weeks ago” (requires GNU `date` or compatible + with the `-d` option). +* `tabulate` search expression allows easy generation of a table of matching + dreams and arbitrary headers you select. See `dr help tabulate` for details. +* New `-t` switch to `find` search expression allows calculating the proportion + of dreams matched. + +Bugs fixed: + +* Message no longer erroneously suggests running `help dr` instead of `dr help`. +* `-f` option to `dr cat` is now honored as documented. + + Changes in 2.1.0 ----------------