update lexicon for latest docs

Closes #2699

Fixes issues with:
* 'string' function synopsis
* Redirection display issues
* Better file & path detection
* Rendering of % & @ chars in both html and man
* @ symbol in tutorial

Improves robustness by implementing an @EOL marker to prevent hold buffer dumping extra chars after the end of an expression.

Added new '{{' and '}}' meta-chars for when you want curly braces in a regexp that was previously tripping up the lexicon.

Improve man/html presentation consistency for
* string
* printf
* prompt_pwd
* type

Use cli-styling for 'practical' examples.

Add <bs> tag for presenting content with preceding backslash.

Signed-off-by: Mark Griffiths <mark@thebespokepixel.com>

Doxyfile

@@ -274,6 +274,8 @@ ALIASES += "span{2}=\2"
 ALIASES += "spcl{2}=\2"
 
 ALIASES += "bksl{1}=\\\1"
+ALIASES += "pcnt{1}=\%\1"
+ALIASES += "atat{1}=\@"
 
 # This tag can be used to specify a number of word-keyword mappings (TCL only).
 # A mapping has the form "name=value". For example adding "class=itcl::class"

Doxyfile.help

@@ -274,6 +274,8 @@ ALIASES += "span{2}=\2"
 ALIASES += "spcl{2}=\2"
 
 ALIASES += "bksl{1}=\\\1"
+ALIASES += "pcnt{1}=\%\1"
+ALIASES += "atat{1}=\@"
 
 # This tag can be used to specify a number of word-keyword mappings (TCL only).
 # A mapping has the form "name=value". For example adding "class=itcl::class"

Doxyfile.user

@@ -274,6 +274,8 @@ ALIASES += "span{2}=<span style=\"\1\">\2</span>"
 ALIASES += "spcl{2}=<span class=\"\1\">\2</span>"
 
 ALIASES += "bksl{1}=<span>\</span>\1"
+ALIASES += "pcnt{1}=<span>%</span>\1"
+ALIASES += "atat{1}=<span>@</span>"
 
 # This tag can be used to specify a number of word-keyword mappings (TCL only).
 # A mapping has the form "name=value". For example adding "class=itcl::class"

Makefile.in

@@ -103,7 +103,7 @@ FISH_OBJS := obj/function.o obj/builtin.o obj/complete.o obj/env.o obj/exec.o \
     obj/parse_productions.o obj/parse_execution.o obj/pager.o obj/utf8.o \
     obj/fish_version.o obj/wcstringutil.o
 
-FISH_INDENT_OBJS := obj/fish_indent.o obj/print_help.o $(FISH_OBJS) 
+FISH_INDENT_OBJS := obj/fish_indent.o obj/print_help.o $(FISH_OBJS)
 
 #
 # Additional files used by builtin.o
@@ -276,9 +276,9 @@ doc: $(HDR_FILES_SRC) Doxyfile.user $(HTML_SRC) $(HELP_SRC) doc.h \
 	(cat Doxyfile.user; echo INPUT_FILTER=./lexicon_filter; \
 	echo PROJECT_NUMBER=$(FISH_BUILD_VERSION) | $(SED) "s/-.*//") | \
 		doxygen - && touch user_doc; \
-	cd user_doc/html && rm -f bc_s.png bdwn.png closed.png ftv2*.png \
+	cd user_doc/html && rm -f arrow*.png bc_s.png bdwn.png closed.png \
-		nav*.png open.png sync_*.png tab*.* doxygen.* dynsections.js \
+		doc.png folder*.png ftv2*.png nav*.png open.png splitbar.png \
-		jquery.js pages.html
+		sync_*.png tab*.* doxygen.* dynsections.js jquery.js pages.html
 
 #
 # PDF version of the source code documentation.

doc_src/FORMATTING.md

@@ -160,6 +160,8 @@ The following can be used in \\fish blocks to render some fish scenarios. These
 - `<error>`: \<error\>This would be shown as an error.\</error\>
 - `<asis>`: \<asis\>This test will not be parsed for fish markup.\</asis\>
 - `<outp>`: \<outp\>This would be rendered as command/script output.\</outp\>
+- `<bs>`: Render the contents with a preceding backslash. Useful when presenting output.
+- `{{` and `}}`: Required when wanting curly braces in regular expression example.
 
 ### Prompts and cursors
 

doc_src/fish_right_prompt.txt

@@ -20,7 +20,7 @@ A simple right prompt:
 
 \fish
 function fish_right_prompt -d "Write out the right prompt"
-    date "+%m/%d/%y"
+    date '+%m/%d/%y'
 end
 \endfish
 

doc_src/index.hdr.in

@@ -528,22 +528,24 @@ The above code demonstrates how to use multiple '`$`' symbols to expand the valu
 Lists adjacent to other lists or strings are expanded as cartesian products:
 
 Examples:
-\fish
+\fish{cli-dark}
-echo {good,bad}" apples"
+>_ echo {good,bad}" apples"
-# Outputs 'good apples bad apples'
+<outp>good apples bad apples</outp>
 
-set -l a x y z
+>_ set -l a x y z
-set -l b 1 2 3
+>_ set -l b 1 2 3
-echo $a$b
-# Outputs 'x1 y1 z1 x2 y2 z2 x3 y3 z3'
-echo $a"-"$b
-# Outputs 'x-1 y-1 z-1 x-2 y-2 z-2 x-3 y-3 z-3'
 
-echo {x,y,z}$b
+>_ echo $a$b
-# Outputs 'x1 y1 z1 x2 y2 z2 x3 y3 z3'
+<outp>x1 y1 z1 x2 y2 z2 x3 y3 z3</outp>
 
-echo {$b}word
+>_ echo $a"-"$b
-# Outputs '1word 2word 3word'
+<outp>x-1 y-1 z-1 x-2 y-2 z-2 x-3 y-3 z-3</outp>
+
+>_ echo {x,y,z}$b
+<outp>x1 y1 z1 x2 y2 z2 x3 y3 z3</outp>
+
+>_ echo {$b}word
+<outp>1word 2word 3word</outp>
 \endfish
 
 Be careful when you try to use braces to separate variable names from text. The dangers noted in the last example above can be avoided by wrapping the variable in double quotes instead of braces (`echo "$b"word`).

doc_src/printf.txt

@@ -60,7 +60,7 @@ This file has been imported from the printf in GNU Coreutils version 6.9. If you
 \subsection printf-example Example
 
 \fish
-printf '\%s\\t\%s\n' flounder fish
+printf '%s\\t%s\\n' flounder fish
 \endfish
 Will print "flounder	fish" (separated with a tab character), followed by a newline character. This is useful for writing completions, as fish expects completion scripts to output the option followed by the description, separated with a tab character.
 

doc_src/prompt_pwd.txt

@@ -13,16 +13,19 @@ To change the number of characters per path component, set $fish_prompt_pwd_dir_
 
 \subsection prompt_pwd-example Examples
 
-\fish
+\fish{cli-dark}
 >_ cd ~/
 >_ echo $PWD
-/home/alfa
+<outp>/home/alfa</outp>
+
 >_ prompt_pwd
-~
+<outp>~</outp>
+
 >_ cd /tmp/banana/sausage/with/mustard
 >_ prompt_pwd
-/t/b/s/w/mustard
+<outp>/t/b/s/w/mustard</outp>
+
 >_ set -g fish_prompt_pwd_dir_length 3
 >_ prompt_pwd
-/tmp/ban/sau/wit/mustard
+<outp>/tmp/ban/sau/wit/mustard</outp>
 \endfish

doc_src/string.txt

@@ -3,18 +3,13 @@
 \subsection string-synopsis Synopsis
 \fish{synopsis}
 string length [(-q | --quiet)] [STRING...]
-string sub [(-s | --start) START] [(-l | --length) LENGTH]
+string sub [(-s | --start) START] [(-l | --length) LENGTH] [(-q | --quiet)] [STRING...]
-           [(-q | --quiet)] [STRING...]
+string split [(-m | --max) MAX] [(-r | --right)] [(-q | --quiet)] SEP [STRING...]
-string split [(-m | --max) MAX] [(-r | --right)] [(-q | --quiet)]
-             SEP [STRING...]
 string join [(-q | --quiet)] SEP [STRING...]
-string trim [(-l | --left)] [(-r | --right)] [(-c | --chars CHARS)]
+string trim [(-l | --left)] [(-r | --right)] [(-c | --chars CHARS)] [(-q | --quiet)] [STRING...]
-            [(-q | --quiet)] [STRING...]
 string escape [(-n | --no-quoted)] [STRING...]
-string match [(-a | --all)] [(-i | --ignore-case)] [(-r | --regex)]
+string match [(-a | --all)] [(-i | --ignore-case)] [(-r | --regex)] [(-n | --index)] [(-q | --quiet)] PATTERN [STRING...]
-             [(-n | --index)] [(-q | --quiet)] PATTERN [STRING...]
+string replace [(-a | --all)] [(-i | --ignore-case)] [(-r | --regex)] [(-q | --quiet)] PATTERN REPLACEMENT [STRING...]
-string replace [(-a | --all)] [(-i | --ignore-case)] [(-r | --regex)]
-               [(-q | --quiet)] PATTERN REPLACEMENT [STRING...]
 \endfish
 
 
@@ -51,150 +46,129 @@ The following subcommands are available:
 
 \subsection string-example Examples
 
-\fish
+\fish{cli-dark}
-string length 'hello, world'
+>_ string length 'hello, world'
-# Output:
+<outp>12</outp>
-# 12
 
-string length -q $str
+>_ set str foo
+>_ string length -q $str; echo $status
+1
 # Equivalent to test -n $str
 \endfish
 
-\fish
+\fish{cli-dark}
-string sub --length 2 abcde
+>_ string sub --length 2 abcde
-# Output:
+<outp>ab</outp>
-# ab
 
-string sub -s 2 -l 2 abcde
+>_ string sub -s 2 -l 2 abcde
-# Output:
+<outp>bc</outp>
-# bc
 
-string sub --start=-2 abcde
+>_ string sub --start=-2 abcde
-# Output:
+<outp>de</outp>
-# de
 \endfish
 
-\fish
+\fish{cli-dark}
-string split . example.com
+>_ string split . example.com
-# Output:
+<outp>example</outp>
-# example
+<outp>com</outp>
-# com
 
-string split -r -m1 / /usr/local/bin/fish
+>_ string split -r -m1 / /usr/local/bin/fish
-# Output:
+<outp>/usr/local/bin</outp>
-# /usr/local/bin
+<outp>fish</outp>
-# fish
 
-string split '' abc
+>_ string split '' abc
-# Output:
+<outp>a</outp>
-# a
+<outp>b</outp>
-# b
+<outp>c</outp>
-# c
 \endfish
 
-\fish
+\fish{cli-dark}
-seq 3 | string join ...
+>_ seq 3 | string join ...
-# Output:
+<outp>1...2...3</outp>
-# 1...2...3
 \endfish
 
-\fish
+\fish{cli-dark}
-string trim ' abc  '
+>_ string trim ' abc  '
-# Output:
+<outp>abc</outp>
-# abc
 
-string trim --right --chars=yz xyzzy zany
+>_ string trim --right --chars=yz xyzzy zany
-# Output:
+<outp>x</outp>
-# x
+<outp>zan</outp>
-# zan
 \endfish
 
-\fish
+\fish{cli-dark}
-echo \x07 | string escape
+>_ echo \\x07 | string escape
-# Output:
+<bs>c</bs>g
-# \\cg
 \endfish
 
-\fish
+\subsection string-example-match-glob Match Glob Examples
-# string match glob examples
 
-string match '?' a
+\fish{cli-dark}
-# Output:
+>_ string match '?' a
-# a
+<outp>a</outp>
 
-string match 'a*b' axxb
+>_ string match 'a*b' axxb
-# Output:
+<outp>axxb</outp>
-# axxb
 
-string match -i 'a??B' Axxb
+>_ string match -i 'a??B' Axxb
-# Output:
+<outp>Axxb</outp>
-# Axxb
 
-echo 'ok?' | string match '*\\?'
+>_ echo 'ok?' | string match '*\\?'
-# Output:
+>_ <outp>ok?</outp>
-# ok?
-
-# string match regex examples
-
-string match -r 'cat|dog|fish' 'nice dog'
-# Output:
-# dog
-
-string match -r '(\\d\\d?):(\\d\\d):(\\d\\d)' 2:34:56
-# Output:
-# 2:34:56
-# 2
-# 34
-# 56
-
-string match -r '^(\\w{2,4})\\g1$' papa mud murmur
-# Output:
-# papa
-# pa
-# murmur
-# mur
-
-string match -r -a -n at ratatat
-# Output:
-# 2 2
-# 4 2
-# 6 2
-
-string match -r -i '0x[0-9a-f]{1,8}' 'int magic = 0xBadC0de;'
-# Output:
-# 0xBadC0de
 \endfish
 
-\fish
+\subsection string-example-match-regex Match Regex Examples
 
-# string replace literal examples
+\fish{cli-dark}
+>_ string match -r 'cat|dog|fish' 'nice dog'
+<outp>dog</outp>
 
-string replace is was 'blue is my favorite'
+>_ string match -r '(\\d\\d?):(\\d\\d):(\\d\\d)' <asis>2:34:56</asis>
-# Output:
+<outp>2:34:56</outp>
-# blue was my favorite
+<outp>2</outp>
+<outp>34</outp>
+<outp>56</outp>
 
-string replace 3rd last 1st 2nd 3rd
+>_ string match -r '^(\\w{{2,4}})\\g1$' papa mud murmur
-# Output:
+<outp>papa</outp>
-# 1st
+<outp>pa</outp>
-# 2nd
+<outp>murmur</outp>
-# last
+<outp>mur</outp>
 
-string replace -a ' ' _ 'spaces to underscores'
+>_ string match -r -a -n at ratatat
-# Output:
+<outp>2 2</outp>
-# spaces_to_underscores
+<outp>4 2</outp>
+<outp>6 2</outp>
 
-# string replace regex examples
+>_ string match -r -i '0x[0-9a-f]{{1,8}}' 'int magic = 0xBadC0de;'
-
+<outp>0xBadC0de</outp>
-string replace -r -a '[^\\d.]+' ' ' '0 one two 3.14 four 5x'
+\endfish
-# Output:
+
-# 0 3.14 5
+\subsection string-example-replace-literal Replace Literal Examples
-
+
-string replace -r '(\\w+)\\s+(\\w+)' '$2 $1 $$' 'left right'
+\fish{cli-dark}
-# Output:
+>_ string replace is was 'blue is my favorite'
-# right left $
+<outp>blue was my favorite</outp>
-
+
-string replace -r '\\s*newline\\s*' '\n' 'put a newline here'
+>_ string replace 3rd last 1st 2nd 3rd
-# Output:
+<outp>1st</outp>
-# put a
+<outp>2nd</outp>
-# here
+<outp>last</outp>
+
+>_ string replace -a ' ' _ 'spaces to underscores'
+<outp>spaces_to_underscores</outp>
+\endfish
+
+\subsection string-example-replace-Regex Replace Regex Examples
+
+\fish{cli-dark}
+>_ string replace -r -a '[^\\d.]+' ' ' '0 one two 3.14 four 5x'
+<outp>0 3.14 5</outp>
+
+>_ string replace -r '(\\w+)\\s+(\\w+)' '$2 $1 $$' 'left right'
+<outp>right left $</outp>
+
+>_ string replace -r '\\s*newline\\s*' '\\n' 'put a newline here'
+<outp>put a</outp>
+<outp>here</outp>
 \endfish

doc_src/type.txt

@@ -26,7 +26,7 @@ The following options are available:
 
 \subsection type-example Example
 
-\fish
+\fish{cli-dark}
-type fg
+>_ type fg
-# Outputs the string 'fg is a shell builtin'.
+<outp>fg is a builtin</outp>
 \endfish

doc_src/user_doc.css

@@ -174,8 +174,8 @@ tt, code, pre, .fish {
 .comment, .suggest { color: #555; }
 .command, .function, .binary { color: #223aa4; }
 .argument, .path, .file  { color: #5961cf; }
-.string { color: #6c6d08; }
 .operator, .variable, .match, .history { color: #1c8885; }
+.string, .string .operator { color: #858904; }
 
 /* Synopsis variant */
 .synopsis {

lexicon_filter.in

@@ -46,7 +46,12 @@
     # Then if it's inline. Remove and process immediately...
     /^\\fish.*$/ {
         # Catch @ symbol
-        s/@/(at)/
+        s/@/@at/g
+        # Catch & symbol
+        s/&\([^a-z]\)/@amp\1/g
+        # Catch {{ & }} symbols
+        s/{{/@curlyL/g
+        s/}}/@curlyR/g
         s/^\\fish//
         s/\\endfish//
         b html
@@ -56,7 +61,12 @@
     # Inside \fish block. Process...
     /\\endfish/!{
         # Catch @ symbol
-        s/@/((d))/
+        s/@/@at/g
+        # Catch & symbol
+        s/&\([^a-z]\)/@amp\1/g
+        # Catch {{ & }} symbols
+        s/{{/@curlyL/g
+        s/}}/@curlyR/g
         # Preprocess HTML and HTML-like formatting
         /<[^>]*>/ {
             b html
@@ -103,6 +113,9 @@ s|</i>|}|
 s|<u>|@undr{|
 s|<u [^>]*>|@undr{|
 s|</u>|}|
+# Backslash (when escaping output)
+s|<bs>|@bksl{|
+s|</bs>|}|
 t html
 #.
 # Some handy non-standard extensions
@@ -224,11 +237,11 @@ s/ \\$/ @bksl{ }/
 #.
 # Normal Directory
 s|mkdir |mkdir :|
-s|\([~/:][/]*[.A-Za-z_0-9/-]*\)\\ |\1=|g
+s|\([~/:][/]*[.A-Za-z_0-9*/-]*\)\\ |\1=|g
-s| \([~/][/]*[.A-Za-z_0-9/=-]*\)| \\\
+s| \([~/][/]*[.A-Za-z_0-9*/=-]*\)| \\\
 <@path{\1}\
 |g
-s| \(:[/]*[.A-Za-z_0-9/=-]*\)| \\\
+s| \(:[/]*[.A-Za-z_0-9*/=-]*\)| \\\
 <@path{\1}\
 |g
 t protect
@@ -255,6 +268,7 @@ s|^\([a-zA-Z][{},a-zA-Z0-9%*._/?!-]*\)|@args{\1}|g
 # Pick up loose text after markup.
 s/\([})]\)\([a-zA-Z0-9+%*.,][,a-zA-Z0-9%*._/?!-]*\);/\1@args{\2};/g
 s/\([})]\)\([a-zA-Z0-9+%*.,][,a-zA-Z0-9%*._/?!-]*\)$/\1@args{\2}/g
+s/\([})]\)\([a-zA-Z0-9+%*.,][,a-zA-Z0-9%*._/?!-]*\)@EOL/\1@args{\2}/g
 #.
 # Uncomment the following 2 lines (ss) to log the pattern buffer.
 s/^.*$/Pattern : &/w lexicon.log
@@ -279,6 +293,8 @@ s,\([^\\ ]*\)\\\n\([^<]*\)<\(@[^}]*[}\\]\),\1\3\2,
 t join
 # Clean up stray new lines
 s/\n//g
+# Clean up past @EOL
+s/@EOL.*$//g
 #.
 # Uncomment the folowing two lines (ss) to log the buffer before 'cleaning'.
 s/^.*$/PreClean: &/w lexicon.log
@@ -475,16 +491,17 @@ x
 #.
 # Mark up sesitive character entities.
 #.
-# We comment out this target because it isn't referenced and if we don't we
-# get warnings about "unused label 'entities'".
-#:entities
 s/</\&lt;/g
 s/>/\&gt;/g
-s/((d))/@/g
+s/@amp/\&amp;/g
+s/@curlyL/\{/g
+s/@curlyR/\}/g
+s/@at/@atat{ }/g
 #.
 # Final post processing
 s/};\([^]]\)/}@redr{;}\1/g
 s/};$/}@redr{;}/
+s/@sglq{}/''/
 s/ \[\([@(]\)/ @args{[}\1/g
 s/ \[\([A-Z]*\) / @args{[\1} /g
 s/@args{\([a-zA-Z0-9_.]*\)}\]/@args{\1]}/g
@@ -496,7 +513,9 @@ s/ \]$/ @args{]}/g
 s/\]}\]$/]]}/
 s/\\\([()]\)/@optr{@bksl{\1}}/g
 s/\([()]\)/@optr{\1}/g
+s/\\\\\([cdgnstwx?]\)/@bksl{\1}/g
 s/\\n/@bksl{n}/
+s/%\([diouxXfgGeEsbmy]\)/@pcnt{\1}/g
 s/ \\$//
 #.
 # Uncomment the folowing two lines (ss) to log the final output, sent to Doxygen.
@@ -516,6 +535,8 @@ b
 #.
 # Move protected content to hold space and mark up other entities.
 :protect
+# Add an 'End of Line' marker
+s/$/@EOL/
 s/^.*$/Input   : &/w lexicon.log
 s/^Input   : //
 h
@@ -536,6 +557,7 @@ x
     s/[^\<]*//
     s/^ *\\\n//g
     s/\n *\\//g
+    s/\n@EOL//g
     s/[()] \\//g
     s/^[^\<][^@][^\\]*//
     s/\n[]|;) ][^\\]*\\//
@@ -559,11 +581,13 @@ s/^[a-z][a-z]* \n//
 # Swap the buffers back.
 x
 #.
-# A special case. Tidy up after commands.
+# A special case. Tidy up after performing command substitution.
 # Redirectors
 s/\([^{|] *\)|/\1@redr{|}/g
-s/&$/@redr{\&amp;}/
+s/\&@EOL$/@redr{@amp}@EOL/g
-s/\([^{&] *\)&[^a-z]/\1@redr{\&amp;}/g
+s/@amp@EOL$/@redr{@amp}@EOL/g
+s/\([<>]\)@amp\([0-9]\)/@redr{\1@amp\2}/g
+s/\([^{&] *\)&[^@a-z]/\1@redr{\&amp;}/g
 s/\([^{<>^] *\)\([0-9]* *[<>^][<>^]*[^@][a-zA-Z0-9./_-]*\)/\1@redr{\2}/g
 s/\\}/}\\/g
 #.
@@ -575,11 +599,9 @@ s/[[][0-9$a-zA-Z_;. -]*]/@args{&}/g
 s/\($[$]*\)\([A-Za-z_0-9][A-Za-z_0-9]*\)/@vars{@optr{\1}\2}/g
 #.
 # Files
-s/\([^@]\)\([A-Za-z0-9_-][A-Za-z0-9_-]*\.[a-z0-9*][a-z0-9*]*\)/\1@fsfo{\2}/g
+/@at/ ! {
-#.
+    s/\([A-Za-z0-9_*-][A-Za-z0-9_*-]*\.[a-z0-9*][a-z0-9*]*\)/@fsfo{\1}/g
-# We comment out this target because it isn't referenced and if we don't we
+}
-# get warnings about "unused label 'commands'".
-#:commands
 #.
 #### This section is built in the Makefile. Just some formatting examples. #####
 #.