eshtrans/frontend: Don't trim trailing tokens from scanners

Before we'd get things like this:

    Input:  $(foo bar)
    Output: $(foo )

    Input:  $(($(foo bar) + 1))
    Output: $(($(foo ) + 1))

Now we get things like this:

    Input:  $(foo bar)
    Output: $(foo bar )

    Input:  $(($(foo bar) + 1))
    Output: $(($(foo bar ) + 1))

1 files changed, 12 insertions, 26 deletions

diff --git a/eshtrans/frontend/lexer.esh b/eshtrans/frontend/lexer.esh
index 31670c1..ec80ac3 100644
--- a/eshtrans/frontend/lexer.esh
+++ b/eshtrans/frontend/lexer.esh
@@ -218,7 +218,6 @@ next_here()
 	local line=
 	local word=
 	local res=
-	local wordexp=
 
 	# Dequeue the here-document.
 	here="${here_queue%%${RS}*}"
@@ -276,9 +275,8 @@ next_here()
 					res="${res#*${RS}}"
 					c="${res%%${RS}*}"
 					res="${res#*${RS}}"
-					wordexp="${res%%${RS}*}"
 					lineno=$((${lineno} + ${ln_off}))
-					line="${line}${wordexp}"
+					line="${line}${res}"
 					continue
 				else
 					line="${line}${c}"
@@ -359,7 +357,6 @@ next_word()
 	local prev_c="${1}"
 	shift 1
 	local res=
-	local word=
 
 	if ! res="$(scan_word false "${prev_c}")"; then
 		exit 1
@@ -368,15 +365,14 @@ next_word()
 	res="${res#*${RS}}"
 	c="${res%%${RS}*}"
 	res="${res#*${RS}}"
-	word="${res%%${RS}*}"
 
 	# We must advance lineno because scan_word() was run in a subshell.
 	lineno=$((${lineno} + ${ln_off}))
-	tok="T_WORD${US}${word}"
+	tok="T_WORD${US}${res}"
 
 	if ${here_awaiting_end}; then
-		here_queue="${here_queue}${US}${word}"
-		case "${word}" in
+		here_queue="${here_queue}${US}${res}"
+		case "${res}" in
 			*\\*|*'"'*|*"'"*)
 				here_queue="${here_queue}${US}true"
 				;;
@@ -403,7 +399,6 @@ scan_word()
 	local quoted=
 	local tmp_c=
 	local res=
-	local wordexp=
 
 	lines=0
 	word=''
@@ -460,11 +455,10 @@ scan_word()
 				res="${res#*${RS}}"
 				c="${res%%${RS}*}"
 				res="${res#*${RS}}"
-				wordexp="${res%%${RS}*}"
 				# We must advance lineno because scan_wordexp()
 				# was run in a subshell.
 				lineno=$((${lineno} + ${ln_off}))
-				word="${word}${wordexp}"
+				word="${word}${res}"
 				# scan_wordexp() leaves behind an unused
 				# character, so we should skip the lgetc() call
 				# below.
@@ -562,8 +556,6 @@ scan_word()
 scan_wordexp()
 {
 	local res=
-	local toks=
-	local param=
 
 	wordexp=''
 	ln_off=0
@@ -591,9 +583,8 @@ scan_wordexp()
 					res="${res#*${RS}}"
 					c="${res%%${RS}*}"
 					res="${res#*${RS}}"
-					toks="${res%%${RS}*}"
 					lineno=${ln_off}
-					wordexp="\$(${SOH}C${STX}${toks}"
+					wordexp="\$(${SOH}C${STX}${res}"
 					wordexp="${wordexp}${ETX})"
 					# ")" is recognized in run_sublexer().
 					;;
@@ -607,9 +598,8 @@ scan_wordexp()
 			res="${res#*${RS}}"
 			c="${res%%${RS}*}"
 			res="${res#*${RS}}"
-			param="${res%%${RS}*}"
 			lineno=$((${lineno} + ${ln_off}))
-			wordexp="\$${param}"
+			wordexp="\$${res}"
 			;;
 	esac
 
@@ -622,7 +612,6 @@ scan_wordexp_param_brace()
 	local mod=
 	local res=
 	local param=
-	local word=
 
 	mod=true
 
@@ -640,7 +629,7 @@ scan_wordexp_param_brace()
 					res="${res#*${RS}}"
 					c="${res%%${RS}*}"
 					res="${res#*${RS}}"
-					param="${res%%${RS}*}"
+					param="${res}"
 					lineno=$((${lineno} + ${ln_off}))
 					# Disable modifications.
 					mod=false
@@ -659,7 +648,7 @@ scan_wordexp_param_brace()
 			res="${res#*${RS}}"
 			c="${res%%${RS}*}"
 			res="${res#*${RS}}"
-			param="${res%%${RS}*}"
+			param="${res}"
 			lineno=$((${lineno} + ${ln_off}))
 			;;
 	esac
@@ -718,12 +707,11 @@ scan_wordexp_param_brace()
 		res="${res#*${RS}}"
 		c="${res%%${RS}*}"
 		res="${res#*${RS}}"
-		word="${res%%${RS}*}"
 		# We must advance lineno because scan_word() was run in a
 		# subshell.
 		lineno=$((${lineno} + ${ln_off}))
-		wordexp="${wordexp}${word}"
-		dbg "param mod word: '$word'"
+		wordexp="${wordexp}${res}"
+		dbg "param mod word: '$res'"
 	fi
 
 	# Check for right brace.
@@ -791,7 +779,6 @@ scan_wordexp_arith()
 	local arith=
 	local paren_lvl=
 	local res=
-	local sub_wordexp=
 
 	arith=''
 	paren_lvl=0
@@ -830,11 +817,10 @@ scan_wordexp_arith()
 				res="${res#*${RS}}"
 				c="${res%%${RS}*}"
 				res="${res#*${RS}}"
-				sub_wordexp="${res%%${RS}*}"
 				# We must advance lineno because scan_wordexp()
 				# was run in a subshell.
 				lineno=$((${lineno} + ${ln_off}))
-				arith="${arith}${sub_wordexp}"
+				arith="${arith}${res}"
 				continue
 				;;
 			*)