From 2cdacbdce4c9909f294dce5d350b9cf3130eddb0 Mon Sep 17 00:00:00 2001
From: Mahmoud Al-Qudsi <mqudsi@neosmart.net>
Date: Mon, 12 Mar 2018 08:08:03 -0500
Subject: [PATCH] Restore sanity to line continuations

Line continuations (i.e. escaped new lines) now make sense again. With
the smart pipe support (pipes continue on to next line) recently added,
this hack to have continuations ignore comments makes no sense.

This is valid code:

```fish
echo hello |
# comment here
tr -d 'l'
```

this isn't:
```fish
echo hello | \
# comment here
tr -d 'l'
```

Reverts @snnw's 318daaffb2d2551523a84acaaab57d79deb1cf22
Closes #2928. Closes #2929.
---
 src/tokenizer.cpp | 16 +++++++---------
 src/tokenizer.h   |  2 --
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 65e262204..1cd4adc0c 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -416,11 +416,12 @@ maybe_t<tok_t> tokenizer_t::tok_next() {
         return none();
     }
 
+    bool line_continued = false;
     // Consume non-newline whitespace. If we get an escaped newline, mark it and continue past it.
     for (;;) {
         if (this->buff[0] == L'\\' && this->buff[1] == L'\n') {
+            line_continued = true;
             this->buff += 2;
-            this->continue_line_after_comment = true;
         } else if (iswspace_not_nl(this->buff[0])) {
             this->buff++;
         } else {
@@ -428,15 +429,12 @@ maybe_t<tok_t> tokenizer_t::tok_next() {
         }
     }
 
-    while (*this->buff == L'#') {
+    if (*this->buff == L'#') {
         // We have a comment, walk over the comment.
         const wchar_t *comment_start = this->buff;
         while (this->buff[0] != L'\n' && this->buff[0] != L'\0') this->buff++;
         size_t comment_len = this->buff - comment_start;
 
-        // If we are going to continue after the comment, skip any trailing newline.
-        if (this->buff[0] == L'\n' && this->continue_line_after_comment) this->buff++;
-
         // Maybe return the comment.
         if (this->show_comments) {
             tok_t result;
@@ -445,11 +443,12 @@ maybe_t<tok_t> tokenizer_t::tok_next() {
             result.length = comment_len;
             return result;
         }
+        if (line_continued) {
+            return none();
+        }
         while (iswspace_not_nl(this->buff[0])) this->buff++;
     }
 
-    // We made it past the comments and ate any trailing newlines we wanted to ignore.
-    this->continue_line_after_comment = false;
     size_t start_pos = this->buff - this->start;
 
     tok_t result;
@@ -468,8 +467,7 @@ maybe_t<tok_t> tokenizer_t::tok_next() {
             // Hack: when we get a newline, swallow as many as we can. This compresses multiple
             // subsequent newlines into a single one.
             if (!this->show_blank_lines) {
-                while (*this->buff == L'\n' || *this->buff == 13 /* CR */ || *this->buff == ' ' ||
-                       *this->buff == '\t') {
+                while (is_whitespace(*this->buff)) {
                     this->buff++;
                 }
             }
diff --git a/src/tokenizer.h b/src/tokenizer.h
index 8ce6618a7..a3e6ef782 100644
--- a/src/tokenizer.h
+++ b/src/tokenizer.h
@@ -104,8 +104,6 @@ class tokenizer_t {
     bool show_comments{false};
     /// Whether all blank lines are returned.
     bool show_blank_lines{false};
-    /// Whether to continue the previous line after the comment.
-    bool continue_line_after_comment{false};
 
     tok_t call_error(tokenizer_error *error_type, const wchar_t *token_start,
                      const wchar_t *error_loc);