Only discard command comment if prev token was comment (#3628)

This fixes issues where a file with multiple def commands with their own
doc comments, some of the comments would be discarded.

crates/nu-parser/src/lex/lexer.rs

@@ -364,11 +364,21 @@ pub fn parse_block(tokens: Vec<Token>) -> (LiteBlock, Option<ParseError>) {
                 // If we encounter two newline characters in a row, use a special eoleol event,
                 // which allows the parser to discard comments that shouldn't be treated as
                 // documentation for the following item.
-                if let Some(Token {
+                let last_was_comment = std::matches!(
-                    contents: TokenContents::Eol,
+                    parser.prev_token,
-                    ..
+                    Some(Token {
-                }) = tokens.peek()
+                        contents: TokenContents::Comment(..),
-                {
+                        ..
+                    })
+                );
+                let next_is_eol = std::matches!(
+                    tokens.peek(),
+                    Some(Token {
+                        contents: TokenContents::Eol,
+                        ..
+                    })
+                );
+                if last_was_comment && next_is_eol {
                     tokens.next();
                     parser.eoleol();
                 } else {

crates/nu-parser/src/lex/tests.rs

@@ -131,6 +131,35 @@ def e [] {echo hi}
         );
     }
 
+    #[test]
+    fn lex_multi_comments() {
+        let input = r#"
+#A comment
+def e [] {echo hi}
+
+#Another comment
+def e2 [] {echo hello}
+            "#;
+
+        let (result, err) = lex(input, 0);
+        assert!(err.is_none());
+
+        let span1 = span(2, 11);
+        assert_eq!(result[1].span, span1);
+        assert_eq!(
+            result[1].contents,
+            TokenContents::Comment(LiteComment::new("A comment".to_string().spanned(span1)))
+        );
+        let span2 = span(33, 48);
+        assert_eq!(result[9].span, span2);
+        assert_eq!(
+            result[9].contents,
+            TokenContents::Comment(LiteComment::new(
+                "Another comment".to_string().spanned(span2)
+            ))
+        );
+    }
+
     #[test]
     fn def_comment_with_single_quote() {
         let input = r#"def f [] {
@@ -311,11 +340,89 @@ def my_echo [arg] { echo $arg }
             ])
         );
     }
+
+    #[test]
+    fn two_commands_with_comments() {
+        let code = r#"
+# My echo
+# * It's much better :)
+def my_echo [arg] { echo $arg }
+
+# My echo2
+# * It's even better!
+def my_echo2 [arg] { echo $arg }
+        "#;
+        let (result, err) = lex(code, 0);
+        assert!(err.is_none());
+        let (result, err) = parse_block(result);
+        assert!(err.is_none());
+
+        assert_eq!(result.block.len(), 2);
+        assert_eq!(result.block[0].pipelines.len(), 1);
+        assert_eq!(result.block[0].pipelines[0].commands.len(), 1);
+        assert_eq!(result.block[0].pipelines[0].commands[0].parts.len(), 4);
+        assert_eq!(
+            result.block[0].pipelines[0].commands[0].comments,
+            Some(vec![
+                LiteComment::new_with_ws(
+                    " ".to_string().spanned(Span::new(2, 3)),
+                    "My echo".to_string().spanned(Span::new(3, 10))
+                ),
+                LiteComment::new_with_ws(
+                    " ".to_string().spanned(Span::new(12, 13)),
+                    "* It's much better :)"
+                        .to_string()
+                        .spanned(Span::new(13, 34))
+                )
+            ])
+        );
+
+        assert_eq!(result.block[1].pipelines.len(), 1);
+        assert_eq!(result.block[1].pipelines[0].commands.len(), 1);
+        assert_eq!(result.block[1].pipelines[0].commands[0].parts.len(), 4);
+        assert_eq!(
+            result.block[1].pipelines[0].commands[0].comments,
+            Some(vec![
+                LiteComment::new_with_ws(
+                    " ".to_string().spanned(Span::new(69, 70)),
+                    "My echo2".to_string().spanned(Span::new(70, 78))
+                ),
+                LiteComment::new_with_ws(
+                    " ".to_string().spanned(Span::new(80, 81)),
+                    "* It's even better!"
+                        .to_string()
+                        .spanned(Span::new(81, 100))
+                )
+            ])
+        );
+    }
+
     #[test]
     fn discarded_comment() {
         let code = r#"
 # This comment gets discarded, because of the following empty line
 
+echo 42
+        "#;
+        let (result, err) = lex(code, 0);
+        assert!(err.is_none());
+        // assert_eq!(format!("{:?}", result), "");
+        let (result, err) = parse_block(result);
+        assert!(err.is_none());
+        assert_eq!(result.block.len(), 1);
+        assert_eq!(result.block[0].pipelines.len(), 1);
+        assert_eq!(result.block[0].pipelines[0].commands.len(), 1);
+        assert_eq!(result.block[0].pipelines[0].commands[0].parts.len(), 2);
+        assert_eq!(result.block[0].pipelines[0].commands[0].comments, None);
+    }
+
+    #[test]
+    fn discarded_comment_multi_newline() {
+        let code = r#"
+# This comment gets discarded, because of the following empty line
+
+
+
 echo 42
         "#;
         let (result, err) = lex(code, 0);

crates/nu-parser/tests/main.rs

@@ -23,3 +23,44 @@ fn defs_contain_comment_in_help() {
         assert!(actual.out.contains("I comment and test. I am a good boy."));
     });
 }
+
+#[test]
+fn defs_contain_multiple_comments_in_help() {
+    Playground::setup("comment_test_2", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContent(
+            "my_def.nu",
+            r#"
+                # I comment and test. I am a good boy.
+                def comment_philosphy [] {
+                    echo It’s not a bug – it’s an undocumented feature. (Anonymous)
+                }
+
+                # I comment and test all my functions. I am a very good boy.
+                def comment_philosphy_2 [] {
+                    echo It’s not a bug – it’s an undocumented feature. (Anonymous)
+                }
+
+
+                # I comment and test all my functions. I am the best boy.
+                def comment_philosphy_3 [] {
+                    echo It’s not a bug – it’s an undocumented feature. (Anonymous)
+                }
+                "#,
+        )]);
+
+        let actual = nu!(cwd: dirs.test(), r#"
+            source my_def.nu
+            help comment_philosphy
+            help comment_philosphy_2
+            help comment_philosphy_3
+            "#);
+
+        assert!(actual.out.contains("I comment and test. I am a good boy."));
+        assert!(actual
+            .out
+            .contains("I comment and test all my functions. I am a very good boy."));
+        assert!(actual
+            .out
+            .contains("I comment and test all my functions. I am the best boy."));
+    });
+}