diff --git a/GRAMMAR.md b/GRAMMAR.md index 4fea4d8246..54ec6b8057 100644 --- a/GRAMMAR.md +++ b/GRAMMAR.md @@ -26,6 +26,9 @@ LINE_PREFIX = @-|-@|@|- TEXT = recipe text, only matches in a recipe body ``` +Inside delimiters, including `()`, `[], `{}`, and `{{}}`, `COMMENT` and +`NEWLINE` are treated as whitespace. + grammar syntax -------------- diff --git a/README.md b/README.md index 2376144eeb..4963af7218 100644 --- a/README.md +++ b/README.md @@ -4585,6 +4585,11 @@ $ just --dump > formatted-justfile The `--dump` command can be used with `--dump-format json` to print a JSON representation of a `justfile`. +Comments inside delimiters, such as `[]` and `()` are ignored.master +However, they are not preserved by the formatter. To prevent comments from being +silently dropped, the formatter will return an error if the justfile has +comments which it cannot preserve. + Cached Recipes -------------- diff --git a/src/ast.rs b/src/ast.rs index 7b01b11f97..e57f4c4b0a 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -5,6 +5,7 @@ use super::*; /// are performed by the `Analyzer`, which produces a `Justfile` from an `Ast`. #[derive(Debug, Clone)] pub(crate) struct Ast<'src> { + pub(crate) dropped_comment: Option>, pub(crate) items: Vec>, pub(crate) list_features: Vec<(ListFeature, Token<'src>)>, pub(crate) module_path: Modulepath, diff --git a/src/compiler.rs b/src/compiler.rs index ed702f7c3e..00c4b31876 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -263,8 +263,8 @@ impl Compiler { #[cfg(test)] pub(crate) fn test_compile(src: &str) -> CompileResult { - let tokens = Lexer::test_lex(src)?; - let ast = Parser::parse_tokens(&mut Numerator::new(), &tokens)?; + let lex = Lexer::test_lex(src)?; + let ast = Parser::parse_lex(&lex, &mut Numerator::new())?; let root = Utf8PathBuf::from("justfile"); let mut asts: HashMap<(Modulepath, Utf8PathBuf), Ast> = HashMap::new(); asts.insert((Modulepath::default(), root.clone()), ast); diff --git a/src/error.rs b/src/error.rs index 348dc8b683..43be8411bd 100644 --- a/src/error.rs +++ b/src/error.rs @@ -169,6 +169,9 @@ pub(crate) enum Error<'src> { switch: Switch, }, FormatCheckFoundDiff, + FormatDroppedComment { + token: Token<'src>, + }, FunctionCall { function: Name<'src>, message: String, @@ -395,7 +398,9 @@ impl<'src> Error<'src> { Some(module.token) } Self::Assert { name, .. } => Some(**name), - Self::Backtick { token, .. } | Self::RegexCompile { token, .. } => Some(*token), + Self::Backtick { token, .. } + | Self::FormatDroppedComment { token } + | Self::RegexCompile { token, .. } => Some(*token), Self::Compile { compile_error } => Some(compile_error.context()), Self::Const { const_error } => Some(const_error.context()), Self::FunctionCall { function, .. } => Some(function.token), @@ -810,6 +815,9 @@ impl ColorDisplay for Error<'_> { FormatCheckFoundDiff => { write!(f, "formatted justfile differs from original")?; } + FormatDroppedComment { .. } => { + write!(f, "formatting would drop comment")?; + } FunctionCall { message, .. } => { write!(f, "{message}")?; } diff --git a/src/lex.rs b/src/lex.rs new file mode 100644 index 0000000000..a47086635b --- /dev/null +++ b/src/lex.rs @@ -0,0 +1,6 @@ +use super::*; + +pub(crate) struct Lex<'src> { + pub(crate) dropped_comment: Option>, + pub(crate) tokens: Vec>, +} diff --git a/src/lexer.rs b/src/lexer.rs index 6f8b182ddd..32cc169cb4 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -14,6 +14,7 @@ static DEDENT_RE: LazyLock = pub(crate) struct Lexer<'src> { /// Char iterator chars: Chars<'src>, + dropped_comment: Option>, /// Indentation stack indentation: Vec<&'src str>, /// Interpolation token start stack @@ -46,12 +47,12 @@ impl<'src> Lexer<'src> { pub(crate) const INTERPOLATION_START: &'static str = "{{"; /// Lex `src` - pub(crate) fn lex(path: &'src Utf8Path, src: &'src str) -> CompileResult<'src, Vec>> { + pub(crate) fn lex(path: &'src Utf8Path, src: &'src str) -> CompileResult<'src, Lex<'src>> { Self::new(path, src).tokenize() } #[cfg(test)] - pub(crate) fn test_lex(src: &'src str) -> CompileResult<'src, Vec>> { + pub(crate) fn test_lex(src: &'src str) -> CompileResult<'src, Lex<'src>> { Self::new("justfile".as_ref(), src).tokenize() } @@ -68,6 +69,7 @@ impl<'src> Lexer<'src> { Self { chars, + dropped_comment: None, indentation: vec![""], interpolation_stack: Vec::new(), lines: src.lines().count(), @@ -205,6 +207,10 @@ impl<'src> Lexer<'src> { self.token_start = self.token_end; } + fn last_token(&self) -> Token<'src> { + *self.tokens.last().unwrap() + } + /// Create an internal error with `message` fn internal_error(&self, message: impl Into) -> CompileError<'src> { // Use `self.token_end` as the location of the error @@ -273,7 +279,7 @@ impl<'src> Lexer<'src> { } /// Consume the text and produce a series of tokens - fn tokenize(mut self) -> CompileResult<'src, Vec>> { + fn tokenize(mut self) -> CompileResult<'src, Lex<'src>> { loop { if self.token_start.column == 0 { self.lex_line_start()?; @@ -307,7 +313,10 @@ impl<'src> Lexer<'src> { assert_eq!(self.token_start.offset, self.src.len()); assert_eq!(self.indentation.len(), 1); - Ok(self.tokens) + Ok(Lex { + dropped_comment: self.dropped_comment, + tokens: self.tokens, + }) } /// Handle blank lines and indentation @@ -594,9 +603,7 @@ impl<'src> Lexer<'src> { NewlineCarriageReturn => self.lex_double(Eol), Interpolation => { self.lex_double(InterpolationStart)?; - self - .interpolation_stack - .push(self.tokens[self.tokens.len() - 1]); + self.interpolation_stack.push(self.last_token()); Ok(()) } EndOfFile => Ok(()), @@ -811,7 +818,12 @@ impl<'src> Lexer<'src> { self.advance()?; } - self.token(Comment); + if self.open_delimiters_or_interpolation() { + self.token(Whitespace); + self.dropped_comment.get_or_insert(self.last_token()); + } else { + self.token(Comment); + } Ok(()) } @@ -959,7 +971,7 @@ mod tests { text.to_owned() }; - let have = Lexer::test_lex(&text).unwrap(); + let have = Lexer::test_lex(&text).unwrap().tokens; let have_kinds = have .iter() diff --git a/src/lib.rs b/src/lib.rs index 29a9eaf366..effdd9c755 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,6 +73,7 @@ pub(crate) use { keyed::Keyed, keyword::Keyword, layer::Layer, + lex::Lex, lexer::Lexer, line::Line, list::List, @@ -313,6 +314,7 @@ mod justfile; mod keyed; mod keyword; mod layer; +mod lex; mod lexer; mod line; mod list; diff --git a/src/parser.rs b/src/parser.rs index b9bee2d248..e0090893c0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -24,6 +24,7 @@ use {super::*, TokenKind::*}; /// token, the set is cleared. If the parser finds a token which is unexpected, /// the elements of the set are printed in the resultant error message. pub(crate) struct Parser<'run, 'src> { + dropped_comment: Option>, expected_tokens: BTreeSet, file_depth: u32, import_offsets: Vec, @@ -39,16 +40,17 @@ pub(crate) struct Parser<'run, 'src> { } impl<'run, 'src> Parser<'run, 'src> { - /// Parse `tokens` into an `Ast` + /// Parse `lex` into an `Ast` pub(crate) fn parse( file_depth: u32, import_offsets: &[usize], + lex: &'run Lex<'src>, module_namepath: Option<&'run Namepath<'src>>, numerator: &'run mut Numerator, - tokens: &'run [Token<'src>], working_directory: &'run Utf8Path, ) -> CompileResult<'src, Ast<'src>> { Self { + dropped_comment: lex.dropped_comment, expected_tokens: BTreeSet::new(), file_depth, import_offsets: import_offsets.to_vec(), @@ -58,7 +60,7 @@ impl<'run, 'src> Parser<'run, 'src> { next_token: 0, numerator, recursion_depth: 0, - tokens, + tokens: &lex.tokens, unstable_features: BTreeSet::new(), working_directory, } @@ -71,24 +73,24 @@ impl<'run, 'src> Parser<'run, 'src> { source: &Source<'src>, src: &'src str, ) -> CompileResult<'src, Ast<'src>> { - let tokens = Lexer::lex(path, src)?; + let lex = Lexer::lex(path, src)?; Parser::parse( source.file_depth, &source.import_offsets, + &lex, source.namepath.as_ref(), numerator, - &tokens, &source.working_directory, ) } #[cfg(test)] - pub(crate) fn parse_tokens( + pub(crate) fn parse_lex( + lex: &'run Lex<'src>, numerator: &'run mut Numerator, - tokens: &'run [Token<'src>], ) -> CompileResult<'src, Ast<'src>> { - Self::parse(0, &[], None, numerator, tokens, "".as_ref()) + Self::parse(0, &[], lex, None, numerator, "".as_ref()) } fn error(&self, kind: CompileErrorKind<'src>) -> CompileResult<'src, CompileError<'src>> { @@ -511,6 +513,7 @@ impl<'run, 'src> Parser<'run, 'src> { } Ok(Ast { + dropped_comment: self.dropped_comment, items: self.items, list_features: self.list_features, module_path: self.module_namepath.map(Into::into).unwrap_or_default(), @@ -1931,15 +1934,15 @@ mod tests { fn test(text: &str, want: Tree) { let unindented = unindent(text); - let tokens = Lexer::test_lex(&unindented).expect("lexing failed"); - let have = Parser::parse_tokens(&mut Numerator::new(), &tokens) + let lex = Lexer::test_lex(&unindented).expect("lexing failed"); + let have = Parser::parse_lex(&lex, &mut Numerator::new()) .expect("parsing failed") .tree(); if have != want { eprintln!("parsed text: {unindented}"); eprintln!("expected: {want}"); eprintln!("but got: {have}"); - eprintln!("tokens: {tokens:?}"); + eprintln!("tokens: {:?}", lex.tokens); panic!(); } } @@ -1984,9 +1987,9 @@ mod tests { length: usize, kind: CompileErrorKind, ) { - let tokens = Lexer::test_lex(src).expect("Lexing failed in parse test..."); + let lex = Lexer::test_lex(src).expect("Lexing failed in parse test..."); - match Parser::parse_tokens(&mut Numerator::new(), &tokens) { + match Parser::parse_lex(&lex, &mut Numerator::new()) { Ok(_) => panic!("Parsing unexpectedly succeeded"), Err(have) => { let want = CompileError { @@ -2015,9 +2018,9 @@ mod tests { length: usize, found: TokenKind, ) { - let tokens = Lexer::test_lex(src).expect("Lexing failed in parse test..."); + let lex = Lexer::test_lex(src).expect("Lexing failed in parse test..."); - match Parser::parse_tokens(&mut Numerator::new(), &tokens) { + match Parser::parse_lex(&lex, &mut Numerator::new()) { Ok(_) => panic!("Parsing unexpectedly succeeded"), Err(have) => { assert_eq!( diff --git a/src/subcommand.rs b/src/subcommand.rs index d5aa5ba66d..56e389acb5 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -489,9 +489,7 @@ impl Subcommand { } fn format<'src>(config: &Config, loader: &'src Loader, search: &Search) -> RunResult<'src> { - let root = search.justfile_parent(); - - let (path, src) = loader.load(root, &search.justfile)?; + let (path, src) = loader.load(&search.justfile, &search.justfile)?; let ast = Parser::parse_source( &mut Numerator::new(), @@ -500,6 +498,10 @@ impl Subcommand { src, )?; + if let Some(token) = ast.dropped_comment { + return Err(Error::FormatDroppedComment { token }); + } + let formatted = ast .color_display( config diff --git a/src/testing.rs b/src/testing.rs index 797fafc514..4065e4d959 100644 --- a/src/testing.rs +++ b/src/testing.rs @@ -57,10 +57,10 @@ pub(crate) fn analysis_error( length: usize, kind: CompileErrorKind, ) { - let tokens = Lexer::test_lex(src).expect("Lexing failed in parse test..."); + let lex = Lexer::test_lex(src).expect("Lexing failed in parse test..."); - let ast = Parser::parse_tokens(&mut Numerator::new(), &tokens) - .expect("Parsing failed in analysis test..."); + let ast = + Parser::parse_lex(&lex, &mut Numerator::new()).expect("Parsing failed in analysis test..."); let root = Utf8PathBuf::from("justfile"); let mut asts: HashMap<(Modulepath, Utf8PathBuf), Ast> = HashMap::new(); diff --git a/tests/delimiters.rs b/tests/delimiters.rs index 143af509d0..fa9cd07d90 100644 --- a/tests/delimiters.rs +++ b/tests/delimiters.rs @@ -124,3 +124,14 @@ fn interpolation_continuation() { .stdout("ab\n") .success(); } + +#[test] +fn comments_in_delimiters_are_ignored() { + assert_eval( + "( + # foo + 'bar' + )", + "bar", + ); +} diff --git a/tests/format.rs b/tests/format.rs index 5655e4d905..8006335881 100644 --- a/tests/format.rs +++ b/tests/format.rs @@ -94,6 +94,30 @@ fn check_found_diff() { .failure(); } +#[test] +fn comment_in_delimiter_error() { + Test::new() + .arg("--fmt") + .justfile( + " + x := ( + # foo + 'bar' + ) + ", + ) + .stderr( + " + error: formatting would drop comment + ——▶ justfile:2:3 + │ + 2 │ # foo + │ ^^^^^ + ", + ) + .failure(); +} + #[test] fn check_found_diff_quiet() { Test::new() diff --git a/tests/interpolation.rs b/tests/interpolation.rs index 57bca46d1e..2234d76982 100644 --- a/tests/interpolation.rs +++ b/tests/interpolation.rs @@ -46,29 +46,6 @@ fn eol_without_continuation_in_interpolation() { .success(); } -#[test] -fn comment_in_interopolation() { - Test::new() - .justfile( - " - foo: - echo {{ # hello - 'a' - }} - ", - ) - .stderr( - " - error: expected backtick, '!', '[', identifier, '(', '/', or string, but found comment - ——▶ justfile:2:11 - │ - 2 │ echo {{ # hello - │ ^^^^^^^ - ", - ) - .failure(); -} - #[test] fn indent_and_dedent_are_ignored_in_interpolation() { Test::new()