Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions GRAMMAR.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ LINE_PREFIX = @-|-@|@|-
TEXT = recipe text, only matches in a recipe body
```

Inside delimiters, including `()`, `[], `{}`, and `{{}}`, `COMMENT` and
`NEWLINE` are treated as whitespace.

grammar syntax
--------------

Expand Down
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4585,6 +4585,11 @@ $ just --dump > formatted-justfile
The `--dump` command can be used with `--dump-format json` to print a JSON
representation of a `justfile`.

Comments inside delimiters, such as `[]` and `()` are ignored.<sup>master</sup>
However, they are not preserved by the formatter. To prevent comments from being
silently dropped, the formatter will return an error if the justfile has
comments which it cannot preserve.

Cached Recipes
--------------

Expand Down
1 change: 1 addition & 0 deletions src/ast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use super::*;
/// are performed by the `Analyzer`, which produces a `Justfile` from an `Ast`.
#[derive(Debug, Clone)]
pub(crate) struct Ast<'src> {
pub(crate) dropped_comment: Option<Token<'src>>,
pub(crate) items: Vec<Item<'src>>,
pub(crate) list_features: Vec<(ListFeature, Token<'src>)>,
pub(crate) module_path: Modulepath,
Expand Down
4 changes: 2 additions & 2 deletions src/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,8 @@ impl Compiler {

#[cfg(test)]
pub(crate) fn test_compile(src: &str) -> CompileResult<Justfile> {
let tokens = Lexer::test_lex(src)?;
let ast = Parser::parse_tokens(&mut Numerator::new(), &tokens)?;
let lex = Lexer::test_lex(src)?;
let ast = Parser::parse_lex(&lex, &mut Numerator::new())?;
let root = Utf8PathBuf::from("justfile");
let mut asts: HashMap<(Modulepath, Utf8PathBuf), Ast> = HashMap::new();
asts.insert((Modulepath::default(), root.clone()), ast);
Expand Down
10 changes: 9 additions & 1 deletion src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,9 @@ pub(crate) enum Error<'src> {
switch: Switch,
},
FormatCheckFoundDiff,
FormatDroppedComment {
token: Token<'src>,
},
FunctionCall {
function: Name<'src>,
message: String,
Expand Down Expand Up @@ -395,7 +398,9 @@ impl<'src> Error<'src> {
Some(module.token)
}
Self::Assert { name, .. } => Some(**name),
Self::Backtick { token, .. } | Self::RegexCompile { token, .. } => Some(*token),
Self::Backtick { token, .. }
| Self::FormatDroppedComment { token }
| Self::RegexCompile { token, .. } => Some(*token),
Self::Compile { compile_error } => Some(compile_error.context()),
Self::Const { const_error } => Some(const_error.context()),
Self::FunctionCall { function, .. } => Some(function.token),
Expand Down Expand Up @@ -810,6 +815,9 @@ impl ColorDisplay for Error<'_> {
FormatCheckFoundDiff => {
write!(f, "formatted justfile differs from original")?;
}
FormatDroppedComment { .. } => {
write!(f, "formatting would drop comment")?;
}
FunctionCall { message, .. } => {
write!(f, "{message}")?;
}
Expand Down
6 changes: 6 additions & 0 deletions src/lex.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
use super::*;

pub(crate) struct Lex<'src> {
pub(crate) dropped_comment: Option<Token<'src>>,
pub(crate) tokens: Vec<Token<'src>>,
}
30 changes: 21 additions & 9 deletions src/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ static DEDENT_RE: LazyLock<Regex> =
pub(crate) struct Lexer<'src> {
/// Char iterator
chars: Chars<'src>,
dropped_comment: Option<Token<'src>>,
/// Indentation stack
indentation: Vec<&'src str>,
/// Interpolation token start stack
Expand Down Expand Up @@ -46,12 +47,12 @@ impl<'src> Lexer<'src> {
pub(crate) const INTERPOLATION_START: &'static str = "{{";

/// Lex `src`
pub(crate) fn lex(path: &'src Utf8Path, src: &'src str) -> CompileResult<'src, Vec<Token<'src>>> {
pub(crate) fn lex(path: &'src Utf8Path, src: &'src str) -> CompileResult<'src, Lex<'src>> {
Self::new(path, src).tokenize()
}

#[cfg(test)]
pub(crate) fn test_lex(src: &'src str) -> CompileResult<'src, Vec<Token<'src>>> {
pub(crate) fn test_lex(src: &'src str) -> CompileResult<'src, Lex<'src>> {
Self::new("justfile".as_ref(), src).tokenize()
}

Expand All @@ -68,6 +69,7 @@ impl<'src> Lexer<'src> {

Self {
chars,
dropped_comment: None,
indentation: vec![""],
interpolation_stack: Vec::new(),
lines: src.lines().count(),
Expand Down Expand Up @@ -205,6 +207,10 @@ impl<'src> Lexer<'src> {
self.token_start = self.token_end;
}

fn last_token(&self) -> Token<'src> {
*self.tokens.last().unwrap()
}

/// Create an internal error with `message`
fn internal_error(&self, message: impl Into<String>) -> CompileError<'src> {
// Use `self.token_end` as the location of the error
Expand Down Expand Up @@ -273,7 +279,7 @@ impl<'src> Lexer<'src> {
}

/// Consume the text and produce a series of tokens
fn tokenize(mut self) -> CompileResult<'src, Vec<Token<'src>>> {
fn tokenize(mut self) -> CompileResult<'src, Lex<'src>> {
loop {
if self.token_start.column == 0 {
self.lex_line_start()?;
Expand Down Expand Up @@ -307,7 +313,10 @@ impl<'src> Lexer<'src> {
assert_eq!(self.token_start.offset, self.src.len());
assert_eq!(self.indentation.len(), 1);

Ok(self.tokens)
Ok(Lex {
dropped_comment: self.dropped_comment,
tokens: self.tokens,
})
}

/// Handle blank lines and indentation
Expand Down Expand Up @@ -594,9 +603,7 @@ impl<'src> Lexer<'src> {
NewlineCarriageReturn => self.lex_double(Eol),
Interpolation => {
self.lex_double(InterpolationStart)?;
self
.interpolation_stack
.push(self.tokens[self.tokens.len() - 1]);
self.interpolation_stack.push(self.last_token());
Ok(())
}
EndOfFile => Ok(()),
Expand Down Expand Up @@ -811,7 +818,12 @@ impl<'src> Lexer<'src> {
self.advance()?;
}

self.token(Comment);
if self.open_delimiters_or_interpolation() {
self.token(Whitespace);
self.dropped_comment.get_or_insert(self.last_token());
} else {
self.token(Comment);
}

Ok(())
}
Expand Down Expand Up @@ -959,7 +971,7 @@ mod tests {
text.to_owned()
};

let have = Lexer::test_lex(&text).unwrap();
let have = Lexer::test_lex(&text).unwrap().tokens;

let have_kinds = have
.iter()
Expand Down
2 changes: 2 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ pub(crate) use {
keyed::Keyed,
keyword::Keyword,
layer::Layer,
lex::Lex,
lexer::Lexer,
line::Line,
list::List,
Expand Down Expand Up @@ -313,6 +314,7 @@ mod justfile;
mod keyed;
mod keyword;
mod layer;
mod lex;
mod lexer;
mod line;
mod list;
Expand Down
33 changes: 18 additions & 15 deletions src/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use {super::*, TokenKind::*};
/// token, the set is cleared. If the parser finds a token which is unexpected,
/// the elements of the set are printed in the resultant error message.
pub(crate) struct Parser<'run, 'src> {
dropped_comment: Option<Token<'src>>,
expected_tokens: BTreeSet<TokenKind>,
file_depth: u32,
import_offsets: Vec<usize>,
Expand All @@ -39,16 +40,17 @@ pub(crate) struct Parser<'run, 'src> {
}

impl<'run, 'src> Parser<'run, 'src> {
/// Parse `tokens` into an `Ast`
/// Parse `lex` into an `Ast`
pub(crate) fn parse(
file_depth: u32,
import_offsets: &[usize],
lex: &'run Lex<'src>,
module_namepath: Option<&'run Namepath<'src>>,
numerator: &'run mut Numerator,
tokens: &'run [Token<'src>],
working_directory: &'run Utf8Path,
) -> CompileResult<'src, Ast<'src>> {
Self {
dropped_comment: lex.dropped_comment,
expected_tokens: BTreeSet::new(),
file_depth,
import_offsets: import_offsets.to_vec(),
Expand All @@ -58,7 +60,7 @@ impl<'run, 'src> Parser<'run, 'src> {
next_token: 0,
numerator,
recursion_depth: 0,
tokens,
tokens: &lex.tokens,
unstable_features: BTreeSet::new(),
working_directory,
}
Expand All @@ -71,24 +73,24 @@ impl<'run, 'src> Parser<'run, 'src> {
source: &Source<'src>,
src: &'src str,
) -> CompileResult<'src, Ast<'src>> {
let tokens = Lexer::lex(path, src)?;
let lex = Lexer::lex(path, src)?;

Parser::parse(
source.file_depth,
&source.import_offsets,
&lex,
source.namepath.as_ref(),
numerator,
&tokens,
&source.working_directory,
)
}

#[cfg(test)]
pub(crate) fn parse_tokens(
pub(crate) fn parse_lex(
lex: &'run Lex<'src>,
numerator: &'run mut Numerator,
tokens: &'run [Token<'src>],
) -> CompileResult<'src, Ast<'src>> {
Self::parse(0, &[], None, numerator, tokens, "".as_ref())
Self::parse(0, &[], lex, None, numerator, "".as_ref())
}

fn error(&self, kind: CompileErrorKind<'src>) -> CompileResult<'src, CompileError<'src>> {
Expand Down Expand Up @@ -511,6 +513,7 @@ impl<'run, 'src> Parser<'run, 'src> {
}

Ok(Ast {
dropped_comment: self.dropped_comment,
items: self.items,
list_features: self.list_features,
module_path: self.module_namepath.map(Into::into).unwrap_or_default(),
Expand Down Expand Up @@ -1931,15 +1934,15 @@ mod tests {

fn test(text: &str, want: Tree) {
let unindented = unindent(text);
let tokens = Lexer::test_lex(&unindented).expect("lexing failed");
let have = Parser::parse_tokens(&mut Numerator::new(), &tokens)
let lex = Lexer::test_lex(&unindented).expect("lexing failed");
let have = Parser::parse_lex(&lex, &mut Numerator::new())
.expect("parsing failed")
.tree();
if have != want {
eprintln!("parsed text: {unindented}");
eprintln!("expected: {want}");
eprintln!("but got: {have}");
eprintln!("tokens: {tokens:?}");
eprintln!("tokens: {:?}", lex.tokens);
panic!();
}
}
Expand Down Expand Up @@ -1984,9 +1987,9 @@ mod tests {
length: usize,
kind: CompileErrorKind,
) {
let tokens = Lexer::test_lex(src).expect("Lexing failed in parse test...");
let lex = Lexer::test_lex(src).expect("Lexing failed in parse test...");

match Parser::parse_tokens(&mut Numerator::new(), &tokens) {
match Parser::parse_lex(&lex, &mut Numerator::new()) {
Ok(_) => panic!("Parsing unexpectedly succeeded"),
Err(have) => {
let want = CompileError {
Expand Down Expand Up @@ -2015,9 +2018,9 @@ mod tests {
length: usize,
found: TokenKind,
) {
let tokens = Lexer::test_lex(src).expect("Lexing failed in parse test...");
let lex = Lexer::test_lex(src).expect("Lexing failed in parse test...");

match Parser::parse_tokens(&mut Numerator::new(), &tokens) {
match Parser::parse_lex(&lex, &mut Numerator::new()) {
Ok(_) => panic!("Parsing unexpectedly succeeded"),
Err(have) => {
assert_eq!(
Expand Down
8 changes: 5 additions & 3 deletions src/subcommand.rs
Original file line number Diff line number Diff line change
Expand Up @@ -489,9 +489,7 @@ impl Subcommand {
}

fn format<'src>(config: &Config, loader: &'src Loader, search: &Search) -> RunResult<'src> {
let root = search.justfile_parent();

let (path, src) = loader.load(root, &search.justfile)?;
let (path, src) = loader.load(&search.justfile, &search.justfile)?;

let ast = Parser::parse_source(
&mut Numerator::new(),
Expand All @@ -500,6 +498,10 @@ impl Subcommand {
src,
)?;

if let Some(token) = ast.dropped_comment {
return Err(Error::FormatDroppedComment { token });
}

let formatted = ast
.color_display(
config
Expand Down
6 changes: 3 additions & 3 deletions src/testing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,10 @@ pub(crate) fn analysis_error(
length: usize,
kind: CompileErrorKind,
) {
let tokens = Lexer::test_lex(src).expect("Lexing failed in parse test...");
let lex = Lexer::test_lex(src).expect("Lexing failed in parse test...");

let ast = Parser::parse_tokens(&mut Numerator::new(), &tokens)
.expect("Parsing failed in analysis test...");
let ast =
Parser::parse_lex(&lex, &mut Numerator::new()).expect("Parsing failed in analysis test...");

let root = Utf8PathBuf::from("justfile");
let mut asts: HashMap<(Modulepath, Utf8PathBuf), Ast> = HashMap::new();
Expand Down
11 changes: 11 additions & 0 deletions tests/delimiters.rs
Original file line number Diff line number Diff line change
Expand Up @@ -124,3 +124,14 @@ fn interpolation_continuation() {
.stdout("ab\n")
.success();
}

#[test]
fn comments_in_delimiters_are_ignored() {
assert_eval(
"(
# foo
'bar'
)",
"bar",
);
}
24 changes: 24 additions & 0 deletions tests/format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,30 @@ fn check_found_diff() {
.failure();
}

#[test]
fn comment_in_delimiter_error() {
Test::new()
.arg("--fmt")
.justfile(
"
x := (
# foo
'bar'
)
",
)
.stderr(
"
error: formatting would drop comment
——▶ justfile:2:3
│
2 │ # foo
│ ^^^^^
",
)
.failure();
}

#[test]
fn check_found_diff_quiet() {
Test::new()
Expand Down
Loading
Loading