repo_description: 🤖 Just a command runner
code: use {super::*, CompileErrorKind::*, TokenKind::*};
  
  static DEDENT_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[ \t\n]*\n[^ \t\n]").unwrap());
  
  /// Just language lexer
  ///
  /// The lexer proceeds character-by-character, as opposed to using regular
  /// expressions to lex tokens or semi-tokens at a time. As a result, it is
  /// verbose and straightforward. Just used to have a regex-based lexer, which
  /// was slower and generally godawful.  However, this should not be taken as a
  /// slight against regular expressions, the lexer was just idiosyncratically
  /// bad.
  pub(crate) struct Lexer<'src> {
    /// Char iterator
    chars: Chars<'src>,
    /// Indentation stack
    indentation: Vec<&'src str>,
    /// Interpolation token start stack
    interpolation_stack: Vec<Token<'src>>,
    /// Source text line count
    lines: usize,
    /// Next character to be lexed
    next: Option<char>,
    /// Current open delimiters
    open_delimiters: Vec<(Delimiter, usize)>,
    /// Path to source file
    path: &'src Path,
    /// Inside recipe body
    recipe_body: bool,
    /// Next indent will start a recipe body
    recipe_body_pending: bool,
    /// Source text
    src: &'src str,
    /// Current token end
    token_end: Position,
    /// Current token start
    token_start: Position,
    /// Tokens
    tokens: Vec<Token<'src>>,
  }
  
  impl<'src> Lexer<'src> {
    pub(crate) const INTERPOLATION_END: &'static str = "}}";
    pub(crate) const INTERPOLATION_ESCAPE: &'static str = "{{{{";
    pub(crate) const INTERPOLATION_START: &'static str = "{{";
  
    /// Lex `src`
    pub(crate) fn lex(path: &'src Path, src: &'src str) -> CompileResult<'src, Vec<Token<'src>>> {
      Self::new(path, src).tokenize()
    }
  
    #[cfg(test)]
    pub(crate) fn test_lex(src: &'src str) -> CompileResult<'src, Vec<Token<'src>>> {
      Self::new("justfile".as_ref(), src).tokenize()
    }
  
    /// Create a new Lexer to lex `src`
    fn new(path: &'src Path, src: &'src str) -> Self {
      let mut chars = src.chars();
      let next = chars.next();
  
      let start = Position {
        offset: 0,
        column: 0,
        line: 0,
      };
  
      Self {
        chars,
        indentation: vec![""],
        interpolation_stack: Vec::new(),
        lines: src.lines().count(),
        next,
        open_delimiters: Vec::new(),
        path,
        recipe_body: false,
        recipe_body_pending: false,
        src,
        token_end: start,
        token_start: start,
        tokens: Vec::new(),
      }
    }
  
    /// Advance over the character in `self.next`, updating `self.token_end`
    /// accordingly.
    fn advance(&mut self) -> CompileResult<'src> {
      match self.next {
        Some(c) => {
          let len_utf8 = c.len_utf8();
  
          self.token_end.offset += len_utf8;
          self.token_end.column += len_utf8;
  
          if c == '\n' && self.token_end.line + 1 < self.lines {
            self.token_end.column = 0;
            self.token_end.line += 1;
          }
  
          self.next = self.chars.next();
  
          Ok(())
        }
        None => Err(self.internal_error("Lexer advanced past end of text")),
      }
    }
  
    /// Lexeme of in-progress token
    fn lexeme(&self) -> &'src str {
      &self.src[self.token_start.offset..self.token_end.offset]
    }
  
    /// Length of current token
    fn current_token_length(&self) -> usize {
      self.token_end.offset - self.token_start.offset
    }
  
    fn accepted(&mut self, c: char) -> CompileResult<'src, bool> {
      if self.next_is(c) {
        self.advance()?;
        Ok(true)
      } else {
        Ok(false)
      }
    }
  
    fn presume(&mut self, c: char) -> CompileResult<'src> {
      if !self.next_is(c) {
        return Err(self.internal_error(format!("Lexer presumed character `{c}`")));
      }
  
      self.advance()?;
  
      Ok(())
    }
  
    fn presume_str(&mut self, s: &str) -> CompileResult<'src> {
      for c in s.chars() {
        self.presume(c)?;
      }
  
      Ok(())
    }
  
    /// Is next character c?
    fn next_is(&self, c: char) -> bool {
      self.next == Some(c)
    }
  
    /// Is next character ' ' or '\t'?
    fn next_is_whitespace(&self) -> bool {
      self.next_is(' ') || self.next_is('\t')
    }
  
    /// Un-lexed text
    fn rest(&self) -> &'src str {
      &self.src[self.token_end.offset..]
    }
  
    /// Check if unlexed text begins with prefix
    fn rest_starts_with(&self, prefix: &str) -> bool {
      self.rest().starts_with(prefix)
    }
  
    /// Does rest start with "\n" or "\r\n"?
    fn at_eol(&self) -> bool {
      self.next_is('\n') || self.rest_starts_with("\r\n")
    }
  
    /// Are we at end-of-file?
    fn at_eof(&self) -> bool {
      self.rest().is_empty()
    }
  
    /// Are we at end-of-line or end-of-file?
    fn at_eol_or_eof(&self) -> bool {
      self.at_eol() || self.at_eof()
    }
  
    /// Get current indentation
    fn indentation(&self) -> &'src str {
      self.indentation.last().unwrap()
    }
  
    /// Are we currently indented
    fn indented(&self) -> bool {
      !self.indentation().is_empty()
    }
  
    /// Create a new token with `kind` whose lexeme is between `self.token_start`
    /// and `self.token_end`
    fn token(&mut self, kind: TokenKind) {
      self.tokens.push(Token {
        column: self.token_start.column,
        kind,
        length: self.token_end.offset - self.token_start.offset,
        line: self.token_start.line,
        offset: self.token_start.offset,
        path: self.path,
        src: self.src,
      });
  
      // Set `token_start` to point after the lexed token
      self.token_start = self.token_end;
    }
  
    /// Create an internal error with `message`
    fn internal_error(&self, message: impl Into<String>) -> CompileError<'src> {
      // Use `self.token_end` as the location of the error
      let token = Token {
        src: self.src,
        offset: self.token_end.offset,
        line: self.token_end.line,
        column: self.token_end.column,
        length: 0,
        kind: Unspecified,
        path: self.path,
      };
      CompileError::new(
        token,
        Internal {
          message: message.into(),
        },
      )
    }
  
    /// Create a compilation error with `kind`
    fn error(&self, kind: CompileErrorKind<'src>) -> CompileError<'src> {
      // Use the in-progress token span as the location of the error.
  
      // The width of the error site to highlight depends on the kind of error:
      let length = match kind {
        UnterminatedString | UnterminatedBacktick => {
          let Some(kind) = StringKind::from_token_start(self.lexeme()) else {
            return self.internal_error("Lexer::error: expected string or backtick token start");
          };
          kind.delimiter().len()
        }
        // highlight the full token
        _ => self.lexeme().len(),
      };
  
      let token = Token {
        kind: Unspecified,
        src: self.src,
        offset: self.token_start.offset,
        line: self.token_start.line,
        column: self.token_start.column,
        length,
        path: self.path,
      };
  
      CompileError::new(token, kind)
    }
  
    fn unterminated_interpolation_error(interpolation_start: Token<'src>) -> CompileError<'src> {
      CompileError::new(interpolation_start, UnterminatedInterpolation)
    }
  
    /// True if `c` can be the first character of an identifier
    pub(crate) fn is_identifier_start(c: char) -> bool {
      matches!(c, 'a'..='z' | 'A'..='Z' | '_')
    }
  
    /// True if `c` can be a continuation character of an identifier
    pub(crate) fn is_identifier_continue(c: char) -> bool {
      Self::is_identifier_start(c) || matches!(c, '0'..='9' | '-')
    }
  
    /// Consume the text and produce a series of tokens
    fn tokenize(mut self) -> CompileResult<'src, Vec<Token<'src>>> {
      loop {
        if self.token_start.column == 0 {
          self.lex_line_start()?;
        }
  
        match self.next {
          Some(first) => {
            if let Some(&interpolation_start) = self.interpolation_stack.last() {
              self.lex_interpolation(interpolation_start, first)?;
            } else if self.recipe_body {
              self.lex_body()?;
            } else {
              self.lex_normal(first)?;
            }
          }
          None => break,
        }
      }
  
      if let Some(&interpolation_start) = self.interpolation_stack.last() {
        return Err(Self::unterminated_interpolation_error(interpolation_start));
      }
  
      while self.indented() {
        self.lex_dedent();
      }
  
      self.token(Eof);
  
      assert_eq!(self.token_start.offset, self.token_end.offset);
      assert_eq!(self.token_start.offset, self.src.len());
      assert_eq!(self.indentation.len(), 1);
  
      Ok(self.tokens)
    }
  
    /// Handle blank lines and indentation
    fn lex_line_start(&mut self) -> CompileResult<'src> {
      enum Indentation<'src> {
        // Line only contains whitespace
        Blank,
        // Indentation continues
        Continue,
        // Indentation decreases
        Decrease,
        // Indentation isn't consistent
        Inconsistent,
        // Indentation increases
        Increase,
        // Indentation mixes spaces and tabs
        Mixed { whitespace: &'src str },
      }
  
      use Indentation::*;
  
      let nonblank_index = self
        .rest()
        .char_indices()
        .skip_while(|&(_, c)| c == ' ' || c == '\t')
        .map(|(i, _)| i)
        .next()
        .unwrap_or_else(|| self.rest().len());
  
      let rest = &self.rest()[nonblank_index..];
  
      let whitespace = &self.rest()[..nonblank_index];
  
      if self.open_delimiters_or_interpolation() {
        if !whitespace.is_empty() {
          while self.next_is_whitespace() {
            self.advance()?;
          }
  
          self.token(Whitespace);
        }
  
        return Ok(());
      }
  
      let body_whitespace = &whitespace[..whitespace
        .char_indices()
        .take(self.indentation().chars().count())
        .map(|(i, _c)| i)
        .next()
        .unwrap_or(0)];
  
      let spaces = whitespace.chars().any(|c| c == ' ');
      let tabs = whitespace.chars().any(|c| c == '\t');
  
      let body_spaces = body_whitespace.chars().any(|c| c == ' ');
      let body_tabs = body_whitespace.chars().any(|c| c == '\t');
  
      #[allow(clippy::if_same_then_else)]
      let indentation = if rest.starts_with('\n') || rest.starts_with("\r\n") || rest.is_empty() {
        Blank
      } else if whitespace == self.indentation() {
        Continue
      } else if self.indentation.contains(&whitespace) {
        Decrease
      } else if self.recipe_body && whitespace.starts_with(self.indentation()) {
        Continue
      } else if self.recipe_body && body_spaces && body_tabs {
        Mixed {
          whitespace: body_whitespace,
        }
      } else if !self.recipe_body && spaces && tabs {
        Mixed { whitespace }
      } else if whitespace.len() < self.indentation().len() {
        Inconsistent
      } else if self.recipe_body
        && body_whitespace.len() >= self.indentation().len()
        && !body_whitespace.starts_with(self.indentation())
      {
        Inconsistent
      } else if whitespace.len() >= self.indentation().len()
        && !whitespace.starts_with(self.indentation())
      {
        Inconsistent
      } else {
        Increase
      };
  
      match indentation {
        Blank => {
          if DEDENT_RE.is_match(self.rest()) {
            while self.indented() {
              self.lex_dedent();
            }
          }
  
          if !whitespace.is_empty() {
            while self.next_is_whitespace() {
              self.advance()?;
            }
  
            self.token(Whitespace);
          }
  
          Ok(())
        }
        Continue => {
          if !self.indentation().is_empty() {
            for _ in self.indentation().chars() {
              self.advance()?;
            }
  
            self.token(Whitespace);
          }
  
          Ok(())
        }
        Decrease => {
          while self.indentation() != whitespace {
            self.lex_dedent();
          }
  
          if !whitespace.is_empty() {
            while self.next_is_whitespace() {
              self.advance()?;
            }
  
            self.token(Whitespace);
          }
  
          Ok(())
        }
        Mixed { whitespace } => {
          for _ in whitespace.chars() {
            self.advance()?;
          }
  
          Err(self.error(MixedLeadingWhitespace { whitespace }))
        }
        Inconsistent => {
          for _ in whitespace.chars() {
            self.advance()?;
          }
  
          Err(self.error(InconsistentLeadingWhitespace {
            expected: self.indentation(),
            found: whitespace,
          }))
        }
        Increase => {
          while self.next_is_whitespace() {
            self.advance()?;
          }
  
          let indentation = self.lexeme();
          self.indentation.push(indentation);
          self.token(Indent);
          if self.recipe_body_pending {
            self.recipe_body = true;
          }
  
          Ok(())
        }
      }
    }
  
    /// Lex token beginning with `start` outside of a recipe body
    fn lex_normal(&mut self, start: char) -> CompileResult<'src> {
      match start {
        ' ' | '\t' => self.lex_whitespace(),
        '!' if self.rest().starts_with("!include") => Err(self.error(Include)),
        '!' => self.lex_choices('!', &[('=', BangEquals), ('~', BangTilde)], None),
        '#' => self.lex_comment(),
        '$' => self.lex_single(Dollar),
        '&' => self.lex_digraph('&', '&', AmpersandAmpersand),
        '(' => self.lex_delimiter(ParenL),
        ')' => self.lex_delimiter(ParenR),
        '*' => self.lex_single(Asterisk),
        '+' => self.lex_single(Plus),
        ',' => self.lex_single(Comma),
        '/' => self.lex_single(Slash),
        ':' => self.lex_colon(),
        '=' => self.lex_choices(
          '=',
          &[('=', EqualsEquals), ('~', EqualsTilde)],
          Some(Equals),
        ),
        '?' => self.lex_single(QuestionMark),
        '@' => self.lex_single(At),
        '[' => self.lex_delimiter(BracketL),
        '\\' => self.lex_escape(),
        '\n' | '\r' => self.lex_eol(),
        '\u{feff}' => self.lex_single(ByteOrderMark),
        ']' => self.lex_delimiter(BracketR),
        '`' | '"' | '\'' => self.lex_string(None),
        '{' => self.lex_delimiter(BraceL),
        '|' => self.lex_digraph('|', '|', BarBar),
        '}' => {
          let format_string_kind = self.open_delimiters.last().and_then(|(delimiter, _line)| {
            if !self.rest().starts_with(Self::INTERPOLATION_END) {
              None
            } else if let Delimiter::FormatString(kind) = delimiter {
              Some(kind)
            } else {
              None
            }
          });
  
          if let Some(format_string_kind) = format_string_kind {
            self.lex_string(Some(*format_string_kind))
          } else {
            self.lex_delimiter(BraceR)
          }
        }
        _ if Self::is_identifier_start(start) => self.lex_identifier(),
        _ => {
          self.advance()?;
          Err(self.error(UnknownStartOfToken { start }))
        }
      }
    }
  
    /// Lex token beginning with `start` inside an interpolation
    fn lex_interpolation(
      &mut self,
      interpolation_start: Token<'src>,
      start: char,
    ) -> CompileResult<'src> {
      if self.rest_starts_with(Self::INTERPOLATION_END) && self.open_delimiters.is_empty() {
        // end current interpolation
        if self.interpolation_stack.pop().is_none() {
          self.presume_str(Self::INTERPOLATION_END)?;
          return Err(self.internal_error(
            "Lexer::lex_interpolation found `}}` but was called with empty interpolation stack.",
          ));
        }
        // Emit interpolation end token
        self.lex_double(InterpolationEnd)
      } else if self.at_eof() && self.open_delimiters.is_empty() {
        // Return unterminated interpolation error that highlights the opening
        // {{
        Err(Self::unterminated_interpolation_error(interpolation_start))
      } else {
        // Otherwise lex as per normal
        self.lex_normal(start)
      }
    }
  
    /// Lex token while in recipe body
    fn lex_body(&mut self) -> CompileResult<'src> {
      enum Terminator {
        EndOfFile,
        Interpolation,
        Newline,
        NewlineCarriageReturn,
      }
  
      use Terminator::*;
  
      let terminator = loop {
        if self.rest_starts_with(Self::INTERPOLATION_ESCAPE) {
          self.presume_str(Self::INTERPOLATION_ESCAPE)?;
          continue;
        }
  
        if self.rest_starts_with("\n") {
          break Newline;
        }
  
        if self.rest_starts_with("\r\n") {
          break NewlineCarriageReturn;
        }
  
        if self.rest_starts_with(Self::INTERPOLATION_START) {
          break Interpolation;
        }
  
        if self.at_eof() {
          break EndOfFile;
        }
  
        self.advance()?;
      };
  
      // emit text token containing text so far
      if self.current_token_length() > 0 {
        self.token(Text);
      }
  
      match terminator {
        Newline => self.lex_single(Eol),
        NewlineCarriageReturn => self.lex_double(Eol),
        Interpolation => {
          self.lex_double(InterpolationStart)?;
          self
            .interpolation_stack
            .push(self.tokens[self.tokens.len() - 1]);
          Ok(())
        }
        EndOfFile => Ok(()),
      }
    }
  
    fn lex_dedent(&mut self) {
      assert_eq!(self.current_token_length(), 0);
      self.token(Dedent);
      self.indentation.pop();
      self.recipe_body_pending = false;
      self.recipe_body = false;
    }
  
    /// Lex a single-character token
    fn lex_single(&mut self, kind: TokenKind) -> CompileResult<'src> {
      self.advance()?;
      self.token(kind);
      Ok(())
    }
  
    /// Lex a double-character token
    fn lex_double(&mut self, kind: TokenKind) -> CompileResult<'src> {
      self.advance()?;
      self.advance()?;
      self.token(kind);
      Ok(())
    }
  
    /// Lex a double-character token of kind `then` if the second character of
    /// that token would be `second`, otherwise lex a single-character token of
    /// kind `otherwise`
    fn lex_choices(
      &mut self,
      first: char,
      choices: &[(char, TokenKind)],
      otherwise: Option<TokenKind>,
    ) -> CompileResult<'src> {
      self.presume(first)?;
  
      for (second, then) in choices {
        if self.accepted(*second)? {
          self.token(*then);
          return Ok(());
        }
      }
  
      if let Some(token) = otherwise {
        self.token(token);
      } else {
        // Emit an unspecified token to consume the current character,
        self.token(Unspecified);
  
        let expected = choices.iter().map(|choice| choice.0).collect();
  
        if self.at_eof() {
          return Err(self.error(UnexpectedEndOfToken { expected }));
        }
  
        // …and advance past another character,
        self.advance()?;
  
        // …so that the error we produce highlights the unexpected character.
        return Err(self.error(UnexpectedCharacter { expected }));
      }
  
      Ok(())
    }
  
    /// Lex an opening or closing delimiter
    fn lex_delimiter(&mut self, kind: TokenKind) -> CompileResult<'src> {
      match kind {
        BraceL => self.open_delimiter(Delimiter::Brace),
        BraceR => self.close_delimiter(Delimiter::Brace)?,
        BracketL => self.open_delimiter(Delimiter::Bracket),
        BracketR => self.close_delimiter(Delimiter::Bracket)?,
        ParenL => self.open_delimiter(Delimiter::Paren),
        ParenR => self.close_delimiter(Delimiter::Paren)?,
        _ => {
          return Err(self.internal_error(format!(
            "Lexer::lex_delimiter called with non-delimiter token: `{kind}`",
          )));
        }
      }
  
      // Emit the delimiter token
      self.lex_single(kind)?;
  
      Ok(())
    }
  
    /// Push a delimiter onto the open delimiter stack
    fn open_delimiter(&mut self, delimiter: Delimiter) {
      self
        .open_delimiters
        .push((delimiter, self.token_start.line));
    }
  
    /// Pop a delimiter from the open delimiter stack and error if incorrect type
    fn close_delimiter(&mut self, close: Delimiter) -> CompileResult<'src> {
      match self.open_delimiters.pop() {
        Some((open, _)) if open == close => Ok(()),
        Some((open, open_line)) => Err(self.error(MismatchedClosingDelimiter {
          open,
          close,
          open_line,
        })),
        None => Err(self.error(UnexpectedClosingDelimiter { close })),
      }
    }
  
    /// Return true if there are any unclosed delimiters
    fn open_delimiters_or_interpolation(&self) -> bool {
      !self.open_delimiters.is_empty() || !self.interpolation_stack.is_empty()
    }
  
    /// Lex a two-character digraph
    fn lex_digraph(&mut self, left: char, right: char, token: TokenKind) -> CompileResult<'src> {
      self.presume(left)?;
  
      if self.accepted(right)? {
        self.token(token);
        Ok(())
      } else {
        // Emit an unspecified token to consume the current character,
        self.token(Unspecified);
  
        if self.at_eof() {
          return Err(self.error(UnexpectedEndOfToken {
            expected: vec![right],
          }));
        }
  
        // …and advance past another character,
        self.advance()?;
  
        // …so that the error we produce highlights the unexpected character.
        Err(self.error(UnexpectedCharacter {
          expected: vec![right],
        }))
      }
    }
  
    /// Lex a token starting with ':'
    fn lex_colon(&mut self) -> CompileResult<'src> {
      self.presume(':')?;
  
      if self.accepted('=')? {
        self.token(ColonEquals);
      } else if self.accepted(':')? {
        self.token(ColonColon);
      } else {
        self.token(Colon);
        self.recipe_body_pending = true;
      }
  
      Ok(())
    }
  
    /// Lex an token starting with '\' escape
    fn lex_escape(&mut self) -> CompileResult<'src> {
      self.presume('\\')?;
  
      // Treat newline escaped with \ as whitespace
      if self.accepted('\n')? {
        while self.next_is_whitespace() {
          self.advance()?;
        }
        self.token(Whitespace);
      } else if self.accepted('\r')? {
        if !self.accepted('\n')? {
          return Err(self.error(UnpairedCarriageReturn));
        }
        while self.next_is_whitespace() {
          self.advance()?;
        }
        self.token(Whitespace);
      } else if let Some(character) = self.next {
        return Err(self.error(InvalidEscapeSequence { character }));
      }
  
      Ok(())
    }
  
    /// Lex a carriage return and line feed
    fn lex_eol(&mut self) -> CompileResult<'src> {
      if self.accepted('\r')? {
        if !self.accepted('\n')? {
          return Err(self.error(UnpairedCarriageReturn));
        }
      } else {
        self.presume('\n')?;
      }
  
      // Emit eol if there are no open delimiters, otherwise emit whitespace.
      if self.open_delimiters_or_interpolation() {
        self.token(Whitespace);
      } else {
        self.token(Eol);
      }
  
      Ok(())
    }
  
    /// Lex name: [a-zA-Z_][a-zA-Z0-9_]*
    fn lex_identifier(&mut self) -> CompileResult<'src> {
      self.advance()?;
  
      while let Some(c) = self.next {
        if !Self::is_identifier_continue(c) {
          break;
        }
  
        self.advance()?;
      }
  
      self.token(Identifier);
  
      Ok(())
    }
  
    /// Lex comment: #[^\r\n]
    fn lex_comment(&mut self) -> CompileResult<'src> {
      self.presume('#')?;
  
      while !self.at_eol_or_eof() {
        self.advance()?;
      }
  
      self.token(Comment);
  
      Ok(())
    }
  
    /// Lex whitespace: [ \t]+
    fn lex_whitespace(&mut self) -> CompileResult<'src> {
      while self.next_is_whitespace() {
        self.advance()?;
      }
  
      self.token(Whitespace);
  
      Ok(())
    }
  
    /// Lex a backtick, cooked string, or raw string.
    ///
    /// Backtick:      ``[^`]*``
    /// Cooked string: "[^"]*" # also processes escape sequences
    /// Raw string:    '[^']*'
    fn lex_string(&mut self, format_string_kind: Option<StringKind>) -> CompileResult<'src> {
      let format = format_string_kind.is_some()
        || self.tokens.last().is_some_and(|token| {
          token.kind == TokenKind::Identifier && token.lexeme() == Keyword::F.lexeme()
        });
  
      let kind = if let Some(kind) = format_string_kind {
        self.presume_str(Self::INTERPOLATION_END)?;
        kind
      } else {
        let Some(kind) = StringKind::from_token_start(self.rest()) else {
          self.advance()?;
          return Err(self.internal_error("Lexer::lex_string: invalid string start"));
        };
        self.presume_str(kind.delimiter())?;
        kind
      };
  
      let mut escape = false;
  
      loop {
        if self.next.is_none() {
          return Err(self.error(kind.unterminated_error_kind()));
        } else if !escape && kind.processes_escape_sequences() && self.next_is('\\') {
          escape = true;
        } else if escape && kind.processes_escape_sequences() && self.next_is('u') {
          escape = false;
        } else if format && self.rest_starts_with(Self::INTERPOLATION_ESCAPE) {
          escape = false;
          self.advance()?;
          self.advance()?;
          self.advance()?;
        } else if !escape
          && (self.rest_starts_with(kind.delimiter())
            || format && self.rest_starts_with(Self::INTERPOLATION_START))
        {
          break;
        } else {
          escape = false;
        }
  
        self.advance()?;
      }
  
      if format && self.rest_starts_with(Self::INTERPOLATION_START) {
        self.presume_str(Self::INTERPOLATION_START)?;
        if format_string_kind.is_some() {
          self.token(FormatStringContinue);
        } else {
          self.token(FormatStringStart);
          self.open_delimiter(Delimiter::FormatString(kind));
        }
      } else {
        self.presume_str(kind.delimiter())?;
  
        if let Some(format_string_kind) = format_string_kind {
          self.close_delimiter(Delimiter::FormatString(format_string_kind))?;
          self.token(FormatStringEnd);
        } else {
          self.token(kind.token_kind());
        }
      }
  
      Ok(())
    }
  }
  
  #[cfg(test)]
  mod tests {
    use super::*;
  
    use pretty_assertions::assert_eq;
  
    macro_rules! test {
      {
        name:     $name:ident,
        text:     $text:expr,
        tokens:   ($($kind:ident $(: $lexeme:literal)?),* $(,)?)$(,)?
      } => {
        #[test]
        fn $name() {
          let kinds: &[TokenKind] = &[$($kind,)* Eof];
  
          let lexemes: &[&str] = &[$(lexeme!($kind $(, $lexeme)?),)* ""];
  
          test($text, true, kinds, lexemes);
        }
      };
      {
        name:     $name:ident,
        text:     $text:expr,
        tokens:   ($($kind:ident $(: $lexeme:literal)?),* $(,)?)$(,)?
        unindent: $unindent:expr,
      } => {
        #[test]
        fn $name() {
          let kinds: &[TokenKind] = &[$($kind,)* Eof];
  
          let lexemes: &[&str] = &[$(lexeme!($kind $(, $lexeme)?),)* ""];
  
          test($text, $unindent, kinds, lexemes);
        }
      }
    }
  
    macro_rules! lexeme {
      {
        $kind:ident, $lexeme:literal
      } => {
        $lexeme
      };
      {
        $kind:ident
      } => {
        default_lexeme($kind)
      }
    }
  
    #[track_caller]
    fn test(text: &str, unindent_text: bool, want_kinds: &[TokenKind], want_lexemes: &[&str]) {
      let text = if unindent_text {
        unindent(text)
      } else {
        text.to_owned()
      };
  
      let have = Lexer::test_lex(&text).unwrap();
  
      let have_kinds = have
        .iter()
        .map(|token| token.kind)
        .collect::<Vec<TokenKind>>();
  
      let have_lexemes = have.iter().map(Token::lexeme).collect::<Vec<&str>>();
  
      assert_eq!(have_kinds, want_kinds, "token kind mismatch");
      assert_eq!(have_lexemes, want_lexemes, "token lexeme mismatch");
  
      let mut roundtrip = String::new();
  
      for lexeme in have_lexemes {
        roundtrip.push_str(lexeme);
      }
  
      assert_eq!(roundtrip, text, "roundtrip mismatch");
  
      let lines = text.lines().count();
  
      let mut offset = 0;
      let mut line = 0;
      let mut column = 0;
  
      for token in have {
        assert_eq!(token.offset, offset, "offset mismatch");
        assert_eq!(token.line, line, "line mismatch");
        assert_eq!(token.lexeme().len(), token.length, "length mismatch");
        assert_eq!(token.column, column, "column mismatch");
  
        for c in token.lexeme().chars() {
          if c == '\n' && line + 1 < lines {
            line += 1;
            column = 0;
          } else {
            column += c.len_utf8();
          }
        }
  
        offset += token.length;
      }
    }
  
    fn default_lexeme(kind: TokenKind) -> &'static str {
      match kind {
        // Fixed lexemes
        AmpersandAmpersand => "&&",
        Asterisk => "*",
        At => "@",
        BangEquals => "!=",
        BangTilde => "!~",
        BarBar => "||",
        BraceL => "{",
        BraceR => "}",
        BracketL => "[",
        BracketR => "]",
        ByteOrderMark => "\u{feff}",
        Colon => ":",
        ColonColon => "::",
        ColonEquals => ":=",
        Comma => ",",
        Dollar => "$",
        Eol => "\n",
        Equals => "=",
        EqualsEquals => "==",
        EqualsTilde => "=~",
        Indent => "  ",
        InterpolationEnd => "}}",
        InterpolationStart => "{{",
        ParenL => "(",
        ParenR => ")",
        Plus => "+",
        QuestionMark => "?",
        Slash => "/",
        Whitespace => " ",
  
        // Empty lexemes
        Dedent | Eof => "",
  
        // Variable lexemes
        Backtick | Comment | FormatStringContinue | FormatStringEnd | FormatStringStart
        | Identifier | StringToken | Text | Unspecified => {
          panic!("Token {kind:?} has no default lexeme")
        }
      }
    }
  
    macro_rules! error {
      (
        name:   $name:ident,
        input:  $input:expr,
        offset: $offset:expr,
        line:   $line:expr,
        column: $column:expr,
        width:  $width:expr,
        kind:   $kind:expr,
      ) => {
        #[test]
        fn $name() {
          error($input, $offset, $line, $column, $width, $kind);
        }
      };
    }
  
    #[track_caller]
    fn error(
      src: &str,
      offset: usize,
      line: usize,
      column: usize,
      length: usize,
      kind: CompileErrorKind,
    ) {
      match Lexer::test_lex(src) {
        Ok(_) => panic!("Lexing succeeded but expected"),
        Err(have) => {
          let want = CompileError {
            token: Token {
              kind: have.token.kind,
              src,
              offset,
              line,
              column,
              length,
              path: "justfile".as_ref(),
            },
            kind: kind.into(),
          };
          assert_eq!(have, want);
        }
      }
    }
  
    test! {
      name:   name_new,
      text:   "foo",
      tokens: (Identifier:"foo"),
    }
  
    test! {
      name:   comment,
      text:   "# hello",
      tokens: (Comment:"# hello"),
    }
  
    test! {
      name:   backtick,
      text:   "`echo`",
      tokens: (Backtick:"`echo`"),
    }
  
    test! {
      name:   backtick_multi_line,
      text:   "`echo\necho`",
      tokens: (Backtick:"`echo\necho`"),
    }
  
    test! {
      name:   raw_string,
      text:   "'hello'",
      tokens: (StringToken:"'hello'"),
    }
  
    test! {
      name:   raw_string_multi_line,
      text:   "'hello\ngoodbye'",
      tokens: (StringToken:"'hello\ngoodbye'"),
    }
  
    test! {
      name:   cooked_string,
      text:   "\"hello\"",
      tokens: (StringToken:"\"hello\""),
    }
  
    test! {
      name:   cooked_string_multi_line,
      text:   "\"hello\ngoodbye\"",
      tokens: (StringToken:"\"hello\ngoodbye\""),
    }
  
    test! {
      name:   cooked_multiline_string,
      text:   "\"\"\"hello\ngoodbye\"\"\"",
      tokens: (StringToken:"\"\"\"hello\ngoodbye\"\"\""),
    }
  
    test! {
      name:   ampersand_ampersand,
      text:   "&&",
      tokens: (AmpersandAmpersand),
    }
  
    test! {
      name:   equals,
      text:   "=",
      tokens: (Equals),
    }
  
    test! {
      name:   equals_equals,
      text:   "==",
      tokens: (EqualsEquals),
    }
  
    test! {
      name:   bang_equals,
      text:   "!=",
      tokens: (BangEquals),
    }
  
    test! {
      name:   brace_l,
      text:   "{",
      tokens: (BraceL),
    }
  
    test! {
      name:   brace_r,
      text:   "{}",
      tokens: (BraceL, BraceR),
    }
  
    test! {
      name:   brace_lll,
      text:   "{{{",
      tokens: (BraceL, BraceL, BraceL),
    }
  
    test! {
      name:   brace_rrr,
      text:   "{{{}}}",
      tokens: (BraceL, BraceL, BraceL, BraceR, BraceR, BraceR),
    }
  
    test! {
      name:   dollar,
      text:   "$",
      tokens: (Dollar),
    }
  
    test! {
      name:   export_concatenation,
      text:   "export foo = 'foo' + 'bar'",
      tokens: (
        Identifier:"export",
        Whitespace,
        Identifier:"foo",
        Whitespace,
        Equals,
        Whitespace,
        StringToken:"'foo'",
        Whitespace,
        Plus,
        Whitespace,
        StringToken:"'bar'",
      )
    }
  
    test! {
      name: export_complex,
      text: "export foo = ('foo' + 'bar') + `baz`",
      tokens: (
        Identifier:"export",
        Whitespace,
        Identifier:"foo",
        Whitespace,
        Equals,
        Whitespace,
        ParenL,
        StringToken:"'foo'",
        Whitespace,
        Plus,
        Whitespace,
        StringToken:"'bar'",
        ParenR,
        Whitespace,
        Plus,
        Whitespace,
        Backtick:"`baz`",
      ),
    }
  
    test! {
      name:     eol_linefeed,
      text:     "\n",
      tokens:   (Eol),
      unindent: false,
    }
  
    test! {
      name:     eol_carriage_return_linefeed,
      text:     "\r\n",
      tokens:   (Eol:"\r\n"),
      unindent: false,
    }
  
    test! {
      name:   indented_line,
      text:   "foo:\n a",
      tokens: (Identifier:"foo", Colon, Eol, Indent:" ", Text:"a", Dedent),
    }
  
    test! {
      name:   indented_normal,
      text:   "
        a
          b
          c
      ",
      tokens: (
        Identifier:"a",
        Eol,
        Indent:"  ",
        Identifier:"b",
        Eol,
        Whitespace:"  ",
        Identifier:"c",
        Eol,
        Dedent,
      ),
    }
  
    test! {
      name:   indented_normal_nonempty_blank,
      text:   "a\n  b\n\t\t\n  c\n",
      tokens: (
        Identifier:"a",
        Eol,
        Indent:"  ",
        Identifier:"b",
        Eol,
        Whitespace:"\t\t",
        Eol,
        Whitespace:"  ",
        Identifier:"c",
        Eol,
        Dedent,
      ),
      unindent: false,
    }
  
    test! {
      name:   indented_normal_multiple,
      text:   "
        a
          b
            c
      ",
      tokens: (
        Identifier:"a",
        Eol,
        Indent:"  ",
        Identifier:"b",
        Eol,
        Indent:"    ",
        Identifier:"c",
        Eol,
        Dedent,
        Dedent,
      ),
    }
  
    test! {
      name:   indent_indent_dedent_indent,
      text:   "
        a
          b
            c
          d
            e
      ",
      tokens: (
        Identifier:"a",
        Eol,
        Indent:"  ",
          Identifier:"b",
          Eol,
          Indent:"    ",
            Identifier:"c",
            Eol,
          Dedent,
          Whitespace:"  ",
          Identifier:"d",
          Eol,
          Indent:"    ",
            Identifier:"e",
            Eol,
          Dedent,
        Dedent,
      ),
    }
  
    test! {
      name:   indent_recipe_dedent_indent,
      text:   "
        a
          b:
            c
          d
            e
      ",
      tokens: (
        Identifier:"a",
        Eol,
        Indent:"  ",
          Identifier:"b",
          Colon,
          Eol,
          Indent:"    ",
            Text:"c",
            Eol,
          Dedent,
          Whitespace:"  ",
          Identifier:"d",
          Eol,
          Indent:"    ",
            Identifier:"e",
            Eol,
          Dedent,
        Dedent,
      ),
    }
  
    test! {
      name: indented_block,
      text: "
        foo:
          a
          b
          c
      ",
      tokens: (
        Identifier:"foo",
        Colon,
        Eol,
        Indent,
        Text:"a",
        Eol,
        Whitespace:"  ",
        Text:"b",
        Eol,
        Whitespace:"  ",
        Text:"c",
        Eol,
        Dedent,
      )
    }
  
    test! {
      name: brace_escape,
      text: "
        foo:
          {{{{
      ",
      tokens: (
        Identifier:"foo",
        Colon,
        Eol,
        Indent,
        Text:"{{{{",
        Eol,
        Dedent,
      )
    }
  
    test! {
      name: indented_block_followed_by_item,
      text: "
        foo:
          a
        b:
      ",
      tokens: (
        Identifier:"foo",
        Colon,
        Eol,
        Indent,
        Text:"a",
        Eol,
        Dedent,
        Identifier:"b",
        Colon,
        Eol,
      )
    }
  
    test! {
      name: indented_block_followed_by_blank,
      text: "
        foo:
            a
  
        b:
      ",
      tokens: (
        Identifier:"foo",
        Colon,
        Eol,
        Indent:"    ",
        Text:"a",
        Eol,
        Dedent,
        Eol,
        Identifier:"b",
        Colon,
        Eol,
      ),
    }
  
    test! {
      name: indented_line_containing_unpaired_carriage_return,
      text: "foo:\n \r \n",
      tokens: (
        Identifier:"foo",
        Colon,
        Eol,
        Indent:" ",
        Text:"\r ",
        Eol,
        Dedent,
      ),
      unindent: false,
    }
  
    test! {
      name: indented_blocks,
      text: "
        b: a
          @mv a b
  
        a:
          @touch F
          @touch a
  
        d: c
          @rm c
  
        c: b
          @mv b c
      ",
      tokens: (
        Identifier:"b",
        Colon,
        Whitespace,
        Identifier:"a",
        Eol,
        Indent,
        Text:"@mv a b",
        Eol,
        Dedent,
        Eol,
        Identifier:"a",
        Colon,
        Eol,
        Indent,
        Text:"@touch F",
        Eol,
        Whitespace:"  ",
        Text:"@touch a",
        Eol,
        Dedent,
        Eol,
        Identifier:"d",
        Colon,
        Whitespace,
        Identifier:"c",
        Eol,
        Indent,
        Text:"@rm c",
        Eol,
        Dedent,
        Eol,
        Identifier:"c",
        Colon,
        Whitespace,
        Identifier:"b",
        Eol,
        Indent,
        Text:"@mv b c",
        Eol,
        Dedent
      ),
    }
  
    test! {
      name: interpolation_empty,
      text: "hello:\n echo {{}}",
      tokens: (
        Identifier:"hello",
        Colon,
        Eol,
        Indent:" ",
        Text:"echo ",
        InterpolationStart,
        InterpolationEnd,
        Dedent,
      ),
    }
  
    test! {
      name: interpolation_expression,
      text: "hello:\n echo {{`echo hello` + `echo goodbye`}}",
      tokens: (
        Identifier:"hello",
        Colon,
        Eol,
        Indent:" ",
        Text:"echo ",
        InterpolationStart,
        Backtick:"`echo hello`",
        Whitespace,
        Plus,
        Whitespace,
        Backtick:"`echo goodbye`",
        InterpolationEnd,
        Dedent,
      ),
    }
  
    test! {
      name: interpolation_raw_multiline_string,
      text: "hello:\n echo {{'\n'}}",
      tokens: (
        Identifier:"hello",
        Colon,
        Eol,
        Indent:" ",
        Text:"echo ",
        InterpolationStart,
        StringToken:"'\n'",
        InterpolationEnd,
        Dedent,
      ),
    }
  
    test! {
      name: tokenize_names,
      text: "
        foo
        bar-bob
        b-bob_asdfAAAA
        test123
      ",
      tokens: (
        Identifier:"foo",
        Eol,
        Identifier:"bar-bob",
        Eol,
        Identifier:"b-bob_asdfAAAA",
        Eol,
        Identifier:"test123",
        Eol,
      ),
    }
  
    test! {
      name: tokenize_indented_line,
      text: "foo:\n a",
      tokens: (
        Identifier:"foo",
        Colon,
        Eol,
        Indent:" ",
        Text:"a",
        Dedent,
      ),
    }
  
    test! {
      name: tokenize_indented_block,
      text: "
        foo:
          a
          b
          c
      ",
      tokens: (
        Identifier:"foo",
        Colon,
        Eol,
        Indent,
        Text:"a",
        Eol,
        Whitespace:"  ",
        Text:"b",
        Eol,
        Whitespace:"  ",
        Text:"c",
        Eol,
        Dedent,
      ),
    }
  
    test! {
      name: tokenize_strings,
      text: r#"a = "'a'" + '"b"' + "'c'" + '"d"'#echo hello"#,
      tokens: (
        Identifier:"a",
        Whitespace,
        Equals,
        Whitespace,
        StringToken:"\"'a'\"",
        Whitespace,
        Plus,
        Whitespace,
        StringToken:"'\"b\"'",
        Whitespace,
        Plus,
        Whitespace,
        StringToken:"\"'c'\"",
        Whitespace,
        Plus,
        Whitespace,
        StringToken:"'\"d\"'",
        Comment:"#echo hello",
      )
    }
  
    test! {
      name: tokenize_recipe_interpolation_eol,
      text: "
        foo: # some comment
         {{hello}}
      ",
      tokens: (
        Identifier:"foo",
        Colon,
        Whitespace,
        Comment:"# some comment",
        Eol,
        Indent:" ",
        InterpolationStart,
        Identifier:"hello",
        InterpolationEnd,
        Eol,
        Dedent
      ),
    }
  
    test! {
      name: tokenize_recipe_interpolation_eof,
      text: "foo: # more comments
   {{hello}}
  # another comment
  ",
      tokens: (
        Identifier:"foo",
        Colon,
        Whitespace,
        Comment:"# more comments",
        Eol,
        Indent:" ",
        InterpolationStart,
        Identifier:"hello",
        InterpolationEnd,
        Eol,
        Dedent,
        Comment:"# another comment",
        Eol,
      ),
    }
  
    test! {
      name: tokenize_recipe_complex_interpolation_expression,
      text: "foo: #lol\n {{a + b + \"z\" + blarg}}",
      tokens: (
        Identifier:"foo",
        Colon,
        Whitespace:" ",
        Comment:"#lol",
        Eol,
        Indent:" ",
        InterpolationStart,
        Identifier:"a",
        Whitespace,
        Plus,
        Whitespace,
        Identifier:"b",
        Whitespace,
        Plus,
        Whitespace,
        StringToken:"\"z\"",
        Whitespace,
        Plus,
        Whitespace,
        Identifier:"blarg",
        InterpolationEnd,
        Dedent,
      ),
    }
  
    test! {
      name: tokenize_recipe_multiple_interpolations,
      text: "foo:,#ok\n {{a}}0{{b}}1{{c}}",
      tokens: (
        Identifier:"foo",
        Colon,
        Comma,
        Comment:"#ok",
        Eol,
        Indent:" ",
        InterpolationStart,
        Identifier:"a",
        InterpolationEnd,
        Text:"0",
        InterpolationStart,
        Identifier:"b",
        InterpolationEnd,
        Text:"1",
        InterpolationStart,
        Identifier:"c",
        InterpolationEnd,
        Dedent,
  
      ),
    }
  
    test! {
      name: tokenize_junk,
      text: "
        bob
  
        hello blah blah blah : a b c #whatever
      ",
      tokens: (
        Identifier:"bob",
        Eol,
        Eol,
        Identifier:"hello",
        Whitespace,
        Identifier:"blah",
        Whitespace,
        Identifier:"blah",
        Whitespace,
        Identifier:"blah",
        Whitespace,
        Colon,
        Whitespace,
        Identifier:"a",
        Whitespace,
        Identifier:"b",
        Whitespace,
        Identifier:"c",
        Whitespace,
        Comment:"#whatever",
        Eol,
      )
    }
  
    test! {
      name: tokenize_continued_indent,
      text: "
        hello:
          foo
  
          bar
  
        baz
      ",
      tokens: (
        Identifier:"hello",
        Colon,
        Eol,
        Indent,
        Text:"foo",
        Eol,
        Eol,
        Whitespace:"  ",
        Text:"bar",
        Eol,
        Dedent,
        Eol,
        Identifier:"baz",
        Eol,
      ),
    }
  
    test! {
      name: tokenize_empty_lines,
      text: "
  
        # this does something
        hello:
          asdf
          bsdf
  
          csdf
  
          dsdf # whatever
  
        # yolo
      ",
      tokens: (
        Eol,
        Comment:"# this does something",
        Eol,
        Identifier:"hello",
        Colon,
        Eol,
        Indent,
        Text:"asdf",
        Eol,
        Whitespace:"  ",
        Text:"bsdf",
        Eol,
        Eol,
        Whitespace:"  ",
        Text:"csdf",
        Eol,
        Eol,
        Whitespace:"  ",
        Text:"dsdf # whatever",
        Eol,
        Dedent,
        Eol,
        Comment:"# yolo",
        Eol,
      ),
    }
  
    test! {
      name: tokenize_comment_before_variable,
      text: "
        #
        A='1'
        echo:
          echo {{A}}
      ",
      tokens: (
        Comment:"#",
        Eol,
        Identifier:"A",
        Equals,
        StringToken:"'1'",
        Eol,
        Identifier:"echo",
        Colon,
        Eol,
        Indent,
        Text:"echo ",
        InterpolationStart,
        Identifier:"A",
        InterpolationEnd,
        Eol,
        Dedent,
      ),
    }
  
    test! {
      name: tokenize_interpolation_backticks,
      text: "hello:\n echo {{`echo hello` + `echo goodbye`}}",
      tokens: (
        Identifier:"hello",
        Colon,
        Eol,
        Indent:" ",
        Text:"echo ",
        InterpolationStart,
        Backtick:"`echo hello`",
        Whitespace,
        Plus,
        Whitespace,
        Backtick:"`echo goodbye`",
        InterpolationEnd,
        Dedent
      ),
    }
  
    test! {
      name: tokenize_empty_interpolation,
      text: "hello:\n echo {{}}",
      tokens: (
        Identifier:"hello",
        Colon,
        Eol,
        Indent:" ",
        Text:"echo ",
        InterpolationStart,
        InterpolationEnd,
        Dedent,
      ),
    }
  
    test! {
      name: tokenize_assignment_backticks,
      text: "a = `echo hello` + `echo goodbye`",
      tokens: (
        Identifier:"a",
        Whitespace,
        Equals,
        Whitespace,
        Backtick:"`echo hello`",
        Whitespace,
        Plus,
        Whitespace,
        Backtick:"`echo goodbye`",
      ),
    }
  
    test! {
      name: tokenize_multiple,
      text: "
  
        hello:
          a
          b
  
          c
  
          d
  
        # hello
        bob:
          frank
         \t
      ",
      tokens: (
        Eol,
        Identifier:"hello",
        Colon,
        Eol,
        Indent,
        Text:"a",
        Eol,
        Whitespace:"  ",
        Text:"b",
        Eol,
        Eol,
        Whitespace:"  ",
        Text:"c",
        Eol,
        Eol,
        Whitespace:"  ",
        Text:"d",
        Eol,
        Dedent,
        Eol,
        Comment:"# hello",
        Eol,
        Identifier:"bob",
        Colon,
        Eol,
        Indent:"  ",
        Text:"frank",
        Eol,
        Eol,
        Dedent,
      ),
    }
  
    test! {
      name: tokenize_comment,
      text: "a:=#",
      tokens: (
        Identifier:"a",
        ColonEquals,
        Comment:"#",
      ),
    }
  
    test! {
      name: tokenize_comment_with_bang,
      text: "a:=#foo!",
      tokens: (
        Identifier:"a",
        ColonEquals,
        Comment:"#foo!",
      ),
    }
  
    test! {
      name: tokenize_order,
      text: "
        b: a
          @mv a b
  
        a:
          @touch F
          @touch a
  
        d: c
          @rm c
  
        c: b
          @mv b c
      ",
      tokens: (
        Identifier:"b",
        Colon,
        Whitespace,
        Identifier:"a",
        Eol,
        Indent,
        Text:"@mv a b",
        Eol,
        Dedent,
        Eol,
        Identifier:"a",
        Colon,
        Eol,
        Indent,
        Text:"@touch F",
        Eol,
        Whitespace:"  ",
        Text:"@touch a",
        Eol,
        Dedent,
        Eol,
        Identifier:"d",
        Colon,
        Whitespace,
        Identifier:"c",
        Eol,
        Indent,
        Text:"@rm c",
        Eol,
        Dedent,
        Eol,
        Identifier:"c",
        Colon,
        Whitespace,
        Identifier:"b",
        Eol,
        Indent,
        Text:"@mv b c",
        Eol,
        Dedent,
      ),
    }
  
    test! {
      name: tokenize_parens,
      text: "((())) ()abc(+",
      tokens: (
        ParenL,
        ParenL,
        ParenL,
        ParenR,
        ParenR,
        ParenR,
        Whitespace,
        ParenL,
        ParenR,
        Identifier:"abc",
        ParenL,
        Plus,
      ),
    }
  
    test! {
      name: crlf_newline,
      text: "#\r\n#asdf\r\n",
      tokens: (
        Comment:"#",
        Eol:"\r\n",
        Comment:"#asdf",
        Eol:"\r\n",
      ),
    }
  
    test! {
      name: multiple_recipes,
      text: "a:\n  foo\nb:",
      tokens: (
        Identifier:"a",
        Colon,
        Eol,
        Indent:"  ",
        Text:"foo",
        Eol,
        Dedent,
        Identifier:"b",
        Colon,
      ),
    }
  
    test! {
      name:   brackets,
      text:   "[][]",
      tokens: (BracketL, BracketR, BracketL, BracketR),
    }
  
    test! {
      name:   open_delimiter_eol,
      text:   "[\n](\n){\n}",
      tokens: (
        BracketL, Whitespace:"\n", BracketR,
        ParenL, Whitespace:"\n", ParenR,
        BraceL, Whitespace:"\n", BraceR
      ),
    }
  
    test! {
      name:   format_string_empty,
      text:   "f''",
      tokens: (
        Identifier: "f",
        StringToken: "''",
      ),
    }
  
    test! {
      name:   format_string_identifier,
      text:   "f'{{foo}}'",
      tokens: (
        Identifier: "f",
        FormatStringStart: "'{{",
        Identifier: "foo",
        FormatStringEnd: "}}'",
      ),
    }
  
    test! {
      name:   format_string_continue,
      text:   "f'{{foo}}bar{{baz}}'",
      tokens: (
        Identifier: "f",
        FormatStringStart: "'{{",
        Identifier: "foo",
        FormatStringContinue: "}}bar{{",
        Identifier: "baz",
        FormatStringEnd: "}}'",
      ),
    }
  
    test! {
      name:   format_string_whitespace,
      text:   "f '{{foo}}'",
      tokens: (
        Identifier: "f",
        Whitespace,
        StringToken: "'{{foo}}'",
      ),
    }
  
    test! {
      name:   format_string_wrong_identifier,
      text:   "g'{{foo}}'",
      tokens: (
        Identifier: "g",
        StringToken: "'{{foo}}'",
      ),
    }
  
    test! {
      name:   format_string_followed_by_recipe,
      text:   "foo := f'{{'foo'}}{{'bar'}}'\nbar:",
      tokens: (
        Identifier: "foo",
        Whitespace: " ",
        ColonEquals: ":=",
        Whitespace: " ",
        Identifier: "f",
        FormatStringStart: "'{{",
        StringToken: "'foo'",
        FormatStringContinue: "}}{{",
        StringToken: "'bar'",
        FormatStringEnd: "}}'",
        Eol: "\n",
        Identifier: "bar",
        Colon,
      ),
    }
  
    test! {
      name:   indented_format_string_followed_by_recipe,
      text:   "foo := f'''{{'foo'}}{{'bar'}}'''\nbar:",
      tokens: (
        Identifier: "foo",
        Whitespace: " ",
        ColonEquals: ":=",
        Whitespace: " ",
        Identifier: "f",
        FormatStringStart: "'''{{",
        StringToken: "'foo'",
        FormatStringContinue: "}}{{",
        StringToken: "'bar'",
        FormatStringEnd: "}}'''",
        Eol: "\n",
        Identifier: "bar",
        Colon,
      ),
    }
  
    test! {
      name:   tokenize_recipe_after_body,
      text:   "foo:\n echo FOO\n\nfoo:",
      tokens: (
        Identifier: "foo",
        Colon,
        Eol,
        Indent: " ",
        Text: "echo FOO",
        Eol,
        Dedent,
        Eol,
        Identifier: "foo",
        Colon,
      ),
    }
  
    error! {
      name:  tokenize_space_then_tab,
      input: "a:
   0
   1
  \t2
  ",
      offset: 9,
      line:   3,
      column: 0,
      width:  1,
      kind:   InconsistentLeadingWhitespace{expected: " ", found: "\t"},
    }
  
    error! {
      name:  tokenize_tabs_then_tab_space,
      input: "a:
  \t\t0
  \t\t 1
  \t  2
  ",
      offset: 12,
      line:   3,
      column: 0,
      width:  3,
      kind:   InconsistentLeadingWhitespace{expected: "\t\t", found: "\t  "},
    }
  
    error! {
      name:   tokenize_unknown,
      input:  "%",
      offset: 0,
      line:   0,
      column: 0,
      width:  1,
      kind:   UnknownStartOfToken { start: '%'},
    }
  
    error! {
      name:   unterminated_string_with_escapes,
      input:  r#"a = "\n\t\r\"\\"#,
      offset: 4,
      line:   0,
      column: 4,
      width:  1,
      kind:   UnterminatedString,
    }
  
    error! {
      name:   unterminated_raw_string,
      input:  "r a='asdf",
      offset: 4,
      line:   0,
      column: 4,
      width:  1,
      kind:   UnterminatedString,
    }
  
    error! {
      name:   unterminated_interpolation,
      input:  "foo:\n echo {{
    ",
      offset: 11,
      line:   1,
      column: 6,
      width:  2,
      kind:   UnterminatedInterpolation,
    }
  
    error! {
      name:   unterminated_backtick,
      input:  "`echo",
      offset: 0,
      line:   0,
      column: 0,
      width:  1,
      kind:   UnterminatedBacktick,
    }
  
    error! {
      name:   unpaired_carriage_return,
      input:  "foo\rbar",
      offset: 3,
      line:   0,
      column: 3,
      width:  1,
      kind:   UnpairedCarriageReturn,
    }
  
    error! {
      name:   invalid_name_start_dash,
      input:  "-foo",
      offset: 0,
      line:   0,
      column: 0,
      width:  1,
      kind:   UnknownStartOfToken{ start: '-'},
    }
  
    error! {
      name:   invalid_name_start_digit,
      input:  "0foo",
      offset: 0,
      line:   0,
      column: 0,
      width:  1,
      kind:   UnknownStartOfToken { start: '0' },
    }
  
    error! {
      name:   unterminated_string,
      input:  r#"a = ""#,
      offset: 4,
      line:   0,
      column: 4,
      width:  1,
      kind:   UnterminatedString,
    }
  
    error! {
      name:   mixed_leading_whitespace_recipe,
      input:  "a:\n\t echo hello",
      offset: 3,
      line:   1,
      column: 0,
      width:  2,
      kind:   MixedLeadingWhitespace{whitespace: "\t "},
    }
  
    error! {
      name:   mixed_leading_whitespace_normal,
      input:  "a\n\t echo hello",
      offset: 2,
      line:   1,
      column: 0,
      width:  2,
      kind:   MixedLeadingWhitespace{whitespace: "\t "},
    }
  
    error! {
      name:   mixed_leading_whitespace_indent,
      input:  "a\n foo\n \tbar",
      offset: 7,
      line:   2,
      column: 0,
      width:  2,
      kind:   MixedLeadingWhitespace{whitespace: " \t"},
    }
  
    error! {
      name:   bad_dedent,
      input:  "a\n foo\n   bar\n  baz",
      offset: 14,
      line:   3,
      column: 0,
      width:  2,
      kind:   InconsistentLeadingWhitespace{expected: "   ", found: "  "},
    }
  
    error! {
      name:   unclosed_interpolation_delimiter,
      input:  "a:\n echo {{ foo",
      offset: 9,
      line:   1,
      column: 6,
      width:  2,
      kind:   UnterminatedInterpolation,
    }
  
    error! {
      name:   unexpected_character_after_at,
      input:  "@%",
      offset: 1,
      line:   0,
      column: 1,
      width:  1,
      kind:   UnknownStartOfToken { start: '%'},
    }
  
    error! {
      name:   mismatched_closing_brace,
      input:  "(]",
      offset: 1,
      line:   0,
      column: 1,
      width:  0,
      kind:   MismatchedClosingDelimiter {
        open:      Delimiter::Paren,
        close:     Delimiter::Bracket,
        open_line: 0,
      },
    }
  
    error! {
      name:   ampersand_eof,
      input:  "&",
      offset: 1,
      line:   0,
      column: 1,
      width:  0,
      kind:   UnexpectedEndOfToken {
        expected: vec!['&'],
      },
    }
  
    error! {
      name:   ampersand_unexpected,
      input:  "&%",
      offset: 1,
      line:   0,
      column: 1,
      width:  1,
      kind:   UnexpectedCharacter {
        expected: vec!['&'],
      },
    }
  
    error! {
      name:   bang_eof,
      input:  "!",
      offset: 1,
      line:   0,
      column: 1,
      width:  0,
      kind:   UnexpectedEndOfToken {
        expected: vec!['=', '~'],
      },
    }
  
    error! {
      name:   unclosed_parenthesis_in_interpolation,
      input:  "a:\n echo {{foo(}}",
      offset:  15,
      line:   1,
      column: 12,
      width:  0,
      kind:   MismatchedClosingDelimiter {
        close: Delimiter::Brace,
        open: Delimiter::Paren,
        open_line: 1,
      },
    }
  
    #[test]
    fn presume_error() {
      let compile_error = Lexer::new("justfile".as_ref(), "!")
        .presume('-')
        .unwrap_err();
      assert_matches!(
        compile_error.token,
        Token {
          offset: 0,
          line: 0,
          column: 0,
          length: 0,
          src: "!",
          kind: Unspecified,
          path: _,
        }
      );
      assert_matches!(&*compile_error.kind,
          Internal { message }
          if message == "Lexer presumed character `-`"
      );
  
      assert_eq!(
        Error::Compile { compile_error }
          .color_display(Color::never())
          .to_string(),
        "error: Internal error, this may indicate a bug in just: Lexer presumed character `-`
  consider filing an issue: https://github.com/casey/just/issues/new
   ——▶ justfile:1:1
    │
  1 │ !
    │ ^"
      );
    }
  }
  