content: // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
  // for details. All rights reserved. Use of this source code is governed by a
  // BSD-style license that can be found in the LICENSE file.
  
  part of scanner;
  
  abstract class Scanner {
    Token tokenize();
  
    factory Scanner(SourceFile file, {bool includeComments: false}) {
      if (file is Utf8BytesSourceFile) {
        return new Utf8BytesScanner(file, includeComments: includeComments);
      } else {
        return new StringScanner(file, includeComments: includeComments);
      }
    }
  }
  
  abstract class AbstractScanner implements Scanner {
    final bool includeComments;
  
    /**
     * The string offset for the next token that will be created.
     *
     * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values
     * are different. One string character can be encoded using multiple UTF-8
     * bytes.
     */
    int tokenStart = -1;
  
    /**
     * A pointer to the token stream created by this scanner. The first token
     * is a special token and not part of the source file. This is an
     * implementation detail to avoids special cases in the scanner. This token
     * is not exposed to clients of the scanner, which are expected to invoke
     * [firstToken] to access the token stream.
     */
    final Token tokens = new SymbolToken(EOF_INFO, -1);
  
    /**
     * A pointer to the last scanned token.
     */
    Token tail;
  
    /**
     * The source file that is being scanned. This field can be [:null:].
     * If the source file is available, the scanner assigns its [:lineStarts:] and
     * [:length:] fields at the end of [tokenize].
     */
    final SourceFile file;
  
    final List<int> lineStarts = <int>[0];
  
    AbstractScanner(this.file, this.includeComments) {
      this.tail = this.tokens;
    }
  
    /**
     * Advances and returns the next character.
     *
     * If the next character is non-ASCII, then the returned value depends on the
     * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while
     * the [StringScanner] returns a UTF-16 code unit.
     *
     * The scanner ensures that [advance] is not invoked after it returned [$EOF].
     * This allows implementations to omit bound checks if the data structure ends
     * with '0'.
     */
    int advance();
  
    /**
     * Returns the current unicode character.
     *
     * If the current character is ASCII, then it is returned unchanged.
     *
     * The [Utf8BytesScanner] decodes the next unicode code point starting at the
     * current position. Note that every unicode character is returned as a single
     * code point, that is, for '\u{1d11e}' it returns 119070, and the following
     * [advance] returns the next character.
     *
     * The [StringScanner] returns the current character unchanged, which might
     * be a surrogate character. In the case of '\u{1d11e}', it returns the first
     * code unit 55348, and the following [advance] returns the second code unit
     * 56606.
     *
     * Invoking [currentAsUnicode] multiple times is safe, i.e.,
     * [:currentAsUnicode(next) == currentAsUnicode(currentAsUnicode(next)):].
     */
    int currentAsUnicode(int next);
  
    /**
     * Returns the character at the next poisition. Like in [advance], the
     * [Utf8BytesScanner] returns a UTF-8 byte, while the [StringScanner] returns
     * a UTF-16 code unit.
     */
    int peek();
  
    /**
     * Notifies the scanner that unicode characters were detected in either a
     * comment or a string literal between [startScanOffset] and the current
     * scan offset.
     */
    void handleUnicode(int startScanOffset);
  
    /**
     * Returns the current scan offset.
     *
     * In the [Utf8BytesScanner] this is the offset into the byte list, in the
     * [StringScanner] the offset in the source string.
     */
    int get scanOffset;
  
    /**
     * Returns the current string offset.
     *
     * In the [StringScanner] this is identical to the [scanOffset]. In the
     * [Utf8BytesScanner] it is computed based on encountered UTF-8 characters.
     */
    int get stringOffset;
  
    /**
     * Returns the first token scanned by this [Scanner].
     */
    Token firstToken();
  
    /**
     * Returns the last token scanned by this [Scanner].
     */
    Token previousToken();
  
    /**
     * Notifies that a new token starts at current offset.
     */
    void beginToken() {
      tokenStart = stringOffset;
    }
  
    /**
     * Appends a substring from the scan offset [:start:] to the current
     * [:scanOffset:] plus the [:extraOffset:]. For example, if the current
     * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the
     * substring string [5,9).
     *
     * Note that [extraOffset] can only be used if the covered character(s) are
     * known to be ASCII.
     */
    void appendSubstringToken(PrecedenceInfo info, int start,
                              bool asciiOnly, [int extraOffset]);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendStringToken(PrecedenceInfo info, String value);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendPrecedenceToken(PrecedenceInfo info);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    int select(int choice, PrecedenceInfo yes, PrecedenceInfo no);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendKeywordToken(Keyword keyword);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendEofToken();
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendWhiteSpace(int next);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void lineFeedInMultiline();
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendBeginGroup(PrecedenceInfo info);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    int appendEndGroup(PrecedenceInfo info, int openKind);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendGt(PrecedenceInfo info);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendGtGt(PrecedenceInfo info);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void appendComment(start, bool asciiOnly);
  
    /** Documentation in subclass [ArrayBasedScanner]. */
    void discardOpenLt();
  
    // TODO(ahe): Move this class to implementation.
  
    Token tokenize() {
      int next = advance();
      while (!identical(next, $EOF)) {
        next = bigSwitch(next);
      }
      appendEofToken();
  
      if (file != null) {
        file.length = stringOffset;
        // One additional line start at the end, see [SourceFile.lineStarts].
        lineStarts.add(stringOffset + 1);
        file.lineStarts = lineStarts;
      }
  
      return firstToken();
    }
  
    int bigSwitch(int next) {
      beginToken();
      if (identical(next, $SPACE) || identical(next, $TAB)
          || identical(next, $LF) || identical(next, $CR)) {
        appendWhiteSpace(next);
        next = advance();
        // Sequences of spaces are common, so advance through them fast.
        while (identical(next, $SPACE)) {
          // We don't invoke [:appendWhiteSpace(next):] here for efficiency,
          // assuming that it does not do anything for space characters.
          next = advance();
        }
        return next;
      }
  
      if ($a <= next && next <= $z) {
        if (identical($r, next)) {
          return tokenizeRawStringKeywordOrIdentifier(next);
        }
        return tokenizeKeywordOrIdentifier(next, true);
      }
  
      if (($A <= next && next <= $Z) ||
          identical(next, $_) ||
          identical(next, $$)) {
        return tokenizeIdentifier(next, scanOffset, true);
      }
  
      if (identical(next, $LT)) {
        return tokenizeLessThan(next);
      }
  
      if (identical(next, $GT)) {
        return tokenizeGreaterThan(next);
      }
  
      if (identical(next, $EQ)) {
        return tokenizeEquals(next);
      }
  
      if (identical(next, $BANG)) {
        return tokenizeExclamation(next);
      }
  
      if (identical(next, $PLUS)) {
        return tokenizePlus(next);
      }
  
      if (identical(next, $MINUS)) {
        return tokenizeMinus(next);
      }
  
      if (identical(next, $STAR)) {
        return tokenizeMultiply(next);
      }
  
      if (identical(next, $PERCENT)) {
        return tokenizePercent(next);
      }
  
      if (identical(next, $AMPERSAND)) {
        return tokenizeAmpersand(next);
      }
  
      if (identical(next, $BAR)) {
        return tokenizeBar(next);
      }
  
      if (identical(next, $CARET)) {
        return tokenizeCaret(next);
      }
  
      if (identical(next, $OPEN_SQUARE_BRACKET)) {
        return tokenizeOpenSquareBracket(next);
      }
  
      if (identical(next, $TILDE)) {
        return tokenizeTilde(next);
      }
  
      if (identical(next, $BACKSLASH)) {
        appendPrecedenceToken(BACKSLASH_INFO);
        return advance();
      }
  
      if (identical(next, $HASH)) {
        return tokenizeTag(next);
      }
  
      if (identical(next, $OPEN_PAREN)) {
        appendBeginGroup(OPEN_PAREN_INFO);
        return advance();
      }
  
      if (identical(next, $CLOSE_PAREN)) {
        return appendEndGroup(CLOSE_PAREN_INFO, OPEN_PAREN_TOKEN);
      }
  
      if (identical(next, $COMMA)) {
        appendPrecedenceToken(COMMA_INFO);
        return advance();
      }
  
      if (identical(next, $COLON)) {
        appendPrecedenceToken(COLON_INFO);
        return advance();
      }
  
      if (identical(next, $SEMICOLON)) {
        appendPrecedenceToken(SEMICOLON_INFO);
        // Type parameters and arguments cannot contain semicolon.
        discardOpenLt();
        return advance();
      }
  
      if (identical(next, $QUESTION)) {
        appendPrecedenceToken(QUESTION_INFO);
        return advance();
      }
  
      if (identical(next, $CLOSE_SQUARE_BRACKET)) {
        return appendEndGroup(CLOSE_SQUARE_BRACKET_INFO,
                              OPEN_SQUARE_BRACKET_TOKEN);
      }
  
      if (identical(next, $BACKPING)) {
        appendPrecedenceToken(BACKPING_INFO);
        return advance();
      }
  
      if (identical(next, $OPEN_CURLY_BRACKET)) {
        appendBeginGroup(OPEN_CURLY_BRACKET_INFO);
        return advance();
      }
  
      if (identical(next, $CLOSE_CURLY_BRACKET)) {
        return appendEndGroup(CLOSE_CURLY_BRACKET_INFO,
                              OPEN_CURLY_BRACKET_TOKEN);
      }
  
      if (identical(next, $SLASH)) {
        return tokenizeSlashOrComment(next);
      }
  
      if (identical(next, $AT)) {
        return tokenizeAt(next);
      }
  
      if (identical(next, $DQ) || identical(next, $SQ)) {
        return tokenizeString(next, scanOffset, false);
      }
  
      if (identical(next, $PERIOD)) {
        return tokenizeDotsOrNumber(next);
      }
  
      if (identical(next, $0)) {
        return tokenizeHexOrNumber(next);
      }
  
      // TODO(ahe): Would a range check be faster?
      if (identical(next, $1) || identical(next, $2) || identical(next, $3)
          || identical(next, $4) ||  identical(next, $5) || identical(next, $6)
          || identical(next, $7) || identical(next, $8) || identical(next, $9)) {
        return tokenizeNumber(next);
      }
  
      if (identical(next, $EOF)) {
        return $EOF;
      }
      if (next < 0x1f) {
        return error("unexpected character $next");
      }
  
      next = currentAsUnicode(next);
  
      // The following are non-ASCII characters.
  
      if (identical(next, $NBSP)) {
        appendWhiteSpace(next);
        return advance();
      }
  
      return error("unexpected unicode character $next");
    }
  
    int tokenizeTag(int next) {
      // # or #!.*[\n\r]
      if (scanOffset == 0) {
        if (identical(peek(), $BANG)) {
          int start = scanOffset + 1;
          bool asciiOnly = true;
          do {
            next = advance();
            if (next > 127) asciiOnly = false;
          } while (!identical(next, $LF) &&
                   !identical(next, $CR) &&
                   !identical(next, $EOF));
          if (!asciiOnly) handleUnicode(start);
          return next;
        }
      }
      appendPrecedenceToken(HASH_INFO);
      return advance();
    }
  
    int tokenizeTilde(int next) {
      // ~ ~/ ~/=
      next = advance();
      if (identical(next, $SLASH)) {
        return select($EQ, TILDE_SLASH_EQ_INFO, TILDE_SLASH_INFO);
      } else {
        appendPrecedenceToken(TILDE_INFO);
        return next;
      }
    }
  
    int tokenizeOpenSquareBracket(int next) {
      // [ [] []=
      next = advance();
      if (identical(next, $CLOSE_SQUARE_BRACKET)) {
        Token token = previousToken();
        if (token is KeywordToken &&
            identical(token.keyword.syntax, 'operator')) {
          return select($EQ, INDEX_EQ_INFO, INDEX_INFO);
        }
      }
      appendBeginGroup(OPEN_SQUARE_BRACKET_INFO);
      return next;
    }
  
    int tokenizeCaret(int next) {
      // ^ ^=
      return select($EQ, CARET_EQ_INFO, CARET_INFO);
    }
  
    int tokenizeBar(int next) {
      // | || |=
      next = advance();
      if (identical(next, $BAR)) {
        appendPrecedenceToken(BAR_BAR_INFO);
        return advance();
      } else if (identical(next, $EQ)) {
        appendPrecedenceToken(BAR_EQ_INFO);
        return advance();
      } else {
        appendPrecedenceToken(BAR_INFO);
        return next;
      }
    }
  
    int tokenizeAmpersand(int next) {
      // && &= &
      next = advance();
      if (identical(next, $AMPERSAND)) {
        appendPrecedenceToken(AMPERSAND_AMPERSAND_INFO);
        return advance();
      } else if (identical(next, $EQ)) {
        appendPrecedenceToken(AMPERSAND_EQ_INFO);
        return advance();
      } else {
        appendPrecedenceToken(AMPERSAND_INFO);
        return next;
      }
    }
  
    int tokenizePercent(int next) {
      // % %=
      return select($EQ, PERCENT_EQ_INFO, PERCENT_INFO);
    }
  
    int tokenizeMultiply(int next) {
      // * *=
      return select($EQ, STAR_EQ_INFO, STAR_INFO);
    }
  
    int tokenizeMinus(int next) {
      // - -- -=
      next = advance();
      if (identical(next, $MINUS)) {
        appendPrecedenceToken(MINUS_MINUS_INFO);
        return advance();
      } else if (identical(next, $EQ)) {
        appendPrecedenceToken(MINUS_EQ_INFO);
        return advance();
      } else {
        appendPrecedenceToken(MINUS_INFO);
        return next;
      }
    }
  
    int tokenizePlus(int next) {
      // + ++ +=
      next = advance();
      if (identical($PLUS, next)) {
        appendPrecedenceToken(PLUS_PLUS_INFO);
        return advance();
      } else if (identical($EQ, next)) {
        appendPrecedenceToken(PLUS_EQ_INFO);
        return advance();
      } else {
        appendPrecedenceToken(PLUS_INFO);
        return next;
      }
    }
  
    int tokenizeExclamation(int next) {
      // ! !=
      // !== is kept for user-friendly error reporting.
  
      next = advance();
      if (identical(next, $EQ)) {
        return select($EQ, BANG_EQ_EQ_INFO, BANG_EQ_INFO);
      }
      appendPrecedenceToken(BANG_INFO);
      return next;
    }
  
    int tokenizeEquals(int next) {
      // = == =>
      // === is kept for user-friendly error reporting.
  
      // Type parameters and arguments cannot contain any token that
      // starts with '='.
      discardOpenLt();
  
      next = advance();
      if (identical(next, $EQ)) {
        return select($EQ, EQ_EQ_EQ_INFO, EQ_EQ_INFO);
      } else if (identical(next, $GT)) {
        appendPrecedenceToken(FUNCTION_INFO);
        return advance();
      }
      appendPrecedenceToken(EQ_INFO);
      return next;
    }
  
    int tokenizeGreaterThan(int next) {
      // > >= >> >>=
      next = advance();
      if (identical($EQ, next)) {
        appendPrecedenceToken(GT_EQ_INFO);
        return advance();
      } else if (identical($GT, next)) {
        next = advance();
        if (identical($EQ, next)) {
          appendPrecedenceToken(GT_GT_EQ_INFO);
          return advance();
        } else {
          appendGtGt(GT_GT_INFO);
          return next;
        }
      } else {
        appendGt(GT_INFO);
        return next;
      }
    }
  
    int tokenizeLessThan(int next) {
      // < <= << <<=
      next = advance();
      if (identical($EQ, next)) {
        appendPrecedenceToken(LT_EQ_INFO);
        return advance();
      } else if (identical($LT, next)) {
        return select($EQ, LT_LT_EQ_INFO, LT_LT_INFO);
      } else {
        appendBeginGroup(LT_INFO);
        return next;
      }
    }
  
    int tokenizeNumber(int next) {
      int start = scanOffset;
      while (true) {
        next = advance();
        if ($0 <= next && next <= $9) {
          continue;
        } else if (identical(next, $e) || identical(next, $E)) {
          return tokenizeFractionPart(next, start);
        } else {
          if (identical(next, $PERIOD)) {
            int nextnext = peek();
            if ($0 <= nextnext && nextnext <= $9) {
              return tokenizeFractionPart(advance(), start);
            }
          }
          appendSubstringToken(INT_INFO, start, true);
          return next;
        }
      }
    }
  
    int tokenizeHexOrNumber(int next) {
      int x = peek();
      if (identical(x, $x) || identical(x, $X)) {
        return tokenizeHex(next);
      }
      return tokenizeNumber(next);
    }
  
    int tokenizeHex(int next) {
      int start = scanOffset;
      next = advance(); // Advance past the $x or $X.
      bool hasDigits = false;
      while (true) {
        next = advance();
        if (($0 <= next && next <= $9)
            || ($A <= next && next <= $F)
            || ($a <= next && next <= $f)) {
          hasDigits = true;
        } else {
          if (!hasDigits) {
            return error("hex digit expected");
          }
          appendSubstringToken(HEXADECIMAL_INFO, start, true);
          return next;
        }
      }
    }
  
    int tokenizeDotsOrNumber(int next) {
      int start = scanOffset;
      next = advance();
      if (($0 <= next && next <= $9)) {
        return tokenizeFractionPart(next, start);
      } else if (identical($PERIOD, next)) {
        return select($PERIOD, PERIOD_PERIOD_PERIOD_INFO, PERIOD_PERIOD_INFO);
      } else {
        appendPrecedenceToken(PERIOD_INFO);
        return next;
      }
    }
  
    int tokenizeFractionPart(int next, int start) {
      bool done = false;
      bool hasDigit = false;
      LOOP: while (!done) {
        if ($0 <= next && next <= $9) {
          hasDigit = true;
        } else if (identical($e, next) || identical($E, next)) {
          hasDigit = true;
          next = tokenizeExponent(advance());
          done = true;
          continue LOOP;
        } else {
          done = true;
          continue LOOP;
        }
        next = advance();
      }
      if (!hasDigit) {
        // Reduce offset, we already advanced to the token past the period.
        appendSubstringToken(INT_INFO, start, true, -1);
  
        // TODO(ahe): Wrong offset for the period. Cannot call beginToken because
        // the scanner already advanced past the period.
        if (identical($PERIOD, next)) {
          return select($PERIOD, PERIOD_PERIOD_PERIOD_INFO, PERIOD_PERIOD_INFO);
        }
        appendPrecedenceToken(PERIOD_INFO);
        return next;
      }
      appendSubstringToken(DOUBLE_INFO, start, true);
      return next;
    }
  
    int tokenizeExponent(int next) {
      if (identical(next, $PLUS) || identical(next, $MINUS)) {
        next = advance();
      }
      bool hasDigits = false;
      while (true) {
        if ($0 <= next && next <= $9) {
          hasDigits = true;
        } else {
          if (!hasDigits) {
            return error("digit expected");
          }
          return next;
        }
        next = advance();
      }
    }
  
    int tokenizeSlashOrComment(int next) {
      int start = scanOffset;
      next = advance();
      if (identical($STAR, next)) {
        return tokenizeMultiLineComment(next, start);
      } else if (identical($SLASH, next)) {
        return tokenizeSingleLineComment(next, start);
      } else if (identical($EQ, next)) {
        appendPrecedenceToken(SLASH_EQ_INFO);
        return advance();
      } else {
        appendPrecedenceToken(SLASH_INFO);
        return next;
      }
    }
  
    int tokenizeSingleLineComment(int next, int start) {
      bool asciiOnly = true;
      while (true) {
        next = advance();
        if (next > 127) asciiOnly = false;
        if (identical($LF, next) ||
            identical($CR, next) ||
            identical($EOF, next)) {
          if (!asciiOnly) handleUnicode(start);
          appendComment(start, asciiOnly);
          return next;
        }
      }
    }
  
  
    int tokenizeMultiLineComment(int next, int start) {
      bool asciiOnlyComment = true; // Track if the entire comment is ASCII.
      bool asciiOnlyLines = true; // Track ASCII since the last handleUnicode.
      int unicodeStart = start;
      int nesting = 1;
      next = advance();
      while (true) {
        if (identical($EOF, next)) {
          if (!asciiOnlyLines) handleUnicode(unicodeStart);
          appendStringToken(BAD_INPUT_INFO, "unterminated multi-line comment");
          return next;
        } else if (identical($STAR, next)) {
          next = advance();
          if (identical($SLASH, next)) {
            --nesting;
            if (0 == nesting) {
              if (!asciiOnlyLines) handleUnicode(unicodeStart);
              next = advance();
              appendComment(start, asciiOnlyComment);
              return next;
            } else {
              next = advance();
            }
          }
        } else if (identical($SLASH, next)) {
          next = advance();
          if (identical($STAR, next)) {
            next = advance();
            ++nesting;
          }
        } else if (identical(next, $LF)) {
          if (!asciiOnlyLines) {
            // Synchronize the string offset in the utf8 scanner.
            handleUnicode(unicodeStart);
            asciiOnlyLines = true;
            unicodeStart = scanOffset;
          }
          lineFeedInMultiline();
          next = advance();
        } else {
          if (next > 127) {
            asciiOnlyLines = false;
            asciiOnlyComment = false;
          }
          next = advance();
        }
      }
    }
  
    int tokenizeRawStringKeywordOrIdentifier(int next) {
      // [next] is $r.
      int nextnext = peek();
      if (identical(nextnext, $DQ) || identical(nextnext, $SQ)) {
        int start = scanOffset;
        next = advance();
        return tokenizeString(next, start, true);
      }
      return tokenizeKeywordOrIdentifier(next, true);
    }
  
    int tokenizeKeywordOrIdentifier(int next, bool allowDollar) {
      KeywordState state = KeywordState.KEYWORD_STATE;
      int start = scanOffset;
      while (state != null && $a <= next && next <= $z) {
        state = state.next(next);
        next = advance();
      }
      if (state == null || state.keyword == null) {
        return tokenizeIdentifier(next, start, allowDollar);
      }
      if (($A <= next && next <= $Z) ||
          ($0 <= next && next <= $9) ||
          identical(next, $_) ||
          identical(next, $$)) {
        return tokenizeIdentifier(next, start, allowDollar);
      } else {
        appendKeywordToken(state.keyword);
        return next;
      }
    }
  
    /**
     * [allowDollar] can exclude '$', which is not allowed as part of a string
     * interpolation identifier.
     */
    int tokenizeIdentifier(int next, int start, bool allowDollar) {
      while (true) {
        if (($a <= next && next <= $z) ||
            ($A <= next && next <= $Z) ||
            ($0 <= next && next <= $9) ||
            identical(next, $_) ||
            (identical(next, $$) && allowDollar)) {
          next = advance();
        } else {
          // Identifier ends here.
          if (start == scanOffset) {
            return error("expected identifier");
          } else {
            appendSubstringToken(IDENTIFIER_INFO, start, true);
          }
          return next;
        }
      }
    }
  
    int tokenizeAt(int next) {
      appendPrecedenceToken(AT_INFO);
      return advance();
    }
  
    int tokenizeString(int next, int start, bool raw) {
      int quoteChar = next;
      next = advance();
      if (identical(quoteChar, next)) {
        next = advance();
        if (identical(quoteChar, next)) {
          // Multiline string.
          return tokenizeMultiLineString(quoteChar, start, raw);
        } else {
          // Empty string.
          appendSubstringToken(STRING_INFO, start, true);
          return next;
        }
      }
      if (raw) {
        return tokenizeSingleLineRawString(next, quoteChar, start);
      } else {
        return tokenizeSingleLineString(next, quoteChar, start);
      }
    }
  
    /**
     * [next] is the first character after the quote.
     * [start] is the scanOffset of the quote.
     *
     * The token contains a substring of the source file, including the
     * string quotes, backslashes for escaping. For interpolated strings,
     * the parts before and after are separate tokens.
     *
     *   "a $b c"
     *
     * gives StringToken("a $), StringToken(b) and StringToken( c").
     */
    int tokenizeSingleLineString(int next, int quoteChar, int start) {
      bool asciiOnly = true;
      while (!identical(next, quoteChar)) {
        if (identical(next, $BACKSLASH)) {
          next = advance();
        } else if (identical(next, $$)) {
          if (!asciiOnly) handleUnicode(start);
          next = tokenizeStringInterpolation(start, asciiOnly);
          start = scanOffset;
          asciiOnly = true;
          continue;
        }
        if (next <= $CR
            && (identical(next, $LF) ||
                identical(next, $CR) ||
                identical(next, $EOF))) {
          if (!asciiOnly) handleUnicode(start);
          return error("unterminated string literal");
        }
        if (next > 127) asciiOnly = false;
        next = advance();
      }
      if (!asciiOnly) handleUnicode(start);
      // Advance past the quote character.
      next = advance();
      appendSubstringToken(STRING_INFO, start, asciiOnly);
      return next;
    }
  
    int tokenizeStringInterpolation(int start, bool asciiOnly) {
      appendSubstringToken(STRING_INFO, start, asciiOnly);
      beginToken(); // $ starts here.
      int next = advance();
      if (identical(next, $OPEN_CURLY_BRACKET)) {
        return tokenizeInterpolatedExpression(next);
      } else {
        return tokenizeInterpolatedIdentifier(next);
      }
    }
  
    int tokenizeInterpolatedExpression(int next) {
      appendBeginGroup(STRING_INTERPOLATION_INFO);
      beginToken(); // The expression starts here.
      next = advance(); // Move past the curly bracket.
      while (!identical(next, $EOF) && !identical(next, $STX)) {
        next = bigSwitch(next);
      }
      if (identical(next, $EOF)) return next;
      next = advance();  // Move past the $STX.
      beginToken(); // The string interpolation suffix starts here.
      return next;
    }
  
    int tokenizeInterpolatedIdentifier(int next) {
      appendPrecedenceToken(STRING_INTERPOLATION_IDENTIFIER_INFO);
      beginToken(); // The identifier starts here.
      next = tokenizeKeywordOrIdentifier(next, false);
      beginToken(); // The string interpolation suffix starts here.
      return next;
    }
  
    int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
      bool asciiOnly = true;
      next = advance(); // Advance past the quote.
      while (next != $EOF) {
        if (identical(next, quoteChar)) {
          if (!asciiOnly) handleUnicode(start);
          next = advance();
          appendSubstringToken(STRING_INFO, start, asciiOnly);
          return next;
        } else if (identical(next, $LF) || identical(next, $CR)) {
          if (!asciiOnly) handleUnicode(start);
          return error("unterminated string literal");
        } else if (next > 127) {
          asciiOnly = false;
        }
        next = advance();
      }
      if (!asciiOnly) handleUnicode(start);
      return error("unterminated string literal");
    }
  
    int tokenizeMultiLineRawString(int quoteChar, int start) {
      bool asciiOnlyString = true;
      bool asciiOnlyLine = true;
      int unicodeStart = start;
      int next = advance(); // Advance past the (last) quote (of three).
      outer: while (!identical(next, $EOF)) {
        while (!identical(next, quoteChar)) {
          if (identical(next, $LF)) {
            if (!asciiOnlyLine) {
              // Synchronize the string offset in the utf8 scanner.
              handleUnicode(unicodeStart);
              asciiOnlyLine = true;
              unicodeStart = scanOffset;
            }
            lineFeedInMultiline();
          } else if (next > 127) {
            asciiOnlyLine = false;
            asciiOnlyString = false;
          }
          next = advance();
          if (identical(next, $EOF)) break outer;
        }
        next = advance();
        if (identical(next, quoteChar)) {
          next = advance();
          if (identical(next, quoteChar)) {
            if (!asciiOnlyLine) handleUnicode(unicodeStart);
            next = advance();
            appendSubstringToken(STRING_INFO, start, asciiOnlyString);
            return next;
          }
        }
      }
      if (!asciiOnlyLine) handleUnicode(unicodeStart);
      return error("unterminated string literal");
    }
  
    int tokenizeMultiLineString(int quoteChar, int start, bool raw) {
      if (raw) return tokenizeMultiLineRawString(quoteChar, start);
      bool asciiOnlyString = true;
      bool asciiOnlyLine = true;
      int unicodeStart = start;
      int next = advance(); // Advance past the (last) quote (of three).
      while (!identical(next, $EOF)) {
        if (identical(next, $$)) {
          if (!asciiOnlyLine) handleUnicode(unicodeStart);
          next = tokenizeStringInterpolation(start, asciiOnlyString);
          start = scanOffset;
          unicodeStart = start;
          asciiOnlyString = true; // A new string token is created for the rest.
          asciiOnlyLine = true;
          continue;
        }
        if (identical(next, quoteChar)) {
          next = advance();
          if (identical(next, quoteChar)) {
            next = advance();
            if (identical(next, quoteChar)) {
              if (!asciiOnlyLine) handleUnicode(unicodeStart);
              next = advance();
              appendSubstringToken(STRING_INFO, start, asciiOnlyString);
              return next;
            }
          }
          continue;
        }
        if (identical(next, $BACKSLASH)) {
          next = advance();
          if (identical(next, $EOF)) break;
        }
        if (identical(next, $LF)) {
          if (!asciiOnlyLine) {
            // Synchronize the string offset in the utf8 scanner.
            handleUnicode(unicodeStart);
            asciiOnlyLine = true;
            unicodeStart = scanOffset;
          }
          lineFeedInMultiline();
        } else if (next > 127) {
          asciiOnlyString = false;
          asciiOnlyLine = false;
        }
        next = advance();
      }
      if (!asciiOnlyLine) handleUnicode(unicodeStart);
      return error("unterminated string literal");
    }
  
    int error(String message) {
      appendStringToken(BAD_INPUT_INFO, message);
      return advance(); // Ensure progress.
    }
  
    void unmatchedBeginGroup(BeginGroupToken begin) {
      String error = 'unmatched "${begin.stringValue}"';
      Token close =
          new StringToken.fromString(
              BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true);
  
      // We want to ensure that unmatched BeginGroupTokens are reported
      // as errors. However, the rest of the parser assumes the groups
      // are well-balanced and will never look at the endGroup
      // token. This is a nice property that allows us to skip quickly
      // over correct code. By inserting an additional error token in
      // the stream, we can keep ignoring endGroup tokens.
      //
      // [begin] --next--> [tail]
      // [begin] --endG--> [close] --next--> [next] --next--> [tail]
      //
      // This allows the parser to skip from [begin] via endGroup to [close] and
      // ignore the [close] token (assuming it's correct), then the error will be
      // reported when parsing the [next] token.
  
      Token next = new StringToken.fromString(
          BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true);
      begin.endGroup = close;
      close.next = next;
      next.next = begin.next;
    }
  }
  