tree-sitter-shellspec/src/scanner.c

#include "tree_sitter/array.h"
#include "tree_sitter/parser.h"

#include <assert.h>
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <wctype.h>

enum TokenType {
  HEREDOC_START,
  SIMPLE_HEREDOC_BODY,
  HEREDOC_BODY_BEGINNING,
  HEREDOC_CONTENT,
  HEREDOC_END,
  FILE_DESCRIPTOR,
  EMPTY_VALUE,
  CONCAT,
  VARIABLE_NAME,
  TEST_OPERATOR,
  REGEX,
  REGEX_NO_SLASH,
  REGEX_NO_SPACE,
  EXPANSION_WORD,
  EXTGLOB_PATTERN,
  BARE_DOLLAR,
  BRACE_START,
  IMMEDIATE_DOUBLE_HASH,
  EXTERNAL_EXPANSION_SYM_HASH,
  EXTERNAL_EXPANSION_SYM_BANG,
  EXTERNAL_EXPANSION_SYM_EQUAL,
  CLOSING_BRACE,
  CLOSING_BRACKET,
  HEREDOC_ARROW,
  HEREDOC_ARROW_DASH,
  NEWLINE,
  OPENING_PAREN,
  ESAC,
  ERROR_RECOVERY,
};

typedef Array(char) String;

typedef struct {
  bool is_raw;
  bool started;
  bool allows_indent;
  String delimiter;
  String current_leading_word;
} Heredoc;

#define heredoc_new()                                                          \
  {                                                                            \
      .is_raw = false,                                                         \
      .started = false,                                                        \
      .allows_indent = false,                                                  \
      .delimiter = array_new(),                                                \
      .current_leading_word = array_new(),                                     \
  };

typedef struct {
  uint8_t last_glob_paren_depth;
  bool ext_was_in_double_quote;
  bool ext_saw_outside_quote;
  Array(Heredoc) heredocs;
} Scanner;

static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }

static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }

static inline bool in_error_recovery(const bool *valid_symbols) {
  return valid_symbols[ERROR_RECOVERY];
}

static inline void reset_string(String *string) {
  if (string->size > 0) {
    memset(string->contents, 0, string->size);
    array_clear(string);
  }
}

static inline void reset_heredoc(Heredoc *heredoc) {
  heredoc->is_raw = false;
  heredoc->started = false;
  heredoc->allows_indent = false;
  reset_string(&heredoc->delimiter);
}

static inline void reset(Scanner *scanner) {
  for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
    reset_heredoc(array_get(&scanner->heredocs, i));
  }
}

static unsigned serialize(Scanner *scanner, char *buffer) {
  uint32_t size = 0;

  buffer[size++] = (char)scanner->last_glob_paren_depth;
  buffer[size++] = (char)scanner->ext_was_in_double_quote;
  buffer[size++] = (char)scanner->ext_saw_outside_quote;
  buffer[size++] = (char)scanner->heredocs.size;

  for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
    Heredoc *heredoc = array_get(&scanner->heredocs, i);
    if (size + 3 + sizeof(uint32_t) + heredoc->delimiter.size >=
        TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
      return 0;
    }

    buffer[size++] = (char)heredoc->is_raw;
    buffer[size++] = (char)heredoc->started;
    buffer[size++] = (char)heredoc->allows_indent;

    memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(uint32_t));
    size += sizeof(uint32_t);
    if (heredoc->delimiter.size > 0) {
      memcpy(&buffer[size], heredoc->delimiter.contents,
            heredoc->delimiter.size);
      size += heredoc->delimiter.size;
    }
  }
  return size;
}

static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
  if (length == 0) {
    // Fully clear heredocs to avoid stale stack entries after reset
    for (uint32_t i = 0; i < scanner->heredocs.size; i++) {
      Heredoc *h = array_get(&scanner->heredocs, i);
      array_delete(&h->current_leading_word);
      array_delete(&h->delimiter);
    }
    array_clear(&scanner->heredocs);
    reset(scanner);
  } else {
    uint32_t size = 0;
    scanner->last_glob_paren_depth = buffer[size++];
    scanner->ext_was_in_double_quote = buffer[size++];
    scanner->ext_saw_outside_quote = buffer[size++];
    uint32_t heredoc_count = (unsigned char)buffer[size++];
    for (uint32_t i = 0; i < heredoc_count; i++) {
      Heredoc *heredoc = NULL;
      if (i < scanner->heredocs.size) {
        heredoc = array_get(&scanner->heredocs, i);
      } else {
        Heredoc new_heredoc = heredoc_new();
        array_push(&scanner->heredocs, new_heredoc);
        heredoc = array_back(&scanner->heredocs);
      }

      heredoc->is_raw = buffer[size++];
      heredoc->started = buffer[size++];
      heredoc->allows_indent = buffer[size++];

      memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(uint32_t));
      size += sizeof(uint32_t);
      array_reserve(&heredoc->delimiter, heredoc->delimiter.size > 0 ? heredoc->delimiter.size : 1);

      if (heredoc->delimiter.size > 0) {
        memcpy(heredoc->delimiter.contents, &buffer[size],
              heredoc->delimiter.size);
        size += heredoc->delimiter.size;
        // Ensure NUL termination for safety
        if (heredoc->delimiter.contents[heredoc->delimiter.size - 1] != '\0') {
          array_reserve(&heredoc->delimiter, heredoc->delimiter.size + 1);
          heredoc->delimiter.contents[heredoc->delimiter.size] = '\0';
          heredoc->delimiter.size++;
        }
      }
    }
    assert(size == length);
  }
}

/**
  * Consume a "word" in POSIX parlance, and returns it unquoted.
  *
  * This is an approximate implementation that doesn't deal with any
  * POSIX-mandated substitution, and assumes the default value for
  * IFS.
  */
static bool advance_word(TSLexer *lexer, String *unquoted_word) {
  bool empty = true;

  int32_t quote = 0;
  if (lexer->lookahead == '\'' || lexer->lookahead == '"') {
    quote = lexer->lookahead;
    advance(lexer);
  }

  while (lexer->lookahead &&
        !(quote ? lexer->lookahead == quote || lexer->lookahead == '\r' ||
                      lexer->lookahead == '\n'
                : iswspace(lexer->lookahead))) {
    if (lexer->lookahead == '\\') {
      advance(lexer);
      if (!lexer->lookahead) {
        return false;
      }
    }
    empty = false;
    array_push(unquoted_word, lexer->lookahead);
    advance(lexer);
  }
  array_push(unquoted_word, '\0');

  if (quote && lexer->lookahead == quote) {
    advance(lexer);
  }

  return !empty;
}

static inline bool scan_bare_dollar(TSLexer *lexer) {
  while (iswspace(lexer->lookahead) && lexer->lookahead != '\n' &&
        !lexer->eof(lexer)) {
    skip(lexer);
  }

  if (lexer->lookahead == '$') {
    advance(lexer);
    lexer->result_symbol = BARE_DOLLAR;
    lexer->mark_end(lexer);
    return iswspace(lexer->lookahead) || lexer->eof(lexer) ||
          lexer->lookahead == '\"';
  }

  return false;
}

static bool scan_heredoc_start(Heredoc *heredoc, TSLexer *lexer) {
  while (iswspace(lexer->lookahead)) {
    skip(lexer);
  }

  lexer->result_symbol = HEREDOC_START;
  heredoc->is_raw = lexer->lookahead == '\'' || lexer->lookahead == '"' ||
                    lexer->lookahead == '\\';

  bool found_delimiter = advance_word(lexer, &heredoc->delimiter);
  if (!found_delimiter) {
    reset_string(&heredoc->delimiter);
    return false;
  }
  return found_delimiter;
}

static bool scan_heredoc_end_identifier(Heredoc *heredoc, TSLexer *lexer) {
  reset_string(&heredoc->current_leading_word);
  // Scan the first 'n' characters on this line, to see if they match the
  // heredoc delimiter
  int32_t size = 0;
  if (heredoc->delimiter.size > 0) {
    while (lexer->lookahead != '\0' && lexer->lookahead != '\n' &&
          (int32_t)*array_get(&heredoc->delimiter, size) == lexer->lookahead &&
          heredoc->current_leading_word.size < heredoc->delimiter.size) {
      array_push(&heredoc->current_leading_word, lexer->lookahead);
      advance(lexer);
      size++;
    }
  }
  array_push(&heredoc->current_leading_word, '\0');
  return heredoc->delimiter.size == 0
            ? false
            : strcmp(heredoc->current_leading_word.contents,
                      heredoc->delimiter.contents) == 0;
}

static bool scan_heredoc_content(Scanner *scanner, TSLexer *lexer,
                                enum TokenType middle_type,
                                enum TokenType end_type) {
  bool did_advance = false;
  Heredoc *heredoc = array_back(&scanner->heredocs);

  for (;;) {
    switch (lexer->lookahead) {
    case '\0': {
      if (lexer->eof(lexer) && did_advance) {
        reset_heredoc(heredoc);
        lexer->result_symbol = end_type;
        return true;
      }
      return false;
    }

    case '\\': {
      did_advance = true;
      advance(lexer);
      advance(lexer);
      break;
    }

    case '$': {
      if (heredoc->is_raw) {
        did_advance = true;
        advance(lexer);
        break;
      }
      if (did_advance) {
        lexer->mark_end(lexer);
        lexer->result_symbol = middle_type;
        heredoc->started = true;
        advance(lexer);
        if (iswalpha(lexer->lookahead) || lexer->lookahead == '{' ||
            lexer->lookahead == '(') {
          return true;
        }
        break;
      }
      if (middle_type == HEREDOC_BODY_BEGINNING &&
          lexer->get_column(lexer) == 0) {
        lexer->result_symbol = middle_type;
        heredoc->started = true;
        return true;
      }
      return false;
    }

    case '\n': {
      if (!did_advance) {
        skip(lexer);
      } else {
        advance(lexer);
      }
      did_advance = true;
      if (heredoc->allows_indent) {
        while (iswspace(lexer->lookahead)) {
          advance(lexer);
        }
      }
      lexer->result_symbol = heredoc->started ? middle_type : end_type;
      lexer->mark_end(lexer);
      if (scan_heredoc_end_identifier(heredoc, lexer)) {
        if (lexer->result_symbol == HEREDOC_END) {
          array_pop(&scanner->heredocs);
        }
        return true;
      }
      break;
    }

    default: {
      if (lexer->get_column(lexer) == 0) {
        // an alternative is to check the starting column of the
        // heredoc body and track that statefully
        while (iswspace(lexer->lookahead)) {
          if (did_advance) {
            advance(lexer);
          } else {
            skip(lexer);
          }
        }
        if (end_type != SIMPLE_HEREDOC_BODY) {
          lexer->result_symbol = middle_type;
          if (scan_heredoc_end_identifier(heredoc, lexer)) {
            return true;
          }
        }
        if (end_type == SIMPLE_HEREDOC_BODY) {
          lexer->result_symbol = end_type;
          lexer->mark_end(lexer);
          if (scan_heredoc_end_identifier(heredoc, lexer)) {
            return true;
          }
        }
      }
      did_advance = true;
      advance(lexer);
      break;
    }
    }
  }
}

static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
  if (valid_symbols[CONCAT] && !in_error_recovery(valid_symbols)) {
    if (!(lexer->lookahead == 0 || iswspace(lexer->lookahead) ||
          lexer->lookahead == '>' || lexer->lookahead == '<' ||
          lexer->lookahead == ')' || lexer->lookahead == '(' ||
          lexer->lookahead == ';' || lexer->lookahead == '&' ||
          lexer->lookahead == '|' ||
          (lexer->lookahead == '}' && valid_symbols[CLOSING_BRACE]) ||
          (lexer->lookahead == ']' && valid_symbols[CLOSING_BRACKET]))) {
      lexer->result_symbol = CONCAT;
      // So for a`b`, we want to return a concat. We check if the
      // 2nd backtick has whitespace after it, and if it does we
      // return concat.
      if (lexer->lookahead == '`') {
        lexer->mark_end(lexer);
        advance(lexer);
        while (lexer->lookahead != '`' && !lexer->eof(lexer)) {
          advance(lexer);
        }
        if (lexer->eof(lexer)) {
          return false;
        }
        if (lexer->lookahead == '`') {
          advance(lexer);
        }
        return iswspace(lexer->lookahead) || lexer->eof(lexer);
      }
      // strings w/ expansions that contains escaped quotes or
      // backslashes need this to return a concat
      if (lexer->lookahead == '\\') {
        lexer->mark_end(lexer);
        advance(lexer);
        if (lexer->lookahead == '"' || lexer->lookahead == '\'' ||
            lexer->lookahead == '\\') {
          return true;
        }
        if (lexer->eof(lexer)) {
          return false;
        }
      } else {
        return true;
      }
    }
    if (iswspace(lexer->lookahead) && valid_symbols[CLOSING_BRACE] &&
        !valid_symbols[EXPANSION_WORD]) {
      lexer->result_symbol = CONCAT;
      return true;
    }
  }

  if (valid_symbols[IMMEDIATE_DOUBLE_HASH] &&
      !in_error_recovery(valid_symbols)) {
    // advance two # and ensure not } after
    if (lexer->lookahead == '#') {
      lexer->mark_end(lexer);
      advance(lexer);
      if (lexer->lookahead == '#') {
        advance(lexer);
        if (lexer->lookahead != '}') {
          lexer->result_symbol = IMMEDIATE_DOUBLE_HASH;
          lexer->mark_end(lexer);
          return true;
        }
      }
    }
  }

  if (valid_symbols[EXTERNAL_EXPANSION_SYM_HASH] &&
      !in_error_recovery(valid_symbols)) {
    if (lexer->lookahead == '#' || lexer->lookahead == '=' ||
        lexer->lookahead == '!') {
      lexer->result_symbol =
          lexer->lookahead == '#'   ? EXTERNAL_EXPANSION_SYM_HASH
          : lexer->lookahead == '!' ? EXTERNAL_EXPANSION_SYM_BANG
                                    : EXTERNAL_EXPANSION_SYM_EQUAL;
      advance(lexer);
      lexer->mark_end(lexer);
      while (lexer->lookahead == '#' || lexer->lookahead == '=' ||
            lexer->lookahead == '!') {
        advance(lexer);
      }
      while (iswspace(lexer->lookahead)) {
        skip(lexer);
      }
      if (lexer->lookahead == '}') {
        return true;
      }
      return false;
    }
  }

  if (valid_symbols[EMPTY_VALUE]) {
    if (iswspace(lexer->lookahead) || lexer->eof(lexer) ||
        lexer->lookahead == ';' || lexer->lookahead == '&') {
      lexer->result_symbol = EMPTY_VALUE;
      return true;
    }
  }

  if ((valid_symbols[HEREDOC_BODY_BEGINNING] ||
      valid_symbols[SIMPLE_HEREDOC_BODY]) &&
      scanner->heredocs.size > 0 && !array_back(&scanner->heredocs)->started &&
      !in_error_recovery(valid_symbols)) {
    return scan_heredoc_content(scanner, lexer, HEREDOC_BODY_BEGINNING,
                                SIMPLE_HEREDOC_BODY);
  }

  if (valid_symbols[HEREDOC_END] && scanner->heredocs.size > 0) {
    Heredoc *heredoc = array_back(&scanner->heredocs);
    if (scan_heredoc_end_identifier(heredoc, lexer)) {
      array_delete(&heredoc->current_leading_word);
      array_delete(&heredoc->delimiter);
      array_pop(&scanner->heredocs);
      lexer->result_symbol = HEREDOC_END;
      return true;
    }
  }

  if (valid_symbols[HEREDOC_CONTENT] && scanner->heredocs.size > 0 &&
      array_back(&scanner->heredocs)->started &&
      !in_error_recovery(valid_symbols)) {
    return scan_heredoc_content(scanner, lexer, HEREDOC_CONTENT, HEREDOC_END);
  }

  if (valid_symbols[HEREDOC_START] && !in_error_recovery(valid_symbols) &&
      scanner->heredocs.size > 0) {
    return scan_heredoc_start(array_back(&scanner->heredocs), lexer);
  }

  if (valid_symbols[TEST_OPERATOR] && !valid_symbols[EXPANSION_WORD]) {
    while (iswspace(lexer->lookahead) && lexer->lookahead != '\n') {
      skip(lexer);
    }

    if (lexer->lookahead == '\\') {
      if (valid_symbols[EXTGLOB_PATTERN]) {
        goto extglob_pattern;
      }
      if (valid_symbols[REGEX_NO_SPACE]) {
        goto regex;
      }
      skip(lexer);

      if (lexer->eof(lexer)) {
        return false;
      }

      if (lexer->lookahead == '\r') {
        skip(lexer);
        if (lexer->lookahead == '\n') {
          skip(lexer);
        }
      } else if (lexer->lookahead == '\n') {
        skip(lexer);
      } else {
        return false;
      }

      while (iswspace(lexer->lookahead)) {
        skip(lexer);
      }
    }

    if (lexer->lookahead == '\n' && !valid_symbols[NEWLINE]) {
      skip(lexer);

      while (iswspace(lexer->lookahead)) {
        skip(lexer);
      }
    }

    if (lexer->lookahead == '-') {
      advance(lexer);

      bool advanced_once = false;
      while (iswalpha(lexer->lookahead)) {
        advanced_once = true;
        advance(lexer);
      }

      if (iswspace(lexer->lookahead) && advanced_once) {
        lexer->mark_end(lexer);
        advance(lexer);
        if (lexer->lookahead == '}' && valid_symbols[CLOSING_BRACE]) {
          if (valid_symbols[EXPANSION_WORD]) {
            lexer->mark_end(lexer);
            lexer->result_symbol = EXPANSION_WORD;
            return true;
          }
          return false;
        }
        lexer->result_symbol = TEST_OPERATOR;
        return true;
      }
      if (iswspace(lexer->lookahead) && valid_symbols[EXTGLOB_PATTERN]) {
        lexer->result_symbol = EXTGLOB_PATTERN;
        return true;
      }
    }

    if (valid_symbols[BARE_DOLLAR] && !in_error_recovery(valid_symbols) &&
        scan_bare_dollar(lexer)) {
      return true;
    }
  }

  if ((valid_symbols[VARIABLE_NAME] || valid_symbols[FILE_DESCRIPTOR] ||
      valid_symbols[HEREDOC_ARROW]) &&
      !valid_symbols[REGEX_NO_SLASH] && !in_error_recovery(valid_symbols)) {
    for (;;) {
      if ((lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
          lexer->lookahead == '\r' ||
          (lexer->lookahead == '\n' && !valid_symbols[NEWLINE])) &&
          !valid_symbols[EXPANSION_WORD]) {
        skip(lexer);
      } else if (lexer->lookahead == '\\') {
        skip(lexer);

        if (lexer->eof(lexer)) {
          lexer->mark_end(lexer);
          lexer->result_symbol = VARIABLE_NAME;
          return true;
        }

        if (lexer->lookahead == '\r') {
          skip(lexer);
        }
        if (lexer->lookahead == '\n') {
          skip(lexer);
        } else {
          if (lexer->lookahead == '\\' && valid_symbols[EXPANSION_WORD]) {
            goto expansion_word;
          }
          return false;
        }
      } else {
        break;
      }
    }

    // no '*', '@', '?', '-', '$', '0', '_'
    if (!valid_symbols[EXPANSION_WORD] &&
        (lexer->lookahead == '*' || lexer->lookahead == '@' ||
        lexer->lookahead == '?' || lexer->lookahead == '-' ||
        lexer->lookahead == '0' || lexer->lookahead == '_')) {
      lexer->mark_end(lexer);
      advance(lexer);
      if (lexer->lookahead == '=' || lexer->lookahead == '[' ||
          lexer->lookahead == ':' || lexer->lookahead == '-' ||
          lexer->lookahead == '%' || lexer->lookahead == '#' ||
          lexer->lookahead == '/') {
        return false;
      }
      if (valid_symbols[EXTGLOB_PATTERN] && iswspace(lexer->lookahead)) {
        lexer->mark_end(lexer);
        lexer->result_symbol = EXTGLOB_PATTERN;
        return true;
      }
    }

    if (valid_symbols[HEREDOC_ARROW] && lexer->lookahead == '<') {
      advance(lexer);
      if (lexer->lookahead == '<') {
        advance(lexer);
        if (lexer->lookahead == '-') {
          advance(lexer);
          Heredoc heredoc = heredoc_new();
          heredoc.allows_indent = true;
          array_push(&scanner->heredocs, heredoc);
          lexer->result_symbol = HEREDOC_ARROW_DASH;
        } else if (lexer->lookahead == '<' || lexer->lookahead == '=') {
          return false;
        } else {
          Heredoc heredoc = heredoc_new();
          array_push(&scanner->heredocs, heredoc);
          lexer->result_symbol = HEREDOC_ARROW;
        }
        return true;
      }
      return false;
    }

    bool is_number = true;
    if (iswdigit(lexer->lookahead)) {
      advance(lexer);
    } else if (iswalpha(lexer->lookahead) || lexer->lookahead == '_') {
      is_number = false;
      advance(lexer);
    } else {
      if (lexer->lookahead == '{') {
        goto brace_start;
      }
      if (valid_symbols[EXPANSION_WORD]) {
        goto expansion_word;
      }
      if (valid_symbols[EXTGLOB_PATTERN]) {
        goto extglob_pattern;
      }
      return false;
    }

    for (;;) {
      if (iswdigit(lexer->lookahead)) {
        advance(lexer);
      } else if (iswalpha(lexer->lookahead) || lexer->lookahead == '_') {
        is_number = false;
        advance(lexer);
      } else {
        break;
      }
    }

    if (is_number && valid_symbols[FILE_DESCRIPTOR] &&
        (lexer->lookahead == '>' || lexer->lookahead == '<')) {
      lexer->result_symbol = FILE_DESCRIPTOR;
      return true;
    }

    if (valid_symbols[VARIABLE_NAME]) {
      if (lexer->lookahead == '+') {
        lexer->mark_end(lexer);
        advance(lexer);
        if (lexer->lookahead == '=' || lexer->lookahead == ':' ||
            valid_symbols[CLOSING_BRACE]) {
          lexer->result_symbol = VARIABLE_NAME;
          return true;
        }
        return false;
      }
      if (lexer->lookahead == '/') {
        return false;
      }
      if (lexer->lookahead == '=' || lexer->lookahead == '[' ||
          (lexer->lookahead == ':' && !valid_symbols[CLOSING_BRACE] &&
          !valid_symbols
              [OPENING_PAREN]) || // TODO(amaanq): more cases for regular word
                                  // chars but not variable names for function
                                  // words, only handling : for now? #235
          lexer->lookahead == '%' ||
          (lexer->lookahead == '#' && !is_number) || lexer->lookahead == '@' ||
          (lexer->lookahead == '-' && valid_symbols[CLOSING_BRACE])) {
        lexer->mark_end(lexer);
        lexer->result_symbol = VARIABLE_NAME;
        return true;
      }

      if (lexer->lookahead == '?') {
        lexer->mark_end(lexer);
        advance(lexer);
        lexer->result_symbol = VARIABLE_NAME;
        return iswalpha(lexer->lookahead);
      }
    }

    return false;
  }

  if (valid_symbols[BARE_DOLLAR] && !in_error_recovery(valid_symbols) &&
      scan_bare_dollar(lexer)) {
    return true;
  }

regex:
  if ((valid_symbols[REGEX] || valid_symbols[REGEX_NO_SLASH] ||
      valid_symbols[REGEX_NO_SPACE]) &&
      !in_error_recovery(valid_symbols)) {
    if (valid_symbols[REGEX] || valid_symbols[REGEX_NO_SPACE]) {
      while (iswspace(lexer->lookahead)) {
        skip(lexer);
      }
    }

    if ((lexer->lookahead != '"' && lexer->lookahead != '\'') ||
        ((lexer->lookahead == '$' || lexer->lookahead == '\'') &&
        valid_symbols[REGEX_NO_SLASH]) ||
        (lexer->lookahead == '\'' && valid_symbols[REGEX_NO_SPACE])) {
      typedef struct {
        bool done;
        bool advanced_once;
        bool found_non_alnumdollarunderdash;
        bool last_was_escape;
        bool in_single_quote;
        uint32_t paren_depth;
        uint32_t bracket_depth;
        uint32_t brace_depth;
      } State;

      if (lexer->lookahead == '$' && valid_symbols[REGEX_NO_SLASH]) {
        lexer->mark_end(lexer);
        advance(lexer);
        if (lexer->lookahead == '(') {
          return false;
        }
      }

      lexer->mark_end(lexer);

      State state = {false, false, false, false, false, 0, 0, 0};
      while (!state.done) {
        if (state.in_single_quote) {
          if (lexer->lookahead == '\'') {
            state.in_single_quote = false;
            advance(lexer);
            lexer->mark_end(lexer);
          }
        }
        switch (lexer->lookahead) {
        case '\\':
          state.last_was_escape = true;
          break;
        case '\0':
          return false;
        case '(':
          state.paren_depth++;
          state.last_was_escape = false;
          break;
        case '[':
          state.bracket_depth++;
          state.last_was_escape = false;
          break;
        case '{':
          if (!state.last_was_escape) {
            state.brace_depth++;
          }
          state.last_was_escape = false;
          break;
        case ')':
          if (state.paren_depth == 0) {
            state.done = true;
          }
          state.paren_depth--;
          state.last_was_escape = false;
          break;
        case ']':
          if (state.bracket_depth == 0) {
            state.done = true;
          }
          state.bracket_depth--;
          state.last_was_escape = false;
          break;
        case '}':
          if (state.brace_depth == 0) {
            state.done = true;
          }
          state.brace_depth--;
          state.last_was_escape = false;
          break;
        case '\'':
          // Enter or exit a single-quoted string.
          state.in_single_quote = !state.in_single_quote;
          advance(lexer);
          state.advanced_once = true;
          state.last_was_escape = false;
          continue;
        default:
          state.last_was_escape = false;
          break;
        }

        if (!state.done) {
          if (valid_symbols[REGEX]) {
            bool was_space =
                !state.in_single_quote && iswspace(lexer->lookahead);
            advance(lexer);
            state.advanced_once = true;
            if (!was_space || state.paren_depth > 0) {
              lexer->mark_end(lexer);
            }
          } else if (valid_symbols[REGEX_NO_SLASH]) {
            if (lexer->lookahead == '/') {
              lexer->mark_end(lexer);
              lexer->result_symbol = REGEX_NO_SLASH;
              return state.advanced_once;
            }
            if (lexer->lookahead == '\\') {
              advance(lexer);
              state.advanced_once = true;
              if (!lexer->eof(lexer) && lexer->lookahead != '[' &&
                  lexer->lookahead != '/') {
                advance(lexer);
                lexer->mark_end(lexer);
              }
            } else {
              bool was_space =
                  !state.in_single_quote && iswspace(lexer->lookahead);
              advance(lexer);
              state.advanced_once = true;
              if (!was_space) {
                lexer->mark_end(lexer);
              }
            }
          } else if (valid_symbols[REGEX_NO_SPACE]) {
            if (lexer->lookahead == '\\') {
              state.found_non_alnumdollarunderdash = true;
              advance(lexer);
              if (!lexer->eof(lexer)) {
                advance(lexer);
              }
            } else if (lexer->lookahead == '$') {
              lexer->mark_end(lexer);
              advance(lexer);
              // do not parse a command
              // substitution
              if (lexer->lookahead == '(') {
                return false;
              }
              // end $ always means regex, e.g.
              // 99999999$
              if (iswspace(lexer->lookahead)) {
                lexer->result_symbol = REGEX_NO_SPACE;
                lexer->mark_end(lexer);
                return true;
              }
            } else {
              bool was_space =
                  !state.in_single_quote && iswspace(lexer->lookahead);
              if (was_space && state.paren_depth == 0) {
                lexer->mark_end(lexer);
                lexer->result_symbol = REGEX_NO_SPACE;
                return state.found_non_alnumdollarunderdash;
              }
              if (!iswalnum(lexer->lookahead) && lexer->lookahead != '$' &&
                  lexer->lookahead != '-' && lexer->lookahead != '_') {
                state.found_non_alnumdollarunderdash = true;
              }
              advance(lexer);
            }
          }
        }
      }

      lexer->result_symbol = valid_symbols[REGEX_NO_SLASH]   ? REGEX_NO_SLASH
                            : valid_symbols[REGEX_NO_SPACE] ? REGEX_NO_SPACE
                                                            : REGEX;
      if (valid_symbols[REGEX] && !state.advanced_once) {
        return false;
      }
      return true;
    }
  }

extglob_pattern:
  if (valid_symbols[EXTGLOB_PATTERN] && !in_error_recovery(valid_symbols)) {
    // first skip ws, then check for ? * + @ !
    while (iswspace(lexer->lookahead)) {
      skip(lexer);
    }

    if (lexer->lookahead == '?' || lexer->lookahead == '*' ||
        lexer->lookahead == '+' || lexer->lookahead == '@' ||
        lexer->lookahead == '!' || lexer->lookahead == '-' ||
        lexer->lookahead == ')' || lexer->lookahead == '\\' ||
        lexer->lookahead == '.' || lexer->lookahead == '[' ||
        (iswalpha(lexer->lookahead))) {
      if (lexer->lookahead == '\\') {
        advance(lexer);
        if ((iswspace(lexer->lookahead) || lexer->lookahead == '"') &&
            lexer->lookahead != '\r' && lexer->lookahead != '\n') {
          advance(lexer);
        } else {
          return false;
        }
      }

      if (lexer->lookahead == ')' && scanner->last_glob_paren_depth == 0) {
        lexer->mark_end(lexer);
        advance(lexer);

        if (iswspace(lexer->lookahead)) {
          return false;
        }
      }

      lexer->mark_end(lexer);
      bool was_non_alpha = !iswalpha(lexer->lookahead);
      if (lexer->lookahead != '[') {
        // no esac
        if (lexer->lookahead == 'e') {
          lexer->mark_end(lexer);
          advance(lexer);
          if (lexer->lookahead == 's') {
            advance(lexer);
            if (lexer->lookahead == 'a') {
              advance(lexer);
              if (lexer->lookahead == 'c') {
                advance(lexer);
                if (iswspace(lexer->lookahead)) {
                  return false;
                }
              }
            }
          }
        } else {
          advance(lexer);
        }
      }

      // -\w is just a word, find something else special
      if (lexer->lookahead == '-') {
        lexer->mark_end(lexer);
        advance(lexer);
        while (iswalnum(lexer->lookahead)) {
          advance(lexer);
        }

        if (lexer->lookahead == ')' || lexer->lookahead == '\\' ||
            lexer->lookahead == '.') {
          return false;
        }
        lexer->mark_end(lexer);
      }

      // case item -) or *)
      if (lexer->lookahead == ')' && scanner->last_glob_paren_depth == 0) {
        lexer->mark_end(lexer);
        advance(lexer);
        if (iswspace(lexer->lookahead)) {
          lexer->result_symbol = EXTGLOB_PATTERN;
          return was_non_alpha;
        }
      }

      if (iswspace(lexer->lookahead)) {
        lexer->mark_end(lexer);
        lexer->result_symbol = EXTGLOB_PATTERN;
        scanner->last_glob_paren_depth = 0;
        return true;
      }

      if (lexer->lookahead == '$') {
        lexer->mark_end(lexer);
        advance(lexer);
        if (lexer->lookahead == '{' || lexer->lookahead == '(') {
          lexer->result_symbol = EXTGLOB_PATTERN;
          return true;
        }
      }

      if (lexer->lookahead == '|') {
        lexer->mark_end(lexer);
        advance(lexer);
        lexer->result_symbol = EXTGLOB_PATTERN;
        return true;
      }

      if (!iswalnum(lexer->lookahead) && lexer->lookahead != '(' &&
          lexer->lookahead != '"' && lexer->lookahead != '[' &&
          lexer->lookahead != '?' && lexer->lookahead != '/' &&
          lexer->lookahead != '\\' && lexer->lookahead != '_' &&
          lexer->lookahead != '*') {
        return false;
      }

      typedef struct {
        bool done;
        bool saw_non_alphadot;
        uint32_t paren_depth;
        uint32_t bracket_depth;
        uint32_t brace_depth;
      } State;

      State state = {false, was_non_alpha, scanner->last_glob_paren_depth, 0,
                    0};
      while (!state.done) {
        switch (lexer->lookahead) {
        case '\0':
          return false;
        case '(':
          state.paren_depth++;
          break;
        case '[':
          state.bracket_depth++;
          break;
        case '{':
          state.brace_depth++;
          break;
        case ')':
          if (state.paren_depth == 0) {
            state.done = true;
          }
          state.paren_depth--;
          break;
        case ']':
          if (state.bracket_depth == 0) {
            state.done = true;
          }
          state.bracket_depth--;
          break;
        case '}':
          if (state.brace_depth == 0) {
            state.done = true;
          }
          state.brace_depth--;
          break;
        }

        if (lexer->lookahead == '|') {
          lexer->mark_end(lexer);
          advance(lexer);
          if (state.paren_depth == 0 && state.bracket_depth == 0 &&
              state.brace_depth == 0) {
            lexer->result_symbol = EXTGLOB_PATTERN;
            return true;
          }
        }

        if (!state.done) {
          bool was_space = iswspace(lexer->lookahead);
          if (lexer->lookahead == '$') {
            lexer->mark_end(lexer);
            if (!iswalpha(lexer->lookahead) && lexer->lookahead != '.' &&
                lexer->lookahead != '\\') {
              state.saw_non_alphadot = true;
            }
            advance(lexer);
            if (lexer->lookahead == '(' || lexer->lookahead == '{') {
              lexer->result_symbol = EXTGLOB_PATTERN;
              scanner->last_glob_paren_depth = state.paren_depth;
              return state.saw_non_alphadot;
            }
          }
          if (was_space) {
            lexer->mark_end(lexer);
            lexer->result_symbol = EXTGLOB_PATTERN;
            scanner->last_glob_paren_depth = 0;
            return state.saw_non_alphadot;
          }
          if (lexer->lookahead == '"') {
            lexer->mark_end(lexer);
            lexer->result_symbol = EXTGLOB_PATTERN;
            scanner->last_glob_paren_depth = 0;
            return state.saw_non_alphadot;
          }
          if (lexer->lookahead == '\\') {
            if (!iswalpha(lexer->lookahead) && lexer->lookahead != '.' &&
                lexer->lookahead != '\\') {
              state.saw_non_alphadot = true;
            }
            advance(lexer);
            if (iswspace(lexer->lookahead) || lexer->lookahead == '"') {
              advance(lexer);
            }
          } else {
            if (!iswalpha(lexer->lookahead) && lexer->lookahead != '.' &&
                lexer->lookahead != '\\') {
              state.saw_non_alphadot = true;
            }
            advance(lexer);
          }
          if (!was_space) {
            lexer->mark_end(lexer);
          }
        }
      }

      lexer->result_symbol = EXTGLOB_PATTERN;
      scanner->last_glob_paren_depth = 0;
      return state.saw_non_alphadot;
    }
    scanner->last_glob_paren_depth = 0;

    return false;
  }

expansion_word:
  if (valid_symbols[EXPANSION_WORD]) {
    bool advanced_once = false;
    bool advance_once_space = false;
    for (;;) {
      if (lexer->lookahead == '\"') {
        return false;
      }
      if (lexer->lookahead == '$') {
        lexer->mark_end(lexer);
        advance(lexer);
        if (lexer->lookahead == '{' || lexer->lookahead == '(' ||
            lexer->lookahead == '\'' || iswalnum(lexer->lookahead)) {
          lexer->result_symbol = EXPANSION_WORD;
          return advanced_once;
        }
        advanced_once = true;
      }

      if (lexer->lookahead == '}') {
        lexer->mark_end(lexer);
        lexer->result_symbol = EXPANSION_WORD;
        return advanced_once || advance_once_space;
      }

      if (lexer->lookahead == '(' && !(advanced_once || advance_once_space)) {
        lexer->mark_end(lexer);
        advance(lexer);
        while (lexer->lookahead != ')' && !lexer->eof(lexer)) {
          // if we find a $( or ${ assume this is valid and is
          // a garbage concatenation of some weird word + an
          // expansion
          // I wonder where this can fail
          if (lexer->lookahead == '$') {
            lexer->mark_end(lexer);
            advance(lexer);
            if (lexer->lookahead == '{' || lexer->lookahead == '(' ||
                lexer->lookahead == '\'' || iswalnum(lexer->lookahead)) {
              lexer->result_symbol = EXPANSION_WORD;
              return advanced_once;
            }
            advanced_once = true;
          } else {
            advanced_once = advanced_once || !iswspace(lexer->lookahead);
            advance_once_space =
                advance_once_space || iswspace(lexer->lookahead);
            advance(lexer);
          }
        }
        lexer->mark_end(lexer);
        if (lexer->lookahead == ')') {
          advanced_once = true;
          advance(lexer);
          lexer->mark_end(lexer);
          if (lexer->lookahead == '}') {
            return false;
          }
        } else {
          return false;
        }
      }

      if (lexer->lookahead == '\'') {
        return false;
      }

      if (lexer->eof(lexer)) {
        return false;
      }
      advanced_once = advanced_once || !iswspace(lexer->lookahead);
      advance_once_space = advance_once_space || iswspace(lexer->lookahead);
      advance(lexer);
    }
  }

brace_start:
  if (valid_symbols[BRACE_START] && !in_error_recovery(valid_symbols)) {
    while (iswspace(lexer->lookahead)) {
      skip(lexer);
    }

    if (lexer->lookahead != '{') {
      return false;
    }

    advance(lexer);
    lexer->mark_end(lexer);

    while (isdigit(lexer->lookahead)) {
      advance(lexer);
    }

    if (lexer->lookahead != '.') {
      return false;
    }
    advance(lexer);

    if (lexer->lookahead != '.') {
      return false;
    }
    advance(lexer);

    while (isdigit(lexer->lookahead)) {
      advance(lexer);
    }

    if (lexer->lookahead != '}') {
      return false;
    }

    lexer->result_symbol = BRACE_START;
    return true;
  }

  return false;
}

void *tree_sitter_shellspec_external_scanner_create() {
  Scanner *scanner = calloc(1, sizeof(Scanner));
  array_init(&scanner->heredocs);
  return scanner;
}

bool tree_sitter_shellspec_external_scanner_scan(void *payload, TSLexer *lexer,
                                                const bool *valid_symbols) {
  Scanner *scanner = (Scanner *)payload;
  return scan(scanner, lexer, valid_symbols);
}

unsigned tree_sitter_shellspec_external_scanner_serialize(void *payload,
                                                          char *state) {
  Scanner *scanner = (Scanner *)payload;
  return serialize(scanner, state);
}

void tree_sitter_shellspec_external_scanner_deserialize(void *payload,
                                                        const char *state,
                                                        unsigned length) {
  Scanner *scanner = (Scanner *)payload;
  deserialize(scanner, state, length);
}

void tree_sitter_shellspec_external_scanner_destroy(void *payload) {
  Scanner *scanner = (Scanner *)payload;
  for (size_t i = 0; i < scanner->heredocs.size; i++) {
    Heredoc *heredoc = array_get(&scanner->heredocs, i);
    array_delete(&heredoc->current_leading_word);
    array_delete(&heredoc->delimiter);
  }
  array_delete(&scanner->heredocs);
  free(scanner);
}