carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-09-27 19:00:11 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity
Files
b9bd7e1eff59e7f90a052d75fffcb123eaddb365
carbon-lang/explorer/syntax/lexer.lpp
T
Jon Ross-Perkins 39155d34ae Remove experimental continuation support (#2776) 
Removes `__continuation`, `__await`, and `__run`.

In part here, the discussion was that while the feature had been useful for validating the early explorer design, it's no longer needed for that role as the explorer is now quite robust. Continuations have been experimental and, at this point, don't have an owner pushing to a proposal.

The triggering factor is that, as we push to address fuzzer issues, I ran into a crash bug in this code; basically, `fn Main() -> i32 { __await; return 0; }`. When I mentioned this, the reaction seemed to trend towards removal of the feature.
2023-04-19 14:22:43 -07:00

452 lines
18 KiB
Plaintext
Raw Blame History

/*
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
Exceptions. See /LICENSE for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
%{
#include <cstdlib>
#include "common/check.h"
#include "common/error.h"
#include "explorer/syntax/lex_helper.h"
#include "explorer/syntax/lex_scan_helper.h"
#include "explorer/syntax/parse_and_lex_context.h"
#include "explorer/syntax/parser.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/FormatVariadic.h"
%}
/* Turn off legacy bits we don't need. */
%option noyywrap nounput nodefault
%option reentrant
/* Lexing a token immediately after consuming some whitespace. */
%s AFTER_WHITESPACE
/*
* Lexing a token immediately after consuming an operand-ending token:
* a closing bracket, identifier, or literal.
*/
%s AFTER_OPERAND
/* table-begin */
ABSTRACT "abstract"
ADDR "addr"
ALIAS "alias"
AMPERSAND "&"
AMPERSAND_EQUAL "&="
AND "and"
API "api"
ARROW "->"
AS "as"
AUTO "auto"
BASE "base"
BOOL "bool"
BREAK "break"
CARET "^"
CARET_EQUAL "^="
CASE "case"
CHOICE "choice"
CLASS "class"
COLON ":"
COLON_BANG ":!"
COMMA ","
CONSTRAINT "constraint"
CONTINUE "continue"
DEFAULT "default"
DESTRUCTOR "destructor"
DOUBLE_ARROW "=>"
ELSE "else"
EQUAL "="
EQUAL_EQUAL "=="
EXTENDS "extends"
EXTERNAL "external"
FALSE "false"
FN "fn"
FN_TYPE "__Fn"
FOR "for"
FORALL "forall"
GREATER ">"
GREATER_EQUAL ">="
GREATER_GREATER ">>"
GREATER_GREATER_EQUAL ">>="
IF "if"
IMPL "impl"
IMPLS "impls"
IMPORT "import"
IN "in"
INTERFACE "interface"
LEFT_CURLY_BRACE "{"
LEFT_PARENTHESIS "("
LEFT_SQUARE_BRACKET "["
LESS "<"
LESS_EQUAL "<="
LESS_LESS "<<"
LESS_LESS_EQUAL "<<="
LET "let"
LIBRARY "library"
MATCH "match"
MATCH_FIRST "__match_first"
MINUS "-"
MINUS_EQUAL "-="
MINUS_MINUS "--"
MIX "__mix"
MIXIN "__mixin"
NAMESPACE "namespace"
NOT "not"
NOT_EQUAL "!="
OR "or"
PACKAGE "package"
PERCENT "%"
PERCENT_EQUAL "%="
PERIOD "."
PIPE "|"
PIPE_EQUAL "|="
PLUS "+"
PLUS_EQUAL "+="
PLUS_PLUS "++"
RETURN "return"
RETURNED "returned"
RIGHT_CURLY_BRACE "}"
RIGHT_PARENTHESIS ")"
RIGHT_SQUARE_BRACKET "]"
SELF "Self"
SEMICOLON ";"
SLASH "/"
SLASH_EQUAL "/="
STAR_EQUAL "*="
STRING "String"
TEMPLATE "template"
THEN "then"
TRUE "true"
TYPE "type"
UNDERSCORE "_"
UNIMPL_EXAMPLE "__unimplemented_example_infix"
VAR "var"
VIRTUAL "virtual"
WHERE "where"
WHILE "while"
/* table-end */
/* This should be kept table-like, but isn't automatic due to spaces. */
identifier [A-Za-z_][A-Za-z0-9_]*
/* TODO: Remove Print special casing once we have variadics or overloads. */
intrinsic_identifier (Print|__intrinsic_[A-Za-z0-9_]*)
sized_type_literal [iuf][1-9][0-9]*
integer_literal [0-9]+
horizontal_whitespace [ \t\r]
whitespace [ \t\r\n]
one_line_comment \/\/[^\n]*\n
operand_start [(A-Za-z0-9_\"]
%%
%{
// Code run each time yylex is called.
// Begin with an empty token span starting where its previous end was.
context.current_token_position.step();
%}
/* table-begin */
{ABSTRACT} { return CARBON_SIMPLE_TOKEN(ABSTRACT); }
{ADDR} { return CARBON_SIMPLE_TOKEN(ADDR); }
{ALIAS} { return CARBON_SIMPLE_TOKEN(ALIAS); }
{AMPERSAND_EQUAL} { return CARBON_SIMPLE_TOKEN(AMPERSAND_EQUAL); }
{AMPERSAND} { return CARBON_SIMPLE_TOKEN(AMPERSAND); }
{AND} { return CARBON_SIMPLE_TOKEN(AND); }
{API} { return CARBON_SIMPLE_TOKEN(API); }
{ARROW} { return CARBON_SIMPLE_TOKEN(ARROW); }
{AS} { return CARBON_SIMPLE_TOKEN(AS); }
{AUTO} { return CARBON_SIMPLE_TOKEN(AUTO); }
{BASE} { return CARBON_SIMPLE_TOKEN(BASE); }
{BOOL} { return CARBON_SIMPLE_TOKEN(BOOL); }
{BREAK} { return CARBON_SIMPLE_TOKEN(BREAK); }
{CARET_EQUAL} { return CARBON_SIMPLE_TOKEN(CARET_EQUAL); }
{CARET} { return CARBON_SIMPLE_TOKEN(CARET); }
{CASE} { return CARBON_SIMPLE_TOKEN(CASE); }
{CHOICE} { return CARBON_SIMPLE_TOKEN(CHOICE); }
{CLASS} { return CARBON_SIMPLE_TOKEN(CLASS); }
{COLON_BANG} { return CARBON_SIMPLE_TOKEN(COLON_BANG); }
{COLON} { return CARBON_SIMPLE_TOKEN(COLON); }
{COMMA} { return CARBON_SIMPLE_TOKEN(COMMA); }
{CONSTRAINT} { return CARBON_SIMPLE_TOKEN(CONSTRAINT); }
{CONTINUE} { return CARBON_SIMPLE_TOKEN(CONTINUE); }
{DEFAULT} { return CARBON_SIMPLE_TOKEN(DEFAULT); }
{DESTRUCTOR} { return CARBON_SIMPLE_TOKEN(DESTRUCTOR); }
{DOUBLE_ARROW} { return CARBON_SIMPLE_TOKEN(DOUBLE_ARROW); }
{ELSE} { return CARBON_SIMPLE_TOKEN(ELSE); }
{EQUAL_EQUAL} { return CARBON_SIMPLE_TOKEN(EQUAL_EQUAL); }
{EQUAL} { return CARBON_SIMPLE_TOKEN(EQUAL); }
{EXTENDS} { return CARBON_SIMPLE_TOKEN(EXTENDS); }
{EXTERNAL} { return CARBON_SIMPLE_TOKEN(EXTERNAL); }
{FALSE} { return CARBON_SIMPLE_TOKEN(FALSE); }
{FN_TYPE} { return CARBON_SIMPLE_TOKEN(FN_TYPE); }
{FN} { return CARBON_SIMPLE_TOKEN(FN); }
{FORALL} { return CARBON_SIMPLE_TOKEN(FORALL); }
{FOR} { return CARBON_SIMPLE_TOKEN(FOR); }
{GREATER_EQUAL} { return CARBON_SIMPLE_TOKEN(GREATER_EQUAL); }
{GREATER_GREATER_EQUAL} { return CARBON_SIMPLE_TOKEN(GREATER_GREATER_EQUAL); }
{GREATER_GREATER} { return CARBON_SIMPLE_TOKEN(GREATER_GREATER); }
{GREATER} { return CARBON_SIMPLE_TOKEN(GREATER); }
{IF} { return CARBON_SIMPLE_TOKEN(IF); }
{IMPLS} { return CARBON_SIMPLE_TOKEN(IMPLS); }
{IMPL} { return CARBON_SIMPLE_TOKEN(IMPL); }
{IMPORT} { return CARBON_SIMPLE_TOKEN(IMPORT); }
{INTERFACE} { return CARBON_SIMPLE_TOKEN(INTERFACE); }
{IN} { return CARBON_SIMPLE_TOKEN(IN); }
{LEFT_CURLY_BRACE} { return CARBON_SIMPLE_TOKEN(LEFT_CURLY_BRACE); }
{LEFT_PARENTHESIS} { return CARBON_SIMPLE_TOKEN(LEFT_PARENTHESIS); }
{LEFT_SQUARE_BRACKET} { return CARBON_SIMPLE_TOKEN(LEFT_SQUARE_BRACKET); }
{LESS_EQUAL} { return CARBON_SIMPLE_TOKEN(LESS_EQUAL); }
{LESS_LESS_EQUAL} { return CARBON_SIMPLE_TOKEN(LESS_LESS_EQUAL); }
{LESS_LESS} { return CARBON_SIMPLE_TOKEN(LESS_LESS); }
{LESS} { return CARBON_SIMPLE_TOKEN(LESS); }
{LET} { return CARBON_SIMPLE_TOKEN(LET); }
{LIBRARY} { return CARBON_SIMPLE_TOKEN(LIBRARY); }
{MATCH_FIRST} { return CARBON_SIMPLE_TOKEN(MATCH_FIRST); }
{MATCH} { return CARBON_SIMPLE_TOKEN(MATCH); }
{MINUS_EQUAL} { return CARBON_SIMPLE_TOKEN(MINUS_EQUAL); }
{MINUS_MINUS} { return CARBON_SIMPLE_TOKEN(MINUS_MINUS); }
{MINUS} { return CARBON_SIMPLE_TOKEN(MINUS); }
{MIXIN} { return CARBON_SIMPLE_TOKEN(MIXIN); }
{MIX} { return CARBON_SIMPLE_TOKEN(MIX); }
{NAMESPACE} { return CARBON_SIMPLE_TOKEN(NAMESPACE); }
{NOT_EQUAL} { return CARBON_SIMPLE_TOKEN(NOT_EQUAL); }
{NOT} { return CARBON_SIMPLE_TOKEN(NOT); }
{OR} { return CARBON_SIMPLE_TOKEN(OR); }
{PACKAGE} { return CARBON_SIMPLE_TOKEN(PACKAGE); }
{PERCENT_EQUAL} { return CARBON_SIMPLE_TOKEN(PERCENT_EQUAL); }
{PERCENT} { return CARBON_SIMPLE_TOKEN(PERCENT); }
{PERIOD} { return CARBON_SIMPLE_TOKEN(PERIOD); }
{PIPE_EQUAL} { return CARBON_SIMPLE_TOKEN(PIPE_EQUAL); }
{PIPE} { return CARBON_SIMPLE_TOKEN(PIPE); }
{PLUS_EQUAL} { return CARBON_SIMPLE_TOKEN(PLUS_EQUAL); }
{PLUS_PLUS} { return CARBON_SIMPLE_TOKEN(PLUS_PLUS); }
{PLUS} { return CARBON_SIMPLE_TOKEN(PLUS); }
{RETURNED} { return CARBON_SIMPLE_TOKEN(RETURNED); }
{RETURN} { return CARBON_SIMPLE_TOKEN(RETURN); }
{SELF} { return CARBON_SIMPLE_TOKEN(SELF); }
{SEMICOLON} { return CARBON_SIMPLE_TOKEN(SEMICOLON); }
{SLASH_EQUAL} { return CARBON_SIMPLE_TOKEN(SLASH_EQUAL); }
{SLASH} { return CARBON_SIMPLE_TOKEN(SLASH); }
{STAR_EQUAL} { return CARBON_SIMPLE_TOKEN(STAR_EQUAL); }
{STRING} { return CARBON_SIMPLE_TOKEN(STRING); }
{TEMPLATE} { return CARBON_SIMPLE_TOKEN(TEMPLATE); }
{THEN} { return CARBON_SIMPLE_TOKEN(THEN); }
{TRUE} { return CARBON_SIMPLE_TOKEN(TRUE); }
{TYPE} { return CARBON_SIMPLE_TOKEN(TYPE); }
{UNDERSCORE} { return CARBON_SIMPLE_TOKEN(UNDERSCORE); }
{UNIMPL_EXAMPLE} { return CARBON_SIMPLE_TOKEN(UNIMPL_EXAMPLE); }
{VAR} { return CARBON_SIMPLE_TOKEN(VAR); }
{VIRTUAL} { return CARBON_SIMPLE_TOKEN(VIRTUAL); }
{WHERE} { return CARBON_SIMPLE_TOKEN(WHERE); }
{WHILE} { return CARBON_SIMPLE_TOKEN(WHILE); }
/* table-end */
/* More modern Bisons provide make_EOF. */
<<EOF>> { return CARBON_SIMPLE_TOKEN(END_OF_FILE); }
{RIGHT_PARENTHESIS} {
BEGIN(AFTER_OPERAND);
return CARBON_SIMPLE_TOKEN(RIGHT_PARENTHESIS);
}
{RIGHT_CURLY_BRACE} {
BEGIN(AFTER_OPERAND);
return CARBON_SIMPLE_TOKEN(RIGHT_CURLY_BRACE);
}
{RIGHT_SQUARE_BRACKET} {
BEGIN(AFTER_OPERAND);
return CARBON_SIMPLE_TOKEN(RIGHT_SQUARE_BRACKET);
}
/*
* For a `*` operator, we look at whitespace and local context to determine the
* arity and fixity. There are two ways to write a binary operator:
*
* 1) Whitespace on both sides.
* 2) Whitespace on neither side, and the previous token is considered to be
* the end of an operand, and the next token is considered to be the start
* of an operand.
*
* Otherwise, the operator is unary, but we also check for whitespace to help
* the parser enforce the rule that whitespace is not permitted between the
* operator and its operand, leading to three more cases:
*
* 3) Whitespace before (but implicitly not after, because that would give a
* longer match and hit case 1): this can only be a prefix operator.
* 4) Whitespace after and not before: this can only be a postfix operator.
* 5) No whitespace on either side (otherwise the longest match would take us
* to case 4): this is a unary operator and could be either prefix or
* postfix.
*/
/* `*` operator case 1: */
<AFTER_WHITESPACE>"*"{whitespace}+ {
BEGIN(AFTER_WHITESPACE);
return CARBON_SIMPLE_TOKEN(BINARY_STAR);
}
/* `*` operator case 2: */
<AFTER_OPERAND>"*"/{operand_start} { return CARBON_SIMPLE_TOKEN(BINARY_STAR); }
/* `*` operator case 3: */
<AFTER_WHITESPACE>"*" { return CARBON_SIMPLE_TOKEN(PREFIX_STAR); }
/* `*` operator case 4: */
<INITIAL,AFTER_OPERAND>"*"{whitespace}+ {
BEGIN(AFTER_WHITESPACE);
return CARBON_SIMPLE_TOKEN(POSTFIX_STAR);
}
/* `*` operator case 5: */
<INITIAL,AFTER_OPERAND>"*" { return CARBON_SIMPLE_TOKEN(UNARY_STAR); }
{sized_type_literal} {
BEGIN(AFTER_OPERAND);
return CARBON_ARG_TOKEN(sized_type_literal, yytext);
}
{intrinsic_identifier} {
BEGIN(AFTER_OPERAND);
Carbon::ErrorOr<Carbon::IntrinsicExpression::Intrinsic> intrinsic =
Carbon::IntrinsicExpression::FindIntrinsic(yytext, context.source_loc());
if (intrinsic.ok()) {
return CARBON_ARG_TOKEN(intrinsic_identifier, *intrinsic);
} else {
return context.RecordSyntaxError(std::move(intrinsic).error());
}
}
{identifier} {
BEGIN(AFTER_OPERAND);
return CARBON_ARG_TOKEN(identifier, yytext);
}
{integer_literal} {
BEGIN(AFTER_OPERAND);
int val = 0;
if (!llvm::to_integer(yytext, val)) {
return context.RecordSyntaxError(
llvm::formatv("Invalid integer literal: {0}", yytext));
}
return CARBON_ARG_TOKEN(integer_literal, val);
}
#*\" {
// Found a double quote character.
Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context);
const std::string& s = str_lex_helper.str();
const int hashtag_num = s.find_first_of('"');
if (!str_lex_helper.Advance()) {
return CARBON_SIMPLE_TOKEN(END_OF_FILE);
}
while (!str_lex_helper.is_eof()) {
switch (str_lex_helper.last_char()) {
case '\n': // Fall through.
case '\v': // Fall through.
case '\f': // Fall through.
case '\r':
return context.RecordSyntaxError(
llvm::formatv("missing closing quote in single-line string: {0}",
str_lex_helper.str()));
break;
case '"':
if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
// Reach closing quotes, break out of the loop.
return Carbon::ProcessSingleLineString(str_lex_helper.str(), context,
hashtag_num);
}
break;
case '\\':
if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
// Read the escaped char.
if (!str_lex_helper.Advance()) {
continue;
}
// Read the next char.
str_lex_helper.Advance();
}
break;
default:
str_lex_helper.Advance();
}
}
return CARBON_SIMPLE_TOKEN(END_OF_FILE);
}
#*\'\'\' {
// Multi-line string literal.
Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context);
const std::string& s = str_lex_helper.str();
const int hashtag_num = s.find_first_of('\'');
if (!str_lex_helper.Advance()) {
return CARBON_SIMPLE_TOKEN(END_OF_FILE);
}
while (!str_lex_helper.is_eof()) {
switch (str_lex_helper.last_char()) {
case '\'':
// Check for 2 more '\''s on block string.
if (!(str_lex_helper.Advance() && str_lex_helper.last_char() == '\'')) {
continue;
}
if (!(str_lex_helper.Advance() && str_lex_helper.last_char() == '\'')) {
continue;
}
// Now we are at the last ' of '''.
if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
// Reach closing quotes, break out of the loop.
return Carbon::ProcessMultiLineString(str_lex_helper.str(), context,
hashtag_num);
}
break;
case '\\':
if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
// Read the escaped char.
if (!str_lex_helper.Advance()) {
continue;
}
// Read the next char.
str_lex_helper.Advance();
}
break;
default:
str_lex_helper.Advance();
}
}
return CARBON_SIMPLE_TOKEN(END_OF_FILE);
}
{one_line_comment} {
// Advance end by 1 line, resetting the column to zero.
context.current_token_position.lines(1);
// Make the span empty by setting start to end.
context.current_token_position.step();
}
{horizontal_whitespace}+ {
// Make the span empty by setting start to end.
context.current_token_position.step();
BEGIN(AFTER_WHITESPACE);
}
\n+ {
// Advance end by yyleng lines, resetting the column to zero.
context.current_token_position.lines(yyleng);
// Make the span empty by setting start to end.
context.current_token_position.step();
BEGIN(AFTER_WHITESPACE);
}
. {
return context.RecordSyntaxError(
llvm::formatv("invalid character '\\x{0}' in source file.",
llvm::toHex(llvm::StringRef(yytext, 1))));
}
%%
auto YyinputWrapper(yyscan_t yyscanner) -> int { return yyinput(yyscanner); }
Reference in New Issue View Git Blame Copy Permalink