carbon-lang/executable_semantics/syntax/lexer.lpp

/*
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
Exceptions. See /LICENSE for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/

%{
#include <cstdlib>
#include <iostream>
#include "executable_semantics/syntax/parse_and_lex_context.h"
%}

/* Turn off legacy bits we don't need */
%option noyywrap nounput nodefault noinput

/* maintains the number of the current line read from input in the
   global variable yylineno.
*/
%option yylineno

AND               "and"
ARROW             "->"
AUTO              "auto"
BOOL              "Bool"
BREAK             "break"
CASE              "case"
CHOICE            "choice"
ONE_LINE_COMMENT  \/\/[^\n]*\n
CONTINUE          "continue"
DBLARROW          "=>"
DEFAULT           "default"
ELSE              "else"
EQUAL_EQUAL       "=="
FALSE             "false"
FN                "fn"
FNTY              "fnty"
IF                "if"
INT               "Int"
MATCH             "match"
NOT               "not"
OR                "or"
RETURN            "return"
STRUCT            "struct"
TRUE              "true"
TYPE              "Type"
VAR               "var"
WHILE             "while"
CONTINUATION_TYPE "__Continuation"
CONTINUATION      "__continuation"
RUN               "__run"
AWAIT             "__await"

identifier    [A-Za-z_][A-Za-z0-9_]*
integer_literal   [0-9]+
horizontal_whitespace [ \t\r]

%{
  // This macro is expanded to run each time a token is recognized.
  //
  // Advances the current token position by yyleng columns without changing
  // the line number.
  # define YY_USER_ACTION context.current_token_position.columns(yyleng);
%}

%%

%{
  // Code run each time yylex is called.

  // Begin with an empty token span starting where its previous end was.
  context.current_token_position.step();
%}

{AND}      { return yy::parser::make_AND(context.current_token_position); }
{ARROW}    { return yy::parser::make_ARROW(context.current_token_position); }
{AUTO}     { return yy::parser::make_AUTO(context.current_token_position); }
{BOOL}     { return yy::parser::make_BOOL(context.current_token_position); }
{BREAK}    { return yy::parser::make_BREAK(context.current_token_position); }
{CASE}     { return yy::parser::make_CASE(context.current_token_position); }
{CHOICE}   { return yy::parser::make_CHOICE(context.current_token_position); }
{CONTINUE} { return yy::parser::make_CONTINUE(context.current_token_position); }
{DBLARROW} { return yy::parser::make_DBLARROW(context.current_token_position); }
{DEFAULT}  { return yy::parser::make_DEFAULT(context.current_token_position); }
{ELSE}     { return yy::parser::make_ELSE(context.current_token_position); }
"=="       { return yy::parser::make_EQUAL_EQUAL(context.current_token_position); }
{FALSE}    { return yy::parser::make_FALSE(context.current_token_position); }
{FN}       { return yy::parser::make_FN(context.current_token_position); }
{FNTY}     { return yy::parser::make_FNTY(context.current_token_position); }
{IF}       { return yy::parser::make_IF(context.current_token_position); }
{INT}      { return yy::parser::make_INT(context.current_token_position); }
{MATCH}    { return yy::parser::make_MATCH(context.current_token_position); }
{NOT}      { return yy::parser::make_NOT(context.current_token_position); }
{OR}       { return yy::parser::make_OR(context.current_token_position); }
{RETURN}   { return yy::parser::make_RETURN(context.current_token_position); }
{STRUCT}   { return yy::parser::make_STRUCT(context.current_token_position); }
{TRUE}     { return yy::parser::make_TRUE(context.current_token_position); }
{TYPE}     { return yy::parser::make_TYPE(context.current_token_position); }
{VAR}      { return yy::parser::make_VAR(context.current_token_position); }
{WHILE}    { return yy::parser::make_WHILE(context.current_token_position); }
{CONTINUATION_TYPE} { return yy::parser::make_CONTINUATION_TYPE(context.current_token_position); }
{CONTINUATION}    { return yy::parser::make_CONTINUATION(context.current_token_position); }
{RUN}      { return yy::parser::make_RUN(context.current_token_position); }
{AWAIT}    { return yy::parser::make_AWAIT(context.current_token_position); }

"=" return yy::parser::make_EQUAL(context.current_token_position);
"-" return yy::parser::make_MINUS(context.current_token_position);
"+" return yy::parser::make_PLUS(context.current_token_position);
"*" return yy::parser::make_STAR(context.current_token_position);
"/" return yy::parser::make_SLASH(context.current_token_position);
"(" return yy::parser::make_LEFT_PARENTHESIS(context.current_token_position);
")" return yy::parser::make_RIGHT_PARENTHESIS(context.current_token_position);
"{" return yy::parser::make_LEFT_CURLY_BRACE(context.current_token_position);
"}" return yy::parser::make_RIGHT_CURLY_BRACE(context.current_token_position);
"[" return yy::parser::make_LEFT_SQUARE_BRACKET(context.current_token_position);
"]" return yy::parser::make_RIGHT_SQUARE_BRACKET(context.current_token_position);
"." return yy::parser::make_PERIOD(context.current_token_position);
"," return yy::parser::make_COMMA(context.current_token_position);
";" return yy::parser::make_SEMICOLON(context.current_token_position);
":" return yy::parser::make_COLON(context.current_token_position);

{identifier} {
  int n = strlen(yytext);
  auto r = reinterpret_cast<char*>(malloc((n + 1) * sizeof(char)));
  strncpy(r, yytext, n + 1);
  return yy::parser::make_identifier(r, context.current_token_position);
}

{integer_literal} {
  auto r = atof(yytext);
  return yy::parser::make_integer_literal(r, context.current_token_position);
}

{ONE_LINE_COMMENT} {
  // Advance end by 1 line, resetting the column to zero.
  context.current_token_position.lines(1);
  // Make the span empty by setting start to end.
  context.current_token_position.step();
}

{horizontal_whitespace}+ {
  // Make the span empty by setting start to end.
  context.current_token_position.step();
}

\n+ {
  // Advance end by yyleng lines, resetting the column to zero.
  context.current_token_position.lines(yyleng);
  // Make the span empty by setting start to end.
  context.current_token_position.step();
}

. {
  std::cerr << context.current_token_position << ": invalid character '"
            << yytext[0] << "' in source file." << std::endl;
  std::exit(1);
}

<<EOF>>    {
  // A more modern Bison would give us make_EOF.
  return yy::parser::make_END_OF_FILE(context.current_token_position);
}

%%