carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 22:02:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Add table sorting functionality (#796)

Browse Source 
Also apply it to the list of tokens. This way there's no need for "sort order" comments.

This was a side-effect of me trying to merge in api/impl/library/package, and thinking "why am I doing this manually?"



Co-authored-by: Geoff Romer <gromer@google.com>
This commit is contained in:
Jon Meow
2021-09-02 17:28:37 -07:00
committed by GitHub
co-authored by Geoff Romer
parent a03536a196
commit 820081b8c1
4 changed files with 137 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
executable_semantics/syntax/format_grammar.py
+79 -33
View File
@@ -33,11 +33,6 @@ _COLS = 80
# An arbitrary separator to use when formatting multiple code segments.
_FORMAT_SEPARATOR = "\n// CLANG FORMAT CODE SEGMENT SEPARATOR\n"
# The table begin and end comments, including table-bounding newlines.
_TABLE_BEGIN = "/* Table begin. */\n"
_TABLE_END = "\n/* Table end. */"
_TABLE_END_WITH_SPACE = "\n /* Table end. */"
@dataclass
class _CppCode:
@@ -66,6 +61,16 @@ class _Table:
content: str
def _print_header(msg: str) -> None:
"""Prints a header, mostly for debug output."""
print(("=== %s " % msg).ljust(79, "="))
def _print_footer() -> None:
"""Prints a footer, mostly for debug output."""
print("=" * 79)
def _parse_args() -> argparse.Namespace:
"""Parses command-line arguments and flags."""
parser = argparse.ArgumentParser(description=__doc__)
@@ -132,9 +137,9 @@ def _add_text_segment(
"""Adds a text segment to the list."""
text_segments.append(segment)
if debug:
print("=== Text segment ===")
_print_header("Text segment")
print(segment)
print("====================")
_print_footer()
def _maybe_add_cpp_segment(
@@ -191,7 +196,7 @@ def _maybe_add_cpp_segment(
has_percent,
)
if debug:
print("=== C++ segment ===")
_print_header("C++ segment")
print(cpp_segment.content)
print(
"Structure: { at %d; } at %d; %%: %s"
@@ -201,7 +206,7 @@ def _maybe_add_cpp_segment(
cpp_segment.has_percent,
)
)
print("===================")
_print_footer()
# Record the code segment.
if close_brace_indent not in cpp_segments:
@@ -212,12 +217,16 @@ def _maybe_add_cpp_segment(
return (end, True)
def _parse_block_comment(
def _parse_comment(
content: str,
text_segments: List[Optional[str]],
table_segments: List[_Table],
text_segment_start: int,
cursor: int,
comment_start_str: str,
comment_end_str: str,
table_start_str: str,
table_end_pattern: str,
debug: bool,
) -> Tuple[int, int]:
"""Parses a comment, possibly adding a table segment.
@@ -226,31 +235,35 @@ def _parse_block_comment(
new_segment_start may or may not change.
"""
# Skip over block comments.
comment_end = content.find("*/", cursor + 2)
comment_end = content.find(comment_end_str, cursor + len(comment_start_str))
if comment_end == -1:
exit(
"failed to find end of /* comment: %s"
% content[cursor : cursor + 20]
"failed to find end of %s comment: %s"
% (comment_start_str, content[cursor : cursor + 20])
)
comment_end += 2
if content[cursor : comment_end + 1] == _TABLE_BEGIN:
for table_end_style in (_TABLE_END, _TABLE_END_WITH_SPACE):
table_end = content.find(table_end_style, comment_end)
if table_end != -1:
break
if table_end == -1:
if comment_end_str != "\n":
comment_end += len(comment_end_str)
if content[cursor : comment_end + 1] == table_start_str:
m = re.compile(table_end_pattern).search(content, comment_end)
if not m:
exit(
"failed to find end of table: %s"
% content[comment_end + 1 : comment_end + 20]
"failed to find end of table: `%s`"
% content[comment_end : comment_end + 20]
)
_add_text_segment(
text_segments, content[text_segment_start : comment_end + 1], debug
)
text_segments.append(None)
table_segments.append(
_Table(len(text_segments) - 1, content[comment_end + 1 : table_end])
table_segment = _Table(
len(text_segments) - 1, content[comment_end + 1 : m.start()]
)
return table_end, table_end + len(_TABLE_END) - 1
table_segments.append(table_segment)
if debug:
_print_header("Table segment")
print(table_segment.content)
_print_footer()
return m.start(), m.end()
else:
return text_segment_start, comment_end - 1
@@ -268,7 +281,7 @@ def _parse_segments(
- table_segments is a list of _Table objects.
"""
i = 0
segment_start = 0
text_segment_start = 0
text_segments: List[Optional[str]] = []
cpp_segments: Dict[int, List[_CppCode]] = {}
table_segments: List[_Table] = []
@@ -278,20 +291,47 @@ def _parse_segments(
# Skip over strings.
i = _find_string_end(content, i + 1)
elif c == "/" and content[i + 1 : i + 2] == "*":
segment_start, i = _parse_block_comment(
content, text_segments, table_segments, segment_start, i, debug
text_segment_start, i = _parse_comment(
content=content,
text_segments=text_segments,
table_segments=table_segments,
text_segment_start=text_segment_start,
cursor=i,
comment_start_str="/*",
comment_end_str="*/",
table_start_str="/* table-begin */\n",
table_end_pattern=r"\n\s*/\* table-end \*/\n",
debug=debug,
)
elif c == "/" and content[i + 1 : i + 2] == "/":
text_segment_start, i = _parse_comment(
content=content,
text_segments=text_segments,
table_segments=table_segments,
text_segment_start=text_segment_start,
cursor=i,
comment_start_str="//",
comment_end_str="\n",
table_start_str="// table-begin\n",
table_end_pattern=r"\n\s*// table-end\n",
debug=debug,
)
elif c == "\\":
# Skip over escapes.
i += 1
elif c == "{":
i, added = _maybe_add_cpp_segment(
content, text_segments, cpp_segments, segment_start, i, debug
content,
text_segments,
cpp_segments,
text_segment_start,
i,
debug,
)
if added:
segment_start = i + 1
text_segment_start = i + 1
i += 1
_add_text_segment(text_segments, content[segment_start:], debug)
_add_text_segment(text_segments, content[text_segment_start:], debug)
return text_segments, cpp_segments, table_segments
@@ -358,7 +398,9 @@ def _format_table_segments(
) -> None:
"""Formats table segments."""
for table in table_segments:
lines = table.content.strip().splitlines()
# Split, removing empty lines.
lines = [line for line in table.content.splitlines() if line]
lines.sort()
rows: List[List[str]] = []
col_widths: List[int] = []
for row_index in range(len(lines)):
@@ -377,8 +419,12 @@ def _format_table_segments(
col_widths[col_index] = max(
col_widths[col_index], len(cols[col_index])
)
# Count prefix spaces for the indent.
m = re.match("^ *", lines[0])
assert m
indent = m.end()
# The last column should not add spaces.
row_format = " ".join(
row_format = (" " * indent) + " ".join(
["%%-%ds" % width for width in col_widths[:-1]] + ["%s"]
)
text_segments[table.segment_index] = "\n".join(
executable_semantics/syntax/format_grammar_test.py
+51 -9
View File
@@ -91,18 +91,18 @@ class TestFormatGrammar(unittest.TestCase):
self.assertEqual(
format_grammar._parse_segments(
"content\n"
"/* Table begin. */\n"
"/* table-begin */\n"
"{VAR} { return SIMPLE_TOKEN(VAR); }\n"
"{WHILE} { return SIMPLE_TOKEN(WHILE); }\n"
"/* Table end. */\n"
"/* table-end */\n"
"more content\n",
False,
),
(
[
"content\n" "/* Table begin. */\n",
"content\n" "/* table-begin */\n",
None,
"\n" "/* Table end. */\n" "more content\n",
"\n" "/* table-end */\n" "more content\n",
],
{},
[
@@ -119,18 +119,18 @@ class TestFormatGrammar(unittest.TestCase):
self.assertEqual(
format_grammar._parse_segments(
"content\n"
" /* Table begin. */\n"
" /* table-begin */\n"
"{VAR} { return SIMPLE_TOKEN(VAR); }\n"
"{WHILE} { return SIMPLE_TOKEN(WHILE); }\n"
" /* Table end. */\n"
" /* table-end */\n"
"more content\n",
False,
),
(
[
"content\n /* Table begin. */\n",
"content\n /* table-begin */\n",
None,
"\n /* Table end. */\nmore content\n",
"\n /* table-end */\nmore content\n",
],
{},
[
@@ -143,6 +143,29 @@ class TestFormatGrammar(unittest.TestCase):
),
)
def test_table_tokens(self):
self.assertEqual(
format_grammar._parse_segments(
"%tokens\n"
" // Comment\n"
" // table-begin\n"
" VAR\n"
" WHILE\n"
" // table-end\n"
" MORE\n",
False,
),
(
[
"%tokens\n" " // Comment\n" " // table-begin\n",
None,
"\n" " // table-end\n" " MORE\n",
],
{},
[format_grammar._Table(1, " VAR\n" " WHILE")],
),
)
def test_format_table_defines(self):
text_segments = [None]
format_grammar._format_table_segments(
@@ -150,8 +173,8 @@ class TestFormatGrammar(unittest.TestCase):
[
format_grammar._Table(
0,
'CONTINUE "continue"\n'
'DEFAULT "default"\n'
'CONTINUE "continue"\n'
'DOUBLE_ARROW "=>"',
)
],
@@ -187,6 +210,25 @@ class TestFormatGrammar(unittest.TestCase):
],
)
def test_format_table_tokens(self):
text_segments = [None]
format_grammar._format_table_segments(
text_segments,
[
format_grammar._Table(
0,
" AND\n" " CONTINUE\n" " BREAK",
)
],
False,
)
self.assertEqual(
text_segments,
[
" AND\n" " BREAK\n" " CONTINUE",
],
)
if __name__ == "__main__":
unittest.main()
executable_semantics/syntax/lexer.lpp
+5 -5
View File
@@ -28,7 +28,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
%s AFTER_OPERAND
/* Table begin. */
/* table-begin */
AND "and"
API "api"
ARROW "->"
@@ -80,7 +80,7 @@ TYPE "Type"
UNDERSCORE "_"
VAR "var"
WHILE "while"
/* Table end. */
/* table-end */
/* This should be kept table-like, but isn't automatic due to spaces. */
identifier [A-Za-z_][A-Za-z0-9_]*
@@ -122,7 +122,7 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\"
context.current_token_position.step();
%}
/* Table begin. */
/* table-begin */
{AND} { return SIMPLE_TOKEN(AND); }
{API} { return SIMPLE_TOKEN(API); }
{ARROW} { return SIMPLE_TOKEN(ARROW); }
@@ -150,8 +150,8 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\"
{IF} { return SIMPLE_TOKEN(IF); }
{IMPL} { return SIMPLE_TOKEN(IMPL); }
{IMPORT} { return SIMPLE_TOKEN(IMPORT); }
{LEFT_PARENTHESIS} { return SIMPLE_TOKEN(LEFT_PARENTHESIS); }
{LEFT_CURLY_BRACE} { return SIMPLE_TOKEN(LEFT_CURLY_BRACE); }
{LEFT_PARENTHESIS} { return SIMPLE_TOKEN(LEFT_PARENTHESIS); }
{LEFT_SQUARE_BRACKET} { return SIMPLE_TOKEN(LEFT_SQUARE_BRACKET); }
{LIBRARY} { return SIMPLE_TOKEN(LIBRARY); }
{MATCH} { return SIMPLE_TOKEN(MATCH); }
@@ -171,7 +171,7 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\"
{UNDERSCORE} { return SIMPLE_TOKEN(UNDERSCORE); }
{VAR} { return SIMPLE_TOKEN(VAR); }
{WHILE} { return SIMPLE_TOKEN(WHILE); }
/* Table end. */
/* table-end */
/* More modern Bisons provide make_EOF. */
<<EOF>> { return SIMPLE_TOKEN(END_OF_FILE); }
executable_semantics/syntax/parser.ypp
+2
View File
@@ -137,6 +137,7 @@
%token
// Most tokens have their spelling defined in lexer.lpp.
// table-begin
AND
API
ARROW
@@ -188,6 +189,7 @@
UNDERSCORE
VAR
WHILE
// table-end
// Used to track EOF.
END_OF_FILE 0
// Only used for precedence.