mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-11-16 08:08:10 +00:00
Specifically, after lexing a comment line, look at the next line and see if it starts with an identical sequence of indent, comment '/'s and character after the '/'s. If so, skip it as part of a block of comments. This skips repeatedly diagnosing the same erroneous comment introducer after the first one in a block, but that seems like a feature rather than a bug. The big motivation is to make sure the lexer is minimally impacted by the length of comment blocks and skips them as efficiently as possible. While they aren't exactly common, large block comments do come up and it'd be unfortunate for those to actually slow down the toolchain. It also happens that this is particularly easy to do because we're just looking to see if we see the same prefix byte sequence. With SIMD we can typically handle the most common indents with just a few instructions. Because of the diagnostic differences, I've included a scalar fallback that replicates the functionality but has no limit on indent size or CPU features. I've also added testing to cover this behavior. The only non-noise benchmark changes are as expected the comment ones, with a nice improvement across the board: ``` BM_CommentLines/1/0/0 15.8ms ± 2% 15.6ms ± 2% -0.87% (p=0.004 n=19+19) BM_CommentLines/4/0/0 20.2ms ± 1% 18.8ms ± 1% -6.75% (p=0.000 n=18+18) BM_CommentLines/128/0/0 221ms ± 1% 167ms ± 1% -24.44% (p=0.000 n=20+19) BM_CommentLines/1/30/0 16.6ms ± 3% 16.5ms ± 3% ~ (p=0.175 n=19+20) BM_CommentLines/4/30/0 26.1ms ± 1% 24.8ms ± 2% -5.05% (p=0.000 n=18+19) BM_CommentLines/128/30/0 233ms ± 1% 185ms ± 1% -20.38% (p=0.000 n=19+20) BM_CommentLines/1/70/0 19.2ms ± 1% 19.0ms ± 2% -0.66% (p=0.016 n=19+20) BM_CommentLines/4/70/0 27.9ms ± 1% 26.6ms ± 1% -4.63% (p=0.000 n=19+19) BM_CommentLines/128/70/0 251ms ± 1% 213ms ± 1% -15.18% (p=0.000 n=20+18) BM_CommentLines/1/0/2 15.9ms ± 1% 15.8ms ± 2% ~ (p=0.061 n=19+19) BM_CommentLines/4/0/2 20.5ms ± 2% 19.0ms ± 2% -7.53% (p=0.000 n=20+20) BM_CommentLines/128/0/2 213ms ± 1% 153ms ± 1% -28.18% (p=0.000 n=19+20) BM_CommentLines/1/30/2 16.8ms ± 2% 16.7ms ± 3% ~ (p=0.134 n=20+20) BM_CommentLines/4/30/2 26.6ms ± 1% 25.2ms ± 3% -5.50% (p=0.000 n=20+20) BM_CommentLines/128/30/2 238ms ± 1% 187ms ± 2% -21.49% (p=0.000 n=17+19) BM_CommentLines/1/70/2 19.3ms ± 1% 19.4ms ± 3% ~ (p=0.407 n=17+20) BM_CommentLines/4/70/2 28.2ms ± 1% 26.9ms ± 2% -4.70% (p=0.000 n=19+19) BM_CommentLines/128/70/2 257ms ± 2% 214ms ± 1% -16.52% (p=0.000 n=20+18) BM_CommentLines/1/0/8 16.3ms ± 2% 16.1ms ± 2% -1.22% (p=0.001 n=20+20) BM_CommentLines/4/0/8 22.7ms ± 2% 20.4ms ± 2% -10.20% (p=0.000 n=20+20) BM_CommentLines/128/0/8 244ms ± 1% 153ms ± 1% -37.26% (p=0.000 n=20+18) BM_CommentLines/1/30/8 17.3ms ± 2% 17.2ms ± 3% ~ (p=0.192 n=20+20) BM_CommentLines/4/30/8 28.0ms ± 2% 25.6ms ± 3% -8.46% (p=0.000 n=19+18) BM_CommentLines/128/30/8 272ms ± 1% 196ms ± 2% -27.90% (p=0.000 n=18+20) BM_CommentLines/1/70/8 19.9ms ± 2% 19.9ms ± 2% ~ (p=0.531 n=20+19) BM_CommentLines/4/70/8 29.3ms ± 1% 27.3ms ± 1% -6.87% (p=0.000 n=19+19) BM_CommentLines/128/70/8 292ms ± 1% 228ms ± 1% -21.97% (p=0.000 n=20+19) ``` --------- Co-authored-by: Richard Smith <richard@metafoo.co.uk>
51 lines
4.7 KiB
Plaintext
51 lines
4.7 KiB
Plaintext
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
// Trailing comments should produce errors, but we should continue to parse (and
|
|
// diagnose) code errors beyond them.
|
|
|
|
var a: i32 = 1; // trailing comment
|
|
var b: 32 = 13; // more trailing comment
|
|
var c: i32 = 0.4; // still more trailing comment
|
|
|
|
// We keep the auto-update marker below the above so check lines don't disrupt
|
|
// the trailing comment lines with similar prefixes.
|
|
// AUTOUPDATE
|
|
// CHECK:STDERR: fail_trailing_comments.carbon:[[@LINE-7]]:19: ERROR: Trailing comments are not permitted.
|
|
// CHECK:STDERR: var a: i32 = 1; // trailing comment
|
|
// CHECK:STDERR: ^
|
|
// CHECK:STDERR: fail_trailing_comments.carbon:[[@LINE-9]]:19: ERROR: Trailing comments are not permitted.
|
|
// CHECK:STDERR: var b: 32 = 13; // more trailing comment
|
|
// CHECK:STDERR: ^
|
|
// CHECK:STDERR: fail_trailing_comments.carbon:[[@LINE-11]]:19: ERROR: Trailing comments are not permitted.
|
|
// CHECK:STDERR: var c: i32 = 0.4; // still more trailing comment
|
|
// CHECK:STDERR: ^
|
|
// CHECK:STDOUT: - filename: fail_trailing_comments.carbon
|
|
// CHECK:STDOUT: tokens: [
|
|
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 1, kind: 'Var', line: {{ *}}[[@LINE-19]], column: 1, indent: 1, spelling: 'var', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-20]], column: 5, indent: 1, spelling: 'a', identifier: 0 },
|
|
// CHECK:STDOUT: { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-21]], column: 6, indent: 1, spelling: ':', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 4, kind: 'IntegerTypeLiteral', line: {{ *}}[[@LINE-22]], column: 8, indent: 1, spelling: 'i32', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 5, kind: 'Equal', line: {{ *}}[[@LINE-23]], column: 12, indent: 1, spelling: '=', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 6, kind: 'IntegerLiteral', line: {{ *}}[[@LINE-24]], column: 14, indent: 1, spelling: '1', value: `1` },
|
|
// CHECK:STDOUT: { index: 7, kind: 'Semi', line: {{ *}}[[@LINE-25]], column: 15, indent: 1, spelling: ';', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 8, kind: 'Var', line: {{ *}}[[@LINE-25]], column: 1, indent: 1, spelling: 'var', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 9, kind: 'Identifier', line: {{ *}}[[@LINE-26]], column: 5, indent: 1, spelling: 'b', identifier: 1 },
|
|
// CHECK:STDOUT: { index: 10, kind: 'Colon', line: {{ *}}[[@LINE-27]], column: 6, indent: 1, spelling: ':', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 11, kind: 'IntegerLiteral', line: {{ *}}[[@LINE-28]], column: 8, indent: 1, spelling: '32', value: `32`, has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 12, kind: 'Equal', line: {{ *}}[[@LINE-29]], column: 11, indent: 1, spelling: '=', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 13, kind: 'IntegerLiteral', line: {{ *}}[[@LINE-30]], column: 13, indent: 1, spelling: '13', value: `13` },
|
|
// CHECK:STDOUT: { index: 14, kind: 'Semi', line: {{ *}}[[@LINE-31]], column: 15, indent: 1, spelling: ';', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 15, kind: 'Var', line: {{ *}}[[@LINE-31]], column: 1, indent: 1, spelling: 'var', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 16, kind: 'Identifier', line: {{ *}}[[@LINE-32]], column: 5, indent: 1, spelling: 'c', identifier: 2 },
|
|
// CHECK:STDOUT: { index: 17, kind: 'Colon', line: {{ *}}[[@LINE-33]], column: 6, indent: 1, spelling: ':', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 18, kind: 'IntegerTypeLiteral', line: {{ *}}[[@LINE-34]], column: 8, indent: 1, spelling: 'i32', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 19, kind: 'Equal', line: {{ *}}[[@LINE-35]], column: 12, indent: 1, spelling: '=', has_trailing_space: true },
|
|
// CHECK:STDOUT: { index: 20, kind: 'RealLiteral', line: {{ *}}[[@LINE-36]], column: 14, indent: 1, spelling: '0.4', value: `4*10^-1` },
|
|
// CHECK:STDOUT: { index: 21, kind: 'Semi', line: {{ *}}[[@LINE-37]], column: 17, indent: 1, spelling: ';', has_trailing_space: true },
|
|
|
|
// CHECK:STDOUT: { index: 22, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
|
|
// CHECK:STDOUT: ]
|