carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 14:51:03 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Initial implementation of raw string literals (#1304)

Browse Source 
* test cases for raw string literals

* raw string literal implementation

* match as block string if starting with triple ", and better error message for simple string

except for *#"""#*

* fix broken test case

block string  literal cannot be one line

* test cases for raw string literals

* raw string literal implementation

* match as block string if starting with triple ", and better error message for simple string

except for *#"""#*

* fix broken test case

block string  literal cannot be one line

* removed unused initial value

* rename flag to indicate multi-line string and remove comment

* use * to get value from std::optional

* clean-ups

* removed skip_scan flag and directly return in case of a single line string starting with #+\'\'\'

* Updated error message: simple string -> single-line string.

Co-authored-by: josh11b <josh11b@users.noreply.github.com>

* Updated test cases according to changes in error message

* Removed counting_hashtag flag.

* Implemented ScanHelper class to handle scanning

* Fixed explanation of ReadHashTags.

* Addressed PR comment.

* Clarify that scan_helper holds the source text.

* Addressed PR comments.

* Updated error messages in test cases.

* Added const keyword to return type of GetCurrentStr().

* addressed PR comments.

1. Moved ScanHelper class to lex_scan_helper.h and lex_scan_helper.cpp.
2. Moved ReadHashTags and Process* functions to lex_scan_helper.cpp. Moved YY_USER_ACTION, SIMPLE_TOKEN and ARG_TOKEN to lex_helper.h. Added a wrapper function YyinputWrapper to call static function yyinput in lexer.lpp.
3. Renamed ScanHelper with StringLexHelper.
4. Modified BUILD accordingly.
5. Renamed data members and functions.

* Addressed PR comments.

1. Adjusted order to keep ret usage close.
2. Used resize to construct the string to avoid creation of temp string.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Removed the multi_line flag and skip_read field to improve readability.

* Copied default parameter value to definition of UnescapeStringLiteral.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Copied default parameter value to definition of ParseBlockStringLiteral.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Prefix CARBON_ to SIMPLE_TOKEN and ARG_TOKEN macros.

* Rollback redefinition of arguments.

* Updated comment on the flex macro.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Updated wording.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Moved the EOF error out of the loop.

* Removed duplicated declaration.

* Changed type of `hashtag_num` and `leading_quotes` to int.

* Minor fix: string copy.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Added comment on YyinputWrapper.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Garmmar in comment.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Added check of eof before readling next char.

* Minor updates based on PR comments.

* Minor changes to address PR comments.

* Used a clearer way to calculate `hashtag_num` and `leading_quotes`. Switched back to indicate muti-line string with a flag.

* Directly copy StringRef for compilation error message.

* Make str_with_quote const as we don't change it.

Co-authored-by: josh11b <josh11b@users.noreply.github.com>

* Added TODO for unsupported cases.

* Fixed a typo.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

Co-authored-by: josh11b <josh11b@users.noreply.github.com>
Co-authored-by: Jon Ross-Perkins <jperkins@google.com>
This commit is contained in:
SlaterLatiao
2022-06-22 15:38:47 -07:00
committed by GitHub
co-authored by Jon Ross-Perkins josh11b
parent a1be2a8a38
commit 8d0f3364d8
22 changed files with 683 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files
explorer/testdata/string/block_file_type_indicator.carbon
Vendored
+24
View File
@@ -0,0 +1,24 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
// CHECK: result: 0
package ExplorerTest api;
fn Main() -> i32 {
var s: String = """filetype
A "block" ""string"" literal
with file type indicator.
""";
if (s == "A \"block\" \"\"string\"\" literal\n with file type indicator.\n") {
return 0;
} else {
return 1;
}
}
explorer/testdata/string/fail_newline.carbon
Vendored
+2 -1
View File
@@ -11,7 +11,8 @@
package ExplorerTest api;
fn Main() -> i32 {
// CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_newline.carbon:[[@LINE+1]]: invalid character '\x22' in source file.
// CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_newline.carbon:[[@LINE+2]]: missing closing quote in single-line string: "new
// CHECK-EMPTY:
Print("new
line");
return 0;
explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon
Vendored
+20
View File
@@ -0,0 +1,20 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{not} %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
package ExplorerTest api;
fn Main() -> i32 {
// CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon:[[@LINE+1]]: Unexpected end of file
var s: String = ##"""
error: there are more #s on the left than the right.
"""#;
return 0;
}
explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon
Vendored
+20
View File
@@ -0,0 +1,20 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{not} %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
package ExplorerTest api;
fn Main() -> i32 {
// CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file.
var s: String = #"""
error: there are more #s on the right than the left.
"""##;
return 0;
}
explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon
Vendored
+20
View File
@@ -0,0 +1,20 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{not} %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
package ExplorerTest api;
fn Main() -> i32 {
// CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """
var s: String = #"""
error: closing """# is not on its own line.
"""#;
return 0;
}
explorer/testdata/string/fail_raw_block_single_line.carbon
Vendored
+23
View File
@@ -0,0 +1,23 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{not} %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
package ExplorerTest api;
fn CompareStr(s: String) -> i32 {
// CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_single_line.carbon:[[@LINE+1]]: Invalid block string: Too few lines
if (s == #"""raw string literal starting with """#) {
return 0;
}
return 1;
}
fn Main() -> i32 {
return CompareStr("\"\"raw string literal starting with \"\"");
}
explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon
Vendored
+24
View File
@@ -0,0 +1,24 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{not} %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
package ExplorerTest api;
fn CompareStr(s: String) -> i32 {
// CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon:[[@LINE+2]]: missing closing quote in single-line string: ##"str"#) {
// CHECK-EMPTY:
if (s == ##"str"#) {
return 0;
}
return 1;
}
fn Main() -> i32 {
return CompareStr("str");
}
explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon
Vendored
+23
View File
@@ -0,0 +1,23 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{not} %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
package ExplorerTest api;
fn CompareStr(s: String) -> i32 {
// CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file.
if (s == "str"#) {
return 0;
}
return 1;
}
fn Main() -> i32 {
return CompareStr("str");
}
explorer/testdata/string/raw_basic.carbon
Vendored
+23
View File
@@ -0,0 +1,23 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
// CHECK: result: 0
package ExplorerTest api;
fn CompareStr(s: String) -> i32 {
if (s == #"str"#) {
return 0;
}
return 1;
}
fn Main() -> i32 {
return CompareStr("str");
}
explorer/testdata/string/raw_basic_multi_hashtag.carbon
Vendored
+23
View File
@@ -0,0 +1,23 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
// CHECK: result: 0
package ExplorerTest api;
fn CompareStr(s: String) -> i32 {
if (s == ##"str"##) {
return 0;
}
return 1;
}
fn Main() -> i32 {
return CompareStr(#####"str"#####);
}
explorer/testdata/string/raw_block.carbon
Vendored
+24
View File
@@ -0,0 +1,24 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
// CHECK: result: 0
package ExplorerTest api;
fn Main() -> i32 {
var s: String = #"""
A "block" ""string"" literal
with indent.
"""#;
if (s == #"A \#"block\#" \#"\#"string\#"\#" literal\#n with indent.\#n"#) {
return 0;
} else {
return 1;
}
}
explorer/testdata/string/raw_block_escaped_back_slash.carbon
Vendored
+24
View File
@@ -0,0 +1,24 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
// CHECK: result: 0
package ExplorerTest api;
fn Main() -> i32 {
var s: String = #"""
A block string literal
\
"""#;
if (s == "A block string literal\n\\\n") {
return 0;
} else {
return 1;
}
}
explorer/testdata/string/raw_block_escaped_triple_quotes.carbon
Vendored
+24
View File
@@ -0,0 +1,24 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
// CHECK: result: 0
package ExplorerTest api;
fn Main() -> i32 {
var s: String = #"""
A block string literal
\#"""#
"""#;
if (s == "A block string literal\n\"\"\"#\n") {
return 0;
} else {
return 1;
}
}
explorer/testdata/string/raw_nesting.carbon
Vendored
+23
View File
@@ -0,0 +1,23 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{explorer} %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s
// AUTOUPDATE: %{explorer} %s
// CHECK: result: 0
package ExplorerTest api;
fn CompareStr(s: String) -> i32 {
if (s == ##"#"str"#"##) {
return 0;
}
return 1;
}
fn Main() -> i32 {
return CompareStr("#\"str\"#");
}