carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-09-24 22:02:23 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Expand source stats to roughly cover more tokens. (#3277)

Browse Source 
This is pretty rough both in code-quality and what it tries to cover.
The coverage approximation is probably fine as it mostly just gives some
vague idea of the distributions we should aim at when evaluating
performance. We still want performance with unusual distributions to be
good.

Happy to improve the code quality if folks want, and especially with
suggestions on what would help. For this kind of quick scripting, it
seemed "fine" to me.
This commit is contained in:
Chandler Carruth
2023-10-11 00:50:00 +00:00
committed by GitHub
parent 6f5934a505
commit 0e8e94d8bf
1 changed files with 125 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/source_stats.py
+125 -2
View File
@@ -27,6 +27,12 @@ LINE_RE = re.compile(
(?P<class_intro>\b(class|struct)\s+(?P<class_name>\w+)\b)|
(?P<end_open_curly>{\s*(?P<open_curly_trailing_comment>//.*)?)|
(?P<trailing_comment>//.*)|
(?P<internal_comment>/\*.*\*/)|
(?P<string_literal>"([^"]|\\")*"|'([^']|\\')*')|
(?P<float_literal>\b(0[xb][0-9a-fA-F']*|[0-9][0-9']*)\.[0-9a-fA-F']*([eEpP][0-9a-fA-F']*)?)|
(?P<int_literal>\b(0[xb][0-9a-fA-F']+|[0-9][0-9']*)([eEpP][0-9a-fA-F']*)?)|
(?P<symbol>[\[\]{}(),.;]|[-+=!@#$%^&*/?|<>]+)|
(?P<keyword>\b(auto|bool|break|case|catch|char|class|const|continue|default|do|double|else|enum|explicit|extern|false|float|for|friend|goto|if|inline|int|long|mutable|namespace|new|nullptr|operator|private|protected|public|return|short|signed|sizeof|static|struct|switch|template|this|throw|true|try|typedef|union|unsigned|using|virtual|void|while)\b)|
(?P<id>\b\w+\b)
""",
re.X,
@@ -44,6 +50,23 @@ class Stats:
comment_line_widths: Counter[int] = field(default_factory=lambda: Counter())
lines_with_trailing_comments: int = 0
classes: int = 0
internal_comments: int = 0
string_literals: int = 0
string_literals_per_line: Counter[int] = field(
default_factory=lambda: Counter()
)
int_literals: int = 0
int_literals_per_line: Counter[int] = field(
default_factory=lambda: Counter()
)
float_literals: int = 0
float_literals_per_line: Counter[int] = field(
default_factory=lambda: Counter()
)
symbols: int = 0
symbols_per_line: Counter[int] = field(default_factory=lambda: Counter())
keywords: int = 0
keywords_per_line: Counter[int] = field(default_factory=lambda: Counter())
identifiers: int = 0
identifier_widths: Counter[int] = field(default_factory=lambda: Counter())
ids_per_line: Counter[int] = field(default_factory=lambda: Counter())
@@ -56,6 +79,17 @@ class Stats:
self.comment_line_widths.update(other.comment_line_widths)
self.lines_with_trailing_comments += other.lines_with_trailing_comments
self.classes += other.classes
self.internal_comments += other.internal_comments
self.string_literals += other.string_literals
self.string_literals_per_line.update(other.string_literals_per_line)
self.int_literals += other.int_literals
self.int_literals_per_line.update(other.int_literals_per_line)
self.float_literals += other.float_literals
self.float_literals_per_line.update(other.float_literals_per_line)
self.symbols += other.symbols
self.symbols_per_line.update(other.symbols_per_line)
self.keywords += other.keywords
self.keywords_per_line.update(other.keywords_per_line)
self.identifiers += other.identifiers
self.identifier_widths.update(other.identifier_widths)
self.ids_per_line.update(other.ids_per_line)
@@ -82,6 +116,11 @@ def scan_file(file: Path) -> Stats:
else:
stats.comment_line_widths[len(line)] += 1
continue
line_string_literals = 0
line_int_literals = 0
line_float_literals = 0
line_symbols = 0
line_keywords = 0
line_identifiers = 0
for m in re.finditer(LINE_RE, line):
if m.group("trailing_comment"):
@@ -89,10 +128,23 @@ def scan_file(file: Path) -> Stats:
break
if m.group("class_intro"):
stats.classes += 1
line_keywords += 1
line_identifiers += 1
stats.identifier_widths[len(m.group("class_name"))] += 1
elif m.group("end_open_curly"):
pass
line_symbols += 1
elif m.group("internal_comment"):
stats.internal_comments += 1
elif m.group("string_literal"):
line_string_literals += 1
elif m.group("int_literal"):
line_int_literals += 1
elif m.group("float_literal"):
line_float_literals += 1
elif m.group("symbol"):
line_symbols += 1
elif m.group("keyword"):
line_keywords += 1
else:
assert m.group("id"), "Line is '%s', and match is '%s'" % (
line,
@@ -100,6 +152,16 @@ def scan_file(file: Path) -> Stats:
)
line_identifiers += 1
stats.identifier_widths[len(m.group("id"))] += 1
stats.string_literals += line_string_literals
stats.string_literals_per_line[line_string_literals] += 1
stats.int_literals += line_int_literals
stats.int_literals_per_line[line_int_literals] += 1
stats.float_literals += line_float_literals
stats.float_literals_per_line[line_float_literals] += 1
stats.symbols += line_symbols
stats.symbols_per_line[line_symbols] += 1
stats.keywords += line_keywords
stats.keywords_per_line[line_keywords] += 1
stats.identifiers += line_identifiers
stats.ids_per_line[line_identifiers] += 1
return stats
@@ -136,19 +198,57 @@ Comment lines: %(comment_lines)d
Empty comment lines: %(empty_comment_lines)d
Lines with trailing comments: %(lines_with_trailing_comments)d
Classes: %(classes)d
Internal comments: %(internal_comments)d
String literals: %(string_literals)d
Int literals: %(int_literals)d
Float literals: %(float_literals)d
Symbols: %(symbols)d
Keywords: %(keywords)d
IDs: %(identifiers)d"""
% asdict(stats)
)
tokens = (
stats.string_literals
+ stats.int_literals
+ stats.float_literals
+ stats.symbols
+ stats.keywords
+ stats.identifiers
)
print(
f"""
Fraction of blank lines: {stats.blank_lines / stats.lines}
Fraction of comment lines: {stats.comment_lines / stats.lines}
Total counted tokens: {tokens}
Fraction string literals: {stats.string_literals / tokens}
Fraction int literals: {stats.int_literals / tokens}
Fraction float literals: {stats.float_literals / tokens}
Fraction symbols: {stats.symbols / tokens}
Fraction keywords: {stats.keywords / tokens}
Fraction IDs: {stats.identifiers / tokens}
"""
)
def print_histogram(
title: str, data: Dict[int, int], column_format: str
) -> None:
print()
print(title)
key_min = min(data.keys())
key_max = max(data.keys()) + 1
values = [data.get(k, 0) for k in range(key_min, key_max)]
keys = [column_format % k for k in range(key_min, key_max)]
total = sum(values)
median = key_min
count = total
for k in range(key_min, key_max):
count -= data.get(k, 0)
if count <= total / 2:
median = k
break
print(title + f" (median: {median})")
fig = tpl.figure()
fig.barh(values, keys)
fig.show()
@@ -156,6 +256,29 @@ IDs: %(identifiers)d"""
print_histogram(
"## Comment line widths ##", stats.comment_line_widths, "%d columns"
)
print_histogram(
"## String literals per line ##",
stats.string_literals_per_line,
"%d literals",
)
print_histogram(
"## Int literals per line ##",
stats.int_literals_per_line,
"%d literals",
)
print_histogram(
"## Float literals per line ##",
stats.float_literals_per_line,
"%d literals",
)
print_histogram(
"## Symbols per line ##", stats.symbols_per_line, "%d symbols"
)
print_histogram(
"## Keywords per line ##", stats.keywords_per_line, "%d keywords"
)
print_histogram("## ID widths ##", stats.identifier_widths, "%d characters")
print_histogram("## IDs per line ##", stats.ids_per_line, "%d ids")