mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 21:00:13 +01:00
Chandler Carruth
b344f3af12
Not sure how these got missed when moving other things to `uv`, but this should clean them up. The bump to Python 3.12 is so that we can use `@override` with the simple import from `typing`. This is needed by the newest versions of `ty` to do type checking. Added the relevant `@override` annotations. Assisted-by: Antigravity with Gemini
318 lines
11 KiB
Python
Executable File
318 lines
11 KiB
Python
Executable File
#!/usr/bin/env -S uv run --script
|
|
|
|
# /// script
|
|
# requires-python = ">=3.12"
|
|
# ///
|
|
|
|
|
|
"""Script to compute statistics about source code."""
|
|
|
|
from __future__ import annotations
|
|
|
|
__copyright__ = """
|
|
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
Exceptions. See /LICENSE for license information.
|
|
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
"""
|
|
|
|
import argparse
|
|
import math
|
|
import re
|
|
from collections import Counter
|
|
from dataclasses import asdict, dataclass, field
|
|
from multiprocessing import Pool
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import termplotlib as tpl # type: ignore
|
|
from alive_progress import alive_bar # type: ignore
|
|
|
|
BLANK_RE = re.compile(r"\s*")
|
|
COMMENT_RE = re.compile(r"\s*///*\s*")
|
|
LINE_RE = re.compile(
|
|
r"""
|
|
(?P<class_intro>\b(class|struct)\s+(?P<class_name>\w+)\b)|
|
|
(?P<end_open_curly>{\s*(?P<open_curly_trailing_comment>//.*)?)|
|
|
(?P<trailing_comment>//.*)|
|
|
(?P<internal_comment>/\*.*\*/)|
|
|
(?P<string_literal>"([^"]|\\")*"|'([^']|\\')*')|
|
|
(?P<float_literal>\b(0[xb][0-9a-fA-F']*|[0-9][0-9']*)\.[0-9a-fA-F']*([eEpP][0-9a-fA-F']*)?)|
|
|
(?P<int_literal>\b(0[xb][0-9a-fA-F']+|[0-9][0-9']*)([eEpP][0-9a-fA-F']*)?)|
|
|
(?P<symbol>[\[\]{}(),.;]|[-+=!@#$%^&*/?|<>]+)|
|
|
(?P<keyword>\b(auto|bool|break|case|catch|char|class|const|continue|default|do|double|else|enum|explicit|extern|false|float|for|friend|goto|if|inline|int|long|mutable|namespace|new|nullptr|operator|private|protected|public|return|short|signed|sizeof|static|struct|switch|template|this|throw|true|try|typedef|union|unsigned|using|virtual|void|while)\b)|
|
|
(?P<id>\b\w+\b)
|
|
""",
|
|
re.X,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Stats:
|
|
"""Stats collected while scanning source files"""
|
|
|
|
lines: int = 0
|
|
blank_lines: int = 0
|
|
comment_lines: int = 0
|
|
empty_comment_lines: int = 0
|
|
comment_line_widths: Counter[int] = field(default_factory=lambda: Counter())
|
|
lines_with_trailing_comments: int = 0
|
|
classes: int = 0
|
|
internal_comments: int = 0
|
|
string_literals: int = 0
|
|
string_literals_per_line: Counter[int] = field(
|
|
default_factory=lambda: Counter()
|
|
)
|
|
int_literals: int = 0
|
|
int_literals_per_line: Counter[int] = field(
|
|
default_factory=lambda: Counter()
|
|
)
|
|
float_literals: int = 0
|
|
float_literals_per_line: Counter[int] = field(
|
|
default_factory=lambda: Counter()
|
|
)
|
|
symbols: int = 0
|
|
symbols_per_line: Counter[int] = field(default_factory=lambda: Counter())
|
|
keywords: int = 0
|
|
keywords_per_line: Counter[int] = field(default_factory=lambda: Counter())
|
|
identifiers: int = 0
|
|
identifier_widths: Counter[int] = field(default_factory=lambda: Counter())
|
|
ids_per_line: Counter[int] = field(default_factory=lambda: Counter())
|
|
unique_ids_per_ten_lines: Counter[int] = field(
|
|
default_factory=lambda: Counter()
|
|
)
|
|
|
|
def accumulate(self, other: Stats) -> None:
|
|
self.lines += other.lines
|
|
self.blank_lines += other.blank_lines
|
|
self.empty_comment_lines += other.empty_comment_lines
|
|
self.comment_lines += other.comment_lines
|
|
self.comment_line_widths.update(other.comment_line_widths)
|
|
self.lines_with_trailing_comments += other.lines_with_trailing_comments
|
|
self.classes += other.classes
|
|
self.internal_comments += other.internal_comments
|
|
self.string_literals += other.string_literals
|
|
self.string_literals_per_line.update(other.string_literals_per_line)
|
|
self.int_literals += other.int_literals
|
|
self.int_literals_per_line.update(other.int_literals_per_line)
|
|
self.float_literals += other.float_literals
|
|
self.float_literals_per_line.update(other.float_literals_per_line)
|
|
self.symbols += other.symbols
|
|
self.symbols_per_line.update(other.symbols_per_line)
|
|
self.keywords += other.keywords
|
|
self.keywords_per_line.update(other.keywords_per_line)
|
|
self.identifiers += other.identifiers
|
|
self.identifier_widths.update(other.identifier_widths)
|
|
self.ids_per_line.update(other.ids_per_line)
|
|
self.unique_ids_per_ten_lines.update(other.unique_ids_per_ten_lines)
|
|
|
|
|
|
def scan_file(file: Path) -> Stats:
|
|
"""Scans the provided file and accumulates stats."""
|
|
stats = Stats()
|
|
unique_ids = set()
|
|
for line in file.open():
|
|
# Strip off the line endings.
|
|
line = line.rstrip("\r\n")
|
|
# Skip over super long lines that are often URLs or structured data that
|
|
# doesn't match "normal" source code patterns.
|
|
if len(line) > 80:
|
|
continue
|
|
stats.lines += 1
|
|
if re.fullmatch(BLANK_RE, line):
|
|
stats.blank_lines += 1
|
|
continue
|
|
if m := re.match(COMMENT_RE, line):
|
|
stats.comment_lines += 1
|
|
if m.end() == len(line):
|
|
stats.empty_comment_lines += 1
|
|
else:
|
|
stats.comment_line_widths[len(line)] += 1
|
|
continue
|
|
line_string_literals = 0
|
|
line_int_literals = 0
|
|
line_float_literals = 0
|
|
line_symbols = 0
|
|
line_keywords = 0
|
|
line_identifiers = 0
|
|
for m in re.finditer(LINE_RE, line):
|
|
if m.group("trailing_comment"):
|
|
stats.lines_with_trailing_comments += 1
|
|
break
|
|
if m.group("class_intro"):
|
|
stats.classes += 1
|
|
line_keywords += 1
|
|
line_identifiers += 1
|
|
stats.identifier_widths[len(m.group("class_name"))] += 1
|
|
elif m.group("end_open_curly"):
|
|
line_symbols += 1
|
|
elif m.group("internal_comment"):
|
|
stats.internal_comments += 1
|
|
elif m.group("string_literal"):
|
|
line_string_literals += 1
|
|
elif m.group("int_literal"):
|
|
line_int_literals += 1
|
|
elif m.group("float_literal"):
|
|
line_float_literals += 1
|
|
elif m.group("symbol"):
|
|
line_symbols += 1
|
|
elif m.group("keyword"):
|
|
line_keywords += 1
|
|
else:
|
|
assert m.group("id"), "Line is '%s', and match is '%s'" % (
|
|
line,
|
|
line[m.start() : m.end()],
|
|
)
|
|
line_identifiers += 1
|
|
stats.identifier_widths[len(m.group("id"))] += 1
|
|
unique_ids.add(m.group("id"))
|
|
stats.string_literals += line_string_literals
|
|
stats.string_literals_per_line[line_string_literals] += 1
|
|
stats.int_literals += line_int_literals
|
|
stats.int_literals_per_line[line_int_literals] += 1
|
|
stats.float_literals += line_float_literals
|
|
stats.float_literals_per_line[line_float_literals] += 1
|
|
stats.symbols += line_symbols
|
|
stats.symbols_per_line[line_symbols] += 1
|
|
stats.keywords += line_keywords
|
|
stats.keywords_per_line[line_keywords] += 1
|
|
stats.identifiers += line_identifiers
|
|
stats.ids_per_line[line_identifiers] += 1
|
|
if stats.lines > 0:
|
|
stats.unique_ids_per_ten_lines[
|
|
math.ceil((len(unique_ids) * 10) / stats.lines)
|
|
] += 1
|
|
return stats
|
|
|
|
|
|
def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace:
|
|
"""Parsers command-line arguments and flags."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"files",
|
|
metavar="FILE",
|
|
type=Path,
|
|
nargs="+",
|
|
help="A file to scan while collecting statistics.",
|
|
)
|
|
return parser.parse_args(args=args)
|
|
|
|
|
|
def main() -> None:
|
|
parsed_args = parse_args()
|
|
stats = Stats()
|
|
with alive_bar(len(parsed_args.files)) as bar:
|
|
with Pool() as p:
|
|
for file_stats in p.imap_unordered(scan_file, parsed_args.files):
|
|
stats.accumulate(file_stats)
|
|
bar()
|
|
|
|
print(
|
|
"""
|
|
## Stats ##
|
|
Lines: %(lines)d
|
|
Blank lines: %(blank_lines)d
|
|
Comment lines: %(comment_lines)d
|
|
Empty comment lines: %(empty_comment_lines)d
|
|
Lines with trailing comments: %(lines_with_trailing_comments)d
|
|
Classes: %(classes)d
|
|
Internal comments: %(internal_comments)d
|
|
String literals: %(string_literals)d
|
|
Int literals: %(int_literals)d
|
|
Float literals: %(float_literals)d
|
|
Symbols: %(symbols)d
|
|
Keywords: %(keywords)d
|
|
IDs: %(identifiers)d"""
|
|
% asdict(stats)
|
|
)
|
|
|
|
tokens = (
|
|
stats.string_literals
|
|
+ stats.int_literals
|
|
+ stats.float_literals
|
|
+ stats.symbols
|
|
+ stats.keywords
|
|
+ stats.identifiers
|
|
)
|
|
print(f"""
|
|
Fraction of blank lines: {stats.blank_lines / stats.lines}
|
|
Fraction of comment lines: {stats.comment_lines / stats.lines}
|
|
|
|
Total counted tokens: {tokens}
|
|
Fraction string literals: {stats.string_literals / tokens}
|
|
Fraction int literals: {stats.int_literals / tokens}
|
|
Fraction float literals: {stats.float_literals / tokens}
|
|
Fraction symbols: {stats.symbols / tokens}
|
|
Fraction keywords: {stats.keywords / tokens}
|
|
Fraction IDs: {stats.identifiers / tokens}
|
|
""")
|
|
|
|
def print_histogram(
|
|
title: str, data: dict[int, int], column_format: str
|
|
) -> None:
|
|
print()
|
|
key_min = min(data.keys())
|
|
key_max = max(data.keys()) + 1
|
|
values = [data.get(k, 0) for k in range(key_min, key_max)]
|
|
keys = [column_format % k for k in range(key_min, key_max)]
|
|
total = sum(values)
|
|
median = key_min
|
|
p90 = key_min
|
|
p95 = key_min
|
|
p99 = key_min
|
|
count = total
|
|
for k in range(key_min, key_max):
|
|
count -= data.get(k, 0)
|
|
if median == key_min and count <= total / 2:
|
|
median = k
|
|
if p90 == key_min and count <= total / 10:
|
|
p90 = k
|
|
if p95 == key_min and count <= total / 20:
|
|
p95 = k
|
|
if p99 == key_min and count <= total / 100:
|
|
p99 = k
|
|
|
|
print(
|
|
title + f" (median: {median}, p90: {p90}, p95: {p95}, p99: {p99})"
|
|
)
|
|
fig = tpl.figure()
|
|
fig.barh(values, keys)
|
|
fig.show()
|
|
|
|
print_histogram(
|
|
"## Comment line widths ##", stats.comment_line_widths, "%d columns"
|
|
)
|
|
|
|
print_histogram(
|
|
"## String literals per line ##",
|
|
stats.string_literals_per_line,
|
|
"%d literals",
|
|
)
|
|
print_histogram(
|
|
"## Int literals per line ##",
|
|
stats.int_literals_per_line,
|
|
"%d literals",
|
|
)
|
|
print_histogram(
|
|
"## Float literals per line ##",
|
|
stats.float_literals_per_line,
|
|
"%d literals",
|
|
)
|
|
print_histogram(
|
|
"## Symbols per line ##", stats.symbols_per_line, "%d symbols"
|
|
)
|
|
print_histogram(
|
|
"## Keywords per line ##", stats.keywords_per_line, "%d keywords"
|
|
)
|
|
|
|
print_histogram("## ID widths ##", stats.identifier_widths, "%d characters")
|
|
print_histogram("## IDs per line ##", stats.ids_per_line, "%d ids")
|
|
print_histogram(
|
|
"## Unique IDs per 10 lines ##",
|
|
stats.unique_ids_per_ten_lines,
|
|
"%d ids",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|