Files
carbon-lang/toolchain/lex/tokenized_buffer.h
T
Jon Ross-PerkinsandChandler Carruth cafcd88882 Split lexing logic and storage to separate files. (#3365)
Just reorganizing logic a little, trying to mirror the direction we've
gone with check, lower, etc. That is, lex.h contains a function `Lex`
that is used directly.

Note, I'm avoiding making meaningful changes here. It could in theory
still affect inlining in benchmarks, but I'm not seeing an impact.

Before:

```
------------------------------------------------------------------------------------------------------
Benchmark                                            Time             CPU   Iterations UserCounters...
------------------------------------------------------------------------------------------------------
BM_ValidKeywords                               2784949 ns      2784867 ns          249 bytes_per_second=214.452M/s tokens_per_second=35.9084M/s
BM_ValidKeywordsAsRawIdentifiers               3222597 ns      3222551 ns          210 bytes_per_second=244.513M/s tokens_per_second=31.0313M/s
BM_RawIdentifierFocus                          5907836 ns      5907518 ns          103 bytes_per_second=264.873M/s tokens_per_second=16.9276M/s
BM_ValidIdentifiers<1, 64, false>              6255128 ns      6254297 ns          105 bytes_per_second=235.488M/s tokens_per_second=15.989M/s
BM_ValidIdentifiers<1, 1, true>                3677630 ns      3677398 ns          192 bytes_per_second=77.7999M/s tokens_per_second=27.1931M/s
BM_ValidIdentifiers<3, 5, true>                5427693 ns      5427116 ns          110 bytes_per_second=105.434M/s tokens_per_second=18.426M/s
BM_ValidIdentifiers<3, 16, true>               5063246 ns      5062761 ns          115 bytes_per_second=216.623M/s tokens_per_second=19.7521M/s
BM_ValidIdentifiers<12, 64, true>              5518589 ns      5518118 ns          100 bytes_per_second=691.264M/s tokens_per_second=18.1221M/s
BM_ValidIdentifiers<16, 16, true>              4890776 ns      4890782 ns          112 bytes_per_second=350.989M/s tokens_per_second=20.4466M/s
BM_ValidIdentifiers<24, 24, true>              4974729 ns      4974582 ns          112 bytes_per_second=498.444M/s tokens_per_second=20.1022M/s
BM_ValidIdentifiers<32, 32, true>              5517583 ns      5517085 ns           99 bytes_per_second=587.718M/s tokens_per_second=18.1255M/s
BM_ValidIdentifiers<48, 48, true>              5914759 ns      5914222 ns           94 bytes_per_second=806.255M/s tokens_per_second=16.9084M/s
BM_ValidIdentifiers<64, 64, true>              7556040 ns      7556036 ns           77 bytes_per_second=833.009M/s tokens_per_second=13.2345M/s
BM_ValidIdentifiers<80, 80, true>              7739113 ns      7737696 ns           76 bytes_per_second=1010.65M/s tokens_per_second=12.9237M/s
BM_HorizontalWhitespace/1                      5015062 ns      5014443 ns          108 bytes_per_second=114.111M/s tokens_per_second=19.9424M/s
BM_HorizontalWhitespace/4                      5165496 ns      5165425 ns          111 bytes_per_second=166.163M/s tokens_per_second=19.3595M/s
BM_HorizontalWhitespace/16                     5616796 ns      5616447 ns          102 bytes_per_second=356.578M/s tokens_per_second=17.8049M/s
BM_HorizontalWhitespace/64                     7912904 ns      7912346 ns           78 bytes_per_second=831.648M/s tokens_per_second=12.6385M/s
BM_HorizontalWhitespace/128                   11086218 ns     11083155 ns           57 bytes_per_second=1.11759G/s tokens_per_second=9.0227M/s
BM_RandomSource                                4796549 ns      4795733 ns          145 bytes_per_second=216.783M/s lines_per_second=6.61943M/s tokens_per_second=20.8519M/s
BM_GroupingSymbols/1/0/0                       3937151 ns      3936581 ns          176 bytes_per_second=216.914M/s lines_per_second=19.0521M/s tokens_per_second=25.4028M/s
BM_GroupingSymbols/2/0/0                       3000029 ns      2999506 ns          239 bytes_per_second=243.125M/s lines_per_second=27.7812M/s tokens_per_second=33.3388M/s
BM_GroupingSymbols/3/0/0                       2729059 ns      2728834 ns          251 bytes_per_second=261.26M/s lines_per_second=32.065M/s tokens_per_second=36.6457M/s
BM_GroupingSymbols/4/0/0                       2432363 ns      2432209 ns          291 bytes_per_second=304.738M/s lines_per_second=37.0034M/s tokens_per_second=41.1149M/s
BM_GroupingSymbols/8/0/0                       2144080 ns      2143967 ns          326 bytes_per_second=468.547M/s lines_per_second=44.0468M/s tokens_per_second=46.6425M/s
BM_GroupingSymbols/16/0/0                      2290055 ns      2289711 ns          308 bytes_per_second=741.709M/s lines_per_second=42.3866M/s tokens_per_second=43.6736M/s
BM_GroupingSymbols/32/0/0                      3102790 ns      3102186 ns          220 bytes_per_second=1027.1M/s lines_per_second=31.7437M/s tokens_per_second=32.2353M/s
BM_GroupingSymbols/0/1/0                       3406964 ns      3406421 ns          207 bytes_per_second=222.677M/s lines_per_second=7.33908M/s tokens_per_second=29.3563M/s
BM_GroupingSymbols/0/2/0                       2244101 ns      2244006 ns          312 bytes_per_second=239.985M/s lines_per_second=7.42689M/s tokens_per_second=44.5632M/s
BM_GroupingSymbols/0/3/0                       1976449 ns      1976080 ns          347 bytes_per_second=216M/s lines_per_second=6.32566M/s tokens_per_second=50.6052M/s
BM_GroupingSymbols/0/4/0                       1600539 ns      1600325 ns          429 bytes_per_second=224.777M/s lines_per_second=6.24873M/s tokens_per_second=62.4873M/s
BM_GroupingSymbols/0/8/0                       1085044 ns      1084941 ns          646 bytes_per_second=222.765M/s lines_per_second=5.12009M/s tokens_per_second=92.1709M/s
BM_GroupingSymbols/0/16/0                       824804 ns       824756 ns          817 bytes_per_second=209.166M/s lines_per_second=3.5659M/s tokens_per_second=121.248M/s
BM_GroupingSymbols/0/32/0                       685741 ns       685741 ns         1004 bytes_per_second=196.576M/s lines_per_second=2.20929M/s tokens_per_second=145.828M/s
BM_GroupingSymbols/0/0/1                       3467507 ns      3467077 ns          206 bytes_per_second=218.781M/s lines_per_second=7.21069M/s tokens_per_second=28.8427M/s
BM_GroupingSymbols/0/0/2                       2284154 ns      2283937 ns          309 bytes_per_second=235.79M/s lines_per_second=7.29705M/s tokens_per_second=43.784M/s
BM_GroupingSymbols/0/0/3                       1965548 ns      1965225 ns          356 bytes_per_second=217.193M/s lines_per_second=6.36059M/s tokens_per_second=50.8848M/s
BM_GroupingSymbols/0/0/4                       1623965 ns      1623725 ns          440 bytes_per_second=221.538M/s lines_per_second=6.15868M/s tokens_per_second=61.5868M/s
BM_GroupingSymbols/0/0/8                       1080601 ns      1080452 ns          650 bytes_per_second=223.691M/s lines_per_second=5.14137M/s tokens_per_second=92.5539M/s
BM_GroupingSymbols/0/0/16                       840820 ns       840677 ns          828 bytes_per_second=205.205M/s lines_per_second=3.49837M/s tokens_per_second=118.952M/s
BM_GroupingSymbols/0/0/32                       707793 ns       707734 ns          991 bytes_per_second=190.467M/s lines_per_second=2.14063M/s tokens_per_second=141.296M/s
BM_GroupingSymbols/32/1/0                      2804159 ns      2803869 ns          245 bytes_per_second=1103.7M/s lines_per_second=34.0779M/s tokens_per_second=35.665M/s
BM_GroupingSymbols/32/2/0                      2676229 ns      2675855 ns          261 bytes_per_second=1123.78M/s lines_per_second=34.688M/s tokens_per_second=37.3712M/s
BM_GroupingSymbols/32/3/0                      2652102 ns      2651836 ns          262 bytes_per_second=1103.24M/s lines_per_second=34.0217M/s tokens_per_second=37.7097M/s
BM_GroupingSymbols/32/4/0                      2600677 ns      2600552 ns          268 bytes_per_second=1096.17M/s lines_per_second=33.7678M/s tokens_per_second=38.4534M/s
BM_GroupingSymbols/32/8/0                      2382017 ns      2381869 ns          294 bytes_per_second=1083.42M/s lines_per_second=33.2659M/s tokens_per_second=41.9838M/s
BM_GroupingSymbols/32/16/0                     2102340 ns      2102243 ns          325 bytes_per_second=1034.45M/s lines_per_second=31.5377M/s tokens_per_second=47.5682M/s
BM_GroupingSymbols/32/32/0                     1745079 ns      1744914 ns          403 bytes_per_second=952.64M/s lines_per_second=28.6461M/s tokens_per_second=57.3094M/s
BM_GroupingSymbols/32/32/1                     1701414 ns      1701255 ns          415 bytes_per_second=962.73M/s lines_per_second=28.9228M/s tokens_per_second=58.7802M/s
BM_GroupingSymbols/32/32/2                     1688503 ns      1688121 ns          415 bytes_per_second=957.019M/s lines_per_second=28.7242M/s tokens_per_second=59.2375M/s
BM_GroupingSymbols/32/32/3                     1679701 ns      1679499 ns          419 bytes_per_second=948.651M/s lines_per_second=28.446M/s tokens_per_second=59.5416M/s
BM_GroupingSymbols/32/32/4                     1647815 ns      1647816 ns          427 bytes_per_second=953.342M/s lines_per_second=28.559M/s tokens_per_second=60.6864M/s
BM_GroupingSymbols/32/32/8                     1581283 ns      1581075 ns          450 bytes_per_second=942.39M/s lines_per_second=28.1201M/s tokens_per_second=63.2481M/s
BM_GroupingSymbols/32/32/16                    1445591 ns      1445526 ns          477 bytes_per_second=936.105M/s lines_per_second=27.7442M/s tokens_per_second=69.179M/s
BM_GroupingSymbols/32/32/32                    1296335 ns      1296094 ns          544 bytes_per_second=882.943M/s lines_per_second=25.8276M/s tokens_per_second=77.1549M/s
BM_BlankLines/1                                5774442 ns      5774454 ns          107 bytes_per_second=99.0921M/s lines_per_second=17.3175M/s tokens_per_second=17.3177M/s
BM_BlankLines/4                                8712135 ns      8711977 ns           75 bytes_per_second=98.5198M/s lines_per_second=45.9133M/s tokens_per_second=11.4785M/s
BM_BlankLines/16                              32313782 ns     32307655 ns           22 bytes_per_second=61.9884M/s lines_per_second=49.5234M/s tokens_per_second=3.09524M/s
BM_BlankLines/64                              87562163 ns     87543476 ns            7 bytes_per_second=75.166M/s lines_per_second=73.1058M/s tokens_per_second=1.14229M/s
BM_BlankLines/128                            160336428 ns    160298644 ns            4 bytes_per_second=79.1257M/s lines_per_second=79.8502M/s tokens_per_second=623.836k/s
BM_CommentLines/1/0/0                          7298959 ns      7298427 ns           89 bytes_per_second=117.601M/s lines_per_second=27.4029M/s tokens_per_second=13.7016M/s
BM_CommentLines/4/0/0                         10002223 ns     10002239 ns           67 bytes_per_second=171.622M/s lines_per_second=49.9883M/s tokens_per_second=9.99776M/s
BM_CommentLines/128/0/0                      143356660 ns    143356412 ns            5 bytes_per_second=259.444M/s lines_per_second=89.9846M/s tokens_per_second=697.562k/s
BM_CommentLines/1/30/0                         7421994 ns      7421289 ns           87 bytes_per_second=501.166M/s lines_per_second=26.9492M/s tokens_per_second=13.4747M/s
BM_CommentLines/4/30/0                        11304903 ns     11303972 ns           56 bytes_per_second=1.13696G/s lines_per_second=44.2318M/s tokens_per_second=8.84645M/s
BM_CommentLines/128/30/0                     156244144 ns    156217089 ns            4 bytes_per_second=2.52178G/s lines_per_second=82.5766M/s tokens_per_second=640.135k/s
BM_CommentLines/1/70/0                         7486688 ns      7485340 ns           80 bytes_per_second=1006.49M/s lines_per_second=26.7186M/s tokens_per_second=13.3594M/s
BM_CommentLines/4/70/0                        14762881 ns     14761417 ns           45 bytes_per_second=1.88011G/s lines_per_second=33.8717M/s tokens_per_second=6.77442M/s
BM_CommentLines/128/70/0                     179933089 ns    179903592 ns            4 bytes_per_second=4.84025G/s lines_per_second=71.7044M/s tokens_per_second=555.853k/s
BM_CommentLines/1/0/2                          7473119 ns      7472370 ns           87 bytes_per_second=140.389M/s lines_per_second=26.765M/s tokens_per_second=13.3826M/s
BM_CommentLines/4/0/2                          9702727 ns      9700268 ns           66 bytes_per_second=255.615M/s lines_per_second=51.5445M/s tokens_per_second=10.309M/s
BM_CommentLines/128/0/2                      140504132 ns    140480254 ns            5 bytes_per_second=438.544M/s lines_per_second=91.8269M/s tokens_per_second=711.844k/s
BM_CommentLines/1/30/2                         7530847 ns      7529227 ns           82 bytes_per_second=519.314M/s lines_per_second=26.5629M/s tokens_per_second=13.2816M/s
BM_CommentLines/4/30/2                        11646677 ns     11644972 ns           56 bytes_per_second=1.16764G/s lines_per_second=42.9366M/s tokens_per_second=8.5874M/s
BM_CommentLines/128/30/2                     161292392 ns    161273904 ns            4 bytes_per_second=2.59054G/s lines_per_second=79.9873M/s tokens_per_second=620.063k/s
BM_CommentLines/1/70/2                         7700751 ns      7700365 ns           83 bytes_per_second=1003.16M/s lines_per_second=25.9725M/s tokens_per_second=12.9864M/s
BM_CommentLines/4/70/2                        14133018 ns     14131523 ns           45 bytes_per_second=2.01664G/s lines_per_second=35.3815M/s tokens_per_second=7.07638M/s
BM_CommentLines/128/70/2                     179085026 ns    179057072 ns            4 bytes_per_second=4.99628G/s lines_per_second=72.0433M/s tokens_per_second=558.481k/s
BM_CommentLines/1/0/8                          7792060 ns      7791951 ns           83 bytes_per_second=208.065M/s lines_per_second=25.6673M/s tokens_per_second=12.8338M/s
BM_CommentLines/4/0/8                         10329194 ns     10329006 ns           61 bytes_per_second=461.644M/s lines_per_second=48.4069M/s tokens_per_second=9.68147M/s
BM_CommentLines/128/0/8                      140917902 ns    140917975 ns            5 bytes_per_second=956.927M/s lines_per_second=91.5417M/s tokens_per_second=709.633k/s
BM_CommentLines/1/30/8                         7813308 ns      7811702 ns           82 bytes_per_second=573.784M/s lines_per_second=25.6024M/s tokens_per_second=12.8013M/s
BM_CommentLines/4/30/8                        12052779 ns     12051440 ns           54 bytes_per_second=1.31373G/s lines_per_second=41.4884M/s tokens_per_second=8.29776M/s
BM_CommentLines/128/30/8                     163767409 ns    163753783 ns            4 bytes_per_second=2.9881G/s lines_per_second=78.776M/s tokens_per_second=610.673k/s
BM_CommentLines/1/70/8                         8188137 ns      8187614 ns           80 bytes_per_second=1013.35M/s lines_per_second=24.4269M/s tokens_per_second=12.2136M/s
BM_CommentLines/4/70/8                        15723650 ns     15721559 ns           43 bytes_per_second=1.95485G/s lines_per_second=31.8031M/s tokens_per_second=6.36069M/s
BM_CommentLines/128/70/8                     182579515 ns    182554119 ns            4 bytes_per_second=5.29237G/s lines_per_second=70.6633M/s tokens_per_second=547.783k/s
BM_SpeedOfLightStrCpy                            27009 ns        27006 ns        25887 bytes_per_second=37.594G/s lines_per_second=1.17547G/s tokens_per_second=3.70286G/s
BM_SpeedOfLightDispatch<1>                     1850860 ns      1850660 ns          381 bytes_per_second=561.764M/s lines_per_second=17.1533M/s tokens_per_second=54.0348M/s
BM_SpeedOfLightDispatch<2>                     1876473 ns      1876369 ns          347 bytes_per_second=554.067M/s lines_per_second=16.9183M/s tokens_per_second=53.2944M/s
BM_SpeedOfLightDispatch<4>                     2208441 ns      2208443 ns          314 bytes_per_second=470.755M/s lines_per_second=14.3744M/s tokens_per_second=45.2808M/s
BM_SpeedOfLightDispatch<8>                     2824174 ns      2824137 ns          250 bytes_per_second=368.125M/s lines_per_second=11.2406M/s tokens_per_second=35.409M/s
BM_SpeedOfLightDispatch<16>                    4306633 ns      4306325 ns          165 bytes_per_second=241.42M/s lines_per_second=7.37172M/s tokens_per_second=23.2217M/s
BM_SpeedOfLightDispatch<32>                    6300653 ns      6300102 ns          113 bytes_per_second=165.019M/s lines_per_second=5.03881M/s tokens_per_second=15.8728M/s
BM_SpeedOfLightDispatch<MaxDispatchTargets>    8879663 ns      8878510 ns           79 bytes_per_second=117.096M/s lines_per_second=3.57549M/s tokens_per_second=11.2632M/s
```

After:

```
------------------------------------------------------------------------------------------------------
Benchmark                                            Time             CPU   Iterations UserCounters...
------------------------------------------------------------------------------------------------------
BM_ValidKeywords                               2821833 ns      2821832 ns          247 bytes_per_second=211.642M/s tokens_per_second=35.438M/s
BM_ValidKeywordsAsRawIdentifiers               3204326 ns      3203964 ns          216 bytes_per_second=245.931M/s tokens_per_second=31.2113M/s
BM_RawIdentifierFocus                          6076723 ns      6076107 ns           98 bytes_per_second=257.524M/s tokens_per_second=16.4579M/s
BM_ValidIdentifiers<1, 64, false>              6303093 ns      6302632 ns          101 bytes_per_second=233.682M/s tokens_per_second=15.8664M/s
BM_ValidIdentifiers<1, 1, true>                3687668 ns      3687338 ns          194 bytes_per_second=77.5902M/s tokens_per_second=27.1198M/s
BM_ValidIdentifiers<3, 5, true>                5298920 ns      5298465 ns          106 bytes_per_second=107.994M/s tokens_per_second=18.8734M/s
BM_ValidIdentifiers<3, 16, true>               4880695 ns      4879704 ns          118 bytes_per_second=224.75M/s tokens_per_second=20.493M/s
BM_ValidIdentifiers<12, 64, true>              5413070 ns      5411832 ns          103 bytes_per_second=704.84M/s tokens_per_second=18.478M/s
BM_ValidIdentifiers<16, 16, true>              5052780 ns      5051309 ns          110 bytes_per_second=339.835M/s tokens_per_second=19.7968M/s
BM_ValidIdentifiers<24, 24, true>              5221580 ns      5220851 ns          104 bytes_per_second=474.933M/s tokens_per_second=19.154M/s
BM_ValidIdentifiers<32, 32, true>              5786811 ns      5786806 ns           99 bytes_per_second=560.325M/s tokens_per_second=17.2807M/s
BM_ValidIdentifiers<48, 48, true>              5959506 ns      5959502 ns           96 bytes_per_second=800.129M/s tokens_per_second=16.7799M/s
BM_ValidIdentifiers<64, 64, true>              7831469 ns      7830629 ns           75 bytes_per_second=803.799M/s tokens_per_second=12.7704M/s
BM_ValidIdentifiers<80, 80, true>              7905843 ns      7904727 ns           75 bytes_per_second=989.298M/s tokens_per_second=12.6507M/s
BM_HorizontalWhitespace/1                      5091171 ns      5090660 ns          109 bytes_per_second=112.402M/s tokens_per_second=19.6438M/s
BM_HorizontalWhitespace/4                      5020344 ns      5020344 ns          112 bytes_per_second=170.965M/s tokens_per_second=19.919M/s
BM_HorizontalWhitespace/16                     5846801 ns      5846459 ns           99 bytes_per_second=342.549M/s tokens_per_second=17.1044M/s
BM_HorizontalWhitespace/64                     7803183 ns      7802357 ns           78 bytes_per_second=843.372M/s tokens_per_second=12.8166M/s
BM_HorizontalWhitespace/128                   10600500 ns     10598602 ns           59 bytes_per_second=1.16869G/s tokens_per_second=9.43521M/s
BM_RandomSource                                4824062 ns      4823496 ns          139 bytes_per_second=215.536M/s lines_per_second=6.58133M/s tokens_per_second=20.7319M/s
BM_GroupingSymbols/1/0/0                       4116846 ns      4116549 ns          170 bytes_per_second=207.431M/s lines_per_second=18.2191M/s tokens_per_second=24.2922M/s
BM_GroupingSymbols/2/0/0                       3024336 ns      3024156 ns          236 bytes_per_second=241.144M/s lines_per_second=27.5548M/s tokens_per_second=33.0671M/s
BM_GroupingSymbols/3/0/0                       2789794 ns      2789256 ns          252 bytes_per_second=255.601M/s lines_per_second=31.3704M/s tokens_per_second=35.8519M/s
BM_GroupingSymbols/4/0/0                       2496498 ns      2496237 ns          283 bytes_per_second=296.921M/s lines_per_second=36.0543M/s tokens_per_second=40.0603M/s
BM_GroupingSymbols/8/0/0                       2200846 ns      2200611 ns          313 bytes_per_second=456.487M/s lines_per_second=42.9131M/s tokens_per_second=45.4419M/s
BM_GroupingSymbols/16/0/0                      2415237 ns      2415015 ns          288 bytes_per_second=703.225M/s lines_per_second=40.1873M/s tokens_per_second=41.4076M/s
BM_GroupingSymbols/32/0/0                      3171195 ns      3170504 ns          215 bytes_per_second=1004.97M/s lines_per_second=31.0597M/s tokens_per_second=31.5407M/s
BM_GroupingSymbols/0/1/0                       3627393 ns      3626737 ns          193 bytes_per_second=209.15M/s lines_per_second=6.89325M/s tokens_per_second=27.573M/s
BM_GroupingSymbols/0/2/0                       2501189 ns      2500946 ns          275 bytes_per_second=215.33M/s lines_per_second=6.66388M/s tokens_per_second=39.9849M/s
BM_GroupingSymbols/0/3/0                       2149513 ns      2149282 ns          318 bytes_per_second=198.593M/s lines_per_second=5.8159M/s tokens_per_second=46.5272M/s
BM_GroupingSymbols/0/4/0                       1793658 ns      1793292 ns          384 bytes_per_second=200.59M/s lines_per_second=5.57634M/s tokens_per_second=55.7634M/s
BM_GroupingSymbols/0/8/0                       1302555 ns      1302272 ns          542 bytes_per_second=185.589M/s lines_per_second=4.26562M/s tokens_per_second=76.7889M/s
BM_GroupingSymbols/0/16/0                      1042993 ns      1042818 ns          671 bytes_per_second=165.428M/s lines_per_second=2.82024M/s tokens_per_second=95.8941M/s
BM_GroupingSymbols/0/32/0                       955561 ns       955471 ns          749 bytes_per_second=141.082M/s lines_per_second=1.58561M/s tokens_per_second=104.66M/s
BM_GroupingSymbols/0/0/1                       3659797 ns      3659369 ns          194 bytes_per_second=207.285M/s lines_per_second=6.83178M/s tokens_per_second=27.3271M/s
BM_GroupingSymbols/0/0/2                       2467556 ns      2467190 ns          281 bytes_per_second=218.276M/s lines_per_second=6.75505M/s tokens_per_second=40.5319M/s
BM_GroupingSymbols/0/0/3                       2152274 ns      2151938 ns          326 bytes_per_second=198.348M/s lines_per_second=5.80872M/s tokens_per_second=46.4697M/s
BM_GroupingSymbols/0/0/4                       1805982 ns      1805877 ns          368 bytes_per_second=199.192M/s lines_per_second=5.53747M/s tokens_per_second=55.3747M/s
BM_GroupingSymbols/0/0/8                       1313041 ns      1312833 ns          539 bytes_per_second=184.096M/s lines_per_second=4.23131M/s tokens_per_second=76.1711M/s
BM_GroupingSymbols/0/0/16                      1065565 ns      1065301 ns          659 bytes_per_second=161.937M/s lines_per_second=2.76072M/s tokens_per_second=93.8702M/s
BM_GroupingSymbols/0/0/32                       946630 ns       946514 ns          726 bytes_per_second=142.417M/s lines_per_second=1.60061M/s tokens_per_second=105.651M/s
BM_GroupingSymbols/32/1/0                      2991120 ns      2991121 ns          233 bytes_per_second=1034.6M/s lines_per_second=31.9445M/s tokens_per_second=33.4323M/s
BM_GroupingSymbols/32/2/0                      2893280 ns      2892944 ns          245 bytes_per_second=1039.45M/s lines_per_second=32.085M/s tokens_per_second=34.5669M/s
BM_GroupingSymbols/32/3/0                      2819177 ns      2819024 ns          239 bytes_per_second=1037.81M/s lines_per_second=32.004M/s tokens_per_second=35.4733M/s
BM_GroupingSymbols/32/4/0                      2778376 ns      2778018 ns          249 bytes_per_second=1026.15M/s lines_per_second=31.6107M/s tokens_per_second=35.9969M/s
BM_GroupingSymbols/32/8/0                      2538279 ns      2538279 ns          275 bytes_per_second=1016.65M/s lines_per_second=31.216M/s tokens_per_second=39.3968M/s
BM_GroupingSymbols/32/16/0                     2291819 ns      2291693 ns          305 bytes_per_second=948.937M/s lines_per_second=28.9306M/s tokens_per_second=43.6359M/s
BM_GroupingSymbols/32/32/0                     1943560 ns      1943560 ns          366 bytes_per_second=855.273M/s lines_per_second=25.7183M/s tokens_per_second=51.452M/s
BM_GroupingSymbols/32/32/1                     1902069 ns      1901915 ns          375 bytes_per_second=861.158M/s lines_per_second=25.8713M/s tokens_per_second=52.5786M/s
BM_GroupingSymbols/32/32/2                     1877847 ns      1877752 ns          379 bytes_per_second=860.371M/s lines_per_second=25.8234M/s tokens_per_second=53.2552M/s
BM_GroupingSymbols/32/32/3                     1837280 ns      1837016 ns          381 bytes_per_second=867.308M/s lines_per_second=26.0069M/s tokens_per_second=54.4361M/s
BM_GroupingSymbols/32/32/4                     1841010 ns      1840902 ns          380 bytes_per_second=853.349M/s lines_per_second=25.5636M/s tokens_per_second=54.3212M/s
BM_GroupingSymbols/32/32/8                     1734676 ns      1734437 ns          405 bytes_per_second=859.062M/s lines_per_second=25.6337M/s tokens_per_second=57.6556M/s
BM_GroupingSymbols/32/32/16                    1641169 ns      1640934 ns          422 bytes_per_second=824.63M/s lines_per_second=24.4403M/s tokens_per_second=60.9409M/s
BM_GroupingSymbols/32/32/32                    1506988 ns      1506914 ns          472 bytes_per_second=759.418M/s lines_per_second=22.2143M/s tokens_per_second=66.3608M/s
BM_BlankLines/1                                5658057 ns      5657150 ns          110 bytes_per_second=101.147M/s lines_per_second=17.6766M/s tokens_per_second=17.6767M/s
BM_BlankLines/4                                8346196 ns      8346052 ns           77 bytes_per_second=102.839M/s lines_per_second=47.9264M/s tokens_per_second=11.9817M/s
BM_BlankLines/16                              31147085 ns     31144610 ns           22 bytes_per_second=64.3033M/s lines_per_second=51.3727M/s tokens_per_second=3.21083M/s
BM_BlankLines/64                              83743719 ns     83743762 ns            8 bytes_per_second=78.5765M/s lines_per_second=76.4228M/s tokens_per_second=1.19412M/s
BM_BlankLines/128                            152299627 ns    152274606 ns            4 bytes_per_second=83.2952M/s lines_per_second=84.0578M/s tokens_per_second=656.708k/s
BM_CommentLines/1/0/0                          7535704 ns      7535149 ns           83 bytes_per_second=113.906M/s lines_per_second=26.542M/s tokens_per_second=13.2711M/s
BM_CommentLines/4/0/0                         10107724 ns     10106088 ns           66 bytes_per_second=169.858M/s lines_per_second=49.4746M/s tokens_per_second=9.89503M/s
BM_CommentLines/128/0/0                      130061022 ns    130029826 ns            5 bytes_per_second=286.034M/s lines_per_second=99.207M/s tokens_per_second=769.054k/s
BM_CommentLines/1/30/0                         7773816 ns      7772726 ns           83 bytes_per_second=478.506M/s lines_per_second=25.7307M/s tokens_per_second=12.8655M/s
BM_CommentLines/4/30/0                        11436116 ns     11434452 ns           56 bytes_per_second=1.12398G/s lines_per_second=43.7271M/s tokens_per_second=8.7455M/s
BM_CommentLines/128/30/0                     155047059 ns    155033899 ns            4 bytes_per_second=2.54103G/s lines_per_second=83.2068M/s tokens_per_second=645.02k/s
BM_CommentLines/1/70/0                         8016209 ns      8014861 ns           75 bytes_per_second=939.998M/s lines_per_second=24.9534M/s tokens_per_second=12.4768M/s
BM_CommentLines/4/70/0                        14894752 ns     14891800 ns           44 bytes_per_second=1.86365G/s lines_per_second=33.5752M/s tokens_per_second=6.7151M/s
BM_CommentLines/128/70/0                     176667108 ns    176631061 ns            4 bytes_per_second=4.92993G/s lines_per_second=73.0329M/s tokens_per_second=566.152k/s
BM_CommentLines/1/0/2                          7764475 ns      7763675 ns           84 bytes_per_second=135.121M/s lines_per_second=25.7607M/s tokens_per_second=12.8805M/s
BM_CommentLines/4/0/2                         10238104 ns     10236809 ns           65 bytes_per_second=242.217M/s lines_per_second=48.8429M/s tokens_per_second=9.76867M/s
BM_CommentLines/128/0/2                      130208823 ns    130190640 ns            5 bytes_per_second=473.204M/s lines_per_second=99.0845M/s tokens_per_second=768.104k/s
BM_CommentLines/1/30/2                         7941224 ns      7940584 ns           78 bytes_per_second=492.411M/s lines_per_second=25.1868M/s tokens_per_second=12.5935M/s
BM_CommentLines/4/30/2                        11936879 ns     11934453 ns           56 bytes_per_second=1.13932G/s lines_per_second=41.8951M/s tokens_per_second=8.3791M/s
BM_CommentLines/128/30/2                     156978531 ns    156967142 ns            4 bytes_per_second=2.66162G/s lines_per_second=82.182M/s tokens_per_second=637.076k/s
BM_CommentLines/1/70/2                         8223614 ns      8222923 ns           79 bytes_per_second=939.409M/s lines_per_second=24.322M/s tokens_per_second=12.1611M/s
BM_CommentLines/4/70/2                        15216239 ns     15215047 ns           45 bytes_per_second=1.87303G/s lines_per_second=32.8619M/s tokens_per_second=6.57244M/s
BM_CommentLines/128/70/2                     176810589 ns    176755958 ns            4 bytes_per_second=5.06133G/s lines_per_second=72.9813M/s tokens_per_second=565.752k/s
BM_CommentLines/1/0/8                          7901146 ns      7899003 ns           82 bytes_per_second=205.245M/s lines_per_second=25.3194M/s tokens_per_second=12.6598M/s
BM_CommentLines/4/0/8                         10135919 ns     10134576 ns           66 bytes_per_second=470.501M/s lines_per_second=49.3356M/s tokens_per_second=9.86721M/s
BM_CommentLines/128/0/8                      132206448 ns    132206347 ns            5 bytes_per_second=1019.98M/s lines_per_second=97.5738M/s tokens_per_second=756.393k/s
BM_CommentLines/1/30/8                         7981189 ns      7981202 ns           79 bytes_per_second=561.598M/s lines_per_second=25.0586M/s tokens_per_second=12.5294M/s
BM_CommentLines/4/30/8                        12311389 ns     12309078 ns           54 bytes_per_second=1.28623G/s lines_per_second=40.62M/s tokens_per_second=8.12409M/s
BM_CommentLines/128/30/8                     160432032 ns    160393999 ns            4 bytes_per_second=3.05069G/s lines_per_second=80.4261M/s tokens_per_second=623.465k/s
BM_CommentLines/1/70/8                         8199080 ns      8199078 ns           79 bytes_per_second=1011.93M/s lines_per_second=24.3927M/s tokens_per_second=12.1965M/s
BM_CommentLines/4/70/8                        16087954 ns     16086159 ns           44 bytes_per_second=1.91055G/s lines_per_second=31.0823M/s tokens_per_second=6.21652M/s
BM_CommentLines/128/70/8                     175714908 ns    175675241 ns            4 bytes_per_second=5.4996G/s lines_per_second=73.4302M/s tokens_per_second=569.232k/s
BM_SpeedOfLightStrCpy                            28860 ns        28858 ns        26092 bytes_per_second=35.181G/s lines_per_second=1.10002G/s tokens_per_second=3.46519G/s
BM_SpeedOfLightDispatch<1>                     1823558 ns      1823473 ns          385 bytes_per_second=570.14M/s lines_per_second=17.4091M/s tokens_per_second=54.8404M/s
BM_SpeedOfLightDispatch<2>                     2013453 ns      2013250 ns          343 bytes_per_second=516.396M/s lines_per_second=15.768M/s tokens_per_second=49.6709M/s
BM_SpeedOfLightDispatch<4>                     2225145 ns      2224920 ns          312 bytes_per_second=467.268M/s lines_per_second=14.2679M/s tokens_per_second=44.9454M/s
BM_SpeedOfLightDispatch<8>                     2851730 ns      2851590 ns          251 bytes_per_second=364.581M/s lines_per_second=11.1324M/s tokens_per_second=35.0682M/s
BM_SpeedOfLightDispatch<16>                    4431584 ns      4431156 ns          161 bytes_per_second=234.619M/s lines_per_second=7.16404M/s tokens_per_second=22.5675M/s
BM_SpeedOfLightDispatch<32>                    6229285 ns      6228092 ns          111 bytes_per_second=166.927M/s lines_per_second=5.09707M/s tokens_per_second=16.0563M/s
BM_SpeedOfLightDispatch<MaxDispatchTargets>    8967228 ns      8965583 ns           79 bytes_per_second=115.958M/s lines_per_second=3.54076M/s tokens_per_second=11.1538M/s
```

---------

Co-authored-by: Chandler Carruth <chandlerc@gmail.com>
2023-11-10 23:59:49 +00:00

375 lines
13 KiB
C++

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#ifndef CARBON_TOOLCHAIN_LEX_TOKENIZED_BUFFER_H_
#define CARBON_TOOLCHAIN_LEX_TOKENIZED_BUFFER_H_
#include <cstdint>
#include <iterator>
#include "common/ostream.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/raw_ostream.h"
#include "toolchain/base/index_base.h"
#include "toolchain/base/value_store.h"
#include "toolchain/diagnostics/diagnostic_emitter.h"
#include "toolchain/lex/token_kind.h"
#include "toolchain/source/source_buffer.h"
namespace Carbon::Lex {
class TokenizedBuffer;
// A lightweight handle to a lexed token in a `TokenizedBuffer`.
//
// `Token` objects are designed to be passed by value, not reference or
// pointer. They are also designed to be small and efficient to store in data
// structures.
//
// `Token` objects from the same `TokenizedBuffer` can be compared with each
// other, both for being the same token within the buffer, and to establish
// relative position within the token stream that has been lexed out of the
// buffer. `Token` objects from different `TokenizedBuffer`s cannot be
// meaningfully compared.
//
// All other APIs to query a `Token` are on the `TokenizedBuffer`.
struct Token : public ComparableIndexBase {
static const Token Invalid;
// Comments aren't tokenized, so this is the first token after StartOfFile.
static const Token FirstNonCommentToken;
using ComparableIndexBase::ComparableIndexBase;
};
constexpr Token Token::Invalid(Token::InvalidIndex);
constexpr Token Token::FirstNonCommentToken(1);
// A lightweight handle to a lexed line in a `TokenizedBuffer`.
//
// `Line` objects are designed to be passed by value, not reference or
// pointer. They are also designed to be small and efficient to store in data
// structures.
//
// Each `Line` object refers to a specific line in the source code that was
// lexed. They can be compared directly to establish that they refer to the
// same line or the relative position of different lines within the source.
//
// All other APIs to query a `Line` are on the `TokenizedBuffer`.
struct Line : public ComparableIndexBase {
static const Line Invalid;
using ComparableIndexBase::ComparableIndexBase;
};
constexpr Line Line::Invalid(Line::InvalidIndex);
// Random-access iterator over tokens within the buffer.
class TokenIterator
: public llvm::iterator_facade_base<
TokenIterator, std::random_access_iterator_tag, const Token, int>,
public Printable<TokenIterator> {
public:
TokenIterator() = delete;
explicit TokenIterator(Token token) : token_(token) {}
auto operator==(const TokenIterator& rhs) const -> bool {
return token_ == rhs.token_;
}
auto operator<(const TokenIterator& rhs) const -> bool {
return token_ < rhs.token_;
}
auto operator*() const -> const Token& { return token_; }
using iterator_facade_base::operator-;
auto operator-(const TokenIterator& rhs) const -> int {
return token_.index - rhs.token_.index;
}
auto operator+=(int n) -> TokenIterator& {
token_.index += n;
return *this;
}
auto operator-=(int n) -> TokenIterator& {
token_.index -= n;
return *this;
}
// Prints the raw token index.
auto Print(llvm::raw_ostream& output) const -> void;
private:
friend class TokenizedBuffer;
Token token_;
};
// A diagnostic location translator that maps token locations into source
// buffer locations.
class TokenLocationTranslator : public DiagnosticLocationTranslator<Token> {
public:
explicit TokenLocationTranslator(const TokenizedBuffer* buffer)
: buffer_(buffer) {}
// Map the given token into a diagnostic location.
auto GetLocation(Token token) -> DiagnosticLocation override;
private:
const TokenizedBuffer* buffer_;
};
// A buffer of tokenized Carbon source code.
//
// This is constructed by lexing the source code text into a series of tokens.
// The buffer provides lightweight handles to tokens and other lexed entities,
// as well as iterations to walk the sequence of tokens found in the buffer.
//
// Lexing errors result in a potentially incomplete sequence of tokens and
// `HasError` returning true.
class TokenizedBuffer : public Printable<TokenizedBuffer> {
public:
[[nodiscard]] auto GetKind(Token token) const -> TokenKind;
[[nodiscard]] auto GetLine(Token token) const -> Line;
// Returns the 1-based line number.
[[nodiscard]] auto GetLineNumber(Token token) const -> int;
// Returns the 1-based column number.
[[nodiscard]] auto GetColumnNumber(Token token) const -> int;
// Returns the source text lexed into this token.
[[nodiscard]] auto GetTokenText(Token token) const -> llvm::StringRef;
// Returns the identifier associated with this token. The token kind must be
// an `Identifier`.
[[nodiscard]] auto GetIdentifier(Token token) const -> IdentifierId;
// Returns the value of an `IntegerLiteral()` token.
[[nodiscard]] auto GetIntegerLiteral(Token token) const -> IntegerId;
// Returns the value of an `RealLiteral()` token.
[[nodiscard]] auto GetRealLiteral(Token token) const -> RealId;
// Returns the value of a `StringLiteral()` token.
[[nodiscard]] auto GetStringLiteral(Token token) const -> StringLiteralId;
// Returns the size specified in a `*TypeLiteral()` token.
[[nodiscard]] auto GetTypeLiteralSize(Token token) const
-> const llvm::APInt&;
// Returns the closing token matched with the given opening token.
//
// The given token must be an opening token kind.
[[nodiscard]] auto GetMatchedClosingToken(Token opening_token) const -> Token;
// Returns the opening token matched with the given closing token.
//
// The given token must be a closing token kind.
[[nodiscard]] auto GetMatchedOpeningToken(Token closing_token) const -> Token;
// Returns whether the given token has leading whitespace.
[[nodiscard]] auto HasLeadingWhitespace(Token token) const -> bool;
// Returns whether the given token has trailing whitespace.
[[nodiscard]] auto HasTrailingWhitespace(Token token) const -> bool;
// Returns whether the token was created as part of an error recovery effort.
//
// For example, a closing paren inserted to match an unmatched paren.
[[nodiscard]] auto IsRecoveryToken(Token token) const -> bool;
// Returns the 1-based line number.
[[nodiscard]] auto GetLineNumber(Line line) const -> int;
// Returns the 1-based indentation column number.
[[nodiscard]] auto GetIndentColumnNumber(Line line) const -> int;
// Returns the next line handle.
[[nodiscard]] auto GetNextLine(Line line) const -> Line;
// Returns the previous line handle.
[[nodiscard]] auto GetPrevLine(Line line) const -> Line;
// Prints a description of the tokenized stream to the provided `raw_ostream`.
//
// It prints one line of information for each token in the buffer, including
// the kind of token, where it occurs within the source file, indentation for
// the associated line, the spelling of the token in source, and any
// additional information tracked such as which unique identifier it is or any
// matched grouping token.
//
// Each line is formatted as a YAML record:
//
// clang-format off
// ```
// token: { index: 0, kind: 'Semi', line: 1, column: 1, indent: 1, spelling: ';' }
// ```
// clang-format on
//
// This can be parsed as YAML using tools like `python-yq` combined with `jq`
// on the command line. The format is also reasonably amenable to other
// line-oriented shell tools from `grep` to `awk`.
auto Print(llvm::raw_ostream& output_stream) const -> void;
// Prints a description of a single token. See `Print` for details on the
// format.
auto PrintToken(llvm::raw_ostream& output_stream, Token token) const -> void;
// Returns true if the buffer has errors that were detected at lexing time.
[[nodiscard]] auto has_errors() const -> bool { return has_errors_; }
[[nodiscard]] auto tokens() const -> llvm::iterator_range<TokenIterator> {
return llvm::make_range(TokenIterator(Token(0)),
TokenIterator(Token(token_infos_.size())));
}
[[nodiscard]] auto size() const -> int { return token_infos_.size(); }
[[nodiscard]] auto expected_parse_tree_size() const -> int {
return expected_parse_tree_size_;
}
auto filename() const -> llvm::StringRef { return source_->filename(); }
private:
friend class Lexer;
friend class TokenLocationTranslator;
// A diagnostic location translator that maps token locations into source
// buffer locations.
class SourceBufferLocationTranslator
: public DiagnosticLocationTranslator<const char*> {
public:
explicit SourceBufferLocationTranslator(const TokenizedBuffer* buffer)
: buffer_(buffer) {}
// Map the given position within the source buffer into a diagnostic
// location.
auto GetLocation(const char* loc) -> DiagnosticLocation override;
private:
const TokenizedBuffer* buffer_;
};
// Specifies minimum widths to use when printing a token's fields via
// `printToken`.
struct PrintWidths {
// Widens `this` to the maximum of `this` and `new_width` for each
// dimension.
auto Widen(const PrintWidths& widths) -> void;
int index;
int kind;
int line;
int column;
int indent;
};
struct TokenInfo {
TokenKind kind;
// Whether the token has trailing whitespace.
bool has_trailing_space = false;
// Whether the token was injected artificially during error recovery.
bool is_recovery = false;
// Line on which the Token starts.
Line token_line;
// Zero-based byte offset of the token within its line.
int32_t column;
// We may have up to 32 bits of payload, based on the kind of token.
union {
static_assert(
sizeof(Token) <= sizeof(int32_t),
"Unable to pack token and identifier index into the same space!");
IdentifierId ident_id = IdentifierId::Invalid;
StringLiteralId string_literal_id;
IntegerId integer_id;
RealId real_id;
Token closing_token;
Token opening_token;
int32_t error_length;
};
};
struct LineInfo {
// The length will always be assigned later. Indent may be assigned if
// non-zero.
explicit LineInfo(int64_t start)
: start(start),
length(static_cast<int32_t>(llvm::StringRef::npos)),
indent(0) {}
explicit LineInfo(int64_t start, int32_t length)
: start(start), length(length), indent(0) {}
// Zero-based byte offset of the start of the line within the source buffer
// provided.
int64_t start;
// The byte length of the line. Does not include the newline character (or a
// nul-terminator or EOF).
int32_t length;
// The byte offset from the start of the line of the first non-whitespace
// character.
int32_t indent;
};
// The constructor is merely responsible for trivial initialization of
// members. A working object of this type is built with `Lex::Lex` so that its
// return can indicate if an error was encountered while lexing.
explicit TokenizedBuffer(SharedValueStores& value_stores,
SourceBuffer& source)
: value_stores_(&value_stores), source_(&source) {}
auto GetLineInfo(Line line) -> LineInfo&;
[[nodiscard]] auto GetLineInfo(Line line) const -> const LineInfo&;
auto AddLine(LineInfo info) -> Line;
auto GetTokenInfo(Token token) -> TokenInfo&;
[[nodiscard]] auto GetTokenInfo(Token token) const -> const TokenInfo&;
auto AddToken(TokenInfo info) -> Token;
[[nodiscard]] auto GetTokenPrintWidths(Token token) const -> PrintWidths;
auto PrintToken(llvm::raw_ostream& output_stream, Token token,
PrintWidths widths) const -> void;
// Used to allocate computed string literals.
llvm::BumpPtrAllocator allocator_;
SharedValueStores* value_stores_;
SourceBuffer* source_;
llvm::SmallVector<TokenInfo> token_infos_;
llvm::SmallVector<LineInfo> line_infos_;
// Stores the computed value of string literals so that StringRefs are
// durable.
llvm::SmallVector<std::unique_ptr<std::string>> computed_strings_;
// The number of parse tree nodes that we expect to be created for the tokens
// in this buffer.
int expected_parse_tree_size_ = 0;
bool has_errors_ = false;
};
// A diagnostic emitter that uses positions within a source buffer's text as
// its source of location information.
using LexerDiagnosticEmitter = DiagnosticEmitter<const char*>;
// A diagnostic emitter that uses tokens as its source of location information.
using TokenDiagnosticEmitter = DiagnosticEmitter<Token>;
} // namespace Carbon::Lex
#endif // CARBON_TOOLCHAIN_LEX_TOKENIZED_BUFFER_H_