mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-11-16 07:58:14 +00:00
Just reorganizing logic a little, trying to mirror the direction we've gone with check, lower, etc. That is, lex.h contains a function `Lex` that is used directly. Note, I'm avoiding making meaningful changes here. It could in theory still affect inlining in benchmarks, but I'm not seeing an impact. Before: ``` ------------------------------------------------------------------------------------------------------ Benchmark Time CPU Iterations UserCounters... ------------------------------------------------------------------------------------------------------ BM_ValidKeywords 2784949 ns 2784867 ns 249 bytes_per_second=214.452M/s tokens_per_second=35.9084M/s BM_ValidKeywordsAsRawIdentifiers 3222597 ns 3222551 ns 210 bytes_per_second=244.513M/s tokens_per_second=31.0313M/s BM_RawIdentifierFocus 5907836 ns 5907518 ns 103 bytes_per_second=264.873M/s tokens_per_second=16.9276M/s BM_ValidIdentifiers<1, 64, false> 6255128 ns 6254297 ns 105 bytes_per_second=235.488M/s tokens_per_second=15.989M/s BM_ValidIdentifiers<1, 1, true>3677630ns 3677398 ns 192 bytes_per_second=77.7999M/s tokens_per_second=27.1931M/s BM_ValidIdentifiers<3, 5, true> 5427693 ns 5427116 ns 110 bytes_per_second=105.434M/s tokens_per_second=18.426M/s BM_ValidIdentifiers<3, 16, true> 5063246 ns 5062761 ns 115 bytes_per_second=216.623M/s tokens_per_second=19.7521M/s BM_ValidIdentifiers<12, 64, true>5518589ns 5518118 ns 100 bytes_per_second=691.264M/s tokens_per_second=18.1221M/s BM_ValidIdentifiers<16, 16, true> 4890776 ns 4890782 ns 112 bytes_per_second=350.989M/s tokens_per_second=20.4466M/s BM_ValidIdentifiers<24, 24, true> 4974729 ns 4974582 ns 112 bytes_per_second=498.444M/s tokens_per_second=20.1022M/s BM_ValidIdentifiers<32, 32, true> 5517583 ns 5517085 ns 99 bytes_per_second=587.718M/s tokens_per_second=18.1255M/s BM_ValidIdentifiers<48, 48, true> 5914759 ns 5914222 ns 94 bytes_per_second=806.255M/s tokens_per_second=16.9084M/s BM_ValidIdentifiers<64, 64, true> 7556040 ns 7556036 ns 77 bytes_per_second=833.009M/s tokens_per_second=13.2345M/s BM_ValidIdentifiers<80, 80, true> 7739113 ns 7737696 ns 76 bytes_per_second=1010.65M/s tokens_per_second=12.9237M/s BM_HorizontalWhitespace/1 5015062 ns 5014443 ns 108 bytes_per_second=114.111M/s tokens_per_second=19.9424M/s BM_HorizontalWhitespace/4 5165496 ns 5165425 ns 111 bytes_per_second=166.163M/s tokens_per_second=19.3595M/s BM_HorizontalWhitespace/16 5616796 ns 5616447 ns 102 bytes_per_second=356.578M/s tokens_per_second=17.8049M/s BM_HorizontalWhitespace/64 7912904 ns 7912346 ns 78 bytes_per_second=831.648M/s tokens_per_second=12.6385M/s BM_HorizontalWhitespace/128 11086218 ns 11083155 ns 57 bytes_per_second=1.11759G/s tokens_per_second=9.0227M/s BM_RandomSource 4796549 ns 4795733 ns 145 bytes_per_second=216.783M/s lines_per_second=6.61943M/s tokens_per_second=20.8519M/s BM_GroupingSymbols/1/0/0 3937151 ns 3936581 ns 176 bytes_per_second=216.914M/s lines_per_second=19.0521M/s tokens_per_second=25.4028M/s BM_GroupingSymbols/2/0/0 3000029 ns 2999506 ns 239 bytes_per_second=243.125M/s lines_per_second=27.7812M/s tokens_per_second=33.3388M/s BM_GroupingSymbols/3/0/0 2729059 ns 2728834 ns 251 bytes_per_second=261.26M/s lines_per_second=32.065M/s tokens_per_second=36.6457M/s BM_GroupingSymbols/4/0/0 2432363 ns 2432209 ns 291 bytes_per_second=304.738M/s lines_per_second=37.0034M/s tokens_per_second=41.1149M/s BM_GroupingSymbols/8/0/0 2144080 ns 2143967 ns 326 bytes_per_second=468.547M/s lines_per_second=44.0468M/s tokens_per_second=46.6425M/s BM_GroupingSymbols/16/0/0 2290055 ns 2289711 ns 308 bytes_per_second=741.709M/s lines_per_second=42.3866M/s tokens_per_second=43.6736M/s BM_GroupingSymbols/32/0/0 3102790 ns 3102186 ns 220 bytes_per_second=1027.1M/s lines_per_second=31.7437M/s tokens_per_second=32.2353M/s BM_GroupingSymbols/0/1/0 3406964 ns 3406421 ns 207 bytes_per_second=222.677M/s lines_per_second=7.33908M/s tokens_per_second=29.3563M/s BM_GroupingSymbols/0/2/0 2244101 ns 2244006 ns 312 bytes_per_second=239.985M/s lines_per_second=7.42689M/s tokens_per_second=44.5632M/s BM_GroupingSymbols/0/3/0 1976449 ns 1976080 ns 347 bytes_per_second=216M/s lines_per_second=6.32566M/s tokens_per_second=50.6052M/s BM_GroupingSymbols/0/4/0 1600539 ns 1600325 ns 429 bytes_per_second=224.777M/s lines_per_second=6.24873M/s tokens_per_second=62.4873M/s BM_GroupingSymbols/0/8/0 1085044 ns 1084941 ns 646 bytes_per_second=222.765M/s lines_per_second=5.12009M/s tokens_per_second=92.1709M/s BM_GroupingSymbols/0/16/0 824804 ns 824756 ns 817 bytes_per_second=209.166M/s lines_per_second=3.5659M/s tokens_per_second=121.248M/s BM_GroupingSymbols/0/32/0 685741 ns 685741 ns 1004 bytes_per_second=196.576M/s lines_per_second=2.20929M/s tokens_per_second=145.828M/s BM_GroupingSymbols/0/0/1 3467507 ns 3467077 ns 206 bytes_per_second=218.781M/s lines_per_second=7.21069M/s tokens_per_second=28.8427M/s BM_GroupingSymbols/0/0/2 2284154 ns 2283937 ns 309 bytes_per_second=235.79M/s lines_per_second=7.29705M/s tokens_per_second=43.784M/s BM_GroupingSymbols/0/0/3 1965548 ns 1965225 ns 356 bytes_per_second=217.193M/s lines_per_second=6.36059M/s tokens_per_second=50.8848M/s BM_GroupingSymbols/0/0/4 1623965 ns 1623725 ns 440 bytes_per_second=221.538M/s lines_per_second=6.15868M/s tokens_per_second=61.5868M/s BM_GroupingSymbols/0/0/8 1080601 ns 1080452 ns 650 bytes_per_second=223.691M/s lines_per_second=5.14137M/s tokens_per_second=92.5539M/s BM_GroupingSymbols/0/0/16 840820 ns 840677 ns 828 bytes_per_second=205.205M/s lines_per_second=3.49837M/s tokens_per_second=118.952M/s BM_GroupingSymbols/0/0/32 707793 ns 707734 ns 991 bytes_per_second=190.467M/s lines_per_second=2.14063M/s tokens_per_second=141.296M/s BM_GroupingSymbols/32/1/0 2804159 ns 2803869 ns 245 bytes_per_second=1103.7M/s lines_per_second=34.0779M/s tokens_per_second=35.665M/s BM_GroupingSymbols/32/2/0 2676229 ns 2675855 ns 261 bytes_per_second=1123.78M/s lines_per_second=34.688M/s tokens_per_second=37.3712M/s BM_GroupingSymbols/32/3/0 2652102 ns 2651836 ns 262 bytes_per_second=1103.24M/s lines_per_second=34.0217M/s tokens_per_second=37.7097M/s BM_GroupingSymbols/32/4/0 2600677 ns 2600552 ns 268 bytes_per_second=1096.17M/s lines_per_second=33.7678M/s tokens_per_second=38.4534M/s BM_GroupingSymbols/32/8/0 2382017 ns 2381869 ns 294 bytes_per_second=1083.42M/s lines_per_second=33.2659M/s tokens_per_second=41.9838M/s BM_GroupingSymbols/32/16/0 2102340 ns 2102243 ns 325 bytes_per_second=1034.45M/s lines_per_second=31.5377M/s tokens_per_second=47.5682M/s BM_GroupingSymbols/32/32/0 1745079 ns 1744914 ns 403 bytes_per_second=952.64M/s lines_per_second=28.6461M/s tokens_per_second=57.3094M/s BM_GroupingSymbols/32/32/1 1701414 ns 1701255 ns 415 bytes_per_second=962.73M/s lines_per_second=28.9228M/s tokens_per_second=58.7802M/s BM_GroupingSymbols/32/32/2 1688503 ns 1688121 ns 415 bytes_per_second=957.019M/s lines_per_second=28.7242M/s tokens_per_second=59.2375M/s BM_GroupingSymbols/32/32/3 1679701 ns 1679499 ns 419 bytes_per_second=948.651M/s lines_per_second=28.446M/s tokens_per_second=59.5416M/s BM_GroupingSymbols/32/32/4 1647815 ns 1647816 ns 427 bytes_per_second=953.342M/s lines_per_second=28.559M/s tokens_per_second=60.6864M/s BM_GroupingSymbols/32/32/8 1581283 ns 1581075 ns 450 bytes_per_second=942.39M/s lines_per_second=28.1201M/s tokens_per_second=63.2481M/s BM_GroupingSymbols/32/32/16 1445591 ns 1445526 ns 477 bytes_per_second=936.105M/s lines_per_second=27.7442M/s tokens_per_second=69.179M/s BM_GroupingSymbols/32/32/32 1296335 ns 1296094 ns 544 bytes_per_second=882.943M/s lines_per_second=25.8276M/s tokens_per_second=77.1549M/s BM_BlankLines/1 5774442 ns 5774454 ns 107 bytes_per_second=99.0921M/s lines_per_second=17.3175M/s tokens_per_second=17.3177M/s BM_BlankLines/4 8712135 ns 8711977 ns 75 bytes_per_second=98.5198M/s lines_per_second=45.9133M/s tokens_per_second=11.4785M/s BM_BlankLines/16 32313782 ns 32307655 ns 22 bytes_per_second=61.9884M/s lines_per_second=49.5234M/s tokens_per_second=3.09524M/s BM_BlankLines/64 87562163 ns 87543476 ns 7 bytes_per_second=75.166M/s lines_per_second=73.1058M/s tokens_per_second=1.14229M/s BM_BlankLines/128 160336428 ns 160298644 ns 4 bytes_per_second=79.1257M/s lines_per_second=79.8502M/s tokens_per_second=623.836k/s BM_CommentLines/1/0/0 7298959 ns 7298427 ns 89 bytes_per_second=117.601M/s lines_per_second=27.4029M/s tokens_per_second=13.7016M/s BM_CommentLines/4/0/0 10002223 ns 10002239 ns 67 bytes_per_second=171.622M/s lines_per_second=49.9883M/s tokens_per_second=9.99776M/s BM_CommentLines/128/0/0 143356660 ns 143356412 ns 5 bytes_per_second=259.444M/s lines_per_second=89.9846M/s tokens_per_second=697.562k/s BM_CommentLines/1/30/0 7421994 ns 7421289 ns 87 bytes_per_second=501.166M/s lines_per_second=26.9492M/s tokens_per_second=13.4747M/s BM_CommentLines/4/30/0 11304903 ns 11303972 ns 56 bytes_per_second=1.13696G/s lines_per_second=44.2318M/s tokens_per_second=8.84645M/s BM_CommentLines/128/30/0 156244144 ns 156217089 ns 4 bytes_per_second=2.52178G/s lines_per_second=82.5766M/s tokens_per_second=640.135k/s BM_CommentLines/1/70/0 7486688 ns 7485340 ns 80 bytes_per_second=1006.49M/s lines_per_second=26.7186M/s tokens_per_second=13.3594M/s BM_CommentLines/4/70/0 14762881 ns 14761417 ns 45 bytes_per_second=1.88011G/s lines_per_second=33.8717M/s tokens_per_second=6.77442M/s BM_CommentLines/128/70/0 179933089 ns 179903592 ns 4 bytes_per_second=4.84025G/s lines_per_second=71.7044M/s tokens_per_second=555.853k/s BM_CommentLines/1/0/2 7473119 ns 7472370 ns 87 bytes_per_second=140.389M/s lines_per_second=26.765M/s tokens_per_second=13.3826M/s BM_CommentLines/4/0/2 9702727 ns 9700268 ns 66 bytes_per_second=255.615M/s lines_per_second=51.5445M/s tokens_per_second=10.309M/s BM_CommentLines/128/0/2 140504132 ns 140480254 ns 5 bytes_per_second=438.544M/s lines_per_second=91.8269M/s tokens_per_second=711.844k/s BM_CommentLines/1/30/2 7530847 ns 7529227 ns 82 bytes_per_second=519.314M/s lines_per_second=26.5629M/s tokens_per_second=13.2816M/s BM_CommentLines/4/30/2 11646677 ns 11644972 ns 56 bytes_per_second=1.16764G/s lines_per_second=42.9366M/s tokens_per_second=8.5874M/s BM_CommentLines/128/30/2 161292392 ns 161273904 ns 4 bytes_per_second=2.59054G/s lines_per_second=79.9873M/s tokens_per_second=620.063k/s BM_CommentLines/1/70/2 7700751 ns 7700365 ns 83 bytes_per_second=1003.16M/s lines_per_second=25.9725M/s tokens_per_second=12.9864M/s BM_CommentLines/4/70/2 14133018 ns 14131523 ns 45 bytes_per_second=2.01664G/s lines_per_second=35.3815M/s tokens_per_second=7.07638M/s BM_CommentLines/128/70/2 179085026 ns 179057072 ns 4 bytes_per_second=4.99628G/s lines_per_second=72.0433M/s tokens_per_second=558.481k/s BM_CommentLines/1/0/8 7792060 ns 7791951 ns 83 bytes_per_second=208.065M/s lines_per_second=25.6673M/s tokens_per_second=12.8338M/s BM_CommentLines/4/0/8 10329194 ns 10329006 ns 61 bytes_per_second=461.644M/s lines_per_second=48.4069M/s tokens_per_second=9.68147M/s BM_CommentLines/128/0/8 140917902 ns 140917975 ns 5 bytes_per_second=956.927M/s lines_per_second=91.5417M/s tokens_per_second=709.633k/s BM_CommentLines/1/30/8 7813308 ns 7811702 ns 82 bytes_per_second=573.784M/s lines_per_second=25.6024M/s tokens_per_second=12.8013M/s BM_CommentLines/4/30/8 12052779 ns 12051440 ns 54 bytes_per_second=1.31373G/s lines_per_second=41.4884M/s tokens_per_second=8.29776M/s BM_CommentLines/128/30/8 163767409 ns 163753783 ns 4 bytes_per_second=2.9881G/s lines_per_second=78.776M/s tokens_per_second=610.673k/s BM_CommentLines/1/70/8 8188137 ns 8187614 ns 80 bytes_per_second=1013.35M/s lines_per_second=24.4269M/s tokens_per_second=12.2136M/s BM_CommentLines/4/70/8 15723650 ns 15721559 ns 43 bytes_per_second=1.95485G/s lines_per_second=31.8031M/s tokens_per_second=6.36069M/s BM_CommentLines/128/70/8 182579515 ns 182554119 ns 4 bytes_per_second=5.29237G/s lines_per_second=70.6633M/s tokens_per_second=547.783k/s BM_SpeedOfLightStrCpy 27009 ns 27006 ns 25887 bytes_per_second=37.594G/s lines_per_second=1.17547G/s tokens_per_second=3.70286G/s BM_SpeedOfLightDispatch<1> 1850860 ns 1850660 ns 381 bytes_per_second=561.764M/s lines_per_second=17.1533M/s tokens_per_second=54.0348M/s BM_SpeedOfLightDispatch<2> 1876473 ns 1876369 ns 347 bytes_per_second=554.067M/s lines_per_second=16.9183M/s tokens_per_second=53.2944M/s BM_SpeedOfLightDispatch<4> 2208441 ns 2208443 ns 314 bytes_per_second=470.755M/s lines_per_second=14.3744M/s tokens_per_second=45.2808M/s BM_SpeedOfLightDispatch<8> 2824174 ns 2824137 ns 250 bytes_per_second=368.125M/s lines_per_second=11.2406M/s tokens_per_second=35.409M/s BM_SpeedOfLightDispatch<16> 4306633 ns 4306325 ns 165 bytes_per_second=241.42M/s lines_per_second=7.37172M/s tokens_per_second=23.2217M/s BM_SpeedOfLightDispatch<32> 6300653 ns 6300102 ns 113 bytes_per_second=165.019M/s lines_per_second=5.03881M/s tokens_per_second=15.8728M/s BM_SpeedOfLightDispatch<MaxDispatchTargets> 8879663 ns 8878510 ns 79 bytes_per_second=117.096M/s lines_per_second=3.57549M/s tokens_per_second=11.2632M/s ``` After: ``` ------------------------------------------------------------------------------------------------------ Benchmark Time CPU Iterations UserCounters... ------------------------------------------------------------------------------------------------------ BM_ValidKeywords 2821833 ns 2821832 ns 247 bytes_per_second=211.642M/s tokens_per_second=35.438M/s BM_ValidKeywordsAsRawIdentifiers 3204326 ns 3203964 ns 216 bytes_per_second=245.931M/s tokens_per_second=31.2113M/s BM_RawIdentifierFocus 6076723 ns 6076107 ns 98 bytes_per_second=257.524M/s tokens_per_second=16.4579M/s BM_ValidIdentifiers<1, 64, false> 6303093 ns 6302632 ns 101 bytes_per_second=233.682M/s tokens_per_second=15.8664M/s BM_ValidIdentifiers<1, 1, true> 3687668 ns 3687338 ns 194 bytes_per_second=77.5902M/s tokens_per_second=27.1198M/s BM_ValidIdentifiers<3, 5, true> 5298920 ns 5298465 ns 106 bytes_per_second=107.994M/s tokens_per_second=18.8734M/s BM_ValidIdentifiers<3, 16, true> 4880695 ns 4879704 ns 118 bytes_per_second=224.75M/s tokens_per_second=20.493M/s BM_ValidIdentifiers<12, 64, true> 5413070 ns 5411832 ns 103 bytes_per_second=704.84M/s tokens_per_second=18.478M/s BM_ValidIdentifiers<16, 16, true> 5052780 ns 5051309 ns 110 bytes_per_second=339.835M/s tokens_per_second=19.7968M/s BM_ValidIdentifiers<24, 24, true> 5221580 ns 5220851 ns 104 bytes_per_second=474.933M/s tokens_per_second=19.154M/s BM_ValidIdentifiers<32, 32, true> 5786811 ns 5786806 ns 99 bytes_per_second=560.325M/s tokens_per_second=17.2807M/s BM_ValidIdentifiers<48, 48, true> 5959506 ns 5959502 ns 96 bytes_per_second=800.129M/s tokens_per_second=16.7799M/s BM_ValidIdentifiers<64, 64, true> 7831469 ns 7830629 ns 75 bytes_per_second=803.799M/s tokens_per_second=12.7704M/s BM_ValidIdentifiers<80, 80, true> 7905843 ns 7904727 ns 75 bytes_per_second=989.298M/s tokens_per_second=12.6507M/s BM_HorizontalWhitespace/1 5091171 ns 5090660 ns 109 bytes_per_second=112.402M/s tokens_per_second=19.6438M/s BM_HorizontalWhitespace/4 5020344 ns 5020344 ns 112 bytes_per_second=170.965M/s tokens_per_second=19.919M/s BM_HorizontalWhitespace/16 5846801 ns 5846459 ns 99 bytes_per_second=342.549M/s tokens_per_second=17.1044M/s BM_HorizontalWhitespace/64 7803183 ns 7802357 ns 78 bytes_per_second=843.372M/s tokens_per_second=12.8166M/s BM_HorizontalWhitespace/128 10600500 ns 10598602 ns 59 bytes_per_second=1.16869G/s tokens_per_second=9.43521M/s BM_RandomSource 4824062 ns 4823496 ns 139 bytes_per_second=215.536M/s lines_per_second=6.58133M/s tokens_per_second=20.7319M/s BM_GroupingSymbols/1/0/0 4116846 ns 4116549 ns 170 bytes_per_second=207.431M/s lines_per_second=18.2191M/s tokens_per_second=24.2922M/s BM_GroupingSymbols/2/0/0 3024336 ns 3024156 ns 236 bytes_per_second=241.144M/s lines_per_second=27.5548M/s tokens_per_second=33.0671M/s BM_GroupingSymbols/3/0/0 2789794 ns 2789256 ns 252 bytes_per_second=255.601M/s lines_per_second=31.3704M/s tokens_per_second=35.8519M/s BM_GroupingSymbols/4/0/0 2496498 ns 2496237 ns 283 bytes_per_second=296.921M/s lines_per_second=36.0543M/s tokens_per_second=40.0603M/s BM_GroupingSymbols/8/0/0 2200846 ns 2200611 ns 313 bytes_per_second=456.487M/s lines_per_second=42.9131M/s tokens_per_second=45.4419M/s BM_GroupingSymbols/16/0/0 2415237 ns 2415015 ns 288 bytes_per_second=703.225M/s lines_per_second=40.1873M/s tokens_per_second=41.4076M/s BM_GroupingSymbols/32/0/0 3171195 ns 3170504 ns 215 bytes_per_second=1004.97M/s lines_per_second=31.0597M/s tokens_per_second=31.5407M/s BM_GroupingSymbols/0/1/0 3627393 ns 3626737 ns 193 bytes_per_second=209.15M/s lines_per_second=6.89325M/s tokens_per_second=27.573M/s BM_GroupingSymbols/0/2/0 2501189 ns 2500946 ns 275 bytes_per_second=215.33M/s lines_per_second=6.66388M/s tokens_per_second=39.9849M/s BM_GroupingSymbols/0/3/0 2149513 ns 2149282 ns 318 bytes_per_second=198.593M/s lines_per_second=5.8159M/s tokens_per_second=46.5272M/s BM_GroupingSymbols/0/4/0 1793658 ns 1793292 ns 384 bytes_per_second=200.59M/s lines_per_second=5.57634M/s tokens_per_second=55.7634M/s BM_GroupingSymbols/0/8/0 1302555 ns 1302272 ns 542 bytes_per_second=185.589M/s lines_per_second=4.26562M/s tokens_per_second=76.7889M/s BM_GroupingSymbols/0/16/0 1042993 ns 1042818 ns 671 bytes_per_second=165.428M/s lines_per_second=2.82024M/s tokens_per_second=95.8941M/s BM_GroupingSymbols/0/32/0 955561 ns 955471 ns 749 bytes_per_second=141.082M/s lines_per_second=1.58561M/s tokens_per_second=104.66M/s BM_GroupingSymbols/0/0/1 3659797 ns 3659369 ns 194 bytes_per_second=207.285M/s lines_per_second=6.83178M/s tokens_per_second=27.3271M/s BM_GroupingSymbols/0/0/2 2467556 ns 2467190 ns 281 bytes_per_second=218.276M/s lines_per_second=6.75505M/s tokens_per_second=40.5319M/s BM_GroupingSymbols/0/0/3 2152274 ns 2151938 ns 326 bytes_per_second=198.348M/s lines_per_second=5.80872M/s tokens_per_second=46.4697M/s BM_GroupingSymbols/0/0/4 1805982 ns 1805877 ns 368 bytes_per_second=199.192M/s lines_per_second=5.53747M/s tokens_per_second=55.3747M/s BM_GroupingSymbols/0/0/8 1313041 ns 1312833 ns 539 bytes_per_second=184.096M/s lines_per_second=4.23131M/s tokens_per_second=76.1711M/s BM_GroupingSymbols/0/0/16 1065565 ns 1065301 ns 659 bytes_per_second=161.937M/s lines_per_second=2.76072M/s tokens_per_second=93.8702M/s BM_GroupingSymbols/0/0/32 946630 ns 946514 ns 726 bytes_per_second=142.417M/s lines_per_second=1.60061M/s tokens_per_second=105.651M/s BM_GroupingSymbols/32/1/0 2991120 ns 2991121 ns 233 bytes_per_second=1034.6M/s lines_per_second=31.9445M/s tokens_per_second=33.4323M/s BM_GroupingSymbols/32/2/0 2893280 ns 2892944 ns 245 bytes_per_second=1039.45M/s lines_per_second=32.085M/s tokens_per_second=34.5669M/s BM_GroupingSymbols/32/3/0 2819177 ns 2819024 ns 239 bytes_per_second=1037.81M/s lines_per_second=32.004M/s tokens_per_second=35.4733M/s BM_GroupingSymbols/32/4/0 2778376 ns 2778018 ns 249 bytes_per_second=1026.15M/s lines_per_second=31.6107M/s tokens_per_second=35.9969M/s BM_GroupingSymbols/32/8/0 2538279 ns 2538279 ns 275 bytes_per_second=1016.65M/s lines_per_second=31.216M/s tokens_per_second=39.3968M/s BM_GroupingSymbols/32/16/0 2291819 ns 2291693 ns 305 bytes_per_second=948.937M/s lines_per_second=28.9306M/s tokens_per_second=43.6359M/s BM_GroupingSymbols/32/32/0 1943560 ns 1943560 ns 366 bytes_per_second=855.273M/s lines_per_second=25.7183M/s tokens_per_second=51.452M/s BM_GroupingSymbols/32/32/1 1902069 ns 1901915 ns 375 bytes_per_second=861.158M/s lines_per_second=25.8713M/s tokens_per_second=52.5786M/s BM_GroupingSymbols/32/32/2 1877847 ns 1877752 ns 379 bytes_per_second=860.371M/s lines_per_second=25.8234M/s tokens_per_second=53.2552M/s BM_GroupingSymbols/32/32/3 1837280 ns 1837016 ns 381 bytes_per_second=867.308M/s lines_per_second=26.0069M/s tokens_per_second=54.4361M/s BM_GroupingSymbols/32/32/4 1841010 ns 1840902 ns 380 bytes_per_second=853.349M/s lines_per_second=25.5636M/s tokens_per_second=54.3212M/s BM_GroupingSymbols/32/32/8 1734676 ns 1734437 ns 405 bytes_per_second=859.062M/s lines_per_second=25.6337M/s tokens_per_second=57.6556M/s BM_GroupingSymbols/32/32/16 1641169 ns 1640934 ns 422 bytes_per_second=824.63M/s lines_per_second=24.4403M/s tokens_per_second=60.9409M/s BM_GroupingSymbols/32/32/32 1506988 ns 1506914 ns 472 bytes_per_second=759.418M/s lines_per_second=22.2143M/s tokens_per_second=66.3608M/s BM_BlankLines/1 5658057 ns 5657150 ns 110 bytes_per_second=101.147M/s lines_per_second=17.6766M/s tokens_per_second=17.6767M/s BM_BlankLines/4 8346196 ns 8346052 ns 77 bytes_per_second=102.839M/s lines_per_second=47.9264M/s tokens_per_second=11.9817M/s BM_BlankLines/16 31147085 ns 31144610 ns 22 bytes_per_second=64.3033M/s lines_per_second=51.3727M/s tokens_per_second=3.21083M/s BM_BlankLines/64 83743719 ns 83743762 ns 8 bytes_per_second=78.5765M/s lines_per_second=76.4228M/s tokens_per_second=1.19412M/s BM_BlankLines/128 152299627 ns 152274606 ns 4 bytes_per_second=83.2952M/s lines_per_second=84.0578M/s tokens_per_second=656.708k/s BM_CommentLines/1/0/0 7535704 ns 7535149 ns 83 bytes_per_second=113.906M/s lines_per_second=26.542M/s tokens_per_second=13.2711M/s BM_CommentLines/4/0/0 10107724 ns 10106088 ns 66 bytes_per_second=169.858M/s lines_per_second=49.4746M/s tokens_per_second=9.89503M/s BM_CommentLines/128/0/0 130061022 ns 130029826 ns 5 bytes_per_second=286.034M/s lines_per_second=99.207M/s tokens_per_second=769.054k/s BM_CommentLines/1/30/0 7773816 ns 7772726 ns 83 bytes_per_second=478.506M/s lines_per_second=25.7307M/s tokens_per_second=12.8655M/s BM_CommentLines/4/30/0 11436116 ns 11434452 ns 56 bytes_per_second=1.12398G/s lines_per_second=43.7271M/s tokens_per_second=8.7455M/s BM_CommentLines/128/30/0 155047059 ns 155033899 ns 4 bytes_per_second=2.54103G/s lines_per_second=83.2068M/s tokens_per_second=645.02k/s BM_CommentLines/1/70/0 8016209 ns 8014861 ns 75 bytes_per_second=939.998M/s lines_per_second=24.9534M/s tokens_per_second=12.4768M/s BM_CommentLines/4/70/0 14894752 ns 14891800 ns 44 bytes_per_second=1.86365G/s lines_per_second=33.5752M/s tokens_per_second=6.7151M/s BM_CommentLines/128/70/0 176667108 ns 176631061 ns 4 bytes_per_second=4.92993G/s lines_per_second=73.0329M/s tokens_per_second=566.152k/s BM_CommentLines/1/0/2 7764475 ns 7763675 ns 84 bytes_per_second=135.121M/s lines_per_second=25.7607M/s tokens_per_second=12.8805M/s BM_CommentLines/4/0/2 10238104 ns 10236809 ns 65 bytes_per_second=242.217M/s lines_per_second=48.8429M/s tokens_per_second=9.76867M/s BM_CommentLines/128/0/2 130208823 ns 130190640 ns 5 bytes_per_second=473.204M/s lines_per_second=99.0845M/s tokens_per_second=768.104k/s BM_CommentLines/1/30/2 7941224 ns 7940584 ns 78 bytes_per_second=492.411M/s lines_per_second=25.1868M/s tokens_per_second=12.5935M/s BM_CommentLines/4/30/2 11936879 ns 11934453 ns 56 bytes_per_second=1.13932G/s lines_per_second=41.8951M/s tokens_per_second=8.3791M/s BM_CommentLines/128/30/2 156978531 ns 156967142 ns 4 bytes_per_second=2.66162G/s lines_per_second=82.182M/s tokens_per_second=637.076k/s BM_CommentLines/1/70/2 8223614 ns 8222923 ns 79 bytes_per_second=939.409M/s lines_per_second=24.322M/s tokens_per_second=12.1611M/s BM_CommentLines/4/70/2 15216239 ns 15215047 ns 45 bytes_per_second=1.87303G/s lines_per_second=32.8619M/s tokens_per_second=6.57244M/s BM_CommentLines/128/70/2 176810589 ns 176755958 ns 4 bytes_per_second=5.06133G/s lines_per_second=72.9813M/s tokens_per_second=565.752k/s BM_CommentLines/1/0/8 7901146 ns 7899003 ns 82 bytes_per_second=205.245M/s lines_per_second=25.3194M/s tokens_per_second=12.6598M/s BM_CommentLines/4/0/8 10135919 ns 10134576 ns 66 bytes_per_second=470.501M/s lines_per_second=49.3356M/s tokens_per_second=9.86721M/s BM_CommentLines/128/0/8 132206448 ns 132206347 ns 5 bytes_per_second=1019.98M/s lines_per_second=97.5738M/s tokens_per_second=756.393k/s BM_CommentLines/1/30/8 7981189 ns 7981202 ns 79 bytes_per_second=561.598M/s lines_per_second=25.0586M/s tokens_per_second=12.5294M/s BM_CommentLines/4/30/8 12311389 ns 12309078 ns 54 bytes_per_second=1.28623G/s lines_per_second=40.62M/s tokens_per_second=8.12409M/s BM_CommentLines/128/30/8 160432032 ns 160393999 ns 4 bytes_per_second=3.05069G/s lines_per_second=80.4261M/s tokens_per_second=623.465k/s BM_CommentLines/1/70/8 8199080 ns 8199078 ns 79 bytes_per_second=1011.93M/s lines_per_second=24.3927M/s tokens_per_second=12.1965M/s BM_CommentLines/4/70/8 16087954 ns 16086159 ns 44 bytes_per_second=1.91055G/s lines_per_second=31.0823M/s tokens_per_second=6.21652M/s BM_CommentLines/128/70/8 175714908 ns 175675241 ns 4 bytes_per_second=5.4996G/s lines_per_second=73.4302M/s tokens_per_second=569.232k/s BM_SpeedOfLightStrCpy 28860 ns 28858 ns 26092 bytes_per_second=35.181G/s lines_per_second=1.10002G/s tokens_per_second=3.46519G/s BM_SpeedOfLightDispatch<1> 1823558 ns 1823473 ns 385 bytes_per_second=570.14M/s lines_per_second=17.4091M/s tokens_per_second=54.8404M/s BM_SpeedOfLightDispatch<2> 2013453 ns 2013250 ns 343 bytes_per_second=516.396M/s lines_per_second=15.768M/s tokens_per_second=49.6709M/s BM_SpeedOfLightDispatch<4> 2225145 ns 2224920 ns 312 bytes_per_second=467.268M/s lines_per_second=14.2679M/s tokens_per_second=44.9454M/s BM_SpeedOfLightDispatch<8> 2851730 ns 2851590 ns 251 bytes_per_second=364.581M/s lines_per_second=11.1324M/s tokens_per_second=35.0682M/s BM_SpeedOfLightDispatch<16> 4431584 ns 4431156 ns 161 bytes_per_second=234.619M/s lines_per_second=7.16404M/s tokens_per_second=22.5675M/s BM_SpeedOfLightDispatch<32> 6229285 ns 6228092 ns 111 bytes_per_second=166.927M/s lines_per_second=5.09707M/s tokens_per_second=16.0563M/s BM_SpeedOfLightDispatch<MaxDispatchTargets> 8967228 ns 8965583 ns 79 bytes_per_second=115.958M/s lines_per_second=3.54076M/s tokens_per_second=11.1538M/s ``` --------- Co-authored-by: Chandler Carruth <chandlerc@gmail.com>
375 lines
13 KiB
C++
375 lines
13 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#ifndef CARBON_TOOLCHAIN_LEX_TOKENIZED_BUFFER_H_
|
|
#define CARBON_TOOLCHAIN_LEX_TOKENIZED_BUFFER_H_
|
|
|
|
#include <cstdint>
|
|
#include <iterator>
|
|
|
|
#include "common/ostream.h"
|
|
#include "llvm/ADT/APInt.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/ADT/StringRef.h"
|
|
#include "llvm/ADT/iterator.h"
|
|
#include "llvm/ADT/iterator_range.h"
|
|
#include "llvm/Support/Allocator.h"
|
|
#include "llvm/Support/raw_ostream.h"
|
|
#include "toolchain/base/index_base.h"
|
|
#include "toolchain/base/value_store.h"
|
|
#include "toolchain/diagnostics/diagnostic_emitter.h"
|
|
#include "toolchain/lex/token_kind.h"
|
|
#include "toolchain/source/source_buffer.h"
|
|
|
|
namespace Carbon::Lex {
|
|
|
|
class TokenizedBuffer;
|
|
|
|
// A lightweight handle to a lexed token in a `TokenizedBuffer`.
|
|
//
|
|
// `Token` objects are designed to be passed by value, not reference or
|
|
// pointer. They are also designed to be small and efficient to store in data
|
|
// structures.
|
|
//
|
|
// `Token` objects from the same `TokenizedBuffer` can be compared with each
|
|
// other, both for being the same token within the buffer, and to establish
|
|
// relative position within the token stream that has been lexed out of the
|
|
// buffer. `Token` objects from different `TokenizedBuffer`s cannot be
|
|
// meaningfully compared.
|
|
//
|
|
// All other APIs to query a `Token` are on the `TokenizedBuffer`.
|
|
struct Token : public ComparableIndexBase {
|
|
static const Token Invalid;
|
|
// Comments aren't tokenized, so this is the first token after StartOfFile.
|
|
static const Token FirstNonCommentToken;
|
|
using ComparableIndexBase::ComparableIndexBase;
|
|
};
|
|
|
|
constexpr Token Token::Invalid(Token::InvalidIndex);
|
|
constexpr Token Token::FirstNonCommentToken(1);
|
|
|
|
// A lightweight handle to a lexed line in a `TokenizedBuffer`.
|
|
//
|
|
// `Line` objects are designed to be passed by value, not reference or
|
|
// pointer. They are also designed to be small and efficient to store in data
|
|
// structures.
|
|
//
|
|
// Each `Line` object refers to a specific line in the source code that was
|
|
// lexed. They can be compared directly to establish that they refer to the
|
|
// same line or the relative position of different lines within the source.
|
|
//
|
|
// All other APIs to query a `Line` are on the `TokenizedBuffer`.
|
|
struct Line : public ComparableIndexBase {
|
|
static const Line Invalid;
|
|
using ComparableIndexBase::ComparableIndexBase;
|
|
};
|
|
|
|
constexpr Line Line::Invalid(Line::InvalidIndex);
|
|
|
|
// Random-access iterator over tokens within the buffer.
|
|
class TokenIterator
|
|
: public llvm::iterator_facade_base<
|
|
TokenIterator, std::random_access_iterator_tag, const Token, int>,
|
|
public Printable<TokenIterator> {
|
|
public:
|
|
TokenIterator() = delete;
|
|
|
|
explicit TokenIterator(Token token) : token_(token) {}
|
|
|
|
auto operator==(const TokenIterator& rhs) const -> bool {
|
|
return token_ == rhs.token_;
|
|
}
|
|
auto operator<(const TokenIterator& rhs) const -> bool {
|
|
return token_ < rhs.token_;
|
|
}
|
|
|
|
auto operator*() const -> const Token& { return token_; }
|
|
|
|
using iterator_facade_base::operator-;
|
|
auto operator-(const TokenIterator& rhs) const -> int {
|
|
return token_.index - rhs.token_.index;
|
|
}
|
|
|
|
auto operator+=(int n) -> TokenIterator& {
|
|
token_.index += n;
|
|
return *this;
|
|
}
|
|
auto operator-=(int n) -> TokenIterator& {
|
|
token_.index -= n;
|
|
return *this;
|
|
}
|
|
|
|
// Prints the raw token index.
|
|
auto Print(llvm::raw_ostream& output) const -> void;
|
|
|
|
private:
|
|
friend class TokenizedBuffer;
|
|
|
|
Token token_;
|
|
};
|
|
|
|
// A diagnostic location translator that maps token locations into source
|
|
// buffer locations.
|
|
class TokenLocationTranslator : public DiagnosticLocationTranslator<Token> {
|
|
public:
|
|
explicit TokenLocationTranslator(const TokenizedBuffer* buffer)
|
|
: buffer_(buffer) {}
|
|
|
|
// Map the given token into a diagnostic location.
|
|
auto GetLocation(Token token) -> DiagnosticLocation override;
|
|
|
|
private:
|
|
const TokenizedBuffer* buffer_;
|
|
};
|
|
|
|
// A buffer of tokenized Carbon source code.
|
|
//
|
|
// This is constructed by lexing the source code text into a series of tokens.
|
|
// The buffer provides lightweight handles to tokens and other lexed entities,
|
|
// as well as iterations to walk the sequence of tokens found in the buffer.
|
|
//
|
|
// Lexing errors result in a potentially incomplete sequence of tokens and
|
|
// `HasError` returning true.
|
|
class TokenizedBuffer : public Printable<TokenizedBuffer> {
|
|
public:
|
|
[[nodiscard]] auto GetKind(Token token) const -> TokenKind;
|
|
[[nodiscard]] auto GetLine(Token token) const -> Line;
|
|
|
|
// Returns the 1-based line number.
|
|
[[nodiscard]] auto GetLineNumber(Token token) const -> int;
|
|
|
|
// Returns the 1-based column number.
|
|
[[nodiscard]] auto GetColumnNumber(Token token) const -> int;
|
|
|
|
// Returns the source text lexed into this token.
|
|
[[nodiscard]] auto GetTokenText(Token token) const -> llvm::StringRef;
|
|
|
|
// Returns the identifier associated with this token. The token kind must be
|
|
// an `Identifier`.
|
|
[[nodiscard]] auto GetIdentifier(Token token) const -> IdentifierId;
|
|
|
|
// Returns the value of an `IntegerLiteral()` token.
|
|
[[nodiscard]] auto GetIntegerLiteral(Token token) const -> IntegerId;
|
|
|
|
// Returns the value of an `RealLiteral()` token.
|
|
[[nodiscard]] auto GetRealLiteral(Token token) const -> RealId;
|
|
|
|
// Returns the value of a `StringLiteral()` token.
|
|
[[nodiscard]] auto GetStringLiteral(Token token) const -> StringLiteralId;
|
|
|
|
// Returns the size specified in a `*TypeLiteral()` token.
|
|
[[nodiscard]] auto GetTypeLiteralSize(Token token) const
|
|
-> const llvm::APInt&;
|
|
|
|
// Returns the closing token matched with the given opening token.
|
|
//
|
|
// The given token must be an opening token kind.
|
|
[[nodiscard]] auto GetMatchedClosingToken(Token opening_token) const -> Token;
|
|
|
|
// Returns the opening token matched with the given closing token.
|
|
//
|
|
// The given token must be a closing token kind.
|
|
[[nodiscard]] auto GetMatchedOpeningToken(Token closing_token) const -> Token;
|
|
|
|
// Returns whether the given token has leading whitespace.
|
|
[[nodiscard]] auto HasLeadingWhitespace(Token token) const -> bool;
|
|
// Returns whether the given token has trailing whitespace.
|
|
[[nodiscard]] auto HasTrailingWhitespace(Token token) const -> bool;
|
|
|
|
// Returns whether the token was created as part of an error recovery effort.
|
|
//
|
|
// For example, a closing paren inserted to match an unmatched paren.
|
|
[[nodiscard]] auto IsRecoveryToken(Token token) const -> bool;
|
|
|
|
// Returns the 1-based line number.
|
|
[[nodiscard]] auto GetLineNumber(Line line) const -> int;
|
|
|
|
// Returns the 1-based indentation column number.
|
|
[[nodiscard]] auto GetIndentColumnNumber(Line line) const -> int;
|
|
|
|
// Returns the next line handle.
|
|
[[nodiscard]] auto GetNextLine(Line line) const -> Line;
|
|
|
|
// Returns the previous line handle.
|
|
[[nodiscard]] auto GetPrevLine(Line line) const -> Line;
|
|
|
|
// Prints a description of the tokenized stream to the provided `raw_ostream`.
|
|
//
|
|
// It prints one line of information for each token in the buffer, including
|
|
// the kind of token, where it occurs within the source file, indentation for
|
|
// the associated line, the spelling of the token in source, and any
|
|
// additional information tracked such as which unique identifier it is or any
|
|
// matched grouping token.
|
|
//
|
|
// Each line is formatted as a YAML record:
|
|
//
|
|
// clang-format off
|
|
// ```
|
|
// token: { index: 0, kind: 'Semi', line: 1, column: 1, indent: 1, spelling: ';' }
|
|
// ```
|
|
// clang-format on
|
|
//
|
|
// This can be parsed as YAML using tools like `python-yq` combined with `jq`
|
|
// on the command line. The format is also reasonably amenable to other
|
|
// line-oriented shell tools from `grep` to `awk`.
|
|
auto Print(llvm::raw_ostream& output_stream) const -> void;
|
|
|
|
// Prints a description of a single token. See `Print` for details on the
|
|
// format.
|
|
auto PrintToken(llvm::raw_ostream& output_stream, Token token) const -> void;
|
|
|
|
// Returns true if the buffer has errors that were detected at lexing time.
|
|
[[nodiscard]] auto has_errors() const -> bool { return has_errors_; }
|
|
|
|
[[nodiscard]] auto tokens() const -> llvm::iterator_range<TokenIterator> {
|
|
return llvm::make_range(TokenIterator(Token(0)),
|
|
TokenIterator(Token(token_infos_.size())));
|
|
}
|
|
|
|
[[nodiscard]] auto size() const -> int { return token_infos_.size(); }
|
|
|
|
[[nodiscard]] auto expected_parse_tree_size() const -> int {
|
|
return expected_parse_tree_size_;
|
|
}
|
|
|
|
auto filename() const -> llvm::StringRef { return source_->filename(); }
|
|
|
|
private:
|
|
friend class Lexer;
|
|
friend class TokenLocationTranslator;
|
|
|
|
// A diagnostic location translator that maps token locations into source
|
|
// buffer locations.
|
|
class SourceBufferLocationTranslator
|
|
: public DiagnosticLocationTranslator<const char*> {
|
|
public:
|
|
explicit SourceBufferLocationTranslator(const TokenizedBuffer* buffer)
|
|
: buffer_(buffer) {}
|
|
|
|
// Map the given position within the source buffer into a diagnostic
|
|
// location.
|
|
auto GetLocation(const char* loc) -> DiagnosticLocation override;
|
|
|
|
private:
|
|
const TokenizedBuffer* buffer_;
|
|
};
|
|
|
|
// Specifies minimum widths to use when printing a token's fields via
|
|
// `printToken`.
|
|
struct PrintWidths {
|
|
// Widens `this` to the maximum of `this` and `new_width` for each
|
|
// dimension.
|
|
auto Widen(const PrintWidths& widths) -> void;
|
|
|
|
int index;
|
|
int kind;
|
|
int line;
|
|
int column;
|
|
int indent;
|
|
};
|
|
|
|
struct TokenInfo {
|
|
TokenKind kind;
|
|
|
|
// Whether the token has trailing whitespace.
|
|
bool has_trailing_space = false;
|
|
|
|
// Whether the token was injected artificially during error recovery.
|
|
bool is_recovery = false;
|
|
|
|
// Line on which the Token starts.
|
|
Line token_line;
|
|
|
|
// Zero-based byte offset of the token within its line.
|
|
int32_t column;
|
|
|
|
// We may have up to 32 bits of payload, based on the kind of token.
|
|
union {
|
|
static_assert(
|
|
sizeof(Token) <= sizeof(int32_t),
|
|
"Unable to pack token and identifier index into the same space!");
|
|
|
|
IdentifierId ident_id = IdentifierId::Invalid;
|
|
StringLiteralId string_literal_id;
|
|
IntegerId integer_id;
|
|
RealId real_id;
|
|
Token closing_token;
|
|
Token opening_token;
|
|
int32_t error_length;
|
|
};
|
|
};
|
|
|
|
struct LineInfo {
|
|
// The length will always be assigned later. Indent may be assigned if
|
|
// non-zero.
|
|
explicit LineInfo(int64_t start)
|
|
: start(start),
|
|
length(static_cast<int32_t>(llvm::StringRef::npos)),
|
|
indent(0) {}
|
|
|
|
explicit LineInfo(int64_t start, int32_t length)
|
|
: start(start), length(length), indent(0) {}
|
|
|
|
// Zero-based byte offset of the start of the line within the source buffer
|
|
// provided.
|
|
int64_t start;
|
|
|
|
// The byte length of the line. Does not include the newline character (or a
|
|
// nul-terminator or EOF).
|
|
int32_t length;
|
|
|
|
// The byte offset from the start of the line of the first non-whitespace
|
|
// character.
|
|
int32_t indent;
|
|
};
|
|
|
|
// The constructor is merely responsible for trivial initialization of
|
|
// members. A working object of this type is built with `Lex::Lex` so that its
|
|
// return can indicate if an error was encountered while lexing.
|
|
explicit TokenizedBuffer(SharedValueStores& value_stores,
|
|
SourceBuffer& source)
|
|
: value_stores_(&value_stores), source_(&source) {}
|
|
|
|
auto GetLineInfo(Line line) -> LineInfo&;
|
|
[[nodiscard]] auto GetLineInfo(Line line) const -> const LineInfo&;
|
|
auto AddLine(LineInfo info) -> Line;
|
|
auto GetTokenInfo(Token token) -> TokenInfo&;
|
|
[[nodiscard]] auto GetTokenInfo(Token token) const -> const TokenInfo&;
|
|
auto AddToken(TokenInfo info) -> Token;
|
|
[[nodiscard]] auto GetTokenPrintWidths(Token token) const -> PrintWidths;
|
|
auto PrintToken(llvm::raw_ostream& output_stream, Token token,
|
|
PrintWidths widths) const -> void;
|
|
|
|
// Used to allocate computed string literals.
|
|
llvm::BumpPtrAllocator allocator_;
|
|
|
|
SharedValueStores* value_stores_;
|
|
SourceBuffer* source_;
|
|
|
|
llvm::SmallVector<TokenInfo> token_infos_;
|
|
|
|
llvm::SmallVector<LineInfo> line_infos_;
|
|
|
|
// Stores the computed value of string literals so that StringRefs are
|
|
// durable.
|
|
llvm::SmallVector<std::unique_ptr<std::string>> computed_strings_;
|
|
|
|
// The number of parse tree nodes that we expect to be created for the tokens
|
|
// in this buffer.
|
|
int expected_parse_tree_size_ = 0;
|
|
|
|
bool has_errors_ = false;
|
|
};
|
|
|
|
// A diagnostic emitter that uses positions within a source buffer's text as
|
|
// its source of location information.
|
|
using LexerDiagnosticEmitter = DiagnosticEmitter<const char*>;
|
|
|
|
// A diagnostic emitter that uses tokens as its source of location information.
|
|
using TokenDiagnosticEmitter = DiagnosticEmitter<Token>;
|
|
|
|
} // namespace Carbon::Lex
|
|
|
|
#endif // CARBON_TOOLCHAIN_LEX_TOKENIZED_BUFFER_H_
|