Add sentinel Eof token at the end of the tokenized buffer. (#388)

This has two goals:

1) It allows us to simplify and remove special cases from the parser:
   when we expect a particular token next, we can just check for it
   without needing a special case for end-of-file.

2) It gives us a token to use as a position when emitting diagnostics at
   the end of the file.

Centralize all updating of `position` to `Consume` and `SkipTo`, so that we can in a single place ensure that we never go past the EOF token.

parser/parser_impl.h

@@ -25,11 +25,11 @@ class ParseTree::Parser {
  private:
   struct SubtreeStart;
 
-  explicit Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg)
-      : tree(tree_arg),
-        tokens(tokens_arg),
-        position(tokens.Tokens().begin()),
-        end(tokens.Tokens().end()) {}
+  explicit Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg);
+
+  auto AtEndOfFile() -> bool {
+    return tokens.GetKind(*position) == TokenKind::EndOfFile();
+  }
 
   // Requires (and asserts) that the current position matches the provide
   // `Kind`. Returns the current token and advances to the next position.
@@ -75,6 +75,9 @@ class ParseTree::Parser {
   // returns false.
   auto SkipMatchingGroup() -> bool;
 
+  // Skip forward to the token immediately after the given token.
+  auto SkipTo(TokenizedBuffer::Token t) -> void;
+
   // Skips forward to move past the likely end of a declaration.
   //
   // Looks forward, skipping over any matched symbol groups, to find the next
@@ -125,7 +128,10 @@ class ParseTree::Parser {
   ParseTree& tree;
   TokenizedBuffer& tokens;
 
+  // The current position within the token buffer. Never equal to `end`.
   TokenizedBuffer::TokenIterator position;
+  // The end position of the token buffer. There will always be an `EndOfFile`
+  // token between `position` (inclusive) and `end` (exclusive).
   TokenizedBuffer::TokenIterator end;
 };