Modify lex yaml output to elide FileStart/End in tests. (#4433)

Trying to make split file tests of lex functionality shorter and easier
to read. numeric_literals.carbon in particular has an example of why I'm
interested in this (at the bottom). This also switches from `[]` list
format to `-` list format so that the trailing `]` is removed.

Trimming comments in tokenized_buffer.h because (1) it feels like it's
giving too much detail about what's printed, which has drifted slightly
and (2) it also feels like it's trying to justify YAML output, when
that's just what we're doing in general.

---------

Co-authored-by: Geoff Romer <gromer@google.com>

toolchain/driver/compile_subcommand.cpp

@@ -176,6 +176,16 @@ Dump the tokens to stdout when lexed.
 )""",
       },
       [&](auto& arg_b) { arg_b.Set(&dump_tokens); });
+
+  b.AddFlag(
+      {
+          .name = "omit-file-boundary-tokens",
+          .help = R"""(
+For `--dump-tokens`, omit file start and end boundary tokens.
+)""",
+      },
+      [&](auto& arg_b) { arg_b.Set(&omit_file_boundary_tokens); });
+
   b.AddFlag(
       {
           .name = "dump-parse-tree",
@@ -356,7 +366,8 @@ class CompilationUnit {
             [&] { tokens_ = Lex::Lex(value_stores_, *source_, *consumer_); });
     if (options_.dump_tokens && IncludeInDumps()) {
       consumer_->Flush();
-      driver_env_->output_stream << tokens_;
+      tokens_->Print(driver_env_->output_stream,
+                     options_.omit_file_boundary_tokens);
     }
     if (mem_usage_) {
       mem_usage_->Collect("tokens_", *tokens_);

toolchain/driver/compile_subcommand.h

@@ -45,6 +45,7 @@ struct CompileOptions {
   bool force_obj_output = false;
   bool dump_shared_values = false;
   bool dump_tokens = false;
+  bool omit_file_boundary_tokens = false;
   bool dump_parse_tree = false;
   bool dump_raw_sem_ir = false;
   bool dump_sem_ir = false;

toolchain/lex/testdata/basic_syntax.carbon

@@ -2,29 +2,31 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
+// Dump all tokens so that we include the file start and end.
+// ARGS: compile --phase=lex --dump-tokens %s
+//
 // AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/lex/testdata/basic_syntax.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/basic_syntax.carbon
 // CHECK:STDOUT: - filename: basic_syntax.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index:  0, kind:         'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
+// CHECK:STDOUT:   - { index:  0, kind:         'FileStart', line: {{ *\d+}}, column:   1, indent: 1, spelling: '' }
 
 fn run(String program) {
-// CHECK:STDOUT:     { index:  1, kind:                'Fn', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'fn', has_leading_space: true },
-// CHECK:STDOUT:     { index:  2, kind:        'Identifier', line: {{ *}}[[@LINE-2]], column:  4, indent: 1, spelling: 'run', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index:  3, kind:         'OpenParen', line: {{ *}}[[@LINE-3]], column:  7, indent: 1, spelling: '(', closing_token: 6 },
-// CHECK:STDOUT:     { index:  4, kind: 'StringTypeLiteral', line: {{ *}}[[@LINE-4]], column:  8, indent: 1, spelling: 'String' },
-// CHECK:STDOUT:     { index:  5, kind:        'Identifier', line: {{ *}}[[@LINE-5]], column: 15, indent: 1, spelling: 'program', identifier: 1, has_leading_space: true },
-// CHECK:STDOUT:     { index:  6, kind:        'CloseParen', line: {{ *}}[[@LINE-6]], column: 22, indent: 1, spelling: ')', opening_token: 3 },
-// CHECK:STDOUT:     { index:  7, kind:    'OpenCurlyBrace', line: {{ *}}[[@LINE-7]], column: 24, indent: 1, spelling: '{', closing_token: 11, has_leading_space: true },
+// CHECK:STDOUT:   - { index:  1, kind:                'Fn', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'fn', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  2, kind:        'Identifier', line: {{ *}}[[@LINE-2]], column:   4, indent: 1, spelling: 'run', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  3, kind:         'OpenParen', line: {{ *}}[[@LINE-3]], column:   7, indent: 1, spelling: '(', closing_token: 6 }
+// CHECK:STDOUT:   - { index:  4, kind: 'StringTypeLiteral', line: {{ *}}[[@LINE-4]], column:   8, indent: 1, spelling: 'String' }
+// CHECK:STDOUT:   - { index:  5, kind:        'Identifier', line: {{ *}}[[@LINE-5]], column:  15, indent: 1, spelling: 'program', identifier: 1, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  6, kind:        'CloseParen', line: {{ *}}[[@LINE-6]], column:  22, indent: 1, spelling: ')', opening_token: 3 }
+// CHECK:STDOUT:   - { index:  7, kind:    'OpenCurlyBrace', line: {{ *}}[[@LINE-7]], column:  24, indent: 1, spelling: '{', closing_token: 11, has_leading_space: true }
   return True;
-  // CHECK:STDOUT:     { index:  8, kind:            'Return', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: 'return', has_leading_space: true },
-  // CHECK:STDOUT:     { index:  9, kind:        'Identifier', line: {{ *}}[[@LINE-2]], column: 10, indent: 3, spelling: 'True', identifier: 2, has_leading_space: true },
-  // CHECK:STDOUT:     { index: 10, kind:              'Semi', line: {{ *}}[[@LINE-3]], column: 14, indent: 3, spelling: ';' },
+  // CHECK:STDOUT:   - { index:  8, kind:            'Return', line: {{ *}}[[@LINE-1]], column:   3, indent: 3, spelling: 'return', has_leading_space: true }
+  // CHECK:STDOUT:   - { index:  9, kind:        'Identifier', line: {{ *}}[[@LINE-2]], column:  10, indent: 3, spelling: 'True', identifier: 2, has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 10, kind:              'Semi', line: {{ *}}[[@LINE-3]], column:  14, indent: 3, spelling: ';' }
 }
-// CHECK:STDOUT:     { index: 11, kind:   'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '}', opening_token: 7, has_leading_space: true },
+// CHECK:STDOUT:   - { index: 11, kind:   'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '}', opening_token: 7, has_leading_space: true }
 
-// CHECK:STDOUT:     { index: 12, kind:           'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 12, kind:           'FileEnd', line: {{ *}}[[@LINE+0]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }

toolchain/lex/testdata/fail_bad_comment_introducers.carbon

@@ -7,9 +7,6 @@
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/lex/testdata/fail_bad_comment_introducers.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/fail_bad_comment_introducers.carbon
-// CHECK:STDOUT: - filename: fail_bad_comment_introducers.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
 //
 
 // Comments have to have whitespace after `//` currently.
@@ -58,5 +55,5 @@
 
 // An extra un-indented comment line to anchor the end of the file checks.
 
-// CHECK:STDOUT:     { index: 1, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT: - filename: fail_bad_comment_introducers.carbon
+// CHECK:STDOUT:   tokens:

toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon

@@ -34,9 +34,6 @@
 // CHECK:STDERR: fail_bad_comment_introducers_mid_block_indent_change.carbon:[[@LINE-19]]:3: error: whitespace is required after '//' [NoWhitespaceAfterCommentIntroducer]
 // CHECK:STDERR: //abc
 // CHECK:STDERR:   ^
-// CHECK:STDOUT: - filename: fail_bad_comment_introducers_mid_block_indent_change.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
 
-// CHECK:STDOUT:     { index: 1, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT: - filename: fail_bad_comment_introducers_mid_block_indent_change.carbon
+// CHECK:STDOUT:   tokens:

toolchain/lex/testdata/fail_bad_raw_identifier.carbon

@@ -10,8 +10,7 @@
 
 // --- fail_bad_raw_identifier.carbon
 // CHECK:STDOUT: - filename: fail_bad_raw_identifier.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index:  0, kind:     'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 
 // Missing the character after `#`.
@@ -20,8 +19,8 @@
 // CHECK:STDERR:  ^
 // CHECK:STDERR:
 r#
-// CHECK:STDOUT:     { index:  1, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index:  2, kind:         'Error', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: '#' },
+// CHECK:STDOUT:   - { index:  1, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  2, kind:         'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' }
 
 // Not a valid identifier.
 // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:2: error: encountered unrecognized characters while parsing [UnrecognizedCharacters]
@@ -29,9 +28,9 @@ r#
 // CHECK:STDERR:  ^
 // CHECK:STDERR:
 r#3
-// CHECK:STDOUT:     { index:  3, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index:  4, kind:         'Error', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: '#' },
-// CHECK:STDOUT:     { index:  5, kind:    'IntLiteral', line: {{ *}}[[@LINE-3]], column:  3, indent: 1, spelling: '3', value: `3` },
+// CHECK:STDOUT:   - { index:  3, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  4, kind:         'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' }
+// CHECK:STDOUT:   - { index:  5, kind:    'IntLiteral', line: {{ *}}[[@LINE-3]], column: 3, indent: 1, spelling: '3', value: `3` }
 
 // Non ascii start to identifier.
 // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:2: error: encountered unrecognized characters while parsing [UnrecognizedCharacters]
@@ -39,8 +38,8 @@ r#3
 // CHECK:STDERR:  ^
 // CHECK:STDERR:
 r#á
-// CHECK:STDOUT:     { index:  6, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index:  7, kind:         'Error', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: '#á' },
+// CHECK:STDOUT:   - { index:  6, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  7, kind:         'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#á' }
 
 // Raw `r` identifier doesn't start a second raw identifier.
 // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:4: error: encountered unrecognized characters while parsing [UnrecognizedCharacters]
@@ -48,9 +47,9 @@ r#á
 // CHECK:STDERR:    ^
 // CHECK:STDERR:
 r#r#foo
-// CHECK:STDOUT:     { index:  8, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index:  9, kind:         'Error', line: {{ *}}[[@LINE-2]], column:  4, indent: 1, spelling: '#' },
-// CHECK:STDOUT:     { index: 10, kind:    'Identifier', line: {{ *}}[[@LINE-3]], column:  5, indent: 1, spelling: 'foo', identifier: 1 },
+// CHECK:STDOUT:   - { index:  8, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  9, kind:         'Error', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: '#' }
+// CHECK:STDOUT:   - { index: 10, kind:    'Identifier', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: 'foo', identifier: 1 }
 
 // Other identifier characters don't start a raw identifier.
 // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:2: error: encountered unrecognized characters while parsing [UnrecognizedCharacters]
@@ -58,9 +57,9 @@ r#r#foo
 // CHECK:STDERR:  ^
 // CHECK:STDERR:
 s#foo
-// CHECK:STDOUT:     { index: 11, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 's', identifier: 2, has_leading_space: true },
-// CHECK:STDOUT:     { index: 12, kind:         'Error', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: '#' },
-// CHECK:STDOUT:     { index: 13, kind:    'Identifier', line: {{ *}}[[@LINE-3]], column:  3, indent: 1, spelling: 'foo', identifier: 1 },
+// CHECK:STDOUT:   - { index: 11, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 's', identifier: 2, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 12, kind:         'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' }
+// CHECK:STDOUT:   - { index: 13, kind:    'Identifier', line: {{ *}}[[@LINE-3]], column: 3, indent: 1, spelling: 'foo', identifier: 1 }
 
 // Identifier ending in `r` doesn't start a raw identifier.
 // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:4: error: encountered unrecognized characters while parsing [UnrecognizedCharacters]
@@ -68,9 +67,9 @@ s#foo
 // CHECK:STDERR:    ^
 // CHECK:STDERR:
 arr#foo
-// CHECK:STDOUT:     { index: 14, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'arr', identifier: 3, has_leading_space: true },
-// CHECK:STDOUT:     { index: 15, kind:         'Error', line: {{ *}}[[@LINE-2]], column:  4, indent: 1, spelling: '#' },
-// CHECK:STDOUT:     { index: 16, kind:    'Identifier', line: {{ *}}[[@LINE-3]], column:  5, indent: 1, spelling: 'foo', identifier: 1 },
+// CHECK:STDOUT:   - { index: 14, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'arr', identifier: 3, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 15, kind:         'Error', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: '#' }
+// CHECK:STDOUT:   - { index: 16, kind:    'Identifier', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: 'foo', identifier: 1 }
 
 // Whitespace between `r` and `#` isn't allowed.
 // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:3: error: encountered unrecognized characters while parsing [UnrecognizedCharacters]
@@ -78,29 +77,23 @@ arr#foo
 // CHECK:STDERR:   ^
 // CHECK:STDERR:
 r #foo
-// CHECK:STDOUT:     { index: 17, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index: 18, kind:         'Error', line: {{ *}}[[@LINE-2]], column:  3, indent: 1, spelling: '#', has_leading_space: true },
-// CHECK:STDOUT:     { index: 19, kind:    'Identifier', line: {{ *}}[[@LINE-3]], column:  4, indent: 1, spelling: 'foo', identifier: 1 },
+// CHECK:STDOUT:   - { index: 17, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 18, kind:         'Error', line: {{ *}}[[@LINE-2]], column: 3, indent: 1, spelling: '#', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 19, kind:    'Identifier', line: {{ *}}[[@LINE-3]], column: 4, indent: 1, spelling: 'foo', identifier: 1 }
 
 // This is an `r` identifier followed by a string literal.
 r#"hello"#
-// CHECK:STDOUT:     { index: 20, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index: 21, kind: 'StringLiteral', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: '#"hello"#', value: `hello` },
+// CHECK:STDOUT:   - { index: 20, kind:    'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 21, kind: 'StringLiteral', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#"hello"#', value: `hello` }
 
-// CHECK:STDOUT:     { index: 22, kind:       'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
 // --- fail_hash_at_start_of_file.carbon
 // CHECK:STDOUT: - filename: fail_hash_at_start_of_file.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:  'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 // Ensure that we correctly handle a `#` as the first token in the file.
 // CHECK:STDERR: fail_hash_at_start_of_file.carbon:[[@LINE+3]]:1: error: encountered unrecognized characters while parsing [UnrecognizedCharacters]
 // CHECK:STDERR: #foo
 // CHECK:STDERR: ^
 #foo
-// CHECK:STDOUT:     { index: 1, kind:      'Error', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '#', has_leading_space: true },
-// CHECK:STDOUT:     { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: 'foo', identifier: 0 },
-
-// CHECK:STDOUT:     { index: 3, kind:    'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 1, kind:      'Error', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '#', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column:   2, indent: 1, spelling: 'foo', identifier: 0 }

toolchain/lex/testdata/fail_block_string_second_line.carbon

@@ -17,15 +17,11 @@ var s: String = '''
 // CHECK:STDERR:   error here: '''
 // CHECK:STDERR:   ^
 // CHECK:STDOUT: - filename: fail_block_string_second_line.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:         'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
-// CHECK:STDOUT:     { index: 1, kind:               'Var', line: {{ *}}[[@LINE-17]], column:  1, indent: 1, spelling: 'var', has_leading_space: true },
-// CHECK:STDOUT:     { index: 2, kind:        'Identifier', line: {{ *}}[[@LINE-18]], column:  5, indent: 1, spelling: 's', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index: 3, kind:             'Colon', line: {{ *}}[[@LINE-19]], column:  6, indent: 1, spelling: ':' },
-// CHECK:STDOUT:     { index: 4, kind: 'StringTypeLiteral', line: {{ *}}[[@LINE-20]], column:  8, indent: 1, spelling: 'String', has_leading_space: true },
-// CHECK:STDOUT:     { index: 5, kind:             'Equal', line: {{ *}}[[@LINE-21]], column: 15, indent: 1, spelling: '=', has_leading_space: true },
-// CHECK:STDOUT:     { index: 6, kind:     'StringLiteral', line: {{ *}}[[@LINE-22]], column: 17, indent: 1, spelling: ''''
-// CHECK:STDOUT:   error here: '''', value: `error here: `, has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 7, kind:           'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   tokens:
+// CHECK:STDOUT:   - { index: 1, kind:               'Var', line: {{ *}}[[@LINE-16]], column:  1, indent: 1, spelling: 'var', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 2, kind:        'Identifier', line: {{ *}}[[@LINE-17]], column:  5, indent: 1, spelling: 's', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 3, kind:             'Colon', line: {{ *}}[[@LINE-18]], column:  6, indent: 1, spelling: ':' }
+// CHECK:STDOUT:   - { index: 4, kind: 'StringTypeLiteral', line: {{ *}}[[@LINE-19]], column:  8, indent: 1, spelling: 'String', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 5, kind:             'Equal', line: {{ *}}[[@LINE-20]], column: 15, indent: 1, spelling: '=', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 6, kind:     'StringLiteral', line: {{ *}}[[@LINE-21]], column: 17, indent: 1, spelling: ''''
+// CHECK:STDOUT:   error here: '''', value: `error here: `, has_leading_space: true }

toolchain/lex/testdata/fail_mismatched_brackets.carbon

@@ -8,30 +8,26 @@
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/fail_mismatched_brackets.carbon
 // CHECK:STDOUT: - filename: fail_mismatched_brackets.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:  'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 // CHECK:STDERR: fail_mismatched_brackets.carbon:[[@LINE+4]]:1: error: closing symbol without a corresponding opening symbol [UnmatchedClosing]
 // CHECK:STDERR: }
 // CHECK:STDERR: ^
 // CHECK:STDERR:
 }
-// CHECK:STDOUT:     { index: 1, kind:      'Error', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '}', has_leading_space: true },
+// CHECK:STDOUT:   - { index: 1, kind:      'Error', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '}', has_leading_space: true }
 
 // CHECK:STDERR: fail_mismatched_brackets.carbon:[[@LINE+4]]:3: error: closing symbol without a corresponding opening symbol [UnmatchedClosing]
 // CHECK:STDERR: ( } )
 // CHECK:STDERR:   ^
 // CHECK:STDERR:
 ( } )
-// CHECK:STDOUT:     { index: 2, kind:  'OpenParen', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '(', closing_token: 4, has_leading_space: true },
-// CHECK:STDOUT:     { index: 3, kind:      'Error', line: {{ *}}[[@LINE-2]], column:  3, indent: 1, spelling: '}', has_leading_space: true },
-// CHECK:STDOUT:     { index: 4, kind: 'CloseParen', line: {{ *}}[[@LINE-3]], column:  5, indent: 1, spelling: ')', opening_token: 2, has_leading_space: true },
+// CHECK:STDOUT:   - { index: 2, kind:  'OpenParen', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '(', closing_token: 4, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 3, kind:      'Error', line: {{ *}}[[@LINE-2]], column:   3, indent: 1, spelling: '}', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 4, kind: 'CloseParen', line: {{ *}}[[@LINE-3]], column:   5, indent: 1, spelling: ')', opening_token: 2, has_leading_space: true }
 
 // CHECK:STDERR: fail_mismatched_brackets.carbon:[[@LINE+3]]:1: error: opening symbol without a corresponding closing symbol [UnmatchedOpening]
 // CHECK:STDERR: [
 // CHECK:STDERR: ^
 [
-// CHECK:STDOUT:     { index: 5, kind:      'Error', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '[', has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 6, kind:    'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 5, kind:      'Error', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '[', has_leading_space: true }

toolchain/lex/testdata/fail_mismatched_brackets_2.carbon

@@ -22,21 +22,17 @@ fn F() {
 // CHECK:STDERR: }
 // CHECK:STDERR: ^
 // CHECK:STDOUT: - filename: fail_mismatched_brackets_2.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index:  0, kind:       'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
-// CHECK:STDOUT:     { index:  1, kind:              'Fn', line: {{ *}}[[@LINE-19]], column:  1, indent: 1, spelling: 'fn', has_leading_space: true },
-// CHECK:STDOUT:     { index:  2, kind:      'Identifier', line: {{ *}}[[@LINE-20]], column:  4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index:  3, kind:       'OpenParen', line: {{ *}}[[@LINE-21]], column:  5, indent: 1, spelling: '(', closing_token: 4 },
-// CHECK:STDOUT:     { index:  4, kind:      'CloseParen', line: {{ *}}[[@LINE-22]], column:  6, indent: 1, spelling: ')', opening_token: 3 },
-// CHECK:STDOUT:     { index:  5, kind:  'OpenCurlyBrace', line: {{ *}}[[@LINE-23]], column:  8, indent: 1, spelling: '{', closing_token: 12, has_leading_space: true },
-// CHECK:STDOUT:     { index:  6, kind:              'If', line: {{ *}}[[@LINE-23]], column:  3, indent: 3, spelling: 'if', has_leading_space: true },
-// CHECK:STDOUT:     { index:  7, kind:       'OpenParen', line: {{ *}}[[@LINE-24]], column:  6, indent: 3, spelling: '(', closing_token: 9, has_leading_space: true },
-// CHECK:STDOUT:     { index:  8, kind:      'Identifier', line: {{ *}}[[@LINE-25]], column:  7, indent: 3, spelling: 'thing1', identifier: 1 },
-// CHECK:STDOUT:     { index:  9, kind:      'CloseParen', line: {{ *}}[[@LINE-26]], column: 13, indent: 3, spelling: ')', opening_token: 7 },
-// CHECK:STDOUT:     { index: 10, kind:      'Identifier', line: {{ *}}[[@LINE-26]], column:  5, indent: 5, spelling: 'thing2', identifier: 2, has_leading_space: true },
-// CHECK:STDOUT:     { index: 11, kind:            'Semi', line: {{ *}}[[@LINE-27]], column: 11, indent: 5, spelling: ';' },
-// CHECK:STDOUT:     { index: 12, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-27]], column:  3, indent: 3, spelling: '}', opening_token: 5, has_leading_space: true },
-// CHECK:STDOUT:     { index: 13, kind:           'Error', line: {{ *}}[[@LINE-27]], column:  1, indent: 1, spelling: '}', has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 14, kind:         'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   tokens:
+// CHECK:STDOUT:   - { index:  1, kind:              'Fn', line: {{ *}}[[@LINE-18]], column:   1, indent: 1, spelling: 'fn', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  2, kind:      'Identifier', line: {{ *}}[[@LINE-19]], column:   4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  3, kind:       'OpenParen', line: {{ *}}[[@LINE-20]], column:   5, indent: 1, spelling: '(', closing_token: 4 }
+// CHECK:STDOUT:   - { index:  4, kind:      'CloseParen', line: {{ *}}[[@LINE-21]], column:   6, indent: 1, spelling: ')', opening_token: 3 }
+// CHECK:STDOUT:   - { index:  5, kind:  'OpenCurlyBrace', line: {{ *}}[[@LINE-22]], column:   8, indent: 1, spelling: '{', closing_token: 12, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  6, kind:              'If', line: {{ *}}[[@LINE-22]], column:   3, indent: 3, spelling: 'if', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  7, kind:       'OpenParen', line: {{ *}}[[@LINE-23]], column:   6, indent: 3, spelling: '(', closing_token: 9, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  8, kind:      'Identifier', line: {{ *}}[[@LINE-24]], column:   7, indent: 3, spelling: 'thing1', identifier: 1 }
+// CHECK:STDOUT:   - { index:  9, kind:      'CloseParen', line: {{ *}}[[@LINE-25]], column:  13, indent: 3, spelling: ')', opening_token: 7 }
+// CHECK:STDOUT:   - { index: 10, kind:      'Identifier', line: {{ *}}[[@LINE-25]], column:   5, indent: 5, spelling: 'thing2', identifier: 2, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 11, kind:            'Semi', line: {{ *}}[[@LINE-26]], column:  11, indent: 5, spelling: ';' }
+// CHECK:STDOUT:   - { index: 12, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-26]], column:   3, indent: 3, spelling: '}', opening_token: 5, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 13, kind:           'Error', line: {{ *}}[[@LINE-26]], column:   1, indent: 1, spelling: '}', has_leading_space: true }

toolchain/lex/testdata/fail_multifile.carbon

@@ -10,26 +10,19 @@
 
 // --- fail_a.carbon
 // CHECK:STDOUT: - filename: fail_a.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 // CHECK:STDERR: fail_a.carbon:[[@LINE+4]]:3: error: empty digit sequence in numeric literal [EmptyDigitSequence]
 // CHECK:STDERR: 1.a
 // CHECK:STDERR:   ^
 // CHECK:STDERR:
 1.a
-// CHECK:STDOUT:     { index: 1, kind:     'Error', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '1.a', has_leading_space: true },
+// CHECK:STDOUT:   - { index: 1, kind:     'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.a', has_leading_space: true }
 
-// CHECK:STDOUT:     { index: 2, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
 // --- fail_b.carbon
 // CHECK:STDOUT: - filename: fail_b.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 // CHECK:STDERR: fail_b.carbon:[[@LINE+3]]:3: error: empty digit sequence in numeric literal [EmptyDigitSequence]
 // CHECK:STDERR: 2.b
 // CHECK:STDERR:   ^
 2.b
-// CHECK:STDOUT:     { index: 1, kind:     'Error', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '2.b', has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 2, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 1, kind:     'Error', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '2.b', has_leading_space: true }

toolchain/lex/testdata/fail_trailing_comments.carbon

@@ -28,29 +28,25 @@ var c: i32 = 0.4; // still more trailing comment
 // CHECK:STDERR: var c: i32 = 0.4; // still more trailing comment
 // CHECK:STDERR:                   ^
 // CHECK:STDOUT: - filename: fail_trailing_comments.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index:  0, kind:      'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
-// CHECK:STDOUT:     { index:  1, kind:            'Var', line: {{ *}}[[@LINE-25]], column:  1, indent: 1, spelling: 'var', has_leading_space: true },
-// CHECK:STDOUT:     { index:  2, kind:     'Identifier', line: {{ *}}[[@LINE-26]], column:  5, indent: 1, spelling: 'a', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index:  3, kind:          'Colon', line: {{ *}}[[@LINE-27]], column:  6, indent: 1, spelling: ':' },
-// CHECK:STDOUT:     { index:  4, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-28]], column:  8, indent: 1, spelling: 'i32', has_leading_space: true },
-// CHECK:STDOUT:     { index:  5, kind:          'Equal', line: {{ *}}[[@LINE-29]], column: 12, indent: 1, spelling: '=', has_leading_space: true },
-// CHECK:STDOUT:     { index:  6, kind:     'IntLiteral', line: {{ *}}[[@LINE-30]], column: 14, indent: 1, spelling: '1', value: `1`, has_leading_space: true },
-// CHECK:STDOUT:     { index:  7, kind:           'Semi', line: {{ *}}[[@LINE-31]], column: 15, indent: 1, spelling: ';' },
-// CHECK:STDOUT:     { index:  8, kind:            'Var', line: {{ *}}[[@LINE-31]], column:  1, indent: 1, spelling: 'var', has_leading_space: true },
-// CHECK:STDOUT:     { index:  9, kind:     'Identifier', line: {{ *}}[[@LINE-32]], column:  5, indent: 1, spelling: 'b', identifier: 1, has_leading_space: true },
-// CHECK:STDOUT:     { index: 10, kind:          'Colon', line: {{ *}}[[@LINE-33]], column:  6, indent: 1, spelling: ':' },
-// CHECK:STDOUT:     { index: 11, kind:     'IntLiteral', line: {{ *}}[[@LINE-34]], column:  8, indent: 1, spelling: '32', value: `32`, has_leading_space: true },
-// CHECK:STDOUT:     { index: 12, kind:          'Equal', line: {{ *}}[[@LINE-35]], column: 11, indent: 1, spelling: '=', has_leading_space: true },
-// CHECK:STDOUT:     { index: 13, kind:     'IntLiteral', line: {{ *}}[[@LINE-36]], column: 13, indent: 1, spelling: '13', value: `13`, has_leading_space: true },
-// CHECK:STDOUT:     { index: 14, kind:           'Semi', line: {{ *}}[[@LINE-37]], column: 15, indent: 1, spelling: ';' },
-// CHECK:STDOUT:     { index: 15, kind:            'Var', line: {{ *}}[[@LINE-37]], column:  1, indent: 1, spelling: 'var', has_leading_space: true },
-// CHECK:STDOUT:     { index: 16, kind:     'Identifier', line: {{ *}}[[@LINE-38]], column:  5, indent: 1, spelling: 'c', identifier: 2, has_leading_space: true },
-// CHECK:STDOUT:     { index: 17, kind:          'Colon', line: {{ *}}[[@LINE-39]], column:  6, indent: 1, spelling: ':' },
-// CHECK:STDOUT:     { index: 18, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-40]], column:  8, indent: 1, spelling: 'i32', has_leading_space: true },
-// CHECK:STDOUT:     { index: 19, kind:          'Equal', line: {{ *}}[[@LINE-41]], column: 12, indent: 1, spelling: '=', has_leading_space: true },
-// CHECK:STDOUT:     { index: 20, kind:    'RealLiteral', line: {{ *}}[[@LINE-42]], column: 14, indent: 1, spelling: '0.4', value: `4*10^-1`, has_leading_space: true },
-// CHECK:STDOUT:     { index: 21, kind:           'Semi', line: {{ *}}[[@LINE-43]], column: 17, indent: 1, spelling: ';' },
-
-// CHECK:STDOUT:     { index: 22, kind:        'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   tokens:
+// CHECK:STDOUT:   - { index:  1, kind:            'Var', line: {{ *}}[[@LINE-24]], column:   1, indent: 1, spelling: 'var', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  2, kind:     'Identifier', line: {{ *}}[[@LINE-25]], column:   5, indent: 1, spelling: 'a', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  3, kind:          'Colon', line: {{ *}}[[@LINE-26]], column:   6, indent: 1, spelling: ':' }
+// CHECK:STDOUT:   - { index:  4, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-27]], column:   8, indent: 1, spelling: 'i32', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  5, kind:          'Equal', line: {{ *}}[[@LINE-28]], column:  12, indent: 1, spelling: '=', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  6, kind:     'IntLiteral', line: {{ *}}[[@LINE-29]], column:  14, indent: 1, spelling: '1', value: `1`, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  7, kind:           'Semi', line: {{ *}}[[@LINE-30]], column:  15, indent: 1, spelling: ';' }
+// CHECK:STDOUT:   - { index:  8, kind:            'Var', line: {{ *}}[[@LINE-30]], column:   1, indent: 1, spelling: 'var', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  9, kind:     'Identifier', line: {{ *}}[[@LINE-31]], column:   5, indent: 1, spelling: 'b', identifier: 1, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 10, kind:          'Colon', line: {{ *}}[[@LINE-32]], column:   6, indent: 1, spelling: ':' }
+// CHECK:STDOUT:   - { index: 11, kind:     'IntLiteral', line: {{ *}}[[@LINE-33]], column:   8, indent: 1, spelling: '32', value: `32`, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 12, kind:          'Equal', line: {{ *}}[[@LINE-34]], column:  11, indent: 1, spelling: '=', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 13, kind:     'IntLiteral', line: {{ *}}[[@LINE-35]], column:  13, indent: 1, spelling: '13', value: `13`, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 14, kind:           'Semi', line: {{ *}}[[@LINE-36]], column:  15, indent: 1, spelling: ';' }
+// CHECK:STDOUT:   - { index: 15, kind:            'Var', line: {{ *}}[[@LINE-36]], column:   1, indent: 1, spelling: 'var', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 16, kind:     'Identifier', line: {{ *}}[[@LINE-37]], column:   5, indent: 1, spelling: 'c', identifier: 2, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 17, kind:          'Colon', line: {{ *}}[[@LINE-38]], column:   6, indent: 1, spelling: ':' }
+// CHECK:STDOUT:   - { index: 18, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-39]], column:   8, indent: 1, spelling: 'i32', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 19, kind:          'Equal', line: {{ *}}[[@LINE-40]], column:  12, indent: 1, spelling: '=', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 20, kind:    'RealLiteral', line: {{ *}}[[@LINE-41]], column:  14, indent: 1, spelling: '0.4', value: `4*10^-1`, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 21, kind:           'Semi', line: {{ *}}[[@LINE-42]], column:  17, indent: 1, spelling: ';' }

toolchain/lex/testdata/keywords.carbon

@@ -10,41 +10,31 @@
 
 // --- indented.carbon
 // CHECK:STDOUT: - filename: indented.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
    fn
-   // CHECK:STDOUT:     { index: 1, kind:        'Fn', line: {{ *}}[[@LINE-1]], column:  4, indent: 4, spelling: 'fn', has_leading_space: true },
+   // CHECK:STDOUT:   - { index: 1, kind:        'Fn', line: {{ *}}[[@LINE-1]], column: 4, indent: 4, spelling: 'fn', has_leading_space: true }
 
-// CHECK:STDOUT:     { index: 2, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
 // --- chain.carbon
 // CHECK:STDOUT: - filename: chain.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index:  0, kind:  'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 and or not if else for return var break continue _
-// CHECK:STDOUT:     { index:  1, kind:        'And', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'and', has_leading_space: true },
-// CHECK:STDOUT:     { index:  2, kind:         'Or', line: {{ *}}[[@LINE-2]], column:  5, indent: 1, spelling: 'or', has_leading_space: true },
-// CHECK:STDOUT:     { index:  3, kind:        'Not', line: {{ *}}[[@LINE-3]], column:  8, indent: 1, spelling: 'not', has_leading_space: true },
-// CHECK:STDOUT:     { index:  4, kind:         'If', line: {{ *}}[[@LINE-4]], column: 12, indent: 1, spelling: 'if', has_leading_space: true },
-// CHECK:STDOUT:     { index:  5, kind:       'Else', line: {{ *}}[[@LINE-5]], column: 15, indent: 1, spelling: 'else', has_leading_space: true },
-// CHECK:STDOUT:     { index:  6, kind:        'For', line: {{ *}}[[@LINE-6]], column: 20, indent: 1, spelling: 'for', has_leading_space: true },
-// CHECK:STDOUT:     { index:  7, kind:     'Return', line: {{ *}}[[@LINE-7]], column: 24, indent: 1, spelling: 'return', has_leading_space: true },
-// CHECK:STDOUT:     { index:  8, kind:        'Var', line: {{ *}}[[@LINE-8]], column: 31, indent: 1, spelling: 'var', has_leading_space: true },
-// CHECK:STDOUT:     { index:  9, kind:      'Break', line: {{ *}}[[@LINE-9]], column: 35, indent: 1, spelling: 'break', has_leading_space: true },
-// CHECK:STDOUT:     { index: 10, kind:   'Continue', line: {{ *}}[[@LINE-10]], column: 41, indent: 1, spelling: 'continue', has_leading_space: true },
-// CHECK:STDOUT:     { index: 11, kind: 'Underscore', line: {{ *}}[[@LINE-11]], column: 50, indent: 1, spelling: '_', has_leading_space: true },
+// CHECK:STDOUT:   - { index:  1, kind:        'And', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'and', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  2, kind:         'Or', line: {{ *}}[[@LINE-2]], column:  5, indent: 1, spelling: 'or', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  3, kind:        'Not', line: {{ *}}[[@LINE-3]], column:  8, indent: 1, spelling: 'not', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  4, kind:         'If', line: {{ *}}[[@LINE-4]], column: 12, indent: 1, spelling: 'if', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  5, kind:       'Else', line: {{ *}}[[@LINE-5]], column: 15, indent: 1, spelling: 'else', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  6, kind:        'For', line: {{ *}}[[@LINE-6]], column: 20, indent: 1, spelling: 'for', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  7, kind:     'Return', line: {{ *}}[[@LINE-7]], column: 24, indent: 1, spelling: 'return', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  8, kind:        'Var', line: {{ *}}[[@LINE-8]], column: 31, indent: 1, spelling: 'var', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  9, kind:      'Break', line: {{ *}}[[@LINE-9]], column: 35, indent: 1, spelling: 'break', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 10, kind:   'Continue', line: {{ *}}[[@LINE-10]], column: 41, indent: 1, spelling: 'continue', has_leading_space: true }
+// CHECK:STDOUT:   - { index: 11, kind: 'Underscore', line: {{ *}}[[@LINE-11]], column: 50, indent: 1, spelling: '_', has_leading_space: true }
 
-// CHECK:STDOUT:     { index: 12, kind:    'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
 // --- notakeyword.carbon
 // CHECK:STDOUT: - filename: notakeyword.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:  'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 notakeyword
-// CHECK:STDOUT:     { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'notakeyword', identifier: 0, has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 2, kind:    'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'notakeyword', identifier: 0, has_leading_space: true }

toolchain/lex/testdata/multifile.carbon

@@ -10,24 +10,17 @@
 
 // --- a.carbon
 // CHECK:STDOUT: - filename: a.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:  'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 a;
-// CHECK:STDOUT:     { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'a', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index: 2, kind:       'Semi', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: ';' },
+// CHECK:STDOUT:   - { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 2, kind:       'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';' }
 
-// CHECK:STDOUT:     { index: 3, kind:    'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
 // --- b.carbon
 // CHECK:STDOUT: - filename: b.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:  'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 b;
-// CHECK:STDOUT:     { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'b', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index: 2, kind:       'Semi', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: ';' },
+// CHECK:STDOUT:   - { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'b', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 2, kind:       'Semi', line: {{ *}}[[@LINE-2]], column:   2, indent: 1, spelling: ';' }
 a;
-// CHECK:STDOUT:     { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'a', identifier: 1, has_leading_space: true },
-// CHECK:STDOUT:     { index: 4, kind:       'Semi', line: {{ *}}[[@LINE-2]], column:  2, indent: 1, spelling: ';' },
-
-// CHECK:STDOUT:     { index: 5, kind:    'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'a', identifier: 1, has_leading_space: true }
+// CHECK:STDOUT:   - { index: 4, kind:       'Semi', line: {{ *}}[[@LINE-2]], column:   2, indent: 1, spelling: ';' }

toolchain/lex/testdata/numeric_literals.carbon

@@ -10,112 +10,102 @@
 
 // --- valid.carbon
 // CHECK:STDOUT: - filename: valid.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index:  0, kind:          'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 
 fn F() {
-// CHECK:STDOUT:     { index:  1, kind:                 'Fn', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'fn', has_leading_space: true },
-// CHECK:STDOUT:     { index:  2, kind:         'Identifier', line: {{ *}}[[@LINE-2]], column:  4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true },
-// CHECK:STDOUT:     { index:  3, kind:          'OpenParen', line: {{ *}}[[@LINE-3]], column:  5, indent: 1, spelling: '(', closing_token: 4 },
-// CHECK:STDOUT:     { index:  4, kind:         'CloseParen', line: {{ *}}[[@LINE-4]], column:  6, indent: 1, spelling: ')', opening_token: 3 },
-// CHECK:STDOUT:     { index:  5, kind:     'OpenCurlyBrace', line: {{ *}}[[@LINE-5]], column:  8, indent: 1, spelling: '{', closing_token: 54, has_leading_space: true },
+// CHECK:STDOUT:   - { index:  1, kind:                 'Fn', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'fn', has_leading_space: true }
+// CHECK:STDOUT:   - { index:  2, kind:         'Identifier', line: {{ *}}[[@LINE-2]], column:  4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true }
+// CHECK:STDOUT:   - { index:  3, kind:          'OpenParen', line: {{ *}}[[@LINE-3]], column:  5, indent: 1, spelling: '(', closing_token: 4 }
+// CHECK:STDOUT:   - { index:  4, kind:         'CloseParen', line: {{ *}}[[@LINE-4]], column:  6, indent: 1, spelling: ')', opening_token: 3 }
+// CHECK:STDOUT:   - { index:  5, kind:     'OpenCurlyBrace', line: {{ *}}[[@LINE-5]], column:  8, indent: 1, spelling: '{', closing_token: 54, has_leading_space: true }
   // 8 and 9 trigger special behavior in APInt when mishandling signed versus
   // unsigned, so we pay extra attention to those.
   var ints: [i32; 5] = (
-  // CHECK:STDOUT:     { index:  6, kind:                'Var', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: 'var', has_leading_space: true },
-  // CHECK:STDOUT:     { index:  7, kind:         'Identifier', line: {{ *}}[[@LINE-2]], column:  7, indent: 3, spelling: 'ints', identifier: 1, has_leading_space: true },
-  // CHECK:STDOUT:     { index:  8, kind:              'Colon', line: {{ *}}[[@LINE-3]], column: 11, indent: 3, spelling: ':' },
-  // CHECK:STDOUT:     { index:  9, kind:  'OpenSquareBracket', line: {{ *}}[[@LINE-4]], column: 13, indent: 3, spelling: '[', closing_token: 13, has_leading_space: true },
-  // CHECK:STDOUT:     { index: 10, kind:     'IntTypeLiteral', line: {{ *}}[[@LINE-5]], column: 14, indent: 3, spelling: 'i32' },
-  // CHECK:STDOUT:     { index: 11, kind:               'Semi', line: {{ *}}[[@LINE-6]], column: 17, indent: 3, spelling: ';' },
-  // CHECK:STDOUT:     { index: 12, kind:         'IntLiteral', line: {{ *}}[[@LINE-7]], column: 19, indent: 3, spelling: '5', value: `5`, has_leading_space: true },
-  // CHECK:STDOUT:     { index: 13, kind: 'CloseSquareBracket', line: {{ *}}[[@LINE-8]], column: 20, indent: 3, spelling: ']', opening_token: 9 },
-  // CHECK:STDOUT:     { index: 14, kind:              'Equal', line: {{ *}}[[@LINE-9]], column: 22, indent: 3, spelling: '=', has_leading_space: true },
-  // CHECK:STDOUT:     { index: 15, kind:          'OpenParen', line: {{ *}}[[@LINE-10]], column: 24, indent: 3, spelling: '(', closing_token: 26, has_leading_space: true },
+  // CHECK:STDOUT:   - { index:  6, kind:                'Var', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: 'var', has_leading_space: true }
+  // CHECK:STDOUT:   - { index:  7, kind:         'Identifier', line: {{ *}}[[@LINE-2]], column:  7, indent: 3, spelling: 'ints', identifier: 1, has_leading_space: true }
+  // CHECK:STDOUT:   - { index:  8, kind:              'Colon', line: {{ *}}[[@LINE-3]], column: 11, indent: 3, spelling: ':' }
+  // CHECK:STDOUT:   - { index:  9, kind:  'OpenSquareBracket', line: {{ *}}[[@LINE-4]], column: 13, indent: 3, spelling: '[', closing_token: 13, has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 10, kind:     'IntTypeLiteral', line: {{ *}}[[@LINE-5]], column: 14, indent: 3, spelling: 'i32' }
+  // CHECK:STDOUT:   - { index: 11, kind:               'Semi', line: {{ *}}[[@LINE-6]], column: 17, indent: 3, spelling: ';' }
+  // CHECK:STDOUT:   - { index: 12, kind:         'IntLiteral', line: {{ *}}[[@LINE-7]], column: 19, indent: 3, spelling: '5', value: `5`, has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 13, kind: 'CloseSquareBracket', line: {{ *}}[[@LINE-8]], column: 20, indent: 3, spelling: ']', opening_token: 9 }
+  // CHECK:STDOUT:   - { index: 14, kind:              'Equal', line: {{ *}}[[@LINE-9]], column: 22, indent: 3, spelling: '=', has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 15, kind:          'OpenParen', line: {{ *}}[[@LINE-10]], column: 24, indent: 3, spelling: '(', closing_token: 26, has_leading_space: true }
     8,
-    // CHECK:STDOUT:     { index: 16, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '8', value: `8`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 17, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  6, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 16, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '8', value: `8`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 17, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  6, indent: 5, spelling: ',' }
     9,
-    // CHECK:STDOUT:     { index: 18, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '9', value: `9`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 19, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  6, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 18, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '9', value: `9`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 19, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  6, indent: 5, spelling: ',' }
     0x8,
-    // CHECK:STDOUT:     { index: 20, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '0x8', value: `8`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 21, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  8, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 20, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '0x8', value: `8`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 21, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  8, indent: 5, spelling: ',' }
     0b1000,
-    // CHECK:STDOUT:     { index: 22, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '0b1000', value: `8`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 23, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 11, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 22, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '0b1000', value: `8`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 23, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 11, indent: 5, spelling: ',' }
     39999999999999999993,
-    // CHECK:STDOUT:     { index: 24, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '39999999999999999993', value: `39999999999999999993`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 25, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 25, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 24, kind:         'IntLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '39999999999999999993', value: `39999999999999999993`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 25, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 25, indent: 5, spelling: ',' }
   );
-  // CHECK:STDOUT:     { index: 26, kind:         'CloseParen', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: ')', opening_token: 15, has_leading_space: true },
-  // CHECK:STDOUT:     { index: 27, kind:               'Semi', line: {{ *}}[[@LINE-2]], column:  4, indent: 3, spelling: ';' },
+  // CHECK:STDOUT:   - { index: 26, kind:         'CloseParen', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: ')', opening_token: 15, has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 27, kind:               'Semi', line: {{ *}}[[@LINE-2]], column:  4, indent: 3, spelling: ';' }
   var floats: [f64; 7] = (
-  // CHECK:STDOUT:     { index: 28, kind:                'Var', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: 'var', has_leading_space: true },
-  // CHECK:STDOUT:     { index: 29, kind:         'Identifier', line: {{ *}}[[@LINE-2]], column:  7, indent: 3, spelling: 'floats', identifier: 2, has_leading_space: true },
-  // CHECK:STDOUT:     { index: 30, kind:              'Colon', line: {{ *}}[[@LINE-3]], column: 13, indent: 3, spelling: ':' },
-  // CHECK:STDOUT:     { index: 31, kind:  'OpenSquareBracket', line: {{ *}}[[@LINE-4]], column: 15, indent: 3, spelling: '[', closing_token: 35, has_leading_space: true },
-  // CHECK:STDOUT:     { index: 32, kind:   'FloatTypeLiteral', line: {{ *}}[[@LINE-5]], column: 16, indent: 3, spelling: 'f64' },
-  // CHECK:STDOUT:     { index: 33, kind:               'Semi', line: {{ *}}[[@LINE-6]], column: 19, indent: 3, spelling: ';' },
-  // CHECK:STDOUT:     { index: 34, kind:         'IntLiteral', line: {{ *}}[[@LINE-7]], column: 21, indent: 3, spelling: '7', value: `7`, has_leading_space: true },
-  // CHECK:STDOUT:     { index: 35, kind: 'CloseSquareBracket', line: {{ *}}[[@LINE-8]], column: 22, indent: 3, spelling: ']', opening_token: 31 },
-  // CHECK:STDOUT:     { index: 36, kind:              'Equal', line: {{ *}}[[@LINE-9]], column: 24, indent: 3, spelling: '=', has_leading_space: true },
-  // CHECK:STDOUT:     { index: 37, kind:          'OpenParen', line: {{ *}}[[@LINE-10]], column: 26, indent: 3, spelling: '(', closing_token: 52, has_leading_space: true },
+  // CHECK:STDOUT:   - { index: 28, kind:                'Var', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: 'var', has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 29, kind:         'Identifier', line: {{ *}}[[@LINE-2]], column:  7, indent: 3, spelling: 'floats', identifier: 2, has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 30, kind:              'Colon', line: {{ *}}[[@LINE-3]], column: 13, indent: 3, spelling: ':' }
+  // CHECK:STDOUT:   - { index: 31, kind:  'OpenSquareBracket', line: {{ *}}[[@LINE-4]], column: 15, indent: 3, spelling: '[', closing_token: 35, has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 32, kind:   'FloatTypeLiteral', line: {{ *}}[[@LINE-5]], column: 16, indent: 3, spelling: 'f64' }
+  // CHECK:STDOUT:   - { index: 33, kind:               'Semi', line: {{ *}}[[@LINE-6]], column: 19, indent: 3, spelling: ';' }
+  // CHECK:STDOUT:   - { index: 34, kind:         'IntLiteral', line: {{ *}}[[@LINE-7]], column: 21, indent: 3, spelling: '7', value: `7`, has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 35, kind: 'CloseSquareBracket', line: {{ *}}[[@LINE-8]], column: 22, indent: 3, spelling: ']', opening_token: 31 }
+  // CHECK:STDOUT:   - { index: 36, kind:              'Equal', line: {{ *}}[[@LINE-9]], column: 24, indent: 3, spelling: '=', has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 37, kind:          'OpenParen', line: {{ *}}[[@LINE-10]], column: 26, indent: 3, spelling: '(', closing_token: 52, has_leading_space: true }
     0.9,
-    // CHECK:STDOUT:     { index: 38, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '0.9', value: `9*10^-1`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 39, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  8, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 38, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '0.9', value: `9*10^-1`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 39, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  8, indent: 5, spelling: ',' }
     8.0,
-    // CHECK:STDOUT:     { index: 40, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '8.0', value: `80*10^-1`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 41, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  8, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 40, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '8.0', value: `80*10^-1`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 41, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  8, indent: 5, spelling: ',' }
     80.0,
-    // CHECK:STDOUT:     { index: 42, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '80.0', value: `800*10^-1`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 43, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  9, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 42, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '80.0', value: `800*10^-1`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 43, kind:              'Comma', line: {{ *}}[[@LINE-2]], column:  9, indent: 5, spelling: ',' }
     1.0e7,
-    // CHECK:STDOUT:     { index: 44, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '1.0e7', value: `10*10^6`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 45, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 10, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 44, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '1.0e7', value: `10*10^6`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 45, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 10, indent: 5, spelling: ',' }
     1.0e8,
-    // CHECK:STDOUT:     { index: 46, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '1.0e8', value: `10*10^7`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 47, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 10, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 46, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '1.0e8', value: `10*10^7`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 47, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 10, indent: 5, spelling: ',' }
     1.0e-8,
-    // CHECK:STDOUT:     { index: 48, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '1.0e-8', value: `10*10^-9`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 49, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 11, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 48, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '1.0e-8', value: `10*10^-9`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 49, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 11, indent: 5, spelling: ',' }
     39999999999999999993.0e39999999999999999993,
-    // CHECK:STDOUT:     { index: 50, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '39999999999999999993.0e39999999999999999993', value: `399999999999999999930*10^39999999999999999992`, has_leading_space: true },
-    // CHECK:STDOUT:     { index: 51, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 48, indent: 5, spelling: ',' },
+    // CHECK:STDOUT:   - { index: 50, kind:        'RealLiteral', line: {{ *}}[[@LINE-1]], column:  5, indent: 5, spelling: '39999999999999999993.0e39999999999999999993', value: `399999999999999999930*10^39999999999999999992`, has_leading_space: true }
+    // CHECK:STDOUT:   - { index: 51, kind:              'Comma', line: {{ *}}[[@LINE-2]], column: 48, indent: 5, spelling: ',' }
   );
-  // CHECK:STDOUT:     { index: 52, kind:         'CloseParen', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: ')', opening_token: 37, has_leading_space: true },
-  // CHECK:STDOUT:     { index: 53, kind:               'Semi', line: {{ *}}[[@LINE-2]], column:  4, indent: 3, spelling: ';' },
+  // CHECK:STDOUT:   - { index: 52, kind:         'CloseParen', line: {{ *}}[[@LINE-1]], column:  3, indent: 3, spelling: ')', opening_token: 37, has_leading_space: true }
+  // CHECK:STDOUT:   - { index: 53, kind:               'Semi', line: {{ *}}[[@LINE-2]], column:  4, indent: 3, spelling: ';' }
 }
-// CHECK:STDOUT:     { index: 54, kind:    'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '}', opening_token: 5, has_leading_space: true },
+// CHECK:STDOUT:   - { index: 54, kind:    'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '}', opening_token: 5, has_leading_space: true }
 
 
-// CHECK:STDOUT:     { index: 55, kind:            'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
 // --- fail_binary_real.carbon
 // CHECK:STDOUT: - filename: fail_binary_real.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:   'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 // CHECK:STDERR: fail_binary_real.carbon:[[@LINE+4]]:4: error: binary real number literals are not supported [BinaryRealLiteral]
 // CHECK:STDERR: 0b1.0
 // CHECK:STDERR:    ^
 // CHECK:STDERR:
 0b1.0
-// CHECK:STDOUT:     { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '0b1.0', value: `2*2^-1`, has_leading_space: true },
+// CHECK:STDOUT:   - { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '0b1.0', value: `2*2^-1`, has_leading_space: true }
 
-// CHECK:STDOUT:     { index: 2, kind:     'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
 // --- fail_wrong_real_exponent.carbon
 // CHECK:STDOUT: - filename: fail_wrong_real_exponent.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 // CHECK:STDERR: fail_wrong_real_exponent.carbon:[[@LINE+3]]:4: error: expected 'e' to introduce exponent [WrongRealLiteralExponent]
 // CHECK:STDERR: 1.0r3
 // CHECK:STDERR:    ^
 1.0r3
-// CHECK:STDOUT:     { index: 1, kind:     'Error', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '1.0r3', has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 2, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 1, kind:     'Error', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '1.0r3', has_leading_space: true }

toolchain/lex/testdata/printing_digit_padding.carbon

@@ -8,17 +8,13 @@
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/printing_digit_padding.carbon
 // CHECK:STDOUT: - filename: printing_digit_padding.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind: 'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 ;
-// CHECK:STDOUT:     { index: 1, kind:      'Semi', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: ';', has_leading_space: true },
+// CHECK:STDOUT:   - { index: 1, kind:      'Semi', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: ';', has_leading_space: true }
 
 // Test digit padding with values of 9, 10, and 11.
         ;;;
-        // CHECK:STDOUT:     { index: 2, kind:      'Semi', line: {{ *}}[[@LINE-1]], column:  9, indent: 9, spelling: ';', has_leading_space: true },
-        // CHECK:STDOUT:     { index: 3, kind:      'Semi', line: {{ *}}[[@LINE-2]], column: 10, indent: 9, spelling: ';' },
-        // CHECK:STDOUT:     { index: 4, kind:      'Semi', line: {{ *}}[[@LINE-3]], column: 11, indent: 9, spelling: ';' },
-
-// CHECK:STDOUT:     { index: 5, kind:   'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+        // CHECK:STDOUT:   - { index: 2, kind:      'Semi', line: {{ *}}[[@LINE-1]], column:   9, indent: 9, spelling: ';', has_leading_space: true }
+        // CHECK:STDOUT:   - { index: 3, kind:      'Semi', line: {{ *}}[[@LINE-2]], column:  10, indent: 9, spelling: ';' }
+        // CHECK:STDOUT:   - { index: 4, kind:      'Semi', line: {{ *}}[[@LINE-3]], column:  11, indent: 9, spelling: ';' }

toolchain/lex/testdata/printing_integer_literal.carbon

@@ -8,11 +8,7 @@
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/printing_integer_literal.carbon
 // CHECK:STDOUT: - filename: printing_integer_literal.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:  'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 123
-// CHECK:STDOUT:     { index: 1, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '123', value: `123`, has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 2, kind:    'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 1, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '123', value: `123`, has_leading_space: true }

toolchain/lex/testdata/printing_real_literal.carbon

@@ -8,11 +8,7 @@
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/printing_real_literal.carbon
 // CHECK:STDOUT: - filename: printing_real_literal.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:   'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 2.5
-// CHECK:STDOUT:     { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 2, kind:     'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_leading_space: true }

toolchain/lex/testdata/printing_token.carbon

@@ -8,11 +8,7 @@
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/printing_token.carbon
 // CHECK:STDOUT: - filename: printing_token.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:  'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 0x9
-// CHECK:STDOUT:     { index: 1, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: '0x9', value: `9`, has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 2, kind:    'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 1, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: '0x9', value: `9`, has_leading_space: true }

toolchain/lex/testdata/raw_identifier.carbon

@@ -8,32 +8,28 @@
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/raw_identifier.carbon
 // CHECK:STDOUT: - filename: raw_identifier.carbon
-// CHECK:STDOUT:   tokens: [
-// CHECK:STDOUT:     { index: 0, kind:           'FileStart', line: {{ *\d+}}, column:  1, indent: 1, spelling: '' },
+// CHECK:STDOUT:   tokens:
 
 // A non-keyword identifier.
 r#foo
-// CHECK:STDOUT:     { index: 1, kind:          'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'foo', identifier: 1, has_leading_space: true },
+// CHECK:STDOUT:   - { index: 1, kind:          'Identifier', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'foo', identifier: 1, has_leading_space: true }
 
 // The same non-keyword identifier, for comparison.
 foo
-// CHECK:STDOUT:     { index: 2, kind:          'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'foo', identifier: 1, has_leading_space: true },
+// CHECK:STDOUT:   - { index: 2, kind:          'Identifier', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'foo', identifier: 1, has_leading_space: true }
 
 // A keyword as a raw identifier.
 r#self
-// CHECK:STDOUT:     { index: 3, kind:          'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'self', identifier: 2, has_leading_space: true },
+// CHECK:STDOUT:   - { index: 3, kind:          'Identifier', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'self', identifier: 2, has_leading_space: true }
 
 // The same keyword, for comparison.
 self
-// CHECK:STDOUT:     { index: 4, kind: 'SelfValueIdentifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'self', has_leading_space: true },
+// CHECK:STDOUT:   - { index: 4, kind: 'SelfValueIdentifier', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'self', has_leading_space: true }
 
 // A type literal as a raw identifier.
 r#i32
-// CHECK:STDOUT:     { index: 5, kind:          'Identifier', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'i32', identifier: 3, has_leading_space: true },
+// CHECK:STDOUT:   - { index: 5, kind:          'Identifier', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'i32', identifier: 3, has_leading_space: true }
 
 // The same type literal, for comparison.
 i32
-// CHECK:STDOUT:     { index: 6, kind:      'IntTypeLiteral', line: {{ *}}[[@LINE-1]], column:  1, indent: 1, spelling: 'i32', has_leading_space: true },
-
-// CHECK:STDOUT:     { index: 7, kind:             'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true },
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   - { index: 6, kind:      'IntTypeLiteral', line: {{ *}}[[@LINE-1]], column:   1, indent: 1, spelling: 'i32', has_leading_space: true }

toolchain/lex/tokenized_buffer.cpp

@@ -215,13 +215,10 @@ auto TokenizedBuffer::GetTokenPrintWidths(TokenIndex token) const
   return widths;
 }
 
-auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void {
-  if (tokens().begin() == tokens().end()) {
-    return;
-  }
-
+auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream,
+                            bool omit_file_boundary_tokens) const -> void {
   output_stream << "- filename: " << source_->filename() << "\n"
-                << "  tokens: [\n";
+                << "  tokens:\n";
 
   PrintWidths widths = {};
   widths.index = ComputeDecimalPrintedWidth((token_infos_.size()));
@@ -230,10 +227,15 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void {
   }
 
   for (TokenIndex token : tokens()) {
+    if (omit_file_boundary_tokens) {
+      auto kind = GetKind(token);
+      if (kind == TokenKind::FileStart || kind == TokenKind::FileEnd) {
+        continue;
+      }
+    }
     PrintToken(output_stream, token, widths);
     output_stream << "\n";
   }
-  output_stream << "  ]\n";
 }
 
 auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
@@ -254,7 +256,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
   // justification manually in order to use the dynamically computed widths
   // and get the quotes included.
   output_stream << llvm::formatv(
-      "    { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, "
+      "  - { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, "
       "spelling: '{5}'",
       llvm::format_decimal(token_index, widths.index),
       llvm::right_justify(
@@ -304,7 +306,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
     output_stream << ", recovery: true";
   }
 
-  output_stream << " },";
+  output_stream << " }";
 }
 
 // Find the line index corresponding to a specific byte offset within the source

toolchain/lex/tokenized_buffer.h

@@ -173,26 +173,15 @@ class TokenizedBuffer : public Printable<TokenizedBuffer> {
   // Returns the comment's full text range.
   auto GetCommentText(CommentIndex comment_index) const -> llvm::StringRef;
 
-  // Prints a description of the tokenized stream to the provided `raw_ostream`.
+  // Returns tokens as YAML. This prints the tracked token information on a
+  // single line for each token. We use the single-line format so that output is
+  // compact, and so that tools like `grep` are compatible.
   //
-  // It prints one line of information for each token in the buffer, including
-  // the kind of token, where it occurs within the source file, indentation for
-  // the associated line, the spelling of the token in source, and any
-  // additional information tracked such as which unique identifier it is or any
-  // matched grouping token.
+  // An example token looks like:
   //
-  // Each line is formatted as a YAML record:
-  //
-  // clang-format off
-  // ```
-  // token: { index: 0, kind: 'Semi', line: 1, column: 1, indent: 1, spelling: ';' }
-  // ```
-  // clang-format on
-  //
-  // This can be parsed as YAML using tools like `python-yq` combined with `jq`
-  // on the command line. The format is also reasonably amenable to other
-  // line-oriented shell tools from `grep` to `awk`.
-  auto Print(llvm::raw_ostream& output_stream) const -> void;
+  // - { index: 1, kind: 'Semi', line: 1, column: 1, indent: 1, spelling: ';' }
+  auto Print(llvm::raw_ostream& out,
+             bool omit_file_boundary_tokens = false) const -> void;
 
   // Prints a description of a single token.  See `Print` for details on the
   // format.

toolchain/testing/file_test.cpp

@@ -67,7 +67,7 @@ class ToolchainFileTest : public FileTestBase {
         "compile", "--include-diagnostic-kind", "--phase=" + component_.str()};
 
     if (component_ == "lex") {
-      args.push_back("--dump-tokens");
+      args.insert(args.end(), {"--dump-tokens", "--omit-file-boundary-tokens"});
     } else if (component_ == "parse") {
       args.push_back("--dump-parse-tree");
     } else if (component_ == "check") {