Tokenizer: treat Ctrl+Z as EOF if ms extensions are enabled

src/Diagnostics.zig

@@ -169,6 +169,7 @@ pub const Options = packed struct {
     @"language-extension-token": Kind = .default,
     @"complex-component-init": Kind = .default,
     @"microsoft-include": Kind = .default,
+    @"microsoft-end-of-file": Kind = .default,
 };
 
 const messages = struct {
@@ -2410,7 +2411,6 @@ const messages = struct {
         const opt = "pedantic";
         const extra = .str;
         const kind = .off;
-        const pedantic = true;
     };
     pub const not_floating_type = struct {
         const msg = "argument type '{s}' is not a real floating point type";
@@ -2428,6 +2428,12 @@ const messages = struct {
         const opt = "microsoft-include";
         const kind = .warning;
     };
+    pub const ctrl_z_eof = struct {
+        const msg = "treating Ctrl-Z as end-of-file is a Microsoft extension";
+        const opt = "microsoft-end-of-file";
+        const kind = .off;
+        const pedantic = true;
+    };
 };
 
 list: std.ArrayListUnmanaged(Message) = .{},

src/Preprocessor.zig

@@ -206,11 +206,13 @@ pub fn deinit(pp: *Preprocessor) void {
 
 /// Preprocess a source file, returns eof token.
 pub fn preprocess(pp: *Preprocessor, source: Source) Error!Token {
-    return pp.preprocessExtra(source) catch |er| switch (er) {
+    const eof = pp.preprocessExtra(source) catch |er| switch (er) {
         // This cannot occur in the main file and is handled in `include`.
         error.StopPreprocessing => unreachable,
         else => |e| return e,
     };
+    try eof.checkMsEof(source, pp.comp);
+    return eof;
 }
 
 /// Return the name of the #ifndef guard macro that starts a source, if any.
@@ -2338,10 +2340,11 @@ fn include(pp: *Preprocessor, tokenizer: *Tokenizer, which: Compilation.WhichInc
         pp.verboseLog(first, "include file {s}", .{new_source.path});
     }
 
-    _ = pp.preprocessExtra(new_source) catch |er| switch (er) {
-        error.StopPreprocessing => {},
+    const eof = pp.preprocessExtra(new_source) catch |er| switch (er) {
+        error.StopPreprocessing => return,
         else => |e| return e,
     };
+    try eof.checkMsEof(new_source, pp.comp);
 }
 
 /// tokens that are part of a pragma directive can happen in 3 ways:

src/Tokenizer.zig

@@ -1120,6 +1120,14 @@ pub fn next(self: *Tokenizer) Token {
                     self.index += 1;
                     break;
                 },
+                0x1A => if (self.comp.langopts.ms_extensions) {
+                    id = .eof;
+                    break;
+                } else {
+                    id = .invalid;
+                    self.index += 1;
+                    break;
+                },
                 0x80...0xFF => state = .extended_identifier,
                 else => {
                     id = .invalid;

src/Tree.zig

@@ -77,6 +77,20 @@ pub const Token = struct {
         return copy;
     }
 
+    pub fn checkMsEof(tok: Token, source: Source, comp: *Compilation) !void {
+        std.debug.assert(tok.id == .eof);
+        if (source.buf.len > tok.loc.byte_offset and source.buf[tok.loc.byte_offset] == 0x1A) {
+            try comp.diag.add(.{
+                .tag = .ctrl_z_eof,
+                .loc = .{
+                    .id = source.id,
+                    .byte_offset = tok.loc.byte_offset,
+                    .line = tok.loc.line,
+                },
+            }, &.{});
+        }
+    }
+
     pub const List = std.MultiArrayList(Token);
     pub const Id = Tokenizer.Token.Id;
 };

test/cases/ms-extensions.c

@@ -12,3 +12,5 @@
 #ifndef OTHER_INCLUDED
 #error Microsoft search rule should work
 #endif
+
+don't mind me ;)