Merge pull request #623 from ehaas/expansion-tokens

Store expansion locations separately from tokens
Vexu · Feb 9, 2024 · c72f45c · c72f45c
2 parents e5855df + f21caa8
commit c72f45c
Show file tree

Hide file tree

Showing 11 changed files with 222 additions and 134 deletions.
diff --git a/src/aro/Hideset.zig b/src/aro/Hideset.zig
@@ -81,6 +81,12 @@ pub fn clearRetainingCapacity(self: *Hideset) void {
     self.map.clearRetainingCapacity();
 }
 
+pub fn clearAndFree(self: *Hideset) void {
+    self.map.clearAndFree(self.comp.gpa);
+    self.intersection_map.clearAndFree(self.comp.gpa);
+    self.linked_list.shrinkAndFree(self.comp.gpa, 0);
+}
+
 /// Iterator is invalidated if the underlying MultiArrayList slice is reallocated due to resize
 fn iterator(self: *const Hideset, idx: Index) Iterator {
     return Iterator{

diff --git a/src/aro/Parser.zig b/src/aro/Parser.zig
@@ -403,7 +403,7 @@ pub fn errExtra(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex, extra: Diag
         .tag = tag,
         .loc = loc,
         .extra = extra,
-    }, tok.expansionSlice());
+    }, p.pp.expansionSlice(tok_i));
 }
 
 pub fn errTok(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex) Compilation.Error!void {

diff --git a/src/aro/Preprocessor.zig b/src/aro/Preprocessor.zig
diff --git a/src/aro/Tree.zig b/src/aro/Tree.zig
@@ -12,6 +12,16 @@ const StringInterner = @import("StringInterner.zig");
 
 pub const Token = struct {
     id: Id,
+    loc: Source.Location,
+
+    pub const List = std.MultiArrayList(Token);
+    pub const Id = Tokenizer.Token.Id;
+    pub const NumberPrefix = number_affixes.Prefix;
+    pub const NumberSuffix = number_affixes.Suffix;
+};
+
+pub const TokenWithExpansionLocs = struct {
+    id: Token.Id,
     flags: packed struct {
         expansion_disabled: bool = false,
         is_macro_arg: bool = false,
@@ -22,14 +32,14 @@ pub const Token = struct {
     loc: Source.Location,
     expansion_locs: ?[*]Source.Location = null,
 
-    pub fn expansionSlice(tok: Token) []const Source.Location {
+    pub fn expansionSlice(tok: TokenWithExpansionLocs) []const Source.Location {
         const locs = tok.expansion_locs orelse return &[0]Source.Location{};
         var i: usize = 0;
         while (locs[i].id != .unused) : (i += 1) {}
         return locs[0..i];
     }
 
-    pub fn addExpansionLocation(tok: *Token, gpa: std.mem.Allocator, new: []const Source.Location) !void {
+    pub fn addExpansionLocation(tok: *TokenWithExpansionLocs, gpa: std.mem.Allocator, new: []const Source.Location) !void {
         if (new.len == 0 or tok.id == .whitespace or tok.id == .macro_ws or tok.id == .placemarker) return;
         var list = std.ArrayList(Source.Location).init(gpa);
         defer {
@@ -70,14 +80,14 @@ pub const Token = struct {
         gpa.free(locs[0 .. i + 1]);
     }
 
-    pub fn dupe(tok: Token, gpa: std.mem.Allocator) !Token {
+    pub fn dupe(tok: TokenWithExpansionLocs, gpa: std.mem.Allocator) !TokenWithExpansionLocs {
         var copy = tok;
         copy.expansion_locs = null;
         try copy.addExpansionLocation(gpa, tok.expansionSlice());
         return copy;
     }
 
-    pub fn checkMsEof(tok: Token, source: Source, comp: *Compilation) !void {
+    pub fn checkMsEof(tok: TokenWithExpansionLocs, source: Source, comp: *Compilation) !void {
         std.debug.assert(tok.id == .eof);
         if (source.buf.len > tok.loc.byte_offset and source.buf[tok.loc.byte_offset] == 0x1A) {
             try comp.addDiagnostic(.{
@@ -90,11 +100,6 @@ pub const Token = struct {
             }, &.{});
         }
     }
-
-    pub const List = std.MultiArrayList(Token);
-    pub const Id = Tokenizer.Token.Id;
-    pub const NumberPrefix = number_affixes.Prefix;
-    pub const NumberSuffix = number_affixes.Suffix;
 };
 
 pub const TokenIndex = u32;

diff --git a/src/aro/pragmas/gcc.zig b/src/aro/pragmas/gcc.zig
@@ -80,7 +80,7 @@ fn diagnosticHandler(self: *GCC, pp: *Preprocessor, start_idx: TokenIndex) Pragm
                         .tag = .pragma_requires_string_literal,
                         .loc = diagnostic_tok.loc,
                         .extra = .{ .str = "GCC diagnostic" },
-                    }, diagnostic_tok.expansionSlice());
+                    }, pp.expansionSlice(start_idx));
                 },
                 else => |e| return e,
             };
@@ -90,7 +90,7 @@ fn diagnosticHandler(self: *GCC, pp: *Preprocessor, start_idx: TokenIndex) Pragm
                     .tag = .malformed_warning_check,
                     .loc = next.loc,
                     .extra = .{ .str = "GCC diagnostic" },
-                }, next.expansionSlice());
+                }, pp.expansionSlice(start_idx + 1));
             }
             const new_kind: Diagnostics.Kind = switch (diagnostic) {
                 .ignored => .off,
@@ -116,7 +116,7 @@ fn preprocessorHandler(pragma: *Pragma, pp: *Preprocessor, start_idx: TokenIndex
         return pp.comp.addDiagnostic(.{
         .tag = .unknown_gcc_pragma,
         .loc = directive_tok.loc,
-    }, directive_tok.expansionSlice());
+    }, pp.expansionSlice(start_idx + 1));
 
     switch (gcc_pragma) {
         .warning, .@"error" => {
@@ -126,15 +126,15 @@ fn preprocessorHandler(pragma: *Pragma, pp: *Preprocessor, start_idx: TokenIndex
                         .tag = .pragma_requires_string_literal,
                         .loc = directive_tok.loc,
                         .extra = .{ .str = @tagName(gcc_pragma) },
-                    }, directive_tok.expansionSlice());
+                    }, pp.expansionSlice(start_idx + 1));
                 },
                 else => |e| return e,
             };
             const extra = Diagnostics.Message.Extra{ .str = try pp.comp.diagnostics.arena.allocator().dupe(u8, text) };
             const diagnostic_tag: Diagnostics.Tag = if (gcc_pragma == .warning) .pragma_warning_message else .pragma_error_message;
             return pp.comp.addDiagnostic(
                 .{ .tag = diagnostic_tag, .loc = directive_tok.loc, .extra = extra },
-                directive_tok.expansionSlice(),
+                pp.expansionSlice(start_idx + 1),
             );
         },
         .diagnostic => return self.diagnosticHandler(pp, start_idx + 2) catch |err| switch (err) {
@@ -143,12 +143,12 @@ fn preprocessorHandler(pragma: *Pragma, pp: *Preprocessor, start_idx: TokenIndex
                 return pp.comp.addDiagnostic(.{
                     .tag = .unknown_gcc_pragma_directive,
                     .loc = tok.loc,
-                }, tok.expansionSlice());
+                }, pp.expansionSlice(start_idx + 2));
             },
             else => |e| return e,
         },
         .poison => {
-            var i: usize = 2;
+            var i: u32 = 2;
             while (true) : (i += 1) {
                 const tok = pp.tokens.get(start_idx + i);
                 if (tok.id == .nl) break;
@@ -157,14 +157,14 @@ fn preprocessorHandler(pragma: *Pragma, pp: *Preprocessor, start_idx: TokenIndex
                     return pp.comp.addDiagnostic(.{
                         .tag = .pragma_poison_identifier,
                         .loc = tok.loc,
-                    }, tok.expansionSlice());
+                    }, pp.expansionSlice(start_idx + i));
                 }
                 const str = pp.expandedSlice(tok);
                 if (pp.defines.get(str) != null) {
                     try pp.comp.addDiagnostic(.{
                         .tag = .pragma_poison_macro,
                         .loc = tok.loc,
-                    }, tok.expansionSlice());
+                    }, pp.expansionSlice(start_idx + i));
                 }
                 try pp.poisoned_identifiers.put(str, {});
             }

diff --git a/src/aro/pragmas/message.zig b/src/aro/pragmas/message.zig
@@ -28,7 +28,7 @@ fn deinit(pragma: *Pragma, comp: *Compilation) void {
 
 fn preprocessorHandler(_: *Pragma, pp: *Preprocessor, start_idx: TokenIndex) Pragma.Error!void {
     const message_tok = pp.tokens.get(start_idx);
-    const message_expansion_locs = message_tok.expansionSlice();
+    const message_expansion_locs = pp.expansionSlice(start_idx);
 
     const str = Pragma.pasteTokens(pp, start_idx + 1) catch |err| switch (err) {
         error.ExpectedStringLiteral => {

diff --git a/src/aro/pragmas/once.zig b/src/aro/pragmas/once.zig
@@ -45,7 +45,7 @@ fn preprocessorHandler(pragma: *Pragma, pp: *Preprocessor, start_idx: TokenIndex
         try pp.comp.addDiagnostic(.{
             .tag = .extra_tokens_directive_end,
             .loc = name_tok.loc,
-        }, next.expansionSlice());
+        }, pp.expansionSlice(start_idx + 1));
     }
     const seen = self.preprocess_count == pp.preprocess_count;
     const prev = try self.pragma_once.fetchPut(name_tok.loc.id, {});

diff --git a/src/aro/pragmas/pack.zig b/src/aro/pragmas/pack.zig
@@ -37,7 +37,7 @@ fn parserHandler(pragma: *Pragma, p: *Parser, start_idx: TokenIndex) Compilation
         return p.comp.addDiagnostic(.{
             .tag = .pragma_pack_lparen,
             .loc = l_paren.loc,
-        }, l_paren.expansionSlice());
+        }, p.pp.expansionSlice(idx));
     }
     idx += 1;
 

diff --git a/test/fuzz/fuzz_lib.zig b/test/fuzz/fuzz_lib.zig
@@ -45,7 +45,7 @@ fn processSource(comp: *Compilation, builtin: Source, user_source: Source) !void
 
     _ = try pp.preprocess(builtin);
     const eof = try pp.preprocess(user_source);
-    try pp.tokens.append(pp.comp.gpa, eof);
+    try pp.addToken(eof);
 
     var tree = try Parser.parse(&pp);
     defer tree.deinit();

diff --git a/test/record_runner.zig b/test/record_runner.zig
@@ -281,7 +281,7 @@ fn singleRun(alloc: std.mem.Allocator, test_dir: []const u8, test_case: TestCase
         stats.progress.log("could not preprocess file '{s}': {s}\n", .{ path, @errorName(err) });
         return;
     };
-    try pp.tokens.append(alloc, eof);
+    try pp.addToken(eof);
 
     var tree = try aro.Parser.parse(&pp);
     defer tree.deinit();

diff --git a/test/runner.zig b/test/runner.zig
@@ -80,7 +80,7 @@ fn testOne(allocator: std.mem.Allocator, path: []const u8, test_dir: []const u8)
     _ = try pp.preprocess(user_macros);
 
     const eof = try pp.preprocess(file);
-    try pp.tokens.append(allocator, eof);
+    try pp.addToken(eof);
 
     var tree = try aro.Parser.parse(&pp);
     defer tree.deinit();
@@ -237,7 +237,7 @@ pub fn main() !void {
             progress.log("could not preprocess file '{s}': {s}\n", .{ path, @errorName(err) });
             continue;
         };
-        try pp.tokens.append(gpa, eof);
+        try pp.addToken(eof);
 
         if (pp.defines.get("TESTS_SKIPPED")) |macro| {
             if (macro.is_func or macro.tokens.len != 1 or macro.tokens[0].id != .pp_num) {