Merge pull request #527 from ehaas/strings

Add wide & unicode string literal support
Vexu · Oct 21, 2023 · 070fbf8 · 070fbf8
2 parents 57fe911 + 05a7329
commit 070fbf8
Show file tree

Hide file tree

Showing 20 changed files with 527 additions and 283 deletions.
diff --git a/src/Attribute.zig b/src/Attribute.zig
@@ -60,7 +60,7 @@ pub const ArgumentType = enum {
 
     fn fromType(comptime T: type) ArgumentType {
         return switch (T) {
-            []const u8 => .string,
+            Value.ByteRange => .string,
             Identifier => .identifier,
             u32 => .int,
             Alignment => .alignment,
@@ -263,10 +263,17 @@ fn diagnoseField(
         .bytes => {
             const bytes = val.data.bytes.trim(1); // remove null terminator
             if (wanted == Value.ByteRange) {
+                std.debug.assert(node.tag == .string_literal_expr);
+                if (!node.ty.elemType().is(.char) and !node.ty.elemType().is(.uchar)) {
+                    return Diagnostics.Message{
+                        .tag = .attribute_requires_string,
+                        .extra = .{ .str = decl.name },
+                    };
+                }
                 @field(@field(arguments, decl.name), field.name) = bytes;
                 return null;
             } else if (@typeInfo(wanted) == .Enum and @hasDecl(wanted, "opts") and wanted.opts.enum_kind == .string) {
-                const str = bytes.slice(strings);
+                const str = bytes.slice(strings, .@"1");
                 if (std.meta.stringToEnum(wanted, str)) |enum_val| {
                     @field(@field(arguments, decl.name), field.name) = enum_val;
                     return null;

diff --git a/src/Compilation.zig b/src/Compilation.zig
@@ -238,6 +238,8 @@ pub fn generateBuiltinMacros(comp: *Compilation) !Source {
         \\#define __STDC_NO_COMPLEX__ 1
         \\#define __STDC_NO_THREADS__ 1
         \\#define __STDC_NO_VLA__ 1
+        \\#define __STDC_UTF_16__ 1
+        \\#define __STDC_UTF_32__ 1
         \\
     );
     if (comp.langopts.standard.StdCVersionMacro()) |stdc_version| {
@@ -1428,6 +1430,20 @@ pub fn hasBuiltinFunction(comp: *const Compilation, builtin: Builtin) bool {
     }
 }
 
+pub const CharUnitSize = enum(u32) {
+    @"1" = 1,
+    @"2" = 2,
+    @"4" = 4,
+
+    pub fn Type(comptime self: CharUnitSize) type {
+        return switch (self) {
+            .@"1" => u8,
+            .@"2" => u16,
+            .@"4" => u32,
+        };
+    }
+};
+
 pub const renderErrors = Diagnostics.render;
 
 test "addSourceFromReader" {

diff --git a/src/Diagnostics.zig b/src/Diagnostics.zig
@@ -179,6 +179,7 @@ pub const Options = packed struct {
     @"invalid-source-encoding": Kind = .default,
     @"four-char-constants": Kind = .default,
     @"unknown-escape-sequence": Kind = .default,
+    @"invalid-pp-token": Kind = .default,
 };
 
 const messages = struct {
@@ -2510,6 +2511,42 @@ const messages = struct {
         const opt = "unknown-escape-sequence";
         const extra = .invalid_escape;
     };
+    pub const attribute_requires_string = struct {
+        const msg = "attribute '{s}' requires an ordinary string";
+        const kind = .@"error";
+        const extra = .str;
+    };
+    pub const unterminated_string_literal_warning = struct {
+        const msg = "missing terminating '\"' character";
+        const kind = .warning;
+        const opt = "invalid-pp-token";
+    };
+    pub const unterminated_string_literal_error = struct {
+        const msg = "missing terminating '\"' character";
+        const kind = .@"error";
+    };
+    pub const empty_char_literal_warning = struct {
+        const msg = "empty character constant";
+        const kind = .warning;
+        const opt = "invalid-pp-token";
+    };
+    pub const empty_char_literal_error = struct {
+        const msg = "empty character constant";
+        const kind = .@"error";
+    };
+    pub const unterminated_char_literal_warning = struct {
+        const msg = "missing terminating ' character";
+        const kind = .warning;
+        const opt = "invalid-pp-token";
+    };
+    pub const unterminated_char_literal_error = struct {
+        const msg = "missing terminating ' character";
+        const kind = .@"error";
+    };
+    pub const unterminated_comment = struct {
+        const msg = "unterminated comment";
+        const kind = .@"error";
+    };
 };
 
 list: std.ArrayListUnmanaged(Message) = .{},

diff --git a/src/Ir.zig b/src/Ir.zig
@@ -552,7 +552,7 @@ fn writeValue(ir: Ir, val_ref: Interner.Ref, color: bool, w: anytype) !void {
     switch (v.tag) {
         .unavailable => try w.writeAll(" unavailable"),
         .int => try w.print("{d}", .{v.data.int}),
-        .bytes => try w.print("\"{s}\"", .{v.data.bytes.slice(ir.strings)}),
+        .bytes => try w.print("\"{s}\"", .{v.data.bytes.slice(ir.strings, .@"1")}),
         // std.fmt does @as instead of @floatCast
         .float => try w.print("{d}", .{@as(f64, @floatCast(v.data.float))}),
         else => try w.print("({s})", .{@tagName(v.tag)}),