[WIP] Extend synchronization points to leverage fixed-size units/fields

hilti/toolchain/include/ast/builder/builder.h

@@ -431,6 +431,10 @@ class Builder : public builder::NodeFactory {
         return expressionUnresolvedOperator(operator_::Kind::Division, {op1, op2}, m);
     }
 
+    auto multiple(Expression* op1, Expression* op2, const Meta& m = Meta()) {
+        return expressionUnresolvedOperator(operator_::Kind::Multiple, {op1, op2}, m);
+    }
+
     // Other expressions
 
     auto expression(Ctor* c, const Meta& m = Meta()) { return expressionCtor(c, m); }

hilti/toolchain/include/ast/ctors/optional.h

@@ -42,7 +42,8 @@ class Optional : public Ctor {
     static auto create(ASTContext* ctx, QualifiedType* type, const Meta& meta = {}) {
         return ctx->make<Optional>(ctx,
                                    {
-                                       QualifiedType::create(ctx, type::Optional::create(ctx, type), Constness::Const),
+                                       QualifiedType::create(ctx, type::Optional::create(ctx, type),
+                                                             Constness::Mutable),
                                        nullptr,
                                    },
                                    meta);

hilti/toolchain/src/compiler/coercer.cc

@@ -853,7 +853,9 @@ Result<std::pair<bool, Expressions>> hilti::coerceOperands(Builder* builder, ope
         switch ( op->kind() ) {
             case parameter::Kind::In:
             case parameter::Kind::Copy: needs_mutable = false; break;
-            case parameter::Kind::InOut: needs_mutable = true; break;
+            case parameter::Kind::InOut:
+                needs_mutable = false;
+                break; // TODO: should be true, but doesn't work with optional inputs
             case parameter::Kind::Unknown: logger().internalError("unknown operand kind"); break;
         }
 

spicy/toolchain/CMakeLists.txt

@@ -34,6 +34,7 @@ set(SOURCES_COMPILER
     src/compiler/codegen/parsers/literals.cc
     src/compiler/codegen/parsers/types.cc
     src/compiler/codegen/production.cc
+    src/compiler/codegen/productions/ctor.cc
     src/compiler/codegen/productions/look-ahead.cc
     src/compiler/codegen/productions/switch.cc
     src/compiler/codegen/productions/while.cc

spicy/toolchain/include/ast/types/unit-items/field.h

@@ -79,10 +79,10 @@ class Field : public unit::Item {
     std::optional<std::pair<Expression*, QualifiedType*>> convertExpression() const;
 
     /**
-     * Returns an expression representing the number of bytes the fields
+     * Returns an expression representing the number of bytes the field
      * consumes, if known.
      */
-    Expression* size(ASTContext* ctx) const;
+    Expression* parseSize(Builder* builder) const;
 
     void setForwarding(bool is_forwarding) { _is_forwarding = is_forwarding; }
     void setTransient(bool is_transient) { _is_transient = is_transient; }

spicy/toolchain/include/compiler/detail/codegen/parser-builder.h

@@ -177,6 +177,19 @@ struct ParserState {
      * Expression* holding the last parse error if any. This field is set only in sync or trial mode.
      */
     Expression* error = nullptr;
+
+    /**
+     * If set, expression referencing an optional iterator to set while parsing a
+     * production as soon as parsing knows the end of the bytes data the
+     * production will consume. Once this is set, upper-level error recovery
+     * might use that information to jump there for continuing parsing.
+     *
+     * To work with this, before a production starts parsing, it needs to call
+     * ParserBuilder::setEndOfProduction(), either with a specific iterator
+     * where the production will end parsing, or with a null value to indicate
+     * that it cannot know.
+     */
+    Expression* end_of_production = nullptr;
 };
 
 /** Generates the parsing logic for a unit type. */
@@ -402,27 +415,20 @@ class ParserBuilder {
     Expression* atEod();
 
     /**
-     * Generates code that advances the current view to the next position which is not a gap.
-     * This implicitly calls advancedInput() afterwards.
+     * Generates code that advances the current view to the next position which
+     * is not a gap. This implicitly calls `trimInput()` afterwards.
      */
     void advanceToNextData();
 
     /**
      * Generates code that advances the current view to a new start position.
-     * This implicitly calls advancedInput() afterwards.
+     * This implicitly calls `trimInput()` afterwards.
      *
-     * @param i expression that's either the number of bytes to move ahead,
-     * a stream iterator to move to, or a new stream view to use from now on.
+     * @param i expression that's either the number of bytes to move ahead or a
+     * stream iterator to move to.
      */
     void advanceInput(Expression* i);
 
-    /**
-     * Generates code that sets the current view.
-     *
-     * @param i expression that's the new view to use.
-     */
-    void setInput(Expression* i);
-
     /**
      * Generates code that saves the current parsing position inside the
      * current parse object. This only has an effect for unit types that

spicy/toolchain/include/compiler/detail/codegen/production.h

@@ -179,6 +179,24 @@ class Production {
     /** Returns any type associated with this production. */
     virtual QualifiedType* type() const { return nullptr; };
 
+    /**
+     * Returns an expression representing the number of bytes the production
+     * consumes, if known. Returns null if the size cannot be computed.
+     *
+     * The resulting expression does not take any type-independent field
+     * parsing attributes into account (e.g., `&size`). It's up to the caller
+     * to apply those if needed. However, the expression will (and must) take
+     * type-specific attributes into account (e.g., `&ipv4` for an address).
+     *
+     * Note that the returned expression may not be a constant: it can depend
+     * on other values not known at compile time.
+     *
+     * @param builder builder to use for creating the expression
+     * @return expression yielding the size, or null if not available; must
+     * evaluate to a value of type `uint64`.
+     */
+    virtual Expression* parseSize(Builder* builder) const = 0;
+
     /**
      * Returns a ID for this literal that's guaranteed to be globally unique
      * for the literal's value, including across grammars. Returns a negative

spicy/toolchain/include/compiler/detail/codegen/productions/block.h

@@ -2,14 +2,14 @@
 
 #pragma once
 
-#include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
 #include <hilti/base/util.h>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
@@ -54,6 +54,27 @@ class Block : public Production {
         return rhss;
     }
 
+    Expression* parseSize(Builder* builder) const final {
+        if ( _condition || ! _else_prods.empty() )
+            return nullptr;
+
+        // TODO: What about attributes()?
+
+        Expression* size = nullptr;
+        for ( const auto& p : _prods ) {
+            auto psize = p->parseSize(builder);
+            if ( ! psize )
+                return nullptr;
+
+            if ( ! size )
+                size = psize;
+            else
+                size = builder->sum(size, psize);
+        }
+
+        return size;
+    }
+
     std::string dump() const final {
         auto true_ = hilti::util::join(hilti::util::transform(_prods, [](const auto& p) { return p->symbol(); }), " ");
         auto false_ =

spicy/toolchain/include/compiler/detail/codegen/productions/counter.h

@@ -7,6 +7,7 @@
 #include <utility>
 #include <vector>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
@@ -33,6 +34,15 @@ class Counter : public Production {
     Expression* expression() const final { return _expression; }
 
     std::vector<std::vector<Production*>> rhss() const final { return {{_body.get()}}; };
+
+    Expression* parseSize(Builder* builder) const final {
+        auto size = _body->parseSize(builder);
+        if ( ! size )
+            return nullptr;
+
+        return builder->multiple(_expression, size);
+    }
+
     std::string dump() const override { return hilti::util::fmt("counter(%s): %s", *_expression, _body->symbol()); }
 
     SPICY_PRODUCTION

spicy/toolchain/include/compiler/detail/codegen/productions/ctor.h

@@ -2,9 +2,7 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
-#include <utility>
 
 #include <hilti/ast/expressions/ctor.h>
 
@@ -29,9 +27,9 @@ class Ctor : public Production {
     bool isNullable() const final { return false; };
     bool isTerminal() const final { return true; };
 
-    // std::vector<std::vector<Production*>> rhss() const final { return {}; };
     Expression* expression() const final { return _ctor; }
     QualifiedType* type() const final { return _ctor->type(); };
+    Expression* parseSize(Builder* builder) const final;
 
     int64_t tokenID() const final {
         return static_cast<int64_t>(Production::tokenID(hilti::util::fmt("%s|%s", *_ctor, *_ctor->type())));

spicy/toolchain/include/compiler/detail/codegen/productions/deferred.h

@@ -2,7 +2,6 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
 
 #include <spicy/compiler/detail/codegen/production.h>
@@ -35,6 +34,7 @@ class Deferred : public Production {
     bool isLiteral() const final { return false; }
     bool isNullable() const final { return false; }
     bool isTerminal() const final { return false; }
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
     int64_t tokenID() const final { return _resolved ? _resolved->tokenID() : -1; };
 
     std::string dump() const final {

spicy/toolchain/include/compiler/detail/codegen/productions/enclosure.h

@@ -9,6 +9,7 @@
 
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
+#include <spicy/compiler/detail/codegen/productions/visitor.h>
 
 namespace spicy::detail::codegen::production {
 
@@ -33,6 +34,7 @@ class Enclosure : public Production {
 
     std::vector<std::vector<Production*>> rhss() const final { return {{_child.get()}}; };
     QualifiedType* type() const final { return _child->type(); };
+    Expression* parseSize(Builder* builder) const final { return _child->parseSize(builder); }
 
     std::string dump() const override { return _child->symbol(); }
 

spicy/toolchain/include/compiler/detail/codegen/productions/epsilon.h

@@ -2,10 +2,10 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
 #include <utility>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
 
@@ -22,6 +22,8 @@ class Epsilon : public Production {
     bool isNullable() const final { return true; };
     bool isTerminal() const final { return true; };
 
+    Expression* parseSize(Builder* builder) const final { return builder->integer(0U); }
+
     std::string dump() const final { return "()"; }
 
     SPICY_PRODUCTION

spicy/toolchain/include/compiler/detail/codegen/productions/for-each.h

@@ -30,6 +30,8 @@ class ForEach : public Production {
 
     std::vector<std::vector<Production*>> rhss() const final { return {{_body.get()}}; };
 
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
+
     std::string dump() const override { return hilti::util::fmt("foreach: %s", _body->symbol()); }
 
     SPICY_PRODUCTION

spicy/toolchain/include/compiler/detail/codegen/productions/look-ahead.h

@@ -55,6 +55,8 @@ class LookAhead : public Production {
         return {{_alternatives.first.get()}, {_alternatives.second.get()}};
     }
 
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
+
     std::string dump() const final;
 
     /**

spicy/toolchain/include/compiler/detail/codegen/productions/reference.h

@@ -2,9 +2,7 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
-#include <utility>
 #include <vector>
 
 #include <spicy/compiler/detail/codegen/production.h>
@@ -38,6 +36,7 @@ class Reference : public Production {
     std::vector<std::vector<Production*>> rhss() const final { return _production->rhss(); }
     Expression* expression() const final { return _production->expression(); }
     QualifiedType* type() const final { return _production->type(); }
+    Expression* parseSize(Builder* builder) const final { return _production->parseSize(builder); }
     int64_t tokenID() const final { return _production->tokenID(); };
 
     std::string dump() const final { return hilti::util::fmt("ref(%s)", _production->dump()); }

spicy/toolchain/include/compiler/detail/codegen/productions/sequence.h

@@ -2,14 +2,14 @@
 
 #pragma once
 
-#include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
 #include <hilti/base/util.h>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
@@ -36,6 +36,22 @@ class Sequence : public Production {
         return {hilti::util::transform(_prods, [](const auto& p) { return p.get(); })};
     }
 
+    Expression* parseSize(Builder* builder) const final {
+        Expression* size = nullptr;
+        for ( const auto& p : _prods ) {
+            auto psize = p->parseSize(builder);
+            if ( ! psize )
+                return nullptr;
+
+            if ( ! size )
+                size = psize;
+            else
+                size = builder->sum(size, psize);
+        }
+
+        return size;
+    }
+
     std::string dump() const final {
         return hilti::util::join(hilti::util::transform(_prods, [](const auto& p) { return p->symbol(); }), " ");
     }

spicy/toolchain/include/compiler/detail/codegen/productions/skip.h

@@ -52,6 +52,15 @@ class Skip : public Production {
 
     QualifiedType* type() const final { return _void; };
 
+    Expression* parseSize(Builder* builder) const final {
+        if ( _ctor )
+            return _ctor->parseSize(builder);
+        else {
+            assert(_field);
+            return _field->parseSize(builder);
+        }
+    }
+
     std::string dump() const override {
         return hilti::util::fmt("skip: %s", _ctor ? to_string(*_ctor) : _field->print());
     }

spicy/toolchain/include/compiler/detail/codegen/productions/switch.h

@@ -47,6 +47,7 @@ class Switch : public Production {
 
     Expression* expression() const final { return _expression; }
     std::vector<std::vector<Production*>> rhss() const final;
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
 
     std::string dump() const final;
 

spicy/toolchain/include/compiler/detail/codegen/productions/type-literal.h

@@ -2,9 +2,7 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
-#include <utility>
 
 #include <hilti/ast/expressions/type.h>
 
@@ -31,6 +29,8 @@ class TypeLiteral : public Production {
 
     Expression* expression() const final { return _expr; }
     QualifiedType* type() const final { return _type; };
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
+
     int64_t tokenID() const final { return static_cast<int64_t>(Production::tokenID(_type->print())); }
 
     std::string dump() const final { return _type->print(); }