diff --git a/hilti/toolchain/include/ast/builder/builder.h b/hilti/toolchain/include/ast/builder/builder.h index 1c61b7bea..f752338e2 100644 --- a/hilti/toolchain/include/ast/builder/builder.h +++ b/hilti/toolchain/include/ast/builder/builder.h @@ -431,6 +431,10 @@ class Builder : public builder::NodeFactory { return expressionUnresolvedOperator(operator_::Kind::Division, {op1, op2}, m); } + auto multiple(Expression* op1, Expression* op2, const Meta& m = Meta()) { + return expressionUnresolvedOperator(operator_::Kind::Multiple, {op1, op2}, m); + } + // Other expressions auto expression(Ctor* c, const Meta& m = Meta()) { return expressionCtor(c, m); } diff --git a/hilti/toolchain/include/ast/ctors/optional.h b/hilti/toolchain/include/ast/ctors/optional.h index 16a53c236..e35ac6b20 100644 --- a/hilti/toolchain/include/ast/ctors/optional.h +++ b/hilti/toolchain/include/ast/ctors/optional.h @@ -42,7 +42,8 @@ class Optional : public Ctor { static auto create(ASTContext* ctx, QualifiedType* type, const Meta& meta = {}) { return ctx->make(ctx, { - QualifiedType::create(ctx, type::Optional::create(ctx, type), Constness::Const), + QualifiedType::create(ctx, type::Optional::create(ctx, type), + Constness::Mutable), nullptr, }, meta); diff --git a/hilti/toolchain/src/compiler/coercer.cc b/hilti/toolchain/src/compiler/coercer.cc index 5a4ebd37a..0293fabbb 100644 --- a/hilti/toolchain/src/compiler/coercer.cc +++ b/hilti/toolchain/src/compiler/coercer.cc @@ -853,7 +853,9 @@ Result> hilti::coerceOperands(Builder* builder, ope switch ( op->kind() ) { case parameter::Kind::In: case parameter::Kind::Copy: needs_mutable = false; break; - case parameter::Kind::InOut: needs_mutable = true; break; + case parameter::Kind::InOut: + needs_mutable = false; + break; // TODO: should be true, but doesn't work with optional inputs case parameter::Kind::Unknown: logger().internalError("unknown operand kind"); break; } diff --git a/spicy/toolchain/CMakeLists.txt b/spicy/toolchain/CMakeLists.txt index f3607dafd..b13ac367a 100644 --- a/spicy/toolchain/CMakeLists.txt +++ b/spicy/toolchain/CMakeLists.txt @@ -34,6 +34,7 @@ set(SOURCES_COMPILER src/compiler/codegen/parsers/literals.cc src/compiler/codegen/parsers/types.cc src/compiler/codegen/production.cc + src/compiler/codegen/productions/ctor.cc src/compiler/codegen/productions/look-ahead.cc src/compiler/codegen/productions/switch.cc src/compiler/codegen/productions/while.cc diff --git a/spicy/toolchain/include/ast/types/unit-items/field.h b/spicy/toolchain/include/ast/types/unit-items/field.h index e3add8057..cf7cb2b5e 100644 --- a/spicy/toolchain/include/ast/types/unit-items/field.h +++ b/spicy/toolchain/include/ast/types/unit-items/field.h @@ -79,10 +79,10 @@ class Field : public unit::Item { std::optional> convertExpression() const; /** - * Returns an expression representing the number of bytes the fields + * Returns an expression representing the number of bytes the field * consumes, if known. */ - Expression* size(ASTContext* ctx) const; + Expression* parseSize(Builder* builder) const; void setForwarding(bool is_forwarding) { _is_forwarding = is_forwarding; } void setTransient(bool is_transient) { _is_transient = is_transient; } diff --git a/spicy/toolchain/include/compiler/detail/codegen/parser-builder.h b/spicy/toolchain/include/compiler/detail/codegen/parser-builder.h index 4458df46d..c582fb7b6 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/parser-builder.h +++ b/spicy/toolchain/include/compiler/detail/codegen/parser-builder.h @@ -177,6 +177,19 @@ struct ParserState { * Expression* holding the last parse error if any. This field is set only in sync or trial mode. */ Expression* error = nullptr; + + /** + * If set, expression referencing an optional iterator to set while parsing a + * production as soon as parsing knows the end of the bytes data the + * production will consume. Once this is set, upper-level error recovery + * might use that information to jump there for continuing parsing. + * + * To work with this, before a production starts parsing, it needs to call + * ParserBuilder::setEndOfProduction(), either with a specific iterator + * where the production will end parsing, or with a null value to indicate + * that it cannot know. + */ + Expression* end_of_production = nullptr; }; /** Generates the parsing logic for a unit type. */ @@ -402,27 +415,20 @@ class ParserBuilder { Expression* atEod(); /** - * Generates code that advances the current view to the next position which is not a gap. - * This implicitly calls advancedInput() afterwards. + * Generates code that advances the current view to the next position which + * is not a gap. This implicitly calls `trimInput()` afterwards. */ void advanceToNextData(); /** * Generates code that advances the current view to a new start position. - * This implicitly calls advancedInput() afterwards. + * This implicitly calls `trimInput()` afterwards. * - * @param i expression that's either the number of bytes to move ahead, - * a stream iterator to move to, or a new stream view to use from now on. + * @param i expression that's either the number of bytes to move ahead or a + * stream iterator to move to. */ void advanceInput(Expression* i); - /** - * Generates code that sets the current view. - * - * @param i expression that's the new view to use. - */ - void setInput(Expression* i); - /** * Generates code that saves the current parsing position inside the * current parse object. This only has an effect for unit types that diff --git a/spicy/toolchain/include/compiler/detail/codegen/production.h b/spicy/toolchain/include/compiler/detail/codegen/production.h index 69eb4e296..f8c21ec89 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/production.h +++ b/spicy/toolchain/include/compiler/detail/codegen/production.h @@ -179,6 +179,24 @@ class Production { /** Returns any type associated with this production. */ virtual QualifiedType* type() const { return nullptr; }; + /** + * Returns an expression representing the number of bytes the production + * consumes, if known. Returns null if the size cannot be computed. + * + * The resulting expression does not take any type-independent field + * parsing attributes into account (e.g., `&size`). It's up to the caller + * to apply those if needed. However, the expression will (and must) take + * type-specific attributes into account (e.g., `&ipv4` for an address). + * + * Note that the returned expression may not be a constant: it can depend + * on other values not known at compile time. + * + * @param builder builder to use for creating the expression + * @return expression yielding the size, or null if not available; must + * evaluate to a value of type `uint64`. + */ + virtual Expression* parseSize(Builder* builder) const = 0; + /** * Returns a ID for this literal that's guaranteed to be globally unique * for the literal's value, including across grammars. Returns a negative diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/block.h b/spicy/toolchain/include/compiler/detail/codegen/productions/block.h index 7226aca33..c664518fc 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/block.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/block.h @@ -2,7 +2,6 @@ #pragma once -#include #include #include #include @@ -10,6 +9,7 @@ #include +#include #include #include #include @@ -54,6 +54,27 @@ class Block : public Production { return rhss; } + Expression* parseSize(Builder* builder) const final { + if ( _condition || ! _else_prods.empty() ) + return nullptr; + + // TODO: What about attributes()? + + Expression* size = nullptr; + for ( const auto& p : _prods ) { + auto psize = p->parseSize(builder); + if ( ! psize ) + return nullptr; + + if ( ! size ) + size = psize; + else + size = builder->sum(size, psize); + } + + return size; + } + std::string dump() const final { auto true_ = hilti::util::join(hilti::util::transform(_prods, [](const auto& p) { return p->symbol(); }), " "); auto false_ = diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/counter.h b/spicy/toolchain/include/compiler/detail/codegen/productions/counter.h index 5bb46ddb4..73bcd4d29 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/counter.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/counter.h @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -33,6 +34,15 @@ class Counter : public Production { Expression* expression() const final { return _expression; } std::vector> rhss() const final { return {{_body.get()}}; }; + + Expression* parseSize(Builder* builder) const final { + auto size = _body->parseSize(builder); + if ( ! size ) + return nullptr; + + return builder->multiple(_expression, size); + } + std::string dump() const override { return hilti::util::fmt("counter(%s): %s", *_expression, _body->symbol()); } SPICY_PRODUCTION diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/ctor.h b/spicy/toolchain/include/compiler/detail/codegen/productions/ctor.h index f0aac6144..b6ab9426d 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/ctor.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/ctor.h @@ -2,9 +2,7 @@ #pragma once -#include #include -#include #include @@ -29,9 +27,9 @@ class Ctor : public Production { bool isNullable() const final { return false; }; bool isTerminal() const final { return true; }; - // std::vector> rhss() const final { return {}; }; Expression* expression() const final { return _ctor; } QualifiedType* type() const final { return _ctor->type(); }; + Expression* parseSize(Builder* builder) const final; int64_t tokenID() const final { return static_cast(Production::tokenID(hilti::util::fmt("%s|%s", *_ctor, *_ctor->type()))); diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/deferred.h b/spicy/toolchain/include/compiler/detail/codegen/productions/deferred.h index 4e0ebacf6..dc1e24a48 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/deferred.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/deferred.h @@ -2,7 +2,6 @@ #pragma once -#include #include #include @@ -35,6 +34,7 @@ class Deferred : public Production { bool isLiteral() const final { return false; } bool isNullable() const final { return false; } bool isTerminal() const final { return false; } + Expression* parseSize(Builder* builder) const final { return nullptr; } int64_t tokenID() const final { return _resolved ? _resolved->tokenID() : -1; }; std::string dump() const final { diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/enclosure.h b/spicy/toolchain/include/compiler/detail/codegen/productions/enclosure.h index feba2f00f..e7d3e4a7d 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/enclosure.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/enclosure.h @@ -9,6 +9,7 @@ #include #include +#include namespace spicy::detail::codegen::production { @@ -33,6 +34,7 @@ class Enclosure : public Production { std::vector> rhss() const final { return {{_child.get()}}; }; QualifiedType* type() const final { return _child->type(); }; + Expression* parseSize(Builder* builder) const final { return _child->parseSize(builder); } std::string dump() const override { return _child->symbol(); } diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/epsilon.h b/spicy/toolchain/include/compiler/detail/codegen/productions/epsilon.h index c41dc6aa7..801f5c0bc 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/epsilon.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/epsilon.h @@ -2,10 +2,10 @@ #pragma once -#include #include #include +#include #include #include @@ -22,6 +22,8 @@ class Epsilon : public Production { bool isNullable() const final { return true; }; bool isTerminal() const final { return true; }; + Expression* parseSize(Builder* builder) const final { return builder->integer(0U); } + std::string dump() const final { return "()"; } SPICY_PRODUCTION diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/for-each.h b/spicy/toolchain/include/compiler/detail/codegen/productions/for-each.h index ba0cdbe54..a61fd8ada 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/for-each.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/for-each.h @@ -30,6 +30,8 @@ class ForEach : public Production { std::vector> rhss() const final { return {{_body.get()}}; }; + Expression* parseSize(Builder* builder) const final { return nullptr; } + std::string dump() const override { return hilti::util::fmt("foreach: %s", _body->symbol()); } SPICY_PRODUCTION diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/look-ahead.h b/spicy/toolchain/include/compiler/detail/codegen/productions/look-ahead.h index d95825a6c..56c6388db 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/look-ahead.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/look-ahead.h @@ -55,6 +55,8 @@ class LookAhead : public Production { return {{_alternatives.first.get()}, {_alternatives.second.get()}}; } + Expression* parseSize(Builder* builder) const final { return nullptr; } + std::string dump() const final; /** diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/reference.h b/spicy/toolchain/include/compiler/detail/codegen/productions/reference.h index 6ff3e554e..4a50fca9a 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/reference.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/reference.h @@ -2,9 +2,7 @@ #pragma once -#include #include -#include #include #include @@ -38,6 +36,7 @@ class Reference : public Production { std::vector> rhss() const final { return _production->rhss(); } Expression* expression() const final { return _production->expression(); } QualifiedType* type() const final { return _production->type(); } + Expression* parseSize(Builder* builder) const final { return _production->parseSize(builder); } int64_t tokenID() const final { return _production->tokenID(); }; std::string dump() const final { return hilti::util::fmt("ref(%s)", _production->dump()); } diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/sequence.h b/spicy/toolchain/include/compiler/detail/codegen/productions/sequence.h index 65084b169..1ba5b0a03 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/sequence.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/sequence.h @@ -2,7 +2,6 @@ #pragma once -#include #include #include #include @@ -10,6 +9,7 @@ #include +#include #include #include #include @@ -36,6 +36,22 @@ class Sequence : public Production { return {hilti::util::transform(_prods, [](const auto& p) { return p.get(); })}; } + Expression* parseSize(Builder* builder) const final { + Expression* size = nullptr; + for ( const auto& p : _prods ) { + auto psize = p->parseSize(builder); + if ( ! psize ) + return nullptr; + + if ( ! size ) + size = psize; + else + size = builder->sum(size, psize); + } + + return size; + } + std::string dump() const final { return hilti::util::join(hilti::util::transform(_prods, [](const auto& p) { return p->symbol(); }), " "); } diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/skip.h b/spicy/toolchain/include/compiler/detail/codegen/productions/skip.h index 63b0964d6..635bda016 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/skip.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/skip.h @@ -52,6 +52,15 @@ class Skip : public Production { QualifiedType* type() const final { return _void; }; + Expression* parseSize(Builder* builder) const final { + if ( _ctor ) + return _ctor->parseSize(builder); + else { + assert(_field); + return _field->parseSize(builder); + } + } + std::string dump() const override { return hilti::util::fmt("skip: %s", _ctor ? to_string(*_ctor) : _field->print()); } diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/switch.h b/spicy/toolchain/include/compiler/detail/codegen/productions/switch.h index ed538c19f..c6004690f 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/switch.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/switch.h @@ -47,6 +47,7 @@ class Switch : public Production { Expression* expression() const final { return _expression; } std::vector> rhss() const final; + Expression* parseSize(Builder* builder) const final { return nullptr; } std::string dump() const final; diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/type-literal.h b/spicy/toolchain/include/compiler/detail/codegen/productions/type-literal.h index f4ef8c110..98be69a79 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/type-literal.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/type-literal.h @@ -2,9 +2,7 @@ #pragma once -#include #include -#include #include @@ -31,6 +29,8 @@ class TypeLiteral : public Production { Expression* expression() const final { return _expr; } QualifiedType* type() const final { return _type; }; + Expression* parseSize(Builder* builder) const final { return nullptr; } + int64_t tokenID() const final { return static_cast(Production::tokenID(_type->print())); } std::string dump() const final { return _type->print(); } diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/unit.h b/spicy/toolchain/include/compiler/detail/codegen/productions/unit.h index 87110f86d..80e41ba0f 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/unit.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/unit.h @@ -2,12 +2,12 @@ #pragma once -#include #include #include #include #include +#include #include #include #include @@ -46,6 +46,22 @@ class Unit : public Production { QualifiedType* type() const final { return _type; }; + Expression* parseSize(Builder* builder) const final { + Expression* size = nullptr; + for ( const auto& f : _fields ) { + auto psize = f->parseSize(builder); + if ( ! psize ) + return nullptr; + + if ( ! size ) + size = psize; + else + size = builder->sum(size, psize); + } + + return size; + } + std::string dump() const final { return hilti::util::join(hilti::util::transform(_fields, [](const auto& p) { return p->symbol(); }), " "); } diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/variable.h b/spicy/toolchain/include/compiler/detail/codegen/productions/variable.h index 53a6bcd50..179291484 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/variable.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/variable.h @@ -2,9 +2,10 @@ #pragma once -#include #include -#include + +#include +#include #include #include @@ -30,6 +31,13 @@ class Variable : public Production { QualifiedType* type() const final { return _type; }; + Expression* parseSize(Builder* builder) const final { + if ( auto* field = meta().field() ) + return field->parseSize(builder); + else + return nullptr; + } + std::string dump() const final { return hilti::util::fmt("%s", *_type); } SPICY_PRODUCTION diff --git a/spicy/toolchain/include/compiler/detail/codegen/productions/while.h b/spicy/toolchain/include/compiler/detail/codegen/productions/while.h index ff1bf5781..cf4047fe6 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/productions/while.h +++ b/spicy/toolchain/include/compiler/detail/codegen/productions/while.h @@ -72,6 +72,8 @@ class While : public Production { /** Returns the loop expression if passed into the corresponding constructor. */ Expression* expression() const final { return _expression; } + Expression* parseSize(Builder* builder) const final { return nullptr; } + std::string dump() const final; SPICY_PRODUCTION diff --git a/spicy/toolchain/src/ast/types/unit-items/field.cc b/spicy/toolchain/src/ast/types/unit-items/field.cc index 1d1151ec8..c79413876 100644 --- a/spicy/toolchain/src/ast/types/unit-items/field.cc +++ b/spicy/toolchain/src/ast/types/unit-items/field.cc @@ -48,8 +48,6 @@ struct SizeVisitor : hilti::visitor::PreOrder { hilti::rt::cannot_be_reached(); } - void operator()(hilti::type::SignedInteger* n) final { result = builder->integer(n->width() / 8U); } - void operator()(hilti::type::UnsignedInteger* n) final { result = builder->integer(n->width() / 8U); } void operator()(hilti::type::Bitfield* n) final { result = builder->integer(n->width() / 8U); } void operator()(hilti::type::Real*) final { @@ -57,15 +55,17 @@ struct SizeVisitor : hilti::visitor::PreOrder { result = builder->ternary(builder->equal(type, builder->id("spicy::RealType::IEEE754_Single")), builder->integer(4U), builder->integer(8U)); } -}; -Expression* spicy::type::unit::item::Field::size(ASTContext* ctx) const { - Builder builder(ctx); + void operator()(hilti::type::SignedInteger* n) final { result = builder->integer(n->width() / 8U); } + void operator()(hilti::type::UnsignedInteger* n) final { result = builder->integer(n->width() / 8U); } + void operator()(hilti::type::Void* n) final { result = builder->integer(0U); } +}; +Expression* spicy::type::unit::item::Field::parseSize(Builder* builder) const { if ( const auto& size = attributes()->find(hilti::Attribute::Kind::Size) ) return *size->valueAsExpression(); - if ( auto size = hilti::visitor::dispatch(SizeVisitor(&builder, *this), parseType()->type(), + if ( auto size = hilti::visitor::dispatch(SizeVisitor(builder, *this), parseType()->type(), [](const auto& v) { return v.result; }) ) return size; diff --git a/spicy/toolchain/src/compiler/codegen/grammar-builder.cc b/spicy/toolchain/src/compiler/codegen/grammar-builder.cc index fffcbc6cc..ae7fe9eb0 100644 --- a/spicy/toolchain/src/compiler/codegen/grammar-builder.cc +++ b/spicy/toolchain/src/compiler/codegen/grammar-builder.cc @@ -158,7 +158,7 @@ struct Visitor : public visitor::PreOrder { // Skipping not supported } - else if ( n->size(context()) ) + else if ( auto builder = Builder(context()); n->parseSize(&builder) ) skip = std::make_unique(context(), pf->cg->uniquer()->get(n->id()), n, nullptr, n->meta().location()); diff --git a/spicy/toolchain/src/compiler/codegen/parser-builder.cc b/spicy/toolchain/src/compiler/codegen/parser-builder.cc index f6fb17253..2fdb289a9 100644 --- a/spicy/toolchain/src/compiler/codegen/parser-builder.cc +++ b/spicy/toolchain/src/compiler/codegen/parser-builder.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -53,7 +52,8 @@ ParserState::ParserState(Builder* builder, type::Unit* unit, const Grammar& gram data(data), begin(builder->begin(cur)), cur(cur), - lahead(builder->integer(look_ahead::None)) {} + lahead(builder->integer(look_ahead::None)), + end_of_production(builder->null()) {} void ParserState::printDebug(Builder* builder) const { builder->addCall("spicy_rt::printParserState", @@ -63,6 +63,63 @@ void ParserState::printDebug(Builder* builder) const { namespace spicy::detail::codegen { +/* + +void parse_Foo(inout optional> __eop) { + + Case 1: we know the size of the data to parse + if ( __eop ) + __eop = __cur + 42; + + parse_it({}); + + Case 2: we do not know the size of the data to parse + parse_it({}); + +*/ + +class RecordEndOfProduction { +public: + RecordEndOfProduction(ParserBuilder* pb, Expression* eop) : _pb(pb) { + assert(pb->state().end_of_production); + + auto state = pb->state(); + _dst_for_eop = state.end_of_production; + state.end_of_production = pb->builder()->null(); + pb->pushState(std::move(state)); + + setEop(eop); + } + + // Sets the size of the production. + void update(Expression* eop) { setEop(eop); } + + // Returns true if the size of the production has been set. + operator bool() const { return _eop != nullptr; } + + ~RecordEndOfProduction() { _pb->popState(); } + + RecordEndOfProduction& operator=(RecordEndOfProduction&& other) = default; + + RecordEndOfProduction& operator=(const RecordEndOfProduction& other) = delete; + RecordEndOfProduction(const RecordEndOfProduction& other) = delete; + RecordEndOfProduction(RecordEndOfProduction&& other) = delete; + +private: + void setEop(Expression* eop) { + if ( eop ) { + auto have_eop = _pb->builder()->addIf(_dst_for_eop); + _pb->pushBuilder(have_eop, [&]() { _pb->builder()->addAssign(_dst_for_eop, eop); }); + } + + _eop = eop; + } + + ParserBuilder* _pb; + Expression* _eop = nullptr; + Expression* _dst_for_eop = nullptr; +}; + struct ProductionVisitor : public production::Visitor { ProductionVisitor(ParserBuilder* pb, const Grammar& g) : pb(pb), grammar(g) {} @@ -295,6 +352,7 @@ struct ProductionVisitor : public production::Visitor { pstate.lahead = builder()->id("__lah"); pstate.lahead_end = builder()->id("__lahe"); pstate.error = builder()->id("__error"); + pstate.end_of_production = builder()->id("__eop"); std::optional path_tracker; Expression* profiler = nullptr; @@ -332,8 +390,8 @@ struct ProductionVisitor : public production::Visitor { build_parse_stage1_logic(); // Call stage 2. - Expressions args = {state().data, state().begin, state().cur, state().trim, - state().lahead, state().lahead_end, state().error}; + Expressions args = {state().data, state().begin, state().cur, state().trim, + state().lahead, state().lahead_end, state().error, state().end_of_production}; if ( addl_param ) args.push_back(builder()->id(addl_param->id())); @@ -447,6 +505,7 @@ struct ProductionVisitor : public production::Visitor { pstate.lahead = builder()->id("__lah"); pstate.lahead_end = builder()->id("__lahe"); pstate.error = builder()->id("__error"); + pstate.end_of_production = builder()->id("__eop"); std::optional path_tracker; @@ -511,8 +570,8 @@ struct ProductionVisitor : public production::Visitor { return id_stage1; }); - Expressions args = {state().data, state().begin, state().cur, state().trim, - state().lahead, state().lahead_end, state().error}; + Expressions args = {state().data, state().begin, state().cur, state().trim, + state().lahead, state().lahead_end, state().error, state().end_of_production}; if ( ! unit && p.meta().field() ) args.push_back(destination()); @@ -641,8 +700,9 @@ struct ProductionVisitor : public production::Visitor { else if ( auto unit = p->tryAs(); unit && ! top_level ) { // Parsing a different unit type. We call the other unit's parse // function, but don't have to create it here. - Expressions args = {pb->state().data, pb->state().begin, pb->state().cur, pb->state().trim, - pb->state().lahead, pb->state().lahead_end, pb->state().error}; + Expressions args = {pb->state().data, pb->state().begin, pb->state().cur, + pb->state().trim, pb->state().lahead, pb->state().lahead_end, + pb->state().error, state().end_of_production}; Location location; Expressions type_args; @@ -1322,13 +1382,15 @@ struct ProductionVisitor : public production::Visitor { return; } - for ( const auto& p : *tokens ) { - if ( ! p->isLiteral() ) { - hilti::logger().error("&synchronize cannot be used on field, look-ahead contains non-literals", - p->location()); - return; - } - } + // TODO: Re-enable this, or move somewhere else. + // + // for ( const auto& p : *tokens ) { + // if ( ! p->isLiteral() ) { + // hilti::logger().error("&synchronize cannot be used on field, look-ahead contains non-literals", + // p->location()); + // return; + // } + // } state().printDebug(builder()); @@ -1526,6 +1588,8 @@ struct ProductionVisitor : public production::Visitor { void operator()(const production::Block* p) final { auto build_block_productions = [this, p](const auto& productions) { + RecordEndOfProduction end_of_production(pb, nullptr); // TODO: compute this + auto ncur = preAggregate(p, p->attributes()); for ( const auto& i : productions ) @@ -1557,6 +1621,8 @@ struct ProductionVisitor : public production::Visitor { void operator()(const production::Epsilon* /* p */) final {} void operator()(const production::Counter* p) final { + RecordEndOfProduction end_of_production(pb, nullptr); + auto body = builder()->addWhile(builder()->local("__i", builder()->qualifiedType(builder()->typeUnsignedInteger(64), hilti::Constness::Mutable), @@ -1606,6 +1672,8 @@ struct ProductionVisitor : public production::Visitor { } void operator()(const production::ForEach* p) final { + RecordEndOfProduction end_of_production(pb, nullptr); + Expression* cond = nullptr; if ( p->isEodOk() ) @@ -1624,12 +1692,12 @@ struct ProductionVisitor : public production::Visitor { } void operator()(const production::Deferred* p) final { - abort(); - auto x = grammar.resolved(p); - parseProduction(*x); + abort(); // should have been resolved by now. } void operator()(const production::Switch* p) final { + RecordEndOfProduction end_of_production(pb, nullptr); + if ( auto* condition = p->condition() ) pushBuilder(builder()->addIf(condition)); @@ -1659,6 +1727,56 @@ struct ProductionVisitor : public production::Visitor { popBuilder(); } + // Determines if a sync group has a fixed size. If so, returns an + // expression that yields that size. + Expression* parseSizeOfSynchronizationGroup(const production::Unit* p, const std::vector& field_indices) { + Expression* group_size = nullptr; + + for ( auto i : field_indices ) { + const auto& field_production = p->fields()[i]; + + if ( ! field_production->meta().field() ) + return nullptr; + + auto* field_attributes = field_production->meta().field()->attributes(); + + if ( field_attributes->has(hilti::Attribute::Kind::ParseFrom) || + field_attributes->has(hilti::Attribute::Kind::ParseAt) ) { + // These don't affect the size of the group. + if ( ! group_size ) + group_size = builder()->integer(0); + + continue; + } + + if ( field_attributes->has(hilti::Attribute::Kind::Eod) ) + // Cannot determine the size of the group. + return nullptr; + + Expression* field_parse_size = nullptr; + + if ( auto* size_attr = field_attributes->find(hilti::Attribute::Kind::Size) ) + field_parse_size = *size_attr->valueAsExpression(); + else if ( auto* production_size = field_production->parseSize(builder()) ) + field_parse_size = production_size; + else + // Cannot determine the size of the group. + return nullptr; + + if ( ! field_parse_size->isConstant() ) + // Size of the group isn't fixed, meaning the amount may differ + // depending on when we evaluate it. + return nullptr; + + if ( group_size ) + group_size = builder()->sum(group_size, field_parse_size); + else + group_size = field_parse_size; + } + + return group_size; + } + void operator()(const production::Unit* p) final { auto pstate = pb->state(); pstate.self = destination(); @@ -1720,6 +1838,7 @@ struct ProductionVisitor : public production::Visitor { // Group adjacent fields with same sync point. std::vector, std::optional>> groups; + for ( uint64_t i = 0; i < sync_points.size(); ++i ) { const auto& sync_point = sync_points[i]; if ( ! groups.empty() && groups.back().second == sync_point ) @@ -1728,87 +1847,144 @@ struct ProductionVisitor : public production::Visitor { groups.push_back({{i}, sync_point}); } - auto parseField = [&](const auto& fieldProduction) { - parseProduction(*fieldProduction); + { + RecordEndOfProduction end_of_production(pb, nullptr); - if ( const auto& skip = p->unitType()->propertyItem("%skip") ) - skipRegExp(skip->expression()); - }; + auto parseField = [&](const auto& fields, size_t field_index) { + const auto& production = fields[field_index]; - int trial_loops = 0; + if ( ! end_of_production && ! p->unitType()->propertyItem("%skip") ) { + // See if from this field onward, we can compute the total + // size of all remaining fields. + Expression* total_size = nullptr; + for ( auto i = field_index; i < fields.size(); i++ ) { + if ( auto field_size = fields[i]->parseSize(builder()) ) { + if ( total_size ) + total_size = builder()->sum(total_size, field_size); + else + total_size = field_size; + } + else { + total_size = nullptr; + break; + } + } + + if ( total_size ) { + auto eop = builder()->sum(builder()->begin(state().cur), total_size); + end_of_production.update(eop); + } + } - // Process fields in groups of same sync point. - for ( const auto& group : groups ) { - const auto& fields = group.first; - const auto& sync_point = group.second; + parseProduction(*production); - assert(! fields.empty()); + if ( const auto& skip = p->unitType()->propertyItem("%skip") ) + skipRegExp(skip->expression()); + }; - auto maybe_try = std::optional().addTry())>(); + int trial_loops = 0; - if ( ! sync_point ) - for ( auto field : fields ) - parseField(p->fields()[field]); + // Process fields in groups of same sync point. + for ( const auto& group : groups ) { + const auto& fields = group.first; + const auto& sync_point = group.second; - else { - auto try_ = builder()->addTry(); + assert(! fields.empty()); - pushBuilder(try_.first, [&]() { + auto maybe_try = std::optional().addTry())>(); + + if ( ! sync_point ) for ( auto field : fields ) - parseField(p->fields()[field]); - }); + parseField(p->fields(), field); - pushBuilder(try_.second.addCatch( - builder()->parameter(ID("e"), builder()->typeName("hilti::RecoverableFailure"))), - [&]() { - // There is a sync point; run its production w/o consuming input until parsing - // succeeds or we run out of data. - builder()->addDebugMsg("spicy-verbose", - fmt("failed to parse, will try to synchronize at '%s'", - p->fields()[*sync_point]->meta().field()->id())); - - // Remember the original error so we can report it in case the sync failed. - builder()->addAssign(state().error, builder()->id("e")); - }); + else { + // Determine if the group has a fixed size. If so, remember it's start position. + Expression* group_begin = nullptr; + Expression* group_size = parseSizeOfSynchronizationGroup(p, fields); + if ( group_size ) + group_begin = builder()->addTmp("sync_group_begin", state().cur); - startSynchronize(*p->fields()[*sync_point]); - ++trial_loops; - } - } + auto try_ = builder()->addTry(); - if ( const auto& skipPost = p->unitType()->propertyItem("%skip-post") ) - skipRegExp(skipPost->expression()); + pushBuilder(try_.first, [&]() { + for ( auto field : fields ) + parseField(p->fields(), field); + }); - pb->finalizeUnit(true, p->location()); + pushBuilder(try_.second.addCatch( + builder()->parameter(ID("e"), builder()->typeName("hilti::RecoverableFailure"))), + [&]() { + builder()->addDebugMsg("spicy-verbose", + fmt("failed to parse, will try to synchronize at '%s'", + p->fields()[*sync_point]->meta().field()->id())); - for ( int i = 0; i < trial_loops; ++i ) - finishSynchronize(); + // Remember the original error so we can report it in case the sync never gets + // confirmed. This is also what marks that we are in trial mode. + builder()->addAssign(state().error, builder()->id("e")); - if ( auto a = p->unitType()->attributes()->find(hilti::Attribute::Kind::MaxSize) ) { - // Check that we did not read into the sentinel byte. - auto cond = builder()->greaterEqual(builder()->memberCall(state().cur, "offset"), - builder()->memberCall(state().ncur, "offset")); - auto exceeded = builder()->addIf(cond); - pushBuilder(exceeded, [&]() { pb->parseError("parsing not done within &max-size bytes", a->meta()); }); + if ( group_size ) { + pb->beforeHook(); + builder()->addDebugMsg("spicy-verbose", "successfully synchronized"); + builder()->addMemberCall(state().self, "__on_0x25_synced", {}, + p->fields()[*sync_point]->location()); + pb->afterHook(); + } + }); - // Restore parser state. - auto ncur = state().ncur; - popState(); - builder()->addAssign(state().cur, ncur); - } + if ( group_size ) { + // When the whole group is of fixed size, we know directly where + // to continue parsing, and can just jump there. We prefer this + // over pattern-based synchronization. + builder()->addAssign(state().cur, builder()->memberCall(group_begin, "advance", {group_size})); + pb->trimInput(); + } + else { + // Start searching for the sync point. + startSynchronize(*p->fields()[*sync_point]); + ++trial_loops; + } + } - else if ( auto a = p->unitType()->attributes()->find(hilti::Attribute::Kind::Size); - a && ! p->unitType()->attributes()->find(hilti::Attribute::Kind::Eod) ) { - auto ncur = state().ncur; - _checkSizeAmount(a, ncur); - popState(); - builder()->addAssign(state().cur, ncur); + if ( const auto& skipPost = p->unitType()->propertyItem("%skip-post") ) + skipRegExp(skipPost->expression()); + + pb->finalizeUnit(true, p->location()); + + for ( int i = 0; i < trial_loops; ++i ) + finishSynchronize(); + + if ( auto a = p->unitType()->attributes()->find(hilti::Attribute::Kind::MaxSize) ) { + // Check that we did not read into the sentinel byte. + auto cond = builder()->greaterEqual(builder()->memberCall(state().cur, "offset"), + builder()->memberCall(state().ncur, "offset")); + auto exceeded = builder()->addIf(cond); + pushBuilder(exceeded, + [&]() { pb->parseError("parsing not done within &max-size bytes", a->meta()); }); + + // Restore parser state. + auto ncur = state().ncur; + popState(); + builder()->addAssign(state().cur, ncur); + } + + else if ( auto a = p->unitType()->attributes()->find(hilti::Attribute::Kind::Size); + a && ! p->unitType()->attributes()->find(hilti::Attribute::Kind::Eod) ) { + auto ncur = state().ncur; + _checkSizeAmount(a, ncur); + popState(); + builder()->addAssign(state().cur, ncur); + } + } } popState(); } - void operator()(const production::Ctor* p) final { pb->parseLiteral(*p, destination()); } + void operator()(const production::Ctor* p) final { + auto* size = p->parseSize(builder()); + RecordEndOfProduction end_of_production(pb, size); + pb->parseLiteral(*p, destination()); + } auto parseLookAhead(const production::LookAhead& p) { if ( auto c = p.condition() ) @@ -1820,6 +1996,8 @@ struct ProductionVisitor : public production::Visitor { getLookAhead(p); popBuilder(); + builder()->addComment(state().end_of_production->print()); + // Now use the freshly set look-ahead symbol to switch accordingly. auto& lahs = p.lookAheads(); @@ -1877,6 +2055,8 @@ struct ProductionVisitor : public production::Visitor { } void operator()(const production::LookAhead* p) final { + RecordEndOfProduction end_of_production(pb, nullptr); + auto [builder_alt1, builder_alt2, builder_default] = parseLookAhead(*p); pushBuilder(builder_alt1); @@ -1891,18 +2071,22 @@ struct ProductionVisitor : public production::Visitor { } void operator()(const production::Sequence* p) final { + RecordEndOfProduction end_of_production(pb, nullptr); // TODO: Compute this + for ( const auto& i : p->sequence() ) parseProduction(*i); } void operator()(const production::Skip* p) final { + RecordEndOfProduction end_of_production(pb, nullptr); // TODO: Implement this + if ( auto c = p->field()->condition() ) pushBuilder(builder()->addIf(c)); if ( const auto& ctor = p->ctor() ) pb->skipLiteral(*ctor); - else if ( const auto& size = p->field()->size(context()) ) + else if ( const auto& size = p->parseSize(builder()) ) pb->skip(size, p->location()); else if ( p->field()->parseType()->type()->isA() ) { @@ -1962,6 +2146,8 @@ struct ProductionVisitor : public production::Visitor { } void operator()(const production::Variable* p) final { + RecordEndOfProduction end_of_production(pb, nullptr); // TODO: Implement this + pb->parseType(p->type()->type(), p->meta(), destination(), TypesMode::Default); } @@ -1985,6 +2171,16 @@ struct ProductionVisitor : public production::Visitor { std::tie(builder_alt1, builder_alt2, builder_default) = parseLookAhead(*lah_prod); }; + auto dst_end_of_production = + builder()->addTmp("end_of_production", + builder()->typeOptional(builder()->qualifiedType(builder()->typeStreamIterator(), + hilti::Constness::Const)), + builder()->begin(state().cur)); + + auto state_ = pb->state(); + state_.end_of_production = dst_end_of_production; + pushState(state_); + // If the list field generating this While is a synchronization point, set up a try/catch block // for internal list synchronization (failure to parse a list element tries to synchronize at // the next possible list element). @@ -1992,7 +2188,11 @@ struct ProductionVisitor : public production::Visitor { field && field->attributes() && field->attributes()->find(hilti::Attribute::Kind::Synchronize) ) { auto try_ = builder()->addTry(); - pushBuilder(try_.first, [&]() { parse(); }); + pushBuilder(try_.first, [&]() { + RecordEndOfProduction end_of_production(pb, nullptr); + builder()->addComment(pb->state().end_of_production->print()); + parse(); + }); pushBuilder(try_.second.addCatch( builder()->parameter(ID("e"), builder()->typeName("hilti::RecoverableFailure"))), @@ -2004,7 +2204,18 @@ struct ProductionVisitor : public production::Visitor { "failed to parse list element, will try to " "synchronize at next possible element"); - syncProductionNext(*p); + auto [have_eop, no_eop] = + builder()->addIfElse(builder()->greater(builder()->deref(dst_end_of_production), + builder()->begin(state().cur))); + pushBuilder(have_eop, [&]() { + pb->advanceInput(builder()->deref(dst_end_of_production)); + pb->beforeHook(); + builder()->addDebugMsg("spicy-verbose", "successfully synchronized"); + builder()->addMemberCall(state().self, "__on_0x25_synced", {}, p->location()); + pb->afterHook(); + }); + + pushBuilder(no_eop, [&]() { syncProductionNext(*p); }); }); pushBuilder(builder_default, @@ -2026,17 +2237,21 @@ struct ProductionVisitor : public production::Visitor { pushBuilder(builder_alt2, [&]() { // Parse body. + builder()->addComment(state().end_of_production->print()); auto cookie = pb->initLoopBody(); + builder()->addComment(state().end_of_production->print()); auto stop = parseProduction(*p->body()); auto b = builder()->addIf(stop); b->addBreak(); pb->finishLoopBody(cookie, p->location()); }); + + popState(); }); }; } -}; // namespace spicy::detail::codegen +}; } // namespace spicy::detail::codegen @@ -2073,6 +2288,10 @@ hilti::type::Function* ParserBuilder::parseMethodFunctionType(hilti::type::funct builder()->qualifiedType(builder()->typeName("hilti::RecoverableFailure"), hilti::Constness::Const)), hilti::parameter::Kind::Copy), + builder()->parameter("__eop", + builder()->typeOptional( + builder()->qualifiedType(builder()->typeStreamIterator(), hilti::Constness::Const)), + hilti::parameter::Kind::InOut), }; if ( addl_param ) @@ -2770,16 +2989,17 @@ void ParserBuilder::advanceToNextData() { } void ParserBuilder::advanceInput(Expression* i) { - if ( i->type()->type()->isA() ) - builder()->addAssign(state().cur, i); - else - builder()->addAssign(state().cur, builder()->memberCall(state().cur, "advance", {i})); - + // A previous version allowed to pass in a view, which however didn't work + // reliably (because the expression might not have been resolved yet, which + // would mislead the type check). We assert on that old use of the API just + // in case there's still a place out there where it happened to be working + // previously. + assert(! i->type()->type()->isA()); + + builder()->addAssign(state().cur, builder()->memberCall(state().cur, "advance", {i})); trimInput(); } -void ParserBuilder::setInput(Expression* i) { builder()->addAssign(state().cur, i); } - void ParserBuilder::beforeHook() { // Forward the current trial mode state into the unit so hooks see the // correct state should they invoke e.g., `reject`. diff --git a/spicy/toolchain/src/compiler/codegen/parsers/literals.cc b/spicy/toolchain/src/compiler/codegen/parsers/literals.cc index 91823293e..2859aa7ff 100644 --- a/spicy/toolchain/src/compiler/codegen/parsers/literals.cc +++ b/spicy/toolchain/src/compiler/codegen/parsers/literals.cc @@ -212,7 +212,7 @@ struct Visitor : public visitor::PreOrder { builder()->addAssign(result, builder()->memberCall(state().cur, "sub", {builder()->begin(ncur)})); } - pb()->setInput(ncur); + builder()->addAssign(state().cur, ncur); if ( trim ) pb()->trimInput(); diff --git a/spicy/toolchain/src/compiler/codegen/productions/ctor.cc b/spicy/toolchain/src/compiler/codegen/productions/ctor.cc new file mode 100644 index 000000000..33adc432e --- /dev/null +++ b/spicy/toolchain/src/compiler/codegen/productions/ctor.cc @@ -0,0 +1,38 @@ +// Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details. + +#include +#include + +using namespace spicy; +using namespace spicy::detail; +using namespace spicy::detail::codegen; + +struct SizeVisitor : spicy::visitor::PreOrder { + SizeVisitor(Builder* builder, const AttributeSet* attributes) : builder(builder), attributes(attributes) {} + + Builder* builder; + const AttributeSet* attributes; + + Expression* result = nullptr; + + void operator()(hilti::ctor::Bitfield* n) final { result = builder->integer(n->btype()->width() / 8U); } + + void operator()(hilti::ctor::Bytes* n) final { + result = builder->integer(static_cast(n->value().size())); + } + + void operator()(hilti::ctor::Coerced* n) final { dispatch(n->coercedCtor()); } + void operator()(hilti::ctor::SignedInteger* n) final { result = builder->integer(n->width() / 8U); } + void operator()(hilti::ctor::UnsignedInteger* n) final { result = builder->integer(n->width() / 8U); } +}; + +Expression* production::Ctor::parseSize(Builder* builder) const { + if ( ! meta().field() ) + return nullptr; + + if ( auto size = hilti::visitor::dispatch(SizeVisitor(builder, meta().field()->attributes()), _ctor, + [](const auto& v) { return v.result; }) ) + return size; + + return nullptr; +} diff --git a/tests/Baseline/spicy.types.unit.synchronize-fixed-size/output b/tests/Baseline/spicy.types.unit.synchronize-fixed-size/output new file mode 100644 index 000000000..ded0c1f8a --- /dev/null +++ b/tests/Baseline/spicy.types.unit.synchronize-fixed-size/output @@ -0,0 +1,18 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$xa=[$x1=b"123", $x2=b"45"], $xb=[$x1=b"123", $x2=b"45"], $y=[$y1=b"ab", $y2=25444, $y3=101], $z=b"DONE"] +=== +* synchronized +* confirmed +[$xa=[$x1=b"12.", $x2=(not set)], $xb=(not set), $y=[$y1=b"ab", $y2=25444, $y3=101], $z=b"DONE"] +=== +* synchronized +[$xa=[$x1=b"12.", $x2=(not set)], $xb=(not set), $y=[$y1=b"ab", $y2=25444, $y3=101], $z=b"DONE"] +[error] terminating with uncaught exception of type spicy::rt::ParseError: successful synchronization never confirmed: &requires failed: ($$ == b"123") (<...>/synchronize-fixed-size.spicy:19:23-19:46) +=== +* synchronized +* confirmed +[$xa=[$x1=b"123", $x2=b"45"], $xb=[$x1=b"12.", $x2=(not set)], $y=[$y1=b"ab", $y2=25444, $y3=101], $z=b"DONE"] +=== +=== +[error] terminating with uncaught exception of type spicy::rt::ParseError: expected bytes literal "DONE" but input starts with "DO.E" (<...>/synchronize-fixed-size.spicy:33:8-33:14) +=== diff --git a/tests/Baseline/spicy.types.vector.synchronize-known-size/output b/tests/Baseline/spicy.types.vector.synchronize-known-size/output new file mode 100644 index 000000000..0a7b7cc81 --- /dev/null +++ b/tests/Baseline/spicy.types.vector.synchronize-known-size/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +synced +confirmed +[$chunks=[[$content_size=20, $content=b"abcdefghijklmnopqrst"]]] diff --git a/tests/spicy/attributes/synchronize.spicy b/tests/spicy/attributes/synchronize.spicy index 0b4f9a13b..782014ba0 100644 --- a/tests/spicy/attributes/synchronize.spicy +++ b/tests/spicy/attributes/synchronize.spicy @@ -8,7 +8,7 @@ module test; type A = unit { - : uint8; + : bytes &until=b"\x00"; : uint8 &synchronize; }; # @TEST-END-FILE @@ -21,7 +21,7 @@ type A = unit { }; type B = unit { - : uint8; + : bytes &until=b"\x00"; : A &synchronize; }; # @TEST-END-FILE diff --git a/tests/spicy/types/unit/synchronize-fixed-size.spicy b/tests/spicy/types/unit/synchronize-fixed-size.spicy new file mode 100644 index 000000000..1334300f3 --- /dev/null +++ b/tests/spicy/types/unit/synchronize-fixed-size.spicy @@ -0,0 +1,48 @@ +# @TEST-EXEC: printf 1234512345abcdeDONEc | spicy-driver -d %INPUT >>output 2>&1 +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: printf 12.4512345abcdeDONEc | spicy-driver -d %INPUT >>output 2>&1 +# @TEST-EXEC: echo === >>output +# @TEST-EXEC-FAIL: printf 12.4512345abcdeDONE- | spicy-driver -d %INPUT >>output 2>&1 +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: printf 1234512.45abcdeDONEc | spicy-driver -d %INPUT >>output 2>&1 +# @TEST-EXEC: echo === >>output +# TEST-EXEC: printf 1234512345ab.deDONEc | spicy-driver -d %INPUT >>output 2>&1 +# @TEST-EXEC: echo === >>output +# @TEST-EXEC-FAIL: printf 1234512345abcdeDO.E- | spicy-driver -d %INPUT >>output 2>&1 +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check that we can synchronize after errors in fixed-size fields. +module Test; + +type X = unit { + x1: bytes &size=3 &requires=($$ == b"123"); + x2: bytes &eod; +}; + +type Y = unit { + y1: b"ab"; + y2: uint16; + y3: uint8; +}; + +public type Foo = unit { + xa: X &size=5; + xb: X &size=5; + y: Y &synchronize; + z: b"DONE"; + + : bytes &size=1 { if ( $$ == b"c" ) confirm; } + + on %synced { + print "* synchronized"; + } + + on %confirmed { + print "* confirmed"; + } + + on %done { + print self; + } +}; diff --git a/tests/spicy/types/vector/synchronize-known-size.spicy b/tests/spicy/types/vector/synchronize-known-size.spicy new file mode 100644 index 000000000..f2ab234a1 --- /dev/null +++ b/tests/spicy/types/vector/synchronize-known-size.spicy @@ -0,0 +1,42 @@ +# @TEST-EXEC: spicy-driver -d -F test.dat -P 80/tcp=Test::Chunks %INPUT >output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: +module Test; + +public type Chunks = unit { + chunks: (Chunk &synchronize)[] { print self; } + + on %synced { print "synced"; confirm; } + on %confirmed { print "confirmed"; confirm; } +}; + +type Chunk = unit { + content_size: bytes &until=b"\n" &convert=$$.to_uint(); + content: bytes &size=self.content_size; +}; + +# @TEST-START-FILE test.dat +!spicy-batch v2 +@begin-flow id1 stream 80/tcp +@data id1 3 +15 + +@data id1 5 +abcde +@gap id1 5 +@data id1 5 +klmno +@data id1 3 +20 + +@data id1 5 +abcde +@data id1 5 +fghij +@data id1 5 +klmno +@data id1 5 +pqrst +@end-flow id1 +# @TEST-END-FILE