From 8d53e990c933ccd6829a56e6a77aa6d3ec6427b9 Mon Sep 17 00:00:00 2001 From: Sebastian Zarnekow Date: Thu, 25 Apr 2024 19:25:38 +0200 Subject: [PATCH] Support detachable node models in the context of storage aware resources Signed-off-by: Sebastian Zarnekow --- .../xtext/enumrules/ResourceStorageTest.java | 168 ++++++++++++++++ .../serialization/ResourceStorageTest.java | 185 ++++++++++++++++++ .../CompactStoredNodeModelReader.java | 147 ++++++++++++++ .../CompactStoredNodeModelWriter.java | 107 ++++++++++ ...rageAwareDetachableParseResultWrapper.java | 87 ++++++++ .../xtext/nodemodel/impl/RootNode.java | 1 + .../xtext/resource/ParseResultWrapper.java | 12 ++ .../persistence/ResourceStorageLoadable.java | 37 ++-- .../persistence/ResourceStorageWritable.java | 8 +- .../persistence/StorageAwareResource.java | 10 + 10 files changed, 744 insertions(+), 18 deletions(-) create mode 100644 org.eclipse.xtext.tests/src/org/eclipse/xtext/enumrules/ResourceStorageTest.java create mode 100644 org.eclipse.xtext.tests/src/org/eclipse/xtext/nodemodel/serialization/ResourceStorageTest.java create mode 100644 org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/CompactStoredNodeModelReader.java create mode 100644 org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/CompactStoredNodeModelWriter.java create mode 100644 org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/StorageAwareDetachableParseResultWrapper.java diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/enumrules/ResourceStorageTest.java b/org.eclipse.xtext.tests/src/org/eclipse/xtext/enumrules/ResourceStorageTest.java new file mode 100644 index 00000000000..90777440be2 --- /dev/null +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/enumrules/ResourceStorageTest.java @@ -0,0 +1,168 @@ +package org.eclipse.xtext.enumrules; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.eclipse.emf.common.util.URI; +import org.eclipse.emf.ecore.resource.impl.ExtensibleURIConverterImpl; +import org.eclipse.emf.ecore.util.EcoreUtil; +import org.eclipse.emf.ecore.util.EcoreUtil.EqualityHelper; +import org.eclipse.xtext.enumrules.enumAndReferenceTestLanguage.EntityWithEnumAndReference; +import org.eclipse.xtext.enumrules.enumAndReferenceTestLanguage.EnumAndReferenceTestLanguagePackage; +import org.eclipse.xtext.generator.AbstractFileSystemAccess2; +import org.eclipse.xtext.generator.JavaIoFileSystemAccess; +import org.eclipse.xtext.nodemodel.ICompositeNode; +import org.eclipse.xtext.nodemodel.INode; +import org.eclipse.xtext.nodemodel.detachable.DetachableParseResult; +import org.eclipse.xtext.nodemodel.detachable.StorageAwareDetachableParseResultWrapper; +import org.eclipse.xtext.nodemodel.impl.RootNode; +import org.eclipse.xtext.parser.ParserTestHelper; +import org.eclipse.xtext.resource.ParseResultWrapper; +import org.eclipse.xtext.resource.XtextResource; +import org.eclipse.xtext.resource.persistence.IResourceStorageFacade; +import org.eclipse.xtext.resource.persistence.ResourceStorageFacade; +import org.eclipse.xtext.resource.persistence.ResourceStorageLoadable; +import org.eclipse.xtext.resource.persistence.StorageAwareResource; +import org.eclipse.xtext.util.StringInputStream; +import org.junit.Test; + +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; + +public class ResourceStorageTest extends AbstractEnumRulesTest { + + public static class CustomSetup extends EnumAndReferenceTestLanguageStandaloneSetup { + @SuppressWarnings("unused") + @Override + public Injector createInjector() { + return Guice.createInjector(new EnumAndReferenceTestLanguageRuntimeModule() { + public Class bindIResourceStorageFacade() { + return ResourceStorageFacade.class; + } + @Override + public Class bindXtextResource() { + return StorageAwareResource.class; + } + public Class bindAbstractFileSystemAccess2() { + return JavaIoFileSystemAccess.class; + } + @Override + public Class bindParseResultWrapper() { + return StorageAwareDetachableParseResultWrapper.class; + } + }); + } + } + + + public static class InMemoryURIConverter extends ExtensibleURIConverterImpl { + private final Map models = new HashMap(); + + public void addModel(String uri, String content) { + models.put(URI.createURI(uri), new StringInputStream(content)); + } + + @Override + public boolean exists(URI uri, Map options) { + boolean result = models.containsKey(uri); + if (!result) { + for (URI knownUri : models.keySet()) { + if (uri.toString().endsWith(knownUri.toString())) { + return true; + } + } + } + return result; + } + + @Override + public InputStream createInputStream(URI uri, Map options) throws IOException { + return models.get(uri); + } + } + + private ParserTestHelper helper; + + @Inject + private ResourceStorageFacade resourceStorageFacade; + + @Override + public void setUp() throws Exception { + super.setUp(); + with(CustomSetup.class); + helper = new ParserTestHelper(getResourceFactory(), getParser(), get(Keys.RESOURCE_SET_KEY),getCurrentFileExtension()); + injectMembers(this); + } + + @Test + public void testWriteAndLoad() throws Exception { + String modelAsString = "kindOfKeyword Hoo reference Hoo"; + + StorageAwareResource originalResource = parse(modelAsString); + DetachableParseResult originalParseResult = (DetachableParseResult) originalResource.getParseResult(); + RootNode originalRootNode = (RootNode) originalParseResult.getRootNode(); + + EntityWithEnumAndReference model = (EntityWithEnumAndReference) originalResource.getContents().get(0); + assertEquals("Hoo", model.getName()); + EntityWithEnumAndReference proxy = (EntityWithEnumAndReference) model.eGet(EnumAndReferenceTestLanguagePackage.Literals.ENTITY_WITH_ENUM_AND_REFERENCE__REF, false); + assertTrue(proxy.eIsProxy()); + + EcoreUtil.resolveAll(originalResource); + assertTrue(originalResource.getErrors().isEmpty()); + + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + resourceStorageFacade.setStoreNodeModel(true); + resourceStorageFacade.createResourceStorageWritable(bout).writeResource(originalResource); + ResourceStorageLoadable in = resourceStorageFacade.createResourceStorageLoadable(new ByteArrayInputStream(bout.toByteArray())); + StorageAwareResource reloadedResource = (StorageAwareResource) originalResource.getResourceSet().createResource(URI.createURI("synthetic:/Test." + originalResource.getURI().fileExtension())); + InMemoryURIConverter converter = new InMemoryURIConverter(); + converter.addModel(reloadedResource.getURI().toString(), modelAsString); + reloadedResource.getResourceSet().setURIConverter(converter); + originalResource.getResourceSet().getResources().add(reloadedResource); + reloadedResource.loadFromStorage(in); + EntityWithEnumAndReference reloadedModel = (EntityWithEnumAndReference) reloadedResource.getContents().get(0); + EqualityHelper equalityHelper = new EqualityHelper(); + assertTrue(equalityHelper.equals(model, reloadedModel)); + + DetachableParseResult reloadedParseResult = (DetachableParseResult) reloadedResource.getParseResult(); + RootNode reloadedRootNode = (RootNode) reloadedParseResult.getRootNode(); + + assertEqualNodes(originalRootNode, reloadedRootNode, equalityHelper); + + assertSame(reloadedModel, reloadedModel.getRef()); + } + + protected void assertEqualNodes(RootNode expected, RootNode actual, EqualityHelper equalityHelper) throws IOException { + Iterator expectedIter = expected.getAsTreeIterable().iterator(); + Iterator actualIter = actual.getAsTreeIterable().iterator(); + while(expectedIter.hasNext()) { + assertTrue(actualIter.hasNext()); + doAssertEqualNodes(expectedIter.next(), actualIter.next(), equalityHelper); + } + assertFalse(actualIter.hasNext()); + } + + protected void doAssertEqualNodes(INode expected, INode actual, EqualityHelper equalityHelper) { + assertEquals("class", expected.getClass(), actual.getClass()); + assertEquals("text", expected.getText(), actual.getText()); + assertEquals("total offset", expected.getTotalOffset(), actual.getTotalOffset()); + assertEquals("total length", expected.getTotalLength(), actual.getTotalLength()); + assertEquals("grammar element", expected.getGrammarElement(), actual.getGrammarElement()); + assertEquals("direct semantic element", expected.hasDirectSemanticElement(), actual.hasDirectSemanticElement()); + assertTrue(equalityHelper.equals(expected.getSemanticElement(), actual.getSemanticElement())); + assertEquals("syntax error message", expected.getSyntaxErrorMessage(), actual.getSyntaxErrorMessage()); + if (expected instanceof ICompositeNode) { + assertEquals("lookAhead", ((ICompositeNode) expected).getLookAhead(), ((ICompositeNode) actual).getLookAhead()); + } + } + + protected StorageAwareResource parse(String modelAsString) throws IOException { + return (StorageAwareResource) helper.getResourceFromString(modelAsString); + } +} diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/nodemodel/serialization/ResourceStorageTest.java b/org.eclipse.xtext.tests/src/org/eclipse/xtext/nodemodel/serialization/ResourceStorageTest.java new file mode 100644 index 00000000000..2e0fa1bc96a --- /dev/null +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/nodemodel/serialization/ResourceStorageTest.java @@ -0,0 +1,185 @@ +package org.eclipse.xtext.nodemodel.serialization; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.eclipse.emf.common.util.URI; +import org.eclipse.emf.ecore.resource.impl.ExtensibleURIConverterImpl; +import org.eclipse.emf.ecore.util.EcoreUtil.EqualityHelper; +import org.eclipse.xtext.generator.AbstractFileSystemAccess2; +import org.eclipse.xtext.generator.JavaIoFileSystemAccess; +import org.eclipse.xtext.nodemodel.ICompositeNode; +import org.eclipse.xtext.nodemodel.INode; +import org.eclipse.xtext.nodemodel.detachable.DetachableNodeModelBuilder; +import org.eclipse.xtext.nodemodel.detachable.DetachableParseResult; +import org.eclipse.xtext.nodemodel.detachable.StorageAwareDetachableParseResultWrapper; +import org.eclipse.xtext.nodemodel.impl.NodeModelBuilder; +import org.eclipse.xtext.nodemodel.impl.RootNode; +import org.eclipse.xtext.parser.ParserTestHelper; +import org.eclipse.xtext.parser.impl.PartialParsingHelper; +import org.eclipse.xtext.resource.ParseResultWrapper; +import org.eclipse.xtext.resource.XtextResource; +import org.eclipse.xtext.resource.persistence.IResourceStorageFacade; +import org.eclipse.xtext.resource.persistence.ResourceStorageFacade; +import org.eclipse.xtext.resource.persistence.ResourceStorageLoadable; +import org.eclipse.xtext.resource.persistence.StorageAwareResource; +import org.eclipse.xtext.testlanguages.SimpleExpressionsTestLanguageRuntimeModule; +import org.eclipse.xtext.testlanguages.SimpleExpressionsTestLanguageStandaloneSetup; +import org.eclipse.xtext.testlanguages.simpleExpressions.Expression; +import org.eclipse.xtext.tests.AbstractXtextTests; +import org.eclipse.xtext.util.StringInputStream; +import org.junit.Test; + +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; + +public class ResourceStorageTest extends AbstractXtextTests { + + public static class CustomSetup extends SimpleExpressionsTestLanguageStandaloneSetup { + @SuppressWarnings("unused") + @Override + public Injector createInjector() { + return Guice.createInjector(new SimpleExpressionsTestLanguageRuntimeModule() { + public Class bindIResourceStorageFacade() { + return ResourceStorageFacade.class; + } + @Override + public Class bindXtextResource() { + return StorageAwareResource.class; + } + public Class bindAbstractFileSystemAccess2() { + return JavaIoFileSystemAccess.class; + } + public Class bindParseResultWrapper() { + return StorageAwareDetachableParseResultWrapper.class; + } + public Class bindNodeModelBuilder() { + return DetachableNodeModelBuilder.class; + } + public Class bindPartialParsingHelper() { + return null; + } + }); + } + } + + + public static class InMemoryURIConverter extends ExtensibleURIConverterImpl { + private final Map models = new HashMap(); + + public void addModel(String uri, String content) { + models.put(URI.createURI(uri), new StringInputStream(content)); + } + + @Override + public boolean exists(URI uri, Map options) { + boolean result = models.containsKey(uri); + if (!result) { + for (URI knownUri : models.keySet()) { + if (uri.toString().endsWith(knownUri.toString())) { + return true; + } + } + } + return result; + } + + @Override + public InputStream createInputStream(URI uri, Map options) throws IOException { + return models.get(uri); + } + } + + private ParserTestHelper helper; + + @Inject + private ResourceStorageFacade resourceStorageFacade; + + @Override + public void setUp() throws Exception { + super.setUp(); + with(CustomSetup.class); + helper = new ParserTestHelper(getResourceFactory(), getParser(), get(Keys.RESOURCE_SET_KEY),getCurrentFileExtension()); + injectMembers(this); + } + + @Test + public void testWriteAndLoad_01() throws Exception { + String modelAsString = "(d - e) / e * d // fasdf s"; + doTestWriteAndLoad(modelAsString); + } + + @Test + public void testWriteAndLoad_02() throws Exception { + String modelAsString = "a + b + c + d + e +"; + doTestWriteAndLoad(modelAsString); + } + + void doTestWriteAndLoad(String modelAsString) throws Exception { + StorageAwareResource originalResource = parse(modelAsString); + DetachableParseResult originalParseResult = (DetachableParseResult) originalResource.getParseResult(); + RootNode originalRootNode = (RootNode) originalParseResult.getRootNode(); + + Expression model = (Expression) originalResource.getContents().get(0); + + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + resourceStorageFacade.setStoreNodeModel(true); + resourceStorageFacade.createResourceStorageWritable(bout).writeResource(originalResource); + ResourceStorageLoadable in = resourceStorageFacade.createResourceStorageLoadable(new ByteArrayInputStream(bout.toByteArray())); + StorageAwareResource reloadedResource = (StorageAwareResource) originalResource.getResourceSet().createResource(URI.createURI("synthetic:/Test." + originalResource.getURI().fileExtension())); + InMemoryURIConverter converter = new InMemoryURIConverter(); + converter.addModel(reloadedResource.getURI().toString(), modelAsString); + reloadedResource.getResourceSet().setURIConverter(converter); + originalResource.getResourceSet().getResources().add(reloadedResource); + reloadedResource.loadFromStorage(in); + Expression reloadedModel = (Expression) reloadedResource.getContents().get(0); + EqualityHelper equalityHelper = new EqualityHelper(); + assertTrue(equalityHelper.equals(model, reloadedModel)); + + DetachableParseResult reloadedParseResult = (DetachableParseResult) reloadedResource.getParseResult(); + RootNode reloadedRootNode = (RootNode) reloadedParseResult.getRootNode(); + + assertEqualNodes(originalRootNode, reloadedRootNode, equalityHelper); + + assertEquals(originalParseResult.hasSyntaxErrors(), reloadedParseResult.hasSyntaxErrors()); + + // By default, de-serialized resources don't populate the error list + reloadedResource.relink(); + + assertEquals(originalResource.getErrors().size(), reloadedResource.getErrors().size()); + } + + protected void assertEqualNodes(RootNode expected, RootNode actual, EqualityHelper equalityHelper) throws IOException { + Iterator expectedIter = expected.getAsTreeIterable().iterator(); + Iterator actualIter = actual.getAsTreeIterable().iterator(); + while(expectedIter.hasNext()) { + assertTrue(actualIter.hasNext()); + doAssertEqualNodes(expectedIter.next(), actualIter.next(), equalityHelper); + } + assertFalse(actualIter.hasNext()); + } + + protected void doAssertEqualNodes(INode expected, INode actual, EqualityHelper equalityHelper) { + assertEquals("class", expected.getClass(), actual.getClass()); + assertEquals("text", expected.getText(), actual.getText()); + assertEquals("total offset", expected.getTotalOffset(), actual.getTotalOffset()); + assertEquals("total length", expected.getTotalLength(), actual.getTotalLength()); + assertEquals("grammar element", expected.getGrammarElement(), actual.getGrammarElement()); + assertEquals("direct semantic element", expected.hasDirectSemanticElement(), actual.hasDirectSemanticElement()); + assertTrue(equalityHelper.equals(expected.getSemanticElement(), actual.getSemanticElement())); + assertEquals("syntax error message", expected.getSyntaxErrorMessage(), actual.getSyntaxErrorMessage()); + if (expected instanceof ICompositeNode) { + assertEquals("lookAhead", ((ICompositeNode) expected).getLookAhead(), ((ICompositeNode) actual).getLookAhead()); + } + } + + protected StorageAwareResource parse(String modelAsString) throws IOException { + return (StorageAwareResource) helper.getResourceFromString(modelAsString); + } +} diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/CompactStoredNodeModelReader.java b/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/CompactStoredNodeModelReader.java new file mode 100644 index 00000000000..c388589fa2a --- /dev/null +++ b/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/CompactStoredNodeModelReader.java @@ -0,0 +1,147 @@ +/******************************************************************************* + * Copyright (c) 2024 Sebastian Zarnekow and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.nodemodel.detachable; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; + +import org.eclipse.emf.ecore.EObject; +import org.eclipse.emf.ecore.resource.impl.BinaryResourceImpl; +import org.eclipse.emf.ecore.resource.impl.BinaryResourceImpl.EObjectInputStream; +import org.eclipse.xtext.nodemodel.ICompositeNode; +import org.eclipse.xtext.nodemodel.SyntaxErrorMessage; +import org.eclipse.xtext.nodemodel.impl.AbstractNode; +import org.eclipse.xtext.nodemodel.impl.CompositeNodeWithSemanticElement; +import org.eclipse.xtext.nodemodel.impl.NodeModelInput; +import org.eclipse.xtext.util.Strings; + +/** + * @since 2.35 + */ +public class CompactStoredNodeModelReader extends EObjectInputStream implements NodeModelInput { + + private final Tabulated grammarElementArrays; + private final Tabulated semanticObjects; + private final Tabulated syntaxErrorMessages; + private final DetachableNodeModelBuilder nodeModelBuilder; + private final String content; + private final GrammarElementLookup grammarElements; + private int offset; + + public CompactStoredNodeModelReader(InputStream inputStream, String content, Tabulated semanticObjects, DetachableNodeModelBuilder nodeModelBuilder, GrammarElementLookup grammarElements) throws IOException { + super(inputStream, Collections.singletonMap(BinaryResourceImpl.OPTION_INTERNAL_BUFFER_CAPACITY, 8192)); + + this.content = content; + this.semanticObjects = semanticObjects; + this.nodeModelBuilder = nodeModelBuilder; + this.grammarElements = grammarElements; + this.syntaxErrorMessages = new Tabulated<>(); + this.grammarElementArrays = new Tabulated<>(); + } + + @Override + public AbstractNode readNode(int type) throws IOException { + AbstractNode result = NodeModelInput.super.readNode(type); + if (result instanceof CompositeNodeWithSemanticElement && result.hasDirectSemanticElement()) { + nodeModelBuilder.associateWithSemanticElement((ICompositeNode) result, result.getSemanticElement()); + } + return result; + } + + @Override + protected void readSignature() throws IOException { + // no signature + } + + @Override + public int readLength() throws IOException { + int result = readCompressedInt(); + offset += result; + return result; + } + + @Override + public int currentOffset() { + return offset; + } + + @Override + public Object readGrammarElement() throws IOException { + int id = readCompressedInt(); + if (id == -1) { + return null; + } + if (id >= grammarElements.size()) { + id -= grammarElements.size(); + EObject[] array; + if (id == grammarElementArrays.size()) { + int len = readCompressedInt(); + array = new EObject[len]; + for(int i = 0; i < len; i++) { + array[i] = grammarElements.getGrammarElement(readCompressedInt()); + } + grammarElementArrays.getId(new ArrayReference(array)); + } else { + array = grammarElementArrays.getObject(id).getArray(); + } + return array; + } + return grammarElements.getGrammarElement(id); + } + + @Override + public EObject readSemanticObject() throws IOException { + int id = readCompressedInt(); + if (id == -1) { + return null; + } + return semanticObjects.getObject(id); + } + + @Override + public SyntaxErrorMessage readSyntaxErrorMessage() throws IOException { + int id = readCompressedInt(); + if (id == -1) { + return null; + } + if (id == syntaxErrorMessages.size()) { + String issueCode = readSegmentedString(); + String message = readSegmentedString(); + int issueDataLen = readCompressedInt(); + String[] issueData; + if (issueDataLen == -1) { + issueData = null; + } else if (issueDataLen == 0) { + issueData = Strings.EMPTY_ARRAY; + } else { + issueData = new String[issueDataLen]; + for (int i = 0; i < issueDataLen; i++) { + issueData[i] = readSegmentedString(); + } + } + SyntaxErrorMessage result = new SyntaxErrorMessage(message, issueCode, issueData); + syntaxErrorMessages.getId(result); + return result; + } + return syntaxErrorMessages.getObject(id); + } + + protected boolean hasErrors() { + return syntaxErrorMessages.size() > 0; + } + + @Override + public String readContent() throws IOException { + return content; + } + +} + + diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/CompactStoredNodeModelWriter.java b/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/CompactStoredNodeModelWriter.java new file mode 100644 index 00000000000..c9cd7657e0e --- /dev/null +++ b/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/CompactStoredNodeModelWriter.java @@ -0,0 +1,107 @@ +/******************************************************************************* + * Copyright (c) 2024 Sebastian Zarnekow and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.nodemodel.detachable; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; + +import org.eclipse.emf.ecore.EObject; +import org.eclipse.emf.ecore.resource.impl.BinaryResourceImpl; +import org.eclipse.emf.ecore.resource.impl.BinaryResourceImpl.EObjectOutputStream; +import org.eclipse.xtext.nodemodel.SyntaxErrorMessage; +import org.eclipse.xtext.nodemodel.impl.NodeModelOutput; + +/** + * @since 2.35 + */ +public class CompactStoredNodeModelWriter extends EObjectOutputStream implements NodeModelOutput { + + private final Tabulated grammarElementArrays; + private final Tabulated semanticObjects; + private final Tabulated syntaxErrorMessages; + private final GrammarElementLookup grammarElements; + + public CompactStoredNodeModelWriter(OutputStream out, Tabulated semanticObjects, GrammarElementLookup grammarElements) throws IOException { + super(out, Collections.singletonMap(BinaryResourceImpl.OPTION_INTERNAL_BUFFER_CAPACITY, 8192), + Version.VERSION_1_1, STYLE_DATA_CONVERTER); + + this.grammarElements = grammarElements; + this.syntaxErrorMessages = new Tabulated<>(); + this.grammarElementArrays = new Tabulated<>(); + this.semanticObjects = semanticObjects; + } + + @Override + protected void writeSignature() throws IOException { + // don'T write a signature + } + + @Override + public void writeGrammarElement(Object grammarElementOrArray) throws IOException { + if (grammarElementOrArray != null) { + if (grammarElementOrArray instanceof EObject) { + writeCompressedInt(grammarElements.getId((EObject) grammarElementOrArray)); + } else { + int prevSize = grammarElementArrays.size(); + EObject[] array = (EObject[]) grammarElementOrArray; + int arrayId = grammarElementArrays.getId(new ArrayReference(array)); + writeCompressedInt(grammarElements.size() + arrayId); + if (prevSize == arrayId) { + writeCompressedInt(array.length); + for(EObject element: array) { + writeCompressedInt(grammarElements.getId(element)); + } + } + } + } else { + writeCompressedInt(-1); + } + } + + @Override + public void writeSemanticObject(EObject semanticElement) throws IOException { + if (semanticElement != null) { + writeCompressedInt(semanticObjects.getId(semanticElement)); + } else { + writeCompressedInt(-1); + } + } + + @Override + public void writeSyntaxErrorMessage(SyntaxErrorMessage syntaxErrorMessage) throws IOException { + if (syntaxErrorMessage != null) { + int prevSize = syntaxErrorMessages.size(); + int messageId = syntaxErrorMessages.getId(syntaxErrorMessage); + writeCompressedInt(messageId); + if (prevSize == messageId) { + writeSegmentedString(syntaxErrorMessage.getIssueCode()); + writeSegmentedString(syntaxErrorMessage.getMessage()); + String[] issueData = syntaxErrorMessage.getIssueData(); + if (issueData == null) { + writeCompressedInt(-1); + } else { + writeCompressedInt(issueData.length); + for(String s: issueData) { + writeSegmentedString(s); + } + } + } + } else { + writeCompressedInt(-1); + } + } + + @Override + public void writeContent(String value) throws IOException { + // skip since it is loaded from disk + } +} + + diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/StorageAwareDetachableParseResultWrapper.java b/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/StorageAwareDetachableParseResultWrapper.java new file mode 100644 index 00000000000..0c2624f1e09 --- /dev/null +++ b/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/detachable/StorageAwareDetachableParseResultWrapper.java @@ -0,0 +1,87 @@ +/******************************************************************************* + * Copyright (c) 2024 Sebastian Zarnekow and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.nodemodel.detachable; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; + +import org.apache.log4j.Logger; +import org.eclipse.emf.common.util.TreeIterator; +import org.eclipse.emf.common.util.URI; +import org.eclipse.emf.ecore.EObject; +import org.eclipse.emf.ecore.resource.URIConverter; +import org.eclipse.xtext.EcoreUtil2; +import org.eclipse.xtext.nodemodel.impl.NodeModelBuilder; +import org.eclipse.xtext.nodemodel.impl.RootNode; +import org.eclipse.xtext.parser.IParseResult; +import org.eclipse.xtext.resource.XtextResource; + +import com.google.common.io.CharStreams; +import com.google.inject.Inject; +import com.google.inject.Provider; + +/** + * @since 2.35 + */ +public class StorageAwareDetachableParseResultWrapper extends DetachableParseResultWrapper { + + private static final Logger LOG = Logger.getLogger(DetachableParseResultWrapper.class); + + @Inject + protected Provider nodeModelBuilder; + + @Inject + protected GrammarElementLookup grammarElementLookup; + + @Override + public boolean customWriteNodeModel(XtextResource resource, OutputStream outputStream) throws IOException { + IParseResult parseResult = resource.getParseResult(); + EObject rootElement = parseResult.getRootASTElement(); + Tabulated semanticObjects = semanticObjectIndex(rootElement); + CompactStoredNodeModelWriter writer = new CompactStoredNodeModelWriter(outputStream, semanticObjects, grammarElementLookup); + RootNode rootNode = (RootNode) parseResult.getRootNode(); + writer.writeNode(rootNode); + writer.flush(); + return true; + } + + protected Tabulated semanticObjectIndex(EObject rootElement) { + Tabulated semanticObjects = new Tabulated<>(); + TreeIterator iterator = EcoreUtil2.eAll(rootElement); + while(iterator.hasNext()) { + semanticObjects.getId(iterator.next()); + } + return semanticObjects; + } + + @Override + public boolean customReadNodeModel(XtextResource resource, InputStream inputStream) throws IOException { + // if this is a synthetic resource (i.e. tests or so, don't load the node model) + URIConverter uriConverter = resource.getResourceSet().getURIConverter(); + URI uri = resource.getURI(); + if (!uriConverter.exists(uri, resource.getResourceSet().getLoadOptions())) { + LOG.info("Skipping loading node model for synthetic resource " + uri); + return false; + } + try (InputStreamReader reader = new InputStreamReader(uriConverter.createInputStream(uri), resource.getEncoding())) { + String completeContent = CharStreams.toString(reader); + + DetachableNodeModelBuilder builder = (DetachableNodeModelBuilder) nodeModelBuilder.get(); + EObject semanticRoot = resource.getContents().get(0); + Tabulated semanticObjects = semanticObjectIndex(semanticRoot); + CompactStoredNodeModelReader nodeModelReader = new CompactStoredNodeModelReader(inputStream, completeContent, semanticObjects, builder, grammarElementLookup); + RootNode rootNode = (RootNode) nodeModelReader.readNode(); + IParseResult parseResult = builder.createParseResult(semanticRoot, rootNode, nodeModelReader.hasErrors()); + resource.setParseResult(parseResult); + } + return true; + } +} diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/impl/RootNode.java b/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/impl/RootNode.java index 9ffe0c44352..0a9526371d8 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/impl/RootNode.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/nodemodel/impl/RootNode.java @@ -227,5 +227,6 @@ protected void doReadContent(NodeModelInput in) throws IOException { for(int i = 0; i < lineBreakOffsets.length; i++) { lineBreakOffsets[i] = in.readCompressedInt(); } + this.lineBreakOffsets = lineBreakOffsets; } } diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/resource/ParseResultWrapper.java b/org.eclipse.xtext/src/org/eclipse/xtext/resource/ParseResultWrapper.java index 340d2476695..e13b1d6fe9d 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/resource/ParseResultWrapper.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/resource/ParseResultWrapper.java @@ -8,6 +8,10 @@ *******************************************************************************/ package org.eclipse.xtext.resource; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + import org.eclipse.emf.ecore.EObject; import org.eclipse.emf.ecore.EReference; import org.eclipse.xtext.diagnostics.IDiagnosticProducer; @@ -43,4 +47,12 @@ public IParseResult acquire(IParseResult parseResult) { return parseResult; } + public boolean customWriteNodeModel(XtextResource resource, OutputStream outputStream) throws IOException { + return false; + } + + public boolean customReadNodeModel(XtextResource resource, InputStream inputStream) throws IOException { + return false; + } + } \ No newline at end of file diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/ResourceStorageLoadable.java b/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/ResourceStorageLoadable.java index 02cbd4b5083..f13a6e9f5cd 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/ResourceStorageLoadable.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/ResourceStorageLoadable.java @@ -20,7 +20,9 @@ import java.util.zip.ZipInputStream; import org.apache.log4j.Logger; +import org.eclipse.emf.common.util.URI; import org.eclipse.emf.ecore.InternalEObject; +import org.eclipse.emf.ecore.resource.URIConverter; import org.eclipse.emf.ecore.resource.impl.BinaryResourceImpl; import org.eclipse.xtext.nodemodel.impl.SerializableNodeModel; import org.eclipse.xtext.nodemodel.serialization.DeserializationConversionContext; @@ -89,6 +91,7 @@ protected void handleLoadEObject(InternalEObject loaded, BinaryResourceImpl.EObj protected void readResourceDescription(StorageAwareResource resource, InputStream inputStream) throws IOException { try { + @SuppressWarnings("resource") SerializableResourceDescription description = (SerializableResourceDescription) new ObjectInputStream( inputStream).readObject(); description.updateResourceURI(resource.getURI()); @@ -99,21 +102,25 @@ protected void readResourceDescription(StorageAwareResource resource, InputStrea } protected void readNodeModel(StorageAwareResource resource, InputStream inputStream) throws IOException { - SerializableNodeModel serializableNodeModel = new SerializableNodeModel(resource); - // if this is a synthetic resource (i.e. tests or so, don't load the node model) - if (!resource.getResourceSet().getURIConverter().exists(resource.getURI(), - resource.getResourceSet().getLoadOptions())) { - LOG.info("Skipping loading node model for synthetic resource " + resource.getURI()); - return; - } - try (InputStreamReader reader = new InputStreamReader(resource.getResourceSet().getURIConverter().createInputStream(resource.getURI()), - resource.getEncoding())) { - String completeContent = CharStreams.toString(reader); - DeserializationConversionContext deserializationContext = new DeserializationConversionContext(resource, - completeContent); - serializableNodeModel.readObjectData(new DataInputStream(inputStream), deserializationContext); - resource.setParseResult(new ParseResult(head(resource.getContents()), serializableNodeModel.root, - deserializationContext.hasErrors())); + if (!resource.customReadNodeModel(inputStream)) { + SerializableNodeModel serializableNodeModel = new SerializableNodeModel(resource); + // if this is a synthetic resource (i.e. tests or so, don't load the node model) + URIConverter uriConverter = resource.getResourceSet().getURIConverter(); + URI resourceUri = resource.getURI(); + if (!uriConverter.exists(resourceUri, resource.getResourceSet().getLoadOptions())) { + LOG.info("Skipping loading node model for synthetic resource " + resourceUri); + return; + } + try (InputStreamReader reader = new InputStreamReader( + uriConverter.createInputStream(resourceUri), + resource.getEncoding())) { + String completeContent = CharStreams.toString(reader); + DeserializationConversionContext deserializationContext = new DeserializationConversionContext(resource, + completeContent); + serializableNodeModel.readObjectData(new DataInputStream(inputStream), deserializationContext); + resource.setParseResult(new ParseResult(head(resource.getContents()), serializableNodeModel.root, + deserializationContext.hasErrors())); + } } } diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/ResourceStorageWritable.java b/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/ResourceStorageWritable.java index fd6e85bdce3..54463de817b 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/ResourceStorageWritable.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/ResourceStorageWritable.java @@ -139,9 +139,11 @@ protected void convertExternalURIsToPortableURIs(SerializableResourceDescription } protected void writeNodeModel(StorageAwareResource resource, OutputStream outputStream) throws IOException { - DataOutputStream out = new DataOutputStream(outputStream); - new SerializableNodeModel(resource).writeObjectData(out, new SerializationConversionContext(resource)); - out.flush(); + if (!resource.customWriteNodeModel(outputStream)) { + DataOutputStream out = new DataOutputStream(outputStream); + new SerializableNodeModel(resource).writeObjectData(out, new SerializationConversionContext(resource)); + out.flush(); + } } } diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/StorageAwareResource.java b/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/StorageAwareResource.java index 9982647d8c9..f4d38621ad2 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/StorageAwareResource.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/resource/persistence/StorageAwareResource.java @@ -9,6 +9,8 @@ package org.eclipse.xtext.resource.persistence; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -162,4 +164,12 @@ public IResourceDescription getResourceDescription() { public void setResourceDescription(IResourceDescription resourceDescription) { this.resourceDescription = resourceDescription; } + + protected boolean customWriteNodeModel(OutputStream outputStream) throws IOException { + return getParseResultWrapper().customWriteNodeModel(this, outputStream); + } + + protected boolean customReadNodeModel(InputStream inputStream) throws IOException { + return getParseResultWrapper().customReadNodeModel(this, inputStream); + } }