Adds Files in AASX Serialization (#399)

* Adds Files in AASX Serialization (WIP) * Changes Prefix * Fixes typo * Refactoring * Adds missing Methods (Unimplemented) * Removes Recursion and changes Exception thrown in Client * Removes Debugging code - Adds Checks * Adds Test for AASX Serialization with Files * Revert AasEnvironmentApiHTTPController.java * Change license header and author * Change license header * Adds missing method * Adds missing methods
eclipse-basyx · Aug 30, 2024 · 75d5956 · 75d5956
1 parent 439b403
commit 75d5956
Show file tree

Hide file tree

Showing 22 changed files with 284 additions and 31 deletions.
diff --git a/...rc/main/java/org/eclipse/digitaltwin/basyx/aasenvironment/base/DefaultAASEnvironment.java b/...rc/main/java/org/eclipse/digitaltwin/basyx/aasenvironment/base/DefaultAASEnvironment.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (C) 2023 the Eclipse BaSyx Authors
+ * Copyright (C) 2024 the Eclipse BaSyx Authors
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -24,14 +24,12 @@
  ******************************************************************************/
 package org.eclipse.digitaltwin.basyx.aasenvironment.base;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
+import java.io.*;
+import java.nio.file.Files;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 import org.apache.commons.io.FilenameUtils;
@@ -40,12 +38,8 @@
 import org.eclipse.digitaltwin.aas4j.v3.dataformat.core.SerializationException;
 import org.eclipse.digitaltwin.aas4j.v3.dataformat.json.JsonSerializer;
 import org.eclipse.digitaltwin.aas4j.v3.dataformat.xml.XmlSerializer;
-import org.eclipse.digitaltwin.aas4j.v3.model.AssetAdministrationShell;
-import org.eclipse.digitaltwin.aas4j.v3.model.ConceptDescription;
-import org.eclipse.digitaltwin.aas4j.v3.model.Environment;
-import org.eclipse.digitaltwin.aas4j.v3.model.Submodel;
-import org.eclipse.digitaltwin.aas4j.v3.model.SubmodelElement;
-import org.eclipse.digitaltwin.aas4j.v3.model.Resource;
+import org.eclipse.digitaltwin.aas4j.v3.model.*;
+import org.eclipse.digitaltwin.aas4j.v3.model.File;
 import org.eclipse.digitaltwin.aas4j.v3.model.impl.DefaultEnvironment;
 import org.eclipse.digitaltwin.basyx.aasenvironment.AasEnvironment;
 import org.eclipse.digitaltwin.basyx.aasenvironment.ConceptDescriptionIdCollector;
@@ -60,6 +54,7 @@
 import org.eclipse.digitaltwin.basyx.aasrepository.AasRepository;
 import org.eclipse.digitaltwin.basyx.conceptdescriptionrepository.ConceptDescriptionRepository;
 import org.eclipse.digitaltwin.basyx.core.exceptions.ElementDoesNotExistException;
+import org.eclipse.digitaltwin.basyx.core.exceptions.FileDoesNotExistException;
 import org.eclipse.digitaltwin.basyx.submodelrepository.SubmodelRepository;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -69,7 +64,7 @@
 /**
  * Default implementation of {@link AasEnvironment}
  *
- * @author zhangzai, danish
+ * @author zhangzai, danish, fried
  */
 public class DefaultAASEnvironment implements AasEnvironment {
 
@@ -83,7 +78,8 @@ public class DefaultAASEnvironment implements AasEnvironment {
 	private AASXSerializer aasxSerializer = new AASXSerializer();
 	private MetamodelCloneCreator cloneCreator = new MetamodelCloneCreator();
 	private IdentifiableAssertion checker;
-
+	private static String aasxFilePathPrefix = "/aasx/files/";
+
 	public DefaultAASEnvironment(AasRepository aasRepository, SubmodelRepository submodelRepository, ConceptDescriptionRepository conceptDescriptionRepository) {
 		this.aasRepository = aasRepository;
 		this.submodelRepository = submodelRepository;
@@ -108,11 +104,41 @@ public String createXMLAASEnvironmentSerialization(@Valid List<String> aasIds, @
 	public byte[] createAASXAASEnvironmentSerialization(@Valid List<String> aasIds, @Valid List<String> submodelIds, @Valid boolean includeConceptDescriptions) throws SerializationException, IOException {
 		Environment aasEnvironment = createEnvironment(aasIds, submodelIds, includeConceptDescriptions);
 
+		List<InMemoryFile> relatedFiles = new ArrayList<>();
+		HashMap<String, List<File>> fileSubmodelElements = new HashMap<>();
+
+		aasEnvironment.getSubmodels().forEach(sm-> fileSubmodelElements.put(sm.getId(),getAllFileSubmodelElements(sm)));
+
+		addFilesToRelatedFiles(fileSubmodelElements, relatedFiles);
+		aasEnvironment.getAssetAdministrationShells().forEach(aas->addThumbnailToRelatedFiles(aas.getId(),aas.getAssetInformation().getDefaultThumbnail(),relatedFiles));
+
 		ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-		aasxSerializer.write(aasEnvironment, null, outputStream);
+		aasxSerializer.write(aasEnvironment, relatedFiles, outputStream);
 		return outputStream.toByteArray();
 	}
-
+
+	public List<File> getAllFileSubmodelElements(Submodel submodel) {
+		List<File> files = new ArrayList<>();
+		Stack<SubmodelElement> stack = new Stack<>();
+
+		stack.addAll(submodel.getSubmodelElements());
+
+		while (!stack.isEmpty()) {
+			SubmodelElement submodelElement = stack.pop();
+
+			if (submodelElement instanceof File file) {
+                if(!isURL(file.getValue()))
+					files.add(file);
+			} else if (submodelElement instanceof SubmodelElementCollection collection) {
+				stack.addAll(collection.getValue());
+			} else if (submodelElement instanceof SubmodelElementList list) {
+				stack.addAll(list.getValue());
+			}
+		}
+
+		return files;
+	}
+
 	public void loadEnvironment(CompleteEnvironment completeEnvironment) {
 		Environment environment = completeEnvironment.getEnvironment();
 		List<InMemoryFile> relatedFiles = completeEnvironment.getRelatedFiles();
@@ -295,4 +321,91 @@ private ConceptDescription fetchConceptDescriptionFromRepo(String conceptDescrip
 		}
 	}
 
+	private void addFilesToRelatedFiles(HashMap<String, List<File>> submodelFileSMEMap, List<InMemoryFile> relatedFiles) {
+		submodelFileSMEMap.forEach((submodelId, fileSubmodelElementList) -> fileSubmodelElementList.forEach(file -> {
+			try {
+				if (isFileAlreadyAdded(file.getValue())) {
+					return;
+				}
+				processFile(submodelId, file, relatedFiles);
+			} catch (IOException | NullPointerException | FileDoesNotExistException e) {
+				handleFileError(file);
+			}
+		}));
+	}
+
+	private void processFile(String key, File file, List<InMemoryFile> relatedFiles) throws IOException {
+		byte[] fileContent = getFileContent(key, file);
+		String path = getFilePathInAASX(file.getValue());
+		relatedFiles.add(new InMemoryFile(fileContent, path));
+		file.setValue(path);
+	}
+
+	private void handleFileError(File file) {
+		logger.error("File {} does not exist in the repository", file.getValue());
+	}
+
+
+	private byte[] getFileContent(String submodelId, File file) throws IOException {
+		return submodelRepository.getFileByFilePath(submodelId, file.getValue()).readAllBytes();
+	}
+
+	private boolean isFileAlreadyAdded(String filePath){
+		return filePath.startsWith(aasxFilePathPrefix);
+	}
+
+	private void addThumbnailToRelatedFiles(String aasId, Resource thumbnail, List<InMemoryFile> relatedFiles) {
+		if(!isThumbnailSet(thumbnail)) {
+			logger.info("No thumbnail specified for aas {}", aasId);
+		}else {
+			try {
+				String newPath = getThumbnailPathInAASX(thumbnail.getPath());
+				relatedFiles.add(
+						new InMemoryFile(
+								Files.readAllBytes(aasRepository.getThumbnail(aasId).toPath()),
+								newPath
+						)
+				);
+				thumbnail.setPath(newPath);
+			} catch (IOException | FileDoesNotExistException e) {
+				logger.error("Thumbnail file {} does not exist in the repository", thumbnail.getPath());
+			}
+		}
+    }
+
+	private static boolean isThumbnailSet(Resource thumbnail) {
+		return thumbnail != null && thumbnail.getPath() != null;
+	}
+
+	private static String getThumbnailPathInAASX(String path) {
+		try {
+            return "/" + getHashedFilePath(path) + "." + getFileExtensionFromPath(path);
+        } catch (NoSuchAlgorithmException e) {
+			logger.error("Could not hash thumbnail path {}", path);
+            throw new RuntimeException(e);
+        }
+    }
+
+	private static String getFilePathInAASX(String path) {
+		try {
+            return aasxFilePathPrefix + getHashedFilePath(path) + "." + getFileExtensionFromPath(path);
+        } catch (NoSuchAlgorithmException e) {
+			logger.error("Could not hash file path {}", path);
+            throw new RuntimeException(e);
+        }
+    }
+
+	private static String getFileExtensionFromPath(String file) {
+		String[] split = file.split("\\.");
+		return split[split.length - 1];
+	}
+
+	private static String getHashedFilePath(String filePath) throws NoSuchAlgorithmException {
+		int hashCode = filePath.hashCode();
+		return Integer.toHexString(hashCode);
+	}
+
+	private static boolean isURL(String path) {
+		return path.startsWith("http");
+	}
 }
diff --git a/...st/java/org/eclipse/digitaltwin/basyx/aasenvironment/TestAASEnvironmentSerialization.java b/...st/java/org/eclipse/digitaltwin/basyx/aasenvironment/TestAASEnvironmentSerialization.java
@@ -25,9 +25,6 @@
 
 package org.eclipse.digitaltwin.basyx.aasenvironment;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -39,6 +36,7 @@
 import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
 import org.apache.poi.openxml4j.exceptions.InvalidOperationException;
 import org.eclipse.digitaltwin.aas4j.v3.dataformat.aasx.AASXDeserializer;
+import org.eclipse.digitaltwin.aas4j.v3.dataformat.aasx.InMemoryFile;
 import org.eclipse.digitaltwin.aas4j.v3.dataformat.core.DeserializationException;
 import org.eclipse.digitaltwin.aas4j.v3.dataformat.core.SerializationException;
 import org.eclipse.digitaltwin.aas4j.v3.dataformat.json.JsonDeserializer;
@@ -72,6 +70,8 @@
 import org.junit.Test;
 import org.xml.sax.SAXException;
 
+import static org.junit.Assert.*;
+
 public class TestAASEnvironmentSerialization {
 
 	private static final PaginationInfo NO_LIMIT_PAGINATION_INFO = new PaginationInfo(0, "");
@@ -186,11 +186,16 @@ public static void validateXml(String actual, boolean areAASsIncluded, boolean a
 	public static void checkAASX(InputStream inputStream, boolean areAASsIncluded, boolean areSubmodelsIncluded, boolean includeConceptDescription) throws IOException, InvalidFormatException, DeserializationException {
 		AASXDeserializer aasxDeserializer = new AASXDeserializer(inputStream);
 		Environment environment = aasxDeserializer.read();
-
 		checkAASEnvironment(environment, areAASsIncluded, areSubmodelsIncluded, includeConceptDescription);
 		inputStream.close();
 	}
 
+	public static void checkAASXFiles(InputStream inputStream) throws IOException, InvalidFormatException, DeserializationException {
+		AASXDeserializer aasxDeserializer = new AASXDeserializer(inputStream);
+		List<InMemoryFile> files = aasxDeserializer.getRelatedFiles();
+		assertEquals(2,files.size());
+	}
+
 	public static Collection<ConceptDescription> createDummyConceptDescriptions() {
 		Collection<ConceptDescription> conceptDescriptions = new ArrayList<>();
 

diff --git a/...c/test/java/org/eclipse/basyx/digitaltwin/aasenvironment/http/TestAasEnvironmentHTTP.java b/...c/test/java/org/eclipse/basyx/digitaltwin/aasenvironment/http/TestAasEnvironmentHTTP.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (C) 2023 the Eclipse BaSyx Authors
+ * Copyright (C) 2024 the Eclipse BaSyx Authors
  * 
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -140,6 +140,14 @@ public void testAASEnvironmentSertializationWithAASX() throws IOException, Parse
 		TestAASEnvironmentSerialization.checkAASX(response.getEntity().getContent(), aasIdsIncluded, submodelIdsIncluded, includeConceptDescription);
 	}
 
+	@Test
+	public void testAASEnvironmentSertializationWithAASXAndFiles() throws IOException, ParseException, DeserializationException, InvalidFormatException {
+		CloseableHttpResponse response = executeGetOnURL(createSerializationURLForFiles(), AASX_MIMETYPE);
+		assertEquals(HttpStatus.OK.value(), response.getCode());
+
+		TestAASEnvironmentSerialization.checkAASXFiles(response.getEntity().getContent());
+	}
+
 	@Test
 	public void testAASEnvironmentSertializationWithAASXExcludeCD() throws IOException, ParseException, DeserializationException, InvalidFormatException {
 		boolean includeConceptDescription = false;
@@ -252,6 +260,11 @@ public static String createSerializationURL(boolean includeConceptDescription) {
 				createIdCollection(DummyAASEnvironmentComponent.SUBMODEL_OPERATIONAL_DATA_ID, DummyAASEnvironmentComponent.SUBMODEL_TECHNICAL_DATA_ID), includeConceptDescription);
 	}
 
+	public static String createSerializationURLForFiles() {
+		return getSerializationURL(createIdCollection("https://example.com/ids/AssetAdministrationShell/3982_3381_6308_9332"),
+				createIdCollection("https://example.com/ids/Submodel/3293_1019_6578_9120"), false);
+	}
+
 	public static CloseableHttpResponse executeGetOnURL(String url, String header) throws IOException {
 		CloseableHttpClient client = HttpClients.createDefault();
 		HttpGet getRequest = createGetRequestWithHeader(url, header);

diff --git a/basyx.aasenvironment/basyx.aasenvironment-http/src/test/resources/application.properties b/basyx.aasenvironment/basyx.aasenvironment-http/src/test/resources/application.properties
@@ -1,2 +1,4 @@
 server.port=8081
-basyx.backend = InMemory
+basyx.backend = InMemory
+
+basyx.environment = classpath:fileSerializationTest.aasx
diff --git a/basyx.aasenvironment/basyx.aasenvironment-http/src/test/resources/fileSerializationTest.aasx b/basyx.aasenvironment/basyx.aasenvironment-http/src/test/resources/fileSerializationTest.aasx
diff --git a/...java/org/eclipse/digitaltwin/basyx/submodelrepository/backend/CrudSubmodelRepository.java b/...java/org/eclipse/digitaltwin/basyx/submodelrepository/backend/CrudSubmodelRepository.java
@@ -332,4 +332,11 @@ public void patchSubmodelElements(String submodelId, List<SubmodelElement> submo
 		submodelBackend.save(submodel);
 	}
 
+	@Override
+	public InputStream getFileByFilePath(String submodelId, String filePath) {
+		SubmodelService submodelService = getSubmodelServiceOrThrow(submodelId);
+
+		return submodelService.getFileByFilePath(filePath);
+	}
+
 }
diff --git a/.../org/eclipse/digitaltwin/basyx/submodelrepository/client/ConnectedSubmodelRepository.java b/.../org/eclipse/digitaltwin/basyx/submodelrepository/client/ConnectedSubmodelRepository.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (C) 2023 the Eclipse BaSyx Authors
+ * Copyright (C) 2024 the Eclipse BaSyx Authors
  * 
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -246,4 +246,9 @@ public void patchSubmodelElements(String submodelId, List<SubmodelElement> submo
 		getConnectedSubmodelService(submodelId).patchSubmodelElements(submodelElementList);
 	}
 
+	@Override
+	public InputStream getFileByFilePath(String submodelId, String filePath) {
+		return getConnectedSubmodelService(submodelId).getFileByFilePath(filePath);
+	}
+
 }
diff --git a/...igitaltwin/basyx/submodelrepository/client/TestAuthorizedConnectedSubmodelRepository.java b/...igitaltwin/basyx/submodelrepository/client/TestAuthorizedConnectedSubmodelRepository.java
@@ -155,4 +155,9 @@ public void invokeNonOperation() {
 		// TODO
 		throw new NotInvokableException();
 	}
+
+	@Override
+	public void getFileByFilePath(){
+		// Not Implemented for Client so Override Test
+	}
 }
diff --git a/.../eclipse/digitaltwin/basyx/submodelrepository/client/TestConnectedSubmodelRepository.java b/.../eclipse/digitaltwin/basyx/submodelrepository/client/TestConnectedSubmodelRepository.java
@@ -101,4 +101,9 @@ protected boolean fileExistsInStorage(String fileValue) {
 
 		return file.exists();
 	}
+
+	@Override
+	public void getFileByFilePath(){
+		// Not Implemented for Client so Override Test
+	}
 }
diff --git a/...re/src/main/java/org/eclipse/digitaltwin/basyx/submodelrepository/SubmodelRepository.java b/...re/src/main/java/org/eclipse/digitaltwin/basyx/submodelrepository/SubmodelRepository.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (C) 2023 the Eclipse BaSyx Authors
+ * Copyright (C) 2024 the Eclipse BaSyx Authors
  * 
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -44,7 +44,7 @@
 /**
  * Specifies the overall SubmodelRepository API
  * 
- * @author schnicke, danish, kammognie
+ * @author schnicke, danish, kammognie, fried
  *
  */
 public interface SubmodelRepository {
@@ -272,4 +272,15 @@ public default String getName() {
 	 * @param submodelElementList
 	 */
 	public void patchSubmodelElements(String submodelId, List<SubmodelElement> submodelElementList);
+
+	/**
+	 * Retrieves the file of a file submodelelement via its absolute path
+	 *
+	 * @param submodelId
+	 * 			  the Submodel id
+	 * @param filePath
+	 *            the path of the file
+	 * @return File InputStream
+	 */
+	public InputStream getFileByFilePath(String submodelId, String filePath);
 }
diff --git a/...e/digitaltwin/basyx/submodelrepository/core/SubmodelRepositorySubmodelServiceWrapper.java b/...e/digitaltwin/basyx/submodelrepository/core/SubmodelRepositorySubmodelServiceWrapper.java
@@ -134,4 +134,9 @@ public void deleteFileValue(String idShortPath) throws ElementDoesNotExistExcept
 		repoApi.deleteFileValue(submodelId, idShortPath);
 	}
 
+	@Override
+	public InputStream getFileByFilePath(String filePath) {
+		return repoApi.getFileByFilePath(submodelId, filePath);
+	}
+
 }
diff --git a/...italtwin/basyx/submodelrepository/feature/authorization/AuthorizedSubmodelRepository.java b/...italtwin/basyx/submodelrepository/feature/authorization/AuthorizedSubmodelRepository.java
@@ -243,7 +243,12 @@ public void patchSubmodelElements(String submodelId, List<SubmodelElement> submo
 
 		decorated.patchSubmodelElements(submodelId, submodelElementList);
 	}
-
+
+	@Override
+	public InputStream getFileByFilePath(String submodelId, String filePath) {
+		return decorated.getFileByFilePath(submodelId, filePath);
+	}
+
 	private List<String> getIdAsList(String id) {
 		return new ArrayList<>(Arrays.asList(id));
 	}