chore: Remove obsolete fields from models (#3123)

* chore: Remove obsolete fields from models

* Fix tests, add header

* Refactor python model

* Migrate model

streampipes-client-go/streampipes/model/common.go

@@ -29,10 +29,8 @@ type EventProperty struct {
 	Label              string             `json:"label,omitempty"`
 	Description        string             `json:"description,omitempty"`
 	RuntimeName        string             `json:"runtimeName,omitempty"`
-	Required           bool               `json:"required,omitempty"`
 	DomainProperties   []string           `json:"domainProperties,omitempty"`
 	PropertyScope      string             `json:"propertyScope,omitempty"`
-	Index              int                `json:"index"`
 	RuntimeID          string             `json:"runtimeId,omitempty"`
 	RuntimeType        string             `json:"runtimeType"`
 	MeasurementUnit    string             `json:"measurementUnit,omitempty"`
@@ -44,10 +42,8 @@ type EventProperties struct {
 	Label              string            `json:"label"`
 	Description        string            `json:"description"`
 	RuntimeName        string            `json:"runtimeName"`
-	Required           bool              `json:"required"`
 	DomainProperties   []string          `json:"domainProperties"`
 	PropertyScope      string            `json:"propertyScope"`
-	Index              int               `json:"index"`
 	RuntimeID          string            `json:"runtimeId"`
 	RuntimeType        string            `json:"runtimeType,omitempty"`
 	MeasurementUnit    string            `json:"measurementUnit,omitempty"`

streampipes-client-python/streampipes/functions/utils/data_stream_generator.py

@@ -87,7 +87,6 @@ def create_data_stream(
             EventProperty(  # type: ignore
                 label=attribute_name,
                 runtime_name=attribute_name,
-                index=i,
                 runtime_type=f"http://www.w3.org/2001/XMLSchema#{attribute_type}",
             )
             for i, (attribute_name, attribute_type) in enumerate(attributes.items(), start=1)

streampipes-client-python/streampipes/model/common.py

@@ -109,10 +109,8 @@ class EventProperty(BasicModel):
     label: Optional[StrictStr]
     description: Optional[StrictStr]
     runtime_name: StrictStr
-    required: StrictBool = Field(default=False)
     domain_properties: Optional[List[StrictStr]] = Field(default_factory=list)
     property_scope: Optional[StrictStr] = Field(default="MEASUREMENT_PROPERTY")
-    index: StrictInt = Field(default=0)
     runtime_id: Optional[StrictStr]
     runtime_type: StrictStr = Field(default="http://www.w3.org/2001/XMLSchema#string")
     measurement_unit: Optional[StrictStr]

streampipes-client-python/streampipes/model/resource/data_stream.py

@@ -102,7 +102,6 @@ def __init__(self, **kwargs):
     event_schema: Optional[EventSchema]
     measurement_capability: Optional[List[MeasurementCapability]]
     measurement_object: Optional[List[MeasurementObject]]
-    index: StrictInt = Field(default=0)
     corresponding_adapter_id: Optional[StrictStr]
     category: Optional[List[StrictStr]]
     uri: Optional[StrictStr]

streampipes-client-python/tests/client/test_endpoint.py

@@ -53,10 +53,8 @@ def setUp(self) -> None:
                             "label": "Density",
                             "description": "Denotes the current density of the fluid",
                             "runtimeName": "density",
-                            "required": False,
                             "domainProperties": ["http://schema.org/Number"],
                             "propertyScope": "MEASUREMENT_PROPERTY",
-                            "index": 5,
                             "runtimeId": None,
                             "runtimeType": "http://www.w3.org/2001/XMLSchema#float",
                             "measurementUnit": None,
@@ -68,10 +66,8 @@ def setUp(self) -> None:
                             "label": "Temperature",
                             "description": "Denotes the current temperature in degrees celsius",
                             "runtimeName": "temperature",
-                            "required": False,
                             "domainProperties": ["http://schema.org/Number"],
                             "propertyScope": "MEASUREMENT_PROPERTY",
-                            "index": 4,
                             "runtimeId": None,
                             "runtimeType": "http://www.w3.org/2001/XMLSchema#float",
                             "measurementUnit": "http://codes.wmo.int/common/unit/degC",
@@ -135,10 +131,8 @@ def setUp(self) -> None:
                             "label": "Density",
                             "description": "Denotes the current density of the fluid",
                             "runtimeName": "density",
-                            "required": False,
                             "domainProperties": ["http://schema.org/Number"],
                             "propertyScope": "MEASUREMENT_PROPERTY",
-                            "index": 5,
                             "runtimeId": None,
                             "runtimeType": "http://www.w3.org/2001/XMLSchema#float",
                             "measurementUnit": None,
@@ -150,10 +144,8 @@ def setUp(self) -> None:
                             "label": "Temperature",
                             "description": "Denotes the current temperature in degrees celsius",
                             "runtimeName": "temperature",
-                            "required": False,
                             "domainProperties": ["http://schema.org/Number"],
                             "propertyScope": "MEASUREMENT_PROPERTY",
-                            "index": 4,
                             "runtimeId": None,
                             "runtimeType": "http://www.w3.org/2001/XMLSchema#float",
                             "measurementUnit": "http://codes.wmo.int/common/unit/degC",
@@ -169,7 +161,6 @@ def setUp(self) -> None:
                 },
                 "measurementCapability": None,
                 "measurementObject": None,
-                "index": 0,
                 "correspondingAdapterId": "urn:streampipes.apache.org:spi:org.apache.streampipes.connect."
                 "iiot.adapters.simulator.machine:11934d37-135b-4ef6-b5f1-4f520cc81a43",
                 "category": None,
@@ -294,7 +285,6 @@ def test_endpoint_data_stream_happy_path(self, server_version: MagicMock, http_s
                 "includes_locales",
                 "internally_managed",
                 "measurement_object",
-                "index",
                 "corresponding_adapter_id",
                 "uri",
                 "dom",

streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaProducer.java

@@ -96,12 +96,11 @@ public void connect() {
     LOG.info("Kafka producer: Connecting to " + protocol.getTopicDefinition().getActualTopicName());
     this.brokerUrl = protocol.getBrokerHostname() + ":" + protocol.getKafkaPort();
     this.topic = protocol.getTopicDefinition().getActualTopicName();
-    String zookeeperHost = protocol.getZookeeperHost() + ":" + protocol.getZookeeperPort();
 
     try {
       createKafkaTopic(protocol);
     } catch (ExecutionException | InterruptedException e) {
-      LOG.error("Could not create topic: " + topic + " on broker " + zookeeperHost);
+      LOG.error("Could not create topic: " + topic + " on broker " + brokerUrl);
     }
 
     this.producer = new KafkaProducer<>(makeProperties(protocol, Collections.emptyList()));

streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventProperty.java

@@ -40,25 +40,14 @@
 public abstract class EventProperty {
 
   protected static final String PREFIX = "urn:streampipes.org:spi:";
-  private static final long serialVersionUID = 7079045979946059387L;
 
   private String elementId;
   private String label;
-
   private String description;
-
   private String runtimeName;
-
-  private boolean required;
-
   private List<URI> domainProperties;
-
   private String propertyScope;
-
-  private int index = 0;
-
   private String runtimeId;
-
   private Map<String, Object> additionalMetadata;
 
 
@@ -72,12 +61,10 @@ public EventProperty(EventProperty other) {
     this.elementId = other.getElementId();
     this.label = other.getLabel();
     this.description = other.getDescription();
-    this.required = other.isRequired();
     this.runtimeName = other.getRuntimeName();
     this.domainProperties = other.getDomainProperties();
     this.propertyScope = other.getPropertyScope();
     this.runtimeId = other.getRuntimeId();
-    this.index = other.getIndex();
     this.additionalMetadata = other.getAdditionalMetadata();
   }
 
@@ -110,14 +97,6 @@ public void setRuntimeName(String propertyName) {
     this.runtimeName = propertyName;
   }
 
-  public boolean isRequired() {
-    return required;
-  }
-
-  public void setRequired(boolean required) {
-    this.required = required;
-  }
-
   public List<URI> getDomainProperties() {
     return domainProperties;
   }
@@ -158,14 +137,6 @@ public void setRuntimeId(String runtimeId) {
     this.runtimeId = runtimeId;
   }
 
-  public int getIndex() {
-    return index;
-  }
-
-  public void setIndex(int index) {
-    this.index = index;
-  }
-
   public String getElementId() {
     return elementId;
   }
@@ -184,7 +155,7 @@ public void setAdditionalMetadata(Map<String, Object> additionalMetadata) {
 
   @Override
   public int hashCode() {
-    return Objects.hash(elementId, label, description, runtimeName, required, domainProperties, propertyScope, index,
+    return Objects.hash(elementId, label, description, runtimeName, domainProperties, propertyScope,
         runtimeId);
   }
 
@@ -198,9 +169,7 @@ public boolean equals(Object o) {
     }
     EventProperty that = (EventProperty) o;
 
-    return required == that.required
-           && index == that.index
-           && Objects.equals(label, that.label)
+    return Objects.equals(label, that.label)
            && Objects.equals(description, that.description)
            && Objects.equals(runtimeName, that.runtimeName)
            && Objects.equals(propertyScope, that.propertyScope)
@@ -215,10 +184,8 @@ public String toString() {
            + ", label='" + label + '\''
            + ", description='" + description + '\''
            + ", runtimeName='" + runtimeName + '\''
-           + ", required=" + required
            + ", domainProperties=" + domainProperties
            + ", propertyScope='" + propertyScope + '\''
-           + ", index=" + index
            + ", runtimeId='" + runtimeId + '\''
            + '}';
   }

streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyPrimitive.java

@@ -26,12 +26,8 @@
 
 public class EventPropertyPrimitive extends EventProperty {
 
-  private static final long serialVersionUID = 665989638281665875L;
-
   private String runtimeType;
-
   private URI measurementUnit;
-
   private ValueSpecification valueSpecification;
 
   public EventPropertyPrimitive() {

streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/StaticProperty.java

@@ -52,7 +52,6 @@ public abstract class StaticProperty {
 
   protected boolean optional;
   protected StaticPropertyType staticPropertyType;
-  private int index;
   private String label;
   private String description;
   private String internalName;
@@ -70,13 +69,11 @@ public StaticProperty(StaticPropertyType type) {
   }
 
   public StaticProperty(StaticProperty other) {
-    this.index = other.getIndex();
     this.description = other.getDescription();
     this.internalName = other.getInternalName();
     this.optional = other.isOptional();
     this.staticPropertyType = other.getStaticPropertyType();
     this.label = other.getLabel();
-    this.predefined = other.isPredefined();
   }
 
   public StaticProperty(StaticPropertyType type, String internalName, String label,
@@ -129,22 +126,6 @@ public void setStaticPropertyType(StaticPropertyType staticPropertyType) {
     this.staticPropertyType = staticPropertyType;
   }
 
-  public boolean isPredefined() {
-    return predefined;
-  }
-
-  public void setPredefined(boolean predefined) {
-    this.predefined = predefined;
-  }
-
-  public int getIndex() {
-    return index;
-  }
-
-  public void setIndex(int index) {
-    this.index = index;
-  }
-
   public <T extends StaticProperty> T as(Class<T> targetClass) {
     return targetClass.cast(this);
   }
@@ -163,9 +144,6 @@ public boolean equals(Object o) {
     if (optional != that.optional) {
       return false;
     }
-    if (index != that.index) {
-      return false;
-    }
     if (predefined != that.predefined) {
       return false;
     }
@@ -185,7 +163,6 @@ public boolean equals(Object o) {
   public int hashCode() {
     int result = (optional ? 1 : 0);
     result = 31 * result + (staticPropertyType != null ? staticPropertyType.hashCode() : 0);
-    result = 31 * result + index;
     result = 31 * result + (label != null ? label.hashCode() : 0);
     result = 31 * result + (description != null ? description.hashCode() : 0);
     result = 31 * result + (internalName != null ? internalName.hashCode() : 0);

streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationGenerator.java

@@ -25,12 +25,11 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Collectors;
 
 public class PipelineTemplateInvocationGenerator {
 
-  private SpDataStream spDataStream;
-  private PipelineTemplateDescription pipelineTemplateDescription;
+  private final SpDataStream spDataStream;
+  private final PipelineTemplateDescription pipelineTemplateDescription;
 
   public PipelineTemplateInvocationGenerator(SpDataStream dataStream,
                                              PipelineTemplateDescription pipelineTemplateDescription) {
@@ -46,7 +45,6 @@ public PipelineTemplateInvocation generateInvocation() {
     PipelineTemplateInvocation pipelineTemplateInvocation = new PipelineTemplateInvocation();
     pipelineTemplateInvocation.setStaticProperties(collectStaticProperties(pipeline));
     pipelineTemplateInvocation.setDataStreamId(spDataStream.getElementId());
-    //pipelineTemplateInvocation.setPipelineTemplateDescription(pipelineTemplateDescription);
     pipelineTemplateInvocation.setPipelineTemplateId(pipelineTemplateDescription.getPipelineTemplateId());
     return pipelineTemplateInvocation;
   }
@@ -65,13 +63,4 @@ private List<StaticProperty> collectStaticProperties(Pipeline pipeline) {
 
     return staticProperties;
   }
-
-  private List<StaticProperty> filter(List<StaticProperty> staticProperties) {
-    return staticProperties
-        .stream()
-        // TODO fix (predefined is always true
-        //.filter(sp -> !(sp instanceof MappingProperty))
-        .filter(sp -> !(sp.isPredefined()))
-        .collect(Collectors.toList());
-  }
 }

streampipes-sdk/src/main/java/org/apache/streampipes/sdk/StaticProperties.java

@@ -59,10 +59,6 @@ public static StaticPropertyAlternatives alternatives(Label label, List<StaticPr
     StaticPropertyAlternatives alternativesContainer =
         new StaticPropertyAlternatives(label.getInternalId(), label.getLabel(), label.getDescription());
 
-    for (int i = 0; i < alternatives.size(); i++) {
-      alternatives.get(i).setIndex(i);
-    }
-
     alternativesContainer.setAlternatives(alternatives);
 
     return alternativesContainer;
@@ -219,9 +215,6 @@ public static SupportedProperty supportedDomainProperty(String rdfPropertyUri, b
 
   public static StaticPropertyGroup group(Label label, StaticProperty... sp) {
     List<StaticProperty> staticProperties = Arrays.asList(sp);
-    for (int i = 0; i < staticProperties.size(); i++) {
-      staticProperties.get(i).setIndex(i);
-    }
     return new StaticPropertyGroup(label.getInternalId(), label.getLabel(),
         label.getDescription(), staticProperties);
   }

streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/AbstractConfigurablePipelineElementBuilder.java

@@ -749,9 +749,6 @@ private FreeTextStaticProperty prepareFreeTextStaticProperty(Label label, String
   }
 
   private List<StaticProperty> sortStaticProperties(List<StaticProperty> staticProperties) {
-    for (int i = 0; i < staticProperties.size(); i++) {
-      staticProperties.get(i).setIndex(i);
-    }
     return staticProperties;
   }