merge master and regenerate docker sandbox-bundled manifests

Signed-off-by: Yee Hing Tong <[email protected]>

.github/workflows/flyteidl-buf-publish.yml

@@ -3,7 +3,7 @@ name: Publish flyteidl Buf Package
 on:
   push:
     branches:
-      - artifacts-shell
+      - artifacts-shell-2
       - artifacts
       - master
     paths:

charts/flyte-sandbox/templates/buildkit/deployment.yaml

@@ -13,6 +13,7 @@ spec:
     metadata:
       labels: {{- include "flyte-sandbox.buildkitSelectorLabels" . | nindent 8 }}
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
       containers:
         - name: buildkit

docker/sandbox-bundled/Makefile

@@ -18,8 +18,9 @@ flyte:
 .PHONY: manifests
 manifests:
 	mkdir -p manifests
-	helm dependency update ../../charts/flyte-sandbox
 	helm dependency update ../../charts/flyteagent
+	helm dependency update ../../charts/flyte-binary
+	helm dependency update ../../charts/flyte-sandbox
 	kustomize build \
 		--enable-helm \
 		--load-restrictor=LoadRestrictionsNone \

docker/sandbox-bundled/manifests/complete-agent.yaml

@@ -878,7 +878,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: eTJZeFhSZWF0TEVQVWlybw==
+  haSharedSecret: c1BaWUVKZ3RMNHBNa1A5bQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1450,6 +1450,7 @@ spec:
           periodSeconds: 30
         securityContext:
           privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
 ---
 apiVersion: apps/v1
@@ -1473,7 +1474,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 101b0e951be7ace5815290755725d39275a0f550892b354b9cfa129e0a750f0f
+        checksum/secret: 9c81d342cbc7f96cb7d02181f9f7516d4b6e47f6e1475c039241991ac568b851
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/complete.yaml

@@ -858,7 +858,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: bkVuc0pmTEgwZEp0QUVSbQ==
+  haSharedSecret: YTNTZDQ0WTFQR1pNb2lJRA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1398,6 +1398,7 @@ spec:
           periodSeconds: 30
         securityContext:
           privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
 ---
 apiVersion: apps/v1
@@ -1421,7 +1422,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 63b35e1a77beeb576299ff75680ecfdebf9034eab5c85af9b0c24646b1e3007c
+        checksum/secret: 4f7d40beb7c6c4fa355d9a6a5ebeda49896ecf6bb4030db160e2267ef0e4d1f8
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/dev.yaml

@@ -535,7 +535,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: NDV4Z0JTUVAwRnlCYUFPWQ==
+  haSharedSecret: cGlKcHZrZFRRWllURW9Xdw==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -958,6 +958,7 @@ spec:
           periodSeconds: 30
         securityContext:
           privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
 ---
 apiVersion: apps/v1
@@ -981,7 +982,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 17dd6127a719e8f5716b5cbddc9f2c3ccb88560fca3dc5cad286376afc6271c0
+        checksum/secret: cfd24691ea1c055cf1ebdb9f15b6e46916c8831685a84464932920b9b05cb2a8
       labels:
         app: docker-registry
         release: flyte-sandbox

flyteadmin/pkg/async/cloudevent/implementations/cloudevent_publisher.go

@@ -136,9 +136,7 @@ func (c *CloudEventWrappedPublisher) TransformWorkflowExecutionEvent(ctx context
 	// For now, don't append any additional information unless succeeded
 	if rawEvent.Phase != core.WorkflowExecution_SUCCEEDED {
 		return &event.CloudEventWorkflowExecution{
-			RawEvent:        rawEvent,
-			OutputData:      nil,
-			OutputInterface: nil,
+			RawEvent: rawEvent,
 		}, nil
 	}
 

flyteadmin/pkg/manager/impl/exec_manager_other_test.go

@@ -54,12 +54,16 @@ func TestTrackingBitExtract(t *testing.T) {
 		},
 	}
 
-	trackers := execManager.ExtractArtifactKeys(&lit)
+	var trackers = make(map[string]string)
+	execManager.ExtractArtifactTrackers(trackers, &lit)
 	assert.Equal(t, 1, len(trackers))
 
-	trackers = execManager.ExtractArtifactKeys(&core.Literal{Value: &core.Literal_Map{Map: &inputMap}})
+	trackers = make(map[string]string)
+	execManager.ExtractArtifactTrackers(trackers, &core.Literal{Value: &core.Literal_Map{Map: &inputMap}})
 	assert.Equal(t, 1, len(trackers))
-	trackers = execManager.ExtractArtifactKeys(&core.Literal{Value: &core.Literal_Collection{Collection: &inputColl}})
+
+	trackers = make(map[string]string)
+	execManager.ExtractArtifactTrackers(trackers, &core.Literal{Value: &core.Literal_Collection{Collection: &inputColl}})
 	assert.Equal(t, 1, len(trackers))
-	assert.Equal(t, "proj/domain/name@version", trackers[0])
+	assert.Equal(t, "", trackers["proj/domain/name@version"])
 }

flyteadmin/pkg/manager/impl/execution_manager.go

@@ -47,6 +47,7 @@ import (
 )
 
 const childContainerQueueKey = "child_queue"
+const artifactTrackerKey = "_ua"
 
 // Map of [project] -> map of [domain] -> stop watch
 type projectDomainScopedStopWatchMap = map[string]map[string]*promutils.StopWatch
@@ -700,31 +701,26 @@ func resolveSecurityCtx(ctx context.Context, executionConfigSecurityCtx *core.Se
 	}
 }
 
-// ExtractArtifactKeys pulls out artifact keys from Literals for lineage
-// todo: rename this function to be less confusing
-func (m *ExecutionManager) ExtractArtifactKeys(input *core.Literal) []string {
-	var artifactKeys []string
+// ExtractArtifactTrackers pulls out artifact tracker strings from Literals for lineage
+func (m *ExecutionManager) ExtractArtifactTrackers(artifactTrackers map[string]string, input *core.Literal) {
 
 	if input == nil {
-		return artifactKeys
+		return
 	}
 	if input.GetMetadata() != nil {
-		if artifactKey, ok := input.GetMetadata()["_ua"]; ok {
-			artifactKeys = append(artifactKeys, artifactKey)
+		if tracker, ok := input.GetMetadata()[artifactTrackerKey]; ok {
+			artifactTrackers[tracker] = ""
 		}
 	}
 	if input.GetCollection() != nil {
 		for _, v := range input.GetCollection().Literals {
-			mapKeys := m.ExtractArtifactKeys(v)
-			artifactKeys = append(artifactKeys, mapKeys...)
+			m.ExtractArtifactTrackers(artifactTrackers, v)
 		}
 	} else if input.GetMap() != nil {
 		for _, v := range input.GetMap().Literals {
-			mapKeys := m.ExtractArtifactKeys(v)
-			artifactKeys = append(artifactKeys, mapKeys...)
+			m.ExtractArtifactTrackers(artifactTrackers, v)
 		}
 	}
-	return artifactKeys
 }
 
 // getStringFromInput should be called when a tag or partition value is a binding to an input. the input is looked up
@@ -976,7 +972,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel(
 
 	// TODO: Artifact feature gate, remove when ready
 	var lpExpectedInputs *core.ParameterMap
-	var artifactTrackers []string
+	var artifactTrackers = make(map[string]string)
 	var usedArtifactIDs []*core.ArtifactID
 	if m.artifactRegistry.GetClient() != nil {
 		// Literals may have an artifact key in the metadata field. This is something the artifact service should have
@@ -988,9 +984,8 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel(
 		fixedInputMap := &core.Literal{
 			Value: &core.Literal_Map{Map: launchPlan.Spec.FixedInputs},
 		}
-		artifactTrackers = m.ExtractArtifactKeys(requestInputMap)
-		fixedInputArtifactKeys := m.ExtractArtifactKeys(fixedInputMap)
-		artifactTrackers = append(artifactTrackers, fixedInputArtifactKeys...)
+		m.ExtractArtifactTrackers(artifactTrackers, requestInputMap)
+		m.ExtractArtifactTrackers(artifactTrackers, fixedInputMap)
 
 		// Put together the inputs that we've already resolved so that the artifact querying bit can fill them in.
 		// This is to support artifact queries that depend on other inputs using the {{ .inputs.var }} construct.
@@ -1018,7 +1013,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel(
 		}
 
 		logger.Debugf(ctx, "Resolved launch plan closure expected inputs from [%+v] to [%+v]", launchPlan.Closure.ExpectedInputs, lpExpectedInputs)
-		logger.Debugf(ctx, "Found artifact keys: %v", artifactTrackers)
+		logger.Debugf(ctx, "Found artifact trackers: %v", artifactTrackers)
 		logger.Debugf(ctx, "Found artifact IDs: %v", usedArtifactIDs)
 
 	} else {
@@ -1173,6 +1168,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel(
 	// Publish of event is also gated on the artifact client being available, even though it's not directly required.
 	// TODO: Artifact feature gate, remove when ready
 	if m.artifactRegistry.GetClient() != nil {
+		// TODO: Add principal
 		m.publishExecutionStart(ctx, workflowExecutionID, request.Spec.LaunchPlan, workflow.Id, artifactTrackers, usedArtifactIDs)
 	}
 
@@ -1227,17 +1223,23 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel(
 
 // publishExecutionStart is an event that Admin publishes for artifact lineage.
 func (m *ExecutionManager) publishExecutionStart(ctx context.Context, executionID core.WorkflowExecutionIdentifier,
-	launchPlanID *core.Identifier, workflowID *core.Identifier, inputArtifactKeys []string, usedArtifactIDs []*core.ArtifactID) {
+	launchPlanID *core.Identifier, workflowID *core.Identifier, artifactTrackers map[string]string, usedArtifactIDs []*core.ArtifactID) {
+
+	var artifactTrackerList []string
+	// Use a list instead of the fake set
+	for k := range artifactTrackers {
+		artifactTrackerList = append(artifactTrackerList, k)
+	}
 
-	if len(inputArtifactKeys) > 0 || len(usedArtifactIDs) > 0 {
-		logger.Debugf(ctx, "Sending execution start event for execution [%+v] with input artifact keys [%+v] and used artifact ids [%+v]", executionID, inputArtifactKeys, usedArtifactIDs)
+	if len(artifactTrackerList) > 0 || len(usedArtifactIDs) > 0 {
+		logger.Debugf(ctx, "Sending execution start event for execution [%+v] with trackers [%+v] and artifact ids [%+v]", executionID, artifactTrackerList, usedArtifactIDs)
 
 		request := event.CloudEventExecutionStart{
-			ExecutionId:  &executionID,
-			LaunchPlanId: launchPlanID,
-			WorkflowId:   workflowID,
-			ArtifactIds:  usedArtifactIDs,
-			ArtifactKeys: inputArtifactKeys,
+			ExecutionId:      &executionID,
+			LaunchPlanId:     launchPlanID,
+			WorkflowId:       workflowID,
+			ArtifactIds:      usedArtifactIDs,
+			ArtifactTrackers: artifactTrackerList,
 		}
 		go func() {
 			ceCtx := context.TODO()