[wip] add service

docker/sandbox-bundled/manifests/complete.yaml

@@ -1060,7 +1060,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: Tm9IU3lEa0NLZDB0blM5TQ==
+  haSharedSecret: VUh0UzRWS0RyQ0l1WDBDUw==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1747,7 +1747,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 877fe97609c6f425701bda477b1165c1d2fbc0939da9a118b87b0430a0c8e713
+        checksum/secret: b5ceec74c1c81a90b9f028e7a45c5138a85dd5a20b522ac8e2bcdd7182948f79
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/dev.yaml

@@ -744,7 +744,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: VTJWMndkSk15emRBZ3F0eg==
+  haSharedSecret: VFpQSXduUGEyZkgxQ0IyYQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1304,7 +1304,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: f0e226e3dc585d5f35bb7081b38d51f46a2c9f90802717e69fb70ee70e18178c
+        checksum/secret: db7f477669a3dd30b567a390215b0840d15f6598ea01d2c9a8e640552796e9b4
       labels:
         app: docker-registry
         release: flyte-sandbox

flyteartifacts/Dockerfile

@@ -0,0 +1,43 @@
+FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+
+ARG TARGETARCH
+ENV GOARCH "${TARGETARCH}"
+ENV GOOS linux
+
+RUN apk add git openssh-client make curl
+
+# Create the artifacts directory
+RUN mkdir /artifacts
+
+
+# COPY only the go mod files for efficient caching
+COPY go.mod go.sum /go/src/github.com/flyteorg/flyte/flyteartifacts/
+WORKDIR /go/src/github.com/flyteorg/flyte/flyteartifacts/
+
+
+# Pull dependencies
+RUN go mod download
+
+# COPY the rest of the source code
+COPY . /go/src/github.com/flyteorg/flyte/flyteartifacts/
+
+# This 'linux_compile' target should compile binaries to the /artifacts directory
+# The main entrypoint should be compiled to /artifacts/flyteadmin
+RUN make linux_compile
+
+# update the PATH to include the /artifacts directory
+ENV PATH="/artifacts:${PATH}"
+
+# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
+FROM alpine:3.16
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flyte/
+
+COPY --from=builder /artifacts /bin
+
+# Ensure the latest CA certs are present to authenticate SSL connections.
+RUN apk --update add ca-certificates
+
+RUN addgroup -S flyte && adduser -S flyte -G flyte
+USER flyte
+
+CMD ["artifacts"]

flyteartifacts/Makefile

@@ -0,0 +1,6 @@
+
+.PHONY: linux_compile
+linux_compile: export CGO_ENABLED ?= 0
+linux_compile: export GOOS ?= linux
+linux_compile:
+	go build -o /artifacts/flyteadmin -ldflags=$(LD_FLAGS) ./cmd/

flyteartifacts/artifact_config.yaml

@@ -0,0 +1,2 @@
+artifactsServer:
+  myTestValue: "test from file"

flyteartifacts/cmd/main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"context"
+	sharedCmd "github.com/flyteorg/flyte/flyteartifacts/cmd/shared"
+	"github.com/flyteorg/flyte/flytestdlib/logger"
+
+	_ "net/http/pprof" // Required to serve application.
+)
+
+//
+//var (
+//	cfgFile        string
+//	configAccessor = viper.NewAccessor(config.Options{})
+//)
+//
+//var serveCmd = &cobra.Command{
+//	Use:   "serve",
+//	Short: "Launches the Flyte artifacts server",
+//	RunE: func(cmd *cobra.Command, args []string) error {
+//		ctx := context.Background()
+//		cfg := configuration.ApplicationConfig.GetConfig().(*configuration.ApplicationConfiguration)
+//		fmt.Printf("cfg: [%+v]\n", cfg)
+//		opts := make([]grpc.ServerOption, 0)
+//		return server.Serve(ctx, opts...)
+//	},
+//}
+//
+//// RootCmd represents the base command when called without any subcommands
+//var RootCmd = &cobra.Command{
+//	Use:   "artifacts",
+//	Short: "Fill in later",
+//	Long: `
+//To get started run the serve subcommand which will start a server on localhost:50051
+//`,
+//	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+//		return initConfig(cmd.Flags())
+//	},
+//}
+//
+//func init() {
+//	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+//
+//	// Add persistent flags - persistent flags persist through all sub-commands
+//	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is ./artifact_config.yaml)")
+//
+//	// Allow viper to read the value of the flags
+//	configAccessor.InitializePflags(RootCmd.PersistentFlags())
+//
+//	// Command information
+//	RootCmd.AddCommand(serveCmd)
+//
+//	err := flag.CommandLine.Parse([]string{})
+//	if err != nil {
+//		fmt.Println(err)
+//		os.Exit(-1)
+//	}
+//
+//}
+//
+//func initConfig(flags *pflag.FlagSet) error {
+//	configAccessor = viper.NewAccessor(config.Options{
+//		SearchPaths: []string{cfgFile, "./artifact_config.yaml", ".", "/etc/flyte/config", "$GOPATH/src/github.com/flyteorg/flyte/flyteartifacts"},
+//		StrictMode:  false,
+//	})
+//
+//	logger.Infof(context.TODO(), "Using config file: %v", configAccessor.ConfigFilesUsed())
+//
+//	configAccessor.InitializePflags(flags)
+//
+//	err := flag.CommandLine.Parse([]string{})
+//	if err != nil {
+//		fmt.Println(err)
+//		os.Exit(-1)
+//	}
+//
+//	return configAccessor.UpdateConfig(context.TODO())
+//}
+
+func main() {
+	ctx := context.Background()
+	logger.Infof(ctx, "Beginning Flyte Artifacts Service")
+	rootCmd := sharedCmd.NewRootCmd("artifacts")
+	//if err := RootCmd.ExecuteC(); err != nil {
+	//	fmt.Println(err)
+	//	panic(err)
+	//}
+	err := rootCmd.ExecuteContext(ctx)
+	if err != nil {
+		panic(err)
+	}
+}

flyteartifacts/cmd/shared/root.go

@@ -0,0 +1,108 @@
+package shared
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	sharedCfg "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration/shared"
+	"github.com/flyteorg/flyte/flytestdlib/config"
+	"github.com/flyteorg/flyte/flytestdlib/config/viper"
+	"github.com/flyteorg/flyte/flytestdlib/contextutils"
+	"github.com/flyteorg/flyte/flytestdlib/logger"
+	"github.com/flyteorg/flyte/flytestdlib/profutils"
+	"github.com/flyteorg/flyte/flytestdlib/promutils/labeled"
+	"github.com/flyteorg/flyte/flytestdlib/storage"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"os"
+)
+
+var (
+	cfgFile        string
+	configAccessor = viper.NewAccessor(config.Options{})
+)
+
+//var XXRootCmd = &cobra.Command{
+//	Use:   "artifacts",
+//	Short: "Fill in later",
+//	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+//		return initConfig(cmd.Flags())
+//	},
+//}
+
+// NewRootCmd represents the base command when called without any subcommands
+func NewRootCmd(rootUse string, grpcHook GrpcRegistrationHook, httpHook HttpRegistrationHook) *cobra.Command {
+
+	rootCmd := &cobra.Command{
+		Use:   rootUse,
+		Short: "Short description",
+		Long:  "Long description to be filled in later",
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			err := initConfig(cmd, args)
+			if err != nil {
+				return err
+			}
+
+			go func() {
+				ctx := context.Background()
+				sharedConfig := sharedCfg.SharedServerConfig.GetConfig().(*sharedCfg.ServerConfiguration)
+				err := profutils.StartProfilingServerWithDefaultHandlers(ctx,
+					sharedConfig.Metrics.Port.Port, nil)
+				if err != nil {
+					logger.Panicf(ctx, "Failed to Start profiling and metrics server. Error: %v", err)
+				}
+			}()
+
+			return nil
+		},
+	}
+
+	initSubCommands(rootCmd, grpcHook, httpHook)
+	return rootCmd
+}
+
+func init() {
+	// Set Keys
+	labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey,
+		contextutils.DomainKey, storage.FailureTypeLabel)
+}
+
+func initConfig(cmd *cobra.Command, _ []string) error {
+	configAccessor = viper.NewAccessor(config.Options{
+		SearchPaths: []string{cfgFile, ".", "/etc/flyte/config", "$GOPATH/src/github.com/flyteorg/flyte/flyteartifacts"},
+		StrictMode:  false,
+	})
+
+	fmt.Println("Using config file: ", configAccessor.ConfigFilesUsed())
+
+	// persistent flags were initially bound to the root command so we must bind to the same command to avoid
+	// overriding those initial ones. We need to traverse up to the root command and initialize pflags for that.
+	rootCmd := cmd
+	for rootCmd.Parent() != nil {
+		rootCmd = rootCmd.Parent()
+	}
+
+	configAccessor.InitializePflags(rootCmd.PersistentFlags())
+
+	return configAccessor.UpdateConfig(context.TODO())
+}
+
+func initSubCommands(rootCmd *cobra.Command, grpcHook GrpcRegistrationHook, httpHook HttpRegistrationHook) {
+	// allows ` --logtostderr` to work
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+
+	// Add persistent flags - persistent flags persist through all sub-commands
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "artifact_config.yaml", "config file (default is ./artifact_config.yaml)")
+
+	rootCmd.AddCommand(viper.GetConfigCommand())
+	rootCmd.AddCommand(NewServeCmd(rootCmd.Use, grpcHook, httpHook))
+
+	// Allow viper to read the value of the flags
+	configAccessor.InitializePflags(rootCmd.PersistentFlags())
+
+	err := flag.CommandLine.Parse([]string{})
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(-1)
+	}
+}