Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge kinesis fixes into feature branch #155

Merged
merged 2 commits into from
Dec 12, 2024
Merged

Merge kinesis fixes into feature branch #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7fba240
Fix Kinesis checkpointing and other issues.
gamolina Dec 9, 2024
f4e9c4d
Reversed the AWS v2 upgrade until we can figure out the retry tokens …
gamolina Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Docker/kinesis_entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ fi
if [ -n "$CHECKPOINT_TABLE" ]
then
BOOL_FLAGS=${BOOL_FLAGS}" --checkpoint-table=${CHECKPOINT_TABLE}"
if [ -n "$POST_CHECKPOINT_INIT_DELAY" ]
then
BOOL_FLAGS=${BOOL_FLAGS}" --post-checkpoint-init-delay=${POST_CHECKPOINT_INIT_DELAY}"
fi
fi
if [ -n "$LOG_LEVEL" ]
then
Expand Down
18 changes: 0 additions & 18 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ K := $(foreach exec,$(EXECUTABLES),\
ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
BIN_DIR=bin
BIN_NODE=quanta-node
BIN_LOADER=quanta-loader
BIN_PROXY=quanta-proxy
BIN_KINESIS=quanta-kinesis-consumer
BIN_PRODUCER=quanta-s3-kinesis-producer
Expand Down Expand Up @@ -123,12 +122,6 @@ push_proxy_docker: build_all
docker buildx build --push --platform linux/${GOARCH} -t containerregistry.disney.com/digital/$(BIN_PROXY):${VERSION}-$(GOARCH) --build-arg arch="${GOARCH}" --build-arg platform="${PLATFORM}" -f Docker/DeployProxyDockerfile . \
))

push_loader_docker: build_all
$(foreach GOARCH, $(ARCHITECTURES),\
$(shell export GOARCH=$(GOARCH);\
docker buildx build --push --platform linux/${GOARCH} -t containerregistry.disney.com/digital/$(BIN_LOADER):${VERSION}-$(GOARCH) --build-arg arch="${GOARCH}" --build-arg platform="${PLATFORM}" -f Docker/DeployLoaderDockerfile . \
))

build_proxy_docker: build_all
$(foreach GOARCH, $(ARCHITECTURES),\
$(shell export GOARCH=$(GOARCH);\
Expand Down Expand Up @@ -163,17 +156,6 @@ kcl:
admin:
CGO_ENABLED=0 go build -o ${BIN_DIR}/${BIN_ADMIN} ${LDFLAGS} ${PKG_ADMIN}

loader:
$(foreach GOARCH, $(ARCHITECTURES),\
$(shell export GOARCH=$(GOARCH);\
CGO_ENABLED=0 go build -o $(BIN_DIR)/$(BIN_LOADER)-$(PLATFORM)-$(GOARCH) ${LDFLAGS} ${PKG_LOADER} \
))
$(foreach GOARCH, $(ARCHITECTURES),\
$(shell export GOARCH=$(GOARCH);\
docker buildx build --push --platform linux/${GOARCH} -t containerregistry.disney.com/digital/$(BIN_LOADER):${VERSION}-$(GOARCH) --build-arg arch="${GOARCH}" --build-arg platform="${PLATFORM}" -f Docker/DeployLoaderDockerfile . \
))


producer:
CGO_ENABLED=0 go build -o ${BIN_DIR}/${BIN_PRODUCER} ${LDFLAGS} ${PKG_PRODUCER}
docker build -t containerregistry.disney.com/digital/quanta-s3-producer -f Docker/DeployProducerDockerfile .
Expand Down
13 changes: 11 additions & 2 deletions core/builtins.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1005,6 +1005,9 @@ func (m UUIDBSIMapper) MapValue(attr *Attribute, val interface{},
err = m.MutateBitmap(c, attr.Parent.Name, attr.FieldName, nil, false)
}
return
case int64:
v, _ := val.(int64)
result = big.NewInt(v)
default:
err = fmt.Errorf("%s: No handling for type '%T'", m.String(), val)
}
Expand All @@ -1018,9 +1021,15 @@ func (m UUIDBSIMapper) Render(attr *Attribute, value interface{}) string {
if val, ok := value.(*big.Int); ok {
switch shared.TypeFromString(attr.Type) {
case shared.String:
nuuid, _ := endian.FromBytes(val.Bytes())
b := val.Bytes()
if len(b) == 15 { // Must be a 16 byte buffer
b = append([]byte{0}, b...)
}
nuuid, _ := endian.FromBytes(b)
middleEndian, _ := nuuid.MiddleEndianBytes()
if newUUID, err := uuid.FromBytes(middleEndian); err == nil {
if newUUID, err := uuid.FromBytes(middleEndian); err != nil {
return fmt.Sprintf("ERR = %v", err)
} else {
return newUUID.String()
}
/*
Expand Down
62 changes: 35 additions & 27 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ require (
github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c
github.com/aviddiviner/go-murmur v0.0.0-20150519214947-b9740d71e571
github.com/aws/aws-sdk-go v1.44.146
github.com/aws/aws-sdk-go-v2 v1.16.4
github.com/aws/aws-sdk-go-v2/config v1.15.9
github.com/aws/aws-sdk-go-v2/credentials v1.12.4
github.com/aws/aws-sdk-go-v2/service/kinesis v1.9.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.10
github.com/aws/aws-sdk-go-v2/service/sts v1.16.6
github.com/aws/aws-sdk-go-v2 v1.32.6
github.com/aws/aws-sdk-go-v2/config v1.28.6
github.com/aws/aws-sdk-go-v2/credentials v1.17.47
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.43.3
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.38.0
github.com/aws/aws-sdk-go-v2/service/kinesis v1.32.7
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2
github.com/banzaicloud/logrus-runtime-formatter v0.0.0-20190729070250-5ae5475bae5e
github.com/bbalet/stopwords v1.0.0
github.com/bufbuild/protocompile v0.14.1
Expand Down Expand Up @@ -46,21 +48,20 @@ require (
github.com/prometheus/client_golang v1.11.0
github.com/rlmcpherson/s3gof3r v0.5.0
github.com/siddontang/go-mysql v1.1.0
github.com/sirupsen/logrus v1.8.1
github.com/sirupsen/logrus v1.9.0
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570
github.com/stretchr/testify v1.9.0
github.com/stvp/rendezvous v0.0.0-20151118195501-67b5f26b3e18
github.com/vmware/vmware-go-kcl v1.5.0
github.com/xitongsys/parquet-go v1.5.5-0.20201031234703-4d9f11317375
github.com/xitongsys/parquet-go-source v0.0.0-20220527110425-ba4adb87a31b
github.com/xitongsys/parquet-go v1.6.2
github.com/xitongsys/parquet-go-source v0.0.0-20241021075129-b732d2ac9c9b
golang.org/x/net v0.10.0
golang.org/x/sync v0.8.0
golang.org/x/text v0.9.0
google.golang.org/api v0.103.0
google.golang.org/grpc v1.53.0
google.golang.org/protobuf v1.34.2
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/confluentinc/confluent-kafka-go.v1 v1.4.2
gopkg.in/yaml.v2 v2.4.0
)

Expand All @@ -72,31 +73,35 @@ require (
cloud.google.com/go/storage v1.28.0 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect
github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
github.com/apache/thrift v0.14.2 // indirect
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.7 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.20 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.12.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/confluentinc/confluent-kafka-go v1.4.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/fatih/color v1.9.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
Expand All @@ -109,7 +114,7 @@ require (
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/serf v0.9.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.2 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
Expand All @@ -124,6 +129,7 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.8 // indirect
github.com/pingcap/errors v0.11.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand All @@ -144,3 +150,5 @@ require (
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

//replace github.com/harlow/kinesis-consumer => github.com/guymolinari/kinesis-consumer v0.0.0-20241208210820-a50a0f6e2b44
Loading
Loading