Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

use buf for builds and refactor build process #24

Merged
merged 2 commits into from
Nov 16, 2023
Merged

use buf for builds and refactor build process #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
17243c8
use buf for builds and refactor build process
labkode Nov 16, 2023
92022f9
add verbosity to build chain
labkode Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 0 additions & 33 deletions .drone.yml
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
cs3apis-build
build
17 changes: 1 addition & 16 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,14 @@ RUN apt-get update
RUN apt-get install build-essential curl unzip sudo -y
RUN apt-get install python3-pip python3-full -y

# deps for protoc
RUN cd /tmp && curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v25.0/protoc-25.0-linux-x86_64.zip -o protoc.zip && unzip -o protoc.zip && sudo cp bin/protoc /usr/local/bin/protoc
RUN cd /tmp && curl -sSL https://github.com/uber/prototool/releases/download/v1.10.0/prototool-Linux-x86_64 -o prototool && sudo cp prototool /usr/local/bin/ && sudo chmod u+x /usr/local/bin/prototool
RUN cd /tmp && curl -sSL https://github.com/nilslice/protolock/releases/download/v0.16.0/protolock.20220302T184110Z.linux-amd64.tgz -o protolock.tgz && tar -xzf protolock.tgz && sudo cp protolock /usr/local/bin/
RUN cd /tmp && curl -sSL https://github.com/pseudomuto/protoc-gen-doc/releases/download/v1.5.1/protoc-gen-doc_1.5.1_linux_amd64.tar.gz -o protoc-gen-doc.tar.gz && tar xzfv protoc-gen-doc.tar.gz && sudo cp protoc-gen-doc /usr/local/bin/
RUN go install github.com/golang/protobuf/[email protected]


# deps for python
RUN pip install grpcio grpcio-tools --ignore-installed --break-system-packages

# deps for js
RUN curl -sSL https://github.com/grpc/grpc-web/releases/download/1.5.0/protoc-gen-grpc-web-1.5.0-linux-x86_64 -o /tmp/protoc-gen-grpc-web
RUN sudo mv /tmp/protoc-gen-grpc-web /usr/local/bin/ && sudo chmod u+x /usr/local/bin/protoc-gen-grpc-web

# deps for node.js
RUN sudo apt-get update && sudo apt-get install -y ca-certificates curl gnupg
RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
RUN apt-get update
RUN sudo apt-get install nodejs -y
RUN node -v
RUN npm install protoc-gen-grpc -g -unsafe-perm
RUN npm install -g @bufbuild/buf

# compile build tool and put it into path
ADD . /root/cs3apis-build
Expand Down
263 changes: 42 additions & 221 deletions build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,47 +19,14 @@ var (
gitName = flag.String("git-author-name", "cs3org-bot", "Git author name")
gitSSH = flag.Bool("git-ssh", false, "Use git protocol instead of https for cloning repos")

_only_build = flag.Bool("only-build", false, "Build all protos and languages but do not push to language repos")
_all = flag.Bool("all", false, "Compile, build and publish for all available languages, mean to be run in CI platform like Drone")

_buildProto = flag.Bool("build-proto", false, "Compile Protobuf definitions")

_buildGo = flag.Bool("build-go", false, "Build Go library")
_pushGo = flag.Bool("push-go", false, "Push Go library to github.com/cs3org/go-cs3apis")

_buildPython = flag.Bool("build-python", false, "Build Python library")
_pushPython = flag.Bool("push-python", false, "Push Python library to github.com/cs3org/python-cs3apis")

_buildJs = flag.Bool("build-js", false, "Build Js library")
_pushJs = flag.Bool("push-js", false, "Push Js library to github.com/cs3org/js-cs3apis")

_buildNode = flag.Bool("build-node", false, "Build Node.js library")
_pushNode = flag.Bool("push-node", false, "Push Node.js library to github.com/cs3org/node-cs3apis")
_pushGo = flag.Bool("push-go", false, "Push Go library to github.com/cs3org/go-cs3apis")
_pushPython = flag.Bool("push-python", false, "Push Python library to github.com/cs3org/python-cs3apis")
_pushJs = flag.Bool("push-js", false, "Push Js library to github.com/cs3org/js-cs3apis")
_pushNode = flag.Bool("push-node", false, "Push Node.js library to github.com/cs3org/node-cs3apis")
)

func init() {
flag.Parse()

if *_all {
*_buildProto = true
*_buildGo = true
*_buildPython = true
*_buildJs = true
*_buildNode = true

*_pushGo = true
*_pushPython = true
*_pushJs = true
*_pushNode = true
}

if *_only_build {
*_buildProto = true
*_buildGo = true
*_buildPython = true
*_buildJs = true
*_buildNode = true
}
}

func getProtoOS() string {
Expand Down Expand Up @@ -280,185 +247,62 @@ func findFolders() []string {
return folders
}

func buildProto() {
dir := "."
cmd := exec.Command("prototool", "compile", "--walk-timeout", "10s")
cmd.Dir = dir
run(cmd)

cmd = exec.Command("protolock", "status")
cmd.Dir = dir
run(cmd)

// lint
cmd = exec.Command("prototool", "format", "-w", "--walk-timeout", "10s")
cmd.Dir = dir
run(cmd)
cmd = exec.Command("prototool", "lint", "--walk-timeout", "10s")
cmd.Dir = dir
run(cmd)
cmd = exec.Command("go", "run", "tools/check-license/check-license.go")
cmd.Dir = dir
run(cmd)

os.RemoveAll("docs")
os.MkdirAll("docs", 0755)

files := findProtos()
fmt.Println(files)

args := []string{"--doc_out=./docs", "--doc_opt=html,index.html", "-I=.", "-I=./third_party"}
args = append(args, files...)
cmd = exec.Command("protoc", args...)
run(cmd)
}

func buildGo() {

// Remove build dir
os.RemoveAll("build/go-cs3apis")
os.MkdirAll("build", 0755)

// Clone Go repo and set branch to current branch
clone("cs3org/go-cs3apis", "build")
protoBranch := getGitBranch(".")
goBranch := getGitBranch("build/go-cs3apis")
fmt.Printf("Proto branch: %s\nGo branch: %s\n", protoBranch, goBranch)

if goBranch != protoBranch {
checkout(protoBranch, "build/go-cs3apis")
func generate() {
cwd, err := os.Getwd()
if err != nil {
panic(err)
}
fmt.Println("Starting generation of protobuf language bindings ...")
fmt.Printf("current working directory: %s\n", cwd)

// remove leftovers (existing defs)
os.RemoveAll("build/go-cs3apis/cs3")

cmd := exec.Command("prototool", "generate", "--walk-timeout", "10s")
cmd := exec.Command("git", "config", "--global", "--add", "safe.directory", cwd)
run(cmd)

sed("build/go-cs3apis", ".go", "github.com/cs3org/go-cs3apis/build/go-cs3apis/cs3/", "github.com/cs3org/go-cs3apis/cs3/")

if !isRepoDirty("build/go-cs3apis") {
fmt.Println("Repo is clean, nothing to do")
}

// get proto repo commit id
hash := getCommitID(".")
repo := "build/go-cs3apis"
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}

func buildPython() {

// Remove build dir
os.RemoveAll("build/python-cs3apis")
os.RemoveAll("build")
os.MkdirAll("build", 0755)

// Clone Go repo and set branch to current branch
clone("cs3org/python-cs3apis", "build")
protoBranch := getGitBranch(".")
buildBranch := getGitBranch("build/python-cs3apis")
fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)

if buildBranch != protoBranch {
checkout(protoBranch, "build/python-cs3apis")
}

// remove leftovers (existing defs)
os.RemoveAll("build/python-cs3apis/cs3")

files := findProtos()

args := []string{"-m", "grpc_tools.protoc", "--python_out=./build/python-cs3apis", "-I.", "-I./third_party", "--grpc_python_out=./build/python-cs3apis"}
args = append(args, files...)
cmd := exec.Command("python3", args...)
run(cmd)

modules := findFolders()

var initFiles []string
for _, f := range modules {
initPy := fmt.Sprintf("%s/%s/%s", "build/python-cs3apis", f, "__init__.py")
initFiles = append(initFiles, initPy)
}
languages := []string{"go", "js", "node", "python"}

cmd = exec.Command("touch", initFiles...)
run(cmd)
// prepare language git repos
for _, l := range languages {
target := fmt.Sprintf("%s-cs3apis", l)
fmt.Println("cloning repo for " + target)

// get proto repo commit id
hash := getCommitID(".")
repo := "build/python-cs3apis"
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}
// Clone Go repo and set branch to current branch
clone("cs3org/"+target, "build")
protoBranch := getGitBranch(".")
targetBranch := getGitBranch("build/" + target)
fmt.Printf("Proto branch: %s\n%s branch: %s\n", l, protoBranch, targetBranch)

func buildJS() {
// Remove build dir
os.RemoveAll("build/js-cs3apis")
os.MkdirAll("build", 0755)
if targetBranch != protoBranch {
checkout(protoBranch, "build/"+target)
}

// Clone repo and set branch to current branch
clone("cs3org/js-cs3apis", "build")
protoBranch := getGitBranch(".")
buildBranch := getGitBranch("build/js-cs3apis")
fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)
// remove leftovers (existing defs)
os.RemoveAll(fmt.Sprintf("build/%s/cs3", target))

if buildBranch != protoBranch {
checkout(protoBranch, "build/js-cs3apis")
}

// remove leftovers (existing defs)
os.RemoveAll("build/js-cs3apis/cs3")

files := findProtos()

args := []string{"--js_out=import_style=commonjs:./build/js-cs3apis", "--grpc-web_out=import_style=commonjs,mode=grpcwebtext:./build/js-cs3apis/", "-I.", "-I./third_party"}
args = append(args, files...)
cmd := exec.Command("protoc", args...)
fmt.Println("Generating ...")
cmd = exec.Command("buf", "generate")
run(cmd)

// get proto repo commit id
hash := getCommitID(".")
repo := "build/js-cs3apis"
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}
for _, l := range languages {
target := fmt.Sprintf("%s-cs3apis", l)
fmt.Println("Commiting changes for " + target)

func buildNode() {
// Remove build dir
os.RemoveAll("build/node-cs3apis")
os.MkdirAll("build", 0755)

// Clone repo and set branch to current branch
clone("cs3org/node-cs3apis", "build")
protoBranch := getGitBranch(".")
buildBranch := getGitBranch("build/node-cs3apis")
fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)
if !isRepoDirty("build/" + target) {
fmt.Println("Repo is clean, nothing to do")
}

if buildBranch != protoBranch {
checkout(protoBranch, "build/node-cs3apis")
// get proto repo commit id
hash := getCommitID(".")
repo := "build/" + target
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}

// remove leftovers (existing defs)
os.RemoveAll("build/node-cs3apis/cs3")

files := findProtos()

args1 := []string{"--ts_out=grpc_js:./build/node-cs3apis", "--proto_path=.", "--proto_path=./third_party"}
args1 = append(args1, files...)
cmd1 := exec.Command("protoc-gen-grpc-ts", args1...)
run(cmd1)

args2 := []string{"--js_out=import_style=commonjs,binary:./build/node-cs3apis", "--grpc_out=grpc_js:./build/node-cs3apis/", "--proto_path=.", "--proto_path=./third_party"}
args2 = append(args2, files...)
cmd2 := exec.Command("protoc-gen-grpc", args2...)
run(cmd2)

// get proto repo commit id
hash := getCommitID(".")
repo := "build/node-cs3apis"
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
fmt.Println("Generation done!")
}

func pushPython() {
Expand All @@ -478,46 +322,23 @@ func pushNode() {
}

func main() {
if *_buildProto {
fmt.Println("Compiling and linting protobufs ...")
buildProto()
}

if *_buildGo {
fmt.Println("Building Go ...")
buildGo()
}
generate()

if *_pushGo {
fmt.Println("Pushing Go ...")
pushGo()
}

if *_buildPython {
fmt.Println("Building Python ...")
buildPython()
}

if *_pushPython {
fmt.Println("Pushing Python ...")
pushPython()
}

if *_buildJs {
fmt.Println("Building JS ...")
buildJS()
}

if *_pushJs {
fmt.Println("Pushing Js ...")
pushJS()
}

if *_buildNode {
fmt.Println("Building Node.js ...")
buildNode()
}

if *_pushNode {
fmt.Println("Pushing Node.js ...")
pushNode()
Expand Down