Skip to content

Commit

Permalink
use buf for builds and refactor build process
Browse files Browse the repository at this point in the history
  • Loading branch information
@labkode
labkode committed Nov 16, 2023
1 parent 210226c commit 17243c8
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 270 deletions.
33 changes: 0 additions & 33 deletions .drone.yml
View file

This file was deleted.

1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
cs3apis-build
build
17 changes: 1 addition & 16 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,14 @@ RUN apt-get update
RUN apt-get install build-essential curl unzip sudo -y
RUN apt-get install python3-pip python3-full -y

# deps for protoc
RUN cd /tmp && curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v25.0/protoc-25.0-linux-x86_64.zip -o protoc.zip && unzip -o protoc.zip && sudo cp bin/protoc /usr/local/bin/protoc
RUN cd /tmp && curl -sSL https://github.com/uber/prototool/releases/download/v1.10.0/prototool-Linux-x86_64 -o prototool && sudo cp prototool /usr/local/bin/ && sudo chmod u+x /usr/local/bin/prototool
RUN cd /tmp && curl -sSL https://github.com/nilslice/protolock/releases/download/v0.16.0/protolock.20220302T184110Z.linux-amd64.tgz -o protolock.tgz && tar -xzf protolock.tgz && sudo cp protolock /usr/local/bin/
RUN cd /tmp && curl -sSL https://github.com/pseudomuto/protoc-gen-doc/releases/download/v1.5.1/protoc-gen-doc_1.5.1_linux_amd64.tar.gz -o protoc-gen-doc.tar.gz && tar xzfv protoc-gen-doc.tar.gz && sudo cp protoc-gen-doc /usr/local/bin/
RUN go install github.com/golang/protobuf/[email protected]


# deps for python
RUN pip install grpcio grpcio-tools --ignore-installed --break-system-packages

# deps for js
RUN curl -sSL https://github.com/grpc/grpc-web/releases/download/1.5.0/protoc-gen-grpc-web-1.5.0-linux-x86_64 -o /tmp/protoc-gen-grpc-web
RUN sudo mv /tmp/protoc-gen-grpc-web /usr/local/bin/ && sudo chmod u+x /usr/local/bin/protoc-gen-grpc-web

# deps for node.js
RUN sudo apt-get update && sudo apt-get install -y ca-certificates curl gnupg
RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
RUN apt-get update
RUN sudo apt-get install nodejs -y
RUN node -v
RUN npm install protoc-gen-grpc -g -unsafe-perm
RUN npm install -g @bufbuild/buf

# compile build tool and put it into path
ADD . /root/cs3apis-build
Expand Down
258 changes: 37 additions & 221 deletions build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,47 +19,14 @@ var (
gitName = flag.String("git-author-name", "cs3org-bot", "Git author name")
gitSSH = flag.Bool("git-ssh", false, "Use git protocol instead of https for cloning repos")

_only_build = flag.Bool("only-build", false, "Build all protos and languages but do not push to language repos")
_all = flag.Bool("all", false, "Compile, build and publish for all available languages, mean to be run in CI platform like Drone")

_buildProto = flag.Bool("build-proto", false, "Compile Protobuf definitions")

_buildGo = flag.Bool("build-go", false, "Build Go library")
_pushGo = flag.Bool("push-go", false, "Push Go library to github.com/cs3org/go-cs3apis")

_buildPython = flag.Bool("build-python", false, "Build Python library")
_pushPython = flag.Bool("push-python", false, "Push Python library to github.com/cs3org/python-cs3apis")

_buildJs = flag.Bool("build-js", false, "Build Js library")
_pushJs = flag.Bool("push-js", false, "Push Js library to github.com/cs3org/js-cs3apis")

_buildNode = flag.Bool("build-node", false, "Build Node.js library")
_pushNode = flag.Bool("push-node", false, "Push Node.js library to github.com/cs3org/node-cs3apis")
_pushGo = flag.Bool("push-go", false, "Push Go library to github.com/cs3org/go-cs3apis")
_pushPython = flag.Bool("push-python", false, "Push Python library to github.com/cs3org/python-cs3apis")
_pushJs = flag.Bool("push-js", false, "Push Js library to github.com/cs3org/js-cs3apis")
_pushNode = flag.Bool("push-node", false, "Push Node.js library to github.com/cs3org/node-cs3apis")
)

func init() {
flag.Parse()

if *_all {
*_buildProto = true
*_buildGo = true
*_buildPython = true
*_buildJs = true
*_buildNode = true

*_pushGo = true
*_pushPython = true
*_pushJs = true
*_pushNode = true
}

if *_only_build {
*_buildProto = true
*_buildGo = true
*_buildPython = true
*_buildJs = true
*_buildNode = true
}
}

func getProtoOS() string {
Expand Down Expand Up @@ -280,185 +247,57 @@ func findFolders() []string {
return folders
}

func buildProto() {
dir := "."
cmd := exec.Command("prototool", "compile", "--walk-timeout", "10s")
cmd.Dir = dir
run(cmd)

cmd = exec.Command("protolock", "status")
cmd.Dir = dir
run(cmd)

// lint
cmd = exec.Command("prototool", "format", "-w", "--walk-timeout", "10s")
cmd.Dir = dir
run(cmd)
cmd = exec.Command("prototool", "lint", "--walk-timeout", "10s")
cmd.Dir = dir
run(cmd)
cmd = exec.Command("go", "run", "tools/check-license/check-license.go")
cmd.Dir = dir
run(cmd)

os.RemoveAll("docs")
os.MkdirAll("docs", 0755)

files := findProtos()
fmt.Println(files)

args := []string{"--doc_out=./docs", "--doc_opt=html,index.html", "-I=.", "-I=./third_party"}
args = append(args, files...)
cmd = exec.Command("protoc", args...)
run(cmd)
}

func buildGo() {

// Remove build dir
os.RemoveAll("build/go-cs3apis")
os.MkdirAll("build", 0755)

// Clone Go repo and set branch to current branch
clone("cs3org/go-cs3apis", "build")
protoBranch := getGitBranch(".")
goBranch := getGitBranch("build/go-cs3apis")
fmt.Printf("Proto branch: %s\nGo branch: %s\n", protoBranch, goBranch)

if goBranch != protoBranch {
checkout(protoBranch, "build/go-cs3apis")
func generate() {
cwd, err := os.Getwd()
if err != nil {
panic(err)
}
fmt.Printf("current working directory: %s\n", cwd)

// remove leftovers (existing defs)
os.RemoveAll("build/go-cs3apis/cs3")

cmd := exec.Command("prototool", "generate", "--walk-timeout", "10s")
cmd := exec.Command("git", "config", "--global", "--add", "safe.directory", cwd)
run(cmd)

sed("build/go-cs3apis", ".go", "github.com/cs3org/go-cs3apis/build/go-cs3apis/cs3/", "github.com/cs3org/go-cs3apis/cs3/")

if !isRepoDirty("build/go-cs3apis") {
fmt.Println("Repo is clean, nothing to do")
}

// get proto repo commit id
hash := getCommitID(".")
repo := "build/go-cs3apis"
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}

func buildPython() {

// Remove build dir
os.RemoveAll("build/python-cs3apis")
os.RemoveAll("build")
os.MkdirAll("build", 0755)

// Clone Go repo and set branch to current branch
clone("cs3org/python-cs3apis", "build")
protoBranch := getGitBranch(".")
buildBranch := getGitBranch("build/python-cs3apis")
fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)

if buildBranch != protoBranch {
checkout(protoBranch, "build/python-cs3apis")
}

// remove leftovers (existing defs)
os.RemoveAll("build/python-cs3apis/cs3")

files := findProtos()

args := []string{"-m", "grpc_tools.protoc", "--python_out=./build/python-cs3apis", "-I.", "-I./third_party", "--grpc_python_out=./build/python-cs3apis"}
args = append(args, files...)
cmd := exec.Command("python3", args...)
run(cmd)

modules := findFolders()

var initFiles []string
for _, f := range modules {
initPy := fmt.Sprintf("%s/%s/%s", "build/python-cs3apis", f, "__init__.py")
initFiles = append(initFiles, initPy)
}
languages := []string{"go", "js", "node", "python"}

cmd = exec.Command("touch", initFiles...)
run(cmd)
// prepare language git repos
for _, l := range languages {
target := fmt.Sprintf("%s-cs3apis", l)

// get proto repo commit id
hash := getCommitID(".")
repo := "build/python-cs3apis"
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}
// Clone Go repo and set branch to current branch
clone("cs3org/"+target, "build")
protoBranch := getGitBranch(".")
targetBranch := getGitBranch("build/" + target)
fmt.Printf("Proto branch: %s\n%s branch: %s\n", l, protoBranch, targetBranch)

func buildJS() {
// Remove build dir
os.RemoveAll("build/js-cs3apis")
os.MkdirAll("build", 0755)
if targetBranch != protoBranch {
checkout(protoBranch, "build/"+target)
}

// Clone repo and set branch to current branch
clone("cs3org/js-cs3apis", "build")
protoBranch := getGitBranch(".")
buildBranch := getGitBranch("build/js-cs3apis")
fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)
// remove leftovers (existing defs)
os.RemoveAll(fmt.Sprintf("build/%s/cs3", target))

if buildBranch != protoBranch {
checkout(protoBranch, "build/js-cs3apis")
}

// remove leftovers (existing defs)
os.RemoveAll("build/js-cs3apis/cs3")

files := findProtos()

args := []string{"--js_out=import_style=commonjs:./build/js-cs3apis", "--grpc-web_out=import_style=commonjs,mode=grpcwebtext:./build/js-cs3apis/", "-I.", "-I./third_party"}
args = append(args, files...)
cmd := exec.Command("protoc", args...)
cmd = exec.Command("buf", "generate")
run(cmd)

// get proto repo commit id
hash := getCommitID(".")
repo := "build/js-cs3apis"
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}
for _, l := range languages {
target := fmt.Sprintf("%s-cs3apis", l)

func buildNode() {
// Remove build dir
os.RemoveAll("build/node-cs3apis")
os.MkdirAll("build", 0755)

// Clone repo and set branch to current branch
clone("cs3org/node-cs3apis", "build")
protoBranch := getGitBranch(".")
buildBranch := getGitBranch("build/node-cs3apis")
fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)
if !isRepoDirty("build/" + target) {
fmt.Println("Repo is clean, nothing to do")
}

if buildBranch != protoBranch {
checkout(protoBranch, "build/node-cs3apis")
// get proto repo commit id
hash := getCommitID(".")
repo := "build/" + target
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}

// remove leftovers (existing defs)
os.RemoveAll("build/node-cs3apis/cs3")

files := findProtos()

args1 := []string{"--ts_out=grpc_js:./build/node-cs3apis", "--proto_path=.", "--proto_path=./third_party"}
args1 = append(args1, files...)
cmd1 := exec.Command("protoc-gen-grpc-ts", args1...)
run(cmd1)

args2 := []string{"--js_out=import_style=commonjs,binary:./build/node-cs3apis", "--grpc_out=grpc_js:./build/node-cs3apis/", "--proto_path=.", "--proto_path=./third_party"}
args2 = append(args2, files...)
cmd2 := exec.Command("protoc-gen-grpc", args2...)
run(cmd2)

// get proto repo commit id
hash := getCommitID(".")
repo := "build/node-cs3apis"
msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
commit(repo, msg)
}

func pushPython() {
Expand All @@ -478,46 +317,23 @@ func pushNode() {
}

func main() {
if *_buildProto {
fmt.Println("Compiling and linting protobufs ...")
buildProto()
}

if *_buildGo {
fmt.Println("Building Go ...")
buildGo()
}
generate()

if *_pushGo {
fmt.Println("Pushing Go ...")
pushGo()
}

if *_buildPython {
fmt.Println("Building Python ...")
buildPython()
}

if *_pushPython {
fmt.Println("Pushing Python ...")
pushPython()
}

if *_buildJs {
fmt.Println("Building JS ...")
buildJS()
}

if *_pushJs {
fmt.Println("Pushing Js ...")
pushJS()
}

if *_buildNode {
fmt.Println("Building Node.js ...")
buildNode()
}

if *_pushNode {
fmt.Println("Pushing Node.js ...")
pushNode()
Expand Down

0 comments on commit 17243c8

Please sign in to comment.