Skip to content

Commit

Permalink
Remove dags in dockerignore (#1149)
Browse files Browse the repository at this point in the history
* Remove dags in dockerignore

* Fixing lint and tests
  • Loading branch information
kushalmalani committed Apr 11, 2023
1 parent 00e72eb commit cf1bf58
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 32 deletions.
82 changes: 50 additions & 32 deletions cloud/deploy/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,39 @@ func getRegistryURL(domain string) string {
return registry
}

func removeDagsFromDockerIgnore(fullpath string) error {
f, err := os.Open(fullpath)
if err != nil {
return err
}

defer f.Close()

var bs []byte
buf := bytes.NewBuffer(bs)

scanner := bufio.NewScanner(f)
for scanner.Scan() {
text := scanner.Text()
if text != "dags/" {
_, err = buf.WriteString(text + "\n")
if err != nil {
return err
}
}
}

if err := scanner.Err(); err != nil {
return err
}
err = os.WriteFile(fullpath, bytes.Trim(buf.Bytes(), "\n"), 0o666) //nolint:gosec, gomnd
if err != nil {
return err
}

return nil
}

func deployDags(path, dagsPath, runtimeID string, client astro.Client) (string, error) {
// Check the dags directory
monitoringDagPath := filepath.Join(dagsPath, "astronomer_monitoring_dag.py")
Expand Down Expand Up @@ -258,6 +291,11 @@ func Deploy(deployInput InputDeploy, client astro.Client) error { //nolint
fmt.Sprintf("\n Deployment View: %s", ansi.Bold(deploymentURL)) +
fmt.Sprintf("\n Airflow UI: %s", ansi.Bold(deployInfo.webserverURL)))
} else {
fullpath := filepath.Join(deployInput.Path, ".dockerignore")
err := removeDagsFromDockerIgnore(fullpath)
if err != nil {
return errors.New("Found dags entry in .dockerignore file. Remove this entry and try again")
}
envFileExists, _ := fileutil.Exists(deployInput.EnvFile, nil)
if !envFileExists && deployInput.EnvFile != ".env" {
return fmt.Errorf("%w %s", envFileMissing, deployInput.EnvFile)
Expand Down Expand Up @@ -477,6 +515,17 @@ func buildImageWithoutDags(path string, imageHandler airflow.ImageHandler) error
dockerIgnoreCreate := false
fullpath := filepath.Join(path, ".dockerignore")

defer func() {
// remove dags from .dockerignore file if we set it
if dagsIgnoreSet {
removeDagsFromDockerIgnore(fullpath) //nolint:errcheck
}
// remove created docker ignore file
if dockerIgnoreCreate {
os.Remove(fullpath)
}
}()

fileExist, _ := fileutil.Exists(fullpath, nil)
if !fileExist {
// Create a dockerignore file and add the dags folder entry
Expand Down Expand Up @@ -510,40 +559,9 @@ func buildImageWithoutDags(path string, imageHandler airflow.ImageHandler) error
return err
}

defer func() {
// remove created docker ignore file
if dockerIgnoreCreate {
os.Remove(fullpath)
}
}()

// remove dags from .dockerignore file if we set it
if dagsIgnoreSet {
f, err := os.Open(fullpath)
if err != nil {
return err
}

defer f.Close()

var bs []byte
buf := bytes.NewBuffer(bs)

scanner := bufio.NewScanner(f)
for scanner.Scan() {
text := scanner.Text()
if text != "dags/" {
_, err = buf.WriteString(text + "\n")
if err != nil {
return err
}
}
}

if err := scanner.Err(); err != nil {
return err
}
err = os.WriteFile(fullpath, bytes.Trim(buf.Bytes(), "\n"), 0o666) //nolint:gosec, gomnd
err = removeDagsFromDockerIgnore(fullpath)
if err != nil {
return err
}
Expand Down
2 changes: 2 additions & 0 deletions cloud/deploy/deploy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,8 @@ func TestDeployWithDagsDeploySuccess(t *testing.T) {
os.Mkdir("./testfiles1/", os.ModePerm)
fileutil.WriteStringToFile("./testfiles1/Dockerfile", "FROM quay.io/astronomer/astro-runtime:4.2.5")
fileutil.WriteStringToFile("./testfiles1/.env", "")
fileutil.WriteStringToFile("./testfiles1/.dockerignore", "")

deployInput = InputDeploy{
Path: "./testfiles1/",
RuntimeID: "",
Expand Down

0 comments on commit cf1bf58

Please sign in to comment.