Merge pull request #14496 from transcom/prod-fix-pdfcpu-int

B-22137 & B-22170 update pdfcpu 0.6.0 to latest 0.9.1

Dockerfile

@@ -7,10 +7,16 @@ RUN apt-get update
 RUN apt-get install -y ca-certificates --no-install-recommends
 RUN update-ca-certificates
 
+## Give full perms to tmp for the server to have a read/write location
+RUN mkdir -p /tmp && chmod 777 /tmp
+
 # hadolint ignore=DL3007
 FROM gcr.io/distroless/base-debian11@sha256:ac69aa622ea5dcbca0803ca877d47d069f51bd4282d5c96977e0390d7d256455
 COPY --from=build-env /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 
+# Pull in full perms tmp
+COPY --from=build-env --chown=nonroot:nonroot /tmp /tmp
+
 COPY bin/rds-ca-rsa4096-g1.pem /bin/rds-ca-rsa4096-g1.pem
 COPY bin/rds-ca-2019-root.pem /bin/rds-ca-2019-root.pem
 COPY bin/milmove /bin/milmove
@@ -23,6 +29,9 @@ COPY swagger/* /swagger/
 COPY build /build
 COPY public/static/react-file-viewer /public/static/react-file-viewer
 
+# Keep the tmp!
+VOLUME ["/tmp"]
+
 ENTRYPOINT ["/bin/milmove"]
 
 CMD ["serve", "--logging-level=debug"]

Dockerfile.dp3

@@ -1,3 +1,8 @@
+FROM debian:stable AS build-env
+
+## Give full perms to tmp for the server to have a read/write location
+RUN mkdir -p /tmp && chmod 777 /tmp
+
 # hadolint ignore=DL3007
 FROM gcr.io/distroless/base-debian11@sha256:ac69aa622ea5dcbca0803ca877d47d069f51bd4282d5c96977e0390d7d256455
 
@@ -6,6 +11,9 @@ COPY bin/rds-ca-rsa4096-g1.pem /bin/rds-ca-rsa4096-g1.pem
 
 COPY bin/milmove /bin/milmove
 
+# Pull in full perms tmp
+COPY --from=build-env --chown=nonroot:nonroot /tmp /tmp
+
 # Demo Environment certs
 COPY config/tls/api.demo.dp3.us.chain.der.p7b /config/tls/api.demo.dp3.us.chain.der.p7b
 COPY config/tls/api.demo.dp3.us.crt /config/tls/api.demo.dp3.us.crt
@@ -27,6 +35,9 @@ COPY swagger/* /swagger/
 COPY build /build
 COPY public/static/react-file-viewer /public/static/react-file-viewer
 
+# Keep the tmp!
+VOLUME ["/tmp"]
+
 ENTRYPOINT ["/bin/milmove"]
 
 CMD ["serve", "--logging-level=debug"]

go.mod

@@ -62,7 +62,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/markbates/goth v1.79.0
 	github.com/namsral/flag v1.7.4-pre
-	github.com/pdfcpu/pdfcpu v0.6.0
+	github.com/pdfcpu/pdfcpu v0.9.1
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.7
 	github.com/pterm/pterm v0.12.79
@@ -222,7 +222,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
 	github.com/microcosm-cc/bluemonday v1.0.23 // indirect
@@ -237,7 +237,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/peterbourgon/diskv/v3 v3.0.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/fastuuid v1.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/rs/zerolog v1.29.0 // indirect
@@ -259,7 +259,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
-	golang.org/x/image v0.18.0 // indirect
+	golang.org/x/image v0.21.0 // indirect
 	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect

go.sum

@@ -473,8 +473,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@@ -511,8 +511,8 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 h1:pSCLCl6joCFRnjpeojzOpEYs4q7Vditq8fySFG5ap3Y=
 github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/pdfcpu/pdfcpu v0.6.0 h1:z4kARP5bcWa39TTYMcN/kjBnm7MvhTWjXgeYmkdAGMI=
-github.com/pdfcpu/pdfcpu v0.6.0/go.mod h1:kmpD0rk8YnZj0l3qSeGBlAB+XszHUgNv//ORH/E7EYo=
+github.com/pdfcpu/pdfcpu v0.9.1 h1:q8/KlBdHjkE7ZJU4ofhKG5Rjf7M6L324CVM6BMDySao=
+github.com/pdfcpu/pdfcpu v0.9.1/go.mod h1:fVfOloBzs2+W2VJCCbq60XIxc3yJHAZ0Gahv1oO0gyI=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU=
@@ -547,8 +547,8 @@ github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTK
 github.com/rickar/cal/v2 v2.1.13 h1:FENBPXxDPyL1OWGf9ZdpWGcEiGoSjt0UZED8VOxvK0c=
 github.com/rickar/cal/v2 v2.1.13/go.mod h1:/fdlMcx7GjPlIBibMzOM9gMvDBsrK+mOtRXdTzUqV/A=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -730,8 +730,8 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjs
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
-golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
+golang.org/x/image v0.21.0 h1:c5qV36ajHpdj4Qi0GnE0jUc/yuo33OLFaa0d+crTD5s=
+golang.org/x/image v0.21.0/go.mod h1:vUbsLavqK/W303ZroQQVKQ+Af3Yl6Uz1Ppu5J/cLz78=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=

pkg/paperwork/generator.go

@@ -2,11 +2,15 @@ package paperwork
 
 import (
 	"bytes"
+	_ "embed"
+	"fmt"
 	"image"
 	"image/color"
 	"image/jpeg"
 	"image/png"
 	"io"
+	"log"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -92,16 +96,38 @@ func convertTo8BitPNG(in io.Reader, out io.Writer) error {
 	return nil
 }
 
+// Identifies if a filepath directory is mutable
+// This is needed in order to write config and fonts to filesystem
+// as the pdfcpu package hard-code requires it at this time
+// for initial installation and for form filling
+func isDirMutable(path string) bool {
+	testFile := filepath.Join(path, "tmp")
+	file, err := os.Create(testFile)
+	if err != nil {
+		log.Printf("isDirMutable: failed for %s: %v\n", path, err)
+		return false
+	}
+	file.Close()
+	os.Remove(testFile) // Cleanup the test file, it is mutable here
+	return true
+}
+
 // NewGenerator creates a new Generator.
 func NewGenerator(uploader *uploader.Uploader) (*Generator, error) {
 	// Use in memory filesystem for generation. Purpose is to not write
 	// to hard disk due to restrictions in AWS storage. May need better long term solution.
 	afs := storage.NewMemory(storage.NewMemoryParams("", "")).FileSystem()
 
-	// Disable ConfiDir for AWS deployment purposes.
-	// PDFCPU will attempt to create temp dir using os.create(hard disk).This will prevent it.
-	api.DisableConfigDir()
-	pdfConfig := model.NewDefaultConfiguration()
+	tmpDir := os.TempDir()
+	if !isDirMutable(tmpDir) {
+		return nil, fmt.Errorf("tmp directory (%s) is not mutable, cannot configure default pdfcpu generator settings", tmpDir)
+	}
+	err := api.EnsureDefaultConfigAt(tmpDir)
+	if err != nil {
+		return nil, err
+	}
+
+	pdfConfig := api.LoadConfiguration() // As long as our config was set properly, this will load it and not create a new default config
 	pdfCPU := pdfCPUWrapper{Configuration: pdfConfig}
 
 	directory, err := afs.TempDir("", "generator")
@@ -699,7 +725,7 @@ func (g *Generator) FillPDFForm(jsonData []byte, templateReader io.ReadSeeker, f
 	// Fills form using the template reader with json reader, outputs to byte, to be saved to afero file.
 	formerr := api.FillForm(templateReader, readJSON, buf, conf)
 	if formerr != nil {
-		return nil, err
+		return nil, formerr
 	}
 
 	tempFile, err := g.newTempFileWithName(fileName) // Will use g.newTempFileWithName for proper memory usage, saves the new temp file with the fileName
@@ -729,6 +755,10 @@ func (g *Generator) LockPDFForm(templateReader io.ReadSeeker, fileName string) (
 	buf := new(bytes.Buffer)
 	// Reads all form fields on document as []form.Field
 	fields, err := api.FormFields(templateReader, conf)
+	if err != nil {
+		return nil, err
+	}
+
 	// Assembles them to the API's required []string
 	fieldList := make([]string, len(fields))
 	for i, field := range fields {
@@ -786,9 +816,41 @@ func (g *Generator) MergePDFFilesByContents(_ appcontext.AppContext, fileReaders
 	return mergedFile, nil
 }
 
+// Pdfcpu does not nil check watermarks as of version 0.9.1
+// This map allows us to preemptively nil check before calling the package
+func createMapOfOnlyWatermarkedPages(m map[int][]*model.Watermark) map[int][]*model.Watermark {
+	validMap := make(map[int][]*model.Watermark)
+	for page, wms := range m {
+		// Skip entries where the slice is nil or empty
+		if len(wms) == 0 {
+			continue
+		}
+
+		// Filter out nil pointers from the slice
+		validWms := []*model.Watermark{}
+		for _, wm := range wms {
+			if wm != nil {
+				validWms = append(validWms, wm)
+			}
+		}
+
+		// Only add the page to the valid map if the filtered slice is not empty
+		if len(validWms) > 0 {
+			validMap[page] = validWms
+		}
+	}
+	return validMap
+}
+
 func (g *Generator) AddWatermarks(inputFile afero.File, m map[int][]*model.Watermark) (afero.File, error) {
+	// Preemptive nil check for the map and its contents
+	watermarkMap := createMapOfOnlyWatermarkedPages(m)
+	if watermarkMap[0] == nil {
+		return nil, fmt.Errorf("no watermarks provided for generation")
+	}
+
 	buf := new(bytes.Buffer)
-	err := api.AddWatermarksSliceMap(inputFile, buf, m, g.pdfConfig)
+	err := api.AddWatermarksSliceMap(inputFile, buf, watermarkMap, g.pdfConfig)
 	if err != nil {
 		return nil, err
 	}

pkg/paperwork/generator_test.go

@@ -264,7 +264,7 @@ func (suite *PaperworkSuite) TestCreateMergedPDF() {
 	ctx, err := api.ReadContext(file, generator.pdfConfig)
 	suite.FatalNil(err)
 
-	err = validate.XRefTable(ctx.XRefTable)
+	err = validate.XRefTable(ctx)
 	suite.FatalNil(err)
 
 	suite.Equal(3, ctx.PageCount)
@@ -292,7 +292,7 @@ func (suite *PaperworkSuite) TestCreateMergedPDFByContents() {
 	ctx, err := api.ReadContext(file, generator.pdfConfig)
 	suite.FatalNil(err)
 
-	err = validate.XRefTable(ctx.XRefTable)
+	err = validate.XRefTable(ctx)
 	suite.FatalNil(err)
 
 	suite.Equal(2, ctx.PageCount)

pkg/services/ppmshipment/payment_packet_creator.go

@@ -3,12 +3,9 @@ package ppmshipment
 import (
 	"fmt"
 	"io"
-	"time"
 
 	"github.com/gofrs/uuid"
 	"github.com/pdfcpu/pdfcpu/pkg/pdfcpu"
-	"github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model"
-	"github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
@@ -142,26 +139,9 @@ func (p *paymentPacketCreator) Generate(appCtx appcontext.AppContext, ppmShipmen
 		return nil, fmt.Errorf("%s: %w", errMsgPrefix, err)
 	}
 
-	watermarks, err := buildWaterMarks(bookmarks, p.pdfGenerator)
-	if err != nil {
-		errMsgPrefix = fmt.Sprintf("%s: %s", errMsgPrefix, "failed to generate watermarks for PDF")
-		appCtx.Logger().Error(errMsgPrefix, zap.Error(err))
-		return nil, fmt.Errorf("%s: %w", errMsgPrefix, err)
-	}
-
-	// Apply bookmarks and watermarks based on flag
-	if addWatermarks && len(watermarks) > 0 {
-		pdfWithWatermarks, err := p.pdfGenerator.AddWatermarks(finalMergePdf, watermarks)
-		if err != nil {
-			errMsgPrefix = fmt.Sprintf("%s: %s", errMsgPrefix, "failed to add watermarks to PDF")
-			appCtx.Logger().Error(errMsgPrefix, zap.Error(err))
-			return nil, fmt.Errorf("%s: %w", errMsgPrefix, err)
-		}
-		if addBookmarks {
-			return p.pdfGenerator.AddPdfBookmarks(pdfWithWatermarks, bookmarks)
-		}
-		return pdfWithWatermarks, nil
-	}
+	// It was discovered during implementation of B-21938 that watermarks were not functional.
+	// This is because the watermark func was using bookmarks, not watermarks.
+	// See https://github.com/transcom/mymove/pull/14496 for removal
 
 	if addBookmarks {
 		return p.pdfGenerator.AddPdfBookmarks(finalMergePdf, bookmarks)
@@ -210,51 +190,6 @@ func buildBookMarks(fileNamesToMerge []string, sortedPaymentPacketItems map[int]
 	return bookmarks, nil
 }
 
-// generate watermarks which will serve as page footer labels
-func buildWaterMarks(bookMarks []pdfcpu.Bookmark, pdfGenerator paperwork.Generator) (map[int][]*model.Watermark, error) {
-	m := make(map[int][]*model.Watermark)
-
-	opacity := 1.0
-	onTop := true
-	update := false
-	unit := types.POINTS
-
-	desc := fmt.Sprintf("font:Times-Italic, points:10, sc:1 abs, pos:bc, off:0 8, rot:0, op:%f", opacity)
-
-	creationTimeStamp := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
-	totalPages := bookMarks[len(bookMarks)-1].PageThru
-	currentPage := 1
-	bookMarkIndex := 0
-	for _, bm := range bookMarks {
-		cnt := bm.PageThru - bm.PageFrom
-		for j := 0; j <= cnt; j++ {
-			// do not add watermark on SSW pages
-			if currentPage < 4 {
-				currentPage++
-				continue
-			}
-			wmText := bm.Title
-			// we really can't use the bookmark title for the SSW+Orders.
-			// we will just label it as only Orders
-			if currentPage > 3 && bookMarkIndex == 0 {
-				wmText = "Orders"
-			}
-			wms := make([]*model.Watermark, 0)
-			pagingInfo := fmt.Sprintf("Page %d of %d", currentPage, totalPages)
-			text := fmt.Sprintf("%s - Payment Packet[%s] (Creation Date: %v)", pagingInfo, wmText, creationTimeStamp)
-
-			wm, _ := pdfGenerator.CreateTextWatermark(text, desc, onTop, update, unit)
-			wms = append(wms, wm)
-			// note: use current page because map is 1 based
-			m[currentPage] = wms
-			currentPage++
-		}
-		bookMarkIndex++
-	}
-
-	return m, nil
-}
-
 func buildPaymentPacketItemsMap(ppmShipment *models.PPMShipment) map[int]paymentPacketItem {
 	// items are sorted based on key(int), key represents order index
 	sortedPaymentPacketItems := make(map[int]paymentPacketItem)