From aa904f44cfb2bb371200426b0e7d4295cff49313 Mon Sep 17 00:00:00 2001 From: apocelipes Date: Tue, 22 Oct 2024 08:09:52 +0800 Subject: [PATCH] refactor: using standard libraries to do file operations (#542) --- SCC-OUTPUT-REPORT.html | 38 ++++++++++++++--------------- cmd/badges/example.py | 26 ++++++++++---------- cmd/badges/main.go | 54 ++++++++++++++---------------------------- 3 files changed, 51 insertions(+), 67 deletions(-) diff --git a/SCC-OUTPUT-REPORT.html b/SCC-OUTPUT-REPORT.html index bd0f001a1..08daaacf8 100644 --- a/SCC-OUTPUT-REPORT.html +++ b/SCC-OUTPUT-REPORT.html @@ -13,12 +13,12 @@ Go 27 - 9513 - 1458 + 9495 + 1452 444 - 7611 - 1501 - 252876 + 7599 + 1502 + 252699 4061 processor/workers_test.go @@ -67,9 +67,9 @@ 142 104 430 - 93 - 19515 - 441 + 94 + 19510 + 440 main.go @@ -93,13 +93,13 @@ cmd/badges/main.go - 345 - 59 + 327 + 53 14 - 272 + 260 50 - 8129 - 204 + 7957 + 206 processor/workers_tokei_test.go @@ -294,15 +294,15 @@ Total 27 - 9513 - 1458 + 9495 + 1452 444 - 7611 - 1501 - 252876 + 7599 + 1502 + 252699 4061 - Estimated Cost to Develop (organic) $227,566
Estimated Schedule Effort (organic) 7.84 months
Estimated People Required (organic) 2.58
+ Estimated Cost to Develop (organic) $227,190
Estimated Schedule Effort (organic) 7.83 months
Estimated People Required (organic) 2.58
\ No newline at end of file diff --git a/cmd/badges/example.py b/cmd/badges/example.py index 81ce775c3..1130bec5f 100644 --- a/cmd/badges/example.py +++ b/cmd/badges/example.py @@ -9,6 +9,8 @@ from datetime import datetime, timedelta import time import math +from shutil import rmtree +from tempfile import gettempdir # s3 = boto3.client('s3') bucket_name = 'sloccloccode' @@ -40,7 +42,7 @@ def lambda_handler(event, context): get_process_file(filename=filename, url=url, path=path) - with open('/tmp/' + filename, encoding='utf-8') as f: + with open(Path(gettempdir()) / filename, encoding='utf-8') as f: content = f.read() j = json.loads(content) @@ -100,7 +102,7 @@ def get_process_file(filename, url, path): diff = int(time.time() - unixtime) if diff < 86400: - o.download_file('/tmp/' + filename) + o.download_file(Path(gettempdir()) / filename) else: clone_and_process(filename=filename, url=url, path=path) @@ -110,25 +112,25 @@ def clone_and_process(filename, url, path): download_scc() - os.chdir('/tmp') + os.chdir(gettempdir()) - os.system('rm -rf /tmp/scc-tmp-path') - git.exec_command('clone', '--depth=1', url, 'scc-tmp-path',cwd='/tmp') + rmtree(Path(gettempdir()) / 'scc-tmp-path') + git.exec_command('clone', '--depth=1', url, 'scc-tmp-path', cwd='/tmp') - os.system('./scc -f json -o /tmp/' + filename + ' scc-tmp-path') + os.system('./scc -f json -o ' + str(Path(gettempdir()) / filename) + ' scc-tmp-path') - with open('/tmp/' + filename, 'rb') as f: + with open(Path(gettempdir()) / filename, 'rb') as f: s3.upload_fileobj(f, bucket_name, filename) - os.system('rm -rf /tmp/scc-tmp-path') + rmtree(Path(gettempdir()) / 'scc-tmp-path') def download_scc(): - my_file = Path("/tmp/scc") - if os.path.exists('/tmp/scc') == False: - with open('/tmp/scc', 'wb') as f: + my_file = Path(gettempdir()) / 'scc' + if my_file.exists() == False: + with open(my_file, 'wb') as f: s3.download_fileobj(bucket_name, 'scc', f) - os.system('chmod +x /tmp/scc') + my_file.chmod(0o755) def s3time_to_unix(last_modified): diff --git a/cmd/badges/main.go b/cmd/badges/main.go index 89fd7b42f..05b671bad 100644 --- a/cmd/badges/main.go +++ b/cmd/badges/main.go @@ -10,6 +10,7 @@ import ( "net/url" "os" "os/exec" + "path/filepath" "regexp" "strconv" "strings" @@ -19,9 +20,12 @@ import ( "github.com/rs/zerolog/log" ) -var uniqueCode = "unique_code" -var cache = NewSimpleCache(1000, 86400) -var countingSemaphore = make(chan bool, 1) +var ( + uniqueCode = "unique_code" + cache = NewSimpleCache(1000, 86400) + countingSemaphore = make(chan bool, 1) + tmpDir = os.TempDir() +) func main() { http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -210,15 +214,8 @@ func process(id int, s location) ([]byte, error) { } // Clean target just to be sure - cmdArgs := []string{ - "-rf", - "/tmp/scc-tmp-path-" + strconv.Itoa(id), - } - - cmd := exec.Command("rm", cmdArgs...) - err := cmd.Run() - - if err != nil { + targetPath := filepath.Join(tmpDir, "scc-tmp-path-"+strconv.Itoa(id)) + if err := os.RemoveAll(targetPath); err != nil { return nil, err } @@ -227,10 +224,10 @@ func process(id int, s location) ([]byte, error) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() - cmd = exec.CommandContext(ctx, "git", "clone", "--depth=1", s.String(), "/tmp/scc-tmp-path-"+strconv.Itoa(id)) + cmd := exec.CommandContext(ctx, "git", "clone", "--depth=1", s.String(), targetPath) cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") - _, err = cmd.Output() + _, err := cmd.Output() if ctx.Err() == context.DeadlineExceeded { return nil, err @@ -242,17 +239,18 @@ func process(id int, s location) ([]byte, error) { // Run scc against what we just cloned fileName := processPath(s.String()) + filePath := filepath.Join(tmpDir, fileName) if fileName == "" { return nil, errors.New("processPath returned empty") } - cmdArgs = []string{ + cmdArgs := []string{ "-f", "json", "-o", - "/tmp/" + fileName, - "/tmp/scc-tmp-path-" + strconv.Itoa(id), + filePath, + targetPath, } cmd = exec.Command("scc", cmdArgs...) @@ -261,34 +259,18 @@ func process(id int, s location) ([]byte, error) { return nil, err } - file, err := os.ReadFile("/tmp/" + fileName) + file, err := os.ReadFile(filePath) if err != nil { return nil, err } cache.Add(s.String(), file) // Cleanup - cmdArgs = []string{ - "-rf", - "/tmp/" + fileName, - } - - cmd = exec.Command("rm", cmdArgs...) - err = cmd.Run() - - if err != nil { + if err := os.RemoveAll(filePath); err != nil { return nil, err } - cmdArgs = []string{ - "-rf", - "/tmp/scc-tmp-path-" + strconv.Itoa(id), - } - - cmd = exec.Command("rm", cmdArgs...) - err = cmd.Run() - - if err != nil { + if err := os.RemoveAll(targetPath); err != nil { return nil, err }