Skip to content

Commit

Permalink
refactor: using standard libraries to do file operations (#542)
Browse files Browse the repository at this point in the history
  • Loading branch information
@apocelipes
apocelipes authored Oct 22, 2024
1 parent 10eacba commit aa904f4
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 67 deletions.
38 changes: 19 additions & 19 deletions SCC-OUTPUT-REPORT.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@
<tbody><tr>
<th>Go</th>
<th>27</th>
<th>9513</th>
<th>1458</th>
<th>9495</th>
<th>1452</th>
<th>444</th>
<th>7611</th>
<th>1501</th>
<th>252876</th>
<th>7599</th>
<th>1502</th>
<th>252699</th>
<th>4061</th>
</tr><tr>
<td>processor/workers_test.go</td>
Expand Down Expand Up @@ -67,9 +67,9 @@
<td>142</td>
<td>104</td>
<td>430</td>
<td>93</td>
<td>19515</td>
<td>441</td>
<td>94</td>
<td>19510</td>
<td>440</td>
</tr><tr>
<td>main.go</td>
<td></td>
Expand All @@ -93,13 +93,13 @@
</tr><tr>
<td>cmd/badges/main.go</td>
<td></td>
<td>345</td>
<td>59</td>
<td>327</td>
<td>53</td>
<td>14</td>
<td>272</td>
<td>260</td>
<td>50</td>
<td>8129</td>
<td>204</td>
<td>7957</td>
<td>206</td>
</tr><tr>
<td>processor/workers_tokei_test.go</td>
<td></td>
Expand Down Expand Up @@ -294,15 +294,15 @@
<tfoot><tr>
<th>Total</th>
<th>27</th>
<th>9513</th>
<th>1458</th>
<th>9495</th>
<th>1452</th>
<th>444</th>
<th>7611</th>
<th>1501</th>
<th>252876</th>
<th>7599</th>
<th>1502</th>
<th>252699</th>
<th>4061</th>
</tr>
<tr>
<th colspan="9">Estimated Cost to Develop (organic) $227,566<br>Estimated Schedule Effort (organic) 7.84 months<br>Estimated People Required (organic) 2.58<br></th>
<th colspan="9">Estimated Cost to Develop (organic) $227,190<br>Estimated Schedule Effort (organic) 7.83 months<br>Estimated People Required (organic) 2.58<br></th>
</tr></tfoot>
</table></body></html>
26 changes: 14 additions & 12 deletions cmd/badges/example.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
from datetime import datetime, timedelta
import time
import math
from shutil import rmtree
from tempfile import gettempdir

# s3 = boto3.client('s3')
bucket_name = 'sloccloccode'
Expand Down Expand Up @@ -40,7 +42,7 @@ def lambda_handler(event, context):

get_process_file(filename=filename, url=url, path=path)

with open('/tmp/' + filename, encoding='utf-8') as f:
with open(Path(gettempdir()) / filename, encoding='utf-8') as f:
content = f.read()

j = json.loads(content)
Expand Down Expand Up @@ -100,7 +102,7 @@ def get_process_file(filename, url, path):

diff = int(time.time() - unixtime)
if diff < 86400:
o.download_file('/tmp/' + filename)
o.download_file(Path(gettempdir()) / filename)
else:
clone_and_process(filename=filename, url=url, path=path)

Expand All @@ -110,25 +112,25 @@ def clone_and_process(filename, url, path):

download_scc()

os.chdir('/tmp')
os.chdir(gettempdir())

os.system('rm -rf /tmp/scc-tmp-path')
git.exec_command('clone', '--depth=1', url, 'scc-tmp-path',cwd='/tmp')
rmtree(Path(gettempdir()) / 'scc-tmp-path')
git.exec_command('clone', '--depth=1', url, 'scc-tmp-path', cwd='/tmp')

os.system('./scc -f json -o /tmp/' + filename + ' scc-tmp-path')
os.system('./scc -f json -o ' + str(Path(gettempdir()) / filename) + ' scc-tmp-path')

with open('/tmp/' + filename, 'rb') as f:
with open(Path(gettempdir()) / filename, 'rb') as f:
s3.upload_fileobj(f, bucket_name, filename)

os.system('rm -rf /tmp/scc-tmp-path')
rmtree(Path(gettempdir()) / 'scc-tmp-path')


def download_scc():
my_file = Path("/tmp/scc")
if os.path.exists('/tmp/scc') == False:
with open('/tmp/scc', 'wb') as f:
my_file = Path(gettempdir()) / 'scc'
if my_file.exists() == False:
with open(my_file, 'wb') as f:
s3.download_fileobj(bucket_name, 'scc', f)
os.system('chmod +x /tmp/scc')
my_file.chmod(0o755)


def s3time_to_unix(last_modified):
Expand Down
54 changes: 18 additions & 36 deletions cmd/badges/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
Expand All @@ -19,9 +20,12 @@ import (
"github.com/rs/zerolog/log"
)

var uniqueCode = "unique_code"
var cache = NewSimpleCache(1000, 86400)
var countingSemaphore = make(chan bool, 1)
var (
uniqueCode = "unique_code"
cache = NewSimpleCache(1000, 86400)
countingSemaphore = make(chan bool, 1)
tmpDir = os.TempDir()
)

func main() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -210,15 +214,8 @@ func process(id int, s location) ([]byte, error) {
}

// Clean target just to be sure
cmdArgs := []string{
"-rf",
"/tmp/scc-tmp-path-" + strconv.Itoa(id),
}

cmd := exec.Command("rm", cmdArgs...)
err := cmd.Run()

if err != nil {
targetPath := filepath.Join(tmpDir, "scc-tmp-path-"+strconv.Itoa(id))
if err := os.RemoveAll(targetPath); err != nil {
return nil, err
}

Expand All @@ -227,10 +224,10 @@ func process(id int, s location) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()

cmd = exec.CommandContext(ctx, "git", "clone", "--depth=1", s.String(), "/tmp/scc-tmp-path-"+strconv.Itoa(id))
cmd := exec.CommandContext(ctx, "git", "clone", "--depth=1", s.String(), targetPath)

cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
_, err = cmd.Output()
_, err := cmd.Output()

if ctx.Err() == context.DeadlineExceeded {
return nil, err
Expand All @@ -242,17 +239,18 @@ func process(id int, s location) ([]byte, error) {

// Run scc against what we just cloned
fileName := processPath(s.String())
filePath := filepath.Join(tmpDir, fileName)

if fileName == "" {
return nil, errors.New("processPath returned empty")
}

cmdArgs = []string{
cmdArgs := []string{
"-f",
"json",
"-o",
"/tmp/" + fileName,
"/tmp/scc-tmp-path-" + strconv.Itoa(id),
filePath,
targetPath,
}

cmd = exec.Command("scc", cmdArgs...)
Expand All @@ -261,34 +259,18 @@ func process(id int, s location) ([]byte, error) {
return nil, err
}

file, err := os.ReadFile("/tmp/" + fileName)
file, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
cache.Add(s.String(), file)

// Cleanup
cmdArgs = []string{
"-rf",
"/tmp/" + fileName,
}

cmd = exec.Command("rm", cmdArgs...)
err = cmd.Run()

if err != nil {
if err := os.RemoveAll(filePath); err != nil {
return nil, err
}

cmdArgs = []string{
"-rf",
"/tmp/scc-tmp-path-" + strconv.Itoa(id),
}

cmd = exec.Command("rm", cmdArgs...)
err = cmd.Run()

if err != nil {
if err := os.RemoveAll(targetPath); err != nil {
return nil, err
}

Expand Down

0 comments on commit aa904f4

Please sign in to comment.