first commit

.gitignore

@@ -0,0 +1 @@
+.idea

README.md

@@ -0,0 +1,15 @@
+# Wget based on file list
+
+## Usage
+
+```sh
+source$ cd ~/web/media
+source$ find . -type f | grep -v '.thumb' | grep -v product/cache | sed 's#^./##' > ~/media.txt
+```
+
+scp media.txt onto the target server with a gwget binary
+
+```sh
+target$ cd ~/web/media
+target$ ~/gwget ~/media.txt https://www.somesite.com/media/
+```

gwget.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"os"
+	"fmt"
+	"github.com/labstack/gommon/log"
+	"io/ioutil"
+	"strings"
+	"path/filepath"
+	"net/http"
+	"time"
+	"errors"
+)
+
+func get(url, filename string) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("Panic in %s\n%v\n", url, r)
+		}
+	}()
+
+	var err error
+
+	myClient := &http.Client{}
+	var res *http.Response
+	done := make(chan bool)
+	go func(done chan bool) {
+		res, err = myClient.Get(url)
+		done <- true
+	}(done)
+	select {
+	case <-done:
+		//all OK
+	case <-time.After(10 * time.Second):
+		err = errors.New("Timeout")
+	}
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		log.Printf(err.Error())
+		return
+	}
+	file, _ := os.Create(filename)
+	defer file.Close()
+	file.Write(body)
+}
+
+func main () {
+
+	if len(os.Args) < 2 {
+		fmt.Println("Usage:")
+		fmt.Println("   gwget {url} {file}")
+		fmt.Println("       {url} prefix for the download")
+		fmt.Println("       {file} directory list as argument (dir/dir/file format per line, from find)")
+		fmt.Println()
+		os.Exit(0)
+	}
+
+	url := os.Args[2]
+
+	f,err := os.Open(os.Args[1])
+	if err != nil {
+		log.Panic(err)
+	}
+
+	b, err := ioutil.ReadAll(f)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	lines := strings.Split(string(b),"\n")
+	for _,fullpath := range lines {
+		if _, err := os.Stat(fullpath); os.IsNotExist(err) {
+			dir := filepath.Dir(fullpath)
+			if _, err := os.Stat(dir); os.IsNotExist(err) {
+				os.Mkdir(dir,0777)
+				fmt.Println("Created",dir)
+			}
+			get(url+fullpath, fullpath)
+			fmt.Println("Downloaded",fullpath)
+		}
+	}
+}