From 3e8925d3103b0b00f716c4751d210e6d35edc5cb Mon Sep 17 00:00:00 2001 From: Eduardo <100472175@alumnos.uc3m.es> Date: Tue, 1 Oct 2024 19:06:32 -0500 Subject: [PATCH 1/2] =?UTF-8?q?funciona=20con=20los=20par=C3=A1metros?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 1 + README.md | 20 ++--- go.mod | 1 + main.go | 231 +++++++++++++++++++++++++++++++++++------------------ 5 files changed, 168 insertions(+), 85 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0` is the token available in the preferences panel, and `` specifies the location where you want to save the downloaded files. The `` parameter indicates how many cores you wish to allocate for the download. If you do not provide this, the program will utilize all available cores minus one. + ### Obtaining the token To obtain the token, you must log in to AulaGlobal and go to the preferences panel. There, you will find the token under the "Security keys" section. Copy the token and paste it into the program when prompted. ![Retrieving token](assets/instructions-token.gif) + ## Build from source To build the program from source, you will need to have Go installed on your computer. You can download it from the [official website](https://golang.org/). Once you have installed Go, you can clone the repository and build the program by running the following commands: @@ -47,12 +48,13 @@ go build ## Contributing Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change. + ## Inspiration This project was inspired by the need to download all the files from all the courses in AulaGlobal. This is a tedious task that can take a lot of time, so I decided to create a tool that would allow me to do this in a simple and easy way. Previously, I had created a similar tool in Python, but I decided to create a new one in Go because I wanted to learn more about this language and the faster execution time it offers was very appealing to me. -The idea of using the mobile token to authenticate the user was inspired by the project created by [Josersanvil](github.com/Josersanvil/AulaGlobal-CoursesFiles). I decided to use this method because it is more secure than using the user's password and because this year, the university removed the login with user and password. +The idea of using the mobile token to authenticate the user was inspired by the project created by [Josersanvil](https://github.com/Josersanvil/AulaGlobal-CoursesFiles). I decided to use this method because it is more secure than using the user's password and because this year, the university removed the login with user and password. ## License diff --git a/go.mod b/go.mod index 0724df2..0da8f44 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.4 require ( github.com/fatih/color v1.17.0 github.com/schollz/progressbar/v3 v3.16.0 + github.com/spf13/pflag v1.0.5 ) require ( diff --git a/main.go b/main.go index 6a7dff8..2badb32 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ package main import ( - "flag" + "bufio" "fmt" "io" "log" @@ -16,6 +16,8 @@ import ( "github.com/fatih/color" "github.com/schollz/progressbar/v3" + "github.com/spf13/pflag" + flag "github.com/spf13/pflag" ) const ( @@ -97,7 +99,6 @@ Gets the courses, both name and ID, of a given userID */ func getCourses(token, userID string) ([]Course, error) { url := fmt.Sprintf("https://%s%s?wstoken=%s&wsfunction=core_enrol_get_users_courses&userid=%s", domain, webservice, token, userID) - color.Yellow("Getting courses...\n") resp, err := http.Get(url) if err != nil { @@ -120,8 +121,6 @@ func getCourses(token, userID string) ([]Course, error) { for i, name := range names { courses = append(courses, Course{Name: name[1], ID: ids[i][1]}) } - color.Green("Courses found: %d\n", len(courses)) - return courses, nil } @@ -213,23 +212,54 @@ func catalogFiles(courseName string, token string, files []File, dirPath string, } } -func downloadFile(fileStore FileStore) error { - filePath := fileStore.Dir - fileURL := fileStore.FileURL +func downloadFiles(filesStoreChan <-chan FileStore, maxGoroutines int) { + var wg sync.WaitGroup + semaphore := make(chan struct{}, maxGoroutines) + totalFiles := len(filesStoreChan) - resp, err := http.Get(fileURL) + // Create an atomic counter for completed files + var completedFiles int32 + + bar := progressbar.NewOptions(totalFiles, + progressbar.OptionEnableColorCodes(true), + progressbar.OptionShowBytes(false), + progressbar.OptionSetWidth(20), + progressbar.OptionSetPredictTime(false), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[green]=[reset]", + SaucerHead: "[green]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + })) + + for fileStore := range filesStoreChan { + wg.Add(1) + go func(fileStore FileStore) { + defer wg.Done() + semaphore <- struct{}{} + defer func() { <-semaphore }() + if err := downloadFileWithProgress(fileStore, bar, &completedFiles); err != nil { + fmt.Printf("Error downloading file %s: %v\n", fileStore.FileName, err) + } + }(fileStore) + } + wg.Wait() +} + +func downloadFileWithProgress(fileStore FileStore, bar *progressbar.ProgressBar, completedFiles *int32) error { + resp, err := http.Get(fileStore.FileURL) if err != nil { return fmt.Errorf("error downloading the file: %v", err) } defer resp.Body.Close() - // Create the directory path if it doesn't exist - dir := filepath.Dir(filePath) + dir := filepath.Dir(fileStore.Dir) if err := os.MkdirAll(dir, os.ModePerm); err != nil { return fmt.Errorf("error creating the directory: %v", err) } - out, err := os.Create(filePath) + out, err := os.Create(fileStore.Dir) if err != nil { return fmt.Errorf("error creating the file: %v", err) } @@ -239,18 +269,23 @@ func downloadFile(fileStore FileStore) error { if err != nil { return fmt.Errorf("error copying the file: %v", err) } + + atomic.AddInt32(completedFiles, 1) + bar.Add(1) return nil } func main() { - language, userToken, dirPath, maxGoroutines := parseFlags() + language, userToken, dirPath, maxGoroutines, coursesList := parseFlags() if language == 1 { - color.Cyan("Programa creado por Astrak00: github.com/Astrak00/AGDownloader/ para descargar archivos de Aula Global en la UC3M\n") - fmt.Println("Descargando los archivos a la carpeta", color.BlueString(dirPath)) + color.Cyan("Programa creado por Astrak00: github.com/Astrak00/AGDownloader/ \n" + + "para descargar archivos de Aula Global en la UC3M\n") + fmt.Println("Descargando los archivos a la carpeta", dirPath) } else { - color.Cyan("Program created by Astrak00: github.com/Astrak00/AGDownloader/ to download files from Aula Global at UC3M\n") - fmt.Println("Downloading files to the folder", color.BlueString(dirPath)) + color.Cyan("Program created by Astrak00: github.com/Astrak00/AGDownloader/ \n" + + "to download files from Aula Global at UC3M\n") + fmt.Println("Downloading files to the folder", dirPath) } user, err := getUserInfo(userToken) @@ -258,21 +293,62 @@ func main() { log.Fatalf("Error: Invalid Token: %v", err) } + // Obtain the courses of the user + if language == 1 { + color.Yellow("Obteniendo cursos...\n") + } else { + color.Yellow("Getting courses...\n") + } + courses, err := getCourses(userToken, user.UserID) if err != nil { log.Fatalf("Error getting courses: %v\n", err) } + if language == 1 { + color.Green("Cursos encontrados: %d\n", len(courses)) + } else { + color.Green("Courses found: %d\n", len(courses)) + } filesStoreChan := make(chan FileStore, len(courses)*20) errChan := make(chan error, len(courses)) + // If no courses are given, download all + downloadAll := false + if len(coursesList) != 0 && coursesList[0] == "" { + downloadAll = true + } else if len(coursesList) == 0 { + coursesList = showCourses(courses, language) + } else { + if language == 1 { + color.Yellow("Se descargarán los cursos que contengan: %v\n", coursesList) + } else { + color.Yellow("Courses containint: %v will be downloaded\n", coursesList) + } + } + + // Create a wait group to wait for all the goroutines to finish var wg sync.WaitGroup - for _, course := range courses { - wg.Add(1) - go func(course Course) { - defer wg.Done() - processCourse(course, userToken, dirPath, chan<- error(errChan), chan<- FileStore(filesStoreChan)) - }(course) + if downloadAll { + for _, course := range courses { + wg.Add(1) + go func(course Course) { + defer wg.Done() + processCourse(course, userToken, dirPath, chan<- error(errChan), chan<- FileStore(filesStoreChan)) + }(course) + } + } else { + for _, course := range courses { + for _, courseSearch := range coursesList { + if courseSearch == course.ID || strings.Contains(strings.ToLower(course.Name), courseSearch) { + wg.Add(1) + go func(course Course) { + defer wg.Done() + processCourse(course, userToken, dirPath, chan<- error(errChan), chan<- FileStore(filesStoreChan)) + }(course) + } + } + } } wg.Wait() @@ -301,12 +377,15 @@ func main() { } -func parseFlags() (int, string, string, int) { +func parseFlags() (int, string, string, int, []string) { language := flag.Int("l", 0, "Choose your language: 1: Español, 2:English") - token := flag.String("t", "", "Introduce your Aula Global user security token 'aulaglobalmovil'") - dir := flag.String("d", "courses", "Introduce the directory where you want to save the files") - cores := flag.Int("c", runtime.NumCPU()-1, "Introduce the cores to use in the download") - flag.Parse() + token := flag.String("token", "", "Aula Global user security token 'aulaglobalmovil'") + dir := flag.String("dir", "courses", "Directory where you want to save the files") + cores := flag.Int("p", runtime.NumCPU()-1, "Cores to be used while downloading") + + var courses []string + pflag.StringSliceVar(&courses, "courses", []string{}, "Ids or names of the courses you want to download enclosed in \", separated by spaces. \n\"all\" downloads all courses") + pflag.Parse() if *language == 0 { fmt.Println("Introduce tu idioma: 1: Español, 2:English") @@ -317,7 +396,13 @@ func parseFlags() (int, string, string, int) { *token = promptForToken(*language) } - return *language, *token, *dir, *cores + // If some courses are given, replace the commas with spaces and split the string + if len(courses) == 1 && courses[0] != "" { + courses[0] = strings.ReplaceAll(strings.ToLower(courses[0]), ",", " ") + courses = strings.Split(courses[0], " ") + } + + return *language, *token, *dir, *cores, courses } func promptForToken(language int) string { @@ -342,65 +427,59 @@ func promptForToken(language int) string { } } -func downloadFiles(filesStoreChan <-chan FileStore, maxGoroutines int) { - var wg sync.WaitGroup - semaphore := make(chan struct{}, maxGoroutines) - totalFiles := len(filesStoreChan) - - // Create an atomic counter for completed files - var completedFiles int32 +func promptForCoursesList(language int) string { + var courses string + for { + if language == 1 { + fmt.Println("Introduzca los nombres o ids de los cursos que desea descargar, separados por comas:") + } else { + fmt.Println("Introduce the names or ids of the courses you want to download, separated by commas:") + } + fmt.Scanf("%s", &courses) - bar := progressbar.NewOptions(totalFiles, - progressbar.OptionEnableColorCodes(true), - progressbar.OptionShowBytes(false), - progressbar.OptionSetWidth(20), - progressbar.OptionSetPredictTime(false), - progressbar.OptionSetTheme(progressbar.Theme{ - Saucer: "[green]=[reset]", - SaucerHead: "[green]>[reset]", - SaucerPadding: " ", - BarStart: "[", - BarEnd: "]", - })) + if courses != "" { + return courses + } - for fileStore := range filesStoreChan { - wg.Add(1) - go func(fileStore FileStore) { - defer wg.Done() - semaphore <- struct{}{} - defer func() { <-semaphore }() - if err := downloadFileWithProgress(fileStore, bar, &completedFiles); err != nil { - fmt.Printf("Error downloading file %s: %v\n", fileStore.FileName, err) - } - }(fileStore) + if language == 1 { + color.Red("No se ha introducido ningún curso. Inténtelo de nuevo.") + } else { + color.Red("No course has been introduced. Please try again.") + } } - wg.Wait() } -func downloadFileWithProgress(fileStore FileStore, bar *progressbar.ProgressBar, completedFiles *int32) error { - resp, err := http.Get(fileStore.FileURL) - if err != nil { - return fmt.Errorf("error downloading the file: %v", err) +func showCourses(courses []Course, language int) []string { + + if language == 1 { + color.Yellow("Cursos disponibles:\n") + } else { + color.Yellow("Available courses:\n") } - defer resp.Body.Close() - dir := filepath.Dir(fileStore.Dir) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("error creating the directory: %v", err) + for _, course := range courses { + fmt.Printf("%s -> %s\n", course.ID, course.Name) } - out, err := os.Create(fileStore.Dir) - if err != nil { - return fmt.Errorf("error creating the file: %v", err) + if language == 1 { + color.Yellow("Si quiere descargar todos, pulse enter:\n") + } else { + color.Yellow("If you want to download all, press enter:\n") } - defer out.Close() + fmt.Print("Enter courses (separated by spaces): ") - _, err = io.Copy(out, resp.Body) - if err != nil { - return fmt.Errorf("error copying the file: %v", err) + // Create a new scanner to read from standard input + scanner := bufio.NewScanner(os.Stdin) + // Read the entire line + scanner.Scan() + coursesStr := scanner.Text() + + // Split the input string into a slice of courses + coursesList := strings.Fields(coursesStr) + + for i := range coursesList { + coursesList[i] = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(coursesList[i]), ",", "")) } - atomic.AddInt32(completedFiles, 1) - bar.Add(1) - return nil + return coursesList } From 3f4f26f8a925c0cdaf92e576e7a1c70cd3cdfa7e Mon Sep 17 00:00:00 2001 From: Eduardo <100472175@alumnos.uc3m.es> Date: Tue, 1 Oct 2024 19:28:20 -0500 Subject: [PATCH 2/2] Updated readme with new parameters --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++------ main.go | 29 ++++++++++++++++------------- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index a2a3dd9..077b76d 100644 --- a/README.md +++ b/README.md @@ -17,23 +17,64 @@ You can specify some parameters to customize the download process but if you don ./AGDownloader -h Usage of ./AGDownloader: - --courses strings Ids or names of the courses you want to download enclosed in ", separated by spaces. + --courses strings Ids or names of the courses to be downloaded, enclosed in ", separated by spaces. "all" downloads all courses - --dir string Directory where you want to save the files (default "courses") + --dir string Directory where you want to save the files --l int Choose your language: 1: Español, 2:English - --p int Cores to be used while downloading (default 7) + --p int Cores to be used while downloading (default 4) --token string Aula Global user security token 'aulaglobalmovil' ``` -Here, `` is the token available in the preferences panel, and `` specifies the location where you want to save the downloaded files. The `` parameter indicates how many cores you wish to allocate for the download. If you do not provide this, the program will utilize all available cores minus one. - - ### Obtaining the token To obtain the token, you must log in to AulaGlobal and go to the preferences panel. There, you will find the token under the "Security keys" section. Copy the token and paste it into the program when prompted. ![Retrieving token](assets/instructions-token.gif) +### Example + +This is an example of a full command: + +```bash +./AGDownloader --l 1 --token aaaa1111bbbb2222cccc3333dddd4444 --dir AulaGlobal-Copy --p 5 --courses "Inteligencia Distribuidos 123445" +``` + +This program will run in Spanish, with the secret token aaaa1111bbbb2222cccc3333dddd4444, in the folder AulaGlobal-Copy inside the folder where the program is being run, using 5 cores, and downloading the courses that contain the words "Inteligencia" or "Distributed" in their name or the course with the ID 123445. + +#### Language + +You can choose the language in which the program will run by using the `--l` parameter. The possible values are: +- 1: Spanish +- 2: English + +#### Courses + +You can specify the courses you want to download by using the `--courses` parameter. You can specify the courses by their ID or by their name. If you want to download all the courses, you can use the keyword "all". + +``` +./AGDownload --courses "Ingeniería Inteligencia" +``` +This parameter will download all the courses that contain the words "Ingeniería" or "Inteligencia" in their name. + +#### Directory + +You can specify the directory where you want to save the files by using the `--dir` parameter. You must specify the path to the directory where you want to save the files. If you want to download the files in the same directory where the program is being run, you can put a dot. + +``` +./AGDownload --dir . +``` + +#### Cores + +You can specify the number of cores you want to use while downloading the files by using the `--p` parameter. The default value is 4. + +``` +./AGDownload --p 8 +``` + + + + ## Build from source @@ -43,6 +84,7 @@ To build the program from source, you will need to have Go installed on your com git clone git@github.com:Astrak00/AGDownloader.git go build ``` +This will create an executable file called AGDownloader that you can run. ## Contributing diff --git a/main.go b/main.go index 2badb32..79be26d 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "regexp" - "runtime" "strings" "sync" "sync/atomic" @@ -380,11 +379,11 @@ func main() { func parseFlags() (int, string, string, int, []string) { language := flag.Int("l", 0, "Choose your language: 1: Español, 2:English") token := flag.String("token", "", "Aula Global user security token 'aulaglobalmovil'") - dir := flag.String("dir", "courses", "Directory where you want to save the files") - cores := flag.Int("p", runtime.NumCPU()-1, "Cores to be used while downloading") + dir := flag.String("dir", "", "Directory where you want to save the files") + cores := flag.Int("p", 4, "Cores to be used while downloading") var courses []string - pflag.StringSliceVar(&courses, "courses", []string{}, "Ids or names of the courses you want to download enclosed in \", separated by spaces. \n\"all\" downloads all courses") + pflag.StringSliceVar(&courses, "courses", []string{}, "Ids or names of the courses to be downloaded, enclosed in \", separated by spaces. \n\"all\" downloads all courses") pflag.Parse() if *language == 0 { @@ -392,6 +391,10 @@ func parseFlags() (int, string, string, int, []string) { fmt.Scanf("%d", language) } + if *dir == "" { + *dir = promptForDir(*language) + } + if *token == "" { *token = promptForToken(*language) } @@ -427,24 +430,24 @@ func promptForToken(language int) string { } } -func promptForCoursesList(language int) string { - var courses string +func promptForDir(language int) string { + var dir string for { if language == 1 { - fmt.Println("Introduzca los nombres o ids de los cursos que desea descargar, separados por comas:") + fmt.Println("Introduzca la ruta donde quiere guardar los archivos:") } else { - fmt.Println("Introduce the names or ids of the courses you want to download, separated by commas:") + fmt.Println("Enter the path where you want to save the files:") } - fmt.Scanf("%s", &courses) + fmt.Scanf("%s", &dir) - if courses != "" { - return courses + if dir != "" { + return dir } if language == 1 { - color.Red("No se ha introducido ningún curso. Inténtelo de nuevo.") + color.Red("La ruta introducida no parece estar correcta. Inténtelo de nuevo.") } else { - color.Red("No course has been introduced. Please try again.") + color.Red("The given path does not seem to be right. Please try again.") } } }