diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..2afce539
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,4 @@
+.git
+tests
+sample_settings.yaml
+settings.yaml
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 403fb9aa..8d354cf7 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -39,7 +39,7 @@ jobs:
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
- uses: github/codeql-action/init@v1
+ uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
- uses: github/codeql-action/autobuild@v1
+ uses: github/codeql-action/autobuild@v2
# âšī¸ Command-line programs to run using the OS shell.
# đ https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@v1
+ uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 9432e68a..e93272f6 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -41,9 +41,9 @@ jobs:
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
- flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+ flake8 setup.py udemy_enroller --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+ flake8 setup.py udemy_enroller --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics
- name: Run unittests
run: |
poetry run pytest
@@ -53,4 +53,4 @@ jobs:
UDEMY_PASSWORD: ${{ secrets.UDEMY_PASSWORD }}
CI_TEST: "True"
run: |
- poetry run python udemy_enroller.py --browser=chrome --debug
+ poetry run python run_enroller.py --browser=chrome --debug
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f17dd20b..1eaad55a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [4.1.3] - 2023-03-07
+
+### Added
+- Fixing issues with enrollment and scraping
+- Tidying up of the code structure
+
## [4.1.2] - 2022-06-03
### Added
@@ -116,6 +122,8 @@ can continue as normal
project running locally. Suitable for users who are not looking forward to
contribute.
+[4.1.3]:
+ https://github.com/aapatre/Automatic-Udemy-Course-Enroller-GET-PAID-UDEMY-COURSES-for-FREE/releases/tag/v4.1.3
[4.1.2]:
https://github.com/aapatre/Automatic-Udemy-Course-Enroller-GET-PAID-UDEMY-COURSES-for-FREE/releases/tag/v4.1.2
[4.1.1]:
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..114d5a9b
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,15 @@
+FROM python:3.10-alpine
+
+RUN apk add --no-cache build-base
+
+RUN addgroup -S enroller && adduser -S enroller -G enroller
+USER enroller
+RUN mkdir -p ~/.udemy_enroller
+
+WORKDIR /src
+
+COPY . .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+ENTRYPOINT [ "python", "run_enroller.py" ]
diff --git a/README.md b/README.md
index 6f7ae036..9f449186 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,8 @@
[](https://forthebadge.com)
[](https://forthebadge.com)
-* ALPHA IS A PRE DEVELOPMENT BRANCH, DO NOT EXPECT USER FACING ISSUES TO BE ADDRESSED IN THIS BRANCH!
-
-
* Udemy Coupon Grabber & Course Enroller: Grab FREE Coupons!
+
Do you want to LEARN NEW STUFF for FREE? Don't worry, with the power of
web-scraping and automation, this script will find the necessary Udemy Coupons
@@ -14,48 +12,52 @@ web-scraping and automation, this script will find the necessary Udemy Coupons
**NOTE: THIS PROJECT WILL NOT WORK WITH NON ENGLISH UDEMY.**
+
+
The code scrapes course links and coupons from:
- [tutorialbar.com](https://tutorialbar.com)
- [discudemy.com](https://discudemy.com)
- [coursevania.com](https://coursevania.com)
- - [freebiesglobal.com](https://freebiesglobal.com) -> _New_
+ - [freebiesglobal.com](https://freebiesglobal.com)
+ - [idownloadcoupon.com](https://idownloadcoupon.com) -> _New_
-In case of any bugs or issues, please open an issue in github.
+In case of any bugs or issues, please open an issue in github. Also, don't forget to **Fork & Star the repository if you like it!**
-Also, don't forget to **Fork & Star the repository if you like it!**
+**We are also on _[GitLab](https://gitlab.com/the-automators/Automatic-Udemy-Course-Enroller-GET-PAID-UDEMY-COURSES-for-FREE)_**
+
-**_We are also on [GitLab](https://gitlab.com/the-automators/Automatic-Udemy-Course-Enroller-GET-PAID-UDEMY-COURSES-for-FREE)_**
-
-**_Video Proof:_**
+## Video Proof:
[](https://www.youtube.com/watch?v=tdLsVoraMxw "GET Udemy Courses for FREE with Python | 2 Minute Tuesday")
---
-** **_Disclaimer & WARNINGS:_**
+## Disclaimer & WARNINGS:
-1. **Use** this ONLY for **Educational Purposes!** By using this code you agree
- that **I'm not responsible for any kind of trouble** caused by the code. **THIS PROJECT IS NOT AFFILIATED WITH UDEMY.**
-2. **Make sure web-scraping is legal in your region.**
-3. This is **NOT a hacking script**, i.e., it can't enroll you for a specific
+ 1. Use this ONLY for **Educational Purposes!** By using this code you agree that
+ **I'm not responsible for any kind of trouble** caused by the code. **THIS PROJECT IS NOT AFFILIATED WITH UDEMY.**
+
+ 2. **Make sure web-scraping is legal in your region.**
+
+ 3. This is **NOT a hacking script**, i.e., it can't enroll you for a specific
course! Instead it finds courses that provide coupon links to make the
transaction free and then LEGALLY enroll you to the course!
---
-** Requirements:
+## Requirements:
-*** How to Install the Requirements?
+### How to Install the Requirements?
-**Required Python version:** [Python 3.8+](https://www.python.org/downloads/)
+ __Required Python version:__ [Python 3.8+](https://www.python.org/downloads/)
-**(Windows users only) Required Microsoft Visual C++ 14.0+ version:** [Microsoft Visual C++ 14.0+](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
+ __(Windows users only) Required Microsoft Visual C++ 14.0+ version:__ [Microsoft Visual C++ 14.0+](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
-**You must have pip or poetry installed. Please look up how to install them in your OS.**
+ __You must have pip or poetry installed. Please look up how to install them in your OS.__
Download a release of this project or clone the repository then navigate to the
folder where you placed the files on. Type `pip install -r requirements.txt` to
@@ -63,7 +65,7 @@ get all the requirements installed in one go. Similar instructions applies for p
---
-** Instructions
+## Instructions
Props to Davidd Sargent for making a super simple video tutorial. If you prefer written instructions then continue reading further, else click on the image below for a quick video tutorial:
@@ -71,13 +73,15 @@ Props to Davidd Sargent for making a super simple video tutorial. If you prefer
1 . Install from PyPI `pip install udemy-enroller`
-- Run the script and the cli will guide you through the settings required
+- Run the script and the cli will guide you through the settings required.
- If you decide to save the settings they will be stored in your home directory:
**Windows**:
C:/Users/CurrentUserName/.udemy_enroller
**Linux**:
- /home/username/.udemy_enroller
- **The values in settings.yaml should be in the same language as the site you are browsing on**
+ /home/username/.udemy_enroller
+ **The values in settings.yaml should be in the same language as the site you are browsing on.**
+
+
2 . The script can be passed arguments:
@@ -87,26 +91,33 @@ Props to Davidd Sargent for making a super simple video tutorial. If you prefer
- `--coursevania`: Run the coursevania scraper only
- `--tutorialbar`: Run the tutorialbar scraper only
- `--freebiesglobal`: Run the freebiesglobal scraper only
+- `--idownloadcoupon`: Run the idownloadcoupon scraper only
- `--max-pages=`: Max number of pages to scrape from sites before exiting the script (default is 5)
- `--delete-settings`: Delete existing settings file
- `--delete-cookie`: Delete the cookie file if it exists
-- `--debug`: Enable debug logging
+- `--debug`: Enable debug logging
+
+
3 . Run the script in terminal with your target runner:
- `udemy_enroller`
- `udemy_enroller --browser=chrome`
-- `udemy_enroller --browser=chromium`
+- `udemy_enroller --browser=chromium`
+
+
4 . The bot starts scraping the course links from the first **All Courses** page
on [Tutorial Bar](https://www.tutorialbar.com/all-courses/page/1), [DiscUdemy](https://www.discudemy.com/all), [Coursevania](https://coursevania.com) and [FreebiesGlobal](https://freebiesglobal.com) and starts
enrolling you to Udemy courses. After it has enrolled you to courses from the
first page, it then moves to the next site page and the cycle continues.
-- Stop the script by pressing ctrl+c in terminal to stop the enrollment process.
+- Stop the script by pressing `ctrl+c` in terminal to stop the enrollment process.
-5 . _[New]_ At the end of process a detailed result is shown:
+
+
+5 . *[New]* At the end of process a detailed result is shown:
```
================== Run Statistics ==================
@@ -120,91 +131,122 @@ Total Enrolments: 1705
Savings: âŦ2674.44
================== Run Statistics ==================
```
-
+
+### Docker
+
+Alternatively you can run the script in docker.
+
+To build the image run:
+
+```
+docker build -t udemy_enroller .
+```
+
+After the build is finished you can run your container with one of the commands below (you can pass arguments as you would in the cli):
+
+```
+docker run -it udemy_enroller
+```
+
+After you entered your login credentials and settings detach from the interactive mode by pressing the `Ctrl-P` followed by `Ctrl-Q`.
+
+You can also create a `settings.yaml` file from the `sample_settings.yaml` and mount to the container with the command:
+
+```
+docker run -v $(pwd)/settings.yaml:/home/enroller/.udemy_enroller/settings.yaml udemy_enroller
+```
+
+
---
## FAQs
-*** 1. Can I get a specific course for free with this script?
+__1. Can I get a specific course for free with this script?__
-Unfortunately no, but let me assure you that you may be lucky enough to get a
+ Unfortunately no, but let me assure you that you may be lucky enough to get a
particular course for free when the instructor posts its coupon code in order
to promote it. Also, over time you would build a library of courses by running
the script often and have all the required courses in your collection. In fact,
I made this course after completing a
-[Python automation course](https://www.udemy.com/course/automate/) and selenium,
+[Python automation course](https://www.udemy.com/course/automate/) and [Selenium](https://www.selenium.dev/),
which of course I got for free! :)
+
+
+__2. How does the bot work?__
-*** 2. How does the bot work?
-
-The bot retrieves coupon links from Tutorial Bar, DiscUdemy and Coursevania's lists to cut the prices and
+ The bot retrieves coupon links from [Tutorial Bar](https://www.tutorialbar.com/all-courses/page/1), [DiscUdemy](https://www.discudemy.com/all) and [Coursevania](https://coursevania.com)'s lists to cut the prices and
then uses REST requests to authenticate and enroll to the
courses. Think of it this way: Epic Games & other clients like Steam provide you
a handful of games each week, for free; Only in this case, we need a coupon code
-to make those courses free.
+to make those courses free.
+
-*** 3. How frequently should you run the script?
+__3. How frequently should you run the script?__
-Daily, at least once! I've painstakingly amassed over 4000
+ Daily, at least once! I've painstakingly amassed over 4000
courses in the last four years! And out of those 4000, I've only paid for 4 of
these courses.
So, a mere **0.001%** of courses are **actually paid** in my collection!
Thankfully, you can get more than what I gathered in 4 years, in a matter of
-weeks! đđģ
+weeks! đđģ
-*** 4. Why did I create this?
-It used to be my daily habit to redeem courses and it was an extremely tedious
+__4. Why did I create this?__
+
+ It used to be my daily habit to redeem courses and it was an extremely tedious
task that took around 15 minutes, for 10 courses. And then I suddenly got the
idea to automate it, after I found the automation course mentioned above. I bet,
-it will save your precious time too! :)
+it will save your precious time too! :)
+
-*** 5. The code compiles successfully, but it's taking too long to work! IS there any way to fix that?
+__5. The code compiles successfully, but it's taking too long to work! IS there any way to fix that?__
-Since we are heavily dependent on a third-party site to retrieve coupons links,
+ Since we are heavily dependent on a third-party site to retrieve coupons links,
there may be issues when the site is down. Needless to mention the connectivity
issues too. If everything is working fine, you can see the courses being
-retrieved in the Python console/shell, which may take a while.
+retrieved in the Python console/shell, which may take a while.
+
+
+__6. Which is the best way to run the script?__
+
+ It is recommended to run the script using your terminal and system python.
+
-*** 6. Which is the best way to run the script?
+__7. Which branch to commit against?__
-It is recommended to run the script using your terminal and system python.
+ Pull request should be made on "_develop_" branch.
-*** 7. Which branch to commit against?
-Pull request should be made on "develop" branch.
+__8. What's the roadmap?__
-*** 8. What's the roadmap?
+ Take a look at our [Roadmap here](https://github.com/aapatre/Automatic-Udemy-Course-Enroller-GET-PAID-UDEMY-COURSES-for-FREE/projects/1) and help us on what you want or talk to us about your proposed changes.
-Take a look at our
-[Roadmap here](https://github.com/aapatre/Automatic-Udemy-Course-Enroller-GET-PAID-UDEMY-COURSES-for-FREE/projects/1)
-and help us on what you want or talk to us about your proposed changes.
---
-** Support & Maintenance Notice
+## Support & Maintenance Notice
By using this repo/script, you agree that the authors and contributors are under no obligation to provide support for the script and can discontinue it's development, as and when necessary, without prior notice.
---
-** Supporters
+## Supporters
-*** Jetbrains
+### Jetbrains
[](https://jetbrains.com/?from=udemy-free-course-enroller)
Thanks to [JetBrains](https://jetbrains.com/?from=udemy-free-course-enroller) for supporting us. They are the maker of world class IDE and developer tooling. If you think their product might help you, please support them.
-*** GitBook
+### GitBook
[](https://gitbook.com)
Thanks to [GitBook](https://gitbook.com) for supporting us. GitBook is the best place to track personal notes and ideas for teams. If you think their product might help you, please support them.
-*** GitLab
+### GitLab
[](https://gitlab.com)
diff --git a/pyproject.toml b/pyproject.toml
index fee30f8a..26466de6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,8 +1,17 @@
[tool.poetry]
-name = "automatic-udemy-course-enroller-get-paid-udemy-courses-for-free"
-version = "4.1.2"
+name = "udemy-enroller"
+version = "4.1.3"
description = ""
-authors = [""]
+homepage = "https://github.com/aapatre/Automatic-Udemy-Course-Enroller-GET-PAID-UDEMY-COURSES-for-FREE"
+authors = ["aapatre ", "fakeid ", "cullzie "]
+readme = "README.md"
+keywords = ["udemy", "education", "enroll"]
+classifiers = [
+ "Development Status :: 4 - Beta",
+ "Intended Audience :: Education",
+ "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
+ "Programming Language :: Python :: 3.8",
+]
[tool.poetry.dependencies]
python = "^3.8"
@@ -22,9 +31,16 @@ pytest = "^7.1.2"
pytest-cov = "^3.0.0"
pytest-asyncio = "^0.18.3"
bumpver = "^2022.1116"
+flake8 = "^5.0.4"
+flake8-bugbear = "^22.9.23"
+flake8-docstrings = "^1.6.0"
+flake8-isort = "^5.0.0"
+
+[tool.isort]
+profile = "black"
[tool.bumpver]
-current_version = "4.1.2"
+current_version = "4.1.3"
version_pattern = "MAJOR.MINOR.PATCH"
commit_message = "Bump version {old_version} -> {new_version}"
commit = true
@@ -41,5 +57,8 @@ push = false
]
[build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+udemy_enroller = "udemy_enroller.cli:main"
diff --git a/udemy_enroller.py b/run_enroller.py
similarity index 100%
rename from udemy_enroller.py
rename to run_enroller.py
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 00000000..432b896f
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,6 @@
+[flake8]
+max-line-length = 120
+max-complexity = 10
+per-file-ignores =
+ udemy_enroller/udemy_ui.py: C901
+ udemy_enroller/runner.py: C901
diff --git a/setup.py b/setup.py
index 8d9f03e2..abe3cb5a 100644
--- a/setup.py
+++ b/setup.py
@@ -1,3 +1,4 @@
+"""Setup."""
import pathlib
from setuptools import find_packages, setup
@@ -8,7 +9,7 @@
setup(
name="udemy-enroller",
- version="4.1.2",
+ version="4.1.3",
long_description=long_description,
long_description_content_type="text/markdown",
author="aapatre",
diff --git a/udemy_enroller/__init__.py b/udemy_enroller/__init__.py
index 7a534475..f6a0a844 100644
--- a/udemy_enroller/__init__.py
+++ b/udemy_enroller/__init__.py
@@ -1,8 +1,9 @@
-from .driver_manager import ALL_VALID_BROWSER_STRINGS, DriverManager
-from .logging import load_logging_config
-from .scrapers.manager import ScraperManager
-from .settings import Settings
-from .udemy_rest import UdemyActions, UdemyStatus
-from .udemy_ui import UdemyActionsUI
+"""."""
+from .driver_manager import ALL_VALID_BROWSER_STRINGS, DriverManager # noqa: F401
+from .logger import load_logging_config
+from .scrapers.manager import ScraperManager # noqa: F401
+from .settings import Settings # noqa: F401
+from .udemy_rest import UdemyActions, UdemyStatus # noqa: F401
+from .udemy_ui import UdemyActionsUI # noqa: F401
load_logging_config()
diff --git a/udemy_enroller/cli.py b/udemy_enroller/cli.py
index 26d95ad0..c2a81476 100644
--- a/udemy_enroller/cli.py
+++ b/udemy_enroller/cli.py
@@ -1,10 +1,15 @@
+"""CLI entrypoint for this script."""
import argparse
import logging
+import platform
+import sys
from argparse import Namespace
from typing import Tuple, Union
+from pkg_resources import DistributionNotFound, get_distribution
+
from udemy_enroller import ALL_VALID_BROWSER_STRINGS, DriverManager, Settings
-from udemy_enroller.logging import get_logger
+from udemy_enroller.logger import get_logger
from udemy_enroller.runner import redeem_courses, redeem_courses_ui
logger = get_logger()
@@ -12,42 +17,80 @@
def enable_debug_logging() -> None:
"""
- Enable debug logging for the scripts
+ Enable debug logging for the scripts.
:return: None
"""
logger.setLevel(logging.DEBUG)
for handler in logger.handlers:
handler.setLevel(logging.DEBUG)
- logger.info(f"Enabled debug logging")
+ logger.info("Enabled debug logging")
+
+
+def log_package_details() -> None:
+ """
+ Log details of the package.
+
+ :return: None
+ """
+ try:
+ distribution = get_distribution("udemy_enroller")
+ if distribution:
+ logger.debug(f"Name: {distribution.project_name}")
+ logger.debug(f"Version: {distribution.version}")
+ logger.debug(f"Location: {distribution.location}")
+ except DistributionNotFound:
+ logger.debug("Not installed on python env.")
+
+
+def log_python_version():
+ """
+ Log version of python in use.
+
+ :return: None
+ """
+ logger.debug(f"Python: {sys.version}")
+
+
+def log_os_version():
+ """
+ Log version of the OS.
+
+ :return: None
+ """
+ logger.debug(f"OS: {platform.platform()}")
def determine_if_scraper_enabled(
+ idownloadcoupon_enabled: bool,
freebiesglobal_enabled: bool,
tutorialbar_enabled: bool,
discudemy_enabled: bool,
coursevania_enabled: bool,
-) -> Tuple[bool, bool, bool, bool]:
+) -> Tuple[bool, bool, bool, bool, bool]:
"""
- Determine what scrapers should be enabled and disabled
+ Determine what scrapers should be enabled and disabled.
:return: tuple containing boolean of what scrapers should run
"""
if (
- not freebiesglobal_enabled
+ not idownloadcoupon_enabled
+ and not freebiesglobal_enabled
and not tutorialbar_enabled
and not discudemy_enabled
and not coursevania_enabled
):
# Set all to True
(
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
coursevania_enabled,
- ) = (True, True, True, True)
+ ) = (True, True, True, True, True)
return (
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
@@ -57,6 +100,7 @@ def determine_if_scraper_enabled(
def run(
browser: str,
+ idownloadcoupon_enabled: bool,
freebiesglobal_enabled: bool,
tutorialbar_enabled: bool,
discudemy_enabled: bool,
@@ -66,9 +110,10 @@ def run(
delete_cookie: bool,
):
"""
- Run the udemy enroller script
+ Run the udemy enroller script.
:param str browser: Name of the browser we want to create a driver for
+ :param bool idownloadcoupon_enabled:
:param bool freebiesglobal_enabled:
:param bool tutorialbar_enabled:
:param bool discudemy_enabled:
@@ -84,6 +129,7 @@ def run(
redeem_courses_ui(
dm.driver,
settings,
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
@@ -93,6 +139,7 @@ def run(
else:
redeem_courses(
settings,
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
@@ -103,7 +150,7 @@ def run(
def parse_args() -> Namespace:
"""
- Parse args from the CLI or use the args passed in
+ Parse args from the CLI or use the args passed in.
:return: Args to be used in the script
"""
@@ -116,6 +163,12 @@ def parse_args() -> Namespace:
choices=ALL_VALID_BROWSER_STRINGS,
help="Browser to use for Udemy Enroller",
)
+ parser.add_argument(
+ "--idownloadcoupon",
+ action="store_true",
+ default=False,
+ help="Run idownloadcoupon scraper",
+ )
parser.add_argument(
"--freebiesglobal",
action="store_true",
@@ -128,28 +181,24 @@ def parse_args() -> Namespace:
default=False,
help="Run tutorialbar scraper",
)
-
parser.add_argument(
"--discudemy",
action="store_true",
default=False,
help="Run discudemy scraper",
)
-
parser.add_argument(
"--coursevania",
action="store_true",
default=False,
help="Run coursevania scraper",
)
-
parser.add_argument(
"--max-pages",
type=int,
default=5,
- help=f"Max pages to scrape from sites (if pagination exists) (Default is 5)",
+ help="Max pages to scrape from sites (if pagination exists) (Default is 5)",
)
-
parser.add_argument(
"--delete-settings",
action="store_true",
@@ -170,26 +219,34 @@ def parse_args() -> Namespace:
help="Enable debug logging",
)
- args = parser.parse_args()
-
- return args
+ return parser.parse_args()
def main():
+ """Entrypoint for scripts."""
args = parse_args()
if args:
if args.debug:
enable_debug_logging()
+ log_package_details()
+ log_python_version()
+ log_os_version()
(
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
coursevania_enabled,
) = determine_if_scraper_enabled(
- args.freebiesglobal, args.tutorialbar, args.discudemy, args.coursevania
+ args.idownloadcoupon,
+ args.freebiesglobal,
+ args.tutorialbar,
+ args.discudemy,
+ args.coursevania,
)
run(
args.browser,
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
diff --git a/udemy_enroller/driver_manager.py b/udemy_enroller/driver_manager.py
index d0d46d36..32f21536 100644
--- a/udemy_enroller/driver_manager.py
+++ b/udemy_enroller/driver_manager.py
@@ -1,3 +1,4 @@
+"""Webdriver manager."""
from selenium import webdriver
from selenium.webdriver.chrome.options import Options as ChromeOptions
from webdriver_manager.chrome import ChromeDriverManager
@@ -6,7 +7,7 @@
from webdriver_manager.microsoft import EdgeChromiumDriverManager, IEDriverManager
from webdriver_manager.opera import OperaDriverManager
-from udemy_enroller.logging import get_logger
+from udemy_enroller.logger import get_logger
logger = get_logger()
@@ -21,7 +22,10 @@
class DriverManager:
+ """Webdriver manager."""
+
def __init__(self, browser: str, is_ci_build: bool = False):
+ """Initialize."""
self.driver = None
self.options = None
self.browser = browser
@@ -30,11 +34,10 @@ def __init__(self, browser: str, is_ci_build: bool = False):
def _init_driver(self):
"""
- Initialize the correct web driver based on the users requested browser
+ Initialize the correct web driver based on the users requested browser.
:return: None
"""
-
if self.browser.lower() in VALID_CHROME_STRINGS:
if self.is_ci_build:
self.options = self._build_ci_options_chrome()
@@ -76,7 +79,7 @@ def _init_driver(self):
@staticmethod
def _build_ci_options_chrome():
"""
- Build chrome options required to run in CI
+ Build chrome options required to run in CI.
:return:
"""
diff --git a/udemy_enroller/exceptions.py b/udemy_enroller/exceptions.py
index ba632a21..7fc377ec 100644
--- a/udemy_enroller/exceptions.py
+++ b/udemy_enroller/exceptions.py
@@ -1,14 +1,13 @@
+"""Custom exception module."""
+
+
class RobotException(Exception):
- """
- You have been identified as a robot on Udemy site
- """
+ """You have been identified as a robot on Udemy site."""
pass
class LoginException(Exception):
- """
- You have failed to login to the Udemy site
- """
+ """You have failed to login to the Udemy site."""
pass
diff --git a/udemy_enroller/http.py b/udemy_enroller/http_utils.py
similarity index 77%
rename from udemy_enroller/http.py
rename to udemy_enroller/http_utils.py
index 235755d8..4b62b076 100644
--- a/udemy_enroller/http.py
+++ b/udemy_enroller/http_utils.py
@@ -1,13 +1,14 @@
+"""HTTP helpers."""
import aiohttp
-from udemy_enroller.logging import get_logger
+from udemy_enroller.logger import get_logger
logger = get_logger()
-async def get(url, headers=None):
+async def http_get(url, headers=None):
"""
- Send REST get request to the url passed in
+ Send REST get request to the url passed in.
:param url: The Url to get call get request on
:param headers: The headers to pass with the get request
diff --git a/udemy_enroller/logging.py b/udemy_enroller/logger.py
similarity index 88%
rename from udemy_enroller/logging.py
rename to udemy_enroller/logger.py
index f0e5eac4..3d12b30e 100644
--- a/udemy_enroller/logging.py
+++ b/udemy_enroller/logger.py
@@ -1,3 +1,4 @@
+"""Logger utilities."""
import logging
import logging.config
import os
@@ -6,22 +7,20 @@
class CustomFileHandler(logging.FileHandler):
- """
- Allows us to log to the app directory
- """
+ """Allows us to log to the app directory."""
def __init__(self, file_name="app.log", mode="a"):
+ """Initialize."""
log_file_path = os.path.join(get_app_dir(), file_name)
super(CustomFileHandler, self).__init__(log_file_path, mode)
def load_logging_config() -> None:
"""
- Load logging configuration
+ Load logging configuration.
:return: None
"""
-
my_logger = logging.getLogger("udemy_enroller")
my_logger.setLevel(logging.INFO)
@@ -41,7 +40,7 @@ def load_logging_config() -> None:
def get_logger() -> logging.Logger:
"""
- Convenience method to load the app logger
+ Get the app logger.
:return: An instance of the app logger
"""
diff --git a/udemy_enroller/runner.py b/udemy_enroller/runner.py
index 607cfccf..748d5f63 100644
--- a/udemy_enroller/runner.py
+++ b/udemy_enroller/runner.py
@@ -1,3 +1,4 @@
+"""Runner."""
import asyncio
import random
import time
@@ -17,14 +18,14 @@
UdemyStatus,
exceptions,
)
-from udemy_enroller.logging import get_logger
+from udemy_enroller.logger import get_logger
logger = get_logger()
def _redeem_courses(settings: Settings, scrapers: ScraperManager) -> None:
"""
- Method to scrape courses from the supported sites and enroll in them on udemy
+ Scrape courses from the supported sites and enroll in them on udemy.
:param Settings settings: Core settings used for Udemy
:param ScraperManager scrapers:
@@ -59,7 +60,7 @@ def _redeem_courses(settings: Settings, scrapers: ScraperManager) -> None:
if settings.is_ci_build:
logger.info("We have attempted to subscribe to 1 udemy course")
logger.info("Ending test")
- return
+ return # noqa: B012
else:
udemy_actions.stats.table()
logger.info("All scrapers complete")
@@ -68,6 +69,7 @@ def _redeem_courses(settings: Settings, scrapers: ScraperManager) -> None:
def redeem_courses(
settings: Settings,
+ idownloadcoupon_enabled: bool,
freebiesglobal_enabled: bool,
tutorialbar_enabled: bool,
discudemy_enabled: bool,
@@ -75,9 +77,10 @@ def redeem_courses(
max_pages: Union[int, None],
) -> None:
"""
- Wrapper of _redeem_courses which catches unhandled exceptions
+ Wrap _redeem_courses to catch unhandled exceptions.
:param Settings settings: Core settings used for Udemy
+ :param bool idownloadcoupon_enabled: Boolean signifying if idownloadcoupon scraper should run
:param bool freebiesglobal_enabled: Boolean signifying if freebiesglobal scraper should run
:param bool tutorialbar_enabled: Boolean signifying if tutorialbar scraper should run
:param bool discudemy_enabled: Boolean signifying if discudemy scraper should run
@@ -87,6 +90,7 @@ def redeem_courses(
"""
try:
scrapers = ScraperManager(
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
@@ -104,7 +108,7 @@ def _redeem_courses_ui(
scrapers: ScraperManager,
) -> None:
"""
- Method to scrape courses from the supported sites and enroll in them on udemy.
+ Scrape courses from the supported sites and enroll in them on udemy.
:param WebDriver driver: WebDriver to use to complete enrolment
:param Settings settings: Core settings used for Udemy
@@ -150,7 +154,7 @@ def _redeem_courses_ui(
if settings.is_ci_build:
logger.info("We have attempted to subscribe to 1 udemy course")
logger.info("Ending test")
- return
+ return # noqa: B012
else:
udemy_actions.stats.table()
logger.info("All scrapers complete")
@@ -160,6 +164,7 @@ def _redeem_courses_ui(
def redeem_courses_ui(
driver,
settings: Settings,
+ idownloadcoupon_enabled: bool,
freebiesglobal_enabled: bool,
tutorialbar_enabled: bool,
discudemy_enabled: bool,
@@ -167,10 +172,11 @@ def redeem_courses_ui(
max_pages: Union[int, None],
) -> None:
"""
- Wrapper of _redeem_courses so we always close browser on completion
+ Wrap _redeem_courses so we always close browser on completion.
:param WebDriver driver: WebDriver to use to complete enrolment
:param Settings settings: Core settings used for Udemy
+ :param bool idownloadcoupon_enabled: Boolean signifying if idownloadcoupon scraper should run
:param bool freebiesglobal_enabled: Boolean signifying if freebiesglobal scraper should run
:param bool tutorialbar_enabled: Boolean signifying if tutorialbar scraper should run
:param bool discudemy_enabled: Boolean signifying if discudemy scraper should run
@@ -178,9 +184,9 @@ def redeem_courses_ui(
:param int max_pages: Max pages to scrape from sites (if pagination exists)
:return:
"""
-
try:
scrapers = ScraperManager(
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
diff --git a/udemy_enroller/scrapers/__init__.py b/udemy_enroller/scrapers/__init__.py
index e69de29b..c8882251 100644
--- a/udemy_enroller/scrapers/__init__.py
+++ b/udemy_enroller/scrapers/__init__.py
@@ -0,0 +1 @@
+"""Scrapers module."""
diff --git a/udemy_enroller/scrapers/base_scraper.py b/udemy_enroller/scrapers/base_scraper.py
index 57ce7f86..ed5ccd78 100644
--- a/udemy_enroller/scrapers/base_scraper.py
+++ b/udemy_enroller/scrapers/base_scraper.py
@@ -1,6 +1,8 @@
+"""Base Scraper."""
import datetime
import logging
import re
+import typing
from abc import ABC, abstractmethod
from enum import Enum
from typing import Optional
@@ -9,13 +11,18 @@
class ScraperStates(Enum):
+ """Scraper states."""
+
DISABLED = "DISABLED"
RUNNING = "RUNNING"
COMPLETE = "COMPLETE"
class BaseScraper(ABC):
+ """Base scraper logic."""
+
def __init__(self):
+ """Initialize."""
self._state = None
self.scraper_name = None
self.max_pages = None
@@ -24,40 +31,50 @@ def __init__(self):
@abstractmethod
async def run(self):
+ """Run method that must be implemented in subclasses."""
return
@abstractmethod
- async def get_links(self):
+ async def get_links(self) -> typing.List[str]:
+ """Get links method that must be implemented in subclasses."""
return
@property
- def state(self):
+ def state(self) -> str:
+ """State property."""
return self._state
@state.setter
- def state(self, value):
+ def state(self, value) -> None:
+ """Set the state of the scraper."""
if any([ss for ss in ScraperStates if ss.value == value]):
self._state = value
- def set_state_disabled(self):
+ def set_state_disabled(self) -> None:
+ """Set state to disable."""
self.state = ScraperStates.DISABLED.value
logger.info(f"{self.scraper_name} scraper disabled")
- def set_state_running(self):
+ def set_state_running(self) -> None:
+ """Set state to running."""
self.state = ScraperStates.RUNNING.value
logger.info(f"{self.scraper_name} scraper is running")
- def set_state_complete(self):
+ def set_state_complete(self) -> None:
+ """Set state to complete."""
self.state = ScraperStates.COMPLETE.value
logger.info(f"{self.scraper_name} scraper complete")
- def is_disabled(self):
+ def is_disabled(self) -> bool:
+ """Determine whether a scraper is disabled."""
return self.state == ScraperStates.DISABLED.value
- def is_complete(self):
+ def is_complete(self) -> bool:
+ """Determine whether a scraper has completed."""
return self.state == ScraperStates.COMPLETE.value
- def should_run(self):
+ def should_run(self) -> bool:
+ """Determine whether a scraper should run."""
should_run = not self.is_disabled() and not self.is_complete()
if should_run:
self.set_state_running()
@@ -65,12 +82,16 @@ def should_run(self):
@staticmethod
def time_run(func):
+ """Log execution time of the function that is wrapped."""
+
async def wrapper(self):
start_time = datetime.datetime.utcnow()
try:
response = await func(self)
except Exception as e:
- logger.error(f"Error while running {self.scraper_name} scraper: {e}")
+ logger.exception(
+ f"Error while running {self.scraper_name} scraper: {e}"
+ )
self.is_complete()
return []
end_time = datetime.datetime.utcnow()
@@ -83,11 +104,10 @@ async def wrapper(self):
def max_pages_reached(self) -> bool:
"""
- Returns boolean of whether or not we should continue checking tutorialbar.com
+ Return a boolean of whether we should continue checking site.
:return:
"""
-
should_run = True
if self.max_pages is not None:
@@ -108,11 +128,11 @@ def max_pages_reached(self) -> bool:
return should_run
@staticmethod
- def validate_coupon_url(url) -> Optional[str]:
+ def validate_coupon_url(url: str) -> Optional[str]:
"""
- Validate the udemy coupon url passed in
- If it matches the pattern it is returned else it returns None
+ Validate the udemy coupon url passed in.
+ If it matches the pattern it is returned else it returns None
:param url: The url to check the udemy coupon pattern for
:return: The validated url or None
"""
diff --git a/udemy_enroller/scrapers/coursevania.py b/udemy_enroller/scrapers/coursevania.py
index 80616056..44adca56 100644
--- a/udemy_enroller/scrapers/coursevania.py
+++ b/udemy_enroller/scrapers/coursevania.py
@@ -1,25 +1,25 @@
+"""Coursevania Scraper."""
import asyncio
import json
-import logging
from typing import List
from urllib.parse import urlencode
from bs4 import BeautifulSoup
-from udemy_enroller.http import get
+from udemy_enroller.http_utils import http_get
+from udemy_enroller.logger import get_logger
from udemy_enroller.scrapers.base_scraper import BaseScraper
-logger = logging.getLogger("udemy_enroller")
+logger = get_logger()
class CoursevaniaScraper(BaseScraper):
- """
- Contains any logic related to scraping of data from coursevania.com
- """
+ """Contains any logic related to scraping of data from coursevania.com."""
DOMAIN = "https://coursevania.com"
def __init__(self, enabled, max_pages=None):
+ """Initialize."""
super().__init__()
self.scraper_name = "coursevania"
if not enabled:
@@ -31,7 +31,7 @@ def __init__(self, enabled, max_pages=None):
@BaseScraper.time_run
async def run(self) -> List:
"""
- Called to gather the udemy links
+ Gather the udemy links.
:return: List of udemy course links
"""
@@ -44,7 +44,7 @@ async def run(self) -> List:
async def get_links(self):
"""
- Scrape udemy links from coursevania.com
+ Scrape udemy links from coursevania.com.
:return: List of udemy course urls
"""
@@ -61,12 +61,12 @@ async def get_links(self):
async def load_nonce(self) -> None:
"""
- Load the nonce value needed to load the correct page data
+ Load the nonce value needed to load the correct page data.
:return: None
"""
if self._nonce is None:
- response = await get(f"{self.DOMAIN}/courses")
+ response = await http_get(f"{self.DOMAIN}/courses")
if response is not None:
soup = BeautifulSoup(response, "html.parser")
for script_element in soup.find_all("script"):
@@ -78,7 +78,7 @@ async def load_nonce(self) -> None:
async def get_course_links(self) -> List:
"""
- Gets the url of pages which contain the udemy link we want to get
+ Get the url of pages which contain the udemy link we want to get.
:return: list of pages on coursevania.com that contain Udemy coupons
"""
@@ -102,7 +102,7 @@ async def get_course_links(self) -> List:
"TE": "Trailers",
}
query_string = urlencode(query_params)
- response = await get(
+ response = await http_get(
f"{self.DOMAIN}/wp-admin/admin-ajax.php?{query_string}", headers=headers
)
if response is not None:
@@ -119,12 +119,12 @@ async def get_course_links(self) -> List:
@staticmethod
async def get_udemy_course_link(url: str) -> str:
"""
- Gets the udemy course link
+ Get the udemy course link.
:param str url: The url to scrape data from
:return: Coupon link of the udemy course
"""
- text = await get(url)
+ text = await http_get(url)
if text is not None:
soup = BeautifulSoup(text.decode("utf-8"), "html.parser")
udemy_link = (
@@ -134,7 +134,7 @@ async def get_udemy_course_link(url: str) -> str:
async def gather_udemy_course_links(self, courses: List[str]):
"""
- Async fetching of the udemy course links from coursevania.com
+ Async fetching of the udemy course links from coursevania.com.
:param list courses: A list of coursevania.com course links we want to fetch the udemy links for
:return: list of udemy links
diff --git a/udemy_enroller/scrapers/discudemy.py b/udemy_enroller/scrapers/discudemy.py
index fdb791f8..37f2a3b8 100644
--- a/udemy_enroller/scrapers/discudemy.py
+++ b/udemy_enroller/scrapers/discudemy.py
@@ -1,23 +1,23 @@
+"""Discudemy scraper."""
import asyncio
-import logging
from typing import List
from bs4 import BeautifulSoup
-from udemy_enroller.http import get
+from udemy_enroller.http_utils import http_get
+from udemy_enroller.logger import get_logger
from udemy_enroller.scrapers.base_scraper import BaseScraper
-logger = logging.getLogger("udemy_enroller")
+logger = get_logger()
class DiscUdemyScraper(BaseScraper):
- """
- Contains any logic related to scraping of data from discudemy.com
- """
+ """Contains any logic related to scraping of data from discudemy.com."""
DOMAIN = "https://discudemy.com"
def __init__(self, enabled, max_pages=None):
+ """Initialize."""
super().__init__()
self.scraper_name = "discudemy"
if not enabled:
@@ -27,7 +27,7 @@ def __init__(self, enabled, max_pages=None):
@BaseScraper.time_run
async def run(self) -> List:
"""
- Called to gather the udemy links
+ Gathers the udemy links.
:return: List of udemy course links
"""
@@ -40,13 +40,13 @@ async def run(self) -> List:
async def get_links(self) -> List:
"""
- Scrape udemy links from discudemy.com
+ Scrape udemy links from discudemy.com.
:return: List of udemy course urls
"""
discudemy_links = []
self.current_page += 1
- coupons_data = await get(f"{self.DOMAIN}/all/{self.current_page}")
+ coupons_data = await http_get(f"{self.DOMAIN}/all/{self.current_page}")
soup = BeautifulSoup(coupons_data.decode("utf-8"), "html.parser")
for course_card in soup.find_all("a", class_="card-header"):
url_end = course_card["href"].split("/")[-1]
@@ -64,13 +64,12 @@ async def get_links(self) -> List:
@classmethod
async def get_udemy_course_link(cls, url: str) -> str:
"""
- Gets the udemy course link
+ Get the udemy course link.
:param str url: The url to scrape data from
:return: Coupon link of the udemy course
"""
-
- data = await get(url)
+ data = await http_get(url)
soup = BeautifulSoup(data.decode("utf-8"), "html.parser")
for link in soup.find_all("a", href=True):
udemy_link = cls.validate_coupon_url(link["href"])
@@ -79,7 +78,7 @@ async def get_udemy_course_link(cls, url: str) -> str:
async def gather_udemy_course_links(self, courses: List[str]):
"""
- Async fetching of the udemy course links from discudemy.com
+ Async fetching of the udemy course links from discudemy.com.
:param list courses: A list of discudemy.com course links we want to fetch the udemy links for
:return: list of udemy links
@@ -93,12 +92,11 @@ async def gather_udemy_course_links(self, courses: List[str]):
@staticmethod
def _get_last_page(soup: BeautifulSoup) -> int:
"""
- Extract the last page number to scrape
+ Extract the last page number to scrape.
:param soup:
:return: The last page number to scrape
"""
-
return max(
[
int(i.text)
diff --git a/udemy_enroller/scrapers/freebiesglobal.py b/udemy_enroller/scrapers/freebiesglobal.py
index a9116966..cf3faa33 100644
--- a/udemy_enroller/scrapers/freebiesglobal.py
+++ b/udemy_enroller/scrapers/freebiesglobal.py
@@ -1,23 +1,23 @@
+"""Freebiesglobal Scraper."""
import asyncio
-import logging
from typing import List
from bs4 import BeautifulSoup
-from udemy_enroller.http import get
+from udemy_enroller.http_utils import http_get
+from udemy_enroller.logger import get_logger
from udemy_enroller.scrapers.base_scraper import BaseScraper
-logger = logging.getLogger("udemy_enroller")
+logger = get_logger()
class FreebiesglobalScraper(BaseScraper):
- """
- Contains any logic related to scraping of data from Freebiesglobal.com
- """
+ """Contains any logic related to scraping of data from Freebiesglobal.com."""
DOMAIN = "https://freebiesglobal.com"
def __init__(self, enabled, max_pages=None):
+ """Initialize."""
super().__init__()
self.scraper_name = "freebiesglobal"
if not enabled:
@@ -27,7 +27,7 @@ def __init__(self, enabled, max_pages=None):
@BaseScraper.time_run
async def run(self) -> List:
"""
- Called to gather the udemy links
+ Gathers the udemy links.
:return: List of udemy course links
"""
@@ -40,13 +40,13 @@ async def run(self) -> List:
async def get_links(self) -> List:
"""
- Scrape udemy links from freebiesglobal.com
+ Scrape udemy links from freebiesglobal.com.
:return: List of udemy course urls
"""
freebiesglobal_links = []
self.current_page += 1
- coupons_data = await get(
+ coupons_data = await http_get(
f"{self.DOMAIN}/dealstore/udemy/page/{self.current_page}"
)
soup = BeautifulSoup(coupons_data.decode("utf-8"), "html.parser")
@@ -69,13 +69,12 @@ async def get_links(self) -> List:
@classmethod
async def get_udemy_course_link(cls, url: str) -> str:
"""
- Gets the udemy course link
+ Get the udemy course link.
:param str url: The url to scrape data from
:return: Coupon link of the udemy course
"""
-
- data = await get(url)
+ data = await http_get(url)
soup = BeautifulSoup(data.decode("utf-8"), "html.parser")
for link in soup.find_all("a", class_="re_track_btn"):
udemy_link = cls.validate_coupon_url(link["href"])
@@ -85,7 +84,7 @@ async def get_udemy_course_link(cls, url: str) -> str:
async def gather_udemy_course_links(self, courses: List[str]):
"""
- Async fetching of the udemy course links from freebiesglobal.com
+ Async fetching of the udemy course links from freebiesglobal.com.
:param list courses: A list of freebiesglobal.com course links we want to fetch the udemy links for
:return: list of udemy links
@@ -99,12 +98,11 @@ async def gather_udemy_course_links(self, courses: List[str]):
@staticmethod
def _get_last_page(soup: BeautifulSoup) -> int:
"""
- Extract the last page number to scrape
+ Extract the last page number to scrape.
:param soup:
:return: The last page number to scrape
"""
-
return max(
[
int(i.text)
diff --git a/udemy_enroller/scrapers/idownloadcoupon.py b/udemy_enroller/scrapers/idownloadcoupon.py
new file mode 100644
index 00000000..84becd1e
--- /dev/null
+++ b/udemy_enroller/scrapers/idownloadcoupon.py
@@ -0,0 +1,106 @@
+"""IDownloadCoupon scraper."""
+import asyncio
+import urllib.parse
+from typing import List
+
+from bs4 import BeautifulSoup
+
+from udemy_enroller.http_utils import http_get
+from udemy_enroller.logger import get_logger
+from udemy_enroller.scrapers.base_scraper import BaseScraper
+
+logger = get_logger()
+
+
+class IDownloadCouponScraper(BaseScraper):
+ """Contains any logic related to scraping of site data."""
+
+ DOMAIN = "https://www.idownloadcoupon.com"
+
+ def __init__(self, enabled, max_pages=None):
+ """Initialize."""
+ super().__init__()
+ self.scraper_name = "idownloadcoupon"
+ if not enabled:
+ self.set_state_disabled()
+ self.last_page = None
+ self.max_pages = max_pages
+
+ @BaseScraper.time_run
+ async def run(self) -> List:
+ """
+ Run the steps to scrape links.
+
+ :return: list of udemy coupon links
+ """
+ links = await self.get_links()
+ self.max_pages_reached()
+ return links
+
+ async def get_links(self):
+ """
+ Scrape udemy links.
+
+ :return: List of udemy course urls
+ """
+ self.current_page += 1
+ course_links = await self.get_course_links(
+ f"{self.DOMAIN}/page/{self.current_page}/"
+ )
+
+ logger.info(
+ f"Page: {self.current_page} of {self.last_page} scraped from {self.scraper_name}"
+ )
+ udemy_links = await self.gather_udemy_course_links(course_links)
+
+ for counter, course in enumerate(udemy_links):
+ logger.debug(f"Received Link {counter + 1} : {course}")
+
+ return udemy_links
+
+ async def get_course_links(self, url: str) -> List:
+ """
+ Get the url of pages which contain the udemy link we want to get.
+
+ :param str url: The url to scrape data from
+ :return: list of pages that contain Udemy coupons
+ """
+ text = await http_get(url)
+ if text is not None:
+ soup = BeautifulSoup(text.decode("utf-8"), "html.parser")
+
+ links = soup.find_all("li", class_="product")
+ course_links = [link.find_all("a")[1].get("href") for link in links]
+
+ self.last_page = int(
+ soup.find("ul", class_="page-numbers")
+ .find_all("a", class_="page-numbers")[-2]
+ .text.replace(",", "")
+ )
+
+ return course_links
+
+ @classmethod
+ async def get_udemy_course_link(cls, url: str) -> str:
+ """
+ Get the udemy course link.
+
+ :param str url: The url to scrape data from
+ :return: Coupon link of the udemy course
+ """
+ urls = url.split("murl=")
+ if urls:
+ return cls.validate_coupon_url(urllib.parse.unquote(urls[1]))
+
+ async def gather_udemy_course_links(self, courses: List[str]):
+ """
+ Async fetching of the udemy course links.
+
+ :param list courses: A list of course links we want to fetch the udemy links for
+ :return: list of udemy links
+ """
+ return [
+ link
+ for link in await asyncio.gather(*map(self.get_udemy_course_link, courses))
+ if link is not None
+ ]
diff --git a/udemy_enroller/scrapers/manager.py b/udemy_enroller/scrapers/manager.py
index d03e4677..19f7a7ea 100644
--- a/udemy_enroller/scrapers/manager.py
+++ b/udemy_enroller/scrapers/manager.py
@@ -1,22 +1,32 @@
+"""Manager for scapers."""
import asyncio
+import typing
from functools import reduce
-from typing import List
from udemy_enroller.scrapers.coursevania import CoursevaniaScraper
from udemy_enroller.scrapers.discudemy import DiscUdemyScraper
from udemy_enroller.scrapers.freebiesglobal import FreebiesglobalScraper
+from udemy_enroller.scrapers.idownloadcoupon import IDownloadCouponScraper
from udemy_enroller.scrapers.tutorialbar import TutorialBarScraper
class ScraperManager:
+ """Manages the scrapers."""
+
def __init__(
self,
+ idownloadcoupon_enabled,
freebiesglobal_enabled,
tutorialbar_enabled,
discudemy_enabled,
coursevania_enabled,
max_pages,
):
+ """Initialize."""
+ self.idownloadcoupons_scraper = IDownloadCouponScraper(
+ idownloadcoupon_enabled, max_pages=max_pages
+ )
+
self.freebiesglobal_scraper = FreebiesglobalScraper(
freebiesglobal_enabled, max_pages=max_pages
)
@@ -30,15 +40,16 @@ def __init__(
coursevania_enabled, max_pages=max_pages
)
self._scrapers = (
+ self.idownloadcoupons_scraper,
self.freebiesglobal_scraper,
self.tutorialbar_scraper,
self.discudemy_scraper,
self.coursevania_scraper,
)
- async def run(self) -> List:
+ async def run(self) -> typing.List[str]:
"""
- Runs any enabled scrapers and returns a list of links
+ Run any enabled scrapers and returns a list of links.
:return: list
"""
@@ -51,9 +62,9 @@ async def run(self) -> List:
)
return urls
- def _enabled_scrapers(self) -> List:
+ def _enabled_scrapers(self) -> typing.List:
"""
- Returns a list of scrapers that should run
+ Return a list of scrapers that should run.
:return:
"""
diff --git a/udemy_enroller/scrapers/tutorialbar.py b/udemy_enroller/scrapers/tutorialbar.py
index b293d446..efac7ddd 100644
--- a/udemy_enroller/scrapers/tutorialbar.py
+++ b/udemy_enroller/scrapers/tutorialbar.py
@@ -1,24 +1,24 @@
+"""Tutorialbar scraper."""
import asyncio
-import logging
from typing import List
from bs4 import BeautifulSoup
-from udemy_enroller.http import get
+from udemy_enroller.http_utils import http_get
+from udemy_enroller.logger import get_logger
from udemy_enroller.scrapers.base_scraper import BaseScraper
-logger = logging.getLogger("udemy_enroller")
+logger = get_logger()
class TutorialBarScraper(BaseScraper):
- """
- Contains any logic related to scraping of data from tutorialbar.com
- """
+ """Contains any logic related to scraping of data from tutorialbar.com."""
DOMAIN = "https://www.tutorialbar.com"
AD_DOMAINS = ("https://amzn", "https://bit.ly")
def __init__(self, enabled, max_pages=None):
+ """Initialize."""
super().__init__()
self.scraper_name = "tutorialbar"
if not enabled:
@@ -29,7 +29,7 @@ def __init__(self, enabled, max_pages=None):
@BaseScraper.time_run
async def run(self) -> List:
"""
- Runs the steps to scrape links from tutorialbar.com
+ Run the steps to scrape links from tutorialbar.com.
:return: list of udemy coupon links
"""
@@ -39,7 +39,7 @@ async def run(self) -> List:
async def get_links(self):
"""
- Scrape udemy links from tutorialbar.com
+ Scrape udemy links from tutorialbar.com.
:return: List of udemy course urls
"""
@@ -61,7 +61,7 @@ async def get_links(self):
def _filter_ad_domains(self, udemy_links) -> List:
"""
- Filter out any known ad domains from the links scraped
+ Filter out any known ad domains from the links scraped.
:param list udemy_links: List of urls to filter ad domains from
:return: A list of filtered urls
@@ -77,12 +77,12 @@ def _filter_ad_domains(self, udemy_links) -> List:
async def get_course_links(self, url: str) -> List:
"""
- Gets the url of pages which contain the udemy link we want to get
+ Get the url of pages which contain the udemy link we want to get.
:param str url: The url to scrape data from
:return: list of pages on tutorialbar.com that contain Udemy coupons
"""
- text = await get(url)
+ text = await http_get(url)
if text is not None:
soup = BeautifulSoup(text.decode("utf-8"), "html.parser")
@@ -100,13 +100,12 @@ async def get_course_links(self, url: str) -> List:
@staticmethod
async def get_udemy_course_link(url: str) -> str:
"""
- Gets the udemy course link
+ Get the udemy course link.
:param str url: The url to scrape data from
:return: Coupon link of the udemy course
"""
-
- text = await get(url)
+ text = await http_get(url)
if text is not None:
soup = BeautifulSoup(text.decode("utf-8"), "html.parser")
udemy_link = (
@@ -116,7 +115,7 @@ async def get_udemy_course_link(url: str) -> str:
async def gather_udemy_course_links(self, courses: List[str]):
"""
- Async fetching of the udemy course links from tutorialbar.com
+ Async fetching of the udemy course links from tutorialbar.com.
:param list courses: A list of tutorialbar.com course links we want to fetch the udemy links for
:return: list of udemy links
diff --git a/udemy_enroller/settings.py b/udemy_enroller/settings.py
index 747d023b..dbb16399 100644
--- a/udemy_enroller/settings.py
+++ b/udemy_enroller/settings.py
@@ -1,3 +1,4 @@
+"""Settings."""
import getpass
import os.path
from distutils.util import strtobool
@@ -5,20 +6,19 @@
from ruamel.yaml import YAML, dump
-from udemy_enroller.logging import get_logger
+from udemy_enroller.logger import get_logger
from udemy_enroller.utils import get_app_dir
logger = get_logger()
class Settings:
- """
- Contains all logic related to the scripts settings
- """
+ """Contains all logic related to the scripts settings."""
def __init__(
self, delete_settings=False, delete_cookie=False, settings_path="settings.yaml"
):
+ """Initialize."""
self.email = None
self.password = None
self.zip_code = None
@@ -38,7 +38,7 @@ def __init__(
def _init_settings(self) -> None:
"""
- Initialize the settings to be used in the script
+ Initialize the settings to be used in the script.
:return:
"""
@@ -52,7 +52,7 @@ def _init_settings(self) -> None:
def _load_ci_settings(self):
"""
- Load environment variables for CI run
+ Load environment variables for CI run.
:return:
"""
@@ -62,7 +62,7 @@ def _load_ci_settings(self):
def _load_user_settings(self) -> Dict:
"""
- Loads the settings from the yaml file if it exists
+ Load the settings from the yaml file if it exists.
:return: dictionary containing the script settings
"""
@@ -84,7 +84,7 @@ def _load_user_settings(self) -> Dict:
def _generate_settings(self) -> None:
"""
- Generate the settings for the script
+ Generate the settings for the script.
:return:
"""
@@ -96,7 +96,7 @@ def _generate_settings(self) -> None:
def _get_email(self, prompt_save=True) -> Tuple[str, bool]:
"""
- Get input from user on the email to use for udemy
+ Get input from user on the email to use for udemy.
:return: The users udemy email and if it should be saved
"""
@@ -113,7 +113,7 @@ def _get_email(self, prompt_save=True) -> Tuple[str, bool]:
def _get_password(self, prompt_save=True) -> Tuple[str, bool]:
"""
- Get input from user on the password to use for udemy
+ Get input from user on the password to use for udemy.
:return: The users udemy password and if it should be saved
"""
@@ -133,7 +133,7 @@ def _get_password(self, prompt_save=True) -> Tuple[str, bool]:
@staticmethod
def _get_zip_code() -> str:
"""
- Get input from user on the zip code to use for udemy
+ Get input from user on the zip code to use for udemy.
:return: The users udemy zip code
"""
@@ -143,7 +143,7 @@ def _get_zip_code() -> str:
@staticmethod
def _get_languages() -> List[str]:
"""
- Get input from user on the languages they want to get courses in
+ Get input from user on the languages they want to get courses in.
:return: list of languages the user wants to redeem udemy courses in
"""
@@ -154,9 +154,11 @@ def _get_languages() -> List[str]:
@staticmethod
def _get_categories() -> List[str]:
- """Gets the categories the user wants.
+ """
+ Get the categories the user wants.
- :return: list of categories the user wants."""
+ :return: list of categories the user wants.
+ """
categories = input(
"Please enter in a list of comma separated values of"
" the course categories you like, for example:\n"
@@ -170,7 +172,7 @@ def _get_categories() -> List[str]:
def _save_settings(self) -> None:
"""
- Confirm if the user wants to save settings to file
+ Confirm if the user wants to save settings to file.
:return:
"""
@@ -190,7 +192,7 @@ def _save_settings(self) -> None:
# Log some details for the user
if not self._should_store_email:
- logger.info(f"Your email has not been saved to settings.")
+ logger.info("Your email has not been saved to settings.")
if not self._should_store_password:
logger.info("Your password has not been saved to settings.")
if not self._should_store_email or not self._should_store_password:
@@ -200,7 +202,7 @@ def _save_settings(self) -> None:
def delete_settings(self) -> None:
"""
- Delete the settings file
+ Delete the settings file.
:return: None
"""
@@ -216,7 +218,7 @@ def delete_settings(self) -> None:
def delete_cookie(self) -> None:
"""
- Delete the cookie file
+ Delete the cookie file.
:return: None
"""
@@ -228,7 +230,7 @@ def delete_cookie(self) -> None:
def prompt_email(self) -> None:
"""
- Prompt for Udemy email only. Does not prompt for saving
+ Prompt for Udemy email only. Does not prompt for saving.
:return: None
"""
@@ -236,7 +238,7 @@ def prompt_email(self) -> None:
def prompt_password(self) -> None:
"""
- Prompt for Udemy password only. Does not prompt for saving
+ Prompt for Udemy password only. Does not prompt for saving.
:return: None
"""
diff --git a/udemy_enroller/udemy_rest.py b/udemy_enroller/udemy_rest.py
index 980b2b46..554ce720 100644
--- a/udemy_enroller/udemy_rest.py
+++ b/udemy_enroller/udemy_rest.py
@@ -1,3 +1,4 @@
+"""Udemy REST."""
import json
import os
import re
@@ -10,7 +11,7 @@
from bs4 import BeautifulSoup
from cloudscraper import create_scraper
-from udemy_enroller.logging import get_logger
+from udemy_enroller.logger import get_logger
from udemy_enroller.settings import Settings
from udemy_enroller.utils import get_app_dir
@@ -18,9 +19,7 @@
def format_requests(func):
- """
- Convenience method for handling requests response
- """
+ """Handle requests response."""
def formatting(*args, **kwargs):
result = func(*args, **kwargs)
@@ -32,6 +31,8 @@ def formatting(*args, **kwargs):
@dataclass(unsafe_hash=True)
class RunStatistics:
+ """Gather statistics on courses enrolled in."""
+
prices: List[float] = field(default_factory=list)
expired: int = 0
@@ -49,9 +50,11 @@ class RunStatistics:
currency_symbol = "$"
def savings(self):
+ """Calculate the savings made from enrolling to these courses."""
return sum(self.prices) or 0
def table(self):
+ """Log table of statistics to output."""
logger.info("================== Run Statistics ==================")
logger.info(f"Enrolled: {self.enrolled}")
logger.info(f"Unwanted Category: {self.unwanted_category}")
@@ -66,9 +69,7 @@ def table(self):
class UdemyStatus(Enum):
- """
- Possible statuses of udemy course
- """
+ """Possible statuses of udemy course."""
ALREADY_ENROLLED = "ALREADY_ENROLLED"
ENROLLED = "ENROLLED"
@@ -78,6 +79,8 @@ class UdemyStatus(Enum):
class UdemyActions:
+ """Udemy Actions."""
+
LOGIN_URL = "https://www.udemy.com/join/login-popup/?locale=en_US"
MY_COURSES = (
"https://www.udemy.com/api-2.0/users/me/subscribed-courses/?ordering=-last_accessed&fields["
@@ -106,6 +109,7 @@ class UdemyActions:
}
def __init__(self, settings: Settings, cookie_file_name: str = ".cookie"):
+ """Initialize."""
self.settings = settings
self.user_has_preferences = self.settings.categories or self.settings.languages
self.session = requests.Session()
@@ -120,7 +124,8 @@ def __init__(self, settings: Settings, cookie_file_name: str = ".cookie"):
def login(self, retry=False) -> None:
"""
- Login to Udemy using REST api
+ Login to Udemy using REST api.
+
Saves login cookies for future use
:return: None
@@ -129,8 +134,7 @@ def login(self, retry=False) -> None:
if cookie_details is None:
response = self.udemy_scraper.get(self.LOGIN_URL)
soup = BeautifulSoup(response.content, "html.parser")
- csrf_element = soup.find("input", {"name": "csrfmiddlewaretoken"}) or {}
- csrf_token = csrf_element.get("value")
+ csrf_token = response.cookies.get("csrftoken")
if csrf_token is None:
raise Exception("Unable to get csrf_token")
@@ -207,7 +211,7 @@ def login(self, retry=False) -> None:
def load_my_courses(self) -> List:
"""
- Loads users currently enrolled courses from Udemy
+ Load users currently enrolled courses from Udemy.
:return: List of logged in users courses
"""
@@ -215,10 +219,12 @@ def load_my_courses(self) -> List:
all_courses = list()
page_size = 100
- my_courses = self.my_courses(1, page_size)
+ page = 1
+ my_courses = self.my_courses(page, page_size)
all_courses.extend(my_courses["results"])
- total_pages = my_courses["count"] // page_size
- for page in range(2, total_pages + 2):
+
+ while "next" in my_courses and my_courses["next"] is not None:
+ page += 1
my_courses = self.my_courses(page, page_size)
if "results" in my_courses:
all_courses.extend(my_courses["results"])
@@ -229,7 +235,7 @@ def load_my_courses(self) -> List:
@format_requests
def load_user_details(self):
"""
- Load the current users details
+ Load the current users details.
:return: Dict containing the users details
"""
@@ -237,7 +243,7 @@ def load_user_details(self):
def is_enrolled(self, course_id: int) -> bool:
"""
- Check if the user is currently enrolled in the course based on course_id passed in
+ Check if the user is currently enrolled in the course based on course_id passed in.
:param int course_id: Check if the course_id is in the users current courses
:return:
@@ -246,7 +252,7 @@ def is_enrolled(self, course_id: int) -> bool:
def _add_enrolled_course(self, course_id):
"""
- Add enrolled course to the list of enrolled course ids
+ Add enrolled course to the list of enrolled course ids.
:param int course_id: The course_id to add to the list
:return:
@@ -258,7 +264,7 @@ def is_coupon_valid(
self, course_id: int, coupon_code: str, course_identifier: str
) -> bool:
"""
- Check if the coupon is valid for a course
+ Check if the coupon is valid for a course.
:param int course_id: Id of the course to check the coupon against
:param str coupon_code: Coupon to apply to the course
@@ -294,7 +300,7 @@ def is_preferred_language(
self, course_details: Dict, course_identifier: str
) -> bool:
"""
- Check if the course is in one of the languages preferred by the user
+ Check if the course is in one of the languages preferred by the user.
:param dict course_details: Dictionary containing course details from Udemy
:param str course_identifier: Name of the course used for logging
@@ -314,7 +320,7 @@ def is_preferred_category(
self, course_details: Dict, course_identifier: str
) -> bool:
"""
- Check if the course is in one of the categories preferred by the user
+ Check if the course is in one of the categories preferred by the user.
:param dict course_details: Dictionary containing course details from Udemy
:param str course_identifier: Name of the course used for logging
@@ -336,7 +342,7 @@ def is_preferred_category(
@format_requests
def my_courses(self, page: int, page_size: int) -> Dict:
"""
- Load the current logged in users courses
+ Load the current logged in users courses.
:param int page: page number to load
:param int page_size: number of courses to load per page
@@ -347,7 +353,7 @@ def my_courses(self, page: int, page_size: int) -> Dict:
@format_requests
def coupon_details(self, course_id: int, coupon_code: str) -> Dict:
"""
- Check that the coupon is valid for the current course
+ Check that the coupon is valid for the current course.
:param int course_id: Id of the course to check the coupon against
:param str coupon_code: The coupon_code to check against the course
@@ -358,7 +364,7 @@ def coupon_details(self, course_id: int, coupon_code: str) -> Dict:
@format_requests
def course_details(self, course_id: int) -> Dict:
"""
- Retrieves details relating to the course passed in
+ Retrieve details relating to the course passed in.
:param int course_id: Id of the course to get the details of
:return: dictionary containing the course details
@@ -367,7 +373,7 @@ def course_details(self, course_id: int) -> Dict:
def enroll(self, course_link: str) -> str:
"""
- Enroll the current user in the course provided
+ Enroll the current user in the course provided.
:param str course_link: Link to the course with valid coupon attached
:return: str representing the status of the enrolment
@@ -409,7 +415,7 @@ def enroll(self, course_link: str) -> str:
def _get_course_id(self, url: str) -> int:
"""
- Get the course id from the url provided
+ Get the course id from the url provided.
:param str url: Udemy url to fetch the course from
:return: int representing the course id
@@ -428,7 +434,7 @@ def _checkout(
retry: bool = False,
) -> str:
"""
- Checkout process for the course and coupon provided
+ Checkout process for the course and coupon provided.
:param int course_id: The course id of the course to enroll in
:param str coupon_code: The coupon code to apply on checkout
@@ -465,7 +471,7 @@ def _checkout(
def _build_checkout_payload(self, course_id: int, coupon_code: str) -> Dict:
"""
- Build the payload for checkout
+ Build the payload for checkout.
:param int course_id: The course id to checkout
:param str coupon_code: The coupon code to use at checkout
@@ -489,7 +495,7 @@ def _build_checkout_payload(self, course_id: int, coupon_code: str) -> Dict:
def _cache_cookies(self, cookies: Dict) -> None:
"""
- Caches cookies for future logins
+ Cache cookies for future logins.
:param cookies:
:return:
@@ -500,7 +506,7 @@ def _cache_cookies(self, cookies: Dict) -> None:
def _load_cookies(self) -> Dict:
"""
- Loads existing cookie file
+ Load existing cookie file.
:return:
"""
@@ -516,7 +522,7 @@ def _load_cookies(self) -> Dict:
def _delete_cookies(self) -> None:
"""
- Remove existing cookie file
+ Remove existing cookie file.
:return:
"""
diff --git a/udemy_enroller/udemy_ui.py b/udemy_enroller/udemy_ui.py
index a7c34f9b..170c791f 100644
--- a/udemy_enroller/udemy_ui.py
+++ b/udemy_enroller/udemy_ui.py
@@ -1,3 +1,4 @@
+"""Udemy UI."""
import time
from dataclasses import dataclass, field
from datetime import datetime
@@ -13,7 +14,7 @@
from selenium.webdriver.support.ui import WebDriverWait
from udemy_enroller.exceptions import LoginException, RobotException
-from udemy_enroller.logging import get_logger
+from udemy_enroller.logger import get_logger
from udemy_enroller.settings import Settings
logger = get_logger()
@@ -21,6 +22,8 @@
@dataclass(unsafe_hash=True)
class RunStatistics:
+ """Gather statistics on courses enrolled in."""
+
prices: List[Decimal] = field(default_factory=list)
expired: int = 0
@@ -33,10 +36,12 @@ class RunStatistics:
currency_symbol = None
- def savings(self):
+ def savings(self) -> int:
+ """Calculate the savings made from enrolling to these courses."""
return sum(self.prices) or 0
def table(self):
+ """Log table of statistics to output."""
# Only show the table if we have something to show
if self.prices:
if self.currency_symbol is None:
@@ -59,9 +64,7 @@ def table(self):
class UdemyStatus(Enum):
- """
- Possible statuses of udemy course
- """
+ """Possible statuses of udemy course."""
ALREADY_ENROLLED = "ALREADY_ENROLLED"
ENROLLED = "ENROLLED"
@@ -71,13 +74,12 @@ class UdemyStatus(Enum):
class UdemyActionsUI:
- """
- Contains any logic related to interacting with udemy website
- """
+ """Contains any logic related to interacting with udemy website."""
DOMAIN = "https://www.udemy.com"
def __init__(self, driver: WebDriver, settings: Settings):
+ """Initialize."""
self.driver = driver
self.settings = settings
self.logged_in = False
@@ -86,7 +88,7 @@ def __init__(self, driver: WebDriver, settings: Settings):
def login(self, is_retry=False) -> None:
"""
- Login to your udemy account
+ Login to your udemy account.
:param bool is_retry: Is this is a login retry and we still have captcha raise RobotException
@@ -141,7 +143,7 @@ def login(self, is_retry=False) -> None:
def enroll(self, url: str) -> str:
"""
- Redeems the course url passed in
+ Redeems the course url passed in.
:param str url: URL of the course to redeem
:return: A string detailing course status
@@ -197,8 +199,8 @@ def enroll(self, url: str) -> str:
self.settings.zip_code
)
- # After you put the zip code in, the page refreshes itself and disables the enroll button for a split
- # second.
+ # After you put the zip code in, the page refreshes itself and disables
+ # the enroll button for a split second.
enroll_button_is_clickable = EC.element_to_be_clickable(
(By.XPATH, enroll_button_xpath)
)
@@ -298,8 +300,12 @@ def _check_categories(self, course_identifier):
breadcrumbs: WebElement = self.driver.find_element_by_class_name(
breadcrumbs_path
)
- breadcrumbs = breadcrumbs.find_elements_by_class_name(breadcrumbs_text_path)
- breadcrumb_text = [bc.text for bc in breadcrumbs] # Get only the text
+ breadcrumb_elements = breadcrumbs.find_elements_by_class_name(
+ breadcrumbs_text_path
+ )
+ breadcrumb_text = [
+ bc.text for bc in breadcrumb_elements
+ ] # Get only the text
for category in self.settings.categories:
if category in breadcrumb_text:
@@ -351,7 +357,7 @@ def _check_price(self, course_name):
def _check_if_robot(self) -> bool:
"""
- Simply checks if the captcha element is present on login if email/password elements are not
+ Simply checks if the captcha element is present on login if email/password elements are not.
:return: Bool
"""
diff --git a/udemy_enroller/utils.py b/udemy_enroller/utils.py
index 4f183dac..20930cd4 100644
--- a/udemy_enroller/utils.py
+++ b/udemy_enroller/utils.py
@@ -1,9 +1,10 @@
+"""Utility functions."""
import os
def get_app_dir() -> str:
"""
- Gets the app directory where all data related to the script is stored
+ Get the app directory where all data related to the script is stored.
:return:
"""