docs: minor updates (#144)

* docs: update docs dependencies
* dev: update dev dependencies + Dockerimage
* dev: minor refactoring
* dev: remove `mypy` chekcs + workflow changes

.github/workflows/main.yaml

@@ -20,7 +20,7 @@ jobs:
 
       - run: make dev-env
       - run: make pylint-full
-      - run: make mypy
+      # - run: make mypy
 
   Docs:
     runs-on: ubuntu-latest
@@ -33,16 +33,16 @@ jobs:
       - run: make docs-build
 
   Build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [ Static-Analysis, Docs ]
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.6", "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -55,6 +55,6 @@ jobs:
       - run: make tests
 
       - name: codecov.io
-        if: matrix.python-version == "3.11"
+        if: matrix.python-version == 3.11
         run: bash <(curl -s https://codecov.io/bash)
 

Dockerfile

@@ -1,5 +1,5 @@
 # initial builder
-FROM docker.io/python:3.7-slim-stretch as BUILD
+FROM docker.io/python:3.11-slim as BUILD
 
 COPY  . /tmp
 WORKDIR /tmp
@@ -18,13 +18,12 @@ RUN grep "# install" requirements.txt -A100 > docker.requirments.txt \
     && sed -i 's/\/usr\/local/\/usr/g'  /usr/local/bin/deadlinks
 
 
-FROM gcr.io/distroless/python3:latest
+FROM gcr.io/distroless/python3-debian12:debug
 LABEL maintainer "Oleg Butuzov <[email protected]>"
 
-ENV PYTHONPATH=/usr/local/lib/python3.7/site-packages
+ENV PYTHONPATH=/usr/local/lib/python3.11/site-packages
 
 COPY --from=BUILD ${PYTHONPATH} ${PYTHONPATH}
-# COPY --from=BUILD /usr/local/lib/libpython3.7m.so.1.0 /usr/lib/x86_64-linux-gnu/
 COPY --from=BUILD /usr/local/bin/deadlinks /usr/local/bin/
 
 WORKDIR /github/workspace

deadlinks/__init__.py

@@ -24,30 +24,21 @@
 
 # -- Imports -------------------------------------------------------------------
 
-from .url import URL
-from .link import Link
-from .index import Index
+from .__version__ import __version__ as version
 from .baseurl import BaseURL
 from .crawler import Crawler
+from .exceptions import (DeadlinksIgnoredURL, DeadlinksSettingsBase, DeadlinksSettingsChange,
+                         DeadlinksSettingsDomains, DeadlinksSettingsPath, DeadlinksSettingsPathes,
+                         DeadlinksSettingsRetry, DeadlinksSettingsRoot, DeadlinksSettingsThreads)
+from .index import Index
+from .link import Link
 from .request import request, user_agent
 from .settings import Settings
 from .status import Status
-
-from .__version__ import __version__ as version
+from .url import URL
 
 # -- Exports  ------------------------------------------------------------------
 
-from .exceptions import (
-    DeadlinksIgnoredURL,
-    DeadlinksSettingsThreads,
-    DeadlinksSettingsBase,
-    DeadlinksSettingsRetry,
-    DeadlinksSettingsRoot,
-    DeadlinksSettingsChange,
-    DeadlinksSettingsDomains,
-    DeadlinksSettingsPathes,
-    DeadlinksSettingsPath,
-)
 
 __all__ = [
     'version',

deadlinks/__main__.py

@@ -26,26 +26,19 @@
 
 import click
 
-from .settings import Settings
+# Version and App.
+from .__version__ import __app_package__ as name
+from .__version__ import __app_version__ as version
+# CLI implementation related
+from .clicker import Clicker, Options, register_exports, register_options, validate_url
 from .crawler import Crawler
 from .exceptions import DeadlinksException
-
-# CLI implementation related
-from .clicker import (register_options, register_exports)
-from .clicker import Options
-from .clicker import (Clicker, validate_url)
-
 # Exporters
-from .exporters import Export #pylint: disable-msg=W0611
 from .exporters import exporters
-
 # Default and Specified Options
 from .options import default_options as general_options
 from .serving.options import default_options as serving_options
-
-# Version and App.
-from .__version__ import __app_version__ as version
-from .__version__ import __app_package__ as name
+from .settings import Settings
 
 # -- Implementation ------------------------------------------------------------
 

deadlinks/clicker.py

@@ -24,16 +24,16 @@
 
 # -- Imports -------------------------------------------------------------------
 
-from typing import (Dict, List, Tuple, Union, Any, Callable, Sequence)
-from textwrap import dedent
 from collections import OrderedDict
+from textwrap import dedent
+from typing import Any, Callable, Dict, List, Sequence, Tuple, Union
 
-from click import Option, Argument
-from click import Command, Context
+from click import Argument, Command, Context
 from click import HelpFormatter as Formatter
+from click import Option
 
-from .link import Link
 from .__version__ import __app_package__ as app
+from .link import Link
 
 # -- Typing Decorators -----~---------------------------------------------------
 OptionsValues = Union[str, bool, int, List[str], List[Callable]]
@@ -204,7 +204,7 @@ def format_examples(
         with formatter.section('Usage Examples'):
             formatter.write("\n")
             for line in self.EXAMPLES.split("\n"):
-                formatter.write("  {}\n".format(line))
+                formatter.write(f"  {line}\n")
 
     def modify(self, ctx: Context) -> None:
         """ Modifing params based on context. """
@@ -235,12 +235,12 @@ def validate_url(ctx: Context, param: Argument, value: str) -> str:
     url = Link(value)
 
     if not url.is_valid() and not url.scheme:
-        return "http://%s" % value
+        return f"http://{value}"
 
     if "@" in url.path:
         return value
 
     if not url.is_valid() and url.scheme and url.path:
-        return "http://%s:%s" % (url.scheme, url.path)
+        return f"http://{url.scheme}:{url.path}"
 
     return value

deadlinks/crawler.py

@@ -24,25 +24,20 @@
 
 # -- Imports -------------------------------------------------------------------
 
-from typing import (List, Tuple, Optional, Dict)
-from types import FrameType
-
-from threading import Thread
-from signal import (signal, SIGINT)
+from collections import defaultdict
 from queue import Queue
+from signal import SIGINT, signal
+from threading import Thread
 from time import sleep
+from types import FrameType
+from typing import Dict, List, Optional, Tuple
 
-from .link import Link
-from .status import Status
+from .exceptions import DeadlinksIgnoredURL, DeadlinksRedirectionURL
 from .index import Index
+from .link import Link
 from .robots_txt import RobotsTxt
 from .settings import Settings
-from .exceptions import (
-    DeadlinksIgnoredURL,
-    DeadlinksRedirectionURL,
-)
-
-from collections import defaultdict
+from .status import Status
 
 # -- Implementation ------------------------------------------------------------
 
@@ -259,7 +254,8 @@ def internal(self, links: List[Link]) -> List[Link]:
             return links
 
         domain = self.settings.base.domain
-        mapper = lambda x: x.url().replace(domain, 'internal')
+        def mapper(x):
+            return x.url().replace(domain, "internal")
 
         return list(map(Link, map(mapper, links)))
 

deadlinks/exporters/__init__.py

@@ -24,10 +24,10 @@
 
 # -- Imports -------------------------------------------------------------------
 
-from typing import Dict #pylint: disable-msg=W0611
+from typing import Dict
 
-from .export import Export
 from .default import Default
+from .export import Export
 
 # -- Exports -------------------------------------------------------------------
 
@@ -36,6 +36,6 @@
     'Export',
 ]
 
-exporters = { # type Dict[str, Export]
+exporters: Dict[str, Export] = {
     'default': Default,
 }

deadlinks/exporters/default.py

@@ -24,17 +24,16 @@
 
 # -- Imports -------------------------------------------------------------------
 
-from typing import (Dict, Tuple, List, Sequence, Any) #pylint: disable-msg=W0611
-
-from threading import Thread
 from os import name as os_name
+from threading import Thread
 from time import sleep
+from typing import Dict, List, Sequence, Tuple  # pylint: disable-msg=W0611
 
 import click
 
-from .export import Export
-from ..crawler import Crawler
 from ..clicker import OptionRaw
+from ..crawler import Crawler
+from .export import Export
 
 # -- Implementation ------------------------------------------------------------
 
@@ -180,7 +179,7 @@ def _generate(self, key: str) -> str:
 
         param_color = click.style(param, fg=self.params_colors[param])
 
-        return '\n'.join(map(lambda x: "[ {} ] {}".format(param_color, x), links))
+        return '\n'.join(map(lambda x: f"[ {param_color} ] {x}", links))
 
     def report(self) -> None:
 
@@ -217,7 +216,7 @@ def report(self) -> None:
         click.echo(("-"*split_line_len) + "\033[?25h")
 
         # show some url report(s)
-        show = list(self._opts.get('show', [])) # type: Sequence[str]
+        show: Sequence[str] = list(self._opts.get('show', []))
 
         if 'none' in show:
             return

deadlinks/exporters/export.py

@@ -24,11 +24,11 @@
 
 # -- Imports -------------------------------------------------------------------
 
-from typing import (Dict, Tuple)
 from abc import ABC, abstractmethod
+from typing import Dict, Tuple
 
-from ..crawler import Crawler
 from ..clicker import OptionsList
+from ..crawler import Crawler
 
 # -- Abstract ------------------------------------------------------------------
 

deadlinks/index.py

@@ -24,7 +24,7 @@
 
 # -- Imports -------------------------------------------------------------------
 
-from typing import (Dict, List, Iterator, Callable) #pylint: disable-msg=W0611
+from typing import Callable, Dict, Iterator, List  # pylint: disable-msg=W0611
 
 from .link import Link
 from .status import Status
@@ -69,38 +69,32 @@ def put(self, link: Link) -> None:
     def all(self) -> List[Link]:
         """ Return links in the index (but not UNDEFINED). """
 
-        lmbd = lambda x: x.status != Status.UNDEFINED
-        return self._filter(lmbd)
+        return self._filter(lambda x: x.status != Status.UNDEFINED)
 
     def succeed(self) -> List[Link]:
         """ Filters succeed urls from index. """
 
-        lmbd = lambda x: x.status == Status.FOUND
-        return self._filter(lmbd)
+        return self._filter(lambda x: x.status == Status.FOUND)
 
     def redirected(self) -> List[Link]:
         """ Filters succeed urls from index. """
 
-        lmbd = lambda x: x.status == Status.REDIRECTION
-        return self._filter(lmbd)
+        return self._filter(lambda x: x.status == Status.REDIRECTION)
 
     def failed(self) -> List[Link]:
         """ Filters failed urls from index. """
 
-        lmbd = lambda x: x.status == Status.NOT_FOUND
-        return self._filter(lmbd)
+        return self._filter(lambda x: x.status == Status.NOT_FOUND)
 
     def ignored(self) -> List[Link]:
         """ Filters failed urls from index. """
 
-        lmbd = lambda x: x.status == Status.IGNORED
-        return self._filter(lmbd)
+        return self._filter(lambda x: x.status == Status.IGNORED)
 
     def undefined(self) -> List[Link]:
         """ Filters undefined urls from index. """
 
-        lmbd = lambda x: x.status == Status.UNDEFINED
-        return self._filter(lmbd)
+        return self._filter(lambda x: x.status == Status.UNDEFINED)
 
     def update(self, url: Link, status: Status, message: str) -> None:
         """ wraps access to updating url status and gathering stats """
@@ -118,4 +112,4 @@ def get_stats(self) -> Dict[Status, int]:
 
     def _filter(self, lambda_func: Callable[[Link], bool]) -> List[Link]:
         """ Filters  values according lambda. """
-        return list(sorted(filter(lambda_func, self._index.values()))) # type: ignore
+        return list(sorted(filter(lambda_func, self._index.values())))

deadlinks/link.py

@@ -24,8 +24,8 @@
 
 # -- Imports -------------------------------------------------------------------
 
-from typing import Union
 from functools import total_ordering
+from typing import Union
 
 from .url import URL
 
@@ -51,7 +51,7 @@ def is_external(self, url: Union[URL, str]) -> bool:
         if isinstance(url, str):
             url = URL(url)
         elif not isinstance(url, URL):
-            raise TypeError("url of type {}".format(type(url)))
+            raise TypeError(f"url of type {type(url)}")
 
         base_scheme, this_scheme = url.scheme, self.scheme
         base, this = url.domain, self.domain

deadlinks/options.py

@@ -26,13 +26,13 @@
 
 from typing import List
 
-from click import IntRange, Choice
+from click import Choice, IntRange
 
-from .clicker import OptionRaw #pylint: disable-msg=W0611
+from .clicker import OptionRaw
 
 # -- Options -------------------------------------------------------------------
 
-default_options = [] # type: List[OptionRaw]
+default_options: List[OptionRaw] = []
 
 # Index External URLs ------------------------------------------------------
 default_options.append((