Run latest version of flake8 and black (#369)

modyn/common/__init__.py

@@ -1,6 +1,7 @@
 """
 This module contains classes shared among several components.
 """
+
 # TODO(#243): extend package with additional classes and functions
 
 import os

modyn/common/example_extension/__init__.py

@@ -1,6 +1,7 @@
 """
 This module contains classes shared among several components.
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/common/grpc/__init__.py

@@ -1,6 +1,7 @@
 """
 This submodule implements functions to run gRPC servers using multiprocessing.
 """
+
 import os
 
 from .grpc_helpers import GenericGRPCServer  # noqa: F401

modyn/common/trigger_sample/__init__.py

@@ -1,6 +1,7 @@
 """
 This submodule implements necessary functions to persist a calculated trigger training set to disk.
 """
+
 import os
 
 from .trigger_sample_storage import ArrayWrapper, TriggerSampleStorage  # noqa: F401

modyn/database/__init__.py

@@ -1,6 +1,7 @@
 """
 Abstract database connection class.
 """
+
 import os
 
 from .partition_by_meta import PartitionByMeta  # noqa: F401

modyn/evaluator/internal/grpc/evaluator_grpc_servicer.py

@@ -1,4 +1,5 @@
 """Evaluator GRPC servicer."""
+
 import json
 import logging
 import multiprocessing as mp

modyn/metadata_database/__init__.py

@@ -3,6 +3,7 @@
 The models are used to abstract the database operations.
 This allows the storage module to be used with different databases.
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/metadata_database/metadata_base.py

@@ -1,4 +1,5 @@
 """Base class for metadata database."""
+
 from sqlalchemy.orm import DeclarativeBase
 
 

modyn/metadata_database/models/__init__.py

@@ -3,6 +3,7 @@
 The models are used to abstract the database operations.
 This allows the storage module to be used with different databases.
 """
+
 import os
 
 from .pipelines import Pipeline  # noqa: F401

modyn/metadata_database/models/trained_models.py

@@ -1,4 +1,5 @@
 """Trained models downloaded from trainer server"""
+
 from datetime import datetime
 
 from modyn.metadata_database.metadata_base import MetadataBase

modyn/metadata_database/utils/__init__.py

@@ -3,6 +3,7 @@
 The models are used to abstract the database operations.
 This allows the storage module to be used with different databases.
 """
+
 import os
 
 from .model_storage_strategy_config import ModelStorageStrategyConfig  # noqa: F401

modyn/metadata_processor/__init__.py

@@ -1,6 +1,7 @@
 """
 This submodule provides the functionality for the Metadata Processor server component.
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/metadata_processor/internal/grpc/metadata_processor_server.py

@@ -1,4 +1,5 @@
 """GRPC Server Context Manager"""
+
 import logging
 from concurrent import futures
 

modyn/metadata_processor/processor_strategies/__init__.py

@@ -2,6 +2,7 @@
 This module containes extensions of the MetadataProcessorStrategy class that
 implement custom processing strategies.
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/metadata_processor/processor_strategies/processor_strategy_type.py

@@ -1,4 +1,5 @@
 """Processor strategy type and exception."""
+
 from enum import Enum
 
 

modyn/models/__init__.py

@@ -1,6 +1,7 @@
 """Models.
 
 """
+
 import os
 
 from .articlenet.articlenet import ArticleNet  # noqa: F401

modyn/models/articlenet/__init__.py

@@ -1,6 +1,7 @@
 """
 DistilBert classifier
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/models/dlrm/__init__.py

@@ -1,6 +1,7 @@
 """
 DLRM
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/models/dummy/__init__.py

@@ -1,6 +1,7 @@
 """
 Dummy
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/models/fmownet/__init__.py

@@ -1,6 +1,7 @@
 """
 Densenet121 + linear layer
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/models/resnet152/__init__.py

@@ -1,6 +1,7 @@
 """
 ResNet152
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/models/resnet18/__init__.py

@@ -1,6 +1,7 @@
 """
 ResNet18
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/models/resnet50/__init__.py

@@ -1,6 +1,7 @@
 """
 ResNet50
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/models/tokenizers/__init__.py

@@ -1,6 +1,7 @@
 """
 Bert Tokenizer for NLP tasks
 """
+
 import os
 
 from .distill_bert_tokenizer import DistilBertTokenizerTransform  # noqa: F401

modyn/models/yearbooknet/__init__.py

@@ -1,6 +1,7 @@
 """
 Custom CNN for Yearbook dataset
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/selector/__init__.py

@@ -1,6 +1,7 @@
 """
 This submodule provides the base functionality for the selector server component.
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/selector/internal/selector_strategies/__init__.py

@@ -2,6 +2,7 @@
 This submodule contains extensions of the Selector class that implement
 concrete selection strategies.
 """
+
 import os
 
 from .abstract_selection_strategy import AbstractSelectionStrategy  # noqa: F401

modyn/selector/internal/selector_strategies/coreset_strategy.py

@@ -104,9 +104,11 @@ def _session_callback(session: Session) -> Any:
                 session.query(SelectorStateMetadata.sample_key)
                 .filter(
                     SelectorStateMetadata.pipeline_id == self._pipeline_id,
-                    SelectorStateMetadata.seen_in_trigger_id >= self._next_trigger_id - self.tail_triggers
-                    if self.tail_triggers is not None
-                    else True,
+                    (
+                        SelectorStateMetadata.seen_in_trigger_id >= self._next_trigger_id - self.tail_triggers
+                        if self.tail_triggers is not None
+                        else True
+                    ),
                 )
                 .count()
             )

modyn/selector/internal/selector_strategies/downsampling_strategies/abstract_downsampling_strategy.py

@@ -4,7 +4,6 @@
 
 
 class AbstractDownsamplingStrategy(ABC):
-
     """
     This abstract strategy is used to represent the common behaviour of downsampling strategies
     like loss-based, importance downsampling (distribution-based methods) and craig&adacore (greedy-based methods).

modyn/selector/internal/selector_strategies/freshness_sampling_strategy.py

@@ -42,7 +42,9 @@ def __init__(self, config: dict, modyn_config: dict, pipeline_id: int, maximum_k
 
         if self.unused_data_ratio < 1 or self.unused_data_ratio > 99:
             raise ValueError(
-                f"Invalid unused data ratio: {self.unused_data_ratio}. We need at least 1% fresh data (otherwise we would always train on the data from first trigger) and at maximum 99% fresh data (otherwise please use NewDataStrategy+reset)."
+                f"Invalid unused data ratio: {self.unused_data_ratio}."
+                + "We need at least 1% fresh data (otherwise we would always train on the data from first trigger) and"
+                + " at maximum 99% fresh data (otherwise please use NewDataStrategy+reset)."
             )
 
         if self.reset_after_trigger:
@@ -112,9 +114,11 @@ def _get_first_trigger_data(self) -> Iterable[list[int]]:
         if self.has_limit:
             # TODO(#179): this assumes limit < len(samples)
             for samples in self._get_all_unused_data():
-                yield random.sample(samples, self.training_set_size_limit) if self.training_set_size_limit < len(
-                    samples
-                ) else samples
+                yield (
+                    random.sample(samples, self.training_set_size_limit)
+                    if self.training_set_size_limit < len(samples)
+                    else samples
+                )
         else:
             yield from self._get_all_unused_data()
 

modyn/selector/internal/selector_strategies/presampling_strategies/abstract_balanced_strategy.py

@@ -72,9 +72,11 @@ def get_presampling_query(
             .label("row_num"),
         ).filter(
             SelectorStateMetadata.pipeline_id == self.pipeline_id,
-            SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
-            if tail_triggers is not None
-            else True,
+            (
+                SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
+                if tail_triggers is not None
+                else True
+            ),
         )
 
         if self.force_required_target_size:
@@ -139,9 +141,11 @@ def _session_callback(session: Session) -> Any:
                 session.query(self.balanced_column, func.count())  # pylint: disable=not-callable
                 .filter(
                     SelectorStateMetadata.pipeline_id == self.pipeline_id,
-                    SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
-                    if tail_triggers is not None
-                    else True,
+                    (
+                        SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
+                        if tail_triggers is not None
+                        else True
+                    ),
                 )
                 .group_by(self.balanced_column)
             )

modyn/selector/internal/selector_strategies/presampling_strategies/no_presampling_strategy.py

@@ -26,9 +26,11 @@ def get_presampling_query(
             select(SelectorStateMetadata.sample_key)
             .filter(
                 SelectorStateMetadata.pipeline_id == self.pipeline_id,
-                SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
-                if tail_triggers is not None
-                else True,
+                (
+                    SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
+                    if tail_triggers is not None
+                    else True
+                ),
             )
             .order_by(asc(SelectorStateMetadata.timestamp))
         )

modyn/selector/internal/selector_strategies/presampling_strategies/random_no_replacement_presampling_strategy.py

@@ -72,9 +72,11 @@ def _get_query_sample_random(self, next_trigger_id: int, tail_triggers: Optional
             select(SelectorStateMetadata.sample_key)
             .filter(
                 SelectorStateMetadata.pipeline_id == self.pipeline_id,
-                SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
-                if tail_triggers is not None
-                else True,
+                (
+                    SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
+                    if tail_triggers is not None
+                    else True
+                ),
                 # just consider points that have not been used since the last complete trigger
                 SelectorStateMetadata.last_used_in_trigger < self.last_complete_trigger,
             )

modyn/selector/internal/selector_strategies/presampling_strategies/random_presampling_strategy.py

@@ -32,9 +32,11 @@ def get_presampling_query(
             select(SelectorStateMetadata.sample_key)
             .filter(
                 SelectorStateMetadata.pipeline_id == self.pipeline_id,
-                SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
-                if tail_triggers is not None
-                else True,
+                (
+                    SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers
+                    if tail_triggers is not None
+                    else True
+                ),
             )
             .order_by(func.random())  # pylint: disable=E1102
             .limit(target_size)

modyn/selector/internal/storage_backend/__init__.py

@@ -1,6 +1,7 @@
 """
 This submodule provides backends for storing the seen samples during a pipeline.
 """
+
 import os
 
 from .abstract_storage_backend import AbstractStorageBackend  # noqa: F401

modyn/selector/internal/storage_backend/database/__init__.py

@@ -1,6 +1,7 @@
 """
 This submodule provides backends for storing the seen samples during a pipeline.
 """
+
 import os
 
 from .database_storage_backend import DatabaseStorageBackend  # noqa: F401

modyn/selector/internal/storage_backend/database/database_storage_backend.py

@@ -139,9 +139,11 @@ def get_available_labels(self, next_trigger_id: int, tail_triggers: Optional[int
                 .filter(
                     SelectorStateMetadata.pipeline_id == self._pipeline_id,
                     SelectorStateMetadata.seen_in_trigger_id < next_trigger_id,
-                    SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers - 1
-                    if tail_triggers is not None
-                    else True,
+                    (
+                        SelectorStateMetadata.seen_in_trigger_id >= next_trigger_id - tail_triggers - 1
+                        if tail_triggers is not None
+                        else True
+                    ),
                 )
                 .distinct()
                 .all()

modyn/selector/internal/storage_backend/local/__init__.py

@@ -1,6 +1,7 @@
 """
 This submodule provides backends for storing the seen samples during a pipeline.
 """
+
 import os
 
 from .local_storage_backend import LocalStorageBackend  # noqa: F401

modyn/supervisor/__init__.py

@@ -1,6 +1,7 @@
 """Supervisor module. The supervisor initiates a pipeline and coordinates all components.
 
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/supervisor/internal/__init__.py

@@ -1,6 +1,7 @@
 """Supervisor module. The supervisor initiates a pipeline and coordinates all components.
 
 """
+
 import os
 
 files = os.listdir(os.path.dirname(__file__))

modyn/supervisor/internal/evaluation_result_writer/__init__.py

@@ -1,6 +1,7 @@
 """Supervisor module. The supervisor initiates a pipeline and coordinates all components.
 
 """
+
 import os
 
 from .abstract_evaluation_result_writer import AbstractEvaluationResultWriter  # noqa: F401