Organize imports

mlprogram/__init__.py

@@ -1 +1,19 @@
+import mlprogram.actions as actions  # noqa
 import mlprogram.builtins as builtins  # noqa
+import mlprogram.collections as collections  # noqa
+import mlprogram.datasets as datasets  # noqa
+import mlprogram.distributed as distributed  # noqa
+import mlprogram.encoders as encoders  # noqa
+import mlprogram.entrypoint as entrypoint  # noqa
+import mlprogram.functools as functools  # noqa
+import mlprogram.logging as logging  # noqa
+# languages
+import mlprogram.metrics as metrics  # noqa
+import mlprogram.pytorch_pfn_extras as pytorch_pfn_extras  # noqa
+import mlprogram.random as random  # noqa
+import mlprogram.transpyle as transpyle  # noqa
+# nn
+# samplers
+# synthesizers
+# transforms
+# utils

mlprogram/actions/action.py

@@ -2,7 +2,7 @@
 from enum import Enum
 from typing import Any, Generic, List, Optional, Tuple, TypeVar, Union
 
-from mlprogram.languages import Root
+from mlprogram.languages.ast import Root
 
 Value = TypeVar("Value")
 

mlprogram/actions/action_sequence.py

@@ -13,7 +13,7 @@
     NodeType,
     Rule,
 )
-from mlprogram.languages import AST, Field, Leaf, Node, Root
+from mlprogram.languages.ast import AST, Field, Leaf, Node, Root
 
 logger = logging.Logger(__name__)
 

mlprogram/datasets/__init__.py

@@ -1,4 +1,5 @@
-import os
-
-DEFAULT_CACHE_DIR = \
-    os.path.join(os.environ["HOME"], ".cache", "mlprogram", "datasets")
+from mlprogram.datasets import deepfix  # noqa
+from mlprogram.datasets import django  # noqa
+from mlprogram.datasets import hearthstone  # noqa
+from mlprogram.datasets import nl2bash  # noqa
+from mlprogram.datasets.constants import DEFAULT_CACHE_DIR  # noqa

mlprogram/datasets/constants.py

@@ -0,0 +1,3 @@
+import os
+
+DEFAULT_CACHE_DIR = os.path.join(os.environ["HOME"], ".cache", "mlprogram", "datasets")

mlprogram/datasets/deepfix/download.py

@@ -8,19 +8,18 @@
 from typing import Callable
 
 from mlprogram import logging
-from mlprogram.builtins import Environment
-from mlprogram.datasets import DEFAULT_CACHE_DIR
-from mlprogram.functools import file_cache
-from mlprogram.utils.data import ListDataset
+from mlprogram.builtins.datatypes import Environment
+from mlprogram.datasets.constants import DEFAULT_CACHE_DIR
+from mlprogram.functools.cache import file_cache
+from mlprogram.utils.data.utils import ListDataset
 
 logger = logging.Logger(__name__)
 
 BASE_PATH = "https://www.cse.iitk.ac.in/users/karkare/prutor/prutor-deepfix-09-12-2017.zip"  # noqa
 
 
 def default_get(src: str, dst: str):
-    with urllib.request.urlopen(src) as src_file, \
-            open(dst, "wb") as dst_file:
+    with urllib.request.urlopen(src) as src_file, open(dst, "wb") as dst_file:
         copyfileobj(src_file, dst_file)
 
 

mlprogram/datasets/deepfix/lexer.py

@@ -44,8 +44,8 @@
 from typing import List, Optional, Tuple
 
 from mlprogram import logging
-from mlprogram.languages import Lexer as BaseLexer
-from mlprogram.languages import Token
+from mlprogram.languages.lexer import Lexer as BaseLexer
+from mlprogram.languages.token import Token
 
 logger = logging.Logger(__name__)
 

mlprogram/datasets/django/download.py

@@ -4,11 +4,11 @@
 import requests
 
 from mlprogram import logging
-from mlprogram.builtins import Environment
-from mlprogram.datasets import DEFAULT_CACHE_DIR
+from mlprogram.builtins.datatypes import Environment
+from mlprogram.datasets.constants import DEFAULT_CACHE_DIR
 from mlprogram.datasets.django.format_annotations import format_annotations
-from mlprogram.functools import file_cache
-from mlprogram.utils.data import ListDataset
+from mlprogram.functools.cache import file_cache
+from mlprogram.utils.data.utils import ListDataset
 
 logger = logging.Logger(__name__)
 
@@ -29,8 +29,7 @@ def download(cache_path: str = os.path.join(DEFAULT_CACHE_DIR, "django.pt"),
     @file_cache(cache_path)
     def _download():
         return {
-            "annotation": format_annotations(
-                get(BASE_PATH + "all.anno").split("\n")),
+            "annotation": format_annotations(get(BASE_PATH + "all.anno").split("\n")),
             "code": get(BASE_PATH + "all.code").split("\n")
         }
     data = _download()

mlprogram/datasets/django/functions.py

@@ -3,7 +3,7 @@
 
 from nltk import tokenize
 
-from mlprogram.languages import Token
+from mlprogram.languages.token import Token
 
 tokenizer = tokenize.WhitespaceTokenizer()
 
@@ -52,8 +52,7 @@ def placeholder(id: int) -> str:
             else:
                 reference.append(Token[str, str](None, word, word))
 
-            vars = list(filter(lambda x: len(x) > 0,
-                               word.split('.')))  # split by '.'
+            vars = list(filter(lambda x: len(x) > 0, word.split('.')))  # split by '.'
             if len(vars) > 1:
                 for v in vars:
                     reference.append(Token(None, v, v))

mlprogram/datasets/django/parser.py

@@ -51,8 +51,8 @@
 import re
 from typing import Callable, List, Optional
 
-from mlprogram.languages import AST
-from mlprogram.languages.python import Parser as BaseParser
+from mlprogram.languages.ast import AST
+from mlprogram.languages.python.parser import Parser as BaseParser
 
 p_elif = re.compile(r'^elif\s?')
 p_else = re.compile(r'^else\s?')

mlprogram/datasets/hearthstone/download.py

@@ -4,10 +4,10 @@
 import requests
 
 from mlprogram import logging
-from mlprogram.builtins import Environment
-from mlprogram.datasets import DEFAULT_CACHE_DIR
-from mlprogram.functools import file_cache
-from mlprogram.utils.data import ListDataset
+from mlprogram.builtins.datatypes import Environment
+from mlprogram.datasets.constants import DEFAULT_CACHE_DIR
+from mlprogram.functools.cache import file_cache
+from mlprogram.utils.data.utils import ListDataset
 
 logger = logging.Logger(__name__)
 
@@ -19,8 +19,7 @@ def default_get(path: str) -> str:
     return requests.get(path).text
 
 
-def download(cache_path: str = os.path.join(DEFAULT_CACHE_DIR,
-                                            "hearthstone.pt"),
+def download(cache_path: str = os.path.join(DEFAULT_CACHE_DIR, "hearthstone.pt"),
              base_path: str = BASE_PATH,
              get: Callable[[str], str] = default_get) \
         -> Dict[str, ListDataset]:

mlprogram/datasets/hearthstone/functions.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from mlprogram.languages import Token
+from mlprogram.languages.token import Token
 
 
 class TokenizeQuery:

mlprogram/datasets/nl2bash/download.py

@@ -4,10 +4,10 @@
 from typing import Dict
 
 from mlprogram import logging
-from mlprogram.builtins import Environment
-from mlprogram.datasets import DEFAULT_CACHE_DIR
-from mlprogram.functools import file_cache
-from mlprogram.utils.data import ListDataset
+from mlprogram.builtins.datatypes import Environment
+from mlprogram.datasets.constants import DEFAULT_CACHE_DIR
+from mlprogram.functools.cache import file_cache
+from mlprogram.utils.data.utils import ListDataset
 
 logger = logging.Logger(__name__)
 

mlprogram/datasets/nl2bash/functions.py

@@ -3,7 +3,7 @@
 
 from nltk import tokenize
 
-from mlprogram.languages import Token
+from mlprogram.languages.token import Token
 
 tokenizer = tokenize.WhitespaceTokenizer()
 

mlprogram/encoders/action_sequence_encoder.py

@@ -5,16 +5,16 @@
 from torchnlp.encoders import LabelEncoder
 
 from mlprogram import logging
-from mlprogram.actions import (
-    ActionSequence,
+from mlprogram.actions.action import (
     ApplyRule,
     CloseVariadicFieldRule,
     ExpandTreeRule,
     GenerateToken,
     NodeType,
     Rule,
 )
-from mlprogram.languages import Token
+from mlprogram.actions.action_sequence import ActionSequence
+from mlprogram.languages.token import Token
 
 logger = logging.Logger(__name__)
 

mlprogram/entrypoint/__init__.py

@@ -1,3 +1,4 @@
+from mlprogram.entrypoint import modules  # noqa
 from mlprogram.entrypoint.evaluate import EvaluateSample  # noqa
 from mlprogram.entrypoint.evaluate import EvaluateSynthesizer  # noqa
 from mlprogram.entrypoint.evaluate import evaluate  # noqa

mlprogram/entrypoint/configs.py

@@ -5,7 +5,7 @@
 from pytorch_pfn_extras.config import Config
 
 from mlprogram.entrypoint.types import types
-from mlprogram.functools import file_cache
+from mlprogram.functools.cache import file_cache
 
 
 def with_file_cache(path, config, types):

mlprogram/entrypoint/evaluate.py

@@ -12,9 +12,9 @@
 from tqdm import tqdm
 
 from mlprogram import distributed, logging
-from mlprogram.builtins import Environment
-from mlprogram.synthesizers import Synthesizer
-from mlprogram.utils.data import ListDataset
+from mlprogram.builtins.datatypes import Environment
+from mlprogram.synthesizers.synthesizer import Synthesizer
+from mlprogram.utils.data.utils import ListDataset
 
 logger = logging.Logger(__name__)
 

mlprogram/entrypoint/modules/__init__.py

@@ -0,0 +1,4 @@
+import mlprogram.entrypoint.modules.fairseq as fairseq  # noqa
+import mlprogram.entrypoint.modules.numpy as numpy  # noqa
+import mlprogram.entrypoint.modules.torch as torch  # noqa
+import mlprogram.entrypoint.modules.torchnlp as torchnlp  # noqa

mlprogram/entrypoint/modules/fairseq.py

@@ -1,5 +1,9 @@
-import fairseq.optim
+try:
+    import fairseq.optim
 
-types = {
-    "fairseq.optim.Adafactor": lambda: fairseq.optim.adafactor.Adafactor
-}
+    types = {
+        "fairseq.optim.Adafactor": lambda: fairseq.optim.adafactor.Adafactor
+    }
+except:  # noqa
+    types = {}
+    pass

mlprogram/entrypoint/train.py

@@ -10,9 +10,9 @@
 from torch.utils.data import DataLoader
 
 from mlprogram import distributed, logging
-from mlprogram.builtins import Environment
+from mlprogram.builtins.datatypes import Environment
 from mlprogram.pytorch_pfn_extras import SaveTopKModel, StopByThreshold
-from mlprogram.synthesizers import Synthesizer
+from mlprogram.synthesizers.synthesizer import Synthesizer
 
 logger = logging.Logger(__name__)
 

mlprogram/entrypoint/types.py

@@ -28,16 +28,11 @@
 import mlprogram.transforms.pbe
 import mlprogram.transforms.text
 import mlprogram.utils.data
+from mlprogram.entrypoint.modules.fairseq import types as fairseq_types
 from mlprogram.entrypoint.modules.numpy import types as numpy_types
 from mlprogram.entrypoint.modules.torch import types as torch_types
 from mlprogram.entrypoint.modules.torchnlp import types as torchnlp_types
 
-try:
-    from mlprogram.entrypoint.modules.fairseq import types as fairseq_types
-except:  # noqa
-    fairseq_types = {}
-    pass
-
 types = {
     "select": lambda key, options: options[key],
 

mlprogram/metrics/error_correct_rate.py

@@ -2,7 +2,8 @@
 
 from torch import nn
 
-from mlprogram.languages import Analyzer, Interpreter
+from mlprogram.languages.analyzer import Analyzer
+from mlprogram.languages.interpreter import Interpreter
 
 Code = TypeVar("Code")
 Error = TypeVar("Error")

mlprogram/metrics/metric.py

@@ -3,9 +3,11 @@
 
 from torch import nn
 
-from mlprogram.builtins import Apply, Environment, Pick
-from mlprogram.functools import Sequence
-from mlprogram.nn import Function
+from mlprogram.builtins.apply import Apply
+from mlprogram.builtins.datatypes import Environment
+from mlprogram.builtins.pick import Pick
+from mlprogram.functools.functions import Sequence
+from mlprogram.nn.function import Function
 
 Value = TypeVar("Value")
 

mlprogram/metrics/test_case_result.py

@@ -2,8 +2,8 @@
 
 from torch import nn
 
-from mlprogram.builtins import Environment
-from mlprogram.languages import Interpreter
+from mlprogram.builtins.datatypes import Environment
+from mlprogram.languages.interpreter import Interpreter
 from mlprogram.metrics.accuracy import Accuracy
 from mlprogram.metrics.metric import use_environment
 

mlprogram/pytorch_pfn_extras.py

@@ -6,7 +6,7 @@
 from torch import nn
 
 from mlprogram import logging
-from mlprogram.collections import TopKElement
+from mlprogram.collections.top_k_element import TopKElement
 
 logger = logging.Logger(__name__)