Demo of versioned dependency install

documentation/demos/dependency_inference/demo.py

@@ -0,0 +1,25 @@
+import infer  # noqa: F401 isort: skip
+import arrow  # ==1.1.1
+import numpy  # ==1.22.4
+import regex
+
+# NB - latest version of numpy is 1.23.2
+print(
+    f"Numpy version is {numpy.__version__} "
+    '(should be "1.22.4", NOT latest version "1.23.2")'
+)
+assert numpy.__version__ == "1.22.4"
+
+# NB - latest version of arrow is 1.2.2
+print(
+    f"Arrow version is {arrow.__version__} "
+    '(should be "1.1.1", NOT latest version "1.2.2")'
+)
+assert arrow.__version__ == "1.1.1"
+
+# NB - latest version of regex is 2.5.119
+print(
+    f"Regex version is {regex.__version__} "
+    '(should be latest version "2.5.119")'
+)
+assert regex.__version__ == "2.5.119"

documentation/demos/dependency_inference/infer.py

@@ -0,0 +1,178 @@
+import builtins
+import inspect
+import subprocess
+import sys
+import tokenize
+from pathlib import Path
+
+if __name__ == "__main__":
+    print("Infer library: import into your code, do not run directly")
+    sys.exit(0)
+
+
+# Let's find the file importing us
+# https://docs.python.org/3/library/inspect.html#the-interpreter-stack
+def get_importing_file():
+    IGNORED_FILES = [
+        "<frozen importlib._bootstrap_external>",
+        "<frozen importlib._bootstrap>",
+    ]
+    curr_frame = inspect.currentframe()
+    try:
+        outer = inspect.getouterframes(curr_frame)
+        for frame in outer:
+            if frame.filename == __file__:
+                continue
+            # consider using code_context here?
+            if frame.filename not in IGNORED_FILES:
+                break
+        else:
+            raise Exception("No import site found")
+        target_file = Path(frame.filename)
+    finally:
+        del frame
+    return target_file
+
+
+# Return a map from {lineno -> comment} for all import statements found
+def get_comment_map(target_file):
+    comment_map = {}
+    with open(target_file, "rb") as fin:
+        in_import = False
+        for token in tokenize.tokenize(fin.readline):
+            if token.type == tokenize.NAME and token.string == "import":
+                in_import = True
+                continue
+            if token.type == tokenize.NEWLINE:
+                in_import = False
+            if token.type == tokenize.COMMENT and in_import:
+                comment_map[token.start[0]] = token.string.strip("#").strip()
+    return comment_map
+
+
+importing_file = get_importing_file()
+comment_map = get_comment_map(importing_file)
+
+print(f"Infer library imported by: {importing_file}")
+print(f"Infer library found the following comments: {comment_map}")
+
+original_import = builtins.__import__
+
+
+def handle_user_relevant_import(name, version_modifier):
+    p = subprocess.run(
+        [sys.executable, "-m", "pip", "show", f"{name}"], capture_output=True
+    )
+    installed_version = ""
+    for line in p.stdout.decode().split("\n"):
+        if line.startswith("Version: "):
+            installed_version = line[9:].strip()
+    # TODO - gross string ops here, comment_version_modifier is '==1.2.1'
+    #        if we have wrong version or no version, install package
+    if version_modifier[2:] != installed_version or installed_version == "":
+        cmd = [
+            str(sys.executable),
+            "-m",
+            "pip",
+            "install",
+            f"{name}{version_modifier}",
+        ]
+        subprocess.run(cmd, check=True)
+
+
+def new_import(name, globals=None, locals=None, fromlist=(), level=0):
+    curr_frame = inspect.currentframe()
+    outer = inspect.getframeinfo(curr_frame.f_back)
+    if outer.filename == str(importing_file):
+        print(f"Infer library detected an import={name}")
+        handle_user_relevant_import(name, comment_map.get(outer.lineno, ""))
+        module = original_import(
+            name,
+            globals=globals,
+            locals=locals,
+            fromlist=fromlist,
+            level=level,
+        )
+    else:
+        module = original_import(
+            name,
+            globals=globals,
+            locals=locals,
+            fromlist=fromlist,
+            level=level,
+        )
+    return module
+
+
+builtins.__import__ = new_import
+
+
+# Based on
+# https://realpython.com/python-import/#example-automatically-install-from-pypi
+# class TestMeta:
+#     @classmethod
+#     def find_spec(cls, name, path, target=None):
+#         global activate_test_meta
+#         global comment_version_modifier
+#         if not activate_test_meta:
+#             return
+#         # print(f"TestMeta: name={name}, path={path}, target={target}:")
+#         module_found = False
+#         for meta in sys.meta_path:
+#             if meta is cls:
+#                 continue
+#             module_spec = meta.find_spec(name, path, target)
+#             # print(f"\t META_SEARCH {meta} -> {module_spec}")
+#             if module_spec is not None:
+#                 module_found = True
+#         if not module_found:
+#             cmd = [
+#                 str(sys.executable),
+#                 "-m",
+#                 "pip",
+#                 "install",
+#                 f"{name}{comment_version_modifier}",
+#             ]
+#             try:
+#                 subprocess.run(cmd, check=True)
+#             except subprocess.CalledProcessError:
+#                 return None
+#         elif comment_version_modifier:
+#             p = subprocess.run(
+#               [sys.executable, '-m', 'pip', 'show', f'{name}'],
+#               capture_output=True,
+#             )
+#             for line in p.stdout.decode().split('\n'):
+#                 if line.startswith('Version: '):
+#                     installed_version = line[9:].strip()
+#             # TODO - gross string ops here
+#             if comment_version_modifier[2:] != installed_version:
+#                 print(f'BAD INSTALL {name}')
+#                 print(comment_version_modifier)
+#                 print(installed_version)
+#                 print()
+#         activate_test_meta = False
+
+
+# sys.meta_path.insert(0, TestMeta)
+
+# TODO:
+#   * If installed version is as requested, and reinstall if not
+#   * Rewrite file with version annotations
+#   * Cleanup
+#   * Summarize findings and intermediate PR
+#   * Multi-file systems
+
+# Thoughts: some imports the user cares about, internal package imports they
+# probably don't.  Can we distinguish between packages users care about and
+# those that they don't?  Do we have to if we only install if package is
+# missing (seems like numpy internally uses ModuleNotFound)
+
+
+# //NEXT :  PIPFINDER gets the comment map and the metafinder will try to
+# install //specified versions (probably use frame inspection to figure out
+# lineno)
+
+# //the trick will be to distinguish imports the user cares about (i.e. in
+# their //packages) vs imports the user doesn't care about (imports internal to
+# numpy)

example_agents/acme_r2d2/trainer.py

@@ -67,15 +67,15 @@ def _improve(self):
             core_state = self.network.rnn.initial_state(batch_size)
         target_core_state = tree.map_structure(tf.identity, core_state)
 
-        # Before training, optionally unroll LSTM for a fixed warmup period.
+        # Before training, optionally unroll LSTM for a fixed warm-up period.
         burn_in_length = self.parameters.burn_in_length
         burn_in_obs = tree.map_structure(
             lambda x: x[:burn_in_length], observations
         )
         _, core_state = self._burn_in(burn_in_obs, core_state)
         _, target_core_state = self._burn_in(burn_in_obs, target_core_state)
 
-        # Don't train on the warmup period.
+        # Don't train on the warm-up period.
         observations, actions, rewards, discounts, extra = tree.map_structure(
             lambda x: x[burn_in_length:],
             (observations, actions, rewards, discounts, extra),