Temp queue download (#434)

* use `context.temp_queue` to download artifacts

This will break if `context.temp_queue` is None.

* `context.temp_queue` should return `context.queue` if not set

* fix tests

* fix errant paste

* coverage fixes

* 32.0.0

HISTORY.rst

@@ -4,6 +4,13 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+[32.0.0] - 2020-02-24
+---------------------
+
+Changed
+~~~~~~~
+- Scriptworker now uses ``context.temp_queue`` to download artifacts.
+
 [31.1.0] - 2020-02-21
 ---------------------
 

src/scriptworker/artifacts.py

@@ -206,9 +206,9 @@ def get_artifact_url(context, task_id, path):
 
     """
     if path.startswith("public/"):
-        url = context.queue.buildUrl("getLatestArtifact", task_id, path)
+        url = context.temp_queue.buildUrl("getLatestArtifact", task_id, path)
     else:
-        url = context.queue.buildSignedUrl(
+        url = context.temp_queue.buildSignedUrl(
             "getLatestArtifact",
             task_id,
             path,

src/scriptworker/context.py

@@ -53,7 +53,7 @@ class Context(object):
     queue = None
     session = None
     task = None
-    temp_queue = None
+    _temp_queue = None
     running_tasks = None
     _credentials = None
     _claim_task = None  # This assumes a single task per worker.
@@ -169,7 +169,10 @@ def temp_credentials(self):
     @temp_credentials.setter
     def temp_credentials(self, credentials):
         self._temp_credentials = credentials
-        self.temp_queue = self.create_queue(self.temp_credentials)
+        if credentials is not None:
+            self.temp_queue = self.create_queue(self.temp_credentials)
+        else:
+            self.temp_queue = None
 
     def write_json(self, path, contents, message):
         """Write json to disk.
@@ -215,6 +218,22 @@ def event_loop(self):
     def event_loop(self, event_loop):
         self._event_loop = event_loop
 
+    @property
+    def temp_queue(self):
+        """dict: The queue for the current task.
+
+        These will have different sets of scopes than the worker queue.
+
+        """
+        if self._temp_queue:
+            return self._temp_queue
+        else:
+            return self.queue
+
+    @temp_queue.setter
+    def temp_queue(self, queue):
+        self._temp_queue = queue
+
     async def populate_projects(self, force=False):
         """Download the ``projects.yml`` file and populate ``self.projects``.
 

src/scriptworker/version.py

@@ -54,7 +54,7 @@ def get_version_string(version: Union[ShortVerType, LongVerType]) -> str:
 
 # 1}}}
 # Semantic versioning 2.0.0  http://semver.org/
-__version__ = (31, 1, 0)
+__version__ = (32, 0, 0)
 __version_string__ = get_version_string(__version__)
 
 

tests/test_artifacts.py

@@ -182,10 +182,10 @@ def buildSignedUrl(*args, **kwargs):
             return expected
 
     context = mock.MagicMock()
-    context.queue = mock.MagicMock()
-    context.queue.options = {"baseUrl": "https://netloc/"}
-    context.queue.buildUrl = buildUrl
-    context.queue.buildSignedUrl = buildSignedUrl
+    context.temp_queue = mock.MagicMock()
+    context.temp_queue.options = {"baseUrl": "https://netloc/"}
+    context.temp_queue.buildUrl = buildUrl
+    context.temp_queue.buildSignedUrl = buildSignedUrl
     assert get_artifact_url(context, "x", path) == expected
 
 

tests/test_context.py

@@ -84,6 +84,12 @@ def test_temp_queue(rw_context, mocker):
     assert taskcluster.aio.Queue.called_once_with(
         options={"rootUrl": rw_context.config["taskcluster_root_url"], "credentials": rw_context.temp_credentials}, session=rw_context.session
     )
+    assert rw_context._temp_queue is not None
+    rw_context.temp_queue = None
+    fake_queue = mocker.MagicMock()
+    rw_context.queue = fake_queue
+    assert rw_context._temp_queue is None
+    assert rw_context.temp_queue is fake_queue
 
 
 @pytest.mark.asyncio

tests/test_integration.py

@@ -143,8 +143,8 @@ async def get_temp_creds_context(config_override=None):
         yield context
 
 
-async def create_task(context, task_id, task_group_id):
-    payload = integration_create_task_payload(context.config, task_group_id)
+async def create_task(context, task_id, task_group_id, **kwargs):
+    payload = integration_create_task_payload(context.config, task_group_id, **kwargs)
     return await context.queue.createTask(task_id, payload)
 
 
@@ -327,7 +327,7 @@ async def test_private_artifacts(context_function):
     task_group_id = task_id = slugid.nice()
     override = {"task_script": ("bash", "-c", ">&2 echo")}
     async with context_function(override) as context:
-        result = await create_task(context, task_id, task_group_id)
+        result = await create_task(context, task_id, task_group_id, scopes=["queue:get-artifact:SampleArtifacts/_/X.txt"])
         assert result["status"]["state"] == "pending"
         path = os.path.join(context.config["artifact_dir"], "SampleArtifacts/_/X.txt")
         utils.makedirs(os.path.dirname(path))

tox.ini

@@ -45,7 +45,7 @@ depends =
 [testenv:report]
 skip_install = true
 commands = coverage report -m
-depends = py37
+depends = py38
 parallel_show_output = true
 
 [testenv:coveralls]

version.json

@@ -1,8 +1,8 @@
 {
     "version":[
-        31,
-        1,
+        32,
+        0,
         0
     ],
-    "version_string":"31.1.0"
+    "version_string":"32.0.0"
 }