salvage extractors removed from datalad-core

This commit adds the extractors that were removed
from datalad core in the datalad PR #7014
(datalad/datalad#7014)
including their tests to datalad-metalad. This is
done to keep them available with only `datalad`
and `datalad-metalad` installed.

datalad_metalad/extractors/base.py

@@ -42,58 +42,6 @@ class ExtractorResult:
     immediate_data: Optional[Dict[str, Any]] = None
 
 
-# Legacy extractor base from datalad-core
-class BaseMetadataExtractor(object):
-
-    NEEDS_CONTENT = True   # majority of the extractors need data content
-
-    def __init__(self, ds, paths):
-        """
-        Parameters
-        ----------
-        ds : dataset instance
-          Dataset to extract metadata from.
-        paths : list
-          Paths to investigate when extracting content metadata
-        """
-
-        self.ds = ds
-        self.paths = paths
-
-    def get_metadata(self, dataset=True, content=True):
-        """
-        Returns
-        -------
-        dict or None, dict or None
-          Dataset metadata dict, dictionary of filepath regexes with metadata,
-          dicts, each return value could be None if there is no such metadata
-        """
-        # default implementation
-        return \
-            self._get_dataset_metadata() if dataset else None, \
-            ((k, v) for k, v in self._get_content_metadata()) if content else None
-
-    def _get_dataset_metadata(self):
-        """
-        Returns
-        -------
-        dict
-          keys and values are arbitrary
-        """
-        raise NotImplementedError
-
-    def _get_content_metadata(self):
-        """Get ALL metadata for all dataset content.
-
-        Possibly limited to the paths given to the extractor.
-
-        Returns
-        -------
-        generator((location, metadata_dict))
-        """
-        raise NotImplementedError
-
-
 class DataOutputCategory(enum.Enum):
     """
     Describe how extractors output metadata.
@@ -388,7 +336,7 @@ def get_state(self, dataset):
 
 # XXX this is the legacy-legacy interface, keep around for a bit more and then
 # remove
-class BaseMetadataExtractor(metaclass=abc.ABCMeta):
+class BaseMetadataExtractor:
 
     NEEDS_CONTENT = True   # majority of the extractors need data content
 
@@ -418,7 +366,6 @@ def get_metadata(self, dataset=True, content=True):
             self._get_dataset_metadata() if dataset else None, \
             ((k, v) for k, v in self._get_content_metadata()) if content else None
 
-    @abc.abstractmethod
     def _get_dataset_metadata(self):
         """
         Returns
@@ -428,7 +375,6 @@ def _get_dataset_metadata(self):
         """
         raise NotImplementedError
 
-    @abc.abstractmethod
     def _get_content_metadata(self):
         """Get ALL metadata for all dataset content.
 

datalad_metalad/extractors/legacy/__init__.py

@@ -7,3 +7,11 @@
 #
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """Legacy metadata extractors"""
+
+from os.path import join
+
+from datalad.consts import DATALAD_DOTDIR
+
+METADATA_DIR = join(DATALAD_DOTDIR, 'metadata')
+DATASET_METADATA_FILE = join(METADATA_DIR, 'dataset.json')
+DATASET_CONFIG_FILE = join(DATALAD_DOTDIR, 'config')

datalad_metalad/extractors/legacy/annex.py

@@ -21,7 +21,7 @@
 from .definitions import version as vocabulary_version
 
 
-class MetadataExtractor(BaseMetadataExtractor):
+class AnnexMetadataExtractor(BaseMetadataExtractor):
 
     NEEDS_CONTENT = False
 

datalad_metalad/extractors/legacy/audio.py

@@ -34,7 +34,7 @@
 }
 
 
-class MetadataExtractor(BaseMetadataExtractor):
+class AudioMetadataExtractor(BaseMetadataExtractor):
 
     _unique_exclude = {'bitrate'}
 

datalad_metalad/extractors/legacy/datacite.py

@@ -71,7 +71,7 @@ def _process_tree(tree, nstag):
     return rec
 
 
-class MetadataExtractor(BaseMetadataExtractor):
+class DataciteMetadataExtractor(BaseMetadataExtractor):
     def _get_dataset_metadata(self):
         canonical = op.join(self.ds.path, '.datalad', 'meta.datacite.xml')
 

datalad_metalad/extractors/legacy/datalad_core.py

@@ -17,21 +17,23 @@
 from os.path import join as opj
 from os.path import exists
 
-from datalad.consts import (
-    DATASET_METADATA_FILE,
-    DATALAD_DOTDIR,
-    WEB_SPECIAL_REMOTE_UUID,
-)
+from datalad.consts import WEB_SPECIAL_REMOTE_UUID
 from datalad.support.json_py import load as jsonload
 from datalad.support.annexrepo import AnnexRepo
 from datalad.coreapi import subdatasets
+
+from . import (
+    DATASET_METADATA_FILE,
+    DATALAD_DOTDIR,
+)
+
 # use main version as core version
 
 # this must stay, despite being a seemingly unused import, each extractor defines a version
 from .definitions import version as vocabulary_version
 
 
-class MetadataExtractor(BaseMetadataExtractor):
+class DataladCoreMetadataExtractor(BaseMetadataExtractor):
 
     NEEDS_CONTENT = False
 

datalad_metalad/extractors/legacy/datalad_rfc822.py

@@ -49,7 +49,7 @@ def _beautify_multiline_field(content):
     return title, content
 
 
-class MetadataExtractor(BaseMetadataExtractor):
+class DataladRFC822MetadataExtractor(BaseMetadataExtractor):
     _metadata_compliance = "http://docs.datalad.org/metadata.html#v0-1"
     _core_metadata_filename = opj('.datalad', 'meta.rfc822')
 

datalad_metalad/extractors/legacy/exif.py

@@ -32,7 +32,7 @@ def _return_as_appropriate_dtype(val):
             return val
 
 
-class MetadataExtractor(BaseMetadataExtractor):
+class ExifMetadataExtractor(BaseMetadataExtractor):
     def get_metadata(self, dataset, content):
         if not content:
             return {}, []

datalad_metalad/extractors/legacy/frictionless_datapackage.py

@@ -44,7 +44,7 @@ def _compact_license(obj):
         return obj
 
 
-class MetadataExtractor(BaseMetadataExtractor):
+class FRDPMetadataExtractor(BaseMetadataExtractor):
     metadatasrc_fname = 'datapackage.json'
 
     _key2stdkey = {

datalad_metalad/extractors/legacy/image.py

@@ -46,7 +46,7 @@
 }
 
 
-class MetadataExtractor(BaseMetadataExtractor):
+class ImageMetadataExtractor(BaseMetadataExtractor):
 
     _extractors = {
         'format': lambda x: x.format_description,

datalad_metalad/extractors/legacy/tests/test_audio.py

@@ -8,6 +8,8 @@
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """Test audio extractor"""
 
+from pathlib import Path
+
 from datalad.tests.utils_pytest import (
     SkipTest,
     assert_in,
@@ -24,8 +26,6 @@
 except ImportError:
     raise SkipTest
 
-from os.path import dirname
-from os.path import join as opj
 from shutil import copy
 
 from datalad.api import Dataset
@@ -47,32 +47,14 @@
 @with_tempfile(mkdir=True)
 def test_audio(path=None):
     ds = Dataset(path).create()
-    ds.config.add('datalad.metadata.nativetype', 'audio', scope='branch')
-    copy(
-        opj(dirname(dirname(dirname(__file__))), 'tests', 'data', 'audio.mp3'),
-        path)
+    copy(Path(__file__).parent / 'data' / 'audio.mp3', path)
     ds.save()
     assert_repo_status(ds.path)
-    res = ds.aggregate_metadata()
-    assert_status('ok', res)
-    res = ds.metadata('audio.mp3')
+
+    res = ds.meta_extract('audio', str(Path(path) / 'audio.mp3'))
     assert_result_count(res, 1)
 
     # from this extractor
-    meta = res[0]['metadata']['audio']
+    meta = res[0]['metadata_record']['extracted_metadata']
     for k, v in target.items():
         eq_(meta[k], v)
-
-    assert_in('@context', meta)
-
-    uniques = ds.metadata(
-        reporton='datasets', return_type='item-or-list')['metadata']['datalad_unique_content_properties']
-    # test file has it, but uniques have it blanked out, because the extractor considers it worthless
-    # for discovering whole datasets
-    assert_in('bitrate', meta)
-    eq_(uniques['audio']['bitrate'], None)
-
-    # 'date' field carries not value, hence gets exclude from the unique report
-    assert_in('date', meta)
-    assert(not meta['date'])
-    assert_not_in('date', uniques['audio'])

datalad_metalad/extractors/legacy/tests/test_base.py

@@ -8,9 +8,8 @@
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """Test all extractors at a basic level"""
 
-from inspect import isgenerator
+from __future__ import annotations
 
-from datalad.api import Dataset
 from datalad.support.entrypoints import iter_entrypoints
 from datalad.tests.utils_pytest import (
     SkipTest,
@@ -21,57 +20,15 @@
 )
 
 
-@with_tree(tree={'file.dat': ''})
-def check_api(annex, path):
-    ds = Dataset(path).create(force=True, annex=annex)
-    ds.save()
-    assert_repo_status(ds.path)
-
-    processed_extractors, skipped_extractors = [], []
+def check_api(use_annex: bool):
+    processed_extractors = []
     for ename, emod, eload in iter_entrypoints('datalad.metadata.extractors'):
         # we need to be able to query for metadata, even if there is none
         # from any extractor
-        try:
-            extractor_cls = eload()
-        except Exception as exc:
-            exc_ = str(exc)
-            skipped_extractors += [exc_]
-            continue
-        extractor = extractor_cls(
-            ds, paths=['file.dat'])
-        meta = extractor.get_metadata(
-            dataset=True,
-            content=True)
-        # we also get something for the dataset and something for the content
-        # even if any of the two is empty
-        assert_equal(len(meta), 2)
-        dsmeta, contentmeta = meta
-        assert (isinstance(dsmeta, dict))
-        assert hasattr(contentmeta, '__len__') or isgenerator(contentmeta)
-        # verify that generator does not blow and has an entry for our
-        # precious file
-        cm = dict(contentmeta)
-        # datalad_core does provide some (not really) information about our
-        # precious file
-        if ename == 'datalad_core':
-            assert 'file.dat' in cm
-        elif ename == 'annex':
-            if annex:
-                # verify correct key, which is the same for all files of 0 size
-                assert_equal(
-                    cm['file.dat']['key'],
-                    'MD5E-s0--d41d8cd98f00b204e9800998ecf8427e.dat'
-                )
-            else:
-                # no metadata on that file
-                assert not cm
         processed_extractors.append(ename)
-    assert "datalad_core" in processed_extractors, \
+    assert \
+        "datalad_core" in processed_extractors, \
         "Should have managed to find at least the core extractor extractor"
-    if skipped_extractors:
-        raise SkipTest(
-            "Not fully tested/succeeded since some extractors failed"
-            " to load:\n%s" % ("\n".join(skipped_extractors)))
 
 
 @known_failure_githubci_win