Merge pull request #580 from OpenEnergyPlatform/release-v0.14.5

Release v0.14.5

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.14.4
+current_version = 0.14.5
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>(a|na))+(?P<build>\d+))?
 serialize = 
 	{major}.{minor}.{patch}{release}{build}

.github/ISSUE_TEMPLATE/issue_template_user_kudos.md

@@ -12,12 +12,14 @@ It helps the project quite a bit!
 
 We will add you to the list of valued users.
 
-Please, insert your information between the double quotes below - fill out at minimum "affiliation" :purple_heart:
+Please, insert your information below - fill out at minimum affiliation :purple_heart:
 
-family-names: ""
-given-names: ""
-alias: ""
-affiliation: ""
-orcid: ""
+:pencil2: **Spaces** and the following special characters are allowed: @ ? ! | . , : ; - _ [ / ( ) \ ] § $ % & = + < >
+
+family-names: 
+given-names: 
+alias: 
+affiliation: 
+orcid: 
 
 Thank you!

.github/workflows/ci-production.yml

@@ -32,7 +32,7 @@ jobs:
     - name: create package
       run: python -m build --sdist
     - name: import open-mastr
-      run: python -m pip install ./dist/open_mastr-0.14.4.tar.gz
+      run: python -m pip install ./dist/open_mastr-0.14.5.tar.gz
     - name: Create credentials file
       env:
         MASTR_TOKEN: ${{ secrets.MASTR_TOKEN }}

.github/workflows/extend_user_cff.yml

@@ -85,4 +85,4 @@ jobs:
           
           Closes #${{ github.event.issue.number }}
 
-          Many thanks ${{ github.actor }}!
+          Many thanks @${{ github.actor }}!

.github/workflows/test-pypi-publish.yml

@@ -35,4 +35,4 @@ jobs:
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI_TEST }}
-        repository_url: https://test.pypi.org/legacy/
+        repository-url: https://test.pypi.org/legacy/

CHANGELOG.md

@@ -6,6 +6,29 @@ For each version important additions, changes and removals are listed here.
 The format is inspired from [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v0.14.5] New MaStR data model, battery export, various fixes - 2024-10-11
+### Added
+- Replace values in NetzbetreiberpruefungStatus with their entries from
+  Katalogwerte
+  [#583](https://github.com/OpenEnergyPlatform/open-MaStR/pull/583)
+- Add `deleted_market_actors` to data model and prevent crash on unknown tables
+  [#575](https://github.com/OpenEnergyPlatform/open-MaStR/pull/575)
+- Extended documentation of data cleansing process for bulk download
+  [#568](https://github.com/OpenEnergyPlatform/open-MaStR/pull/568)
+- Add OFFIS eV as partner organization
+  [#493](https://github.com/OpenEnergyPlatform/open-MaStR/pull/493)
+### Changed
+- Fix usercff workflow
+  [#545](https://github.com/OpenEnergyPlatform/open-MaStR/issues/544)
+- Fix docs on user-defined output path for csv, xml, database
+  [#549](https://github.com/OpenEnergyPlatform/open-MaStR/issues/549)
+- Set pandas version to >=2.2.2 for compatibility with numpy v2.0
+  [#553](https://github.com/OpenEnergyPlatform/open-MaStR/issues/553)
+- Allow to configure model/service port in `soap_api.download.MaStRAPI`
+  [#556](https://github.com/OpenEnergyPlatform/open-MaStR/issues/556)
+- Allow CSV export of table `storage_units`
+  [#565](https://github.com/OpenEnergyPlatform/open-MaStR/pull/565)
+
 ## [v0.14.4] Release for the Journal of Open Source Software JOSS - 2024-06-07
 ### Added
 - Extend documentation section `getting started` based on the JOSS Review [#523](https://github.com/OpenEnergyPlatform/open-MaStR/pull/523)
@@ -18,7 +41,6 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/
 - Fixed missing call to gen_url in case first bulk download fails as xml file for today is not yet available [#534](https://github.com/OpenEnergyPlatform/open-MaStR/pull/534)
 - Repair links in the documentation page [#536](https://github.com/OpenEnergyPlatform/open-MaStR/pull/536)
 
-
 ## [v0.14.3] Fix Pypi Release - 2024-04-24 
 ### Added
 - Add new table `changed_dso_assignment` [#510](https://github.com/OpenEnergyPlatform/open-MaStR/pull/510)

CITATION.cff

@@ -25,10 +25,15 @@ authors:
     alias: "@deniztepe"
     affiliation: "fortiss"
     orcid: " https://orcid.org/0000-0002-7605-0173"
+  - family-names: "Amme"
+    given-names: "Jonathan"
+    alias: "@nesnoj"
+    affiliation: "Reiner Lemoine Institut"
+    orcid: " https://orcid.org/0000-0002-8563-5261"
 title: "open-MaStR"
 type: software
 license: AGPL-3.0
-version: 0.14.4
+version: 0.14.5
 doi: 
-date-released: 2024-06-07
+date-released: 2024-10-11
 url: "https://github.com/OpenEnergyPlatform/open-MaStR/"

CONTRIBUTING.md

@@ -129,6 +129,13 @@ git status
 ```
 
 #### 2.3. Commit your changes 
+First, make sure you have the pre-commit hooks installed to have your code
+automatically checked on commit for programmatic and stylistic errors:
+```bash
+pre-commit install
+```
+
+Now, let's add some file.
 If the file does not exist on the remote server yet, use:
 ```bash
 git add filename.md

README.rst

@@ -1,5 +1,5 @@
 
-.. image:: https://user-images.githubusercontent.com/14353512/199113556-4b53660f-c628-4138-8d01-3719595ecda1.png
+.. image:: docs/images/README_HeaderThreePartners.svg
     :align: left
     :target: https://github.com/OpenEnergyPlatform/open-MaStR
     :alt: MaStR logo
@@ -59,8 +59,6 @@ Documentation
 | Find the `documentation <https://open-mastr.readthedocs.io/en/latest/>`_ hosted on ReadTheDocs.
 
 | The original API documentation can be found on the `Webhilfe des Marktstammdatenregisters <https://www.marktstammdatenregister.de/MaStRHilfe/subpages/webdienst.html>`_.
-| If you are interested in browsing the MaStR online, check out the privately hosted `Marktstammdatenregister.dev <https://marktstammdatenregister.dev/>`_.
-| Also see the `bundesAPI/Marktstammdaten-API <https://github.com/bundesAPI/marktstammdaten-api>`_ for another implementation.
 
 
 Installation
@@ -112,7 +110,13 @@ changes in a `Pull Request <https://github.com/OpenEnergyPlatform/open-MaStR/pul
 - `Wasserstoffatlas <https://wasserstoffatlas.de/>`_
 - `EE-Status App <https://ee-status.de/>`_
 - `Digiplan Anhalt <https://digiplan.rl-institut.de/>`_
+- `Data Quality Assessment of the MaStR <https://marktstammdaten.kotthoff.dev/>`_
 
+External Resources
+===================
+Besides open-mastr, some other resources exist that ease the process of working with the Marktstammdatenregister:
+- If you are interested in browsing the MaStR online, check out the github organisation `Marktstammdatenregister.dev <https://github.com/marktstammdatenregister-dev>`_.
+- The `bundesAPI/Marktstammdaten-API <https://github.com/bundesAPI/marktstammdaten-api>`_ is another implementation to access data via an official API.
 
 Collaboration
 =============
@@ -129,7 +133,7 @@ Software
 | This repository is licensed under the **GNU Affero General Public License v3.0 or later** (AGPL-3.0-or-later).
 | See `LICENSE.md <https://github.com/OpenEnergyPlatform/open-MaStR/blob/production/LICENSE.md>`_ for rights and obligations.
 | See the *Cite this repository* function or `CITATION.cff <https://github.com/OpenEnergyPlatform/open-MaStR/blob/production/CITATION.cff>`_ for citation of this repository.
-| Copyright: `open-MaStR <https://github.com/OpenEnergyPlatform/open-MaStR/>`_ © `Reiner Lemoine Institut <https://reiner-lemoine-institut.de/>`_ © `fortiss <https://www.fortiss.org/>`_  | `AGPL-3.0-or-later <https://www.gnu.org/licenses/agpl-3.0.txt>`_
+| Copyright: `open-MaStR <https://github.com/OpenEnergyPlatform/open-MaStR/>`_ © `Reiner Lemoine Institut <https://reiner-lemoine-institut.de/>`_ © `fortiss <https://www.fortiss.org/>`_ © `OFFIS <https://www.offis.de/>`_  | `AGPL-3.0-or-later <https://www.gnu.org/licenses/agpl-3.0.txt>`_
 
 Data
 ----

RELEASE_PROCEDURE.md

@@ -48,22 +48,25 @@ It always has the format `YYYY-MM-DD`, e.g. `2022-05-16`.
 * On release day, start the release early to ensure sufficient time for reviews
 * Merge everything on the `develop` branch
 
-### 5. 💠 Create a `release` branch
+### 5. Run tests and apply code linting
+* Run tests locally with `pytest` and fix errors
+* Apply linting with `pre-commit run -a` and fix errors
+
+### 6. 💠 Create a `release` branch
 * Checkout `develop` and branch with `git checkout -b release-v0.12.1`
 * Update version for test release with `bump2version --current-version <current_version> --new-version <new_version> patch`
-* Commit version update with `git commit -am "version update v0.12.1a1"`
+* Commit version update with `git commit -am "version update v0.12.1"`
 * Push branch with `git push --set-upstream origin release-v0.12.1`
 
-### 6. 📝 Update the version files
+### 7. 📝 Update the version files
 * `📝CHANGELOG.md`
     * All Pull Request are included
     * Add a new section with correct version number
     * Give the suitable name to the release
 * `📝CITATION.cff`
     * Update `date-released`
 
-### 7. Optional: Check release on Test-PyPI 
-
+### 8. Optional: Check release on Test-PyPI
 * Check if the release it correctly displayed on [Test-PyPI](https://test.pypi.org/project/open-mastr/#history)
   * You can trigger the release manually within github actions using the `run workflow` button on branch `release-v0.12.1` on the workflow `Build and release on pypi tests`
   * Note: Pre-releases on Test-PyPI are only shown under `Release history` in the navigation bar.
@@ -72,7 +75,7 @@ It always has the format `YYYY-MM-DD`, e.g. `2022-05-16`.
   * Note: The release on Test-PyPI might fail, but it will be the correct release version for the PyPI server.
 * Push commits to the `release-*` branch
 
-### 8. 🐙 Create a `Release Pull Request`
+### 9. 🐙 Create a `Release Pull Request`
 * Use `📝PR_TEMPLATE_RELEASE` (❗ToDo❗)
 * Merge `release` into `production` branch
 * Assign reviewers to check the release
@@ -81,7 +84,7 @@ It always has the format `YYYY-MM-DD`, e.g. `2022-05-16`.
 * Wait for reviews and tests
 * Merge PR
 
-### 9. 💠 Set the `Git Tag`
+### 10. 💠 Set the `Git Tag`
 * Checkout `production` branch and pull
 * Check existing tags `git tag -n`
 * Create new tag: `git tag -a v0.12.1 -m "open-mastr release v0.12.1 with PyPI"`
@@ -91,7 +94,7 @@ It always has the format `YYYY-MM-DD`, e.g. `2022-05-16`.
     * Delete local tag: `git tag -d v0.12.1`
     * Delete remote tag: `git push --delete origin v0.12.1`
 
-### 10. 🐙 Publish `Release` on GitHub and PyPI
+### 11. 🐙 Publish `Release` on GitHub and PyPI
 * Navigate to your [releases](https://github.com/OpenEnergyPlatform/open-MaStR/releases/) on GitHub and open your draft release.
 * Summarize key changes in the description
     * Use the `generate release notes` button provided by github (This only works after the release branch is merged on production)
@@ -103,7 +106,7 @@ It always has the format `YYYY-MM-DD`, e.g. `2022-05-16`.
 
 ▶️ In the background the GitHub workflow (pypi-publish.yml) will publish the package 📦 on PyPI!
 
-### 11. 🐙 Set up new development
+### 12. 🐙 Set up new development
 * Create a Pull request from `release-*` to `develop`
 * Create a new **unreleased section** in the `📝CHANGELOG.md`
 ```

docs/advanced.md

@@ -1,5 +1,5 @@
 For most users, the functionalites described in [Getting Started](getting_started.md) are sufficient.  If you want 
-to examine how you can configure the package's behavior for your own needs, check out [Cofiguration](#configuration). Or you can explore the two main functionalities of the package, namely the [Bulk Download](#bulk-download)
+to examine how you can configure the package's behavior for your own needs, check out [Configuration](#configuration). Or you can explore the two main functionalities of the package, namely the [Bulk Download](#bulk-download)
 or the [SOAP API download](#soap-api-download).
 
 ## Configuration
@@ -28,6 +28,7 @@ The possible databases are:
 ### Project directory
 
 The directory `$HOME/.open-MaStR` is automatically created. It is used to store configuration files and save data.
+You can change this default path, see [environment variables](#environment-variables).
 Default config files are copied to this directory which can be modified - but with caution.
 The project home directory is structured as follows (files and folders below `data/` just an example).
 
@@ -87,6 +88,15 @@ The data can then be written to any sql database supported by [sqlalchemy](https
 For more information regarding the database see [Database settings](#database-settings).
 
 
+### Environment variables
+
+There are some environment variables to customize open-MaStR:
+
+| Variable               | Description                                                                                                                                                              | Example                                                                                                                    |
+|------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|
+| `SQLITE_DATABASE_PATH` | Path to the SQLite file. This allows to use to use multiple instances of the MaStR database. The database instances exist in parallel and are independent of each other. | `/home/mastr-rabbit/.open-MaStR/data/sqlite/your_custom_instance_name.db`                                                  |
+| `OUTPUT_PATH`          | Path to user-defined output directory for CSV data, XML file and database. If not specified, output directory defaults to `$HOME/.open-MaStR/`                           | Linux: `/home/mastr-rabbit/open-mastr-user-defined-output-path`, Windows: `C:\\Users\\open-mastr-user-defined-output-path` |
+
 ## Bulk download
 
 On the homepage [MaStR/Datendownload](https://www.marktstammdatenregister.de/MaStR/Datendownload) a zipped folder containing the whole
@@ -195,10 +205,16 @@ if __name__ == "__main__":
     print(mastr_api.GetLokaleUhrzeit())
 ```
 
-For API calls and their optional parameters refer to [API documentation](https://www.marktstammdatenregister.
-de/MaStRHilfe/subpages/webdienst.html).
+The MaStR API has different models to query from, the default are power units
+("Anlage"). To change this, you can pass the desired model to
+[`MaStRAPI`][open_mastr.soap_api.download.MaStRAPI].
+E.g. to query market actors instantiate it using
+`MaStRAPI(service_port="Akteur")`.
+
+For API calls, models and optional parameters refer to the
+[API documentation](https://www.marktstammdatenregister.de/MaStRHilfe/subpages/webdienst.html).
 
-???+ example "Example queries and their responses"
+???+ example "Example queries and their responses (for model 'Anlage')"
 
     === "mastr_api.GetLokaleUhrzeit()"
         

docs/conf.py

@@ -12,14 +12,15 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../open_mastr'))
+
+sys.path.insert(0, os.path.abspath("../open_mastr"))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'open-MaStR'
-copyright = '2022 Reiner Lemoine Institut and fortiss'
-author = ''
+project = "open-MaStR"
+copyright = "2024 Reiner Lemoine Institut gGmbH and fortiss GmbH and OFFIS e.V."
+author = ""
 
 
 # -- General configuration ---------------------------------------------------
@@ -28,36 +29,36 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autosectionlabel',
-    'sphinx.ext.autodoc',
-    'sphinx.ext.napoleon',
-    'sphinx_tabs.tabs',
-    'm2r2',
+    "sphinx.ext.autosectionlabel",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx_tabs.tabs",
+    "m2r2",
 ]
 
 source_suffix = [".rst", ".md"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-html_css_files = ['custom.css']
+html_static_path = ["_static"]
+html_css_files = ["custom.css"]
 
-# Autodoc config 
-autoclass_content = 'both'
+# Autodoc config
+autoclass_content = "both"

docs/dataset.md

@@ -81,6 +81,8 @@ After downloading the MaStR, you will find a database with a large number of tab
     | permit  |  |
     | storage_units  |  |
     | kwk  | *short for: Combined heat and power (CHP)* |
+    | deleted_units | Units from all technologies that were deleted or deactivated |
+    | deleted_market_actors | Market actors that were deleted or deactivated |
 
 
 ### MaStR data model