Merge branch 'feature/epathermostat_2.0' into feature/epa2.0_tau_expe…

…riments

.github/workflows/python-app.yaml

@@ -5,16 +5,15 @@ on: [push]
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         # Apparently 3.10 gets converted to 3.1 if it's not a string. Brilliant.
         python-version: [3.8]
-
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.github/workflows/windows_conda_testing.yaml

@@ -13,9 +13,9 @@ jobs:
 
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: 3.8
     - name: Test with pytest
@@ -31,5 +31,4 @@ jobs:
         pip install --upgrade pip
         pip install -r requirements.txt
         pip install -r dev-requirements.txt
-        pytest --cov-report=term-missing --cov=thermostat
-
+        pytest --cov-report=term-missing --cov=thermostat

.readthedocs.yaml

@@ -0,0 +1,29 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+    # You can also specify other tool versions:
+    # nodejs: "19"
+    # rust: "1.64"
+    # golang: "1.19"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/conf.py
+
+# If using Sphinx, optionally build your docs in additional formats such as PDF
+formats:
+   - pdf
+
+# Optionally declare the Python requirements required to build your docs
+python:
+   install:
+   - requirements: docs/rtd_requirements.txt

CHANGELOG.md

@@ -18,6 +18,7 @@
 * Updated logic to use ZIP Code for weather station lookup and climate zone lookup
 * Updated Climate Zone lookup strategy
 * Added minimum 30 thermostats per climate zone
+* Added checks for 95% of data being present _or_ 50 core days (heating and cooling).
 
 ## Production releases
 

docs/index.rst

@@ -45,6 +45,5 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 Contact
 -------
 
-Please feel free to reach out to either Abhishek Jathar
-([email protected]) or Craig Maloney ([email protected]) with
-questions or for technical support.
+Please feel free to reach out to Abhishek Jathar
+([email protected]) with questions or for technical support.

docs/rtd_requirements.txt

@@ -1,6 +1 @@
-numpy==1.19.4
-pandas>=0.25.3
-python-dateutil==2.8.1
-pytz==2020.4
-six==1.15.0
-pyproj==3.0.0post1
+six==1.16.0

requirements.in

@@ -0,0 +1,18 @@
+certifi>=2022.12.7
+charset-normalizer
+click
+eeweather
+greenlet
+idna
+numpy
+pandas
+pyproj
+python-dateutil
+pytz
+requests
+scipy
+shapely
+six
+sqlalchemy
+urllib3
+zipcodes

requirements.txt

@@ -2,68 +2,69 @@
 # This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
-#    pip-compile requirements.txt
+#    pip-compile
 #
-certifi==2021.5.30
+certifi==2022.12.7
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   pyproj
     #   requests
-charset-normalizer==2.0.4
+charset-normalizer==2.1.1
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   requests
-click==8.0.1
+click==8.1.3
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   eeweather
 eeweather==0.3.24
-    # via -r requirements.txt
-greenlet==1.1.1
+    # via -r requirements.in
+greenlet==2.0.1
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   sqlalchemy
-idna==3.2
+idna==3.4
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   requests
 numpy==1.22.4
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   pandas
     #   scipy
 pandas==1.5.2
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   eeweather
 python-dateutil==2.8.2
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   pandas
-pytz==2021.1
+pytz==2022.7
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   pandas
-requests==2.26.0
+requests==2.28.1
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   eeweather
-scipy==1.7.1
-    # via -r requirements.txt
-shapely==1.7.1
+scipy==1.10.0
+    # via -r requirements.in
+shapely==2.0.0
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   eeweather
 six==1.16.0
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   python-dateutil
-sqlalchemy==1.4.22
-    # via -r requirements.txt
-urllib3==1.26.6
+sqlalchemy==1.4.46
+    # via -r requirements.in
+urllib3==1.26.13
     # via
-    #   -r requirements.txt
+    #   -r requirements.in
     #   requests
 zipcodes==1.2.0
     # via -r requirements.txt
-loguru==0.7.2
+loguru==0.7.2
+    # via -r requirements.in

scripts/lxc/epathermostat_test

@@ -2,18 +2,18 @@
 CONTAINER=epathermostat
 lxc stop $CONTAINER
 lxc delete $CONTAINER
-lxc launch ubuntu:18.04 $CONTAINER -c security.privileged=true
+lxc launch ubuntu:20.04 $CONTAINER -c security.privileged=true
 sleep 3
 lxc config device add $CONTAINER homedir disk source=/home/craig/projects/epathermostat path=/home/ubuntu/epathermostat
 sleep 3
 lxc exec $CONTAINER -- sudo add-apt-repository ppa:deadsnakes/ppa -y
 lxc exec $CONTAINER -- sudo apt install -y \
     build-essential sqlite3 \
+    python3.11 python3.11-dev python3.11-venv \
     python3.10 python3.10-dev python3.10-venv \
     python3.9 python3.9-dev python3.9-venv \
     python3.8 python3.8-dev python3.8-venv \
-    python3.7 python3.7-dev python3.7-venv \
-    python3.6 python3.6-dev python3.6-venv
+    python3.7 python3.7-dev python3.7-venv
 lxc exec $CONTAINER -- sudo chown ubuntu /home/ubuntu
 lxc exec $CONTAINER -- sudo --login --user ubuntu /usr/bin/python3.9 -m venv /home/ubuntu/venv
 lxc exec $CONTAINER -- sudo --login --user ubuntu /home/ubuntu/venv/bin/pip install -r /home/ubuntu/epathermostat/dev-requirements.txt

scripts/metadata_converter.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 import csv
 import glob
-import os
+from pathlib import Path
 
 from datetime import datetime
 
@@ -36,7 +36,7 @@ def main():
 
     for filename in glob.iglob('metadata*.csv'):
         input_filename = filename
-        output_filename = os.path.join("new", filename)
+        output_filename = Path("new") / filename
         with open(output_filename, 'w') as outfile:
             csv_out = csv.DictWriter(outfile, FIELDNAMES)
             csv_out.writeheader()

scripts/multi_thermostat_driver.py

@@ -1,4 +1,8 @@
 from pathlib import Path
+import logging
+import logging.config
+import json
+import csv
 from datetime import date
 from multi_thermostat_driver_func import mult_thermostat_driver
 # These are variables used in the example code. Please tailor these to your
@@ -81,6 +85,119 @@
 CLIMATE_ZONE_INSUFFICIENT_FILEPATH = OUTPUT_DIR / CLIMATE_ZONE_INSUFFICIENT_FILENAME
 ZIP_FILEPATH = OUTPUT_DIR / ZIP_FILENAME
 
+def write_errors(filepath, fieldnames, errors, extrasaction=None):
+    with open(filepath, 'w') as error_file:
+        if extrasaction:
+            writer = csv.DictWriter(
+                error_file,
+                fieldnames=fieldnames,
+                dialect='excel',
+                extrasaction=extrasaction)
+        else:
+            writer = csv.DictWriter(
+                error_file,
+                fieldnames=fieldnames,
+                dialect='excel')
+        writer.writeheader()
+        for error in errors:
+            writer.writerow(error)
+
+
+def count_metadata(filepath):
+    with open(filepath, 'r') as metadata_file:
+        reader = csv.DictReader(metadata_file)
+        return len(list(reader))
+
+
+def main():
+    '''
+    This script processes the thermostat metadata and data files to generate
+    the certification files for submission to EPA.
+    '''
+
+    logging.basicConfig()
+    with open(LOGGING_CONFIG, 'r') as logging_config:
+        logging.config.dictConfig(json.load(logging_config))
+
+    # Uses the 'epathermostat' logging
+    logger = logging.getLogger('epathermostat')
+    logger.debug('Starting...')
+    logging.captureWarnings(CAPTURE_WARNINGS)
+
+    thermostats, import_errors = from_csv(
+        METADATA_FILENAME,
+        verbose=VERBOSE,
+        save_cache=SAVE_CACHE,
+        cache_path=CACHE_PATH)
+
+    # This logs any import errors that might have occurred.
+    if import_errors:
+        # This writes a file with the thermostat ID as part of the file. This
+        # is for your own troubleshooting
+        fieldnames = ['thermostat_id', 'error']
+        write_errors(IMPORT_ERRORS_FILEPATH, fieldnames, import_errors)
+
+        # This writes a file without the thermostat ID as part of the file.
+        # This file is sent as part of the certification to help with
+        # diagnosing issues with missing thermostats
+        fieldnames = ['error']
+        write_errors(SANITIZED_IMPORT_ERRORS_FILEPATH, fieldnames, import_errors, extrasaction='ignore')
+
+    # Check to see how many thermostats we are importing and warn if less than 30%
+    metadata_count = count_metadata(METADATA_FILENAME)
+    thermostat_estimate_count = thermostats.__length_hint__()  # Get a rough estimate of the number of thermostats
+    percent_thermostats_imported = (thermostat_estimate_count / metadata_count) * 100
+    if percent_thermostats_imported < 30:
+        logger.warning(f'Imported {percent_thermostats_imported}% of thermostats, which is less than 30%')
+        logger.warning(f'Please check {IMPORT_ERRORS_FILEPATH} for more details')
+    else:
+        logger.debug(f'Imported {percent_thermostats_imported}% of thermostats')
+
+    metrics = multiple_thermostat_calculate_epa_field_savings_metrics(thermostats)
+
+    metrics_out = metrics_to_csv(metrics, METRICS_FILEPATH)
+
+    stats, insufficient = compute_summary_statistics(metrics_out)
+
+    if insufficient:
+        fieldnames = ['climate_zone', 'count', 'error']
+        write_errors(CLIMATE_ZONE_INSUFFICIENT_FILEPATH, fieldnames, insufficient)
+
+    certification_to_csv(stats, CERTIFICATION_FILEPATH, PRODUCT_ID)
+
+    summary_statistics_to_csv(
+        stats,
+        STATS_FILEPATH,
+        PRODUCT_ID)
+
+    if ADVANCED_STATS:
+        stats_advanced = compute_summary_statistics(
+            metrics_out,
+            advanced_filtering=True)
+
+        summary_statistics_to_csv(
+            stats_advanced,
+            STATS_ADVANCED_FILEPATH,
+            PRODUCT_ID)
+
+    # Compile the files together in a neat package
+    files_to_zip = [
+        CERTIFICATION_FILEPATH,
+        STATS_FILEPATH,
+        ]
+    if ADVANCED_STATS:
+        files_to_zip.append(STATS_ADVANCED_FILEPATH)
+
+    if import_errors:
+        files_to_zip.append(SANITIZED_IMPORT_ERRORS_FILEPATH)
+
+    if insufficient:
+        files_to_zip.append(CLIMATE_ZONE_INSUFFICIENT_FILEPATH)
+
+    with ZipFile(ZIP_FILEPATH, 'w') as certification_zip:
+        for filename in files_to_zip:
+            if filename.exists():
+                certification_zip.write(filename, arcname=filename.name)
 
 if __name__ == '__main__':
     mult_thermostat_driver(