From c97595fdf98d52a377233fe09a779ee3d2bc52c5 Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Sat, 3 Dec 2022 08:32:30 -0800 Subject: [PATCH 01/29] Add function to reset config values of RESET and historyDays Without this when you rerun after a DB reset or history load those functions will repeat. Potential of losing data. If historyDays is non zero then after history is loaded it will reset in the json config to 0 If reset = true in the json config , after database reset it will be changed to false --- src/vuegraf/vuegraf.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 2d294c5..b58112a 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -40,6 +40,16 @@ def getConfigValue(key, defaultValue): return config[key] return defaultValue +# Reset config file if history or DB reset set +# Allows sequential runs without lossing data +def setconfig(configname, configkey, configvalue) : + with open(configFilename, 'r') as newconfigFile: + newconfig = json.load(newconfigFile) + newconfig[configname][configkey] = configvalue + with open(configFilename, 'w') as configout: + json.dump(newconfig, configout, indent=4) + return() + def populateDevices(account): deviceIdMap = {} account['deviceIdMap'] = deviceIdMap @@ -199,6 +209,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd start = "1970-01-01T00:00:00Z" stop = startupTime.isoformat(timespec='seconds') delete_api.delete(start, stop, '_measurement="energy_usage"', bucket=bucket, org=org) + setconfig('influxDb','reset','false') else: info('Using InfluxDB version 1') @@ -217,6 +228,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if config['influxDb']['reset']: info('Resetting database') influx.delete_series(measurement='energy_usage') + setconfig('influxDb','reset','false') historyDays = min(config['influxDb'].get('historyDays', 0), 7) history = historyDays > 0 @@ -271,6 +283,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd break pauseEvent.wait(5) history = False + setconfig('influxDb','historyDays', 0) if not running: break From 3f6bc4883623ed2837ec6fc525f33aa94960d486 Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Sun, 19 Mar 2023 10:35:42 -0700 Subject: [PATCH 02/29] Update README.md --- README.md | 278 ++++-------------------------------------------------- 1 file changed, 17 insertions(+), 261 deletions(-) diff --git a/README.md b/README.md index c90a8b1..217a303 100644 --- a/README.md +++ b/README.md @@ -1,269 +1,25 @@ -![Vuegraf Logo](https://github.com/jertel/vuegraf/blob/master/vuegraf.png?raw=true "Vuegraf Logo") -# Overview +This is a fork of vuegraf from jertel - https://github.com/jertel/vuegraf -The [Emporia Vue](https://emporiaenergy.com "Emporia's Homepage") energy monitoring kit allows homeowners to monitor their electrical usage. It monitors the main feed consumption and up to 8 (or 16 in the newer version) individual branch circuits, and feeds that data back to the Emporia API server. +Goal to add additional features to either be reveiwed/approved and merged back into jertel's code or remain a seperate product. -This project, Vuegraf, fetches those metrics from the Emporia Vue API host and stores the metrics into your own InfluxDB. After installation you will be able to: -* View your energy usage across all circuits on a single graph -* Create alerts to notify when certain energy usage thresholds are exceeded +Additional Features +- [X] Reset config json file values to allow multiple runs without reseting database. +- [X] Database reset +- [X} History Days +- [ ] Add additional scales to the pull allowing to get years of history since minutes/second have a short life at emporia. +
This also allows faster dashboards for creating itmes like daily or monthly graphs. +- [ ] Hours +- [ ] Days +- [ ] Months +- *Note while adding additional scales will keep seconds = true and minutes = false to not impact existing reporting. +- But add additonal tags for sec/min/hour/day/month -This project is not affiliated with _emporia energy_ company. -# Dependencies - -* [Emporia Vue](https://emporiaenergy.com "Emporia Energy") Account - Username and password for the Emporia Vue system are required. -* [Python 3](https://python.org "Python") - With Pip. -* [InfluxDB 2](https://influxdata.com "InfluxDB") - Host, port, org, bucket, and token are all required. - -# Influx - -## Setup - -If you do not yet have a running InfluxDB 2 instance, you will need to set one up. You can do this very quickly by launching an InfluxDB 2 Docker container as follows: - -``` -mkdir -p /home/myuser/influxdb2 -docker run -v /home/myuser/influxdb2:/var/lib/influxdb2 -p 8086:8086 -e INFLUXD_SESSION_LENGTH=432000 --name influxdb influxdb -``` - -Substitute an appropriate host path for the `/home/myuser/influxdb2` location above. Once running, access the web UI at `http://localhost:8086`. It will prompt you for a username, password, organization name, and bucket name. The rest of this document assumes you have entered the word `vuegraf` for all of these inputs. - -Note that the default session timeout for Influx is only 60 minutes, so this command increases the login session to 300 days. - -Once logged in, go to the _Load Data -> API Tokens_ screen and generate a new All Access token with the description of _vuegraf_. Copy the generated token for use in the rest of this document, specifically when referenced as ``. - -## Dashboard - -By default, a new InfluxDB instance will not have any dashboards loaded. You will need to import the included Influx JSON template, or create your own dashboard in order to visualize your energy usage. - -The included template file named `influx_dashboard.json` includes the provided dashboard and accompanying variables to reproduce the visualizations shown below. This dashboard assumes your main device name contains the word `Panel`, such as `House Panel`, or `Right Panel`. If it does not, the Flux queries will need to be adjusted manually to look for your device's name. - -![Influx Dashboard Screenshot](https://github.com/jertel/vuegraf/blob/master/screenshots/influx_dashboard.png?raw=true "Influx Dashboard") - -You will need to apply this template file to your running InfluxDB instance. Copy the `influx_dashboard.json` file into your host's influxdb2 path. If you followed the Setup instructions above, the path would be `/home/myuser/influxdb2`. The below command can be used to perform this step. This command assumes you are running Influx in a container named `influxdb`. - -``` -docker exec influxdb influx apply -f /var/lib/influxdb2/influx_dashboard.json --org vuegraf --force yes -t -``` - -Replace the `` with the All Access Token you generated in the Influx _Load Data -> API Tokens_ screen. - -You're now ready to proceed with the Vuegraf configuration and startup. - -# Configuration - -The configuration allows for the definition of multiple Emporia Vue accounts. This will only be useful to users that need to pull metrics from multiple accounts. This is not needed if you have multiple Vue devices in a single account. Vuegraf will find multiple devices on its own within each account. - -The email address and password must match the credentials used when creating the Emporia Vue account in their mobile app. - -Important: Ensure that sufficient protection is in place on this configuration file, since it contains the plain-text login credentials into the Emporia Vue account. - -A [sample configuration file](https://github.com/jertel/vuegraf/blob/master/vuegraf.json.sample "Sample Vuegraf Configuration File") is provided in this repository, and details are described below. - -## Minimal Configuration -The minimum configuration required to start Vuegraf is shown below. - -```json -{ - "influxDb": { - "version": 2, - "url": "http://my.influxdb.hostname:8086", - "org": "vuegraf", - "bucket": "vuegraf", - "token": "", - "reset": false - }, - "accounts": [ - { - "name": "Primary Residence", - "email": "my@email.address", - "password": "my-emporia-password" - } - ] -} -``` - -## Advanced Configuration - -### Ingesting Historical Data - -If desired, it is possible to have Vuegraf import historical data. To do so, specify a new temporary parameter called `historyDays` inside the `influxDb` section, with an integer value greater than zero. Once restarted, One-minute data from the past `historyDays` days will be ingested into InfluxDB. Emporia currently retains this data for 7 days, and therefore `historyDays` must be less than or equal to `7`. If `historyDays` is set to `0`, no historical data will be ingested into InfluxDB. - -IMPORTANT - If you restart Vuegraf with historyDays still set to a non-zero value then it will _again_ import history data. This will likely cause confusion with your data since you will now have duplicate/overlapping data. For best results, only enable historyDays > 0 for a single run, and then immediately set it back to 0 to avoid this duplicated import data scenario. - -### Channel Names - -To provide more user-friendly names of each Vue device and branch circuit, the following device configuration can be added to the configuration file, within the account block. List each device and circuit in the order that you added them to the Vue mobile app. The channel names do not need to match the names specified in the Vue mobile app but the device names must match. The below example shows two 8-channel Vue devices for a home with two breaker panels. - -Be aware that the included dashboard assumes your device name contains the word "Panel". For best results, consider renaming your Vue device to contain that word, otherwise you will need to manually adjust the included dashboards' queries. - -```json - "devices": [ - { - "name": "Right Panel", - "channels": [ - "Air Conditioner", - "Furnace", - "Coffee Maker", - "Oven", - "Pool Vacuum", - "Pool Filter", - "Refrigerator", - "Office" - ] - }, - { - "name": "Left Panel", - "channels": [ - "Dryer", - "Washer", - "Dishwasher", - "Water Heater", - "Landscape Features", - "Septic Pump", - "Deep Freeze", - "Sprinkler Pump" - ] - } - ] -``` - -# Running -Vuegraf can be run either as a container (recommended), or as a host process. - -## Container (recommended) - -A Docker container is provided at [hub.docker.com](https://hub.docker.com/r/jertel/vuegraf). Refer to the command below to launch Vuegraf as a container. This assumes you have created a folder called `/home/myuser/vuegraf` and placed the vuegraf.json file inside of it. - -```sh -docker run --name vuegraf -d -v /home/myuser/vuegraf:/opt/vuegraf/conf jertel/vuegraf -``` - -## Host Process - -Ensure Python 3 and Pip are both installed. Install the required dependencies: - -```sh -pip install -r src/requirements.txt -``` -or, on some Linux installations: - -```sh -pip3 install -r src/requirements.txt -``` - - -Then run the program via Python, specifying the JSON configuration file path as the only argument: - -```sh -python src/vuegraf/vuegraf.py vuegraf.json -``` -or, on some Linux installations: -```sh -python3 src/vuegraf/vuegraf.py vuegraf.json -``` - -## Alerts - -The included dashboard template contains two alerts which will trigger when either a power outage occurs, or a loss of Vuegraf data. There are various reasons why alerts can be helpful. See the below screenshots which help illustrate how a fully functioning alert and notification rule might look. Note that the included alerts do not send out notifications. To enable outbound notifactions, such as to Slack, you can create a Notification Endpoint and Notification Rule. - -This alert was edited via the text (Flux) interface since the alert edit UI does not yet accommodate advanced alerting inputs. - -Side note: The logo at the top of this documentation satisfies Slack's icon requirements. Consider using it to help quickly distinguish between other alerts. - -![Influx Alert Edit](https://github.com/jertel/vuegraf/blob/master/screenshots/alert_edit.png?raw=true "Influx Alert") - -This notification rule provides an example of how you can have several alerts change the status to crit, but only a single notification rule is required to transmit notifications to external endpoints (such as email or Slack). - -![Influx Notification Rule](https://github.com/jertel/vuegraf/blob/master/screenshots/notification_rule.png?raw=true "Influx Notification Rule") - - -# Additional Topics - -## Per-second Data Details - -By default, Vuegraf will poll every minute to collect the energy usage value over the past 60 seconds. This results in a single value being capture per minute per channel, or 60 values per hour per channel. If you also would like to see per-second values, you can enable the detailed collection, which is polled once per hour, and backfilled over the previous 3600 seconds. This API call is very expensive on the Emporia servers, so it should not be polled more frequently than once per hour. To enable this detailed data, add (or update) the top-level `detailedDataEnabled` configuration value with a value of `true`. - -``` -detailedDataEnabled: true -``` - -Again: - -- `detailed = True` represents backfilled per-second data that is optionally queried from Emporia once every hour. -- `detailed = False` represents the per-minute average data that is collected every minute. - -When building graphs that show a sum of the energy usage, be sure to only include either detailed=true or detailed=false, otherwise your summed values will be higher than expected. Detailed data will take more time for the graphs to query due to the extra data involved. By default, it is set to False so most users can ignore this note. - -## Vue Utility Connect Energy Monitor - -As reported in [discussion #104](https://github.com/jertel/vuegraf/discussions/104), the Utility Connect device is supported without any custom changes. - -## Smart Plugs - -To include an Emporia smart plug in the configuration, add each plug as it's own device, without channels. Again, the name of the Smart Plug device must exactly match the name you gave the device in the Vue app during initial registration. - -```json - devices: [ - { - "name": "Main Panel", - "channels": [ - "Air Conditioner", - "Furnace", - "Coffee Maker", - "Oven", - "Dishwasher", - "Tesla Charger", - "Refrigerator", - "Office" - ] - }, - { - "name": "Projector Plug" - }, - { - "name": "3D-Printer Plug" - } - ] -``` - -## Docker Compose - -For those that want to run Vuegraf using Docker Compose, the following files have been included: `docker-compose.yaml.template` and `docker-compose-run.sh`. Copy the`docker-compose.yaml.template` file to a new file called `docker-compose.yaml`. In the newly copied file, `vuegraf.volumes` values will need to be changed to the same directory you have created your vuegraf.json file. Additionally, adjust the persistent host storage path for the InfluxDB data volume. - -Finally run the `docker-compose-run.sh` script to start up the multi-container application. - -```sh -./docker-compose-run.sh -``` - -## Upgrading from InfluxDB v1 - -Early Vuegraf users still on InfluxDB v1 can upgrade to InfluxDB 2. To do so, stop the Influx v1 container (again, assuming you're using Docker). Then run the following command to install InfluxDB 2 and automatically upgrade your data. - -``` -docker run --rm --pull always -p 8086:8086 \ - -v /home/myuser/influxdb:/var/lib/influxdb \ - -v /home/myuser/influxdb2:/var/lib/influxdb2 \ - -e DOCKER_INFLUXDB_INIT_MODE=upgrade \ - -e DOCKER_INFLUXDB_INIT_USERNAME=vuegraf \ - -e DOCKER_INFLUXDB_INIT_PASSWORD=vuegraf \ - -e DOCKER_INFLUXDB_INIT_ORG=vuegraf \ - -e DOCKER_INFLUXDB_INIT_BUCKET=vuegraf \ - -e DOCKER_INFLUXDB_INIT_RETENTION=1y \ - influxdb -``` - -Adjust the host paths above as necessary, to match the old and new influxdb directories. The upgrade should complete relatively quickly. For reference, a 7GB database, spanning several months, upgrades in about 15 seconds on SSD storage. - -Monitor the console output and once the upgrade completes and the Influx server finishes starting, shut it down (CTRL+C) and then restart the Influx DB using the command referenced earlier in this document. - -Login to the new Influx DB 2 UI from your web browser, using the _vuegraf / vuegraf_ credentials. Go into the _Load Data -> Buckets_ screen and rename the `vue/autogen` bucket to `vuegraf` via the Settings button. - -Finally, apply the dashboard template as instructed earlier in this document. - -# License +# License +As this is a fork, will hour current license and all whishes of jertel's project. + This is a work in progress with plans to work with Jertel in the future to merge back, so all + docuemntaion and usage is back on the main project: (https://github.com/jertel/vuegra) Vuegraf is distributed under the MIT license. From 2c467a32f589dc6c7c2e7b0404dd5d2739a5e114 Mon Sep 17 00:00:00 2001 From: Garrett Gauthier Date: Sun, 19 Mar 2023 10:36:46 -0700 Subject: [PATCH 03/29] modified: src/vuegraf/vuegraf.py Added config json file reset for both history days (0) and database reset (false) --- src/vuegraf/vuegraf.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index b58112a..1ff92f7 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -142,7 +142,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if chanNum in excludedDetailChannelNumbers: continue - if detailedEnabled: + if collectDetails: usage, usage_start_time = account['vue'].get_chart_usage(chan, detailedStartTime, stopTime, scale=Scale.SECOND.value, unit=Unit.KWH.value) index = 0 for kwhUsage in usage: @@ -242,13 +242,15 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd intervalSecs=getConfigValue("updateIntervalSecs", 60) detailedIntervalSecs=getConfigValue("detailedIntervalSecs", 3600) + detailedDataEnabled=getConfigValue("detailedDataEnabled", False); + info('Settings -> updateIntervalSecs: {}, detailedEnabled: {}, detailedIntervalSecs: {}'.format(intervalSecs, detailedDataEnabled, detailedIntervalSecs)) lagSecs=getConfigValue("lagSecs", 5) detailedStartTime = startupTime while running: now = datetime.datetime.utcnow() stopTime = now - datetime.timedelta(seconds=lagSecs) - detailedEnabled = (stopTime - detailedStartTime).total_seconds() >= detailedIntervalSecs + collectDetails = detailedDataEnabled and detailedIntervalSecs > 0 and (stopTime - detailedStartTime).total_seconds() >= detailedIntervalSecs for account in config["accounts"]: if 'vue' not in account: @@ -267,7 +269,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if history: for day in range(historyDays): - info('Loading historical data: {} day ago'.format(day+1)) + info('Loading historical data: {} day(s) ago'.format(day+1)) #Extract second 12h of day historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)-43200) historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day))) @@ -298,7 +300,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd error('Failed to record new usage data: {}'.format(sys.exc_info())) traceback.print_exc() - if detailedEnabled: + if collectDetails: detailedStartTime = stopTime + datetime.timedelta(seconds=1) pauseEvent.wait(intervalSecs) From 62e42ed100646864ea5c0caa4ac899b07bec60b3 Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Sun, 19 Mar 2023 10:41:57 -0700 Subject: [PATCH 04/29] json config file reset Will reset json config file values of: history days = 0 database reset = false Allows for multiple runs without loosing data or rebuilding history. --- src/vuegraf/vuegraf.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 5296a56..b6a572e 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -40,6 +40,16 @@ def getConfigValue(key, defaultValue): return config[key] return defaultValue +# Reset config file if history or DB reset set +# Allows sequential runs without lossing data +def setconfig(configname, configkey, configvalue) : + with open(configFilename, 'r') as newconfigFile: + newconfig = json.load(newconfigFile) + newconfig[configname][configkey] = configvalue + with open(configFilename, 'w') as configout: + json.dump(newconfig, configout, indent=4) + return() + def populateDevices(account): deviceIdMap = {} account['deviceIdMap'] = deviceIdMap @@ -199,6 +209,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd start = "1970-01-01T00:00:00Z" stop = startupTime.isoformat(timespec='seconds') delete_api.delete(start, stop, '_measurement="energy_usage"', bucket=bucket, org=org) + setconfig('influxDb','reset','false') else: info('Using InfluxDB version 1') @@ -217,6 +228,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if config['influxDb']['reset']: info('Resetting database') influx.delete_series(measurement='energy_usage') + setconfig('influxDb','reset','false') historyDays = min(config['influxDb'].get('historyDays', 0), 7) history = historyDays > 0 @@ -273,6 +285,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd break pauseEvent.wait(5) history = False + setconfig('influxDb','historyDays', 0) if not running: break @@ -296,3 +309,4 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd except: error('Fatal error: {}'.format(sys.exc_info())) traceback.print_exc() + From ffc877f8a08feb4df5370409568ac1866d8155e7 Mon Sep 17 00:00:00 2001 From: Garrett Gauthier Date: Wed, 22 Mar 2023 13:58:05 -0700 Subject: [PATCH 05/29] contributor: gauthig modified: src/vuegraf/vuegraf.py config loading changes split historyDays into minutes and days with appropriate max values added parm (collectSummaries) to enable hours/days. default = FALSE enhancement - true/false parms for min,hour,day extractDataPoints - History histLoop var - needed to break day/month setup differently from minute extract Hour / Day extraction loops setup max days of 25 for each api call, received api faliures at 30 day from emporia Hour/Day loop can take a long time Opertunity to create background task and slow down large extracts even more extractDataPoints - Ongoing Day and Hour set to be pulled on run after 2:00pm local time Hours/day sumary will only be pulled daily. used same loop for history pull ISSUE - Used chart for Day, not efficent, when pulling usage had problem with day not pulling past day. if history block histLoop - seperate minute from hour/day pull influxdb v1 - added batch_size to write_points history load can exceed influxdb client size influxdb v2 - batch size built into write api modified: vuegraf.json.sample Added summariesDataEnabled with default to false must be aware of how to split days/hours/minutes from each other --- src/vuegraf/vuegraf.py | 106 ++++++++++++++++++++++++++++++++++------- vuegraf.json.sample | 5 +- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index b6a572e..9a394a3 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -120,7 +120,7 @@ def createDataPoint(account, chanName, watts, timestamp, detailed): } return dataPoint -def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEndTime=None): +def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEndTime=None, histLoop=None): excludedDetailChannelNumbers = ['Balance', 'TotalUsage'] minutesInAnHour = 60 secondsInAMinute = 60 @@ -143,6 +143,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd continue if collectDetails: + #Seconds usage, usage_start_time = account['vue'].get_chart_usage(chan, detailedStartTime, stopTime, scale=Scale.SECOND.value, unit=Unit.KWH.value) index = 0 for kwhUsage in usage: @@ -154,7 +155,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd index += 1 # fetches historical minute data - if historyStartTime is not None and historyEndTime is not None: + if histLoop == "minute" or collectSummaries: usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.MINUTE.value, unit=Unit.KWH.value) index = 0 for kwhUsage in usage: @@ -163,7 +164,61 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd timestamp = historyStartTime + datetime.timedelta(minutes=index) watts = float(minutesInAnHour * wattsInAKw) * kwhUsage usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, False)) + index += 1 + # fetches historical hour data + if histLoop == "day-hour" or collectSummaries: + + #Fetches historical hour data + historyStartTime = stopTime - datetime.timedelta(days=historyDays) + historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) + dayLoop = historyDays / 25 + historyEndTime = historyStartTime + datetime.timedelta(days=25) + historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) + dayLoop = historyDays / 25 + while dayLoop > 0 : + historyEndTime = min(stopTime, historyStartTime + datetime.timedelta(days=25)) + historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) + usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.HOUR.value, unit=Unit.KWH.value) + index = 0 + for kwhUsage in usage: + if kwhUsage is None: + continue + timestamp = usage_start_time + datetime.timedelta(hours=index) + watts = kwhUsage * 1000 + usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Hour")) + index += 1 + dayLoop = dayLoop -1 + historyStartTime = historyStartTime + datetime.timedelta(days=26) + + # fetches historical Day data + historyStartTime = stopTime - datetime.timedelta(days=historyDays) + historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) + dayLoop = historyDays / 25 + historyEndTime = historyStartTime + datetime.timedelta(days=25) + historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) + dayLoop = historyDays / 25 + while dayLoop > 0 : + historyEndTime = min(stopTime, historyStartTime + datetime.timedelta(days=25)) + historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) + usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.DAY.value, unit=Unit.KWH.value) + index = 0 + for kwhUsage in usage: + if kwhUsage is None: + continue + timestamp = usage_start_time + datetime.timedelta(days=index) + watts = kwhUsage * 1000 + usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Day")) + index += 1 + dayLoop = dayLoop -1 + historyStartTime = historyStartTime + datetime.timedelta(days=26) + + + + + + + startupTime = datetime.datetime.utcnow() try: @@ -230,28 +285,30 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd influx.delete_series(measurement='energy_usage') setconfig('influxDb','reset','false') - historyDays = min(config['influxDb'].get('historyDays', 0), 7) + historyMinute = min(config['influxDb'].get('historyDays', 0), 7) + historyDays = min(config['influxDb'].get('historyDays', 0), 720) history = historyDays > 0 running = True - signal.signal(signal.SIGINT, handleExit) signal.signal(signal.SIGHUP, handleExit) - pauseEvent = Event() - intervalSecs=getConfigValue("updateIntervalSecs", 60) detailedIntervalSecs=getConfigValue("detailedIntervalSecs", 3600) - detailedDataEnabled=getConfigValue("detailedDataEnabled", False); - info('Settings -> updateIntervalSecs: {}, detailedEnabled: {}, detailedIntervalSecs: {}'.format(intervalSecs, detailedDataEnabled, detailedIntervalSecs)) + detailedDataEnabled=getConfigValue("detailedDataEnabled", False) + summariesDataEnabled=getConfigValue('summariesDataEnabled', False) + info('Settings -> updateIntervalSecs: {}, detailedEnabled: {}, detailedIntervalSecs: {}, summariesEnabled {}'.format(intervalSecs, detailedDataEnabled, detailedIntervalSecs, summariesDataEnabled)) + info(' History Days {}'.format(historyDays)) lagSecs=getConfigValue("lagSecs", 5) detailedStartTime = startupTime - + #Only pull Hourly data and Full day after 0200 + summariesDate = datetime.datetime.now() + datetime.timedelta(days=1) + summariesDate = summariesDate.replace(hour=2) while running: now = datetime.datetime.utcnow() stopTime = now - datetime.timedelta(seconds=lagSecs) collectDetails = detailedDataEnabled and detailedIntervalSecs > 0 and (stopTime - detailedStartTime).total_seconds() >= detailedIntervalSecs - + collectSummaries = summariesDataEnabled and datetime.datetime.now() >= summariesDate for account in config["accounts"]: if 'vue' not in account: account['vue'] = PyEmVue() @@ -268,19 +325,33 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd extractDataPoints(device, usageDataPoints) if history: - for day in range(historyDays): - info('Loading historical data: {} day(s) ago'.format(day+1)) + for day in range(historyMinute): + histLoop = "**********minute************" + info('Loading historical data - Minutes: {} day(s) ago'.format(day+1)) #Extract second 12h of day historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)-43200) historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day))) for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime) + extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) pauseEvent.wait(5) #Extract first 12h of day historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)) historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day+1))-43200) for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime) + extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) + if not running: + break + pauseEvent.wait(5) + + if summariesDataEnabled: + histLoop = "day-hour" + info('Loading historical data - Days/Hours/Months: {} day(s) ago'.format(historyDays)) + for gid, device in usages.items(): + historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(historyDays)) + historyEndTime = stopTime + historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) + historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) + extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) if not running: break pauseEvent.wait(5) @@ -294,7 +365,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if influxVersion == 2: write_api.write(bucket=bucket, record=usageDataPoints) else: - influx.write_points(usageDataPoints) + influx.write_points(usageDataPoints,batch_size=5000) except: error('Failed to record new usage data: {}'.format(sys.exc_info())) @@ -302,7 +373,9 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if collectDetails: detailedStartTime = stopTime + datetime.timedelta(seconds=1) - + if collectSummaries: + summariesDate = datetime.datetime.now() + datetime.timedelta(days=1) + summariesDate = summariesDate.replace(hour=2) pauseEvent.wait(intervalSecs) info('Finished') @@ -310,3 +383,4 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd error('Fatal error: {}'.format(sys.exc_info())) traceback.print_exc() + diff --git a/vuegraf.json.sample b/vuegraf.json.sample index defba70..91ec5b9 100644 --- a/vuegraf.json.sample +++ b/vuegraf.json.sample @@ -7,9 +7,12 @@ "database": "vue", "reset": false, "ssl_enable": false, - "ssl_verify": true + "ssl_verify": true, + }, "updateIntervalSecs": 60, + "detailedDataEnabled": "true", + "summariesDataEnabled": "false" "accounts": [ { "name": "Primary Residence", From c4ef3d0494bda9d79dd9b040eba9d753430d3ac9 Mon Sep 17 00:00:00 2001 From: Garrett Gauthier Date: Fri, 24 Mar 2023 10:51:16 -0700 Subject: [PATCH 06/29] modified: src/vuegraf/vuegraf.py Fix Hour and Day history Loops from exceeding current date Timestamp is off - Need to fix. --- src/vuegraf/vuegraf.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 9a394a3..4711f7a 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -176,7 +176,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd historyEndTime = historyStartTime + datetime.timedelta(days=25) historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) dayLoop = historyDays / 25 - while dayLoop > 0 : + while dayLoop > 0 and historyStartTime < stopTime : historyEndTime = min(stopTime, historyStartTime + datetime.timedelta(days=25)) historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.HOUR.value, unit=Unit.KWH.value) @@ -198,7 +198,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd historyEndTime = historyStartTime + datetime.timedelta(days=25) historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) dayLoop = historyDays / 25 - while dayLoop > 0 : + while dayLoop > 0 and historyStartTime < stopTime : historyEndTime = min(stopTime, historyStartTime + datetime.timedelta(days=25)) historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.DAY.value, unit=Unit.KWH.value) @@ -212,13 +212,6 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd index += 1 dayLoop = dayLoop -1 historyStartTime = historyStartTime + datetime.timedelta(days=26) - - - - - - - startupTime = datetime.datetime.utcnow() try: @@ -326,7 +319,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if history: for day in range(historyMinute): - histLoop = "**********minute************" + histLoop = "minute" info('Loading historical data - Minutes: {} day(s) ago'.format(day+1)) #Extract second 12h of day historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)-43200) @@ -351,6 +344,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd historyEndTime = stopTime historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) + print('time before call ',stopTime,historyStartTime ,historyEndTime ) extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) if not running: break From 992bf50ad9419b831c47358de83196abb9403e8a Mon Sep 17 00:00:00 2001 From: Garrett Gauthier Date: Sun, 26 Mar 2023 10:36:08 -0700 Subject: [PATCH 07/29] modified: src/vuegraf/vuegraf.py Cleanned up hour/day loops and combined into 1 section removed extra history end/start time setups removed debuging print/info statements --- src/vuegraf/vuegraf.py | 39 ++++++++++----------------------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 4711f7a..f229c5d 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -155,7 +155,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd index += 1 # fetches historical minute data - if histLoop == "minute" or collectSummaries: + if histLoop == "minute" : usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.MINUTE.value, unit=Unit.KWH.value) index = 0 for kwhUsage in usage: @@ -166,17 +166,14 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, False)) index += 1 - # fetches historical hour data + # fetches historical hour/day or just previous day if histLoop == "day-hour" or collectSummaries: - - #Fetches historical hour data historyStartTime = stopTime - datetime.timedelta(days=historyDays) historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) + #only used on history runs, otherwise is zero and will do next loop once. dayLoop = historyDays / 25 - historyEndTime = historyStartTime + datetime.timedelta(days=25) - historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) - dayLoop = historyDays / 25 - while dayLoop > 0 and historyStartTime < stopTime : + while dayLoop >= 0 and historyStartTime < stopTime : + #Fetches hour data historyEndTime = min(stopTime, historyStartTime + datetime.timedelta(days=25)) historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.HOUR.value, unit=Unit.KWH.value) @@ -188,19 +185,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd watts = kwhUsage * 1000 usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Hour")) index += 1 - dayLoop = dayLoop -1 - historyStartTime = historyStartTime + datetime.timedelta(days=26) - - # fetches historical Day data - historyStartTime = stopTime - datetime.timedelta(days=historyDays) - historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) - dayLoop = historyDays / 25 - historyEndTime = historyStartTime + datetime.timedelta(days=25) - historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) - dayLoop = historyDays / 25 - while dayLoop > 0 and historyStartTime < stopTime : - historyEndTime = min(stopTime, historyStartTime + datetime.timedelta(days=25)) - historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) + #Fetches date data usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.DAY.value, unit=Unit.KWH.value) index = 0 for kwhUsage in usage: @@ -209,7 +194,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd timestamp = usage_start_time + datetime.timedelta(days=index) watts = kwhUsage * 1000 usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Day")) - index += 1 + index += 1 dayLoop = dayLoop -1 historyStartTime = historyStartTime + datetime.timedelta(days=26) @@ -257,7 +242,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd start = "1970-01-01T00:00:00Z" stop = startupTime.isoformat(timespec='seconds') delete_api.delete(start, stop, '_measurement="energy_usage"', bucket=bucket, org=org) - setconfig('influxDb','reset','false') + setconfig('influxDb','reset',False) else: info('Using InfluxDB version 1') @@ -273,10 +258,11 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd influx.create_database(config['influxDb']['database']) + print('RESET = ',config['influxDb']['reset']) if config['influxDb']['reset']: info('Resetting database') influx.delete_series(measurement='energy_usage') - setconfig('influxDb','reset','false') + setconfig('influxDb','reset',False) historyMinute = min(config['influxDb'].get('historyDays', 0), 7) historyDays = min(config['influxDb'].get('historyDays', 0), 720) @@ -340,11 +326,6 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd histLoop = "day-hour" info('Loading historical data - Days/Hours/Months: {} day(s) ago'.format(historyDays)) for gid, device in usages.items(): - historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(historyDays)) - historyEndTime = stopTime - historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) - historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) - print('time before call ',stopTime,historyStartTime ,historyEndTime ) extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) if not running: break From d5c1277d8d383267b67568a26e00bb3186df2bdf Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Sun, 26 Mar 2023 10:47:22 -0700 Subject: [PATCH 08/29] Create README.md --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 217a303..990270b 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,17 @@ Goal to add additional features to either be reveiwed/approved and merged back i Additional Features - [X] Reset config json file values to allow multiple runs without reseting database. - [X] Database reset -- [X} History Days -- [ ] Add additional scales to the pull allowing to get years of history since minutes/second have a short life at emporia. +- [X] History Days +- [X] Add additional scales to the pull allowing to get years of history since minutes/second have a short life at emporia.
This also allows faster dashboards for creating itmes like daily or monthly graphs. -- [ ] Hours -- [ ] Days -- [ ] Months +- [X] Hours history and daily pull +- [X] Days history and daily pull - *Note while adding additional scales will keep seconds = true and minutes = false to not impact existing reporting. - But add additonal tags for sec/min/hour/day/month - +- [ ] Add restart funactionality to : +
* json parm to use below logic +
* read last timestamp in influxdb +
* on startup do history loop for gap in time (last timestamp -> now) # License As this is a fork, will hour current license and all whishes of jertel's project. From 5892138f40173d4e113b4600af331dfa00ca7f08 Mon Sep 17 00:00:00 2001 From: Garrett Gauthier Date: Tue, 4 Apr 2023 12:34:23 -0700 Subject: [PATCH 09/29] Cleanup --- src/vuegraf/getdate.py | 135 +++++++++++++++++++++++++++++++++++++++++ src/vuegraf/vuegraf.py | 16 +++-- 2 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 src/vuegraf/getdate.py diff --git a/src/vuegraf/getdate.py b/src/vuegraf/getdate.py new file mode 100644 index 0000000..5848446 --- /dev/null +++ b/src/vuegraf/getdate.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +import datetime +import json +import signal +import sys +import time +import traceback +from threading import Event + +# InfluxDB v1 +import influxdb + + + +# flush=True helps when running in a container without a tty attached +# (alternatively, "python -u" or PYTHONUNBUFFERED will help here) +def log(level, msg): + now = datetime.datetime.utcnow() + print('{} | {} | {}'.format(now, level.ljust(5), msg), flush=True) + +def info(msg): + log("INFO", msg) + +def error(msg): + log("ERROR", msg) + +def handleExit(signum, frame): + global running + error('Caught exit signal') + running = False + pauseEvent.set() + +def getConfigValue(key, defaultValue): + if key in config: + return config[key] + return defaultValue + +# Reset config file if history or DB reset set +# Allows sequential runs without lossing data +def setconfig(configname, configkey, configvalue) : + with open(configFilename, 'r') as newconfigFile: + newconfig = json.load(newconfigFile) + newconfig[configname][configkey] = configvalue + with open(configFilename, 'w') as configout: + json.dump(newconfig, configout, indent=4) + return() + + + +startupTime = datetime.datetime.utcnow() +try: + if len(sys.argv) != 2: + print('Usage: python {} '.format(sys.argv[0])) + sys.exit(1) + + configFilename = sys.argv[1] + config = {} + with open(configFilename) as configFile: + config = json.load(configFile) + + influxVersion = 1 + if 'version' in config['influxDb']: + influxVersion = config['influxDb']['version'] + + bucket = '' + write_api = None + query_api = None + sslVerify = True + + if 'ssl_verify' in config['influxDb']: + sslVerify = config['influxDb']['ssl_verify'] + + if influxVersion == 2: + info('Using InfluxDB version 2') + bucket = config['influxDb']['bucket'] + org = config['influxDb']['org'] + token = config['influxDb']['token'] + url= config['influxDb']['url'] + influx2 = influxdb_client.InfluxDBClient( + url=url, + token=token, + org=org, + verify_ssl=sslVerify + ) + write_api = influx2.write_api(write_options=influxdb_client.client.write_api.SYNCHRONOUS) + query_api = influx2.query_api() + + + else: + info('Using InfluxDB version 1') + + sslEnable = False + if 'ssl_enable' in config['influxDb']: + sslEnable = config['influxDb']['ssl_enable'] + + # Only authenticate to ingress if 'user' entry was provided in config + if 'user' in config['influxDb']: + influx = influxdb.InfluxDBClient(host=config['influxDb']['host'], port=config['influxDb']['port'], username=config['influxDb']['user'], password=config['influxDb']['pass'], database=config['influxDb']['database'], ssl=sslEnable, verify_ssl=sslVerify) + else: + influx = influxdb.InfluxDBClient(host=config['influxDb']['host'], port=config['influxDb']['port'], database=config['influxDb']['database'], ssl=sslEnable, verify_ssl=sslVerify) + + influx.create_database(config['influxDb']['database']) + + print('RESET = ',config['influxDb']['reset']) + if config['influxDb']['reset']: + info('Resetting database') + influx.delete_series(measurement='energy_usage') + setconfig('influxDb','reset',False) + + historyMinute = min(config['influxDb'].get('historyDays', 0), 7) + historyDays = min(config['influxDb'].get('historyDays', 0), 720) + history = historyDays > 0 + + query1 = 'select last(usage) from "energy_usage" where "detailed"=\'Hour\'' + query2 = 'select last(usage) from "energy_usage" where "detailed"=\'Day\'' + query3 = 'select last(usage) from "energy_usage" where "detailed"=\'False\'' + print(query1) + last = influx.query(query1) + print(last) + last = influx.query(query2) + print(last) + last = influx.query(query3) + print(last) + + for record in + + + + +except: + error('Fatal error: {}'.format(sys.exc_info())) + traceback.print_exc() + + diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index f229c5d..d5dbc41 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -168,7 +168,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd index += 1 # fetches historical hour/day or just previous day if histLoop == "day-hour" or collectSummaries: - historyStartTime = stopTime - datetime.timedelta(days=historyDays) + historyStartTime = stopTime - datetime.timedelta(days=max(historyDays,1)) historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) #only used on history runs, otherwise is zero and will do next loop once. dayLoop = historyDays / 25 @@ -178,12 +178,14 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.HOUR.value, unit=Unit.KWH.value) index = 0 + info('CollectSummaries; start="{}"; stop="{}"'.format(historyStartTime,historyEndTime )) for kwhUsage in usage: if kwhUsage is None: continue timestamp = usage_start_time + datetime.timedelta(hours=index) watts = kwhUsage * 1000 usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Hour")) + print(datetime.datetime.now, ) index += 1 #Fetches date data usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.DAY.value, unit=Unit.KWH.value) @@ -197,7 +199,6 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd index += 1 dayLoop = dayLoop -1 historyStartTime = historyStartTime + datetime.timedelta(days=26) - startupTime = datetime.datetime.utcnow() try: if len(sys.argv) != 2: @@ -280,9 +281,10 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd info(' History Days {}'.format(historyDays)) lagSecs=getConfigValue("lagSecs", 5) detailedStartTime = startupTime - #Only pull Hourly data and Full day after 0200 + #Only pull previous days Hourly and full day data and Full day after on next details loop after 00:05 summariesDate = datetime.datetime.now() + datetime.timedelta(days=1) - summariesDate = summariesDate.replace(hour=2) + summariesDate = summariesDate.replace(hour=0, minute=5, second=00, microsecond=00) + info('Set collectSummaries; {}'.format(summariesDate)) while running: now = datetime.datetime.utcnow() stopTime = now - datetime.timedelta(seconds=lagSecs) @@ -336,7 +338,8 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if not running: break - info('Submitting datapoints to database; account="{}"; points={}'.format(account['name'], len(usageDataPoints))) + info('Submitting datapoints to database; account="{}"; points={}; Details="{}"; Summaries="{}"'.format(account['name'], + len(usageDataPoints),collectDetails, collectSummaries )) if influxVersion == 2: write_api.write(bucket=bucket, record=usageDataPoints) else: @@ -350,7 +353,8 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd detailedStartTime = stopTime + datetime.timedelta(seconds=1) if collectSummaries: summariesDate = datetime.datetime.now() + datetime.timedelta(days=1) - summariesDate = summariesDate.replace(hour=2) + summariesDate = summariesDate.replace(hour=0, minute=5, second=00, microsecond=00) + info('Reset collectSummaries; {}'.format(summariesDate)) pauseEvent.wait(intervalSecs) info('Finished') From 96f17846971b8d0ac222660bdbb4b122008e244f Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Sat, 22 Apr 2023 18:07:08 -0700 Subject: [PATCH 10/29] Command Line Parsing Moved history days and reset database to only command line. JSON config file for normal operating parms only. Examples python src/vugraf/vuegraf.py --load-history-days 7 python src/vugraf/vuegraf.py --load-history-days 7 --reset-database Added help -h or --help Changed except SystemExit to not display trace information is exit is called by program, only if program forced exit. Needed for the exit calls from --help or if json confg file is not provided. --- src/vuegraf/vuegraf.py | 180 +++++++++++++++++++---------------------- 1 file changed, 84 insertions(+), 96 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index d5dbc41..ed74e1c 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 +__author__ = 'https://github.com/jertel' +__license__ = 'GPL' +__contributors__ = 'https://github.com/jertel/vuegraf/graphs/contributors' +__version__ = '1.6.0' +__versiondate__ = '04/18/2023' +__maintainer__ = 'https://github.com/jertel' +__github__ = 'https://github.com/jertel/vuegraf' +__status__ = 'Production' + import datetime import json import signal @@ -7,6 +16,7 @@ import time import traceback from threading import Event +import argparse # InfluxDB v1 import influxdb @@ -30,7 +40,7 @@ def error(msg): log("ERROR", msg) def handleExit(signum, frame): - global running + global runninga error('Caught exit signal') running = False pauseEvent.set() @@ -40,16 +50,6 @@ def getConfigValue(key, defaultValue): return config[key] return defaultValue -# Reset config file if history or DB reset set -# Allows sequential runs without lossing data -def setconfig(configname, configkey, configvalue) : - with open(configFilename, 'r') as newconfigFile: - newconfig = json.load(newconfigFile) - newconfig[configname][configkey] = configvalue - with open(configFilename, 'w') as configout: - json.dump(newconfig, configout, indent=4) - return() - def populateDevices(account): deviceIdMap = {} account['deviceIdMap'] = deviceIdMap @@ -120,7 +120,7 @@ def createDataPoint(account, chanName, watts, timestamp, detailed): } return dataPoint -def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEndTime=None, histLoop=None): +def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEndTime=None): excludedDetailChannelNumbers = ['Balance', 'TotalUsage'] minutesInAnHour = 60 secondsInAMinute = 60 @@ -143,7 +143,6 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd continue if collectDetails: - #Seconds usage, usage_start_time = account['vue'].get_chart_usage(chan, detailedStartTime, stopTime, scale=Scale.SECOND.value, unit=Unit.KWH.value) index = 0 for kwhUsage in usage: @@ -155,59 +154,71 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd index += 1 # fetches historical minute data - if histLoop == "minute" : + if historyStartTime is not None and historyEndTime is not None: usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.MINUTE.value, unit=Unit.KWH.value) index = 0 + + + + + + + for kwhUsage in usage: if kwhUsage is None: continue timestamp = historyStartTime + datetime.timedelta(minutes=index) watts = float(minutesInAnHour * wattsInAKw) * kwhUsage usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, False)) - index += 1 - # fetches historical hour/day or just previous day - if histLoop == "day-hour" or collectSummaries: - historyStartTime = stopTime - datetime.timedelta(days=max(historyDays,1)) - historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) - #only used on history runs, otherwise is zero and will do next loop once. - dayLoop = historyDays / 25 - while dayLoop >= 0 and historyStartTime < stopTime : - #Fetches hour data - historyEndTime = min(stopTime, historyStartTime + datetime.timedelta(days=25)) - historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) - usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.HOUR.value, unit=Unit.KWH.value) - index = 0 - info('CollectSummaries; start="{}"; stop="{}"'.format(historyStartTime,historyEndTime )) - for kwhUsage in usage: - if kwhUsage is None: - continue - timestamp = usage_start_time + datetime.timedelta(hours=index) - watts = kwhUsage * 1000 - usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Hour")) - print(datetime.datetime.now, ) - index += 1 - #Fetches date data - usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.DAY.value, unit=Unit.KWH.value) - index = 0 - for kwhUsage in usage: - if kwhUsage is None: - continue - timestamp = usage_start_time + datetime.timedelta(days=index) - watts = kwhUsage * 1000 - usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Day")) - index += 1 - dayLoop = dayLoop -1 - historyStartTime = historyStartTime + datetime.timedelta(days=26) + startupTime = datetime.datetime.utcnow() try: - if len(sys.argv) != 2: - print('Usage: python {} '.format(sys.argv[0])) - sys.exit(1) + #argparse includes default -h / --help as command line input + parser = argparse.ArgumentParser( + prog='vuegraf.py', + description='Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2)', + epilog='For more information go to : ' + __github__ + ) + parser.add_argument( + "configFilename", + help='json config file', + type=str + ) + parser.add_argument( + '--version', + help='display version number', + action='store_true') + parser.add_argument( + '-v', + '--verbose', + help='verbose output - summaries', + action='store_true') + parser.add_argument( + '-q', + '--quiet', + help='do not print anything but errors', + action='store_true') + parser.add_argument( + '--historydays', + help='Starts executin by pulling history of Hours and Day data for specified number of days. example: --load-history-day 60', + type=int, + default=0 + ) + parser.add_argument( + '--resetdatabase', + action='store_true', + default=False, + help='Drop database and create a new one') + + args = parser.parse_args() + + if args.version: + print('vuegraf.py - version: ', __version__) + sys.exit(0) - configFilename = sys.argv[1] config = {} - with open(configFilename) as configFile: + with open(args.configFilename) as configFile: config = json.load(configFile) influxVersion = 1 @@ -237,13 +248,12 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd write_api = influx2.write_api(write_options=influxdb_client.client.write_api.SYNCHRONOUS) query_api = influx2.query_api() - if config['influxDb']['reset']: + with open(args.configFilename) as configFile: info('Resetting database') delete_api = influx2.delete_api() start = "1970-01-01T00:00:00Z" stop = startupTime.isoformat(timespec='seconds') delete_api.delete(start, stop, '_measurement="energy_usage"', bucket=bucket, org=org) - setconfig('influxDb','reset',False) else: info('Using InfluxDB version 1') @@ -259,16 +269,12 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd influx.create_database(config['influxDb']['database']) - print('RESET = ',config['influxDb']['reset']) - if config['influxDb']['reset']: + if args.resetdatabase: info('Resetting database') influx.delete_series(measurement='energy_usage') - setconfig('influxDb','reset',False) - historyMinute = min(config['influxDb'].get('historyDays', 0), 7) - historyDays = min(config['influxDb'].get('historyDays', 0), 720) + historyDays = min(args.historydays, 720) history = historyDays > 0 - running = True signal.signal(signal.SIGINT, handleExit) signal.signal(signal.SIGHUP, handleExit) @@ -276,20 +282,15 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd intervalSecs=getConfigValue("updateIntervalSecs", 60) detailedIntervalSecs=getConfigValue("detailedIntervalSecs", 3600) detailedDataEnabled=getConfigValue("detailedDataEnabled", False) - summariesDataEnabled=getConfigValue('summariesDataEnabled', False) - info('Settings -> updateIntervalSecs: {}, detailedEnabled: {}, detailedIntervalSecs: {}, summariesEnabled {}'.format(intervalSecs, detailedDataEnabled, detailedIntervalSecs, summariesDataEnabled)) - info(' History Days {}'.format(historyDays)) + info('Settings -> updateIntervalSecs: {}, detailedEnabled: {}, detailedIntervalSecs: {}'.format(intervalSecs, detailedDataEnabled, detailedIntervalSecs)) lagSecs=getConfigValue("lagSecs", 5) detailedStartTime = startupTime - #Only pull previous days Hourly and full day data and Full day after on next details loop after 00:05 - summariesDate = datetime.datetime.now() + datetime.timedelta(days=1) - summariesDate = summariesDate.replace(hour=0, minute=5, second=00, microsecond=00) - info('Set collectSummaries; {}'.format(summariesDate)) + while running: now = datetime.datetime.utcnow() stopTime = now - datetime.timedelta(seconds=lagSecs) collectDetails = detailedDataEnabled and detailedIntervalSecs > 0 and (stopTime - detailedStartTime).total_seconds() >= detailedIntervalSecs - collectSummaries = summariesDataEnabled and datetime.datetime.now() >= summariesDate + for account in config["accounts"]: if 'vue' not in account: account['vue'] = PyEmVue() @@ -306,44 +307,32 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd extractDataPoints(device, usageDataPoints) if history: - for day in range(historyMinute): - histLoop = "minute" - info('Loading historical data - Minutes: {} day(s) ago'.format(day+1)) + for day in range(historyDays): + info('Loading historical data: {} day(s) ago'.format(day+1)) #Extract second 12h of day historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)-43200) historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day))) for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) + extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime) pauseEvent.wait(5) #Extract first 12h of day historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)) historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day+1))-43200) for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) - if not running: - break - pauseEvent.wait(5) - - if summariesDataEnabled: - histLoop = "day-hour" - info('Loading historical data - Days/Hours/Months: {} day(s) ago'.format(historyDays)) - for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) + extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime) if not running: break pauseEvent.wait(5) history = False - setconfig('influxDb','historyDays', 0) if not running: break - info('Submitting datapoints to database; account="{}"; points={}; Details="{}"; Summaries="{}"'.format(account['name'], - len(usageDataPoints),collectDetails, collectSummaries )) + info('Submitting datapoints to database; account="{}"; points={}'.format(account['name'], len(usageDataPoints))) if influxVersion == 2: write_api.write(bucket=bucket, record=usageDataPoints) else: - influx.write_points(usageDataPoints,batch_size=5000) + influx.write_points(usageDataPoints) except: error('Failed to record new usage data: {}'.format(sys.exc_info())) @@ -351,15 +340,14 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if collectDetails: detailedStartTime = stopTime + datetime.timedelta(seconds=1) - if collectSummaries: - summariesDate = datetime.datetime.now() + datetime.timedelta(days=1) - summariesDate = summariesDate.replace(hour=0, minute=5, second=00, microsecond=00) - info('Reset collectSummaries; {}'.format(summariesDate)) pauseEvent.wait(intervalSecs) info('Finished') -except: - error('Fatal error: {}'.format(sys.exc_info())) - traceback.print_exc() - - +except SystemExit as e: + #Is sys.exit was 2, then normal syntax exit from help or bad command line, no error message + if e.code == 0 or e.code == 2: + quit(0) + else: + error('Fatal error: {}'.format(sys.exc_info())) + traceback.print_exc() + From 70a820d2a2e0b1478d465a11539c478ddd03fc8c Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Sun, 23 Apr 2023 13:05:47 -0700 Subject: [PATCH 11/29] History pull of hours and days - 720 days max For history, due to the limit of 7 days for minutes, changed to pull hours and days. This allows approximately 2 years of history. Allows for very quick daily and monthly charts as less compute/data is needed. Minutes for history are removed as hourly data for weeks and limit of minute data to only 7 days, little value influx v1 - too many rows for influx.write_points so added batch size and fixed to 5000. --verbose / -v command line provides history start/stop dates in output --- src/vuegraf/vuegraf.py | 57 ++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index ed74e1c..17abc74 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -40,7 +40,7 @@ def error(msg): log("ERROR", msg) def handleExit(signum, frame): - global runninga + global running error('Caught exit signal') running = False pauseEvent.set() @@ -143,6 +143,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd continue if collectDetails: + #Seconds usage, usage_start_time = account['vue'].get_chart_usage(chan, detailedStartTime, stopTime, scale=Scale.SECOND.value, unit=Unit.KWH.value) index = 0 for kwhUsage in usage: @@ -151,25 +152,32 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd timestamp = detailedStartTime + datetime.timedelta(seconds=index) watts = float(secondsInAMinute * minutesInAnHour * wattsInAKw) * kwhUsage usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, True)) + if args.verbose: + info('Get Detailes (Seconds); start="{}"; stop="{}"'.format(detailedStartTime,stopTime )) index += 1 # fetches historical minute data if historyStartTime is not None and historyEndTime is not None: - usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.MINUTE.value, unit=Unit.KWH.value) - index = 0 - - - - - - - + #Hours + usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.HOUR.value, unit=Unit.KWH.value) + index = 0 + for kwhUsage in usage: + if kwhUsage is None: + continue + timestamp = historyStartTime + datetime.timedelta(hours=index) + watts = kwhUsage * 1000 + usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Hour")) + index += 1 + #Days + usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.DAY.value, unit=Unit.KWH.value) + index = 0 for kwhUsage in usage: if kwhUsage is None: continue - timestamp = historyStartTime + datetime.timedelta(minutes=index) - watts = float(minutesInAnHour * wattsInAKw) * kwhUsage - usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, False)) + timestamp = historyStartTime + datetime.timedelta(days=index) + timestamp = timestamp.replace(hour=23, minute=59, second=59,microsecond=999999) + watts = kwhUsage * 1000 + usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Day")) index += 1 startupTime = datetime.datetime.utcnow() @@ -307,21 +315,20 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd extractDataPoints(device, usageDataPoints) if history: - for day in range(historyDays): - info('Loading historical data: {} day(s) ago'.format(day+1)) - #Extract second 12h of day - historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)-43200) - historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day))) - for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime) - pauseEvent.wait(5) - #Extract first 12h of day - historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)) - historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day+1))-43200) + info('Loading historical data: {} day(s) ago'.format(historyDays)) + historyStartTime = stopTime - datetime.timedelta(historyDays) + historyStartTime = historyStartTime.replace(hour=00, minute=00, second=00, microsecond=000000) + while historyStartTime <= stopTime: + historyEndTime = min(historyStartTime + datetime.timedelta(20), stopTime) + historyEndTime = historyEndTime.replace(hour=23, minute=59, second=59,microsecond=999999) + if args.verbose: + info(' {} - {}'.format(historyStartTime,historyEndTime)) for gid, device in usages.items(): extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime) if not running: break + historyStartTime = historyEndTime + datetime.timedelta(1) + historyStartTime = historyStartTime.replace(hour=00, minute=00, second=00, microsecond=000000) pauseEvent.wait(5) history = False @@ -332,7 +339,7 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd if influxVersion == 2: write_api.write(bucket=bucket, record=usageDataPoints) else: - influx.write_points(usageDataPoints) + influx.write_points(usageDataPoints,batch_size=5000) except: error('Failed to record new usage data: {}'.format(sys.exc_info())) From 38803af9de211d82642da8e85f86e9a19fe426a9 Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Mon, 15 May 2023 11:30:44 -0700 Subject: [PATCH 12/29] Changes to be committed: new file: CHANGELOG.md Created change log for current and future version modified: README.md Updated with required instructions and user guide as listed in the changes modified: setup.py Added argparse libary to be installed modified: src/requirements.txt Added argparse libary to be installed modified: vuegraf.json.sample Moved history and reset from sample json as they are now command line options modified: src/vuegraf/vuegraf.py This text of changes is also in CHANGELOG.MD - Replaced Minute with Hour as normal interval since history is limited to 7 days from Emporia on minute data - argparse libary was added, must run pip install -r requirements again in the src directory (or pip3 based on install) - Hour / Day data pulled. Allows for history of 2 years to be pulled. Assists in clean numbers/graphs to see daily monthly usage to compare against utilities reports. - Hour data runs with the get details time, default is 1 hour (3600 seconds). Based on when the program is started, you may be almost 2 hours behind for get hour. - Changed several one time parameters from the json config file to command line entries (History load days, reset database). Example Command Line usage usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename positional arguments: configFilename json config file options: -h, --help show this help message and exit --version display version number -v, --verbose verbose output - summaries -q, --quiet do not print anything but errors --historydays HISTORYDAYS Starts executin by pulling history of Hours and Day data for specified number of days. example: --load-history-day 60 --resetdatabase Drop database and create a new one - Started Changelog for this and future releases - Added project metadata to main program - vuegraf.py, values can be updated through github automations - Added command line parameters with help syntaxt for all values - arge parse lib. - ran pylint and fixed Quote delimiter consitency to all ' Whitespaces Extra lines --- .gitignore | 1 + .vscode/launch.json | 19 +++ CHANGELOG.md | 53 +++++++ README.md | 312 +++++++++++++++++++++++++++++++++++--- setup.py | 3 +- src/requirements.txt | 1 + src/vuegraf/.gitignore | 6 + src/vuegraf/getdate.py | 135 ----------------- src/vuegraf/vuegraf.py | 331 ++++++++++++++++++++++------------------- vuegraf.json.sample | 4 +- 10 files changed, 553 insertions(+), 312 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 CHANGELOG.md create mode 100644 src/vuegraf/.gitignore delete mode 100644 src/vuegraf/getdate.py diff --git a/.gitignore b/.gitignore index ab35261..89c263c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ int docker-compose.yaml dist/ vuegraf.egg-info/ +vugraftobe.py diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..ab659d4 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,19 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "args" : ["/mnt/c/Users/garre/OneDrive/Documents/GitHub/vuegraf/vuegraf.json", "--resetdatabase", "--historydays", "90", "--verbose"], + //"args" : ["/mnt/c/Users/garre/OneDrive/Documents/GitHub/vuegraf/vuegraf.json", "--resetdatabase", "--verbose"], + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..929eb11 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,53 @@ +# 2.TBD.TBD- Future + +## Breaking changes +- TBD + +## New features +- TBD + +## Other changes + + +# 1.6.0 - Released + +## Breaking changes +- Replaced Minute with Hour as normal interval since history is limited to 7 days from Emporia on minute data +- argparse libary was added, must run pip install -r requirements again in the src directory (or pip3 based on install) + +## New features +- Hour / Day data pulled. Allows for history of 2 years to be pulled. Assists in clean numbers/graphs to see daily monthly usage to compare against utilities reports. +- Hour data runs with the get details time, default is 1 hour (3600 seconds). Based on when the program is started, +you may be almost 2 hours behind for get hour. +- Changed several one time parameters from the json config file to command line entries (History load days, reset database). + Example Command Line usage +usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename + +positional arguments: + configFilename json config file + +options: + -h, --help show this help message and exit + --version display version number + -v, --verbose verbose output - summaries + -q, --quiet do not print anything but errors + --historydays HISTORYDAYS + Starts executin by pulling history of Hours and Day data for specified number of days. + example: --load-history-day 60 + --resetdatabase Drop database and create a new one + +## Other changes +- Started Changelog for this and future releases +- Added project metadata to main program - vuegraf.py, values can be updated through github automations +- Added command line paring with help syntaxt for all values - arge parse lib. +- Updated requirements.txt and setup.py with argparse>= 1.4.0 +- Updated vuegraf.json.sample as history and reset database was moved to command line +- Updated Readme.md with above changes +- ran pylint and fixed + Quote delimiter consitency to all ' + Whitespaces + Extra lines + + + +# 1.5.0 diff --git a/README.md b/README.md index 990270b..14665df 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,293 @@ +![Vuegraf Logo](https://github.com/jertel/vuegraf/blob/master/vuegraf.png?raw=true "Vuegraf Logo") -This is a fork of vuegraf from jertel - https://github.com/jertel/vuegraf - -Goal to add additional features to either be reveiwed/approved and merged back into jertel's code or remain a seperate product. - -Additional Features -- [X] Reset config json file values to allow multiple runs without reseting database. -- [X] Database reset -- [X] History Days -- [X] Add additional scales to the pull allowing to get years of history since minutes/second have a short life at emporia. -
This also allows faster dashboards for creating itmes like daily or monthly graphs. -- [X] Hours history and daily pull -- [X] Days history and daily pull -- *Note while adding additional scales will keep seconds = true and minutes = false to not impact existing reporting. -- But add additonal tags for sec/min/hour/day/month -- [ ] Add restart funactionality to : -
* json parm to use below logic -
* read last timestamp in influxdb -
* on startup do history loop for gap in time (last timestamp -> now) - -# License -As this is a fork, will hour current license and all whishes of jertel's project. - This is a work in progress with plans to work with Jertel in the future to merge back, so all - docuemntaion and usage is back on the main project: (https://github.com/jertel/vuegra) +# Overview + +The [Emporia Vue](https://emporiaenergy.com "Emporia's Homepage") energy monitoring kit allows homeowners to monitor their electrical usage. It monitors the main feed consumption and up to 8 (or 16 in the newer version) individual branch circuits, and feeds that data back to the Emporia API server. + +This project, Vuegraf, fetches those metrics from the Emporia Vue API host and stores the metrics into your own InfluxDB. After installation you will be able to: +* View your energy usage across all circuits on a single graph +* Create alerts to notify when certain energy usage thresholds are exceeded + +This project is not affiliated with _emporia energy_ company. + +# Dependencies + +* [Emporia Vue](https://emporiaenergy.com "Emporia Energy") Account - Username and password for the Emporia Vue system are required. +* [Python 3](https://python.org "Python") - With Pip. +* [InfluxDB 2](https://influxdata.com "InfluxDB") - Host, port, org, bucket, and token are all required. + +# Influx + +## Setup + +If you do not yet have a running InfluxDB 2 instance, you will need to set one up. You can do this very quickly by launching an InfluxDB 2 Docker container as follows: + +``` +mkdir -p /home/myuser/influxdb2 +docker run -v /home/myuser/influxdb2:/var/lib/influxdb2 -p 8086:8086 -e INFLUXD_SESSION_LENGTH=432000 --name influxdb influxdb +``` + +Substitute an appropriate host path for the `/home/myuser/influxdb2` location above. Once running, access the web UI at `http://localhost:8086`. It will prompt you for a username, password, organization name, and bucket name. The rest of this document assumes you have entered the word `vuegraf` for all of these inputs. + +Note that the default session timeout for Influx is only 60 minutes, so this command increases the login session to 300 days. + +Once logged in, go to the _Load Data -> API Tokens_ screen and generate a new All Access token with the description of _vuegraf_. Copy the generated token for use in the rest of this document, specifically when referenced as ``. + +## Dashboard + +By default, a new InfluxDB instance will not have any dashboards loaded. You will need to import the included Influx JSON template, or create your own dashboard in order to visualize your energy usage. + +The included template file named `influx_dashboard.json` includes the provided dashboard and accompanying variables to reproduce the visualizations shown below. This dashboard assumes your main device name contains the word `Panel`, such as `House Panel`, or `Right Panel`. If it does not, the Flux queries will need to be adjusted manually to look for your device's name. + +![Influx Dashboard Screenshot](https://github.com/jertel/vuegraf/blob/master/screenshots/influx_dashboard.png?raw=true "Influx Dashboard") + +You will need to apply this template file to your running InfluxDB instance. Copy the `influx_dashboard.json` file into your host's influxdb2 path. If you followed the Setup instructions above, the path would be `/home/myuser/influxdb2`. The below command can be used to perform this step. This command assumes you are running Influx in a container named `influxdb`. + +``` +docker exec influxdb influx apply -f /var/lib/influxdb2/influx_dashboard.json --org vuegraf --force yes -t +``` + +Replace the `` with the All Access Token you generated in the Influx _Load Data -> API Tokens_ screen. + +You're now ready to proceed with the Vuegraf configuration and startup. + +# Configuration + +The configuration allows for the definition of multiple Emporia Vue accounts. This will only be useful to users that need to pull metrics from multiple accounts. This is not needed if you have multiple Vue devices in a single account. Vuegraf will find multiple devices on its own within each account. + +The email address and password must match the credentials used when creating the Emporia Vue account in their mobile app. + +Important: Ensure that sufficient protection is in place on this configuration file, since it contains the plain-text login credentials into the Emporia Vue account. + +A [sample configuration file](https://github.com/jertel/vuegraf/blob/master/vuegraf.json.sample "Sample Vuegraf Configuration File") is provided in this repository, and details are described below. + +## Minimal Configuration +The minimum configuration required to start Vuegraf is shown below. + +```json +{ + "influxDb": { + "version": 2, + "url": "http://my.influxdb.hostname:8086", + "org": "vuegraf", + "bucket": "vuegraf", + "token": "", + }, + "accounts": [ + { + "name": "Primary Residence", + "email": "my@email.address", + "password": "my-emporia-password" + } + ] +} +``` + +## Advanced Configuration + +### Ingesting Historical Data + +If desired, it is possible to have Vuegraf import historical data. To do so, run vuegraf.py with the option historydays parameter with a value between 1 and 720. When ran, it will start by collection all hourly data points up to the specified parameter or max history stored at emporia. It will also collect 1 Day data for each day storing it 23:59:59 for each day. It collects the time using your local server time, but store it in influx on UTC. + +IMPORTANT - If you restart Vuegraf with --historydays still on the command line (or in the dockerfile) it will _again_ import history data. This will likely cause confusion with your data since you will now have duplicate/overlapping data. For best results, only enable historydays single run, and then remove from your command line. +Example: +python3 path/to/vuegraf.py vuegraf.json --historydays 365 + +### Channel Names + +To provide more user-friendly names of each Vue device and branch circuit, the following device configuration can be added to the configuration file, within the account block. List each device and circuit in the order that you added them to the Vue mobile app. The channel names do not need to match the names specified in the Vue mobile app but the device names must match. The below example shows two 8-channel Vue devices for a home with two breaker panels. + +Be aware that the included dashboard assumes your device name contains the word "Panel". For best results, consider renaming your Vue device to contain that word, otherwise you will need to manually adjust the included dashboards' queries. + +```json + "devices": [ + { + "name": "Right Panel", + "channels": [ + "Air Conditioner", + "Furnace", + "Coffee Maker", + "Oven", + "Pool Vacuum", + "Pool Filter", + "Refrigerator", + "Office" + ] + }, + { + "name": "Left Panel", + "channels": [ + "Dryer", + "Washer", + "Dishwasher", + "Water Heater", + "Landscape Features", + "Septic Pump", + "Deep Freeze", + "Sprinkler Pump" + ] + } + ] +``` + +# Running +Vuegraf can be run either as a container (recommended), or as a host process. + +## Container (recommended) + +A Docker container is provided at [hub.docker.com](https://hub.docker.com/r/jertel/vuegraf). Refer to the command below to launch Vuegraf as a container. This assumes you have created a folder called `/home/myuser/vuegraf` and placed the vuegraf.json file inside of it. + +```sh +docker run --name vuegraf -d -v /home/myuser/vuegraf:/opt/vuegraf/conf jertel/vuegraf +``` + +## Host Process + +Ensure Python 3 and Pip are both installed. Install the required dependencies: + +```sh +pip install -r src/requirements.txt +``` +or, on some Linux installations: + +```sh +pip3 install -r src/requirements.txt +``` + + +Then run the program via Python, specifying the JSON configuration file path as the only argument: + +```sh +python src/vuegraf/vuegraf.py vuegraf.json +``` +or, on some Linux installations: +```sh +python3 src/vuegraf/vuegraf.py vuegraf.json +``` + +usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename + +Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2) + +positional arguments: + configFilename json config file + +options: + -h, --help show this help message and exit + --version display version number + -v, --verbose verbose output - summaries + -q, --quiet do not print anything but errors + --historydays HISTORYDAYS + Starts executin by pulling history of Hours and Day data for specified number of days. + example: --load-history-day 60 + --resetdatabase Drop database and create a new one + + + + +## Alerts + +The included dashboard template contains two alerts which will trigger when either a power outage occurs, or a loss of Vuegraf data. There are various reasons why alerts can be helpful. See the below screenshots which help illustrate how a fully functioning alert and notification rule might look. Note that the included alerts do not send out notifications. To enable outbound notifactions, such as to Slack, you can create a Notification Endpoint and Notification Rule. + +This alert was edited via the text (Flux) interface since the alert edit UI does not yet accommodate advanced alerting inputs. + +Side note: The logo at the top of this documentation satisfies Slack's icon requirements. Consider using it to help quickly distinguish between other alerts. + +![Influx Alert Edit](https://github.com/jertel/vuegraf/blob/master/screenshots/alert_edit.png?raw=true "Influx Alert") + +This notification rule provides an example of how you can have several alerts change the status to crit, but only a single notification rule is required to transmit notifications to external endpoints (such as email or Slack). + +![Influx Notification Rule](https://github.com/jertel/vuegraf/blob/master/screenshots/notification_rule.png?raw=true "Influx Notification Rule") + + +# Additional Topics + +## Per-second Data Details + +By default, Vuegraf will poll every minute to collect the energy usage value over the past 60 seconds. This results in a single value being capture per minute per channel, or 60 values per hour per channel. If you also would like to see per-second values, you can enable the detailed collection, which is polled once per hour, and backfilled over the previous 3600 seconds. This API call is very expensive on the Emporia servers, so it should not be polled more frequently than once per hour. To enable this detailed data, add (or update) the top-level `detailedDataEnabled` configuration value with a value of `true`. The details is also what pulls the Hourly datapoint. + +``` +detailedDataEnabled: true +``` + +For every datapoint a tag is stored in influx for the type of measurement + +- `detailed = True` represents backfilled per-second data that is optionally queried from Emporia once every hour. +- `detailed = False` represents the per-minute average data that is collected every minute. +- `detailed = Hour` represents the data summarized in hours +- `detailed - Day` represents a single data point to summarize the entire day + +When building graphs that show a sum of the energy usage, be sure to only include the correct detail tag, otherwise your summed values will be higher than expected. Detailed data will take more time for the graphs to query due to the extra data involved. If you want to have a chart that shows daily data over a long period or even a full year, use the `detailed = Day` tag. +If you are running this on a small server, you might want to look at setting a RETENTION POLICY on influx to remove minute or second data over time. i.e. keep only 30 days fo second data. + +## Vue Utility Connect Energy Monitor + +As reported in [discussion #104](https://github.com/jertel/vuegraf/discussions/104), the Utility Connect device is supported without any custom changes. + +## Smart Plugs + +To include an Emporia smart plug in the configuration, add each plug as it's own device, without channels. Again, the name of the Smart Plug device must exactly match the name you gave the device in the Vue app during initial registration. + +```json + devices: [ + { + "name": "Main Panel", + "channels": [ + "Air Conditioner", + "Furnace", + "Coffee Maker", + "Oven", + "Dishwasher", + "Tesla Charger", + "Refrigerator", + "Office" + ] + }, + { + "name": "Projector Plug" + }, + { + "name": "3D-Printer Plug" + } + ] +``` + +## Docker Compose + +For those that want to run Vuegraf using Docker Compose, the following files have been included: `docker-compose.yaml.template` and `docker-compose-run.sh`. Copy the`docker-compose.yaml.template` file to a new file called `docker-compose.yaml`. In the newly copied file, `vuegraf.volumes` values will need to be changed to the same directory you have created your vuegraf.json file. Additionally, adjust the persistent host storage path for the InfluxDB data volume. + +Finally run the `docker-compose-run.sh` script to start up the multi-container application. + +```sh +./docker-compose-run.sh +``` + +## Upgrading from InfluxDB v1 + +Early Vuegraf users still on InfluxDB v1 can upgrade to InfluxDB 2. To do so, stop the Influx v1 container (again, assuming you're using Docker). Then run the following command to install InfluxDB 2 and automatically upgrade your data. + +``` +docker run --rm --pull always -p 8086:8086 \ + -v /home/myuser/influxdb:/var/lib/influxdb \ + -v /home/myuser/influxdb2:/var/lib/influxdb2 \ + -e DOCKER_INFLUXDB_INIT_MODE=upgrade \ + -e DOCKER_INFLUXDB_INIT_USERNAME=vuegraf \ + -e DOCKER_INFLUXDB_INIT_PASSWORD=vuegraf \ + -e DOCKER_INFLUXDB_INIT_ORG=vuegraf \ + -e DOCKER_INFLUXDB_INIT_BUCKET=vuegraf \ + -e DOCKER_INFLUXDB_INIT_RETENTION=1y \ + influxdb +``` + +Adjust the host paths above as necessary, to match the old and new influxdb directories. The upgrade should complete relatively quickly. For reference, a 7GB database, spanning several months, upgrades in about 15 seconds on SSD storage. + +Monitor the console output and once the upgrade completes and the Influx server finishes starting, shut it down (CTRL+C) and then restart the Influx DB using the command referenced earlier in this document. + +Login to the new Influx DB 2 UI from your web browser, using the _vuegraf / vuegraf_ credentials. Go into the _Load Data -> Buckets_ screen and rename the `vue/autogen` bucket to `vuegraf` via the Settings button. + +Finally, apply the dashboard template as instructed earlier in this document. + +# License Vuegraf is distributed under the MIT license. diff --git a/setup.py b/setup.py index 2e2e23a..2e2e876 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ install_requires=[ 'influxdb>=5.3.1', 'influxdb_client>=1.33.0', - 'pyemvue>=0.16.0' + 'pyemvue>=0.16.0', + 'argparse>= 1.4.0' ] ) diff --git a/src/requirements.txt b/src/requirements.txt index 83da0e2..dbc2021 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,3 +1,4 @@ influxdb >= 5.3.1 influxdb_client >= 1.33.0 pyemvue == 0.16.0 +argparse >= 1.4.0 \ No newline at end of file diff --git a/src/vuegraf/.gitignore b/src/vuegraf/.gitignore new file mode 100644 index 0000000..89c263c --- /dev/null +++ b/src/vuegraf/.gitignore @@ -0,0 +1,6 @@ +vuegraf.json +int +docker-compose.yaml +dist/ +vuegraf.egg-info/ +vugraftobe.py diff --git a/src/vuegraf/getdate.py b/src/vuegraf/getdate.py deleted file mode 100644 index 5848446..0000000 --- a/src/vuegraf/getdate.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 - -import datetime -import json -import signal -import sys -import time -import traceback -from threading import Event - -# InfluxDB v1 -import influxdb - - - -# flush=True helps when running in a container without a tty attached -# (alternatively, "python -u" or PYTHONUNBUFFERED will help here) -def log(level, msg): - now = datetime.datetime.utcnow() - print('{} | {} | {}'.format(now, level.ljust(5), msg), flush=True) - -def info(msg): - log("INFO", msg) - -def error(msg): - log("ERROR", msg) - -def handleExit(signum, frame): - global running - error('Caught exit signal') - running = False - pauseEvent.set() - -def getConfigValue(key, defaultValue): - if key in config: - return config[key] - return defaultValue - -# Reset config file if history or DB reset set -# Allows sequential runs without lossing data -def setconfig(configname, configkey, configvalue) : - with open(configFilename, 'r') as newconfigFile: - newconfig = json.load(newconfigFile) - newconfig[configname][configkey] = configvalue - with open(configFilename, 'w') as configout: - json.dump(newconfig, configout, indent=4) - return() - - - -startupTime = datetime.datetime.utcnow() -try: - if len(sys.argv) != 2: - print('Usage: python {} '.format(sys.argv[0])) - sys.exit(1) - - configFilename = sys.argv[1] - config = {} - with open(configFilename) as configFile: - config = json.load(configFile) - - influxVersion = 1 - if 'version' in config['influxDb']: - influxVersion = config['influxDb']['version'] - - bucket = '' - write_api = None - query_api = None - sslVerify = True - - if 'ssl_verify' in config['influxDb']: - sslVerify = config['influxDb']['ssl_verify'] - - if influxVersion == 2: - info('Using InfluxDB version 2') - bucket = config['influxDb']['bucket'] - org = config['influxDb']['org'] - token = config['influxDb']['token'] - url= config['influxDb']['url'] - influx2 = influxdb_client.InfluxDBClient( - url=url, - token=token, - org=org, - verify_ssl=sslVerify - ) - write_api = influx2.write_api(write_options=influxdb_client.client.write_api.SYNCHRONOUS) - query_api = influx2.query_api() - - - else: - info('Using InfluxDB version 1') - - sslEnable = False - if 'ssl_enable' in config['influxDb']: - sslEnable = config['influxDb']['ssl_enable'] - - # Only authenticate to ingress if 'user' entry was provided in config - if 'user' in config['influxDb']: - influx = influxdb.InfluxDBClient(host=config['influxDb']['host'], port=config['influxDb']['port'], username=config['influxDb']['user'], password=config['influxDb']['pass'], database=config['influxDb']['database'], ssl=sslEnable, verify_ssl=sslVerify) - else: - influx = influxdb.InfluxDBClient(host=config['influxDb']['host'], port=config['influxDb']['port'], database=config['influxDb']['database'], ssl=sslEnable, verify_ssl=sslVerify) - - influx.create_database(config['influxDb']['database']) - - print('RESET = ',config['influxDb']['reset']) - if config['influxDb']['reset']: - info('Resetting database') - influx.delete_series(measurement='energy_usage') - setconfig('influxDb','reset',False) - - historyMinute = min(config['influxDb'].get('historyDays', 0), 7) - historyDays = min(config['influxDb'].get('historyDays', 0), 720) - history = historyDays > 0 - - query1 = 'select last(usage) from "energy_usage" where "detailed"=\'Hour\'' - query2 = 'select last(usage) from "energy_usage" where "detailed"=\'Day\'' - query3 = 'select last(usage) from "energy_usage" where "detailed"=\'False\'' - print(query1) - last = influx.query(query1) - print(last) - last = influx.query(query2) - print(last) - last = influx.query(query3) - print(last) - - for record in - - - - -except: - error('Fatal error: {}'.format(sys.exc_info())) - traceback.print_exc() - - diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index d5dbc41..f1ff5b3 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 +__author__ = 'https://github.com/jertel' +__license__ = 'GPL' +__contributors__ = 'https://github.com/jertel/vuegraf/graphs/contributors' +__version__ = '1.6.0' +__versiondate__ = '04/18/2023' +__maintainer__ = 'https://github.com/jertel' +__github__ = 'https://github.com/jertel/vuegraf' +__status__ = 'Production' + import datetime import json import signal @@ -7,6 +16,8 @@ import time import traceback from threading import Event +import argparse +import pytz # InfluxDB v1 import influxdb @@ -24,10 +35,10 @@ def log(level, msg): print('{} | {} | {}'.format(now, level.ljust(5), msg), flush=True) def info(msg): - log("INFO", msg) + log('INFO', msg) def error(msg): - log("ERROR", msg) + log('ERROR', msg) def handleExit(signum, frame): global running @@ -40,16 +51,6 @@ def getConfigValue(key, defaultValue): return config[key] return defaultValue -# Reset config file if history or DB reset set -# Allows sequential runs without lossing data -def setconfig(configname, configkey, configvalue) : - with open(configFilename, 'r') as newconfigFile: - newconfig = json.load(newconfigFile) - newconfig[configname][configkey] = configvalue - with open(configFilename, 'w') as configout: - json.dump(newconfig, configout, indent=4) - return() - def populateDevices(account): deviceIdMap = {} account['deviceIdMap'] = deviceIdMap @@ -60,17 +61,17 @@ def populateDevices(account): device = account['vue'].populate_device_properties(device) deviceIdMap[device.device_gid] = device for chan in device.channels: - key = "{}-{}".format(device.device_gid, chan.channel_num) + key = '{}-{}'.format(device.device_gid, chan.channel_num) if chan.name is None and chan.channel_num == '1,2,3': chan.name = device.device_name channelIdMap[key] = chan - info("Discovered new channel: {} ({})".format(chan.name, chan.channel_num)) + info('Discovered new channel: {} ({})'.format(chan.name, chan.channel_num)) def lookupDeviceName(account, device_gid): if device_gid not in account['deviceIdMap']: populateDevices(account) - deviceName = "{}".format(device_gid) + deviceName = '{}'.format(device_gid) if device_gid in account['deviceIdMap']: deviceName = account['deviceIdMap'][device_gid].device_name return deviceName @@ -80,7 +81,7 @@ def lookupChannelName(account, chan): populateDevices(account) deviceName = lookupDeviceName(account, chan.device_gid) - name = "{}-{}".format(deviceName, chan.channel_num) + name = '{}-{}'.format(deviceName, chan.channel_num) try: num = int(chan.channel_num) @@ -99,28 +100,28 @@ def lookupChannelName(account, chan): def createDataPoint(account, chanName, watts, timestamp, detailed): dataPoint = None if influxVersion == 2: - dataPoint = influxdb_client.Point("energy_usage") \ - .tag("account_name", account['name']) \ - .tag("device_name", chanName) \ - .tag("detailed", detailed) \ - .field("usage", watts) \ + dataPoint = influxdb_client.Point('energy_usage') \ + .tag('account_name', account['name']) \ + .tag('device_name', chanName) \ + .tag('detailed', detailed) \ + .field('usage', watts) \ .time(time=timestamp) else: dataPoint = { - "measurement": "energy_usage", - "tags": { - "account_name": account['name'], - "device_name": chanName, - "detailed": detailed, + 'measurement': 'energy_usage', + 'tags': { + 'account_name': account['name'], + 'device_name': chanName, + 'detailed': detailed, }, - "fields": { - "usage": watts, + 'fields': { + 'usage': watts, }, - "time": timestamp + 'time': timestamp } return dataPoint -def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEndTime=None, histLoop=None): +def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime=None, historyEndTime=None): excludedDetailChannelNumbers = ['Balance', 'TotalUsage'] minutesInAnHour = 60 secondsInAMinute = 60 @@ -135,9 +136,14 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd kwhUsage = chan.usage if kwhUsage is not None: - watts = float(minutesInAnHour * wattsInAKw) * kwhUsage - timestamp = stopTime - usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, False)) + if pointType is None: + watts = float(minutesInAnHour * wattsInAKw) * kwhUsage + timestamp = stopTime + usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, False)) + elif pointType == 'Day' or pointType == 'Hour' : + watts = kwhUsage * 1000 + timestamp = historyStartTime + usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, pointType)) if chanNum in excludedDetailChannelNumbers: continue @@ -153,61 +159,81 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd watts = float(secondsInAMinute * minutesInAnHour * wattsInAKw) * kwhUsage usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, True)) index += 1 - + + + # fetches historical minute data - if histLoop == "minute" : - usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.MINUTE.value, unit=Unit.KWH.value) + if historyStartTime is not None and historyEndTime is not None: + if args.verbose: + info('Hour-History - {} - start="{}" - stop="{}"'.format(chanName, historyStartTime,historyEndTime )) + #Hours + usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.HOUR.value, unit=Unit.KWH.value) index = 0 for kwhUsage in usage: if kwhUsage is None: continue - timestamp = historyStartTime + datetime.timedelta(minutes=index) - watts = float(minutesInAnHour * wattsInAKw) * kwhUsage - usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, False)) - + timestamp = historyStartTime + datetime.timedelta(hours=index) + watts = kwhUsage * 1000 + usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, 'Hour')) + index += 1 + #Days + usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.DAY.value, unit=Unit.KWH.value) + index = 0 + for kwhUsage in usage: + if kwhUsage is None: + continue + timestamp = historyStartTime + datetime.timedelta(days=index-1) + timestamp = timestamp.replace(hour=23, minute=59, second=59,microsecond=0) + timestamp = timestamp.astimezone(pytz.UTC) + watts = kwhUsage * 1000 + usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, 'Day')) index += 1 - # fetches historical hour/day or just previous day - if histLoop == "day-hour" or collectSummaries: - historyStartTime = stopTime - datetime.timedelta(days=max(historyDays,1)) - historyStartTime = historyStartTime.replace( hour=00, minute=00, second=00, microsecond=00) - #only used on history runs, otherwise is zero and will do next loop once. - dayLoop = historyDays / 25 - while dayLoop >= 0 and historyStartTime < stopTime : - #Fetches hour data - historyEndTime = min(stopTime, historyStartTime + datetime.timedelta(days=25)) - historyEndTime = historyEndTime.replace( hour=23,minute=59, second=59, microsecond=999999) - usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.HOUR.value, unit=Unit.KWH.value) - index = 0 - info('CollectSummaries; start="{}"; stop="{}"'.format(historyStartTime,historyEndTime )) - for kwhUsage in usage: - if kwhUsage is None: - continue - timestamp = usage_start_time + datetime.timedelta(hours=index) - watts = kwhUsage * 1000 - usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Hour")) - print(datetime.datetime.now, ) - index += 1 - #Fetches date data - usage, usage_start_time = account['vue'].get_chart_usage(chan, historyStartTime, historyEndTime, scale=Scale.DAY.value, unit=Unit.KWH.value) - index = 0 - for kwhUsage in usage: - if kwhUsage is None: - continue - timestamp = usage_start_time + datetime.timedelta(days=index) - watts = kwhUsage * 1000 - usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, "Day")) - index += 1 - dayLoop = dayLoop -1 - historyStartTime = historyStartTime + datetime.timedelta(days=26) + startupTime = datetime.datetime.utcnow() try: - if len(sys.argv) != 2: - print('Usage: python {} '.format(sys.argv[0])) - sys.exit(1) + #argparse includes default -h / --help as command line input + parser = argparse.ArgumentParser( + prog='vuegraf.py', + description='Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2)', + epilog='For more information go to : ' + __github__ + ) + parser.add_argument( + 'configFilename', + help='json config file', + type=str + ) + parser.add_argument( + '--version', + help='display version number', + action='store_true') + parser.add_argument( + '-v', + '--verbose', + help='verbose output - summaries', + action='store_true') + parser.add_argument( + '-q', + '--quiet', + help='do not print anything but errors', + action='store_true') + parser.add_argument( + '--historydays', + help='Starts executin by pulling history of Hours and Day data for specified number of days. example: --load-history-day 60', + type=int, + default=0 + ) + parser.add_argument( + '--resetdatabase', + action='store_true', + default=False, + help='Drop database and create a new one') + args = parser.parse_args() + if args.version: + print('vuegraf.py - version: ', __version__) + sys.exit(0) - configFilename = sys.argv[1] config = {} - with open(configFilename) as configFile: + with open(args.configFilename) as configFile: config = json.load(configFile) influxVersion = 1 @@ -237,13 +263,12 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd write_api = influx2.write_api(write_options=influxdb_client.client.write_api.SYNCHRONOUS) query_api = influx2.query_api() - if config['influxDb']['reset']: + with open(args.configFilename) as configFile: info('Resetting database') delete_api = influx2.delete_api() - start = "1970-01-01T00:00:00Z" + start = '1970-01-01T00:00:00Z' stop = startupTime.isoformat(timespec='seconds') - delete_api.delete(start, stop, '_measurement="energy_usage"', bucket=bucket, org=org) - setconfig('influxDb','reset',False) + delete_api.delete(start, stop, '_measurement="energy_usage"', bucket=bucket, org=org) else: info('Using InfluxDB version 1') @@ -259,38 +284,32 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd influx.create_database(config['influxDb']['database']) - print('RESET = ',config['influxDb']['reset']) - if config['influxDb']['reset']: + if args.resetdatabase: info('Resetting database') influx.delete_series(measurement='energy_usage') - setconfig('influxDb','reset',False) - - historyMinute = min(config['influxDb'].get('historyDays', 0), 7) - historyDays = min(config['influxDb'].get('historyDays', 0), 720) + historyDays = 0 + historyDays = min(args.historydays, 720) history = historyDays > 0 - running = True signal.signal(signal.SIGINT, handleExit) signal.signal(signal.SIGHUP, handleExit) pauseEvent = Event() - intervalSecs=getConfigValue("updateIntervalSecs", 60) - detailedIntervalSecs=getConfigValue("detailedIntervalSecs", 3600) - detailedDataEnabled=getConfigValue("detailedDataEnabled", False) - summariesDataEnabled=getConfigValue('summariesDataEnabled', False) - info('Settings -> updateIntervalSecs: {}, detailedEnabled: {}, detailedIntervalSecs: {}, summariesEnabled {}'.format(intervalSecs, detailedDataEnabled, detailedIntervalSecs, summariesDataEnabled)) - info(' History Days {}'.format(historyDays)) - lagSecs=getConfigValue("lagSecs", 5) + intervalSecs=getConfigValue('updateIntervalSecs', 60) + detailedIntervalSecs=getConfigValue('detailedIntervalSecs', 3600) + detailedDataEnabled=getConfigValue('detailedDataEnabled', False) + info('Settings -> updateIntervalSecs: {}, detailedEnabled: {}, detailedIntervalSecs: {}'.format(intervalSecs, detailedDataEnabled, detailedIntervalSecs)) + lagSecs=getConfigValue('lagSecs', 5) detailedStartTime = startupTime - #Only pull previous days Hourly and full day data and Full day after on next details loop after 00:05 - summariesDate = datetime.datetime.now() + datetime.timedelta(days=1) - summariesDate = summariesDate.replace(hour=0, minute=5, second=00, microsecond=00) - info('Set collectSummaries; {}'.format(summariesDate)) + pastDay = datetime.datetime.now() + pastDay = pastDay.replace(hour=23, minute=59, second=59, microsecond=0) + while running: now = datetime.datetime.utcnow() + curDay = datetime.datetime.now() stopTime = now - datetime.timedelta(seconds=lagSecs) collectDetails = detailedDataEnabled and detailedIntervalSecs > 0 and (stopTime - detailedStartTime).total_seconds() >= detailedIntervalSecs - collectSummaries = summariesDataEnabled and datetime.datetime.now() >= summariesDate - for account in config["accounts"]: + + for account in config['accounts']: if 'vue' not in account: account['vue'] = PyEmVue() account['vue'].login(username=account['email'], password=account['password']) @@ -305,61 +324,71 @@ def extractDataPoints(device, usageDataPoints, historyStartTime=None, historyEnd for gid, device in usages.items(): extractDataPoints(device, usageDataPoints) - if history: - for day in range(historyMinute): - histLoop = "minute" - info('Loading historical data - Minutes: {} day(s) ago'.format(day+1)) - #Extract second 12h of day - historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)-43200) - historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day))) - for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) - pauseEvent.wait(5) - #Extract first 12h of day - historyStartTime = stopTime - datetime.timedelta(seconds=3600*24*(day+1)) - historyEndTime = stopTime - datetime.timedelta(seconds=(3600*24*(day+1))-43200) - for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) - if not running: - break - pauseEvent.wait(5) - - if summariesDataEnabled: - histLoop = "day-hour" - info('Loading historical data - Days/Hours/Months: {} day(s) ago'.format(historyDays)) - for gid, device in usages.items(): - extractDataPoints(device, usageDataPoints, historyStartTime, historyEndTime, histLoop) - if not running: - break - pauseEvent.wait(5) - history = False - setconfig('influxDb','historyDays', 0) - - if not running: - break - - info('Submitting datapoints to database; account="{}"; points={}; Details="{}"; Summaries="{}"'.format(account['name'], - len(usageDataPoints),collectDetails, collectSummaries )) - if influxVersion == 2: - write_api.write(bucket=bucket, record=usageDataPoints) - else: - influx.write_points(usageDataPoints,batch_size=5000) + if collectDetails: + pastHour = stopTime - datetime.timedelta(hours=1) + pastHour = pastHour.replace(minute=00, second=00,microsecond=0) + historyStartTime = pastHour + usages = account['vue'].get_device_list_usage(deviceGids, pastHour, scale=Scale.HOUR.value, unit=Unit.KWH.value) + if usages is not None: + usageDataPoints = [] + for gid, device in usages.items(): + extractDataPoints(device, usageDataPoints, 'Hour', historyStartTime) + if args.verbose: + info('Collected Previous Hour: {} '.format(pastHour)) + + if pastDay.day < curDay.day: + usages = account['vue'].get_device_list_usage(deviceGids, pastDay, scale=Scale.DAY.value, unit=Unit.KWH.value) + historyStartTime = pastDay.astimezone(pytz.UTC) + if usages is not None: + usageDataPoints = [] + for gid, device in usages.items(): + extractDataPoints(device, usageDataPoints,'Day', historyStartTime) + if args.verbose: + info('Collected Previous Day: {}Local - {}UTC, '.format(pastDay, historyStartTime)) + pastDay = datetime.datetime.now() + pastDay = pastDay.replace(hour=23, minute=59, second=00, microsecond=0) + + if history: + info('Loading historical data: {} day(s) ago'.format(historyDays)) + historyStartTime = stopTime - datetime.timedelta(historyDays) + historyStartTime = historyStartTime.replace(hour=00, minute=00, second=00, microsecond=000000) + while historyStartTime <= stopTime: + historyEndTime = min(historyStartTime + datetime.timedelta(20), stopTime) + historyEndTime = historyEndTime.replace(hour=23, minute=59, second=59,microsecond=0) + if args.verbose: + info(' {} - {}'.format(historyStartTime,historyEndTime)) + for gid, device in usages.items(): + extractDataPoints(device, usageDataPoints, 'History', historyStartTime, historyEndTime) + if not running: + break + historyStartTime = historyEndTime + datetime.timedelta(1) + historyStartTime = historyStartTime.replace(hour=00, minute=00, second=00, microsecond=000000) + pauseEvent.wait(5) + history = False + + if not running: + break + + info('Submitting datapoints to database; account="{}"; points={}'.format(account['name'], len(usageDataPoints))) + if influxVersion == 2: + write_api.write(bucket=bucket, record=usageDataPoints) + else: + influx.write_points(usageDataPoints,batch_size=5000) except: - error('Failed to record new usage data: {}'.format(sys.exc_info())) + error('Failed to record new usage data: {}'.format(sys.exc_info())) traceback.print_exc() if collectDetails: detailedStartTime = stopTime + datetime.timedelta(seconds=1) - if collectSummaries: - summariesDate = datetime.datetime.now() + datetime.timedelta(days=1) - summariesDate = summariesDate.replace(hour=0, minute=5, second=00, microsecond=00) - info('Reset collectSummaries; {}'.format(summariesDate)) + pauseEvent.wait(intervalSecs) info('Finished') -except: - error('Fatal error: {}'.format(sys.exc_info())) - traceback.print_exc() - - +except SystemExit as e: + #If sys.exit was 2, then normal syntax exit from help or bad command line, no error message + if e.code == 0 or e.code == 2: + quit(0) + else: + error('Fatal error: {}'.format(sys.exc_info())) + traceback.print_exc() diff --git a/vuegraf.json.sample b/vuegraf.json.sample index 91ec5b9..efccadb 100644 --- a/vuegraf.json.sample +++ b/vuegraf.json.sample @@ -7,12 +7,12 @@ "database": "vue", "reset": false, "ssl_enable": false, - "ssl_verify": true, + "ssl_verify": true }, "updateIntervalSecs": 60, "detailedDataEnabled": "true", - "summariesDataEnabled": "false" + "summariesDataEnabled": "false", "accounts": [ { "name": "Primary Residence", From 124d7932f415e70846387fe16bf8ca7c60527f79 Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Mon, 15 May 2023 13:18:07 -0700 Subject: [PATCH 13/29] Update CHANGELOG.md cleanup --- CHANGELOG.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929eb11..e51b27f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,22 +20,22 @@ - Hour data runs with the get details time, default is 1 hour (3600 seconds). Based on when the program is started, you may be almost 2 hours behind for get hour. - Changed several one time parameters from the json config file to command line entries (History load days, reset database). - Example Command Line usage -usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename - -positional arguments: - configFilename json config file - -options: - -h, --help show this help message and exit - --version display version number - -v, --verbose verbose output - summaries - -q, --quiet do not print anything but errors - --historydays HISTORYDAYS - Starts executin by pulling history of Hours and Day data for specified number of days. - example: --load-history-day 60 - --resetdatabase Drop database and create a new one - +

Example Command Line usage +
vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename + +positional arguments:
+ configFilename json config file
+
+
options: +
-h, --help show this help message and exit +
--version display version number +
-v, --verbose verbose output - summaries +
-q, --quiet do not print anything but errors +
--historydays HISTORYDAYS +
Starts executing by pulling history of Hours and Day data for specified number of days. +
example: --historydays 60 +
--resetdatabase Drop database and create a new one +
## Other changes - Started Changelog for this and future releases - Added project metadata to main program - vuegraf.py, values can be updated through github automations From f7526e8d6b6cfa944292e5235244923424b6159d Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Mon, 15 May 2023 13:19:27 -0700 Subject: [PATCH 14/29] Update vuegraf.py history days label inconsitent --- src/vuegraf/vuegraf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 0fef20d..54fa8d2 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -220,7 +220,7 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= action='store_true') parser.add_argument( '--historydays', - help='Starts executin by pulling history of Hours and Day data for specified number of days. example: --load-history-day 60', + help='Starts executin by pulling history of Hours and Day data for specified number of days. example: --historydays 60', type=int, default=0 ) From d288985d3279e150203e63bf29750f3e6833538f Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Mon, 15 May 2023 14:52:07 -0700 Subject: [PATCH 15/29] Delete .vscode directory --- .vscode/launch.json | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index ab659d4..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - - { - "name": "Python: Current File", - "type": "python", - "request": "launch", - "program": "${file}", - "args" : ["/mnt/c/Users/garre/OneDrive/Documents/GitHub/vuegraf/vuegraf.json", "--resetdatabase", "--historydays", "90", "--verbose"], - //"args" : ["/mnt/c/Users/garre/OneDrive/Documents/GitHub/vuegraf/vuegraf.json", "--resetdatabase", "--verbose"], - "console": "integratedTerminal", - "justMyCode": true - } - ] -} \ No newline at end of file From d1b4828a1f60b63d94122d08c6e8fda2ed8291a8 Mon Sep 17 00:00:00 2001 From: Garrett <62046164+gauthig@users.noreply.github.com> Date: Mon, 15 May 2023 15:01:58 -0700 Subject: [PATCH 16/29] Update README.md Added line breaks '
' --- README.md | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 14665df..44973df 100644 --- a/README.md +++ b/README.md @@ -137,10 +137,16 @@ Vuegraf can be run either as a container (recommended), or as a host process. A Docker container is provided at [hub.docker.com](https://hub.docker.com/r/jertel/vuegraf). Refer to the command below to launch Vuegraf as a container. This assumes you have created a folder called `/home/myuser/vuegraf` and placed the vuegraf.json file inside of it. +Normal run with docker ```sh docker run --name vuegraf -d -v /home/myuser/vuegraf:/opt/vuegraf/conf jertel/vuegraf ``` +Recreate database and load 25 days of history +```sh +docker run --name vuegraf -d -v /home/myuser/vuegraf:/opt/vuegraf/conf jertel/vuegraf --resetdatabase --historydays=24 +``` + ## Host Process Ensure Python 3 and Pip are both installed. Install the required dependencies: @@ -165,22 +171,22 @@ or, on some Linux installations: python3 src/vuegraf/vuegraf.py vuegraf.json ``` -usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename - -Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2) - -positional arguments: - configFilename json config file - -options: - -h, --help show this help message and exit - --version display version number - -v, --verbose verbose output - summaries - -q, --quiet do not print anything but errors - --historydays HISTORYDAYS - Starts executin by pulling history of Hours and Day data for specified number of days. - example: --load-history-day 60 - --resetdatabase Drop database and create a new one +Optional Command Line Parameters +
usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename +
+
Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2) +
+
positional arguments: +
configFilename json config file +
+
options: +
-h, --help show this help message and exit +
--version display version number +
-v, --verbose verbose output - summaries +
--historydays HISTORYDAYS +
Starts executing by pulling history of Hours and Day data for specified number of days. +
example: --load-history-day 60 +
--resetdatabase Drop database and create a new one From 1d08c516873d92a74b29e42ab742ff48f82bd589 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 15 May 2023 20:20:11 -0400 Subject: [PATCH 17/29] Update CHANGELOG.md --- CHANGELOG.md | 56 +++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e51b27f..0367ec1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 2.TBD.TBD- Future +# 1.TBD.TBD ## Breaking changes - TBD @@ -7,47 +7,45 @@ - TBD ## Other changes +- TBD - -# 1.6.0 - Released +# 1.6.0 ## Breaking changes - Replaced Minute with Hour as normal interval since history is limited to 7 days from Emporia on minute data -- argparse libary was added, must run pip install -r requirements again in the src directory (or pip3 based on install) +- argparse libary was added, must run pip `install -r requirements.txt` again in the src directory (or pip3 based on install) ## New features -- Hour / Day data pulled. Allows for history of 2 years to be pulled. Assists in clean numbers/graphs to see daily monthly usage to compare against utilities reports. -- Hour data runs with the get details time, default is 1 hour (3600 seconds). Based on when the program is started, -you may be almost 2 hours behind for get hour. -- Changed several one time parameters from the json config file to command line entries (History load days, reset database). -

Example Command Line usage -
vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename - -positional arguments:
+- Hour / Day historic data retrieval: allows for history of up to two years to be pulled. Assists in clean numbers/graphs to see daily monthly usage to compare against utilities reports. +- Hour data runs with the get details time, default is 1 hour (3600 seconds). Based on when the program is started, you may be almost 2 hours behind for get hour. +- Moved one-time parameters out of the json config file. Those parameters are now specified as command line arguments (History load days, reset database). + +Example Command Line usage +``` +vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename + +positional arguments: configFilename json config file
-
-
options: -
-h, --help show this help message and exit -
--version display version number -
-v, --verbose verbose output - summaries -
-q, --quiet do not print anything but errors -
--historydays HISTORYDAYS -
Starts executing by pulling history of Hours and Day data for specified number of days. -
example: --historydays 60 -
--resetdatabase Drop database and create a new one -
+ +options: + -h, --help Show this help message and exit + --version Display version number + -v, --verbose Verbose output - summaries + -q, --quiet Do not print anything but errors + --historydays HISTORYDAYS + Starts executing by pulling history of Hours and Day data for specified number of days. + example: --historydays 60 + --resetdatabase Drop database and create a new one +``` + ## Other changes - Started Changelog for this and future releases - Added project metadata to main program - vuegraf.py, values can be updated through github automations -- Added command line paring with help syntaxt for all values - arge parse lib. +- Added command line pairing with help syntax for all values, via argparse lib. - Updated requirements.txt and setup.py with argparse>= 1.4.0 - Updated vuegraf.json.sample as history and reset database was moved to command line - Updated Readme.md with above changes - ran pylint and fixed - Quote delimiter consitency to all ' + Quote delimiter consistency to all ' Whitespaces Extra lines - - - -# 1.5.0 From 854f45e65f2d6e74bf85584ad743f4f2c974b2d4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 15 May 2023 20:28:03 -0400 Subject: [PATCH 18/29] Update README.md --- README.md | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 44973df..2d80816 100644 --- a/README.md +++ b/README.md @@ -87,11 +87,14 @@ The minimum configuration required to start Vuegraf is shown below. ### Ingesting Historical Data -If desired, it is possible to have Vuegraf import historical data. To do so, run vuegraf.py with the option historydays parameter with a value between 1 and 720. When ran, it will start by collection all hourly data points up to the specified parameter or max history stored at emporia. It will also collect 1 Day data for each day storing it 23:59:59 for each day. It collects the time using your local server time, but store it in influx on UTC. +If desired, it is possible to have Vuegraf import historical data. To do so, run vuegraf.py with the optional `--historydays` parameter with a value between 1 and 720. When this parameter is provided Vuegraf will start and collect all hourly data points up to the specified parameter, or max history available. It will also collect one day's summary data for each day, storing it with the timestamp 23:59:59 for each day. It collects the time using your local server time, but stores it in influxDB in UTC. -IMPORTANT - If you restart Vuegraf with --historydays still on the command line (or in the dockerfile) it will _again_ import history data. This will likely cause confusion with your data since you will now have duplicate/overlapping data. For best results, only enable historydays single run, and then remove from your command line. -Example: +IMPORTANT - If you restart Vuegraf with `--historydays` on the command line (or forget to remove it from the dockerfile) it will import history data _again_. This will likely cause confusion with your data since you will now have duplicate/overlapping data. For best results, only enable `--historydays` on a single run. + +For Example: +``` python3 path/to/vuegraf.py vuegraf.json --historydays 365 +``` ### Channel Names @@ -172,24 +175,23 @@ python3 src/vuegraf/vuegraf.py vuegraf.json ``` Optional Command Line Parameters -
usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename -
-
Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2) -
-
positional arguments: -
configFilename json config file -
-
options: -
-h, --help show this help message and exit -
--version display version number -
-v, --verbose verbose output - summaries -
--historydays HISTORYDAYS -
Starts executing by pulling history of Hours and Day data for specified number of days. -
example: --load-history-day 60 -
--resetdatabase Drop database and create a new one +``` +usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename +Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2) +positional arguments: + configFilename json config file +options: + -h, --help Show this help message and exit + --version Display version number + -v, --verbose Verbose output - summaries + --historydays HISTORYDAYS + Starts executing by pulling history of Hours and Day data for specified number of days. + example: --load-history-day 60 + --resetdatabase Drop database and create a new one +``` ## Alerts @@ -216,15 +218,15 @@ By default, Vuegraf will poll every minute to collect the energy usage value ove detailedDataEnabled: true ``` -For every datapoint a tag is stored in influx for the type of measurement +For every datapoint a tag is stored in InfluxDB for the type of measurement - `detailed = True` represents backfilled per-second data that is optionally queried from Emporia once every hour. - `detailed = False` represents the per-minute average data that is collected every minute. - `detailed = Hour` represents the data summarized in hours -- `detailed - Day` represents a single data point to summarize the entire day +- `detailed = Day` represents a single data point to summarize the entire day When building graphs that show a sum of the energy usage, be sure to only include the correct detail tag, otherwise your summed values will be higher than expected. Detailed data will take more time for the graphs to query due to the extra data involved. If you want to have a chart that shows daily data over a long period or even a full year, use the `detailed = Day` tag. -If you are running this on a small server, you might want to look at setting a RETENTION POLICY on influx to remove minute or second data over time. i.e. keep only 30 days fo second data. +If you are running this on a small server, you might want to look at setting a RETENTION POLICY on your InfluxDB bucket to remove minute or second data over time. For example, it will reduce storage needs if you retain only 30 days of per-_second_ data. ## Vue Utility Connect Energy Monitor From c8cb1abea505c1bab5fbec8dcba7e30be5c12b3c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 15 May 2023 20:30:20 -0400 Subject: [PATCH 19/29] Update vuegraf.py --- src/vuegraf/vuegraf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 54fa8d2..73cb61a 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 __author__ = 'https://github.com/jertel' -__license__ = 'GPL' +__license__ = 'MIT' __contributors__ = 'https://github.com/jertel/vuegraf/graphs/contributors' __version__ = '1.6.0' -__versiondate__ = '04/18/2023' +__versiondate__ = '2023/05/16' __maintainer__ = 'https://github.com/jertel' __github__ = 'https://github.com/jertel/vuegraf' __status__ = 'Production' From 3ee9867adef0ff6ef2b742b3cbcc511b54c6431b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 15 May 2023 20:31:41 -0400 Subject: [PATCH 20/29] Update vuegraf.py --- src/vuegraf/vuegraf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 73cb61a..b6c8000 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -206,21 +206,21 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= ) parser.add_argument( '--version', - help='display version number', + help='Display version number', action='store_true') parser.add_argument( '-v', '--verbose', - help='verbose output - summaries', + help='Verbose output - summaries', action='store_true') parser.add_argument( '-q', '--quiet', - help='do not print anything but errors', + help='Do not print anything but errors', action='store_true') parser.add_argument( '--historydays', - help='Starts executin by pulling history of Hours and Day data for specified number of days. example: --historydays 60', + help='Starts execution by pulling history of Hours and Day data for specified number of days. example: --historydays 60', type=int, default=0 ) @@ -228,7 +228,7 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= '--resetdatabase', action='store_true', default=False, - help='Drop database and create a new one') + help='Drop database and create a new one. USE WITH CAUTION - WILL RESULT IN COMPLETE VUEGRAF DATA LOSS!') args = parser.parse_args() if args.version: print('vuegraf.py - version: ', __version__) From 080b1ba412a7537b49de40599217e597b1ac3202 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 15 May 2023 20:41:48 -0400 Subject: [PATCH 21/29] Update vuegraf.py --- src/vuegraf/vuegraf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index b6c8000..4128d9d 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -159,7 +159,7 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= watts = float(secondsInAMinute * minutesInAnHour * wattsInAKw) * kwhUsage usageDataPoints.append(createDataPoint(account, chanName, watts, timestamp, True)) if args.verbose: - info('Get Detailes (Seconds); start="{}"; stop="{}"'.format(detailedStartTime,stopTime )) + info('Get Details (Seconds); start="{}"; stop="{}"'.format(detailedStartTime,stopTime )) index += 1 @@ -196,8 +196,8 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= #argparse includes default -h / --help as command line input parser = argparse.ArgumentParser( prog='vuegraf.py', - description='Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2)', - epilog='For more information go to : ' + __github__ + description='Pulls data from cloud servers and loads it into an InfluxDB database', + epilog='For more information visit ' + __github__ ) parser.add_argument( 'configFilename', @@ -265,7 +265,7 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= write_api = influx2.write_api(write_options=influxdb_client.client.write_api.SYNCHRONOUS) query_api = influx2.query_api() - with open(args.configFilename) as configFile: + if args.resetdatabase: info('Resetting database') delete_api = influx2.delete_api() start = '1970-01-01T00:00:00Z' From 6404ccea5717eec76a81708829c1daec184a4a73 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 15 May 2023 20:42:24 -0400 Subject: [PATCH 22/29] Update CHANGELOG.md --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0367ec1..84ea1c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ Example Command Line usage vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename positional arguments: - configFilename json config file
+ configFilename json config file options: -h, --help Show this help message and exit @@ -33,8 +33,8 @@ options: -v, --verbose Verbose output - summaries -q, --quiet Do not print anything but errors --historydays HISTORYDAYS - Starts executing by pulling history of Hours and Day data for specified number of days. - example: --historydays 60 + Starts executing by pulling history of Hours and Day data for specified number of days. + example: --historydays 60 --resetdatabase Drop database and create a new one ``` From 7abd4034a2cafd470e24b255ff6de1ae9a0a4190 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 15 May 2023 20:43:36 -0400 Subject: [PATCH 23/29] Update vuegraf.py --- src/vuegraf/vuegraf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 4128d9d..313f0b2 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -289,7 +289,7 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= if args.resetdatabase: info('Resetting database') influx.delete_series(measurement='energy_usage') - historyDays = 0 + historyDays = min(args.historydays, 720) history = historyDays > 0 running = True From 844e67f8188cd8913cc66749820f8bc3d1eceac6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 17 May 2023 08:46:01 -0400 Subject: [PATCH 24/29] Fix reset of v2 --- src/vuegraf/vuegraf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index 313f0b2..dd10c6f 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -269,7 +269,7 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= info('Resetting database') delete_api = influx2.delete_api() start = '1970-01-01T00:00:00Z' - stop = startupTime.isoformat(timespec='seconds') + stop = startupTime.isoformat(timespec='seconds') + 'Z' delete_api.delete(start, stop, '_measurement="energy_usage"', bucket=bucket, org=org) else: info('Using InfluxDB version 1') From 60be82b46b97a6032794ba22b7c5931a40d800d2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 17 May 2023 08:48:14 -0400 Subject: [PATCH 25/29] Improve help instructions --- src/vuegraf/vuegraf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vuegraf/vuegraf.py b/src/vuegraf/vuegraf.py index dd10c6f..7544ce3 100644 --- a/src/vuegraf/vuegraf.py +++ b/src/vuegraf/vuegraf.py @@ -196,12 +196,12 @@ def extractDataPoints(device, usageDataPoints, pointType=None, historyStartTime= #argparse includes default -h / --help as command line input parser = argparse.ArgumentParser( prog='vuegraf.py', - description='Pulls data from cloud servers and loads it into an InfluxDB database', - epilog='For more information visit ' + __github__ + description='Veugraf retrieves data from cloud servers and inserts it into an InfluxDB database.', + epilog='For more information visit: ' + __github__ ) parser.add_argument( 'configFilename', - help='json config file', + help='JSON config file', type=str ) parser.add_argument( From 18b0ff18286a391b706b1e8769b1ae697c61096a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 17 May 2023 08:51:26 -0400 Subject: [PATCH 26/29] Update to reflect usage change --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2d80816..518214a 100644 --- a/README.md +++ b/README.md @@ -178,13 +178,13 @@ Optional Command Line Parameters ``` usage: vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename -Pulls data from Emporia AWS servers and loads it into a influx database (v1 or v2) +Retrieves data from cloud servers and inserts it into an InfluxDB database. positional arguments: - configFilename json config file + configFilename JSON config file options: - -h, --help Show this help message and exit + -h, --help show this help message and exit --version Display version number -v, --verbose Verbose output - summaries --historydays HISTORYDAYS From 474b20c2e18ccc8884ffdd6f39404995983cd341 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 17 May 2023 08:54:17 -0400 Subject: [PATCH 27/29] Update CHANGELOG.md --- CHANGELOG.md | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84ea1c7..528efd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,39 +13,23 @@ ## Breaking changes - Replaced Minute with Hour as normal interval since history is limited to 7 days from Emporia on minute data -- argparse libary was added, must run pip `install -r requirements.txt` again in the src directory (or pip3 based on install) +- argparse libary was added, must run `pip install -r requirements.txt` again in the src directory (or pip3 based on install) ## New features - Hour / Day historic data retrieval: allows for history of up to two years to be pulled. Assists in clean numbers/graphs to see daily monthly usage to compare against utilities reports. - Hour data runs with the get details time, default is 1 hour (3600 seconds). Based on when the program is started, you may be almost 2 hours behind for get hour. -- Moved one-time parameters out of the json config file. Those parameters are now specified as command line arguments (History load days, reset database). - -Example Command Line usage -``` -vuegraf.py [-h] [--version] [-v] [-q] [--historydays HISTORYDAYS] [--resetdatabase] configFilename - -positional arguments: - configFilename json config file - -options: - -h, --help Show this help message and exit - --version Display version number - -v, --verbose Verbose output - summaries - -q, --quiet Do not print anything but errors - --historydays HISTORYDAYS - Starts executing by pulling history of Hours and Day data for specified number of days. - example: --historydays 60 - --resetdatabase Drop database and create a new one -``` +- Moved one-time parameters out of the json config file. Those parameters are now specified as command line arguments (--historydays, --resetdatabase). ## Other changes - Started Changelog for this and future releases -- Added project metadata to main program - vuegraf.py, values can be updated through github automations +- Added project metadata to `vuegraf.py`, values can be updated through github automations - Added command line pairing with help syntax for all values, via argparse lib. -- Updated requirements.txt and setup.py with argparse>= 1.4.0 -- Updated vuegraf.json.sample as history and reset database was moved to command line +- Updated `requirements.txt` and setup.py with `argparse>= 1.4.0` +- Updated `vuegraf.json.sample` as history and reset database was moved to command line - Updated Readme.md with above changes - ran pylint and fixed Quote delimiter consistency to all ' Whitespaces Extra lines + +With sppecial thanks to @gauthig for initiating these 1.6.0 changes! From 583d0052aab219f133b8dc4a1bf53ecd566fb813 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 17 May 2023 08:54:36 -0400 Subject: [PATCH 28/29] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 528efd1..e3631e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,4 +32,4 @@ Whitespaces Extra lines -With sppecial thanks to @gauthig for initiating these 1.6.0 changes! +With special thanks to @gauthig for initiating these 1.6.0 changes! From 7a821fbcc12f0265d1e618d0916660c3e0baf8a7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 May 2023 06:44:38 -0400 Subject: [PATCH 29/29] Update vuegraf.json.sample --- vuegraf.json.sample | 1 - 1 file changed, 1 deletion(-) diff --git a/vuegraf.json.sample b/vuegraf.json.sample index efccadb..54f25d7 100644 --- a/vuegraf.json.sample +++ b/vuegraf.json.sample @@ -12,7 +12,6 @@ }, "updateIntervalSecs": 60, "detailedDataEnabled": "true", - "summariesDataEnabled": "false", "accounts": [ { "name": "Primary Residence",