add outside-docker scripts and README

README.txt

@@ -0,0 +1,53 @@
+Change Log:
+2022/6/15: enable command-line parser, so that users can customize input/output file names, PM2.5 factor, etc. by changing parameters in settings.yaml file. 
+2022/9/5: PM2.5 factor changed from 0 to 0.15; emission input changed from "nei_isrm_summary_onroad.csv" to "nei_isrm_summary_onroad_LDV.csv" (i.e., removed light commercial truck categories, now only 7, instead of 10, SCCs are considered of interest). 
+
+_______________________________________________________________
+
+The LDVEMIS model in BILD-AQ project is designed to translate VMT changes in different scenarios to emission changes of air pollutants such as NOx, VOC, NH3, SOx and PM2.5. 
+
+Input (in "input" folder) includes a tract-level VMT data file (E.g., "Sample_Input_CA.csv") that should be provided either by the user directly or by upstream models, and two other data files that have already been pre-generated by LDVEMIS team. Output of the LDVEMIS model is a csv file that estimates changes of NOx, VOC, NH3, SOx and PM2.5 emissions at each ISRM grid (unit = short ton/year). 
+
+_______________________________________________________________
+Description of files in the LDVEMIS library:
+
+1. run.py - an executable python script that reads settings from settings.yaml and calls LDVEMIS docker to execute the whole simulation.
+
+2. settings.yaml - a user-friendly configuration file for setting high-level parameters, such as input/output file name, emission category, etc. 
+
+3. environment.txt - a list of python packages needed for running run.py. Please make sure to install them before starting. 
+
+4. input/HPMS/network_isrm_*.RData - pre-processed data for simulations inside each of the CONUS states. Each row in this RData is a HPMS road segment by census tract and ISRM grid. Data for four states are still misssing now and will be added later: Idoha, Missouri, Montana, Texas.
+
+5. input/nei_isrm_summary_onroad_LDV.csv - a pre-processed data that allocates 2014 National Emission Inventory (NEI) to each ISRM grids. 7 emission categories (SCCs) are considered of interest, and their description can be found in the documentation folder.
+
+6. input/Sample_Input_CA.csv - a FAKE example of user- or upstream-provided VMT data. In general, this input should provides tract-level VMT data under the baseline ("VMT_base" column) versus in a specific scenario ("VMT_scenarios" column). You can provide VMT data for any number of census tracts, and can also name it whatever you like (but make sure to change "input_fname" accordingly in settings.yaml). 
+
+7. output/Sample_Output_ISRM_CA_Gas_LD_Veh..csv - a FAKE example output that LDVEMIS generates using the FAKE "Sample_Input_CA.csv". Its "VOC/NOx/NH3/SOx/PM25" columns indicate emission changes of those air pollutants at each ISRM grid (unit = short ton/year). 
+
+_______________________________________________________________
+To run the LDVEMIS module:
+
+1) Check environment.txt and make sure you have installed the packages listed. 
+
+2) Make sure your docker engine is running properly by entering "sudo service docker status". If not running,  start it by "sudo service docker start". 
+
+3) Under a directory that you like, ${LDVEMIS_HOME}, put "settings.yaml" and "run.py"
+
+4) Create two folders named "input" and "output"
+
+5) Put nei_isrm_summary_onroad_LDV.csv and HPMS/network_isrm_*.RData in input folder
+
+6) Provide the VMT change data anyname_input.csv (its data should follow the "Sample_Input_CA.csv" format) in input folder
+*** this is the only input required from user/upstream model
+
+7) In settings.yaml, set input_fname to be the input name you used (e.g., "input_fname: anyname_input.csv"). 
+
+8) Update setting in settings.yaml if needed. For example, you can change PM2.5 factor to any value reasonable (default = 0.15). 
+
+9) Under ${LDVEMIS_HOME} directory, enter "python run run.py" to run the main python script
+
+10) When the run is done, check the output (its name can be customized in settings.yaml before the run) in output folder
+
+
+

environment.txt

@@ -0,0 +1,12 @@
+Python version recommended: 3.9.7
+
+
+Python packages needed:
+	from re import L
+	import shutil
+	import sys
+	import os
+	import yaml # version 6.0
+	import docker # version 5.0.3
+	import argparse # version 1.1
+	import logging

run.py

@@ -0,0 +1,153 @@
+from re import L
+import shutil
+import yaml
+import docker
+import os
+import argparse
+import logging
+import sys
+
+
+logging.basicConfig(
+    stream=sys.stdout, level=logging.INFO,
+    format='%(asctime)s %(name)s - %(levelname)s - %(message)s')
+
+
+## parset arguments and settings from settings.yaml and command line
+def parse_args_and_settings(settings_file='settings.yaml'):
+
+    # read settings from config file
+    with open(settings_file) as file:
+        settings = yaml.load(file, Loader=yaml.FullLoader)
+
+    # parse command-line args
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument(
+        '-v', '--verbose', action='store_true', help='print docker stdout')
+    parser.add_argument(
+        '-p', '--pull_latest', action='store_true',
+        help='pull latest docker images before running')
+    args = parser.parse_args()
+
+    # override .yaml settings with command-line values if command-line
+    # values are not False/None
+    if args.verbose:
+        settings.update({'docker_stdout': args.verbose})
+    if args.pull_latest:
+        settings.update({'pull_latest': args.pull_latest})
+
+    return settings
+
+
+## util: format print
+def formatted_print(string, width=50, fill_char='#'):
+    print('\n')
+    if len(string) + 2 > width:
+        width = len(string) + 4
+    string = string.upper()
+    print(fill_char * width)
+    print('{:#^{width}}'.format(' ' + string + ' ', width=width))
+    print(fill_char * width, '\n')
+
+
+## model volume mount defintion, equivalent to 
+## docker run -v host_inputdir:indocker_inputdir 
+def get_ldvemis_docker_vols(settings):
+    host_inputdir        = os.path.abspath(settings['host_inputdir'])
+    host_outputdir       = os.path.abspath(settings['host_outputdir'])
+    indocker_inputdir   = os.path.abspath(settings['indocker_inputdir'])
+    indocker_outputdir  = os.path.abspath(settings['indocker_outputdir'])
+    ldvemis_docker_vols = {
+        host_inputdir: {                    ## source location, aka "local"  
+            'bind': indocker_inputdir,      ## destination loc, aka "remote", "client"
+            'mode': 'rw'},
+        host_outputdir: {
+            'bind': indocker_outputdir,
+            'mode': 'rw'} }
+    return ldvemis_docker_vols
+
+
+
+## format container command
+def get_ldvemis_cmd(settings):
+    input_fname = settings['input_fname']
+    output_fname = settings['output_fname']
+    emis_inputfname = settings['emis_inputfname']
+    emis_category = settings['emis_category']
+    basedir = settings.get('basedir','/')
+    codedir = settings.get('codedir','/')
+    PMfactor = settings.get('PMfactor', 0)
+    formattable_command = settings['formattable_command']
+    ldvemis_cmd = formattable_command.format(input_fname, output_fname, emis_inputfname, emis_category, basedir, codedir,PMfactor)
+    return ldvemis_cmd
+
+
+## pull docker client
+def initialize_docker_client(settings):
+    image_name = settings['docker_image']
+    pull_latest = settings.get('pull_latest', False)
+    client = docker.from_env()
+
+    if pull_latest:
+        print('Pulling latest image for {0}'.format(image_name))
+        client.images.pull(image_name)
+
+    return client 
+
+
+## main function
+def run_model(settings, client):
+
+    # 1. PARSE SETTINGS
+    input_fname = settings['input_fname']
+    output_fname = settings['output_fname']
+    emis_category = settings['emis_category']
+    image_name = settings['docker_image']
+    ldvemis_docker_vols = get_ldvemis_docker_vols(settings)
+    ldvemis_cmd = get_ldvemis_cmd(settings)
+    docker_stdout = settings.get('docker_stdout', False)
+
+
+    # 2. RUN LDVEMIS via docker container client
+    print_str = (
+        "Simulating LDVEMIS with input {0} "
+        "output {1}, emission {2}".format(
+            input_fname, output_fname, emis_category))
+    formatted_print(print_str)
+    ldvemis = client.containers.run(
+        image_name,
+        volumes=ldvemis_docker_vols,
+        command=ldvemis_cmd,
+        stdout=docker_stdout,
+        stderr=True,
+        detach=True)
+    for log in ldvemis.logs(
+            stream=True, stderr=True, stdout=docker_stdout):
+        print(log)
+
+
+    # 3. CLEAN UP
+    ldvemis.remove()
+    logger.info('LDVEMIS Done!')
+
+    return
+
+
+
+
+## main script
+if __name__ == '__main__':
+
+    logger = logging.getLogger(__name__)
+
+    # load args and settings
+    settings = parse_args_and_settings()
+
+    # start docker client
+    client = initialize_docker_client(settings)
+
+    # perform the main run
+    run_model(settings, client)
+
+    # print when finished
+    logger.info("Finished")

settings.yaml

@@ -0,0 +1,31 @@
+############################
+#     LDVEMIS SETTINGS     #
+############################
+# docker settings
+docker_image: ghcr.io/yhanw0719/bild-aq:mainv1.2.3
+docker_stdout: False
+pull_latest: True # False
+
+# io settings
+host_inputdir: /home/ubuntu/Yuhan/Data/BILD-AQ/input/
+host_outputdir: /home/ubuntu/Yuhan/Data/BILD-AQ/output/
+input_fname: Sample_Input_CA.csv ## you can customize it
+output_fname: Sample_Output_ISRM_CA_Onroad_LDV.csv ## you can customize it
+
+# emis input settings
+emis_inputfname: nei_isrm_summary_onroad_LDV.csv
+emis_category: Onroad_LDV 
+
+# set PM 2.5 change factor (% PM25 change = PM25factor * % VMT change)
+PMfactor: 0.15
+
+# path settings inside docker (DO NOT CHANGE UNLESS NECESSARY)
+basedir: /
+codedir: /opt/gitrepo/BILD-AQ
+indocker_inputdir: /input
+indocker_outputdir: /output
+
+# format command line arguments
+formattable_command: "--input_fname {0} --output_fname {1} --emis_inputfname {2} --emis_category {3} --basedir {4} --codedir {5} --PMfactor {6}"
+
+