diff --git a/examples/docker/README b/examples/docker/README new file mode 100644 index 0000000..6d4ac9c --- /dev/null +++ b/examples/docker/README @@ -0,0 +1,118 @@ +This example shows how to: + +1) Build a Docker image that contains everything needed to run trollflow +2) Use the Docker image to produce satellite images from various different + satellites +3) Modify the config files to suit your needs + + +1) Building the Docker image + +a) Install Docker + + Install docker following the instructions from the following links for + your operating system. + + Ubuntu: https://docs.docker.com/install/linux/docker-ce/ubuntu/ + Fedora: https://docs.docker.com/install/linux/docker-ce/fedora/ + OSX: https://docs.docker.com/docker-for-mac/install/ + Windows: https://docs.docker.com/docker-for-windows/install/ + +b) Build the docker image "trollflow" + + After you have a working Docker installation, in commandline, navigate + to the directory where this README file is located + (trollflow-sat/examples/docker/) and issue the following command: + + In linux or OSX: + + docker build -t trollflow -f build/Dockerfile build_dir + + In windows: + + docker build -t trollflow -f build\Dockerfile build_dir + + Depending on the computer and network connection, the build takes from + 5 minutes to 20 minutes. + + +2) Running production + + To test the Docker image to produce images from satellite data, + you'll need some satellite data. This example contains example + configurations for the following satellite instruments and formats: + + - MSG SEVIRI in uncompressed HRIT format + + All the configuration files use UNIX paths, as the Docker image is + based on Ubuntu linux. + +a) Start the trollflow Docker image + + This example shows how to run production for MSG SEVIRI data in + uncompressed HRIT format. To run other examples, replace the + "source" mount-point in the first line after "docker run" to point + to any of the other supplied directories. + + - in linux or OSX: + + docker run \ + -e LOCAL_USER_ID=`id -u $USER` \ + --mount type=bind,source=`pwd`/config_seviri_hrit/,target=/mnt/config/ \ + --mount type=bind,source=`pwd`/logs/,target=/mnt/logs/ \ + --mount type=bind,source=`pwd`/input/,target=/mnt/input/ \ + --mount type=bind,source=`pwd`/output/,target=/mnt/output/ \ + -p 9001:9001 \ + --rm \ + trollflow + + - in windows: + + docker run \ + --mount type=bind,source=%cd%}config_seviri_hrit,target=/mnt/config/ \ + --mount type=bind,source=%cd%\logs,target=/mnt/logs/ \ + --mount type=bind,source=%cd%\input,target=/mnt/input/ \ + --mount type=bind,source=%cd%\output,target=/mnt/output/ \ + -p 9001:9001 \ + --rm \ + trollflow + + This will start trollflow and mount the four subdirectories within + this directory to the trollflow Docker container. + + The processes running inside the container can be controlled and + monitored by opening the following URL in a browser: + + http://127.0.0.1:9001 + + The login credentials, given in supervisord.conf, are "username" and + "password". + +b) Copy files to "input" directory + + Copying files to the mounted "output" subdirectory starts the + processing. Log files are written to "logs" directory, and + following them shows the progress of the processing. The log + messages are also printed in the terminal where the trollflow Docker + container is running. + +c) The results + + In the case of succesful run, the resulting images are saved to + "output" directory. + + +3) Adapting the configuration files + + The example configuration files can be modified as the user sees + fit, but the following things need to be noted: + + - The trollflow Docker image is built to start Supervisord (which + handles the startup of all the necessary software) using the + configuration file placed in /mnt/config/supervisord.conf + + - If the paths for the input, output and/or logs are changed, these + need to be updated also in the configuration files + + - Remember to match the mount-points in the "docker run" command to + the paths define in the configuration files diff --git a/examples/docker/build_dir/Dockerfile b/examples/docker/build_dir/Dockerfile new file mode 100644 index 0000000..f80c65a --- /dev/null +++ b/examples/docker/build_dir/Dockerfile @@ -0,0 +1,50 @@ +FROM continuumio/miniconda3:latest + +RUN apt-get update && apt-get -y --no-install-recommends install \ + gosu + +RUN conda config --add channels conda-forge && \ + conda update conda && \ + conda install satpy \ + pyresample \ + pykdtree \ + trollimage \ + pycoast \ + pyorbital \ + rasterio \ + gdal \ + h5py \ + ipython \ + pyinotify \ + pyzmq \ + h5netcdf \ + netifaces \ + meld3 \ + scp \ + paramiko \ + dpath && \ + conda clean -a -y +ADD requirements.txt /tmp/ +RUN pip install -r /tmp/requirements.txt && \ + rm -r /tmp/requirements.txt && \ + cd /tmp/ && \ + git clone https://github.com/Supervisor/supervisor.git && \ + cd supervisor && python setup.py install && \ + cd /tmp/ && rm -rf /tmp/supervisor/ && \ + git clone https://github.com/Supervisor/superlance.git && \ + cd superlance && python setup.py install && \ + cd /tmp/ && rm -rf /tmp/superlance/ && \ + python -c 'from pyspectral.utils import download_luts; \ + from pyspectral.utils import logging_on; import logging; \ + logging_on(logging.DEBUG); download_luts()' && \ + python -c 'from pyspectral.utils import download_rsr; \ + from pyspectral.utils import logging_on; import logging; \ + logging_on(logging.DEBUG); download_rsr()' && \ + mkdir -p /home/user/.local/share && \ + mv /root/.local/share/pyspectral /home/user/.local/share/ + +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["supervisord", "-c", "/mnt/config/supervisord.conf"] diff --git a/examples/docker/build_dir/entrypoint.sh b/examples/docker/build_dir/entrypoint.sh new file mode 100644 index 0000000..c6ad8ae --- /dev/null +++ b/examples/docker/build_dir/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Add local user +# Either use the LOCAL_USER_ID if passed in at runtime or +# fallback + +USER_ID=${LOCAL_USER_ID:-9001} + +echo "Starting with UID : $USER_ID" +useradd --shell /bin/bash -u $USER_ID -o -c "" -m user +export HOME=/home/user +chown -R ${USER_ID}.${USER_ID} /home/user/ +exec gosu user "$@" diff --git a/examples/docker/build_dir/requirements.txt b/examples/docker/build_dir/requirements.txt new file mode 100644 index 0000000..2e5df18 --- /dev/null +++ b/examples/docker/build_dir/requirements.txt @@ -0,0 +1,6 @@ +posttroll +pytroll-schedule +pytroll-collectors +trollmoves +trollflow +trollflow-sat diff --git a/examples/docker/config_ears_viirs/README b/examples/docker/config_ears_viirs/README new file mode 100644 index 0000000..f337017 --- /dev/null +++ b/examples/docker/config_ears_viirs/README @@ -0,0 +1,38 @@ +This directory contains an example production chain setup for VIIRS +Compact SDR received from EARS network. + +supervisord.conf +---------------- + +Configuration file for Supervisord, which starts all the processing steps. + +trollstalker.ini and trollstalker_logging.ini +--------------------------------------------- + +Configuration file for Trollstalker, which creates posttroll messages +for incoming files, and logging config for it. + +segment_gatherer.yaml +--------------------- + +Configuration file for segment_gatherer, which collects the messages +coming from trollstalker to one message that contains metadata of both +M and DNB channel files for one time slot. + +gatherer.ini +------------ + +Configuration file for gathering granules to cover a geographic area +of interest. + +flow_processor.yaml and flow_processor_logging.ini +-------------------------------------------------- + +Definition of the processing chain and its logging: creation of +composites, resampling to target area, writing to files. + +products.yaml +------------- + +Definition of the products to be generated: composites, target areas, +file formats, filename patterns. diff --git a/examples/docker/config_ears_viirs/flow_processor.yaml b/examples/docker/config_ears_viirs/flow_processor.yaml new file mode 100644 index 0000000..599418e --- /dev/null +++ b/examples/docker/config_ears_viirs/flow_processor.yaml @@ -0,0 +1,58 @@ +config: + product_list: &product_list + /mnt/config/products.yaml + log_config: + /mnt/config/flow_processor_logging.ini + use_utc: True + force_gc: Tue + coverage_check: False + +work: + - type: daemon + name: listener + components: + - class: !!python/object:posttroll.listener.ListenerContainer + topics: + - /collection/ears_viirs + + - type: workflow + name: satpy_compositor + Workflow: + - trollflow_sat.satpy_compositor.SceneLoader: + product_list: *product_list + use_lock: True + instruments: + - viirs + readers: + - viirs_compact + service: + ears_viirs + + - type: workflow + name: satpy_resampler + Workflow: + - trollflow_sat.satpy_resampler.Resampler: + # Use "null" instead of "None" + radius: 2000 + resampler: nearest + mask_area: True + product_list: *product_list + use_lock: True + process_by_area: True + + - type: daemon + name: satpy_writer + components: + - class: !!python/object:trollflow_sat.satpy_writer.DataWriterContainer + nameservers: null + port: 0 + topic: "/image/ears_viirs/{productname}" + use_lock: True + save_settings: + tile: True + blockxsize: 512 + blockysize: 512 + compress: deflate + # Overviews (reduced resolution images) created from the full + # resolution data and embedded in the saved (GeoTIFF) files + # overviews: [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] diff --git a/examples/docker/config_ears_viirs/flow_processor_logging.ini b/examples/docker/config_ears_viirs/flow_processor_logging.ini new file mode 100644 index 0000000..170e568 --- /dev/null +++ b/examples/docker/config_ears_viirs/flow_processor_logging.ini @@ -0,0 +1,34 @@ +[loggers] +keys = root,flow_processor + +[handlers] +keys = consoleHandler,fileHandler + +[formatters] +keys = simpleFormatter + +[logger_root] +level = DEBUG +handlers = consoleHandler,fileHandler + +[logger_flow_processor] +level = DEBUG +handlers = consoleHandler,fileHandler +qualname = flow_processor +propagate = 0 + +[handler_consoleHandler] +class = StreamHandler +level = DEBUG +formatter = simpleFormatter +args = (sys.stdout,) + +[handler_fileHandler] +class = handlers.TimedRotatingFileHandler +level = DEBUG +formatter = simpleFormatter +args = ("/mnt/logs/flow_processor_ears_viirs.log", 'midnight', 1, 28, None, True, True) + +[formatter_simpleFormatter] +format = [%(levelname)s: %(asctime)s: %(name)s] %(message)s +datefmt = diff --git a/examples/docker/config_ears_viirs/gatherer.ini b/examples/docker/config_ears_viirs/gatherer.ini new file mode 100644 index 0000000..4d179a2 --- /dev/null +++ b/examples/docker/config_ears_viirs/gatherer.ini @@ -0,0 +1,17 @@ +[DEFAULT] +regions = euro4 + +[ears_viirs] +pattern = /mnt/input/SVMC_{platform_name}_d{start_date:%Y%m%d}_t{start_time:%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit_number:5d}_c{proctime:%Y%m%d%H%M%S%f}_eum_ops.h5 +format = SDR_compact +type = HDF5 +data_processing_level = 1B +platform_name = Suomi-NPP +sensor = viirs +timeliness = 30 +duration = 85.4 +variant = EARS +orbit_type = polar +service = +topics = /dataset/ears_viirs +publish_topic = /collection/ears_viirs diff --git a/examples/docker/config_ears_viirs/products.yaml b/examples/docker/config_ears_viirs/products.yaml new file mode 100644 index 0000000..99ae7f0 --- /dev/null +++ b/examples/docker/config_ears_viirs/products.yaml @@ -0,0 +1,42 @@ +common: + output_dir: &output_dir + /mnt/output/ + coverage_check: False + fname_pattern: &fname + "{time:%Y%m%d_%H%M}_{platform_name}_{areaname}_{productname}.{format}" + process_by_area: True + +formats: &formats + - format: tif + writer: geotiff + +sun_limit_day: &sunzen_day + 90.0 +sun_limit_night: &sunzen_night + 90.0 + +product_list: &product_list + overview: + productname: overview + output_dir: *output_dir + formats: *formats + sunzen_day_maximum: *sunzen_day + natural_color: + productname: natural_color + output_dir: *output_dir + formats: *formats + sunzen_day_maximum: *sunzen_day + night_overview: + productname: night_overview + output_dir: *output_dir + formats: *formats + sunzen_night_minimum: *sunzen_night + fog: + productname: fog + output_dir: *output_dir + formats: *formats + +product_list: + euro4: + areaname: euro4 + products: *product_list diff --git a/examples/docker/config_ears_viirs/segment_gatherer.yaml b/examples/docker/config_ears_viirs/segment_gatherer.yaml new file mode 100644 index 0000000..15b9cfd --- /dev/null +++ b/examples/docker/config_ears_viirs/segment_gatherer.yaml @@ -0,0 +1,22 @@ +patterns: + msg: + pattern: + "SV{segment}C_{orig_platform_name}_d{start_time:%Y%m%d_t%H%M%S}{start_decimal:1d}_e{end_time:%H%M%S}{end_decimal:1d}_b{orbit_number:5d}_c{proctime:s}_eum_ops.h5" + critical_files: null + wanted_files: :M,:DNB + all_files: :M,:DNB + variable_tags: + - proctime + +timeliness: + 20 + +time_name: + start_time + +posttroll: + topics: + - /segment/ears_viirs + publish_topic: + /dataset/ears_viirs + diff --git a/examples/docker/config_ears_viirs/supervisord.conf b/examples/docker/config_ears_viirs/supervisord.conf new file mode 100644 index 0000000..4a442a0 --- /dev/null +++ b/examples/docker/config_ears_viirs/supervisord.conf @@ -0,0 +1,61 @@ +[inet_http_server] +port = *:9001 +username = username +password = password + +[supervisord] +logfile = /mnt/logs/supervisord.log +logfile_maxbytes = 50MB +logfile_backups = 10 +loglevel = debug +pidfile = /mnt/config/supervisord.pid +nodaemon = true +minfds = 1024 +minprocs = 200 +environment = PPP_CONFIG_DIR="/mnt/config/" + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl = http://127.0.0.1:9001 + +[program:nameserver] +command = /opt/conda/bin/nameserver +redirect_stderr = true +priority = 1 +stdout_logfile = /mnt/logs/supervisor_nameserver_ears_viirs.log +startsecs = 10 +stopsignal = KILL + +[program:trollstalker] +command = /opt/conda/bin/python /opt/conda/bin/trollstalker.py -c /mnt/config/trollstalker.ini -C ears_viirs +redirect_stderr = true +stdout_logfile = /mnt/logs/supervisor_trollstalker-ears_viirs.log +stopsignal = KILL + +[program:flow_processor] +command = /opt/conda/bin/python /opt/conda/bin/flow_processor.py /mnt/config/flow_processor.yaml +redirect_stderr = true +stdout_logfile = /mnt/logs/supervisor_flow_processor_ears_viirs.log +stopsignal = KILL + +[program:segment_gatherer] +command = /opt/conda/bin/python /opt/conda/bin/segment_gatherer.py -c /mnt/config/segment_gatherer.yaml -v -l /mnt/logs/segment_gatherer_ears_viirs.log +redirect_stderr = true +stdout_logfile = /mnt/logs/supervisor_segment_gatherer_ears_viirs.log +stopsignal = KILL ; signal used to kill process (default TERM) + +[program:gatherer] +command = /opt/conda/bin/python /opt/conda/bin/gatherer.py -v -l /mnt/logs/gatherer_ears_viirs.log /mnt/config/gatherer.ini +redirect_stderr = true +stdout_logfile = /mnt/logs/supervisor_gatherer_ears_viirs-stdout.log +stopsignal = KILL + +;[eventlistener:crashmailbatch] +;command = /opt/conda/bin/crashmailbatch --toEmail="someone@somewhere.com" --fromEmail="supervisord@ears_viirs.test.com" -s "Crash alert from supervisord" +;events = PROCESS_STATE,TICK_60 + +;[eventlistener:fatalmailbatch] +;command = /opt/conda/bin/fatalmailbatch --toEmail="someone@somewhere.com" --fromEmail="supervisord@ears_viirs.test.com" -s "Fatal start alert from supervisord" +;events = PROCESS_STATE,TICK_60 diff --git a/examples/docker/config_ears_viirs/trollstalker.ini b/examples/docker/config_ears_viirs/trollstalker.ini new file mode 100644 index 0000000..10d339d --- /dev/null +++ b/examples/docker/config_ears_viirs/trollstalker.ini @@ -0,0 +1,11 @@ +[ears_viirs] +topic = /segment/ears_viirs +directory = /mnt/input +publish_port = +event_names = IN_CLOSE_WRITE,IN_MOVED_TO +stalker_log_config = /mnt/config/trollstalker_logging.ini +loglevel = DEBUG +filepattern = {path}SV{band}C_{platform_name}_d{start_date:%Y%m%d}_t{start_time:%H%M%S}{start_decimal:1d}_e{end_time:%H%M%S}{end_decimal:1d}_b{orbit_number:5d}_c{proctime:%Y%m%d%H%M%S}{proc_decimal:d}_eum_ops.h5 +instruments = viirs +alias_platform_name = npp:Suomi-NPP +history = 10 diff --git a/examples/docker/config_ears_viirs/trollstalker_logging.ini b/examples/docker/config_ears_viirs/trollstalker_logging.ini new file mode 100644 index 0000000..d8bedfd --- /dev/null +++ b/examples/docker/config_ears_viirs/trollstalker_logging.ini @@ -0,0 +1,34 @@ +[loggers] +keys = root,trollstalker + +[handlers] +keys = consoleHandler,fileHandler + +[formatters] +keys = simpleFormatter + +[logger_root] +level = DEBUG +handlers = consoleHandler,fileHandler + +[logger_trollstalker] +level = DEBUG +handlers = consoleHandler,fileHandler +qualname = trollstalker +propagate = 0 + +[handler_consoleHandler] +class = StreamHandler +level = DEBUG +formatter = simpleFormatter +args = (sys.stdout,) + +[handler_fileHandler] +class = handlers.TimedRotatingFileHandler +level = DEBUG +formatter = simpleFormatter +args = ("/mnt/logs/trollstalker_ears_viirs.log", 'midnight', 1, 7, None, True, True) + +[formatter_simpleFormatter] +format = [%(levelname)s: %(asctime)s: %(name)s] %(message)s +datefmt = diff --git a/examples/docker/config_seviri_hrit/README b/examples/docker/config_seviri_hrit/README new file mode 100644 index 0000000..d356103 --- /dev/null +++ b/examples/docker/config_seviri_hrit/README @@ -0,0 +1,34 @@ +This directory contains an example production chain setup for MSG +SEVIRI HRIT files that have been already decompressed. The +decompression isn't included, as the xRITDecompress software needs to +be downloaded personnally from Eumetsat. + +supervisord.conf +---------------- + +Configuration file for Supervisord, which starts all the processing steps. + +trollstalker.ini and trollstalker_logging.ini +--------------------------------------------- + +Configuration file for Trollstalker, which creates posttroll messages +for incoming files, and logging config for it. + +segment_gatherer.yaml +--------------------- + +Configuration file for segment_gatherer, which collects the messages +coming from trollstalker to one message that contains metadata of all +the required files to produce the images. + +flow_processor.yaml and flow_processor_logging.ini +-------------------------------------------------- + +Definition of the processing chain and its logging: creation of +composites, resampling to target area, writing to files. + +products.yaml +------------- + +Definition of the products to be generated: composites, target areas, +file formats, filename patterns. diff --git a/examples/docker/config_seviri_hrit/flow_processor.yaml b/examples/docker/config_seviri_hrit/flow_processor.yaml new file mode 100644 index 0000000..38730e8 --- /dev/null +++ b/examples/docker/config_seviri_hrit/flow_processor.yaml @@ -0,0 +1,63 @@ +config: + product_list: &product_list + /mnt/config/products.yaml + log_config: + /mnt/config/flow_processor_logging.ini + use_utc: True + force_gc: Tue + coverage_check: False + +work: + - type: daemon + name: listener + components: + - class: !!python/object:posttroll.listener.ListenerContainer + topics: + - /dataset/msg0deg + + - type: workflow + name: satpy_compositor + Workflow: + - trollflow_sat.satpy_compositor.SceneLoader: + product_list: *product_list + use_lock: True + instruments: + - seviri + readers: + - seviri_l1b_hrit + service: + msg0deg + + - type: workflow + name: satpy_resampler + Workflow: + - trollflow_sat.satpy_resampler.Resampler: + # Use "null" instead of "None" + # for nearest neighbour + radius: 15000 + # for bilinear + # radius: 50000 + # cache_dir: /mnt/resample_cache/ + resampler: nearest + # resampler: bilinear + mask_area: False + product_list: *product_list + use_lock: True + process_by_area: True + + - type: daemon + name: satpy_writer + components: + - class: !!python/object:trollflow_sat.satpy_writer.DataWriterContainer + nameservers: null + port: 0 + topic: "/image/msg0deg/{productname}" + use_lock: True + save_settings: + tile: True + blockxsize: 512 + blockysize: 512 + compress: deflate + # Overviews (reduced resolution images) created from the full + # resolution data and embedded in the saved (GeoTIFF) files + # overviews: [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] diff --git a/examples/docker/config_seviri_hrit/flow_processor_logging.ini b/examples/docker/config_seviri_hrit/flow_processor_logging.ini new file mode 100644 index 0000000..b5d38d6 --- /dev/null +++ b/examples/docker/config_seviri_hrit/flow_processor_logging.ini @@ -0,0 +1,34 @@ +[loggers] +keys = root,flow_processor + +[handlers] +keys = consoleHandler,fileHandler + +[formatters] +keys = simpleFormatter + +[logger_root] +level = DEBUG +handlers = consoleHandler,fileHandler + +[logger_flow_processor] +level = DEBUG +handlers = consoleHandler,fileHandler +qualname = flow_processor +propagate = 0 + +[handler_consoleHandler] +class = StreamHandler +level = DEBUG +formatter = simpleFormatter +args = (sys.stdout,) + +[handler_fileHandler] +class = handlers.TimedRotatingFileHandler +level = DEBUG +formatter = simpleFormatter +args = ("/mnt/logs/flow_processor_msg0deg.log", 'midnight', 1, 28, None, True, True) + +[formatter_simpleFormatter] +format = [%(levelname)s: %(asctime)s: %(name)s] %(message)s +datefmt = diff --git a/examples/docker/config_seviri_hrit/products.yaml b/examples/docker/config_seviri_hrit/products.yaml new file mode 100644 index 0000000..7d756dc --- /dev/null +++ b/examples/docker/config_seviri_hrit/products.yaml @@ -0,0 +1,42 @@ +common: + output_dir: &output_dir + /mnt/output/ + coverage_check: False + fname_pattern: &fname + "{time:%Y%m%d_%H%M}_{platform_name}_{areaname}_{productname}.{format}" + process_by_area: True + +formats: &formats + - format: tif + writer: geotiff + +sun_limit_day: &sunzen_day + 90.0 +sun_limit_night: &sunzen_night + 90.0 + +product_list: &product_list + overview: + productname: overview + output_dir: *output_dir + formats: *formats + sunzen_day_maximum: *sunzen_day + day_microphysics: + productname: day_microphysics + output_dir: *output_dir + formats: *formats + sunzen_day_maximum: *sunzen_day + airmass: + productname: airmass + output_dir: *output_dir + formats: *formats + night_fog: + productname: night_fog + output_dir: *output_dir + formats: *formats + sunzen_night_minimum: *sunzen_night + +product_list: + euro4: + areaname: euro4 + products: *product_list diff --git a/examples/docker/config_seviri_hrit/segment_gatherer.yaml b/examples/docker/config_seviri_hrit/segment_gatherer.yaml new file mode 100644 index 0000000..0966a22 --- /dev/null +++ b/examples/docker/config_seviri_hrit/segment_gatherer.yaml @@ -0,0 +1,21 @@ +patterns: + msg: + pattern: + "H-000-{orig_platform_name:4s}__-{orig_platform_name:4s}________-{channel_name:_<9s}-{segment:_<9s}-{start_time:%Y%m%d%H%M}-__" + critical_files: :EPI,:PRO + wanted_files: VIS006:000001-000008,VIS008:000001-000008,IR_016:000001-000008,IR_039:000001-000008,WV_062:000001-000008,WV_073:000001-000008,IR_087:000001-000008,IR_097:000001-000008,IR_108:000001-000008,IR_120:000001-000008,IR_134:000001-000008,HRV:000001-000024,:PRO,:EPI + all_files: VIS006:000001-000008,VIS008:000001-000008,IR_016:000001-000008,IR_039:000001-000008,WV_062:000001-000008,WV_073:000001-000008,IR_087:000001-000008,IR_097:000001-000008,IR_108:000001-000008,IR_120:000001-000008,IR_134:000001-000008,HRV:000001-000024,:PRO,:EPI + variable_tags: [] + +timeliness: + 20 + +time_name: + start_time + +posttroll: + topics: + - /segment/msg0deg + publish_topic: + /dataset/msg0deg + diff --git a/examples/docker/config_seviri_hrit/supervisord.conf b/examples/docker/config_seviri_hrit/supervisord.conf new file mode 100644 index 0000000..7d9c2f0 --- /dev/null +++ b/examples/docker/config_seviri_hrit/supervisord.conf @@ -0,0 +1,55 @@ +[inet_http_server] +port = *:9001 +username = username +password = password + +[supervisord] +logfile = /mnt/logs/supervisord.log +logfile_maxbytes = 50MB +logfile_backups = 10 +loglevel = debug +pidfile = /mnt/config/supervisord.pid +nodaemon = true +minfds = 1024 +minprocs = 200 +environment = PPP_CONFIG_DIR="/mnt/config/" + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl = http://127.0.0.1:9001 + +[program:nameserver] +command = /opt/conda/bin/nameserver +redirect_stderr = true +priority = 1 +stdout_logfile = /mnt/logs/supervisor_nameserver_msg0deg.log +startsecs = 10 +stopsignal = KILL + +[program:trollstalker] +command = /opt/conda/bin/python /opt/conda/bin/trollstalker.py -c /mnt/config/trollstalker.ini -C msg_0_deg +redirect_stderr = true +stdout_logfile = /mnt/logs/supervisor_trollstalker-msg.log +stopsignal = KILL + +[program:flow_processor] +command = /opt/conda/bin/python /opt/conda/bin/flow_processor.py /mnt/config/flow_processor.yaml +redirect_stderr = true +stdout_logfile = /mnt/logs/supervisor_flow_processor_msg0deg.log +stopsignal = KILL + +[program:segment_gatherer] +command = /opt/conda/bin/python /opt/conda/bin/segment_gatherer.py -c /mnt/config/segment_gatherer.yaml -v -l /mnt/logs/segment_gatherer_msg0deg.log +redirect_stderr = true +stdout_logfile = /mnt/logs/supervisor_segment_gatherer_msg0deg.log +stopsignal = KILL ; signal used to kill process (default TERM) + +;[eventlistener:crashmailbatch] +;command = /opt/conda/bin/crashmailbatch --toEmail="someone@somewhere.com" --fromEmail="supervisord@msg0deg.test.com" -s "Crash alert from supervisord" +;events = PROCESS_STATE,TICK_60 + +;[eventlistener:fatalmailbatch] +;command = /opt/conda/bin/fatalmailbatch --toEmail="someone@somewhere.com" --fromEmail="supervisord@msg0deg.test.com" -s "Fatal start alert from supervisord" +;events = PROCESS_STATE,TICK_60 diff --git a/examples/docker/config_seviri_hrit/trollstalker.ini b/examples/docker/config_seviri_hrit/trollstalker.ini new file mode 100644 index 0000000..0ecbaae --- /dev/null +++ b/examples/docker/config_seviri_hrit/trollstalker.ini @@ -0,0 +1,11 @@ +[msg_0_deg] +topic = /segment/msg0deg +directory = /mnt/input +publish_port = +event_names = IN_CLOSE_WRITE,IN_MOVED_TO +stalker_log_config = /mnt/config/trollstalker_logging.ini +loglevel = DEBUG +filepattern = {path}H-000-{orig_platform_name:_<6s}-{platform_name}________-{channel_name:_<9s}-{segment:_<9s}-{nominal_time:%Y%m%d%H%M}-__ +instruments = seviri +alias_platform_name = MSG3:Meteosat-10|MSG1:Meteosat-8|MSG2:Meteosat-9|MSG4:Meteosat-11 +history = 10 diff --git a/examples/docker/config_seviri_hrit/trollstalker_logging.ini b/examples/docker/config_seviri_hrit/trollstalker_logging.ini new file mode 100644 index 0000000..e5f8c0c --- /dev/null +++ b/examples/docker/config_seviri_hrit/trollstalker_logging.ini @@ -0,0 +1,34 @@ +[loggers] +keys = root,trollstalker + +[handlers] +keys = consoleHandler,fileHandler + +[formatters] +keys = simpleFormatter + +[logger_root] +level = DEBUG +handlers = consoleHandler,fileHandler + +[logger_trollstalker] +level = DEBUG +handlers = consoleHandler,fileHandler +qualname = trollstalker +propagate = 0 + +[handler_consoleHandler] +class = StreamHandler +level = DEBUG +formatter = simpleFormatter +args = (sys.stdout,) + +[handler_fileHandler] +class = handlers.TimedRotatingFileHandler +level = DEBUG +formatter = simpleFormatter +args = ("/mnt/logs/trollstalker_msg0deg.log", 'midnight', 1, 7, None, True, True) + +[formatter_simpleFormatter] +format = [%(levelname)s: %(asctime)s: %(name)s] %(message)s +datefmt = diff --git a/examples/docker/input/README b/examples/docker/input/README new file mode 100644 index 0000000..1d9ccb3 --- /dev/null +++ b/examples/docker/input/README @@ -0,0 +1 @@ +This is the directory for new incoming satellite data diff --git a/examples/docker/logs/README b/examples/docker/logs/README new file mode 100644 index 0000000..657e139 --- /dev/null +++ b/examples/docker/logs/README @@ -0,0 +1 @@ +Log files will be saved here diff --git a/examples/docker/output/README b/examples/docker/output/README new file mode 100644 index 0000000..4785fb7 --- /dev/null +++ b/examples/docker/output/README @@ -0,0 +1 @@ +Processed images will be saved here