diff --git a/api/py/ai/chronon/repo/run.py b/api/py/ai/chronon/repo/run.py index 88036b095..6c9f5628d 100755 --- a/api/py/ai/chronon/repo/run.py +++ b/api/py/ai/chronon/repo/run.py @@ -237,6 +237,7 @@ def set_runtime_env(args): - Environment variables derived from args (like app_name) - conf.metaData.modeToEnvMap for the mode (set on config) - team environment per context and mode set on teams.json + - production team environment per mode set on teams.json - default team environment per context and mode set on teams.json - Common Environment set in teams.json """ @@ -245,6 +246,7 @@ def set_runtime_env(args): "conf_env": {}, "default_env": {}, "team_env": {}, + "production_team_env ": {}, "cli_args": {}, } conf_type = None @@ -262,7 +264,7 @@ def set_runtime_env(args): ) if args.conf and effective_mode: try: - context, conf_type, team, _ = args.conf.split("/")[-4:] + _, conf_type, team, _ = args.conf.split("/")[-4:] except Exception as e: logging.error( "Invalid conf path: {}, please ensure to supply the relative path to zipline/ folder".format( @@ -272,6 +274,9 @@ def set_runtime_env(args): raise e if not team: team = "default" + # context is the environment in which the job is running, which is provided from the args, + # default to be dev. + context = args.env logging.info( f"Context: {context} -- conf_type: {conf_type} -- team: {team}" ) @@ -298,6 +303,11 @@ def set_runtime_env(args): environment["team_env"] = ( teams_json[team].get(context, {}).get(effective_mode, {}) ) + # If the job is running in dev environment but no dev environment is defined in teams.json, + # use production environment. + environment["production_team_env"] = ( + teams_json[team].get("production", {}).get(effective_mode, {}) + ) environment["default_env"] = ( teams_json.get("default", {}) .get(context, {}) @@ -326,7 +336,7 @@ def set_runtime_env(args): environment["cli_args"]["CHRONON_DRIVER_JAR"] = args.chronon_jar environment["cli_args"]["CHRONON_ONLINE_JAR"] = args.online_jar environment["cli_args"]["CHRONON_ONLINE_CLASS"] = args.online_class - order = ["conf_env", "team_env", "default_env", "common_env", "cli_args"] + order = ["conf_env", "team_env", "production_team_env", "default_env", "common_env", "cli_args"] print("Setting env variables:") for key in os.environ: if any([key in environment[set_key] for set_key in order]): @@ -568,6 +578,12 @@ def set_defaults(parser): required=False, help="Conf param - required for every mode except fetch", ) + parser.add_argument( + "--env", + required=False, + default='dev', + help="Running environment - default to be dev" + ) parser.add_argument("--mode", choices=MODE_ARGS.keys()) parser.add_argument("--ds", help="the end partition to backfill the data") parser.add_argument(