diff --git a/.github/workflows/process_challenge.yml b/.github/workflows/process_challenge.yml new file mode 100644 index 0000000..f3964c8 --- /dev/null +++ b/.github/workflows/process_challenge.yml @@ -0,0 +1,38 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: evalai-challenge +on: + push: + branches: [challenge] + pull_request: + types: [opened, synchronize, reopened, edited] + branches: [challenge] +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7.5 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f github/requirements.txt ]; then pip install -r github/requirements.txt; fi + - name: Validate challenge + run: | + python3 github/challenge_processing_script.py + env: + IS_VALIDATION: 'True' + GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} + - name: Create or update challenge + run: | + python3 github/challenge_processing_script.py + if: ${{github.event_name == 'push' && success()}} + env: + IS_VALIDATION: 'False' + GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d9c78c8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +challenge_config.zip +evaluation_script.zip + +# virtualenv +env/ +venv/ + +# text-editor related files +.vscode/ + +# cache +__pycache__ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..d539c08 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +## How to create a challenge on EvalAI? + +If you are looking for a simple challenge configuration that you can replicate to create a challenge on EvalAI, then you are at the right place. Follow the instructions given below to get started. + +## Directory Structure + +``` +. +├── README.md +├── annotations # Contains the annotations for Dataset splits +│   ├── test_annotations_devsplit.json # Annotations of dev split +│   └── test_annotations_testsplit.json # Annotations for test split +├── challenge_data # Contains scripts to test the evalautaion script locally +│   ├── challenge_1 # Contains evaluation script for the challenge +| ├── __init__.py # Imports the main.py file for evaluation +|    └── main.py # Challenge evaluation script +│   └── __init__.py # Imports the modules which involve evaluation script loading +├── challenge_config.yaml # Configuration file to define challenge setup +├── evaluation_script # Contains the evaluation script +│   ├── __init__.py # Imports the modules that involve annotations loading etc +│   └── main.py # Contains the main `evaluate()` method +├── logo.jpg # Logo image of the challenge +├── submission.json # Sample submission file +├── run.sh # Script to create the challenge configuration zip to be uploaded on EvalAI website +└── templates # Contains challenge related HTML templates + ├── challenge_phase_1_description.html # Challenge Phase 1 description template + ├── challenge_phase_2_description.html # Challenge Phase 2 description template + ├── description.html # Challenge description template + ├── evaluation_details.html # Contains description about how submissions will be evalauted for each challenge phase + ├── submission_guidelines.html # Contains information about how to make submissions to the challenge + └── terms_and_conditions.html # Contains terms and conditions related to the challenge +├── worker # Contains the scripts to test evaluation script locally +│   ├── __init__.py # Imports the module that ionvolves loading evaluation script +│   └── run.py # Contains the code to run the evaluation locally +``` + +## Create challenge using github + +1. Use this repository as [template](https://docs.github.com/en/free-pro-team@latest/github/creating-cloning-and-archiving-repositories/creating-a-repository-from-a-template). + +2. Generate your [github personal acccess token](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) and copy it in clipboard. + +3. Add the github personal access token in the forked repository's [secrets](https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-a-repository) with the name `AUTH_TOKEN`. + +4. Now, go to [EvalAI](https://eval.ai) to fetch the following details - + 1. `evalai_user_auth_token` - Go to [profile page](https://eval.ai/web/profile) after logging in and click on `Get your Auth Token` to copy your auth token. + 2. `host_team_pk` - Go to [host team page](https://eval.ai/web/challenge-host-teams) and copy the `ID` for the team you want to use for challenge creation. + 3. `evalai_host_url` - Use `https://eval.ai` for production server and `https://staging.eval.ai` for staging server. + +5. Create a branch with name `challenge` in the forked repository from the `master` branch. +Note: Only changes in `challenge` branch will be synchronized with challenge on EvalAI. + +6. Add `evalai_user_auth_token` and `host_team_pk` in `github/host_config.json`. + +7. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need. + +8. Commit the changes and push the `challenge` branch in the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). + +9. If challenge config contains errors then a `issue` will be opened automatically in the repository with the errors otherwise the challenge will be created on EvalAI. + +10. Go to [Hosted Challenges](https://eval.ai/web/hosted-challenges) to view your challenge. The challenge will be publicly available once EvalAI admin approves the challenge. + +11. To update the challenge on EvalAI, make changes in the repository and push on `challenge` branch and wait for the build to complete. + +## Create challenge using config + +1. Fork this repository. + +2. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need. + +3. Once you are done making changes, run the command `./run.sh` to generate the `challenge_config.zip`. + +4. Upload the `challenge_config.zip` on [EvalAI](https://eval.ai) to create a challenge on EvalAI. Challenge will be available publicly once EvalAI Admin approves the challenge. + +5. To update the challenge on EvalAI, use the UI to update the details. + +## Test your evaluation script locally + +In order to test the evaluation script locally before uploading it to [EvalAI](https://eval.ai) server, please follow the below instructions - + +1. Copy the evaluation script i.e `__init__.py` , `main.py` and other relevant files from `evaluation_script/` directory to `challenge_data/challenge_1/` directory. + +2. Now, edit `challenge_phase` name, `annotation file` name and `submission file` name in the `worker/run.py` file to the challenge phase codename (which you want to test for), annotation file name in the `annotations/` folder (for specific phase) and corresponding submission file respectively. + +3. Run the command `python -m worker.run` from the directory where `annotations/` `challenge_data/` and `worker/` directories are present. If the command runs successfully, then the evaluation script works locally and will work on the server as well. + +## Facing problems in creating a challenge? + +Please feel free to open issues on our [GitHub Repository](https://github.com/Cloud-CV/EvalAI-Starter/issues) or contact us at team@cloudcv.org if you have issues. diff --git a/annotations/test_annotations_devsplit.json b/annotations/test_annotations_devsplit.json new file mode 100644 index 0000000..f2aed13 --- /dev/null +++ b/annotations/test_annotations_devsplit.json @@ -0,0 +1,3 @@ +{ + "foo": "bar" +} \ No newline at end of file diff --git a/annotations/test_annotations_testsplit.json b/annotations/test_annotations_testsplit.json new file mode 100644 index 0000000..f2aed13 --- /dev/null +++ b/annotations/test_annotations_testsplit.json @@ -0,0 +1,3 @@ +{ + "foo": "bar" +} \ No newline at end of file diff --git a/challenge_config.yaml b/challenge_config.yaml new file mode 100755 index 0000000..ede451a --- /dev/null +++ b/challenge_config.yaml @@ -0,0 +1,145 @@ +# If you are not sure what all these fields mean, please refer our documentation here: +# https://evalai.readthedocs.io/en/latest/configuration.html +title: Random Number Generator Challenge +short_description: Random number generation challenge for each submission +description: templates/description.html +evaluation_details: templates/evaluation_details.html +terms_and_conditions: templates/terms_and_conditions.html +image: logo.jpg +submission_guidelines: templates/submission_guidelines.html +leaderboard_description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras egestas a libero nec sagittis. +evaluation_script: evaluation_script.zip +remote_evaluation: False +is_docker_based: False +start_date: 2019-01-01 00:00:00 +end_date: 2099-05-31 23:59:59 +published: True + +leaderboard: + - id: 1 + schema: + { + "labels": ["Metric1", "Metric2", "Metric3", "Total"], + "default_order_by": "Total", + "metadata": { + "Metric1": { + "sort_ascending": True, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + }, + "Metric2": { + "sort_ascending": True, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + } + } + } + +challenge_phases: + - id: 1 + name: Dev Phase + description: templates/challenge_phase_1_description.html + leaderboard_public: False + is_public: True + is_submission_public: True + start_date: 2019-01-19 00:00:00 + end_date: 2099-04-25 23:59:59 + test_annotation_file: annotations/test_annotations_devsplit.json + codename: dev + max_submissions_per_day: 5 + max_submissions_per_month: 50 + max_submissions: 50 + default_submission_meta_attributes: + - name: method_name + is_visible: True + - name: method_description + is_visible: True + - name: project_url + is_visible: True + - name: publication_url + is_visible: True + submission_meta_attributes: + - name: TextAttribute + description: Sample + type: text + required: False + - name: SingleOptionAttribute + description: Sample + type: radio + options: ["A", "B", "C"] + - name: MultipleChoiceAttribute + description: Sample + type: checkbox + options: ["alpha", "beta", "gamma"] + - name: TrueFalseField + description: Sample + type: boolean + required: True + is_restricted_to_select_one_submission: False + is_partial_submission_evaluation_enabled: False + allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz" + - id: 2 + name: Test Phase + description: templates/challenge_phase_2_description.html + leaderboard_public: True + is_public: True + is_submission_public: True + start_date: 2019-01-01 00:00:00 + end_date: 2099-05-24 23:59:59 + test_annotation_file: annotations/test_annotations_testsplit.json + codename: test + max_submissions_per_day: 5 + max_submissions_per_month: 50 + max_submissions: 50 + default_submission_meta_attributes: + - name: method_name + is_visible: True + - name: method_description + is_visible: True + - name: project_url + is_visible: True + - name: publication_url + is_visible: True + submission_meta_attributes: + - name: TextAttribute + description: Sample + type: text + - name: SingleOptionAttribute + description: Sample + type: radio + options: ["A", "B", "C"] + - name: MultipleChoiceAttribute + description: Sample + type: checkbox + options: ["alpha", "beta", "gamma"] + - name: TrueFalseField + description: Sample + type: boolean + is_restricted_to_select_one_submission: False + is_partial_submission_evaluation_enabled: False + +dataset_splits: + - id: 1 + name: Train Split + codename: train_split + - id: 2 + name: Test Split + codename: test_split + +challenge_phase_splits: + - challenge_phase_id: 1 + leaderboard_id: 1 + dataset_split_id: 1 + visibility: 1 + leaderboard_decimal_precision: 2 + is_leaderboard_order_descending: True + - challenge_phase_id: 2 + leaderboard_id: 1 + dataset_split_id: 1 + visibility: 3 + leaderboard_decimal_precision: 2 + is_leaderboard_order_descending: True + - challenge_phase_id: 2 + leaderboard_id: 1 + dataset_split_id: 2 + visibility: 1 + leaderboard_decimal_precision: 2 + is_leaderboard_order_descending: True diff --git a/challenge_data/__init__.py b/challenge_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/challenge_data/challenge_1/__init__.py b/challenge_data/challenge_1/__init__.py new file mode 100644 index 0000000..f543866 --- /dev/null +++ b/challenge_data/challenge_1/__init__.py @@ -0,0 +1 @@ +from .main import evaluate diff --git a/challenge_data/challenge_1/main.py b/challenge_data/challenge_1/main.py new file mode 100644 index 0000000..1c15565 --- /dev/null +++ b/challenge_data/challenge_1/main.py @@ -0,0 +1,83 @@ +import random + + +def evaluate(test_annotation_file, user_submission_file, phase_codename, **kwargs): + print("Starting Evaluation.....") + print("Submission related metadata:") + """ + Evaluates the submission for a particular challenge phase adn returns score + Arguments: + + `test_annotations_file`: Path to test_annotation_file on the server + `user_submission_file`: Path to file submitted by the user + `phase_codename`: Phase to which submission is made + + `**kwargs`: keyword arguments that contains additional submission + metadata that challenge hosts can use to send slack notification. + You can access the submission metadata + with kwargs['submission_metadata'] + + Example: A sample submission metadata can be accessed like this: + >>> print(kwargs['submission_metadata']) + { + "status": u"running", + "when_made_public": None, + "participant_team": 5, + "input_file": "https://abc.xyz/path/to/submission/file.json", + "execution_time": u"123", + "publication_url": u"ABC", + "challenge_phase": 1, + "created_by": u"ABC", + "stdout_file": "https://abc.xyz/path/to/stdout/file.json", + "method_name": u"Test", + "stderr_file": "https://abc.xyz/path/to/stderr/file.json", + "participant_team_name": u"Test Team", + "project_url": u"http://foo.bar", + "method_description": u"ABC", + "is_public": False, + "submission_result_file": "https://abc.xyz/path/result/file.json", + "id": 123, + "submitted_at": u"2017-03-20T19:22:03.880652Z", + } + """ + print(kwargs["submission_metadata"]) + output = {} + if phase_codename == "dev": + print("Evaluating for Dev Phase") + output["result"] = [ + { + "train_split": { + "Metric1": random.randint(0, 99), + "Metric2": random.randint(0, 99), + "Metric3": random.randint(0, 99), + "Total": random.randint(0, 99), + } + } + ] + # To display the results in the result file + output["submission_result"] = output["result"][0]["train_split"] + print("Completed evaluation for Dev Phase") + elif phase_codename == "test": + print("Evaluating for Test Phase") + output["result"] = [ + { + "train_split": { + "Metric1": random.randint(0, 99), + "Metric2": random.randint(0, 99), + "Metric3": random.randint(0, 99), + "Total": random.randint(0, 99), + } + }, + { + "test_split": { + "Metric1": random.randint(0, 99), + "Metric2": random.randint(0, 99), + "Metric3": random.randint(0, 99), + "Total": random.randint(0, 99), + } + }, + ] + # To display the results in the result file + output["submission_result"] = output["result"][0] + print("Completed evaluation for Test Phase") + return output diff --git a/code_upload_challenge_evaluation/agent/agent.py b/code_upload_challenge_evaluation/agent/agent.py new file mode 100644 index 0000000..f3ea4a6 --- /dev/null +++ b/code_upload_challenge_evaluation/agent/agent.py @@ -0,0 +1,38 @@ +import evaluation_pb2 +import evaluation_pb2_grpc +import grpc +import os +import pickle +import time + +time.sleep(30) + +LOCAL_EVALUATION = os.environ.get("LOCAL_EVALUATION") + +if LOCAL_EVALUATION: + channel = grpc.insecure_channel("environment:8085") +else: + channel = grpc.insecure_channel("localhost:8085") + +stub = evaluation_pb2_grpc.EnvironmentStub(channel) + + +def pack_for_grpc(entity): + return pickle.dumps(entity) + + +def unpack_for_grpc(entity): + return pickle.loads(entity) + + +flag = None + +while not flag: + base = unpack_for_grpc( + stub.act_on_environment( + evaluation_pb2.Package(SerializedEntity=pack_for_grpc(1)) + ).SerializedEntity + ) + flag = base["feedback"][2] + print("Agent Feedback", base["feedback"]) + print("*" * 100) diff --git a/code_upload_challenge_evaluation/docker-compose.yml b/code_upload_challenge_evaluation/docker-compose.yml new file mode 100644 index 0000000..6b27dff --- /dev/null +++ b/code_upload_challenge_evaluation/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3' +services: + environment: + hostname: environment + env_file: + - docker/environment/docker.env + build: + context: ./ + dockerfile: docker/environment/Dockerfile + ports: + - '8085:8085' + + agent: + hostname: agent + env_file: + - docker/agent/docker.env + build: + context: ./ + dockerfile: docker/agent/Dockerfile diff --git a/code_upload_challenge_evaluation/docker/agent/Dockerfile b/code_upload_challenge_evaluation/docker/agent/Dockerfile new file mode 100644 index 0000000..35e7093 --- /dev/null +++ b/code_upload_challenge_evaluation/docker/agent/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.7.5 + +ENV PYTHONUNBUFFERED 1 +ADD ./agent / +ADD ./utils / +ADD requirements/agent.txt / +RUN pip install --upgrade pip +RUN pip install -r agent.txt +CMD [ "python", "agent.py" ] diff --git a/code_upload_challenge_evaluation/docker/agent/docker.env b/code_upload_challenge_evaluation/docker/agent/docker.env new file mode 100644 index 0000000..5406ab9 --- /dev/null +++ b/code_upload_challenge_evaluation/docker/agent/docker.env @@ -0,0 +1 @@ +LOCAL_EVALUATION = True diff --git a/code_upload_challenge_evaluation/docker/environment/Dockerfile b/code_upload_challenge_evaluation/docker/environment/Dockerfile new file mode 100644 index 0000000..edf8857 --- /dev/null +++ b/code_upload_challenge_evaluation/docker/environment/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.7.5 + +ENV PYTHONUNBUFFERED 1 +ADD ./environment / +ADD ./utils / +ADD requirements/environment.txt / +RUN pip install --upgrade pip +RUN pip install -r environment.txt +CMD ["python", "environment.py"] diff --git a/code_upload_challenge_evaluation/docker/environment/docker.env b/code_upload_challenge_evaluation/docker/environment/docker.env new file mode 100644 index 0000000..50a18dd --- /dev/null +++ b/code_upload_challenge_evaluation/docker/environment/docker.env @@ -0,0 +1,4 @@ +AUTH_TOKEN=x +EVALAI_API_SERVER=https://eval.ai +LOCAL_EVALUATION = True +QUEUE_NAME=x diff --git a/code_upload_challenge_evaluation/environment/environment.py b/code_upload_challenge_evaluation/environment/environment.py new file mode 100644 index 0000000..0449564 --- /dev/null +++ b/code_upload_challenge_evaluation/environment/environment.py @@ -0,0 +1,145 @@ +import grpc +import gym +import pickle +import sys +import os +import requests +import json + +from environment_utils import EvalAI_Interface + +from concurrent import futures +import time + +import evaluation_pb2 +import evaluation_pb2_grpc + +LOCAL_EVALUATION = os.environ.get("LOCAL_EVALUATION") +EVALUATION_COMPLETED = False + + +class evaluator_environment: + def __init__(self, environment="CartPole-v0"): + self.score = 0 + self.feedback = None + self.env = gym.make(environment) + self.env.reset() + + def get_action_space(self): + return list(range(self.env.action_space.n)) + + def next_score(self): + self.score += 1 + + +class Environment(evaluation_pb2_grpc.EnvironmentServicer): + def __init__(self, challenge_pk, phase_pk, submission_pk, server): + self.challenge_pk = challenge_pk + self.phase_pk = phase_pk + self.submission_pk = submission_pk + self.server = server + + def get_action_space(self, request, context): + message = pack_for_grpc(env.get_action_space()) + return evaluation_pb2.Package(SerializedEntity=message) + + def act_on_environment(self, request, context): + global EVALUATION_COMPLETED + if not env.feedback or not env.feedback[2]: + action = unpack_for_grpc(request.SerializedEntity) + env.next_score() + env.feedback = env.env.step(action) + if env.feedback[2]: + if not LOCAL_EVALUATION: + update_submission_result( + env, self.challenge_pk, self.phase_pk, self.submission_pk + ) + else: + print("Final Score: {0}".format(env.score)) + print("Stopping Evaluation!") + EVALUATION_COMPLETED = True + return evaluation_pb2.Package( + SerializedEntity=pack_for_grpc( + {"feedback": env.feedback, "current_score": env.score,} + ) + ) + + +env = evaluator_environment() +api = EvalAI_Interface( + AUTH_TOKEN=os.environ.get("AUTH_TOKEN", "x"), + EVALAI_API_SERVER=os.environ.get("EVALAI_API_SERVER", "http://localhost:8000"), +) + + +def pack_for_grpc(entity): + return pickle.dumps(entity) + + +def unpack_for_grpc(entity): + return pickle.loads(entity) + + +def get_action_space(env): + return list(range(env.action_space.n)) + + +def update_submission_result(env, challenge_pk, phase_pk, submission_pk): + submission_data = { + "submission_status": "finished", + "submission": submission_pk, + } + submission_data = { + "challenge_phase": phase_pk, + "submission": submission_pk, + "stdout": "standard_ouput", + "stderr": "standard_error", + "submission_status": "FINISHED", + "result": json.dumps( + [ + { + "split": "train_split", + "show_to_participant": True, + "accuracies": {"score": env.score}, + } + ] + ), + } + api.update_submission_data(submission_data, challenge_pk) + print("Data updated successfully!") + EVALUATION_COMPLETED = True + exit(0) + + +def main(): + if not LOCAL_EVALUATION: + BODY = os.environ.get("BODY") + # Sample example for BODY + # BODY = "{'submitted_image_uri': '937891341272.dkr.ecr.us-east-1.amazonaws.com/cartpole-challenge-203-participant-team-265:bb55f57f-ae44-4e76-96c2-e1ebb5d7b65a', 'submission_pk': 1351, 'phase_pk': '527', 'challenge_pk': '203'}" + BODY = BODY.replace("'", '"') + BODY = json.loads(BODY) + challenge_pk = BODY["challenge_pk"] + phase_pk = BODY["phase_pk"] + submission_pk = BODY["submission_pk"] + else: + challenge_pk = "1" + phase_pk = "1" + submission_pk = "1" + + server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) + evaluation_pb2_grpc.add_EnvironmentServicer_to_server( + Environment(challenge_pk, phase_pk, submission_pk, server), server + ) + print("Starting server. Listening on port 8085.") + server.add_insecure_port("[::]:8085") + server.start() + try: + while not EVALUATION_COMPLETED: + time.sleep(4) + server.stop(0) + except KeyboardInterrupt: + server.stop(0) + + +if __name__ == "__main__": + main() diff --git a/code_upload_challenge_evaluation/environment/environment_utils.py b/code_upload_challenge_evaluation/environment/environment_utils.py new file mode 100644 index 0000000..ff3721e --- /dev/null +++ b/code_upload_challenge_evaluation/environment/environment_utils.py @@ -0,0 +1,44 @@ +import logging +import requests + + +URLS = { + "update_submission_data": "/api/jobs/challenge/{}/update_submission/", +} + + +class EvalAI_Interface: + def __init__(self, AUTH_TOKEN, EVALAI_API_SERVER): + self.AUTH_TOKEN = AUTH_TOKEN + self.EVALAI_API_SERVER = EVALAI_API_SERVER + + def get_request_headers(self): + headers = {"Authorization": "Bearer {}".format(self.AUTH_TOKEN)} + return headers + + def make_request(self, url, method, data=None): + headers = self.get_request_headers() + try: + response = requests.request( + method=method, url=url, headers=headers, data=data, timeout=200 + ) + response.raise_for_status() + print("Successful Status", response.json()) + except requests.exceptions.RequestException as e: + print( + "The worker is not able to establish connection with EvalAI", + response.json(), + ) + raise + return response.json() + + def return_url_per_environment(self, url): + base_url = "{0}".format(self.EVALAI_API_SERVER) + url = "{0}{1}".format(base_url, url) + return url + + def update_submission_data(self, data, challenge_pk): + url = URLS.get("update_submission_data").format(challenge_pk) + url = self.return_url_per_environment(url) + response = self.make_request(url, "PUT", data=data) + return response diff --git a/code_upload_challenge_evaluation/requirements/agent.txt b/code_upload_challenge_evaluation/requirements/agent.txt new file mode 100644 index 0000000..80f5dba --- /dev/null +++ b/code_upload_challenge_evaluation/requirements/agent.txt @@ -0,0 +1,3 @@ +grpcio==1.22.0 +grpcio-tools==1.22.0 +numpy==1.19.4 \ No newline at end of file diff --git a/code_upload_challenge_evaluation/requirements/environment.txt b/code_upload_challenge_evaluation/requirements/environment.txt new file mode 100644 index 0000000..ecf3567 --- /dev/null +++ b/code_upload_challenge_evaluation/requirements/environment.txt @@ -0,0 +1,5 @@ +grpcio==1.22.0 +grpcio-tools==1.22.0 +gym==0.15.4 +requests==2.25.0 +urllib3==1.26.5 \ No newline at end of file diff --git a/code_upload_challenge_evaluation/utils/client.py b/code_upload_challenge_evaluation/utils/client.py new file mode 100644 index 0000000..5b544a3 --- /dev/null +++ b/code_upload_challenge_evaluation/utils/client.py @@ -0,0 +1,25 @@ +import grpc +import digestor_pb2 +import digestor_pb2_grpc + +class DigestorClient(object): + """ + Client for accessing the gRPC functionality + """ + + def __init__(self): + self.host = 'localhost' + self.server_port = 46001 + + self.channel = grpc.insecure_channel( + '{}:{}'.format(self.host, self.server_port)) + + + self.stub = digestor_pb2_grpc.DigestorStub(self.channel) + + def get_digest(self, message): + """ + Client function to call the rpc for GetDigest + """ + to_digest_message =digestor_pb2.DigestMessage(ToDigest=message) + return self.stub.GetDigestor(to_digest_message) \ No newline at end of file diff --git a/code_upload_challenge_evaluation/utils/evaluation.proto b/code_upload_challenge_evaluation/utils/evaluation.proto new file mode 100644 index 0000000..03e16b8 --- /dev/null +++ b/code_upload_challenge_evaluation/utils/evaluation.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package evaluation; + +service Environment{ + rpc get_action_space(Package) returns (Package) {} + rpc act_on_environment(Package) returns (Package) {} +} + +message Package{ + bytes SerializedEntity = 1; +} diff --git a/code_upload_challenge_evaluation/utils/evaluation_pb2.py b/code_upload_challenge_evaluation/utils/evaluation_pb2.py new file mode 100644 index 0000000..a5367fe --- /dev/null +++ b/code_upload_challenge_evaluation/utils/evaluation_pb2.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: evaluation.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='evaluation.proto', + package='evaluation', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x10\x65valuation.proto\x12\nevaluation\"#\n\x07Package\x12\x18\n\x10SerializedEntity\x18\x01 \x01(\x0c\x32\x8f\x01\n\x0b\x45nvironment\x12>\n\x10get_action_space\x12\x13.evaluation.Package\x1a\x13.evaluation.Package\"\x00\x12@\n\x12\x61\x63t_on_environment\x12\x13.evaluation.Package\x1a\x13.evaluation.Package\"\x00\x62\x06proto3') +) + + + + +_PACKAGE = _descriptor.Descriptor( + name='Package', + full_name='evaluation.Package', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='SerializedEntity', full_name='evaluation.Package.SerializedEntity', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=32, + serialized_end=67, +) + +DESCRIPTOR.message_types_by_name['Package'] = _PACKAGE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Package = _reflection.GeneratedProtocolMessageType('Package', (_message.Message,), { + 'DESCRIPTOR' : _PACKAGE, + '__module__' : 'evaluation_pb2' + # @@protoc_insertion_point(class_scope:evaluation.Package) + }) +_sym_db.RegisterMessage(Package) + + + +_ENVIRONMENT = _descriptor.ServiceDescriptor( + name='Environment', + full_name='evaluation.Environment', + file=DESCRIPTOR, + index=0, + serialized_options=None, + serialized_start=70, + serialized_end=213, + methods=[ + _descriptor.MethodDescriptor( + name='get_action_space', + full_name='evaluation.Environment.get_action_space', + index=0, + containing_service=None, + input_type=_PACKAGE, + output_type=_PACKAGE, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='act_on_environment', + full_name='evaluation.Environment.act_on_environment', + index=1, + containing_service=None, + input_type=_PACKAGE, + output_type=_PACKAGE, + serialized_options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_ENVIRONMENT) + +DESCRIPTOR.services_by_name['Environment'] = _ENVIRONMENT + +# @@protoc_insertion_point(module_scope) diff --git a/code_upload_challenge_evaluation/utils/evaluation_pb2_grpc.py b/code_upload_challenge_evaluation/utils/evaluation_pb2_grpc.py new file mode 100644 index 0000000..d420ac8 --- /dev/null +++ b/code_upload_challenge_evaluation/utils/evaluation_pb2_grpc.py @@ -0,0 +1,63 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import evaluation_pb2 as evaluation__pb2 + + +class EnvironmentStub(object): + # missing associated documentation comment in .proto file + pass + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.get_action_space = channel.unary_unary( + '/evaluation.Environment/get_action_space', + request_serializer=evaluation__pb2.Package.SerializeToString, + response_deserializer=evaluation__pb2.Package.FromString, + ) + self.act_on_environment = channel.unary_unary( + '/evaluation.Environment/act_on_environment', + request_serializer=evaluation__pb2.Package.SerializeToString, + response_deserializer=evaluation__pb2.Package.FromString, + ) + + +class EnvironmentServicer(object): + # missing associated documentation comment in .proto file + pass + + def get_action_space(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def act_on_environment(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_EnvironmentServicer_to_server(servicer, server): + rpc_method_handlers = { + 'get_action_space': grpc.unary_unary_rpc_method_handler( + servicer.get_action_space, + request_deserializer=evaluation__pb2.Package.FromString, + response_serializer=evaluation__pb2.Package.SerializeToString, + ), + 'act_on_environment': grpc.unary_unary_rpc_method_handler( + servicer.act_on_environment, + request_deserializer=evaluation__pb2.Package.FromString, + response_serializer=evaluation__pb2.Package.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'evaluation.Environment', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/evaluation_script/__init__.py b/evaluation_script/__init__.py new file mode 100644 index 0000000..3d9cd94 --- /dev/null +++ b/evaluation_script/__init__.py @@ -0,0 +1,43 @@ +""" +# Q. How to install custom python pip packages? + +# A. Uncomment the below code to install the custom python packages. + +import os +import subprocess +import sys +from pathlib import Path + +def install(package): + # Install a pip python package + + # Args: + # package ([str]): Package name with version + + subprocess.check_call([sys.executable, "-m", "pip", "install", package]) + + +def install_local_package(folder_name): + # Install a local python package + + # Args: + # folder_name ([str]): name of the folder placed in evaluation_script/ + + subprocess.check_output( + [ + sys.executable, + "-m", + "pip", + "install", + os.path.join(str(Path(__file__).parent.absolute()) + folder_name), + ] +) + +install("shapely==1.7.1") +install("requests==2.25.1") + +install_local_package("package_folder_name") + +""" + +from .main import evaluate diff --git a/evaluation_script/main.py b/evaluation_script/main.py new file mode 100644 index 0000000..61c73d9 --- /dev/null +++ b/evaluation_script/main.py @@ -0,0 +1,81 @@ +import random + + +def evaluate(test_annotation_file, user_submission_file, phase_codename, **kwargs): + print("Starting Evaluation.....") + """ + Evaluates the submission for a particular challenge phase and returns score + Arguments: + + `test_annotations_file`: Path to test_annotation_file on the server + `user_submission_file`: Path to file submitted by the user + `phase_codename`: Phase to which submission is made + + `**kwargs`: keyword arguments that contains additional submission + metadata that challenge hosts can use to send slack notification. + You can access the submission metadata + with kwargs['submission_metadata'] + + Example: A sample submission metadata can be accessed like this: + >>> print(kwargs['submission_metadata']) + { + 'status': u'running', + 'when_made_public': None, + 'participant_team': 5, + 'input_file': 'https://abc.xyz/path/to/submission/file.json', + 'execution_time': u'123', + 'publication_url': u'ABC', + 'challenge_phase': 1, + 'created_by': u'ABC', + 'stdout_file': 'https://abc.xyz/path/to/stdout/file.json', + 'method_name': u'Test', + 'stderr_file': 'https://abc.xyz/path/to/stderr/file.json', + 'participant_team_name': u'Test Team', + 'project_url': u'http://foo.bar', + 'method_description': u'ABC', + 'is_public': False, + 'submission_result_file': 'https://abc.xyz/path/result/file.json', + 'id': 123, + 'submitted_at': u'2017-03-20T19:22:03.880652Z' + } + """ + output = {} + if phase_codename == "dev": + print("Evaluating for Dev Phase") + output["result"] = [ + { + "train_split": { + "Metric1": random.randint(0, 99), + "Metric2": random.randint(0, 99), + "Metric3": random.randint(0, 99), + "Total": random.randint(0, 99), + } + } + ] + # To display the results in the result file + output["submission_result"] = output["result"][0]["train_split"] + print("Completed evaluation for Dev Phase") + elif phase_codename == "test": + print("Evaluating for Test Phase") + output["result"] = [ + { + "train_split": { + "Metric1": random.randint(0, 99), + "Metric2": random.randint(0, 99), + "Metric3": random.randint(0, 99), + "Total": random.randint(0, 99), + } + }, + { + "test_split": { + "Metric1": random.randint(0, 99), + "Metric2": random.randint(0, 99), + "Metric3": random.randint(0, 99), + "Total": random.randint(0, 99), + } + }, + ] + # To display the results in the result file + output["submission_result"] = output["result"][0] + print("Completed evaluation for Test Phase") + return output diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py new file mode 100644 index 0000000..b0c88bb --- /dev/null +++ b/github/challenge_processing_script.py @@ -0,0 +1,144 @@ +import http +import json +import os +import requests +import sys + +from config import * +from utils import ( + add_pull_request_comment, + check_for_errors, + check_if_merge_or_commit, + check_if_pull_request, + create_challenge_zip_file, + create_github_repository_issue, + get_request_header, + load_host_configs, + validate_token, +) + +sys.dont_write_bytecode = True + +GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT")) + +GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") +if not GITHUB_AUTH_TOKEN: + print( + "Please add your github access token to the repository secrets with the name AUTH_TOKEN" + ) + sys.exit(1) + +HOST_AUTH_TOKEN = None +CHALLENGE_HOST_TEAM_PK = None +EVALAI_HOST_URL = None + + +if __name__ == "__main__": + + configs = load_host_configs(HOST_CONFIG_FILE_PATH) + if configs: + HOST_AUTH_TOKEN = configs[0] + CHALLENGE_HOST_TEAM_PK = configs[1] + EVALAI_HOST_URL = configs[2] + else: + sys.exit(1) + + # Fetching the url + if VALIDATION_STEP == "True": + url = "{}{}".format( + EVALAI_HOST_URL, + CHALLENGE_CONFIG_VALIDATION_URL.format(CHALLENGE_HOST_TEAM_PK), + ) + else: + url = "{}{}".format( + EVALAI_HOST_URL, + CHALLENGE_CREATE_OR_UPDATE_URL.format(CHALLENGE_HOST_TEAM_PK), + ) + + headers = get_request_header(HOST_AUTH_TOKEN) + + # Creating the challenge zip file and storing in a dict to send to EvalAI + create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) + zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") + file = {"zip_configuration": zip_file} + + data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + + try: + response = requests.post(url, data=data, headers=headers, files=file) + + if ( + response.status_code != http.HTTPStatus.OK + and response.status_code != http.HTTPStatus.CREATED + ): + response.raise_for_status() + else: + print("\n" + response.json()["Success"]) + except requests.exceptions.HTTPError as err: + if response.status_code in EVALAI_ERROR_CODES: + is_token_valid = validate_token(response.json()) + if is_token_valid: + error = response.json()["error"] + error_message = "\nFollowing errors occurred while validating the challenge config:\n{}".format( + error + ) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + else: + print( + "\nFollowing errors occurred while validating the challenge config: {}".format( + err + ) + ) + os.environ["CHALLENGE_ERRORS"] = str(err) + except Exception as e: + if VALIDATION_STEP == "True": + error_message = "\nFollowing errors occurred while validating the challenge config: {}".format( + e + ) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + else: + error_message = "\nFollowing errors occurred while processing the challenge config: {}".format( + e + ) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + + zip_file.close() + os.remove(zip_file.name) + + is_valid, errors = check_for_errors() + if not is_valid: + if VALIDATION_STEP == "True" and check_if_pull_request(): + pr_number = GITHUB_CONTEXT["event"]["number"] + add_pull_request_comment( + GITHUB_AUTH_TOKEN, + os.path.basename(GITHUB_REPOSITORY), + pr_number, + errors, + ) + print( + "\nExiting the {} script after failure\n".format( + os.path.basename(__file__) + ) + ) + sys.exit(1) + else: + issue_title = ( + "Following errors occurred while validating the challenge config:" + ) + create_github_repository_issue( + GITHUB_AUTH_TOKEN, + os.path.basename(GITHUB_REPOSITORY), + issue_title, + errors, + ) + print( + "\nExiting the {} script after failure\n".format( + os.path.basename(__file__) + ) + ) + sys.exit(1) + + print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) diff --git a/github/config.py b/github/config.py new file mode 100644 index 0000000..738b69d --- /dev/null +++ b/github/config.py @@ -0,0 +1,28 @@ +import os + + +os.environ["CHALLENGE_ERRORS"] = "False" + +HOST_CONFIG_FILE_PATH = "github/host_config.json" +CHALLENGE_CONFIG_VALIDATION_URL = "/api/challenges/challenge/challenge_host_team/{}/validate_challenge_config/" +CHALLENGE_CREATE_OR_UPDATE_URL = "/api/challenges/challenge/challenge_host_team/{}/create_or_update_github_challenge/" +EVALAI_ERROR_CODES = [400, 401, 406] +API_HOST_URL = "https://eval.ai" +IGNORE_DIRS = [ + ".git", + ".github", + "github", + "code_upload_challenge_evaluation", + "remote_challenge_evaluation", +] +IGNORE_FILES = [ + ".gitignore", + "challenge_config.zip", + "README.md", + "run.sh", + "submission.json", +] +CHALLENGE_ZIP_FILE_PATH = "challenge_config.zip" +GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") +GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME") +VALIDATION_STEP = os.getenv("IS_VALIDATION") diff --git a/github/host_config.json b/github/host_config.json new file mode 100644 index 0000000..e7ead61 --- /dev/null +++ b/github/host_config.json @@ -0,0 +1,5 @@ +{ + "token": "", + "team_pk": "", + "evalai_host_url": "" +} diff --git a/github/requirements.txt b/github/requirements.txt new file mode 100644 index 0000000..555cea3 --- /dev/null +++ b/github/requirements.txt @@ -0,0 +1,2 @@ +PyGithub===1.53 +requests==2.24.0 diff --git a/github/utils.py b/github/utils.py new file mode 100644 index 0000000..2b904db --- /dev/null +++ b/github/utils.py @@ -0,0 +1,180 @@ +import json +import os +import sys +import zipfile + +from config import * +from github import Github + + +def check_for_errors(): + """ + Checks if any errors have been recorded so far during this workflow step and returns the error if so + """ + if os.getenv("CHALLENGE_ERRORS") == "False": + return True, None + return False, os.getenv("CHALLENGE_ERRORS") + + +def check_if_pull_request(): + """ + Returns True if the workflow triggering event is a pull request + """ + if GITHUB_EVENT_NAME == "pull_request": + return True + return False + + +def check_if_merge_or_commit(): + """ + Returns True if the workflow triggering event is either a merge or a direct commit + """ + if GITHUB_EVENT_NAME == "push": + return True + return False + + +def add_pull_request_comment(github_auth_token, repo_name, pr_number, comment_body): + """ + Adds a comment to a pull request + Arguments: + github_auth_token {str}: The auth token of the github user + repo_name {str}: The name of the repository + pr_number {int}: The Pull request number to add a comment + comment_body {str}: The body of the comment + """ + try: + client = Github(github_auth_token) + repo = client.get_user().get_repo(repo_name) + pull = repo.get_pull(pr_number) + pull.create_issue_comment(comment_body) + except Exception as e: + print("There was an error while commenting on the Pull request: {}".format(e)) + + +def create_github_repository_issue( + github_auth_token, repo_name, issue_title, issue_body +): + """ + Creates an issue in a given repository + + Arguments: + github_auth_token {str}: The auth token of the github user + repo_name {str}: The name of the repository + issue_title {int}: The title of the issue to be created + issue_body {str}: The body of the issue to be created + """ + try: + client = Github(github_auth_token) + repo = client.get_user().get_repo(repo_name) + issue = repo.create_issue(issue_title, issue_body) + except Exception as e: + print("There was an error while creating an issue: {}".format(e)) + + +def create_challenge_zip_file(challenge_zip_file_path, ignore_dirs, ignore_files): + """ + Creates the challenge zip file at a given path + + Arguments: + challenge_zip_file_path {str}: The relative path of the created zip file + ignore_dirs {list}: The list of directories to exclude from the zip file + ignore_files {list}: The list of files to exclude from the zip file + """ + working_dir = ( + os.getcwd() + ) # Special case for github. For local. use os.path.dirname(os.getcwd()) + + # Creating evaluation_script.zip file + eval_script_dir = working_dir + "/evaluation_script" + eval_script_zip = zipfile.ZipFile( + "evaluation_script.zip", "w", zipfile.ZIP_DEFLATED + ) + for root, dirs, files in os.walk(eval_script_dir): + for file in files: + file_name = os.path.join(root, file) + name_in_zip_file = ( + file_name[len(eval_script_dir) + 1 :] + if file_name.startswith(eval_script_dir) + else file_name + ) + eval_script_zip.write(file_name, name_in_zip_file) + eval_script_zip.close() + + # Creating the challenge_config.zip file + zipf = zipfile.ZipFile(challenge_zip_file_path, "w", zipfile.ZIP_DEFLATED) + for root, dirs, files in os.walk(working_dir): + parents = root.split("/") + if not set(parents) & set(ignore_dirs): + for file in files: + if file not in ignore_files: + file_name = os.path.join(root, file) + name_in_zip_file = ( + file_name[len(working_dir) + 1 :] + if file_name.startswith(working_dir) + else file_name + ) + zipf.write(file_name, name_in_zip_file) + zipf.close() + + +def get_request_header(token): + """ + Returns user auth token formatted in header for sending requests + + Arguments: + token {str}: The user token to gain access to EvalAI + """ + header = {"Authorization": "Bearer {}".format(token)} + return header + + +def load_host_configs(config_path): + """ + Loads token to be used for sending requests + + Arguments: + config_path {str}: The path of host configs having the user token, team id and the EvalAI host url + """ + config_path = "{}/{}".format(os.getcwd(), config_path) + if os.path.exists(config_path): + with open(config_path, "r") as f: + try: + data = f.read() + except (OSError, IOError) as e: + print("\nAn error occured while loading the host configs: {}".format(e)) + sys.exit(1) + data = json.loads(data) + host_auth_token = data["token"] + challenge_host_team_pk = data["team_pk"] + evalai_host_url = data["evalai_host_url"] + return [host_auth_token, challenge_host_team_pk, evalai_host_url] + else: + error_message = "\nThe host config json file is not present. Please include an auth token, team_pk & evalai_host_url in it: {}".format( + config_path + ) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + return False + + +def validate_token(response): + """ + Function to check if the authentication token provided by user is valid or not + + Arguments: + response {dict}: The response json dict sent back from EvalAI + """ + error = None + if "detail" in response: + if response["detail"] == "Invalid token": + error = "\nThe authentication token you are using isn't valid. Please generate it again.\n" + print(error) + os.environ["CHALLENGE_ERRORS"] = error + return False + if response["detail"] == "Token has expired": + error = "\nSorry, the token has expired. Please generate it again.\n" + print(error) + os.environ["CHALLENGE_ERRORS"] = error + return False + return True diff --git a/logo.jpg b/logo.jpg new file mode 100644 index 0000000..1a0273e Binary files /dev/null and b/logo.jpg differ diff --git a/remote_challenge_evaluation/README.md b/remote_challenge_evaluation/README.md new file mode 100644 index 0000000..7388db3 --- /dev/null +++ b/remote_challenge_evaluation/README.md @@ -0,0 +1,17 @@ +## How to setup remote challenge evaluation using EvalAI :rocket: +If you are looking for setting up a remote challenge evaluation on EvalAI, then you are at the right place. Follow the instructions given below to get started. + +1. Create a challenge on EvalAI using [GitHub](https://github.com/Cloud-CV/EvalAI-Starters#create-challenge-using-github) based challenge creation. + +2. Once the challenge is successfully created, please email EvalAI admin on team@cloudcv.org for sending the `challenge_pk` and `queue_name`. + +3. After receiving the details from the admin, please add these in the `evaluation_script_starter.py`. + +4. Create a new virtual python3 environment for installating the worker requirements. + +5. Install the requirements using `pip install -r requirements.txt`. + +6. For python3, run the worker using `python -m evaluation_script_starter` +## Facing problems in setting up evaluation? + +Please feel free to open issues on our [GitHub Repository](https://github.com/Cloud-CV/EvalAI-Starter/issues) or contact us at team@cloudcv.org if you have issues. diff --git a/remote_challenge_evaluation/eval_ai_interface.py b/remote_challenge_evaluation/eval_ai_interface.py new file mode 100644 index 0000000..4f48f91 --- /dev/null +++ b/remote_challenge_evaluation/eval_ai_interface.py @@ -0,0 +1,148 @@ +import logging + +import requests + +logger = logging.getLogger(__name__) + +URLS = { + "get_message_from_sqs_queue": "/api/jobs/challenge/queues/{}/", + "get_submission_by_pk": "/api/jobs/submission/{}", + "get_challenge_phase_by_pk": "/api/challenges/challenge/phase/{}", + "delete_message_from_sqs_queue": "/api/jobs/queues/{}/", + "update_submission": "/api/jobs/challenge/{}/update_submission/", +} + + +class EvalAI_Interface: + def __init__(self, AUTH_TOKEN, EVALAI_API_SERVER, QUEUE_NAME, CHALLENGE_PK): + """Class to initiate call to EvalAI backend + + Arguments: + AUTH_TOKEN {[string]} -- The authentication token corresponding to EvalAI + EVALAI_API_SERVER {[string]} -- It should be set to https://eval.ai # For production server + QUEUE_NAME {[string]} -- Unique queue name corresponding to every challenge + CHALLENGE_PK {[integer]} -- Primary key corresponding to a challenge + """ + + self.AUTH_TOKEN = AUTH_TOKEN + self.EVALAI_API_SERVER = EVALAI_API_SERVER + self.QUEUE_NAME = QUEUE_NAME + self.CHALLENGE_PK = CHALLENGE_PK + + def get_request_headers(self): + """Function to get the header of the EvalAI request in proper format + + Returns: + [dict]: Authorization header + """ + headers = {"Authorization": "Bearer {}".format(self.AUTH_TOKEN)} + return headers + + def make_request(self, url, method, data=None): + """Function to make request to EvalAI interface + + Args: + url ([str]): URL of the request + method ([str]): Method of the request + data ([dict], optional): Data of the request. Defaults to None. + + Returns: + [JSON]: JSON response data + """ + headers = self.get_request_headers() + try: + response = requests.request( + method=method, url=url, headers=headers, data=data + ) + response.raise_for_status() + except requests.exceptions.RequestException: + logger.info("The server isn't able establish connection with EvalAI") + raise + return response.json() + + def return_url_per_environment(self, url): + """Function to get the URL for API + + Args: + url ([str]): API endpoint url to which the request is to be made + + Returns: + [str]: API endpoint url with EvalAI base url attached + """ + base_url = "{0}".format(self.EVALAI_API_SERVER) + url = "{0}{1}".format(base_url, url) + return url + + def get_message_from_sqs_queue(self): + """Function to get the message from SQS Queue + + Docs: https://eval.ai/api/docs/#operation/get_submission_message_from_queue + + Returns: + [JSON]: JSON response data + """ + url = URLS.get("get_message_from_sqs_queue").format(self.QUEUE_NAME) + url = self.return_url_per_environment(url) + response = self.make_request(url, "GET") + return response + + def delete_message_from_sqs_queue(self, receipt_handle): + """Function to delete the submission message from the queue + + Docs: https://eval.ai/api/docs/#operation/delete_submission_message_from_queue + + Args: + receipt_handle ([str]): Receipt handle of the message to be deleted + + Returns: + [JSON]: JSON response data + """ + url = URLS.get("delete_message_from_sqs_queue").format(self.QUEUE_NAME) + url = self.return_url_per_environment(url) + data = {"receipt_handle": receipt_handle} + response = self.make_request(url, "POST", data) + return response + + def update_submission_data(self, data): + """Function to update the submission data on EvalAI + + Docs: https://eval.ai/api/docs/#operation/update_submission + + Args: + data ([dict]): Data to be updated + + Returns: + [JSON]: JSON response data + """ + url = URLS.get("update_submission").format(self.CHALLENGE_PK) + url = self.return_url_per_environment(url) + response = self.make_request(url, "PUT", data=data) + return response + + def update_submission_status(self, data): + """ + + Docs: https://eval.ai/api/docs/#operation/update_submission + + Args: + data ([dict]): Data to be updated + + Returns: + [JSON]: JSON response data + """ + url = URLS.get("update_submission").format(self.CHALLENGE_PK) + url = self.return_url_per_environment(url) + response = self.make_request(url, "PATCH", data=data) + return response + + def get_submission_by_pk(self, submission_pk): + url = URLS.get("get_submission_by_pk").format(submission_pk) + url = self.return_url_per_environment(url) + response = self.make_request(url, "GET") + return response + + def get_challenge_phase_by_pk(self, phase_pk): + url = URLS.get("get_challenge_phase_by_pk").format(phase_pk) + url = self.return_url_per_environment(url) + response = self.make_request(url, "GET") + return response diff --git a/remote_challenge_evaluation/evaluate.py b/remote_challenge_evaluation/evaluate.py new file mode 100644 index 0000000..297f469 --- /dev/null +++ b/remote_challenge_evaluation/evaluate.py @@ -0,0 +1,76 @@ + + +def evaluate(user_submission_file, phase_codename, test_annotation_file=None, **kwargs): + print("Starting Evaluation.....") + """ + Evaluates the submission for a particular challenge phase and returns score + Arguments: + `user_submission_file`: Path to file submitted by the user + `phase_codename`: Phase to which submission is made + + `test_annotations_file`: Path to test_annotation_file on the server + We recommend setting a default `test_annotation_file` or using `phase_codename` + to select the appropriate file. For example, you could load test annotation file + for current phase as: + ``` + test_annotation_file = json.loads(open("{phase_codename}_path", "r")) + ``` + `**kwargs`: keyword arguments that contains additional submission + metadata that challenge hosts can use to send slack notification. + You can access the submission metadata + with kwargs['submission_metadata'] + Example: A sample submission metadata can be accessed like this: + >>> print(kwargs['submission_metadata']) + { + 'status': u'running', + 'when_made_public': None, + 'participant_team': 5, + 'input_file': 'https://abc.xyz/path/to/submission/file.json', + 'execution_time': u'123', + 'publication_url': u'ABC', + 'challenge_phase': 1, + 'created_by': u'ABC', + 'stdout_file': 'https://abc.xyz/path/to/stdout/file.json', + 'method_name': u'Test', + 'stderr_file': 'https://abc.xyz/path/to/stderr/file.json', + 'participant_team_name': u'Test Team', + 'project_url': u'http://foo.bar', + 'method_description': u'ABC', + 'is_public': False, + 'submission_result_file': 'https://abc.xyz/path/result/file.json', + 'id': 123, + 'submitted_at': u'2017-03-20T19:22:03.880652Z' + } + """ + + ''' + # Load test annotation file for current phase + test_annotation_file = json.loads(open("{phase_codename}_path", "r")) + ''' + output = {} + if phase_codename == "dev": + print("Evaluating for Dev Phase") + output["result"] = [ + { + "split": "train_split", + "show_to_participant": True, + "accuracies": {"Metric1": 90}, + }, + ] + print("Completed evaluation for Dev Phase") + elif phase_codename == "test": + print("Evaluating for Test Phase") + output["result"] = [ + { + "split": "train_split", + "show_to_participant": True, + "accuracies": {"Metric1": 90}, + }, + { + "split": "test_split", + "show_to_participant": False, + "accuracies": {"Metric1": 50, "Metric2": 40}, + }, + ] + print("Completed evaluation for Test Phase") + return output diff --git a/remote_challenge_evaluation/main.py b/remote_challenge_evaluation/main.py new file mode 100644 index 0000000..932ef88 --- /dev/null +++ b/remote_challenge_evaluation/main.py @@ -0,0 +1,108 @@ +import json +import os +import time + +import requests + +from eval_ai_interface import EvalAI_Interface +from evaluate import evaluate + +# Remote Evaluation Meta Data +# See https://evalai.readthedocs.io/en/latest/evaluation_scripts.html#writing-remote-evaluation-script +auth_token = os.environ["AUTH_TOKEN"] +evalai_api_server = os.environ["API_SERVER"] +queue_name = os.environ["QUEUE_NAME"] +challenge_pk = os.environ["CHALLENGE_PK"] +save_dir = os.environ.get("SAVE_DIR", "./") + + +def download(submission, save_dir): + response = requests.get(submission["input_file"]) + submission_file_path = os.path.join( + save_dir, submission["input_file"].split("/")[-1] + ) + with open(submission_file_path, "wb") as f: + f.write(response.content) + return submission_file_path + + +def update_running(evalai, submission_pk): + status_data = { + "submission": submission_pk, + "submission_status": "RUNNING", + } + update_status = evalai.update_submission_status(status_data) + + +def update_failed( + evalai, phase_pk, submission_pk, submission_error, stdout="", metadata="" +): + submission_data = { + "challenge_phase": phase_pk, + "submission": submission_pk, + "stdout": stdout, + "stderr": submission_error, + "submission_status": "FAILED", + "metadata": metadata, + } + update_data = evalai.update_submission_data(submission_data) + + +def update_finished( + evalai, + phase_pk, + submission_pk, + result, + submission_error="", + stdout="", + metadata="", +): + submission_data = { + "challenge_phase": phase_pk, + "submission": submission_pk, + "stdout": stdout, + "stderr": submission_error, + "submission_status": "FINISHED", + "result": result, + "metadata": metadata, + } + update_data = evalai.update_submission_data(submission_data) + + +if __name__ == "__main__": + evalai = EvalAI_Interface(auth_token, evalai_api_server, queue_name, challenge_pk) + + while True: + # Get the message from the queue + message = evalai.get_message_from_sqs_queue() + message_body = message.get("body") + if message_body: + submission_pk = message_body.get("submission_pk") + challenge_pk = message_body.get("challenge_pk") + phase_pk = message_body.get("phase_pk") + # Get submission details -- This will contain the input file URL + submission = evalai.get_submission_by_pk(submission_pk) + challenge_phase = evalai.get_challenge_phase_by_pk(phase_pk) + if ( + submission.get("status") == "finished" + or submission.get("status") == "failed" + or submission.get("status") == "cancelled" + ): + message_receipt_handle = message.get("receipt_handle") + evalai.delete_message_from_sqs_queue(message_receipt_handle) + + else: + if submission.get("status") == "submitted": + update_running(evalai, submission_pk) + submission_file_path = download(submission, save_dir) + try: + results = evaluate( + submission_file_path, challenge_phase["codename"] + ) + update_finished( + evalai, phase_pk, submission_pk, json.dumps(results["result"]) + ) + except Exception as e: + update_failed(evalai, phase_pk, submission_pk, str(e)) + # Poll challenge queue for new submissions + time.sleep(60) diff --git a/remote_challenge_evaluation/requirements.txt b/remote_challenge_evaluation/requirements.txt new file mode 100644 index 0000000..fd7d3e0 --- /dev/null +++ b/remote_challenge_evaluation/requirements.txt @@ -0,0 +1 @@ +requests==2.25.1 \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..0e80046 --- /dev/null +++ b/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Remove already existing zip files +rm evaluation_script.zip +rm challenge_config.zip + +# Create new zip configuration according the updated code +cd evaluation_script +zip -r ../evaluation_script.zip * -x "*.DS_Store" +cd .. +zip -r challenge_config.zip * -x "*.DS_Store" -x "evaluation_script/*" -x "*.git" -x "run.sh" -x "code_upload_challenge_evaluation/*" -x "remote_challenge_evaluation/*" -x "worker/*" -x "challenge_data/*" -x "github/*" -x ".github/*" -x "README.md" diff --git a/submission.json b/submission.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/submission.json @@ -0,0 +1 @@ +{} diff --git a/templates/challenge_phase_1_description.html b/templates/challenge_phase_1_description.html new file mode 100755 index 0000000..98907f5 --- /dev/null +++ b/templates/challenge_phase_1_description.html @@ -0,0 +1 @@ +

"Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?"

\ No newline at end of file diff --git a/templates/challenge_phase_2_description.html b/templates/challenge_phase_2_description.html new file mode 100755 index 0000000..7de79f9 --- /dev/null +++ b/templates/challenge_phase_2_description.html @@ -0,0 +1 @@ +"Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?" \ No newline at end of file diff --git a/templates/description.html b/templates/description.html new file mode 100755 index 0000000..2ee4109 --- /dev/null +++ b/templates/description.html @@ -0,0 +1,3 @@ +

"Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?"

+ +

"Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?"

diff --git a/templates/evaluation_details.html b/templates/evaluation_details.html new file mode 100755 index 0000000..14bf424 --- /dev/null +++ b/templates/evaluation_details.html @@ -0,0 +1 @@ +

"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."

diff --git a/templates/submission_guidelines.html b/templates/submission_guidelines.html new file mode 100755 index 0000000..491fc70 --- /dev/null +++ b/templates/submission_guidelines.html @@ -0,0 +1 @@ +

Submit any blank file here to see a random number generated for your submission. If you get lucky, you might reach the top of the leaderboard.

diff --git a/templates/terms_and_conditions.html b/templates/terms_and_conditions.html new file mode 100755 index 0000000..12e9f60 --- /dev/null +++ b/templates/terms_and_conditions.html @@ -0,0 +1 @@ +

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

diff --git a/worker/__init__.py b/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/worker/run.py b/worker/run.py new file mode 100644 index 0000000..2c8dc10 --- /dev/null +++ b/worker/run.py @@ -0,0 +1,61 @@ +import importlib +import os +import sys + + +def get_curr_working_dir(): + curr_working_dir = os.getcwd() + return curr_working_dir + + +def run(): + current_working_directory = get_curr_working_dir() + sys.path.append("{}".format(current_working_directory)) + sys.path.append("{}/challenge_data/challenge_1".format(current_working_directory)) + + challenge_id = 1 + challenge_phase = "test" # Add the challenge phase codename to be tested + annotation_file_path = "{}/annotations/test_annotations_testsplit.json".format( + current_working_directory + ) # Add the test annotation file path + user_submission_file_path = "{}/submission.json".format( + current_working_directory + ) # Add the sample submission file path + + CHALLENGE_IMPORT_STRING = "challenge_data.challenge_1" + challenge_module = importlib.import_module(CHALLENGE_IMPORT_STRING) + + EVALUATION_SCRIPTS = {} + EVALUATION_SCRIPTS[challenge_id] = challenge_module + print("Trying to evaluate") + submission_metadata = { + "status": u"running", + "when_made_public": None, + "participant_team": 5, + "input_file": "https://abc.xyz/path/to/submission/file.json", + "execution_time": u"123", + "publication_url": u"ABC", + "challenge_phase": 1, + "created_by": u"ABC", + "stdout_file": "https://abc.xyz/path/to/stdout/file.json", + "method_name": u"Test", + "stderr_file": "https://abc.xyz/path/to/stderr/file.json", + "participant_team_name": u"Test Team", + "project_url": u"http://foo.bar", + "method_description": u"ABC", + "is_public": False, + "submission_result_file": "https://abc.xyz/path/result/file.json", + "id": 123, + "submitted_at": u"2017-03-20T19:22:03.880652Z", + } + EVALUATION_SCRIPTS[challenge_id].evaluate( + annotation_file_path, + user_submission_file_path, + challenge_phase, + submission_metadata=submission_metadata, + ) + print("Evaluated Successfully!") + + +if __name__ == "__main__": + run()