ml container: created Github Actions workflow, moved recommendation_p…

…ipeline to root

.github/workflows/ml_update.yml

@@ -0,0 +1,38 @@
+name: Update ML Container
+
+on:
+    push:
+        branches:
+            - main
+            - dev
+            - feat/ML_server
+        paths:
+            - 'recommend/**'
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v2
+
+            - name: Setup SSH Key
+              run: |
+                echo "${{ secrets.PEM }}" > key.pem 
+                chmod 600 key.pem
+                
+            - name: Transfer Recommendation Script
+              run:
+                scp -i key.pem ./recommendation_pipeline.sh ec2-user@${{ secrets.CLOUD_URL }}:~
+
+            - name: Sync ML Code
+              run: rsync -avz ./recommend/ ec2-user@${{ secrets.CLOUD_URL }}:/recommend/
+
+            - name: Restart ML Container & Mount Code
+              run: |
+                ssh -i key.pem -o StrictHostKeyChecking=no ec2-user@${{ secrets.CLOUD_URL }} <<EOF
+                    docker stop ml-container
+                    docker rm ml-container
+                    docker run -d --name ml-container -v /recommend:/app "${{ secrets.DOCKER_USERNAME }}/ml-image:latest"
+                EOF

backend/requirements.txt

@@ -77,7 +77,6 @@ pygments==2.16.1
 pymysql
 pynacl
 pyparsing==3.1.1
-# python==3.11.5
 python-dateutil==2.8.2
 python-decouple==3.8
 python-dotenv==1.0.0

recommend/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.11.5
+
+WORKDIR /app
+
+# create the base image for ML dependencies
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose this port to my backend server
+EXPOSE 8080

recommendation_pipeline.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# go into the backend container and do the first part
+current_time=$(TZ=Asia/Seoul date +%H:%M) 
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Connecting to the backend container..."
+
+docker exec haeng bash <<'EOF'    # change this back to 'EOF'
+current_time=$(TZ=Asia/Seoul date +%H:%M)
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Start collecting data..."    
+python3 transfer.py -c
+
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Start transferring userData & eventData to shared volume..."
+USER_FILE="userData.csv"
+EVENT_FILE="eventData.csv"
+USER_SRC_PATH="/app/data/$USER_FILE"
+EVENT_SRC_PATH="/app/data/$EVENT_FILE"
+VOLUME_PATH="/var/lib/docker/volumes/shared-volume/_data/"
+
+cp $USER_SRC_PATH $VOLUME_PATH
+cp $EVENT_SRC_PATH $VOLUME_PATH
+
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Finished!"
+
+EOF
+
+# go into the ML container and do the second part
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Connecting to the ML container..."
+
+docker exec ml-container bash <<'EOF'
+current_time=$(TZ=Asia/Seoul date +%H:%M)
+
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Start copying userData & eventData from shared volume..."
+VOLUME_PATH="/var/lib/docker/volumes/shared-volume/_data/"
+USER_FILE="userData.csv"
+EVENT_FILE="eventData.csv"
+USER_DST_PATH="/app/data/$USER_FILE"
+EVENT_DST_PATH="/app/data/$EVENT_FILE"
+RECOMMENDATION_FILE="all_recommends_index.csv"
+
+cp $VOLUME_PATH$USER_FILE $USER_DST_PATH
+cp $VOLUME_PATH$EVENT_FILE $EVENT_DST_PATH
+
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Start preprocessing & Extracting embedding..."
+python /app/preprocess.py 1 #User
+python /app/preprocess.py 0 #Event
+
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Compare similarity & Make recommendation..."
+python /app/recommend.py
+cp $RECOMMENDATION_FILE $VOLUME_PATH
+EOF
+
+# go into the backend container and do the third part
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Connecting to the backend container..."
+docker exec haeng bash<<'EOF'
+
+current_time=$(TZ=Asia/Seoul date +%H:%M)
+current_timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+echo "[$current_timestamp] Copying the recommendation results into the container..."
+RECOMMENDATION_FILE="all_recommends_index.csv"
+VOLUME_PATH="/var/lib/docker/volumes/shared-volume/_data/"
+RECOMMENDATION_DST_PATH="/app/data/$RECOMMENDATION_FILE"
+cp $VOLUME_PATH$RECOMMENDATION_FILE $RECOMMENDATION_DST_PATH
+
+echo "[$current_timestamp] Delete redundant recommendations and load to DB..."
+python3 transfer.py -l
+echo "[$current_timestamp] Finished!"
+EOF
+
+