✨ 문제 업데이트 Job 개발 (#2)

* ✨ 문제 업데이트 Job 개발

* ✨ DataSource 분리 및 ECR 배포코드 작성

.github/workflows/cicd.yml

@@ -0,0 +1,61 @@
+name: cicd-dev
+
+on:
+  push:
+    branches: [ "main" ]
+
+env:
+  AWS_REGION: ap-northeast-2
+  ECR_REPOSITORY: new_morandi_batch
+  ECR_REGISTRY: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.ap-northeast-2.amazonaws.com/new_morandi_backend
+  GITHUB_SHA: ${{ github.sha }}
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      # Gradle 빌드를 추가합니다.
+      - name: Set up JDK 17
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
+          java-version: '17'
+
+      - name: Cleanup application.yml
+        run: rm -f src/main/resources/application.yml
+
+      # GitHub Secret에서 application-prod.yml 내용을 불러와 파일로 저장
+      - name: Create application.yml from GitHub Secret
+        run: |
+          mkdir -p src/main/resources
+          echo "${{ secrets.DEV_APPLICATION_YML }}" > src/main/resources/application.yml
+
+
+      - name: Build with Gradle
+        env:
+          ORG_GRADLE_OPTS: "-Duser.timezone=Asia/Seoul"
+        run: ./gradlew clean bootJar -x test
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.ECR_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.ECR_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Login to Private ECR
+        run: aws ecr get-login-password --region ap-northeast-2 | docker login --username AWS --password-stdin ${{ env.ECR_REGISTRY }}
+
+      - name: Build Docker Image
+        run: docker build -t ${{ env.ECR_REGISTRY }}:${{ github.sha }} .
+
+      - name: Push Docker Image to ECR
+        run: docker push ${{ env.ECR_REGISTRY }}:${{ github.sha }}

.gitignore

@@ -1,3 +1,5 @@
+/Users/miiiinju/NewMorandi-Batch/morandi-batch/src/main/resources/application.yml
+
 HELP.md
 .gradle
 build/

Dockerfile

@@ -0,0 +1,4 @@
+FROM openjdk:17-oracle
+WORKDIR /app
+COPY build/libs/*.jar app.jar
+ENTRYPOINT ["java", "-jar", "app.jar"]

build.gradle

@@ -24,6 +24,8 @@ repositories {
 dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-batch'
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
+    //configuration processor
+    implementation 'org.springframework.boot:spring-boot-configuration-processor'
 
     // lombok
     compileOnly 'org.projectlombok:lombok'

src/main/java/kr/co/morandi_batch/MorandiBatchApplication.java

@@ -2,6 +2,8 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.batch.JobLauncherApplicationRunner;
+import org.springframework.context.annotation.Bean;
 
 @SpringBootApplication
 public class MorandiBatchApplication {

src/main/java/kr/co/morandi_batch/batch/config/DataSourceConfig.java

@@ -1,55 +1,41 @@
 package kr.co.morandi_batch.batch.config;
 
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.batch.core.configuration.support.DefaultBatchConfiguration;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.batch.BatchDataSource;
 import org.springframework.boot.jdbc.DataSourceBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Primary;
 import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
-import org.springframework.jdbc.datasource.DataSourceTransactionManager;
-import org.springframework.orm.jpa.JpaTransactionManager;
-import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
-import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
-import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
+import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
 
 import javax.sql.DataSource;
 
 @Configuration
-@EnableJpaRepositories(
-        basePackages = "kr.co.morandi_batch.domain",
-        entityManagerFactoryRef = "businessEntityManager",
-        transactionManagerRef = "businessTransactionManager"
-)
 public class DataSourceConfig {
 
-    @Bean
-    @Primary
-    public LocalContainerEntityManagerFactoryBean businessEntityManager() {
-        LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean();
-
-        em.setDataSource(businessDataSource());
-        em.setPackagesToScan("kr.co.morandi_batch.domain");
-        em.setJpaVendorAdapter(new HibernateJpaVendorAdapter());
-
-        return em;
-    }
-
-    @Primary
-    @Bean(name = "businessDataSource")
-    @ConfigurationProperties(prefix = "spring.datasource.business")
-    public DataSource businessDataSource() {
-        return DataSourceBuilder.create().build();
+    @Bean(name = "batchDataSource")
+    @BatchDataSource
+    public DataSource H2Datasource () {
+        return new EmbeddedDatabaseBuilder()
+                .addScript("classpath:org/springframework/batch/core/schema-drop-h2.sql")
+                .addScript("classpath:org/springframework/batch/core/schema-h2.sql")
+                .setType(EmbeddedDatabaseType.H2)
+                .build();
     }
-
-    @Primary
-    @Bean(name = "businessTransactionManager")
-    public PlatformTransactionManager businessTransactionManager() {
-        JpaTransactionManager transactionManager = new JpaTransactionManager();
-
-        transactionManager.setEntityManagerFactory(businessEntityManager().getObject());
-
-        return transactionManager;
+    @Bean
+    @Primary // 이 DataSource를 애플리케이션의 주 데이터 소스로 지정
+    public DataSource dataSource(@Value("${spring.datasource.driver-class-name}") String driverClassName,
+                                       @Value("${spring.datasource.url}") String url,
+                                       @Value("${spring.datasource.username}") String user,
+                                       @Value("${spring.datasource.password}") String pass) {
+        return DataSourceBuilder.create()
+                .driverClassName(driverClassName)
+                .url(url)
+                .username(user)
+                .password(pass)
+                .build();
     }
-
 }

src/main/java/kr/co/morandi_batch/batch/config/NewProblemJobConfig.java

@@ -2,10 +2,7 @@
 
 import kr.co.morandi_batch.batch.processor.NewProblemProcessor;
 import kr.co.morandi_batch.batch.reader.NewProblemPagingReader;
-import kr.co.morandi_batch.batch.reader.NewProblemReader;
 import kr.co.morandi_batch.batch.reader.dto.ProblemDTO;
-import kr.co.morandi_batch.batch.reader.dto.ProblemsResponse;
-import kr.co.morandi_batch.batch.writer.NewProblemWriter;
 import kr.co.morandi_batch.domain.problem.Problem;
 import lombok.RequiredArgsConstructor;
 import org.springframework.batch.core.Job;
@@ -18,8 +15,6 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.transaction.PlatformTransactionManager;
 
-import java.util.List;
-
 @Configuration
 @RequiredArgsConstructor
 public class NewProblemJobConfig {
@@ -29,6 +24,7 @@ public class NewProblemJobConfig {
     private final JdbcBatchItemWriter<Problem> newProblemWriter;
 
 
+
     @Bean
     Job newBaekjoonProblemJob(JobRepository jobRepository, PlatformTransactionManager transactionManager) {
         return new JobBuilder("newBaekjoonProblemJob", jobRepository)
@@ -41,7 +37,7 @@ Job newBaekjoonProblemJob(JobRepository jobRepository, PlatformTransactionManage
     Step newBaekjoonProblemStep(JobRepository jobRepository, PlatformTransactionManager transactionManger) {
         return new StepBuilder("newBaekjoonProblemStep", jobRepository)
                 .<ProblemDTO, Problem>chunk(50, transactionManger)
-                .allowStartIfComplete(true)
+//                .allowStartIfComplete(true)
                 .reader(newProblemPagingReader)
                 .processor(newProblemProcessor)
                 .writer(newProblemWriter)

src/main/java/kr/co/morandi_batch/batch/config/BaekjoonProblemBatchConfig.java → src/main/java/kr/co/morandi_batch/batch/config/UpdateBaekjoonProblemBatchConfig.java

@@ -1,15 +1,15 @@
 package kr.co.morandi_batch.batch.config;
 
 import kr.co.morandi_batch.batch.pagingCollectionsItemReader.PagingCollectionsItemReader;
-import kr.co.morandi_batch.batch.processor.ProblemProcessor;
+import kr.co.morandi_batch.batch.writer.ProblemUpdateWriter;
 import kr.co.morandi_batch.domain.problem.Problem;
 import lombok.RequiredArgsConstructor;
 import org.springframework.batch.core.Job;
 import org.springframework.batch.core.Step;
-import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
 import org.springframework.batch.core.job.builder.JobBuilder;
 import org.springframework.batch.core.repository.JobRepository;
 import org.springframework.batch.core.step.builder.StepBuilder;
+import org.springframework.batch.item.support.CompositeItemProcessor;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.transaction.PlatformTransactionManager;
@@ -19,30 +19,29 @@
 
 @Configuration
 @RequiredArgsConstructor
-public class BaekjoonProblemBatchConfig {
+public class UpdateBaekjoonProblemBatchConfig {
 
     private final PagingCollectionsItemReader<Problem, List<Problem>> problemPagingCollectionsItemReader;
 
-    private final ProblemProcessor problemProcessor;
+    private final CompositeItemProcessor<List<Problem>, List<Problem>> compositeItemProcessor;
+    private final ProblemUpdateWriter problemUpdateWriter;
 
     @Bean
     Job updateBaekjoonProblemJob(JobRepository jobRepository, PlatformTransactionManager transactionManager) {
         return new JobBuilder("updateBaekjoonProblem", jobRepository)
-                .flow(getProblemStep(jobRepository, transactionManager))
+                .flow(updateProblemStep(jobRepository, transactionManager))
                 .end()
                 .build();
     }
 
     @Bean
-    Step getProblemStep(JobRepository jobRepository, PlatformTransactionManager transactionManger) {
-        return new StepBuilder("getProblemStep", jobRepository)
+    Step updateProblemStep(JobRepository jobRepository, PlatformTransactionManager transactionManger) {
+        return new StepBuilder("updateProblemStep", jobRepository)
                 .<List<Problem>, List<Problem>>chunk(10, transactionManger)
-                .allowStartIfComplete(true)
+//                .allowStartIfComplete(true)
                 .reader(problemPagingCollectionsItemReader)
-                .processor(problemProcessor)
-                .writer(list -> {
-                    System.out.println("dd : ");
-                })
+                .processor(compositeItemProcessor)
+                .writer(problemUpdateWriter)
                 .build();
     }
 

src/main/java/kr/co/morandi_batch/batch/pagingCollectionsItemReader/PagingCollectionsItemReader.java

@@ -157,7 +157,9 @@ protected void distributeResultsByChunk(List<T> resultList) {
 
     }
     protected void doClose() throws Exception {
-        this.entityManager.close();
+        if(this.entityManager != null) {
+            this.entityManager.close();
+        }
         super.doClose();
     }
 }

src/main/java/kr/co/morandi_batch/batch/processor/NewProblemProcessor.java

@@ -12,7 +12,7 @@ public class NewProblemProcessor implements ItemProcessor<ProblemDTO, Problem> {
 
     @Override
     public Problem process(ProblemDTO problemDTO) throws Exception {
-        log.info("Processing problem with ID: {}", problemDTO.getProblemId());
+        log.debug("Processing problem with ID: {}", problemDTO.getProblemId());
         return problemDTO.toEntity();
     }
 }